ceph/src/arrow/dev/release/verify-release-candidate.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Licensed to the Apache Software Foundation (ASF) under one
   4 # or more contributor license agreements.  See the NOTICE file
   5 # distributed with this work for additional information
   6 # regarding copyright ownership.  The ASF licenses this file
   7 # to you under the Apache License, Version 2.0 (the
   8 # "License"); you may not use this file except in compliance
   9 # with the License.  You may obtain a copy of the License at
  10 #
  11 #   http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing,
  14 # software distributed under the License is distributed on an
  15 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16 # KIND, either express or implied.  See the License for the
  17 # specific language governing permissions and limitations
  18 # under the License.
  19 #
  20
  21 # Requirements
  22 # - Ruby >= 2.3
  23 # - Maven >= 3.3.9
  24 # - JDK >=7
  25 # - gcc >= 4.8
  26 # - Node.js >= 11.12 (best way is to use nvm)
  27 # - Go >= 1.15
  28 #
  29 # If using a non-system Boost, set BOOST_ROOT and add Boost libraries to
  30 # LD_LIBRARY_PATH.
  31 #
  32 # To reuse build artifacts between runs set ARROW_TMPDIR environment variable to
  33 # a directory where the temporary files should be placed to, note that this
  34 # directory is not cleaned up automatically.
  35
  36 case $# in
  37   3) ARTIFACT="$1"
  38      VERSION="$2"
  39      RC_NUMBER="$3"
  40      case $ARTIFACT in
  41        source|binaries|wheels) ;;
  42        *) echo "Invalid argument: '${ARTIFACT}', valid options are \
  43 'source', 'binaries', or 'wheels'"
  44           exit 1
  45           ;;
  46      esac
  47      ;;
  48   *) echo "Usage: $0 source|binaries X.Y.Z RC_NUMBER"
  49      exit 1
  50      ;;
  51 esac
  52
  53 set -e
  54 set -x
  55 set -o pipefail
  56
  57 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
  58 ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))"
  59
  60 detect_cuda() {
  61   if ! (which nvcc && which nvidia-smi) > /dev/null; then
  62     return 1
  63   fi
  64
  65   local n_gpus=$(nvidia-smi --list-gpus | wc -l)
  66   return $((${n_gpus} < 1))
  67 }
  68
  69 # Build options for the C++ library
  70
  71 if [ -z "${ARROW_CUDA:-}" ] && detect_cuda; then
  72   ARROW_CUDA=ON
  73 fi
  74 : ${ARROW_CUDA:=OFF}
  75 : ${ARROW_FLIGHT:=ON}
  76 : ${ARROW_GANDIVA:=ON}
  77
  78 ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
  79
  80 download_dist_file() {
  81   curl \
  82     --silent \
  83     --show-error \
  84     --fail \
  85     --location \
  86     --remote-name $ARROW_DIST_URL/$1
  87 }
  88
  89 download_rc_file() {
  90   download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1
  91 }
  92
  93 import_gpg_keys() {
  94   download_dist_file KEYS
  95   gpg --import KEYS
  96 }
  97
  98 fetch_archive() {
  99   local dist_name=$1
 100   download_rc_file ${dist_name}.tar.gz
 101   download_rc_file ${dist_name}.tar.gz.asc
 102   download_rc_file ${dist_name}.tar.gz.sha256
 103   download_rc_file ${dist_name}.tar.gz.sha512
 104   gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
 105   shasum -a 256 -c ${dist_name}.tar.gz.sha256
 106   shasum -a 512 -c ${dist_name}.tar.gz.sha512
 107 }
 108
 109 verify_dir_artifact_signatures() {
 110   # verify the signature and the checksums of each artifact
 111   find $1 -name '*.asc' | while read sigfile; do
 112     artifact=${sigfile/.asc/}
 113     gpg --verify $sigfile $artifact || exit 1
 114
 115     # go into the directory because the checksum files contain only the
 116     # basename of the artifact
 117     pushd $(dirname $artifact)
 118     base_artifact=$(basename $artifact)
 119     if [ -f $base_artifact.sha256 ]; then
 120       shasum -a 256 -c $base_artifact.sha256 || exit 1
 121     fi
 122     shasum -a 512 -c $base_artifact.sha512 || exit 1
 123     popd
 124   done
 125 }
 126
 127 test_binary() {
 128   local download_dir=binaries
 129   mkdir -p ${download_dir}
 130
 131   ${PYTHON:-python} $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
 132          --dest=${download_dir}
 133
 134   verify_dir_artifact_signatures ${download_dir}
 135 }
 136
 137 test_apt() {
 138   for target in "debian:buster" \
 139                 "arm64v8/debian:buster" \
 140                 "debian:bullseye" \
 141                 "arm64v8/debian:bullseye" \
 142                 "debian:bookworm" \
 143                 "arm64v8/debian:bookworm" \
 144                 "ubuntu:bionic" \
 145                 "arm64v8/ubuntu:bionic" \
 146                 "ubuntu:focal" \
 147                 "arm64v8/ubuntu:focal" \
 148                 "ubuntu:hirsute" \
 149                 "arm64v8/ubuntu:hirsute" \
 150                 "ubuntu:impish" \
 151                 "arm64v8/ubuntu:impish"; do \
 152     case "${target}" in
 153       arm64v8/*)
 154         if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then
 155           case "${target}" in
 156           arm64v8/debian:buster|arm64v8/ubuntu:bionic|arm64v8/ubuntu:focal)
 157             ;; # OK
 158           *)
 159             # qemu-user-static in Ubuntu 20.04 has a crash bug:
 160             #   https://bugs.launchpad.net/qemu/+bug/1749393
 161             continue
 162             ;;
 163           esac
 164         else
 165           continue
 166         fi
 167         ;;
 168     esac
 169     if ! docker run --rm -v "${SOURCE_DIR}"/../..:/arrow:delegated \
 170            "${target}" \
 171            /arrow/dev/release/verify-apt.sh \
 172            "${VERSION}" \
 173            "rc"; then
 174       echo "Failed to verify the APT repository for ${target}"
 175       exit 1
 176     fi
 177   done
 178 }
 179
 180 test_yum() {
 181   for target in "almalinux:8" \
 182                 "arm64v8/almalinux:8" \
 183                 "amazonlinux:2" \
 184                 "centos:7" \
 185                 "centos:8" \
 186                 "arm64v8/centos:8"; do
 187     case "${target}" in
 188       arm64v8/*)
 189         if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then
 190           : # OK
 191         else
 192           continue
 193         fi
 194         ;;
 195     esac
 196     if ! docker run --rm -v "${SOURCE_DIR}"/../..:/arrow:delegated \
 197            "${target}" \
 198            /arrow/dev/release/verify-yum.sh \
 199            "${VERSION}" \
 200            "rc"; then
 201       echo "Failed to verify the Yum repository for ${target}"
 202       exit 1
 203     fi
 204   done
 205 }
 206
 207
 208 setup_tempdir() {
 209   cleanup() {
 210     if [ "${TEST_SUCCESS}" = "yes" ]; then
 211       rm -fr "${ARROW_TMPDIR}"
 212     else
 213       echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details."
 214     fi
 215   }
 216
 217   if [ -z "${ARROW_TMPDIR}" ]; then
 218     # clean up automatically if ARROW_TMPDIR is not defined
 219     ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX")
 220     trap cleanup EXIT
 221   else
 222     # don't clean up automatically
 223     mkdir -p "${ARROW_TMPDIR}"
 224   fi
 225 }
 226
 227 setup_miniconda() {
 228   # Setup short-lived miniconda for Python and integration tests
 229   OS="$(uname)"
 230   if [ "${OS}" == "Darwin" ]; then
 231     OS=MacOSX
 232   fi
 233   ARCH="$(uname -m)"
 234   MINICONDA_URL="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${OS}-${ARCH}.sh"
 235
 236   MINICONDA=$PWD/test-miniconda
 237
 238   if [ ! -d "${MINICONDA}" ]; then
 239     # Setup miniconda only if the directory doesn't exist yet
 240     wget -O miniconda.sh $MINICONDA_URL
 241     bash miniconda.sh -b -p $MINICONDA
 242     rm -f miniconda.sh
 243   fi
 244   echo "Installed miniconda at ${MINICONDA}"
 245
 246   . $MINICONDA/etc/profile.d/conda.sh
 247
 248   conda create -n arrow-test -y -q -c conda-forge \
 249     python=3.8 \
 250     nomkl \
 251     numpy \
 252     pandas \
 253     cython
 254   conda activate arrow-test
 255   echo "Using conda environment ${CONDA_PREFIX}"
 256 }
 257
 258 # Build and test Java (Requires newer Maven -- I used 3.3.9)
 259
 260 test_package_java() {
 261   pushd java
 262
 263   mvn test
 264   mvn package
 265
 266   popd
 267 }
 268
 269 # Build and test C++
 270
 271 test_and_install_cpp() {
 272   mkdir -p cpp/build
 273   pushd cpp/build
 274
 275   ARROW_CMAKE_OPTIONS="
 276 ${ARROW_CMAKE_OPTIONS:-}
 277 -DCMAKE_INSTALL_PREFIX=$ARROW_HOME
 278 -DCMAKE_INSTALL_LIBDIR=lib
 279 -DARROW_FLIGHT=${ARROW_FLIGHT}
 280 -DARROW_PLASMA=ON
 281 -DARROW_ORC=ON
 282 -DARROW_PYTHON=ON
 283 -DARROW_GANDIVA=${ARROW_GANDIVA}
 284 -DARROW_PARQUET=ON
 285 -DARROW_DATASET=ON
 286 -DPARQUET_REQUIRE_ENCRYPTION=ON
 287 -DARROW_VERBOSE_THIRDPARTY_BUILD=ON
 288 -DARROW_WITH_BZ2=ON
 289 -DARROW_WITH_ZLIB=ON
 290 -DARROW_WITH_ZSTD=ON
 291 -DARROW_WITH_LZ4=ON
 292 -DARROW_WITH_SNAPPY=ON
 293 -DARROW_WITH_BROTLI=ON
 294 -DARROW_BOOST_USE_SHARED=ON
 295 -DCMAKE_BUILD_TYPE=release
 296 -DARROW_BUILD_TESTS=ON
 297 -DARROW_BUILD_INTEGRATION=ON
 298 -DARROW_CUDA=${ARROW_CUDA}
 299 -DARROW_DEPENDENCY_SOURCE=AUTO
 300 "
 301   cmake $ARROW_CMAKE_OPTIONS ..
 302
 303   make -j$NPROC install
 304
 305   # TODO: ARROW-5036: plasma-serialization_tests broken
 306   # TODO: ARROW-5054: libgtest.so link failure in flight-server-test
 307   LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH ctest \
 308     --exclude-regex "plasma-serialization_tests" \
 309     -j$NPROC \
 310     --output-on-failure \
 311     -L unittest
 312   popd
 313 }
 314
 315 test_csharp() {
 316   pushd csharp
 317
 318   local csharp_bin=${PWD}/bin
 319   mkdir -p ${csharp_bin}
 320
 321   if which dotnet > /dev/null 2>&1; then
 322     if ! which sourcelink > /dev/null 2>&1; then
 323       local dotnet_tools_dir=$HOME/.dotnet/tools
 324       if [ -d "${dotnet_tools_dir}" ]; then
 325         PATH="${dotnet_tools_dir}:$PATH"
 326       fi
 327     fi
 328   else
 329     local dotnet_version=3.1.405
 330     local dotnet_platform=
 331     case "$(uname)" in
 332       Linux)
 333         dotnet_platform=linux
 334         ;;
 335       Darwin)
 336         dotnet_platform=macos
 337         ;;
 338     esac
 339     local dotnet_download_thank_you_url=https://dotnet.microsoft.com/download/thank-you/dotnet-sdk-${dotnet_version}-${dotnet_platform}-x64-binaries
 340     local dotnet_download_url=$( \
 341       curl --location ${dotnet_download_thank_you_url} | \
 342         grep 'window\.open' | \
 343         grep -E -o '[^"]+' | \
 344         sed -n 2p)
 345     curl ${dotnet_download_url} | \
 346       tar xzf - -C ${csharp_bin}
 347     PATH=${csharp_bin}:${PATH}
 348   fi
 349
 350   dotnet test
 351   mv dummy.git ../.git
 352   dotnet pack -c Release
 353   mv ../.git dummy.git
 354
 355   if ! which sourcelink > /dev/null 2>&1; then
 356     dotnet tool install --tool-path ${csharp_bin} sourcelink
 357     PATH=${csharp_bin}:${PATH}
 358     if ! sourcelink --help > /dev/null 2>&1; then
 359       export DOTNET_ROOT=${csharp_bin}
 360     fi
 361   fi
 362
 363   sourcelink test artifacts/Apache.Arrow/Release/netstandard1.3/Apache.Arrow.pdb
 364   sourcelink test artifacts/Apache.Arrow/Release/netcoreapp2.1/Apache.Arrow.pdb
 365
 366   popd
 367 }
 368
 369 # Build and test Python
 370
 371 test_python() {
 372   pushd python
 373
 374   pip install -r requirements-build.txt -r requirements-test.txt
 375
 376   export PYARROW_WITH_DATASET=1
 377   export PYARROW_WITH_PARQUET=1
 378   export PYARROW_WITH_PLASMA=1
 379   if [ "${ARROW_CUDA}" = "ON" ]; then
 380     export PYARROW_WITH_CUDA=1
 381   fi
 382   if [ "${ARROW_FLIGHT}" = "ON" ]; then
 383     export PYARROW_WITH_FLIGHT=1
 384   fi
 385   if [ "${ARROW_GANDIVA}" = "ON" ]; then
 386     export PYARROW_WITH_GANDIVA=1
 387   fi
 388
 389   python setup.py build_ext --inplace
 390   pytest pyarrow -v --pdb
 391
 392   popd
 393 }
 394
 395 test_glib() {
 396   pushd c_glib
 397
 398   pip install meson
 399
 400   meson build --prefix=$ARROW_HOME --libdir=lib
 401   ninja -C build
 402   ninja -C build install
 403
 404   export GI_TYPELIB_PATH=$ARROW_HOME/lib/girepository-1.0:$GI_TYPELIB_PATH
 405
 406   if ! bundle --version; then
 407     gem install --no-document bundler
 408   fi
 409
 410   bundle install --path vendor/bundle
 411   bundle exec ruby test/run-test.rb
 412
 413   popd
 414 }
 415
 416 test_js() {
 417   pushd js
 418
 419   if [ "${INSTALL_NODE}" -gt 0 ]; then
 420     export NVM_DIR="`pwd`/.nvm"
 421     mkdir -p $NVM_DIR
 422     curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | \
 423       PROFILE=/dev/null bash
 424     [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
 425
 426     nvm install --lts
 427     npm install -g yarn
 428   fi
 429
 430   yarn --frozen-lockfile
 431   yarn run-s clean:all lint build
 432   yarn test
 433   popd
 434 }
 435
 436 test_ruby() {
 437   pushd ruby
 438
 439   local modules="red-arrow red-arrow-dataset red-plasma red-parquet"
 440   if [ "${ARROW_CUDA}" = "ON" ]; then
 441     modules="${modules} red-arrow-cuda"
 442   fi
 443   if [ "${ARROW_FLIGHT}" = "ON" ]; then
 444     modules="${modules} red-arrow-flight"
 445   fi
 446   if [ "${ARROW_GANDIVA}" = "ON" ]; then
 447     modules="${modules} red-gandiva"
 448   fi
 449
 450   for module in ${modules}; do
 451     pushd ${module}
 452     bundle install --path vendor/bundle
 453     bundle exec ruby test/run-test.rb
 454     popd
 455   done
 456
 457   popd
 458 }
 459
 460 test_go() {
 461   local VERSION=1.15.14
 462   local ARCH=amd64
 463
 464   if [ "$(uname)" == "Darwin" ]; then
 465     local OS=darwin
 466   else
 467     local OS=linux
 468   fi
 469
 470   local GO_ARCHIVE=go$VERSION.$OS-$ARCH.tar.gz
 471   wget https://dl.google.com/go/$GO_ARCHIVE
 472
 473   mkdir -p local-go
 474   tar -xzf $GO_ARCHIVE -C local-go
 475   rm -f $GO_ARCHIVE
 476
 477   export GOROOT=`pwd`/local-go/go
 478   export GOPATH=`pwd`/local-go/gopath
 479   export PATH=$GOROOT/bin:$GOPATH/bin:$PATH
 480
 481   pushd go/arrow
 482
 483   go get -v ./...
 484   go test ./...
 485   go clean -modcache
 486
 487   popd
 488 }
 489
 490 # Run integration tests
 491 test_integration() {
 492   JAVA_DIR=$PWD/java
 493   CPP_BUILD_DIR=$PWD/cpp/build
 494
 495   export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
 496   export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
 497
 498   pip install -e dev/archery
 499
 500   INTEGRATION_TEST_ARGS=""
 501
 502   if [ "${ARROW_FLIGHT}" = "ON" ]; then
 503     INTEGRATION_TEST_ARGS="${INTEGRATION_TEST_ARGS} --run-flight"
 504   fi
 505
 506   # Flight integration test executable have runtime dependency on
 507   # release/libgtest.so
 508   LD_LIBRARY_PATH=$ARROW_CPP_EXE_PATH:$LD_LIBRARY_PATH \
 509       archery integration \
 510               --with-cpp=${TEST_INTEGRATION_CPP} \
 511               --with-java=${TEST_INTEGRATION_JAVA} \
 512               --with-js=${TEST_INTEGRATION_JS} \
 513               --with-go=${TEST_INTEGRATION_GO} \
 514               $INTEGRATION_TEST_ARGS
 515 }
 516
 517 clone_testing_repositories() {
 518   # Clone testing repositories if not cloned already
 519   if [ ! -d "arrow-testing" ]; then
 520     git clone https://github.com/apache/arrow-testing.git
 521   fi
 522   if [ ! -d "parquet-testing" ]; then
 523     git clone https://github.com/apache/parquet-testing.git
 524   fi
 525   export ARROW_TEST_DATA=$PWD/arrow-testing/data
 526   export PARQUET_TEST_DATA=$PWD/parquet-testing/data
 527 }
 528
 529 test_source_distribution() {
 530   export ARROW_HOME=$ARROW_TMPDIR/install
 531   export PARQUET_HOME=$ARROW_TMPDIR/install
 532   export LD_LIBRARY_PATH=$ARROW_HOME/lib:${LD_LIBRARY_PATH:-}
 533   export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:${PKG_CONFIG_PATH:-}
 534
 535   if [ "$(uname)" == "Darwin" ]; then
 536     NPROC=$(sysctl -n hw.ncpu)
 537   else
 538     NPROC=$(nproc)
 539   fi
 540
 541   clone_testing_repositories
 542
 543   if [ ${TEST_JAVA} -gt 0 ]; then
 544     test_package_java
 545   fi
 546   if [ ${TEST_CPP} -gt 0 ]; then
 547     test_and_install_cpp
 548   fi
 549   if [ ${TEST_CSHARP} -gt 0 ]; then
 550     test_csharp
 551   fi
 552   if [ ${TEST_PYTHON} -gt 0 ]; then
 553     test_python
 554   fi
 555   if [ ${TEST_GLIB} -gt 0 ]; then
 556     test_glib
 557   fi
 558   if [ ${TEST_RUBY} -gt 0 ]; then
 559     test_ruby
 560   fi
 561   if [ ${TEST_JS} -gt 0 ]; then
 562     test_js
 563   fi
 564   if [ ${TEST_GO} -gt 0 ]; then
 565     test_go
 566   fi
 567   if [ ${TEST_INTEGRATION} -gt 0 ]; then
 568     test_integration
 569   fi
 570 }
 571
 572 test_binary_distribution() {
 573   if [ ${TEST_BINARY} -gt 0 ]; then
 574     test_binary
 575   fi
 576   if [ ${TEST_APT} -gt 0 ]; then
 577     test_apt
 578   fi
 579   if [ ${TEST_YUM} -gt 0 ]; then
 580     test_yum
 581   fi
 582 }
 583
 584 test_linux_wheels() {
 585   if [ "$(uname -m)" = "aarch64" ]; then
 586     local arch="aarch64"
 587   else
 588     local arch="x86_64"
 589   fi
 590
 591   local py_arches="3.6m 3.7m 3.8 3.9"
 592   local platform_tags="manylinux_2_12_${arch}.manylinux2010_${arch} manylinux_2_17_${arch}.manylinux2014_${arch}"
 593
 594   for py_arch in ${py_arches}; do
 595     local env=_verify_wheel-${py_arch}
 596     conda create -yq -n ${env} python=${py_arch//[mu]/}
 597     conda activate ${env}
 598     pip install -U pip
 599
 600     for tag in ${platform_tags}; do
 601       # check the mandatory and optional imports
 602       pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[mu.]/}-cp${py_arch//./}-${tag}.whl
 603       INSTALL_PYARROW=OFF ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_DIR}
 604     done
 605
 606     conda deactivate
 607   done
 608 }
 609
 610 test_macos_wheels() {
 611   local py_arches="3.6m 3.7m 3.8 3.9"
 612   local macos_version=$(sw_vers -productVersion)
 613   local macos_short_version=${macos_version:0:5}
 614
 615   local check_s3=ON
 616   local check_flight=ON
 617
 618   # macOS version <= 10.13
 619   if [ $(echo "${macos_short_version}\n10.14" | sort -V | head -n1) == "${macos_short_version}" ]; then
 620     local check_s3=OFF
 621   fi
 622   # apple silicon processor
 623   if [ "$(uname -m)" = "arm64" ]; then
 624     local py_arches="3.8 3.9"
 625     local check_flight=OFF
 626   fi
 627
 628   # verify arch-native wheels inside an arch-native conda environment
 629   for py_arch in ${py_arches}; do
 630     local env=_verify_wheel-${py_arch}
 631     conda create -yq -n ${env} python=${py_arch//m/}
 632     conda activate ${env}
 633     pip install -U pip
 634
 635     # check the mandatory and optional imports
 636     pip install --find-links python-rc/${VERSION}-rc${RC_NUMBER} pyarrow==${VERSION}
 637     INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} ARROW_S3=${check_s3} \
 638       ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_DIR}
 639
 640     conda deactivate
 641   done
 642
 643   # verify arm64 and universal2 wheels using an universal2 python binary
 644   # the interpreter should be installed from python.org:
 645   #   https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg
 646   if [ "$(uname -m)" = "arm64" ]; then
 647     for py_arch in "3.9"; do
 648       local pyver=${py_arch//m/}
 649       local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}"
 650
 651       # create and activate a virtualenv for testing as arm64
 652       for arch in "arm64" "x86_64"; do
 653         local venv="${ARROW_TMPDIR}/test-${arch}-virtualenv"
 654         $python -m virtualenv $venv
 655         source $venv/bin/activate
 656         pip install -U pip
 657
 658         # install pyarrow's universal2 wheel
 659         pip install \
 660             --find-links python-rc/${VERSION}-rc${RC_NUMBER} \
 661             --target $(python -c 'import site; print(site.getsitepackages()[0])') \
 662             --platform macosx_11_0_universal2 \
 663             --only-binary=:all: \
 664             pyarrow==${VERSION}
 665         # check the imports and execute the unittests
 666         INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} ARROW_S3=${check_s3} \
 667           arch -${arch} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_DIR}
 668
 669         deactivate
 670       done
 671     done
 672   fi
 673 }
 674
 675 test_wheels() {
 676   clone_testing_repositories
 677
 678   local download_dir=binaries
 679   mkdir -p ${download_dir}
 680
 681   if [ "$(uname)" == "Darwin" ]; then
 682     local filter_regex=.*macosx.*
 683   else
 684     local filter_regex=.*manylinux.*
 685   fi
 686
 687   python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
 688          --package_type python \
 689          --regex=${filter_regex} \
 690          --dest=${download_dir}
 691
 692   verify_dir_artifact_signatures ${download_dir}
 693
 694   pushd ${download_dir}
 695
 696   if [ "$(uname)" == "Darwin" ]; then
 697     test_macos_wheels
 698   else
 699     test_linux_wheels
 700   fi
 701
 702   popd
 703 }
 704
 705 # By default test all functionalities.
 706 # To deactivate one test, deactivate the test and all of its dependents
 707 # To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
 708
 709 # Install NodeJS locally for running the JavaScript tests rather than using the
 710 # system Node installation, which may be too old.
 711 : ${INSTALL_NODE:=1}
 712
 713 if [ "${ARTIFACT}" == "source" ]; then
 714   : ${TEST_SOURCE:=1}
 715 elif [ "${ARTIFACT}" == "wheels" ]; then
 716   TEST_WHEELS=1
 717 else
 718   TEST_BINARY_DISTRIBUTIONS=1
 719 fi
 720 : ${TEST_SOURCE:=0}
 721 : ${TEST_WHEELS:=0}
 722 : ${TEST_BINARY_DISTRIBUTIONS:=0}
 723
 724 : ${TEST_DEFAULT:=1}
 725 : ${TEST_JAVA:=${TEST_DEFAULT}}
 726 : ${TEST_CPP:=${TEST_DEFAULT}}
 727 : ${TEST_CSHARP:=${TEST_DEFAULT}}
 728 : ${TEST_GLIB:=${TEST_DEFAULT}}
 729 : ${TEST_RUBY:=${TEST_DEFAULT}}
 730 : ${TEST_PYTHON:=${TEST_DEFAULT}}
 731 : ${TEST_JS:=${TEST_DEFAULT}}
 732 : ${TEST_GO:=${TEST_DEFAULT}}
 733 : ${TEST_INTEGRATION:=${TEST_DEFAULT}}
 734 if [ ${TEST_BINARY_DISTRIBUTIONS} -gt 0 ]; then
 735   TEST_BINARY_DISTRIBUTIONS_DEFAULT=${TEST_DEFAULT}
 736 else
 737   TEST_BINARY_DISTRIBUTIONS_DEFAULT=0
 738 fi
 739 : ${TEST_BINARY:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
 740 : ${TEST_APT:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
 741 : ${TEST_YUM:=${TEST_BINARY_DISTRIBUTIONS_DEFAULT}}
 742
 743 # For selective Integration testing, set TEST_DEFAULT=0 TEST_INTEGRATION_X=1 TEST_INTEGRATION_Y=1
 744 : ${TEST_INTEGRATION_CPP:=${TEST_INTEGRATION}}
 745 : ${TEST_INTEGRATION_JAVA:=${TEST_INTEGRATION}}
 746 : ${TEST_INTEGRATION_JS:=${TEST_INTEGRATION}}
 747 : ${TEST_INTEGRATION_GO:=${TEST_INTEGRATION}}
 748
 749 # Automatically test if its activated by a dependent
 750 TEST_GLIB=$((${TEST_GLIB} + ${TEST_RUBY}))
 751 TEST_CPP=$((${TEST_CPP} + ${TEST_GLIB} + ${TEST_PYTHON} + ${TEST_INTEGRATION_CPP}))
 752 TEST_JAVA=$((${TEST_JAVA} + ${TEST_INTEGRATION_JAVA}))
 753 TEST_JS=$((${TEST_JS} + ${TEST_INTEGRATION_JS}))
 754 TEST_GO=$((${TEST_GO} + ${TEST_INTEGRATION_GO}))
 755 TEST_INTEGRATION=$((${TEST_INTEGRATION} + ${TEST_INTEGRATION_CPP} + ${TEST_INTEGRATION_JAVA} + ${TEST_INTEGRATION_JS} + ${TEST_INTEGRATION_GO}))
 756
 757 if [ "${ARTIFACT}" == "source" ]; then
 758   NEED_MINICONDA=$((${TEST_CPP} + ${TEST_INTEGRATION}))
 759 elif [ "${ARTIFACT}" == "wheels" ]; then
 760   NEED_MINICONDA=$((${TEST_WHEELS}))
 761 else
 762   if [ -z "${PYTHON:-}" ]; then
 763     NEED_MINICONDA=$((${TEST_BINARY}))
 764   else
 765     NEED_MINICONDA=0
 766   fi
 767 fi
 768
 769 : ${TEST_ARCHIVE:=apache-arrow-${VERSION}.tar.gz}
 770 case "${TEST_ARCHIVE}" in
 771   /*)
 772    ;;
 773   *)
 774    TEST_ARCHIVE=${PWD}/${TEST_ARCHIVE}
 775    ;;
 776 esac
 777
 778 TEST_SUCCESS=no
 779
 780 setup_tempdir "arrow-${VERSION}"
 781 echo "Working in sandbox ${ARROW_TMPDIR}"
 782 cd ${ARROW_TMPDIR}
 783
 784 if [ ${NEED_MINICONDA} -gt 0 ]; then
 785   setup_miniconda
 786 fi
 787
 788 if [ "${ARTIFACT}" == "source" ]; then
 789   dist_name="apache-arrow-${VERSION}"
 790   if [ ${TEST_SOURCE} -gt 0 ]; then
 791     import_gpg_keys
 792     if [ ! -d "${dist_name}" ]; then
 793       fetch_archive ${dist_name}
 794       tar xf ${dist_name}.tar.gz
 795     fi
 796   else
 797     mkdir -p ${dist_name}
 798     if [ ! -f ${TEST_ARCHIVE} ]; then
 799       echo "${TEST_ARCHIVE} not found"
 800       exit 1
 801     fi
 802     tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1
 803   fi
 804   pushd ${dist_name}
 805   test_source_distribution
 806   popd
 807 elif [ "${ARTIFACT}" == "wheels" ]; then
 808   import_gpg_keys
 809   test_wheels
 810 else
 811   import_gpg_keys
 812   test_binary_distribution
 813 fi
 814
 815 TEST_SUCCESS=yes
 816 echo 'Release candidate looks good!'
 817 exit 0