ceph/install-deps.sh

   1 #!/usr/bin/env bash
   2 # -*- mode:sh; tab-width:8; indent-tabs-mode:t -*-
   3 #
   4 # Ceph distributed storage system
   5 #
   6 # Copyright (C) 2014, 2015 Red Hat <contact@redhat.com>
   7 #
   8 # Author: Loic Dachary <loic@dachary.org>
   9 #
  10 #  This library is free software; you can redistribute it and/or
  11 #  modify it under the terms of the GNU Lesser General Public
  12 #  License as published by the Free Software Foundation; either
  13 #  version 2.1 of the License, or (at your option) any later version.
  14 #
  15 set -e
  16 DIR=/tmp/install-deps.$$
  17 trap "rm -fr $DIR" EXIT
  18 mkdir -p $DIR
  19 if test $(id -u) != 0 ; then
  20     SUDO=sudo
  21 fi
  22 export LC_ALL=C # the following is vulnerable to i18n
  23
  24 ARCH=$(uname -m)
  25
  26 function munge_ceph_spec_in {
  27     local with_seastar=$1
  28     shift
  29     local for_make_check=$1
  30     shift
  31     local OUTFILE=$1
  32     sed -e 's/@//g' < ceph.spec.in > $OUTFILE
  33     # http://rpm.org/user_doc/conditional_builds.html
  34     if $with_seastar; then
  35         sed -i -e 's/%bcond_with seastar/%bcond_without seastar/g' $OUTFILE
  36     fi
  37     if $for_make_check; then
  38         sed -i -e 's/%bcond_with make_check/%bcond_without make_check/g' $OUTFILE
  39     fi
  40 }
  41
  42 function munge_debian_control {
  43     local version=$1
  44     shift
  45     local with_seastar=$1
  46     shift
  47     local for_make_check=$1
  48     shift
  49     local control=$1
  50     case "$version" in
  51         *squeeze*|*wheezy*)
  52             control="/tmp/control.$$"
  53             grep -v babeltrace debian/control > $control
  54             ;;
  55     esac
  56     if $with_seastar; then
  57         sed -i -e 's/^# Crimson[[:space:]]//g' $control
  58     fi
  59     if $for_make_check; then
  60         sed -i 's/^# Make-Check[[:space:]]/             /g' $control
  61     fi
  62     echo $control
  63 }
  64
  65 function ensure_decent_gcc_on_ubuntu {
  66     # point gcc to the one offered by g++-7 if the used one is not
  67     # new enough
  68     local old=$(gcc -dumpfullversion -dumpversion)
  69     local new=$1
  70     local codename=$2
  71     if dpkg --compare-versions $old ge ${new}.0; then
  72         return
  73     fi
  74
  75     if [ ! -f /usr/bin/g++-${new} ]; then
  76         $SUDO tee /etc/apt/sources.list.d/ubuntu-toolchain-r.list <<EOF
  77 deb [lang=none] http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
  78 deb [arch=amd64 lang=none] http://mirror.nullivex.com/ppa/ubuntu-toolchain-r-test $codename main
  79 deb [arch=amd64 lang=none] http://deb.rug.nl/ppa/mirror/ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
  80 EOF
  81         # import PPA's signing key into APT's keyring
  82         cat << ENDOFKEY | $SUDO apt-key add -
  83 -----BEGIN PGP PUBLIC KEY BLOCK-----
  84 Version: SKS 1.1.6
  85 Comment: Hostname: keyserver.ubuntu.com
  86
  87 mI0ESuBvRwEEAMi4cDba7xlKaaoXjO1n1HX8RKrkW+HEIl79nSOSJyvzysajs7zUow/OzCQp
  88 9NswqrDmNuH1+lPTTRNAGtK8r2ouq2rnXT1mTl23dpgHZ9spseR73s4ZBGw/ag4bpU5dNUSt
  89 vfmHhIjVCuiSpNn7cyy1JSSvSs3N2mxteKjXLBf7ABEBAAG0GkxhdW5jaHBhZCBUb29sY2hh
  90 aW4gYnVpbGRziLYEEwECACAFAkrgb0cCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAe
  91 k3eiup7yfzGKA/4xzUqNACSlB+k+DxFFHqkwKa/ziFiAlkLQyyhm+iqz80htRZr7Ls/ZRYZl
  92 0aSU56/hLe0V+TviJ1s8qdN2lamkKdXIAFfavA04nOnTzyIBJ82EAUT3Nh45skMxo4z4iZMN
  93 msyaQpNl/m/lNtOLhR64v5ZybofB2EWkMxUzX8D/FQ==
  94 =LcUQ
  95 -----END PGP PUBLIC KEY BLOCK-----
  96 ENDOFKEY
  97         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y || true
  98         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y g++-${new}
  99     fi
 100
 101     case "$codename" in
 102         trusty)
 103             old=4.8;;
 104         xenial)
 105             old=5;;
 106         bionic)
 107             old=7;;
 108     esac
 109     $SUDO update-alternatives --remove-all gcc || true
 110     $SUDO update-alternatives \
 111          --install /usr/bin/gcc gcc /usr/bin/gcc-${new} 20 \
 112          --slave   /usr/bin/g++ g++ /usr/bin/g++-${new}
 113
 114     if [ -f /usr/bin/g++-${old} ]; then
 115       $SUDO update-alternatives \
 116          --install /usr/bin/gcc gcc /usr/bin/gcc-${old} 10 \
 117          --slave   /usr/bin/g++ g++ /usr/bin/g++-${old}
 118     fi
 119
 120     $SUDO update-alternatives --auto gcc
 121
 122     # cmake uses the latter by default
 123     $SUDO ln -nsf /usr/bin/gcc /usr/bin/${ARCH}-linux-gnu-gcc
 124     $SUDO ln -nsf /usr/bin/g++ /usr/bin/${ARCH}-linux-gnu-g++
 125 }
 126
 127 function install_pkg_on_ubuntu {
 128     local project=$1
 129     shift
 130     local sha1=$1
 131     shift
 132     local codename=$1
 133     shift
 134     local force=$1
 135     shift
 136     local pkgs=$@
 137     local missing_pkgs
 138     if [ $force = "force" ]; then
 139         missing_pkgs="$@"
 140     else
 141         for pkg in $pkgs; do
 142             if ! apt -qq list $pkg 2>/dev/null | grep -q installed; then
 143                 missing_pkgs+=" $pkg"
 144             fi
 145         done
 146     fi
 147     if test -n "$missing_pkgs"; then
 148         local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo"
 149         $SUDO curl --silent --location $shaman_url --output /etc/apt/sources.list.d/$project.list
 150         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true
 151         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs
 152     fi
 153 }
 154
 155 function install_boost_on_ubuntu {
 156     local codename=$1
 157     if apt -qq list ceph-libboost1.67-dev 2>/dev/null | grep -q installed; then
 158         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove 'ceph-libboost.*1.67.*'
 159         $SUDO rm /etc/apt/sources.list.d/ceph-libboost1.67.list
 160     fi
 161     local project=libboost
 162     local ver=1.72
 163     local sha1=1d7c7a00cc3f37e340bae0360191a757b44ec80c
 164     install_pkg_on_ubuntu \
 165         $project \
 166         $sha1 \
 167         $codename \
 168         check \
 169         ceph-libboost-atomic$ver-dev \
 170         ceph-libboost-chrono$ver-dev \
 171         ceph-libboost-container$ver-dev \
 172         ceph-libboost-context$ver-dev \
 173         ceph-libboost-coroutine$ver-dev \
 174         ceph-libboost-date-time$ver-dev \
 175         ceph-libboost-filesystem$ver-dev \
 176         ceph-libboost-iostreams$ver-dev \
 177         ceph-libboost-program-options$ver-dev \
 178         ceph-libboost-python$ver-dev \
 179         ceph-libboost-random$ver-dev \
 180         ceph-libboost-regex$ver-dev \
 181         ceph-libboost-system$ver-dev \
 182         ceph-libboost-test$ver-dev \
 183         ceph-libboost-thread$ver-dev \
 184         ceph-libboost-timer$ver-dev
 185 }
 186
 187 function version_lt {
 188     test $1 != $(echo -e "$1\n$2" | sort -rV | head -n 1)
 189 }
 190
 191 function ensure_decent_gcc_on_rh {
 192     local old=$(gcc -dumpversion)
 193     local expected=5.1
 194     local dts_ver=$1
 195     if version_lt $old $expected; then
 196         if test -t 1; then
 197             # interactive shell
 198             cat <<EOF
 199 Your GCC is too old. Please run following command to add DTS to your environment:
 200
 201 scl enable devtoolset-8 bash
 202
 203 Or add following line to the end of ~/.bashrc to add it permanently:
 204
 205 source scl_source enable devtoolset-8
 206
 207 see https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ for more details.
 208 EOF
 209         else
 210             # non-interactive shell
 211             source /opt/rh/devtoolset-$dts_ver/enable
 212         fi
 213     fi
 214 }
 215
 216 for_make_check=false
 217 if tty -s; then
 218     # interactive
 219     for_make_check=true
 220 elif [ $FOR_MAKE_CHECK ]; then
 221     for_make_check=true
 222 else
 223     for_make_check=false
 224 fi
 225
 226 if [ x$(uname)x = xFreeBSDx ]; then
 227     $SUDO pkg install -yq \
 228         devel/babeltrace \
 229         devel/binutils \
 230         devel/git \
 231         devel/gperf \
 232         devel/gmake \
 233         devel/cmake \
 234         devel/yasm \
 235         devel/boost-all \
 236         devel/boost-python-libs \
 237         devel/valgrind \
 238         devel/pkgconf \
 239         devel/libedit \
 240         devel/libtool \
 241         devel/google-perftools \
 242         lang/cython \
 243         devel/py-virtualenv \
 244         databases/leveldb \
 245         net/openldap24-client \
 246         archivers/snappy \
 247         archivers/liblz4 \
 248         ftp/curl \
 249         misc/e2fsprogs-libuuid \
 250         misc/getopt \
 251         net/socat \
 252         textproc/expat2 \
 253         textproc/gsed \
 254         lang/gawk \
 255         textproc/libxml2 \
 256         textproc/xmlstarlet \
 257         textproc/jq \
 258         textproc/py-sphinx \
 259         emulators/fuse \
 260         java/junit \
 261         lang/python36 \
 262         devel/py-pip \
 263         devel/py-flake8 \
 264         devel/py-tox \
 265         devel/py-argparse \
 266         devel/py-nose \
 267         devel/py-prettytable \
 268         www/py-routes \
 269         www/py-flask \
 270         www/node \
 271         www/npm \
 272         www/fcgi \
 273         security/nss \
 274         security/krb5 \
 275         security/oath-toolkit \
 276         sysutils/flock \
 277         sysutils/fusefs-libs \
 278
 279         # Now use pip to install some extra python modules
 280         pip install pecan
 281
 282     exit
 283 else
 284     [ $WITH_SEASTAR ] && with_seastar=true || with_seastar=false
 285     source /etc/os-release
 286     case "$ID" in
 287     debian|ubuntu|devuan)
 288         echo "Using apt-get to install dependencies"
 289         $SUDO apt-get install -y devscripts equivs
 290         $SUDO apt-get install -y dpkg-dev
 291         case "$VERSION" in
 292             *Bionic*)
 293                 ensure_decent_gcc_on_ubuntu 9 bionic
 294                 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu bionic
 295                 ;;
 296             *Disco*)
 297                 [ ! $NO_BOOST_PKGS ] && apt-get install -y libboost1.67-all-dev
 298                 ;;
 299             *)
 300                 $SUDO apt-get install -y gcc
 301                 ;;
 302         esac
 303         if ! test -r debian/control ; then
 304             echo debian/control is not a readable file
 305             exit 1
 306         fi
 307         touch $DIR/status
 308
 309         backports=""
 310         control=$(munge_debian_control "$VERSION" "$with_seastar" "$for_make_check" "debian/control")
 311         case "$VERSION" in
 312             *squeeze*|*wheezy*)
 313                 backports="-t $codename-backports"
 314                 ;;
 315         esac
 316
 317         # make a metapackage that expresses the build dependencies,
 318         # install it, rm the .deb; then uninstall the package as its
 319         # work is done
 320         $SUDO env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
 321         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
 322         if [ "$control" != "debian/control" ] ; then rm $control; fi
 323         ;;
 324     centos|fedora|rhel|ol|virtuozzo)
 325         yumdnf="dnf"
 326         builddepcmd="dnf -y builddep --allowerasing"
 327         if [[ $ID =~ centos|rhel ]] && version_lt $VERSION_ID 8; then
 328             yumdnf="yum"
 329             builddepcmd="yum-builddep -y --setopt=*.skip_if_unavailable=true"
 330         fi
 331         echo "Using $yumdnf to install dependencies"
 332         if [ "$ID" = "centos" -a "$ARCH" = "aarch64" ]; then
 333             $SUDO yum-config-manager --disable centos-sclo-sclo || true
 334             $SUDO yum-config-manager --disable centos-sclo-rh || true
 335             $SUDO yum remove centos-release-scl || true
 336         fi
 337         case "$ID" in
 338             fedora)
 339                 $SUDO $yumdnf install -y $yumdnf-utils
 340                 ;;
 341             centos|rhel|ol|virtuozzo)
 342                 MAJOR_VERSION="$(echo $VERSION_ID | cut -d. -f1)"
 343                 $SUDO $yumdnf install -y $yumdnf-utils
 344                 rpm --quiet --query epel-release || \
 345                     $SUDO $yumdnf -y install --nogpgcheck https://dl.fedoraproject.org/pub/epel/epel-release-latest-$MAJOR_VERSION.noarch.rpm
 346                 $SUDO rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$MAJOR_VERSION
 347                 $SUDO rm -f /etc/yum.repos.d/dl.fedoraproject.org*
 348                 if test $ID = centos -a $MAJOR_VERSION = 7 ; then
 349                     case "$ARCH" in
 350                         x86_64)
 351                             $SUDO $yumdnf -y install centos-release-scl
 352                             dts_ver=8
 353                             ;;
 354                         aarch64)
 355                             $SUDO $yumdnf -y install centos-release-scl-rh
 356                             $SUDO yum-config-manager --disable centos-sclo-rh
 357                             $SUDO yum-config-manager --enable centos-sclo-rh-testing
 358                             dts_ver=8
 359                             ;;
 360                     esac
 361                 elif test $ID = rhel -a $MAJOR_VERSION = 7 ; then
 362                     $SUDO yum-config-manager \
 363                           --enable rhel-server-rhscl-7-rpms \
 364                           --enable rhel-7-server-optional-rpms \
 365                           --enable rhel-7-server-devtools-rpms
 366                     dts_ver=8
 367                 elif test $ID = centos -a $MAJOR_VERSION = 8 ; then
 368                     # Enable 'powertools' or 'PowerTools' repo
 369                     $SUDO dnf config-manager --set-enabled $(dnf repolist --all 2>/dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}')
 370                     # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies
 371                     $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
 372                     $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
 373                     $SUDO dnf copr enable -y ktdreyer/ceph-el8
 374                 elif test $ID = rhel -a $MAJOR_VERSION = 8 ; then
 375                     $SUDO subscription-manager repos --enable "codeready-builder-for-rhel-8-*-rpms"
 376                     $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
 377                     $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
 378                     $SUDO dnf copr enable -y ktdreyer/ceph-el8
 379                 fi
 380                 ;;
 381         esac
 382         munge_ceph_spec_in $with_seastar $for_make_check $DIR/ceph.spec
 383         # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
 384         $SUDO $yumdnf install -y python3-devel
 385         $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out
 386         [ ${PIPESTATUS[0]} -ne 0 ] && exit 1
 387         if [ -n "$dts_ver" ]; then
 388             ensure_decent_gcc_on_rh $dts_ver
 389         fi
 390         IGNORE_YUM_BUILDEP_ERRORS="ValueError: SELinux policy is not managed or store cannot be accessed."
 391         sed "/$IGNORE_YUM_BUILDEP_ERRORS/d" $DIR/yum-builddep.out | grep -qi "error:" && exit 1
 392         ;;
 393     opensuse*|suse|sles)
 394         echo "Using zypper to install dependencies"
 395         zypp_install="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
 396         $SUDO $zypp_install systemd-rpm-macros rpm-build || exit 1
 397         munge_ceph_spec_in $with_seastar $for_make_check $DIR/ceph.spec
 398         $SUDO $zypp_install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
 399         ;;
 400     alpine)
 401         # for now we need the testing repo for leveldb
 402         TESTREPO="http://nl.alpinelinux.org/alpine/edge/testing"
 403         if ! grep -qF "$TESTREPO" /etc/apk/repositories ; then
 404             $SUDO echo "$TESTREPO" | sudo tee -a /etc/apk/repositories > /dev/null
 405         fi
 406         source alpine/APKBUILD.in
 407         $SUDO apk --update add abuild build-base ccache $makedepends
 408         if id -u build >/dev/null 2>&1 ; then
 409            $SUDO addgroup build abuild
 410         fi
 411         ;;
 412     *)
 413         echo "$ID is unknown, dependencies will have to be installed manually."
 414         exit 1
 415         ;;
 416     esac
 417 fi
 418
 419 function populate_wheelhouse() {
 420     local install=$1
 421     shift
 422
 423     # although pip comes with virtualenv, having a recent version
 424     # of pip matters when it comes to using wheel packages
 425     PIP_OPTS="--timeout 300 --exists-action i"
 426     pip $PIP_OPTS $install \
 427       'setuptools >= 0.8' 'pip >= 7.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1
 428     if test $# != 0 ; then
 429         pip $PIP_OPTS $install $@ || return 1
 430     fi
 431 }
 432
 433 function activate_virtualenv() {
 434     local top_srcdir=$1
 435     local env_dir=$top_srcdir/install-deps-python3
 436
 437     if ! test -d $env_dir ; then
 438         virtualenv --python=python3 ${env_dir}
 439         . $env_dir/bin/activate
 440         if ! populate_wheelhouse install ; then
 441             rm -rf $env_dir
 442             return 1
 443         fi
 444     fi
 445     . $env_dir/bin/activate
 446 }
 447
 448 function preload_wheels_for_tox() {
 449     local ini=$1
 450     shift
 451     pushd . > /dev/null
 452     cd $(dirname $ini)
 453     local require_files=$(ls *requirements*.txt 2>/dev/null) || true
 454     local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true
 455     local require=$(echo -n "$require_files" | sed -e 's/^/-r /')
 456     local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /')
 457     local md5=wheelhouse/md5
 458     if test "$require"; then
 459         if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then
 460             rm -rf wheelhouse
 461         fi
 462     fi
 463     if test "$require" && ! test -d wheelhouse ; then
 464         type python3 > /dev/null 2>&1 || continue
 465         activate_virtualenv $top_srcdir || exit 1
 466         populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1
 467         mv $wip_wheelhouse wheelhouse
 468         md5sum $require_files $constraint_files > $md5
 469     fi
 470     popd > /dev/null
 471 }
 472
 473 # use pip cache if possible but do not store it outside of the source
 474 # tree
 475 # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching
 476 if $for_make_check; then
 477     mkdir -p install-deps-cache
 478     top_srcdir=$(pwd)
 479     export XDG_CACHE_HOME=$top_srcdir/install-deps-cache
 480     wip_wheelhouse=wheelhouse-wip
 481     #
 482     # preload python modules so that tox can run without network access
 483     #
 484     find . -name tox.ini | while read ini ; do
 485         preload_wheels_for_tox $ini
 486     done
 487     rm -rf $top_srcdir/install-deps-python3
 488     rm -rf $XDG_CACHE_HOME
 489     type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false)
 490 fi