X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=ceph%2Finstall-deps.sh;h=0a21dfcb635661fa08bdad17fd7bd6c64d4cee9c;hb=HEAD;hp=8f671db1450a4df695962684685af643d0d9f92e;hpb=eafe8130898c3d7229e1c84c100c2e62e32be0d0;p=ceph.git diff --git a/ceph/install-deps.sh b/ceph/install-deps.sh index 8f671db14..eb773c3eb 100755 --- a/ceph/install-deps.sh +++ b/ceph/install-deps.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# -*- mode:sh; tab-width:8; indent-tabs-mode:t -*- # # Ceph distributed storage system # @@ -19,28 +18,32 @@ mkdir -p $DIR if test $(id -u) != 0 ; then SUDO=sudo fi -export LC_ALL=C # the following is vulnerable to i18n +# enable UTF-8 encoding for programs like pip that expect to +# print more than just ascii chars +export LC_ALL=C.UTF-8 ARCH=$(uname -m) -function install_seastar_deps { - if [ $WITH_SEASTAR ]; then - $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y \ - ragel libc-ares-dev libhwloc-dev libnuma-dev libpciaccess-dev \ - libcrypto++-dev libgnutls28-dev libsctp-dev libprotobuf-dev \ - protobuf-compiler systemtap-sdt-dev libyaml-cpp-dev - fi +function in_jenkins() { + test -n "$JENKINS_HOME" } function munge_ceph_spec_in { + local with_seastar=$1 + shift + local with_zbd=$1 + shift local for_make_check=$1 shift local OUTFILE=$1 sed -e 's/@//g' < ceph.spec.in > $OUTFILE # http://rpm.org/user_doc/conditional_builds.html - if [ $WITH_SEASTAR ]; then + if $with_seastar; then sed -i -e 's/%bcond_with seastar/%bcond_without seastar/g' $OUTFILE fi + if $with_zbd; then + sed -i -e 's/%bcond_with zbd/%bcond_without zbd/g' $OUTFILE + fi if $for_make_check; then sed -i -e 's/%bcond_with make_check/%bcond_without make_check/g' $OUTFILE fi @@ -49,39 +52,34 @@ function munge_ceph_spec_in { function munge_debian_control { local version=$1 shift - local for_make_check=$1 - shift local control=$1 case "$version" in *squeeze*|*wheezy*) - control="/tmp/control.$$" - grep -v babeltrace debian/control > $control - ;; + control="/tmp/control.$$" + grep -v babeltrace debian/control > $control + ;; esac - if $for_make_check; then - sed -i 's/^# Make-Check[[:space:]]/ /g' $control - fi echo $control } function ensure_decent_gcc_on_ubuntu { + in_jenkins && echo "CI_DEBUG: Start ensure_decent_gcc_on_ubuntu() in install-deps.sh" # point gcc to the one offered by g++-7 if the used one is not # new enough local old=$(gcc -dumpfullversion -dumpversion) local new=$1 local codename=$2 if dpkg --compare-versions $old ge ${new}.0; then - return + return fi if [ ! -f /usr/bin/g++-${new} ]; then - $SUDO tee /etc/apt/sources.list.d/ubuntu-toolchain-r.list < /dev/null; then - missing_pkgs+=" $pkg" - fi - done + if [ $force = "force" ]; then + missing_pkgs="$@" + else + for pkg in $pkgs; do + if ! apt -qq list $pkg 2>/dev/null | grep -q installed; then + missing_pkgs+=" $pkg" + in_jenkins && echo "CI_DEBUG: missing_pkgs=$missing_pkgs" + fi + done + fi if test -n "$missing_pkgs"; then - local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo" - $SUDO curl --silent --location $shaman_url --output /etc/apt/sources.list.d/$project.list - $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true - $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs + local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo" + in_jenkins && echo -n "CI_DEBUG: Downloading $shaman_url ... " + $SUDO curl --silent --fail --write-out "%{http_code}" --location $shaman_url --output /etc/apt/sources.list.d/$project.list + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs + fi +} + +boost_ver=1.79 + +function clean_boost_on_ubuntu { + in_jenkins && echo "CI_DEBUG: Running clean_boost_on_ubuntu() in install-deps.sh" + # Find currently installed version. If there are multiple + # versions, they end up newline separated + local installed_ver=$(apt -qq list --installed ceph-libboost*-dev 2>/dev/null | + cut -d' ' -f2 | + cut -d'.' -f1,2 | + sort -u) + # If installed_ver contains whitespace, we can't really count on it, + # but otherwise, bail out if the version installed is the version + # we want. + if test -n "$installed_ver" && + echo -n "$installed_ver" | tr '[:space:]' ' ' | grep -v -q ' '; then + if echo "$installed_ver" | grep -q "^$boost_ver"; then + return + fi + fi + + # Historical packages + $SUDO rm -f /etc/apt/sources.list.d/ceph-libboost*.list + # Currently used + $SUDO rm -f /etc/apt/sources.list.d/libboost.list + # Refresh package list so things aren't in the available list. + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y || true + # Remove all ceph-libboost packages. We have an early return if + # the desired version is already (and the only) version installed, + # so no need to spare it. + if test -n "$installed_ver"; then + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y --fix-missing remove "ceph-libboost*" fi } function install_boost_on_ubuntu { + in_jenkins && echo "CI_DEBUG: Running install_boost_on_ubuntu() in install-deps.sh" + # Once we get to this point, clean_boost_on_ubuntu() should ensure + # that there is no more than one installed version. + local installed_ver=$(apt -qq list --installed ceph-libboost*-dev 2>/dev/null | + grep -e 'libboost[0-9].[0-9]\+-dev' | + cut -d' ' -f2 | + cut -d'.' -f1,2) + if test -n "$installed_ver"; then + if echo "$installed_ver" | grep -q "^$boost_ver"; then + return + fi + fi local codename=$1 + local project=libboost + local sha1=892ab89e76b91b505ffbf083f6fb7f2a666d4132 install_pkg_on_ubuntu \ - ceph-libboost1.67 \ - dd38c27740c1f9a9e6719a07eef84a1369dc168b \ - $codename \ - ceph-libboost-atomic1.67-dev \ - ceph-libboost-chrono1.67-dev \ - ceph-libboost-container1.67-dev \ - ceph-libboost-context1.67-dev \ - ceph-libboost-coroutine1.67-dev \ - ceph-libboost-date-time1.67-dev \ - ceph-libboost-filesystem1.67-dev \ - ceph-libboost-iostreams1.67-dev \ - ceph-libboost-program-options1.67-dev \ - ceph-libboost-python1.67-dev \ - ceph-libboost-random1.67-dev \ - ceph-libboost-regex1.67-dev \ - ceph-libboost-system1.67-dev \ - ceph-libboost-thread1.67-dev \ - ceph-libboost-timer1.67-dev + $project \ + $sha1 \ + $codename \ + check \ + ceph-libboost-atomic${boost_ver}-dev \ + ceph-libboost-chrono${boost_ver}-dev \ + ceph-libboost-container${boost_ver}-dev \ + ceph-libboost-context${boost_ver}-dev \ + ceph-libboost-coroutine${boost_ver}-dev \ + ceph-libboost-date-time${boost_ver}-dev \ + ceph-libboost-filesystem${boost_ver}-dev \ + ceph-libboost-iostreams${boost_ver}-dev \ + ceph-libboost-program-options${boost_ver}-dev \ + ceph-libboost-python${boost_ver}-dev \ + ceph-libboost-random${boost_ver}-dev \ + ceph-libboost-regex${boost_ver}-dev \ + ceph-libboost-system${boost_ver}-dev \ + ceph-libboost-test${boost_ver}-dev \ + ceph-libboost-thread${boost_ver}-dev \ + ceph-libboost-timer${boost_ver}-dev +} + +function install_libzbd_on_ubuntu { + in_jenkins && echo "CI_DEBUG: Running install_libzbd_on_ubuntu() in install-deps.sh" + local codename=$1 + local project=libzbd + local sha1=1fadde94b08fab574b17637c2bebd2b1e7f9127b + install_pkg_on_ubuntu \ + $project \ + $sha1 \ + $codename \ + check \ + libzbd-dev +} + +motr_pkgs_url='https://github.com/Seagate/cortx-motr/releases/download/2.0.0-rgw' + +function install_cortx_motr_on_ubuntu { + if dpkg -l cortx-motr-dev &> /dev/null; then + return + fi + if [ "$(lsb_release -sc)" = "jammy" ]; then + install_pkg_on_ubuntu \ + cortx-motr \ + 39f89fa1c6945040433a913f2687c4b4e6cbeb3f \ + jammy \ + check \ + cortx-motr \ + cortx-motr-dev + else + local deb_arch=$(dpkg --print-architecture) + local motr_pkg="cortx-motr_2.0.0.git3252d623_$deb_arch.deb" + local motr_dev_pkg="cortx-motr-dev_2.0.0.git3252d623_$deb_arch.deb" + $SUDO curl -sL -o/var/cache/apt/archives/$motr_pkg $motr_pkgs_url/$motr_pkg + $SUDO curl -sL -o/var/cache/apt/archives/$motr_dev_pkg $motr_pkgs_url/$motr_dev_pkg + # For some reason libfabric pkg is not available in arm64 version + # of Ubuntu 20.04 (Focal Fossa), so we borrow it from more recent + # versions for now. + if [[ "$deb_arch" == 'arm64' ]]; then + local lf_pkg='libfabric1_1.11.0-2_arm64.deb' + $SUDO curl -sL -o/var/cache/apt/archives/$lf_pkg http://ports.ubuntu.com/pool/universe/libf/libfabric/$lf_pkg + $SUDO apt-get install -y /var/cache/apt/archives/$lf_pkg + fi + $SUDO apt-get install -y /var/cache/apt/archives/{$motr_pkg,$motr_dev_pkg} + $SUDO apt-get install -y libisal-dev + fi } function version_lt { @@ -175,29 +266,36 @@ function version_lt { function ensure_decent_gcc_on_rh { local old=$(gcc -dumpversion) - local expected=5.1 local dts_ver=$1 - if version_lt $old $expected; then - if test -t 1; then - # interactive shell - cat <= 22" | bc)" -ne 0; then - yumdnf="dnf" - builddepcmd="dnf -y builddep --allowerasing" + # make a metapackage that expresses the build dependencies, + # install it, rm the .deb; then uninstall the package as its + # work is done + build_profiles="" + if $for_make_check; then + build_profiles+=",pkg.ceph.check" fi - echo "Using $yumdnf to install dependencies" - if [ "$ID" = "centos" -a "$ARCH" = "aarch64" ]; then - $SUDO yum-config-manager --disable centos-sclo-sclo || true - $SUDO yum-config-manager --disable centos-sclo-rh || true - $SUDO yum remove centos-release-scl || true - fi + if $with_seastar; then + build_profiles+=",pkg.ceph.crimson" + fi + if $with_pmem; then + build_profiles+=",pkg.ceph.pmdk" + fi + + in_jenkins && cat </dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}') + dts_ver=11 + # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies + $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/ + $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save + $SUDO dnf -y module enable javapackages-tools + elif test $ID = rhel -a $MAJOR_VERSION = 8 ; then + dts_ver=11 + $SUDO dnf config-manager --set-enabled "codeready-builder-for-rhel-8-${ARCH}-rpms" + $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/ + $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save + $SUDO dnf -y module enable javapackages-tools fi ;; esac - munge_ceph_spec_in $for_make_check $DIR/ceph.spec - $SUDO $yumdnf install -y \*rpm-macros + munge_ceph_spec_in $with_seastar $with_zbd $for_make_check $DIR/ceph.spec + # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel + $SUDO dnf install -y python3-devel $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out [ ${PIPESTATUS[0]} -ne 0 ] && exit 1 - if [ -n "$dts_ver" ]; then + if [ -n "$dts_ver" ]; then ensure_decent_gcc_on_rh $dts_ver - fi - ! grep -q -i error: $DIR/yum-builddep.out || exit 1 - # for building python-saml and its dependencies - $SUDO $yumdnf install -y xmlsec1 xmlsec1-nss xmlsec1-openssl xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel + fi + IGNORE_YUM_BUILDEP_ERRORS="ValueError: SELinux policy is not managed or store cannot be accessed." + sed "/$IGNORE_YUM_BUILDEP_ERRORS/d" $DIR/yum-builddep.out | grep -i "error:" && exit 1 + # for rgw motr backend build checks + if ! rpm --quiet -q cortx-motr-devel && + { [[ $FOR_MAKE_CHECK ]] || $with_rgw_motr; }; then + $SUDO dnf install -y \ + "$motr_pkgs_url/isa-l-2.30.0-1.el7.${ARCH}.rpm" \ + "$motr_pkgs_url/cortx-motr-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm" \ + "$motr_pkgs_url/cortx-motr-devel-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm" + fi ;; opensuse*|suse|sles) echo "Using zypper to install dependencies" zypp_install="zypper --gpg-auto-import-keys --non-interactive install --no-recommends" - $SUDO $zypp_install systemd-rpm-macros - munge_ceph_spec_in $for_make_check $DIR/ceph.spec + $SUDO $zypp_install systemd-rpm-macros rpm-build || exit 1 + munge_ceph_spec_in $with_seastar false $for_make_check $DIR/ceph.spec $SUDO $zypp_install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1 - $SUDO $zypp_install libxmlsec1-1 libxmlsec1-nss1 libxmlsec1-openssl1 xmlsec1-devel xmlsec1-openssl-devel - ;; - alpine) - # for now we need the testing repo for leveldb - TESTREPO="http://nl.alpinelinux.org/alpine/edge/testing" - if ! grep -qF "$TESTREPO" /etc/apk/repositories ; then - $SUDO echo "$TESTREPO" | sudo tee -a /etc/apk/repositories > /dev/null - fi - source alpine/APKBUILD.in - $SUDO apk --update add abuild build-base ccache $makedepends - if id -u build >/dev/null 2>&1 ; then - $SUDO addgroup build abuild - fi ;; *) echo "$ID is unknown, dependencies will have to be installed manually." - exit 1 + exit 1 ;; esac fi function populate_wheelhouse() { + in_jenkins && echo "CI_DEBUG: Running populate_wheelhouse() in install-deps.sh" local install=$1 shift # although pip comes with virtualenv, having a recent version # of pip matters when it comes to using wheel packages - pip --timeout 300 $install 'setuptools >= 0.8' 'pip >= 7.0' 'wheel >= 0.24' || return 1 + PIP_OPTS="--timeout 300 --exists-action i" + pip $PIP_OPTS $install \ + 'setuptools >= 0.8' 'pip >= 21.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1 if test $# != 0 ; then - pip --timeout 300 $install $@ || return 1 + pip $PIP_OPTS $install $@ || return 1 fi } function activate_virtualenv() { + in_jenkins && echo "CI_DEBUG: Running activate_virtualenv() in install-deps.sh" local top_srcdir=$1 - local interpreter=$2 - local env_dir=$top_srcdir/install-deps-$interpreter + local env_dir=$top_srcdir/install-deps-python3 if ! test -d $env_dir ; then - # Make a temporary virtualenv to get a fresh version of virtualenv - # because CentOS 7 has a buggy old version (v1.10.1) - # https://github.com/pypa/virtualenv/issues/463 - virtualenv ${env_dir}_tmp - # install setuptools before upgrading virtualenv, as the latter needs - # a recent setuptools for setup commands like `extras_require`. - ${env_dir}_tmp/bin/pip install --upgrade setuptools - ${env_dir}_tmp/bin/pip install --upgrade virtualenv - ${env_dir}_tmp/bin/virtualenv --python $interpreter $env_dir - rm -rf ${env_dir}_tmp - + python3 -m venv ${env_dir} . $env_dir/bin/activate if ! populate_wheelhouse install ; then rm -rf $env_dir @@ -437,41 +532,50 @@ function activate_virtualenv() { . $env_dir/bin/activate } +function preload_wheels_for_tox() { + in_jenkins && echo "CI_DEBUG: Running preload_wheels_for_tox() in install-deps.sh" + local ini=$1 + shift + pushd . > /dev/null + cd $(dirname $ini) + local require_files=$(ls *requirements*.txt 2>/dev/null) || true + local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true + local require=$(echo -n "$require_files" | sed -e 's/^/-r /') + local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /') + local md5=wheelhouse/md5 + if test "$require"; then + if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then + rm -rf wheelhouse + fi + fi + if test "$require" && ! test -d wheelhouse ; then + type python3 > /dev/null 2>&1 || continue + activate_virtualenv $top_srcdir || exit 1 + python3 -m pip install --upgrade pip + populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1 + mv $wip_wheelhouse wheelhouse + md5sum $require_files $constraint_files > $md5 + fi + popd > /dev/null +} + # use pip cache if possible but do not store it outside of the source # tree # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching -mkdir -p install-deps-cache -top_srcdir=$(pwd) -export XDG_CACHE_HOME=$top_srcdir/install-deps-cache -wip_wheelhouse=wheelhouse-wip +if $for_make_check; then + mkdir -p install-deps-cache + top_srcdir=$(pwd) + export XDG_CACHE_HOME=$top_srcdir/install-deps-cache + wip_wheelhouse=wheelhouse-wip + # + # preload python modules so that tox can run without network access + # + find . -name tox.ini | while read ini ; do + preload_wheels_for_tox $ini + done + rm -rf $top_srcdir/install-deps-python3 + rm -rf $XDG_CACHE_HOME + type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false) +fi -# -# preload python modules so that tox can run without network access -# -find . -name tox.ini | while read ini ; do - ( - cd $(dirname $ini) - require=$(ls *requirements.txt 2>/dev/null | sed -e 's/^/-r /') - md5=wheelhouse/md5 - if test "$require"; then - if ! test -f $md5 || ! md5sum -c $md5 ; then - rm -rf wheelhouse - fi - fi - if test "$require" && ! test -d wheelhouse ; then - for interpreter in python2.7 python3 ; do - type $interpreter > /dev/null 2>&1 || continue - activate_virtualenv $top_srcdir $interpreter || exit 1 - populate_wheelhouse "wheel -w $wip_wheelhouse" $require || exit 1 - done - mv $wip_wheelhouse wheelhouse - md5sum *requirements.txt > $md5 - fi - ) -done - -for interpreter in python2.7 python3 ; do - rm -rf $top_srcdir/install-deps-$interpreter -done -rm -rf $XDG_CACHE_HOME -git --version || (echo "Dashboard uses git to pull dependencies." ; false) +in_jenkins && echo "CI_DEBUG: End install-deps.sh" || true