2 # -*- mode:sh; tab-width:8; indent-tabs-mode:t -*-
4 # Ceph distributed storage system
6 # Copyright (C) 2014, 2015 Red Hat <contact@redhat.com>
8 # Author: Loic Dachary <loic@dachary.org>
10 # This library is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public
12 # License as published by the Free Software Foundation; either
13 # version 2.1 of the License, or (at your option) any later version.
16 DIR
=/tmp
/install-deps.$$
17 trap "rm -fr $DIR" EXIT
19 if test $
(id
-u) != 0 ; then
22 export LC_ALL
=C
# the following is vulnerable to i18n
26 function munge_ceph_spec_in
{
31 local for_make_check
=$1
36 sed -e 's/@//g' < ceph.spec.
in > $OUTFILE
37 # http://rpm.org/user_doc/conditional_builds.html
38 if $with_seastar; then
39 sed -i -e 's/%bcond_with seastar/%bcond_without seastar/g' $OUTFILE
42 sed -i -e 's/%bcond_with jaeger/%bcond_without jaeger/g' $OUTFILE
45 sed -i -e 's/%bcond_with zbd/%bcond_without zbd/g' $OUTFILE
47 if $for_make_check; then
48 sed -i -e 's/%bcond_with make_check/%bcond_without make_check/g' $OUTFILE
52 function munge_debian_control
{
58 control
="/tmp/control.$$"
59 grep -v babeltrace debian
/control
> $control
63 sed -i -e 's/^# Jaeger[[:space:]]//g' $control
64 sed -i -e 's/^# Crimson libyaml-cpp-dev,/d' $control
69 function ensure_decent_gcc_on_ubuntu
{
70 # point gcc to the one offered by g++-7 if the used one is not
72 local old
=$
(gcc
-dumpfullversion -dumpversion)
75 if dpkg
--compare-versions $old ge
${new}.0; then
79 if [ ! -f /usr
/bin
/g
++-${new} ]; then
80 $SUDO tee /etc
/apt
/sources.list.d
/ubuntu-toolchain-r.list
<<EOF
81 deb [lang=none] http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
82 deb [arch=amd64 lang=none] http://mirror.nullivex.com/ppa/ubuntu-toolchain-r-test $codename main
84 # import PPA's signing key into APT's keyring
85 cat << ENDOFKEY | $SUDO apt-key add -
86 -----BEGIN PGP PUBLIC KEY BLOCK-----
88 Comment: Hostname: keyserver.ubuntu.com
90 mI0ESuBvRwEEAMi4cDba7xlKaaoXjO1n1HX8RKrkW+HEIl79nSOSJyvzysajs7zUow/OzCQp
91 9NswqrDmNuH1+lPTTRNAGtK8r2ouq2rnXT1mTl23dpgHZ9spseR73s4ZBGw/ag4bpU5dNUSt
92 vfmHhIjVCuiSpNn7cyy1JSSvSs3N2mxteKjXLBf7ABEBAAG0GkxhdW5jaHBhZCBUb29sY2hh
93 aW4gYnVpbGRziLYEEwECACAFAkrgb0cCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAe
94 k3eiup7yfzGKA/4xzUqNACSlB+k+DxFFHqkwKa/ziFiAlkLQyyhm+iqz80htRZr7Ls/ZRYZl
95 0aSU56/hLe0V+TviJ1s8qdN2lamkKdXIAFfavA04nOnTzyIBJ82EAUT3Nh45skMxo4z4iZMN
96 msyaQpNl/m/lNtOLhR64v5ZybofB2EWkMxUzX8D/FQ==
98 -----END PGP PUBLIC KEY BLOCK-----
100 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get update
-y || true
101 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
install -y g
++-${new}
112 $SUDO update-alternatives
--remove-all gcc || true
113 $SUDO update-alternatives \
114 --install /usr
/bin
/gcc gcc
/usr
/bin
/gcc-
${new} 20 \
115 --slave /usr
/bin
/g
++ g
++ /usr
/bin
/g
++-${new}
117 if [ -f /usr
/bin
/g
++-${old} ]; then
118 $SUDO update-alternatives \
119 --install /usr
/bin
/gcc gcc
/usr
/bin
/gcc-
${old} 10 \
120 --slave /usr
/bin
/g
++ g
++ /usr
/bin
/g
++-${old}
123 $SUDO update-alternatives
--auto gcc
125 # cmake uses the latter by default
126 $SUDO ln -nsf /usr
/bin
/gcc
/usr
/bin
/${ARCH}-linux-gnu-gcc
127 $SUDO ln -nsf /usr
/bin
/g
++ /usr
/bin
/${ARCH}-linux-gnu-g++
130 function ensure_python3_sphinx_on_ubuntu
{
131 local sphinx_command
=/usr
/bin
/sphinx-build
132 # python-sphinx points $sphinx_command to
133 # ../share/sphinx/scripts/python2/sphinx-build when it's installed
134 # let's "correct" this
135 if test -e $sphinx_command && head -n1 $sphinx_command |
grep -q python$
; then
136 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
-y remove python-sphinx
140 function install_pkg_on_ubuntu
{
151 if [ $force = "force" ]; then
155 if ! apt
-qq list
$pkg 2>/dev
/null |
grep -q installed
; then
156 missing_pkgs
+=" $pkg"
160 if test -n "$missing_pkgs"; then
161 local shaman_url
="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo"
162 $SUDO curl
--silent --location $shaman_url --output /etc
/apt
/sources.list.d
/$project.list
163 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get update
-y -o Acquire
::Languages
=none
-o Acquire
::Translation
=none || true
164 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
install --allow-unauthenticated -y $missing_pkgs
168 function install_boost_on_ubuntu
{
170 local installed_ver
=$
(apt
-qq list
--installed ceph-libboost
*-dev 2>/dev
/null |
171 grep -e 'libboost[0-9].[0-9]\+-dev' |
174 if test -n "$installed_ver"; then
175 if echo "$installed_ver" |
grep -q "^$ver"; then
178 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
-y remove
"ceph-libboost.*${installed_ver}.*"
179 $SUDO rm -f /etc
/apt
/sources.list.d
/ceph-libboost
${installed_ver}.list
183 local project
=libboost
184 local sha1
=7aba8a1882670522ee1d1ee1bba0ea170b292dec
185 install_pkg_on_ubuntu \
190 ceph-libboost-atomic
$ver-dev \
191 ceph-libboost-chrono
$ver-dev \
192 ceph-libboost-container
$ver-dev \
193 ceph-libboost-context
$ver-dev \
194 ceph-libboost-coroutine
$ver-dev \
195 ceph-libboost-date-time
$ver-dev \
196 ceph-libboost-filesystem
$ver-dev \
197 ceph-libboost-iostreams
$ver-dev \
198 ceph-libboost-program-options
$ver-dev \
199 ceph-libboost-python
$ver-dev \
200 ceph-libboost-random
$ver-dev \
201 ceph-libboost-regex
$ver-dev \
202 ceph-libboost-system
$ver-dev \
203 ceph-libboost-test
$ver-dev \
204 ceph-libboost-thread
$ver-dev \
205 ceph-libboost-timer
$ver-dev
208 function install_libzbd_on_ubuntu
{
211 local sha1
=1fadde94b08fab574b17637c2bebd2b1e7f9127b
212 install_pkg_on_ubuntu \
220 function install_libpmem_on_ubuntu
{
223 local sha1
=7c18b4b1413ae965ea8bcbfc69eb9784f9212319
224 install_pkg_on_ubuntu \
233 function version_lt
{
234 test $1 != $
(echo -e "$1\n$2" |
sort -rV |
head -n 1)
241 elif [ $FOR_MAKE_CHECK ]; then
247 if [ x$
(uname
)x
= xFreeBSDx
]; then
248 $SUDO pkg
install -yq \
257 devel
/boost-python-libs \
262 devel
/google-perftools \
265 net
/openldap24-client \
269 misc
/e2fsprogs-libuuid \
276 textproc
/xmlstarlet \
287 devel
/py-prettytable \
296 security
/oath-toolkit \
298 sysutils
/fusefs-libs \
300 # Now use pip to install some extra python modules
305 [ $WITH_SEASTAR ] && with_seastar
=true || with_seastar
=false
306 [ $WITH_JAEGER ] && with_jaeger
=true || with_jaeger
=false
307 [ $WITH_ZBD ] && with_zbd
=true || with_zbd
=false
308 [ $WITH_PMEM ] && with_pmem
=true || with_pmem
=false
309 source /etc
/os-release
311 debian|ubuntu|devuan|elementary|softiron
)
312 echo "Using apt-get to install dependencies"
313 $SUDO apt-get
install -y devscripts equivs
314 $SUDO apt-get
install -y dpkg-dev
315 ensure_python3_sphinx_on_ubuntu
318 ensure_decent_gcc_on_ubuntu
9 bionic
319 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu bionic
320 $with_zbd && install_libzbd_on_ubuntu bionic
323 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu focal
324 $with_zbd && install_libzbd_on_ubuntu focal
325 $with_pmem && install_libpmem_on_ubuntu focal
328 $SUDO apt-get
install -y gcc
331 if ! test -r debian
/control
; then
332 echo debian
/control is not a readable
file
338 control
=$
(munge_debian_control
"$VERSION" "debian/control")
341 backports
="-t $codename-backports"
345 # make a metapackage that expresses the build dependencies,
346 # install it, rm the .deb; then uninstall the package as its
349 if $for_make_check; then
350 build_profiles
+=",pkg.ceph.check"
352 if $with_seastar; then
353 build_profiles
+=",pkg.ceph.crimson"
355 if $with_jaeger; then
356 build_profiles
+=",pkg.ceph.jaeger"
359 $SUDO env DEBIAN_FRONTEND
=noninteractive mk-build-deps \
360 --build-profiles "${build_profiles#,}" \
362 --tool="apt-get -y --no-install-recommends $backports" $control ||
exit 1
363 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
-y remove ceph-build-deps
364 if [ "$control" != "debian/control" ] ; then rm $control; fi
366 centos|fedora|rhel|ol|virtuozzo
)
367 builddepcmd
="dnf -y builddep --allowerasing"
368 echo "Using dnf to install dependencies"
371 $SUDO dnf
install -y dnf-utils
373 centos|rhel|ol|virtuozzo
)
374 MAJOR_VERSION
="$(echo $VERSION_ID | cut -d. -f1)"
375 $SUDO dnf
install -y dnf-utils
376 rpm
--quiet --query epel-release || \
377 $SUDO dnf
-y install --nogpgcheck https
://dl.fedoraproject.org
/pub
/epel
/epel-release-latest-
$MAJOR_VERSION.noarch.rpm
378 $SUDO rpm
--import /etc
/pki
/rpm-gpg
/RPM-GPG-KEY-EPEL-
$MAJOR_VERSION
379 $SUDO rm -f /etc
/yum.repos.d
/dl.fedoraproject.org
*
380 if test $ID = centos
-a $MAJOR_VERSION = 8 ; then
381 # Enable 'powertools' or 'PowerTools' repo
382 $SUDO dnf config-manager
--set-enabled $
(dnf repolist
--all 2>/dev
/null|gawk
'tolower($0) ~ /^powertools\s/{print $1}')
383 # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies
384 $SUDO dnf config-manager
--add-repo http
://apt-mirror.front.sepia.ceph.com
/lab-extras
/8/
385 $SUDO dnf config-manager
--setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck
=0 --save
386 elif test $ID = rhel
-a $MAJOR_VERSION = 8 ; then
387 $SUDO dnf config-manager
--set-enabled "codeready-builder-for-rhel-8-${ARCH}-rpms"
388 $SUDO dnf config-manager
--add-repo http
://apt-mirror.front.sepia.ceph.com
/lab-extras
/8/
389 $SUDO dnf config-manager
--setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck
=0 --save
393 munge_ceph_spec_in
$with_seastar $with_zbd $for_make_check $with_jaeger $DIR/ceph.spec
394 # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
395 $SUDO dnf
install -y python3-devel
396 $SUDO $builddepcmd $DIR/ceph.spec
2>&1 |
tee $DIR/yum-builddep.out
397 [ ${PIPESTATUS[0]} -ne 0 ] && exit 1
398 IGNORE_YUM_BUILDEP_ERRORS
="ValueError: SELinux policy is not managed or store cannot be accessed."
399 sed "/$IGNORE_YUM_BUILDEP_ERRORS/d" $DIR/yum-builddep.out |
grep -qi "error:" && exit 1
402 echo "Using zypper to install dependencies"
403 zypp_install
="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
404 $SUDO $zypp_install systemd-rpm-macros rpm-build ||
exit 1
405 munge_ceph_spec_in
$with_seastar false
$for_make_check $with_jaeger $DIR/ceph.spec
406 $SUDO $zypp_install $
(rpmspec
-q --buildrequires $DIR/ceph.spec
) ||
exit 1
409 echo "$ID is unknown, dependencies will have to be installed manually."
415 function populate_wheelhouse
() {
419 # although pip comes with virtualenv, having a recent version
420 # of pip matters when it comes to using wheel packages
421 PIP_OPTS
="--timeout 300 --exists-action i"
422 pip
$PIP_OPTS $install \
423 'setuptools >= 0.8' 'pip >= 21.0' 'wheel >= 0.24' 'tox >= 2.9.1' ||
return 1
424 if test $# != 0 ; then
425 # '--use-feature=fast-deps --use-deprecated=legacy-resolver' added per
426 # https://github.com/pypa/pip/issues/9818 These should be able to be
427 # removed at some point in the future.
428 pip
--use-feature=fast-deps
--use-deprecated=legacy-resolver
$PIP_OPTS $install $@ ||
return 1
432 function activate_virtualenv
() {
434 local env_dir
=$top_srcdir/install-deps-python3
436 if ! test -d $env_dir ; then
437 python3
-m venv
${env_dir}
438 .
$env_dir/bin
/activate
439 if ! populate_wheelhouse
install ; then
444 .
$env_dir/bin
/activate
447 function preload_wheels_for_tox
() {
452 local require_files
=$
(ls *requirements
*.txt
2>/dev
/null
) || true
453 local constraint_files
=$
(ls *constraints
*.txt
2>/dev
/null
) || true
454 local require
=$
(echo -n "$require_files" |
sed -e 's/^/-r /')
455 local constraint
=$
(echo -n "$constraint_files" |
sed -e 's/^/-c /')
456 local md5
=wheelhouse
/md5
457 if test "$require"; then
458 if ! test -f $md5 ||
! md5sum -c $md5 > /dev
/null
; then
462 if test "$require" && ! test -d wheelhouse
; then
463 type python3
> /dev
/null
2>&1 ||
continue
464 activate_virtualenv
$top_srcdir ||
exit 1
465 populate_wheelhouse
"wheel -w $wip_wheelhouse" $require $constraint ||
exit 1
466 mv $wip_wheelhouse wheelhouse
467 md5sum $require_files $constraint_files > $md5
472 # use pip cache if possible but do not store it outside of the source
474 # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching
475 if $for_make_check; then
476 mkdir
-p install-deps-cache
478 export XDG_CACHE_HOME
=$top_srcdir/install-deps-cache
479 wip_wheelhouse
=wheelhouse-wip
481 # preload python modules so that tox can run without network access
483 find .
-name tox.ini |
while read ini
; do
484 preload_wheels_for_tox
$ini
486 rm -rf $top_srcdir/install-deps-python3
487 rm -rf $XDG_CACHE_HOME
488 type git
> /dev
/null ||
(echo "Dashboard uses git to pull dependencies." ; false
)