2 # -*- mode:sh; tab-width:8; indent-tabs-mode:t -*-
4 # Ceph distributed storage system
6 # Copyright (C) 2014, 2015 Red Hat <contact@redhat.com>
8 # Author: Loic Dachary <loic@dachary.org>
10 # This library is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public
12 # License as published by the Free Software Foundation; either
13 # version 2.1 of the License, or (at your option) any later version.
16 DIR
=/tmp
/install-deps.$$
17 trap "rm -fr $DIR" EXIT
19 if test $
(id
-u) != 0 ; then
22 export LC_ALL
=C
# the following is vulnerable to i18n
26 function munge_ceph_spec_in
{
29 local for_make_check
=$1
32 sed -e 's/@//g' < ceph.spec.
in > $OUTFILE
33 # http://rpm.org/user_doc/conditional_builds.html
34 if $with_seastar; then
35 sed -i -e 's/%bcond_with seastar/%bcond_without seastar/g' $OUTFILE
37 if $for_make_check; then
38 sed -i -e 's/%bcond_with make_check/%bcond_without make_check/g' $OUTFILE
42 function munge_debian_control
{
47 local for_make_check
=$1
52 control
="/tmp/control.$$"
53 grep -v babeltrace debian
/control
> $control
56 if $with_seastar; then
57 sed -i -e 's/^# Crimson[[:space:]]//g' $control
59 if $for_make_check; then
60 sed -i 's/^# Make-Check[[:space:]]/ /g' $control
65 function ensure_decent_gcc_on_ubuntu
{
66 # point gcc to the one offered by g++-7 if the used one is not
68 local old
=$
(gcc
-dumpfullversion -dumpversion)
71 if dpkg
--compare-versions $old ge
${new}.0; then
75 if [ ! -f /usr
/bin
/g
++-${new} ]; then
76 $SUDO tee /etc
/apt
/sources.list.d
/ubuntu-toolchain-r.list
<<EOF
77 deb [lang=none] http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
78 deb [arch=amd64 lang=none] http://mirror.nullivex.com/ppa/ubuntu-toolchain-r-test $codename main
79 deb [arch=amd64 lang=none] http://deb.rug.nl/ppa/mirror/ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
81 # import PPA's signing key into APT's keyring
82 cat << ENDOFKEY | $SUDO apt-key add -
83 -----BEGIN PGP PUBLIC KEY BLOCK-----
85 Comment: Hostname: keyserver.ubuntu.com
87 mI0ESuBvRwEEAMi4cDba7xlKaaoXjO1n1HX8RKrkW+HEIl79nSOSJyvzysajs7zUow/OzCQp
88 9NswqrDmNuH1+lPTTRNAGtK8r2ouq2rnXT1mTl23dpgHZ9spseR73s4ZBGw/ag4bpU5dNUSt
89 vfmHhIjVCuiSpNn7cyy1JSSvSs3N2mxteKjXLBf7ABEBAAG0GkxhdW5jaHBhZCBUb29sY2hh
90 aW4gYnVpbGRziLYEEwECACAFAkrgb0cCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAe
91 k3eiup7yfzGKA/4xzUqNACSlB+k+DxFFHqkwKa/ziFiAlkLQyyhm+iqz80htRZr7Ls/ZRYZl
92 0aSU56/hLe0V+TviJ1s8qdN2lamkKdXIAFfavA04nOnTzyIBJ82EAUT3Nh45skMxo4z4iZMN
93 msyaQpNl/m/lNtOLhR64v5ZybofB2EWkMxUzX8D/FQ==
95 -----END PGP PUBLIC KEY BLOCK-----
97 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get update
-y || true
98 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
install -y g
++-${new}
109 $SUDO update-alternatives
--remove-all gcc || true
110 $SUDO update-alternatives \
111 --install /usr
/bin
/gcc gcc
/usr
/bin
/gcc-
${new} 20 \
112 --slave /usr
/bin
/g
++ g
++ /usr
/bin
/g
++-${new}
114 if [ -f /usr
/bin
/g
++-${old} ]; then
115 $SUDO update-alternatives \
116 --install /usr
/bin
/gcc gcc
/usr
/bin
/gcc-
${old} 10 \
117 --slave /usr
/bin
/g
++ g
++ /usr
/bin
/g
++-${old}
120 $SUDO update-alternatives
--auto gcc
122 # cmake uses the latter by default
123 $SUDO ln -nsf /usr
/bin
/gcc
/usr
/bin
/${ARCH}-linux-gnu-gcc
124 $SUDO ln -nsf /usr
/bin
/g
++ /usr
/bin
/${ARCH}-linux-gnu-g++
127 function install_pkg_on_ubuntu
{
138 if [ $force = "force" ]; then
142 if ! apt
-qq list
$pkg 2>/dev
/null |
grep -q installed
; then
143 missing_pkgs
+=" $pkg"
147 if test -n "$missing_pkgs"; then
148 local shaman_url
="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo"
149 $SUDO curl
--silent --location $shaman_url --output /etc
/apt
/sources.list.d
/$project.list
150 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get update
-y -o Acquire
::Languages
=none
-o Acquire
::Translation
=none || true
151 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
install --allow-unauthenticated -y $missing_pkgs
155 function install_boost_on_ubuntu
{
157 if apt
-qq list ceph-libboost1.67
-dev 2>/dev
/null |
grep -q installed
; then
158 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
-y remove
'ceph-libboost.*1.67.*'
159 $SUDO rm /etc
/apt
/sources.list.d
/ceph-libboost1.67.list
161 local project
=libboost
163 local sha1
=1d7c7a00cc3f37e340bae0360191a757b44ec80c
164 install_pkg_on_ubuntu \
169 ceph-libboost-atomic
$ver-dev \
170 ceph-libboost-chrono
$ver-dev \
171 ceph-libboost-container
$ver-dev \
172 ceph-libboost-context
$ver-dev \
173 ceph-libboost-coroutine
$ver-dev \
174 ceph-libboost-date-time
$ver-dev \
175 ceph-libboost-filesystem
$ver-dev \
176 ceph-libboost-iostreams
$ver-dev \
177 ceph-libboost-program-options
$ver-dev \
178 ceph-libboost-python
$ver-dev \
179 ceph-libboost-random
$ver-dev \
180 ceph-libboost-regex
$ver-dev \
181 ceph-libboost-system
$ver-dev \
182 ceph-libboost-test
$ver-dev \
183 ceph-libboost-thread
$ver-dev \
184 ceph-libboost-timer
$ver-dev
187 function version_lt
{
188 test $1 != $
(echo -e "$1\n$2" |
sort -rV |
head -n 1)
191 function ensure_decent_gcc_on_rh
{
192 local old
=$
(gcc
-dumpversion)
195 if version_lt
$old $expected; then
199 Your GCC is too old. Please run following command to add DTS to your environment:
201 scl enable devtoolset-8 bash
203 Or add following line to the end of ~/.bashrc to add it permanently:
205 source scl_source enable devtoolset-8
207 see https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ for more details.
210 # non-interactive shell
211 source /opt
/rh
/devtoolset-
$dts_ver/enable
220 elif [ $FOR_MAKE_CHECK ]; then
226 if [ x$
(uname
)x
= xFreeBSDx
]; then
227 $SUDO pkg
install -yq \
236 devel
/boost-python-libs \
241 devel
/google-perftools \
243 devel
/py-virtualenv \
245 net
/openldap24-client \
249 misc
/e2fsprogs-libuuid \
256 textproc
/xmlstarlet \
267 devel
/py-prettytable \
275 security
/oath-toolkit \
277 sysutils
/fusefs-libs \
279 # Now use pip to install some extra python modules
284 [ $WITH_SEASTAR ] && with_seastar
=true || with_seastar
=false
285 source /etc
/os-release
287 debian|ubuntu|devuan
)
288 echo "Using apt-get to install dependencies"
289 $SUDO apt-get
install -y devscripts equivs
290 $SUDO apt-get
install -y dpkg-dev
293 ensure_decent_gcc_on_ubuntu
9 bionic
294 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu bionic
297 [ ! $NO_BOOST_PKGS ] && apt-get
install -y libboost1.67
-all-dev
300 $SUDO apt-get
install -y gcc
303 if ! test -r debian
/control
; then
304 echo debian
/control is not a readable
file
310 control
=$
(munge_debian_control
"$VERSION" "$with_seastar" "$for_make_check" "debian/control")
313 backports
="-t $codename-backports"
317 # make a metapackage that expresses the build dependencies,
318 # install it, rm the .deb; then uninstall the package as its
320 $SUDO env DEBIAN_FRONTEND
=noninteractive mk-build-deps
--install --remove --tool="apt-get -y --no-install-recommends $backports" $control ||
exit 1
321 $SUDO env DEBIAN_FRONTEND
=noninteractive apt-get
-y remove ceph-build-deps
322 if [ "$control" != "debian/control" ] ; then rm $control; fi
324 centos|fedora|rhel|ol|virtuozzo
)
326 builddepcmd
="dnf -y builddep --allowerasing"
327 if [[ $ID =~ centos|rhel
]] && version_lt
$VERSION_ID 8; then
329 builddepcmd
="yum-builddep -y --setopt=*.skip_if_unavailable=true"
331 echo "Using $yumdnf to install dependencies"
332 if [ "$ID" = "centos" -a "$ARCH" = "aarch64" ]; then
333 $SUDO yum-config-manager
--disable centos-sclo-sclo || true
334 $SUDO yum-config-manager
--disable centos-sclo-rh || true
335 $SUDO yum remove centos-release-scl || true
339 $SUDO $yumdnf install -y $yumdnf-utils
341 centos|rhel|ol|virtuozzo
)
342 MAJOR_VERSION
="$(echo $VERSION_ID | cut -d. -f1)"
343 $SUDO $yumdnf install -y $yumdnf-utils
344 rpm
--quiet --query epel-release || \
345 $SUDO $yumdnf -y install --nogpgcheck https
://dl.fedoraproject.org
/pub
/epel
/epel-release-latest-
$MAJOR_VERSION.noarch.rpm
346 $SUDO rpm
--import /etc
/pki
/rpm-gpg
/RPM-GPG-KEY-EPEL-
$MAJOR_VERSION
347 $SUDO rm -f /etc
/yum.repos.d
/dl.fedoraproject.org
*
348 if test $ID = centos
-a $MAJOR_VERSION = 7 ; then
351 $SUDO $yumdnf -y install centos-release-scl
355 $SUDO $yumdnf -y install centos-release-scl-rh
356 $SUDO yum-config-manager
--disable centos-sclo-rh
357 $SUDO yum-config-manager
--enable centos-sclo-rh-testing
361 elif test $ID = rhel
-a $MAJOR_VERSION = 7 ; then
362 $SUDO yum-config-manager \
363 --enable rhel-server-rhscl-7-rpms \
364 --enable rhel-7-server-optional-rpms \
365 --enable rhel-7-server-devtools-rpms
367 elif test $ID = centos
-a $MAJOR_VERSION = 8 ; then
368 $SUDO dnf config-manager
--set-enabled PowerTools
369 # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies
370 $SUDO dnf config-manager
--add-repo http
://apt-mirror.front.sepia.ceph.com
/lab-extras
/8/
371 $SUDO dnf config-manager
--setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck
=0 --save
372 $SUDO dnf copr
enable -y ktdreyer
/ceph-el8
373 elif test $ID = rhel
-a $MAJOR_VERSION = 8 ; then
374 $SUDO subscription-manager repos
--enable "codeready-builder-for-rhel-8-*-rpms"
375 $SUDO dnf config-manager
--add-repo http
://apt-mirror.front.sepia.ceph.com
/lab-extras
/8/
376 $SUDO dnf config-manager
--setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck
=0 --save
377 $SUDO dnf copr
enable -y ktdreyer
/ceph-el8
381 munge_ceph_spec_in
$with_seastar $for_make_check $DIR/ceph.spec
382 # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
383 $SUDO $yumdnf install -y python3-devel
384 $SUDO $builddepcmd $DIR/ceph.spec
2>&1 |
tee $DIR/yum-builddep.out
385 [ ${PIPESTATUS[0]} -ne 0 ] && exit 1
386 if [ -n "$dts_ver" ]; then
387 ensure_decent_gcc_on_rh
$dts_ver
389 IGNORE_YUM_BUILDEP_ERRORS
="ValueError: SELinux policy is not managed or store cannot be accessed."
390 sed "/$IGNORE_YUM_BUILDEP_ERRORS/d" $DIR/yum-builddep.out |
grep -qi "error:" && exit 1
393 echo "Using zypper to install dependencies"
394 zypp_install
="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
395 $SUDO $zypp_install systemd-rpm-macros rpm-build ||
exit 1
396 munge_ceph_spec_in
$with_seastar $for_make_check $DIR/ceph.spec
397 $SUDO $zypp_install $
(rpmspec
-q --buildrequires $DIR/ceph.spec
) ||
exit 1
400 # for now we need the testing repo for leveldb
401 TESTREPO
="http://nl.alpinelinux.org/alpine/edge/testing"
402 if ! grep -qF "$TESTREPO" /etc
/apk
/repositories
; then
403 $SUDO echo "$TESTREPO" | sudo
tee -a /etc
/apk
/repositories
> /dev
/null
405 source alpine
/APKBUILD.
in
406 $SUDO apk
--update add abuild build-base ccache
$makedepends
407 if id
-u build
>/dev
/null
2>&1 ; then
408 $SUDO addgroup build abuild
412 echo "$ID is unknown, dependencies will have to be installed manually."
418 function populate_wheelhouse
() {
422 # although pip comes with virtualenv, having a recent version
423 # of pip matters when it comes to using wheel packages
424 PIP_OPTS
="--timeout 300 --exists-action i"
425 pip
$PIP_OPTS $install \
426 'setuptools >= 0.8' 'pip >= 7.0' 'wheel >= 0.24' 'tox >= 2.9.1' ||
return 1
427 if test $# != 0 ; then
428 pip
$PIP_OPTS $install $@ ||
return 1
432 function activate_virtualenv
() {
434 local env_dir
=$top_srcdir/install-deps-python3
436 if ! test -d $env_dir ; then
437 virtualenv
--python=python3
${env_dir}
438 .
$env_dir/bin
/activate
439 if ! populate_wheelhouse
install ; then
444 .
$env_dir/bin
/activate
447 function preload_wheels_for_tox
() {
452 local require_files
=$
(ls *requirements
*.txt
2>/dev
/null
) || true
453 local constraint_files
=$
(ls *constraints
*.txt
2>/dev
/null
) || true
454 local require
=$
(echo -n "$require_files" |
sed -e 's/^/-r /')
455 local constraint
=$
(echo -n "$constraint_files" |
sed -e 's/^/-c /')
456 local md5
=wheelhouse
/md5
457 if test "$require"; then
458 if ! test -f $md5 ||
! md5sum -c $md5 > /dev
/null
; then
462 if test "$require" && ! test -d wheelhouse
; then
463 type python3
> /dev
/null
2>&1 ||
continue
464 activate_virtualenv
$top_srcdir ||
exit 1
465 populate_wheelhouse
"wheel -w $wip_wheelhouse" $require $constraint ||
exit 1
466 mv $wip_wheelhouse wheelhouse
467 md5sum $require_files $constraint_files > $md5
472 # use pip cache if possible but do not store it outside of the source
474 # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching
475 if $for_make_check; then
476 mkdir
-p install-deps-cache
478 export XDG_CACHE_HOME
=$top_srcdir/install-deps-cache
479 wip_wheelhouse
=wheelhouse-wip
481 # preload python modules so that tox can run without network access
483 find .
-name tox.ini |
while read ini
; do
484 preload_wheels_for_tox
$ini
486 rm -rf $top_srcdir/install-deps-python3
487 rm -rf $XDG_CACHE_HOME
488 type git
> /dev
/null ||
(echo "Dashboard uses git to pull dependencies." ; false
)