ceph/run-make-check.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Ceph distributed storage system
   4 #
   5 # Copyright (C) 2014 Red Hat <contact@redhat.com>
   6 #
   7 # Author: Loic Dachary <loic@dachary.org>
   8 #
   9 #  This library is free software; you can redistribute it and/or
  10 #  modify it under the terms of the GNU Lesser General Public
  11 #  License as published by the Free Software Foundation; either
  12 #  version 2.1 of the License, or (at your option) any later version.
  13 #
  14
  15 #
  16 # To just look at what this script will do, run it like this:
  17 #
  18 # $ DRY_RUN=echo ./run-make-check.sh
  19 #
  20
  21 set -e
  22
  23 trap clean_up_after_myself EXIT
  24
  25 ORIGINAL_CCACHE_CONF="$HOME/.ccache/ccache.conf"
  26 SAVED_CCACHE_CONF="$HOME/.run-make-check-saved-ccache-conf"
  27
  28 function save_ccache_conf() {
  29     test -f $ORIGINAL_CCACHE_CONF && cp $ORIGINAL_CCACHE_CONF $SAVED_CCACHE_CONF || true
  30 }
  31
  32 function restore_ccache_conf() {
  33     test -f $SAVED_CCACHE_CONF && mv $SAVED_CCACHE_CONF $ORIGINAL_CCACHE_CONF || true
  34 }
  35
  36 function clean_up_after_myself() {
  37     rm -fr ${CEPH_BUILD_VIRTUALENV:-/tmp}/*virtualenv*
  38     restore_ccache_conf
  39 }
  40
  41 function get_processors() {
  42     if test -n "$NPROC" ; then
  43         echo $NPROC
  44     else
  45         if test $(nproc) -ge 2 ; then
  46             expr $(nproc) / 2
  47         else
  48             echo 1
  49         fi
  50     fi
  51 }
  52
  53 function detect_ceph_dev_pkgs() {
  54     local cmake_opts
  55     local boost_root=/opt/ceph
  56     if test -f $boost_root/include/boost/config.hpp; then
  57         cmake_opts+=" -DWITH_SYSTEM_BOOST=ON -DBOOST_ROOT=$boost_root"
  58     else
  59         cmake_opts+=" -DBOOST_J=$(get_processors)"
  60     fi
  61     echo "$cmake_opts"
  62 }
  63
  64 function run() {
  65     local install_cmd
  66     local which_pkg="which"
  67     source /etc/os-release
  68     if test -f /etc/redhat-release ; then
  69         if ! type bc > /dev/null 2>&1 ; then
  70             echo "Please install bc and re-run."
  71             exit 1
  72         fi
  73         if test "$(echo "$VERSION_ID >= 22" | bc)" -ne 0; then
  74             install_cmd="dnf -y install"
  75         else
  76             install_cmd="yum install -y"
  77         fi
  78     elif type zypper > /dev/null 2>&1 ; then
  79         install_cmd="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
  80     elif type apt-get > /dev/null 2>&1 ; then
  81         install_cmd="apt-get install -y"
  82         which_pkg="debianutils"
  83     fi
  84
  85     if ! type sudo > /dev/null 2>&1 ; then
  86         echo "Please install sudo and re-run. This script assumes it is running"
  87         echo "as a normal user with the ability to run commands as root via sudo."
  88         exit 1
  89     fi
  90     if [ -n "$install_cmd" ]; then
  91         $DRY_RUN sudo $install_cmd ccache $which_pkg
  92     else
  93         echo "WARNING: Don't know how to install packages" >&2
  94         echo "This probably means distribution $ID is not supported by run-make-check.sh" >&2
  95     fi
  96
  97     if ! type ccache > /dev/null 2>&1 ; then
  98         echo "ERROR: ccache could not be installed"
  99         exit 1
 100     fi
 101
 102     if test -f ./install-deps.sh ; then
 103             export WITH_SEASTAR=1
 104             export FOR_MAKE_CHECK=1
 105             $DRY_RUN source ./install-deps.sh || return 1
 106         trap clean_up_after_myself EXIT
 107     fi
 108
 109     # Init defaults after deps are installed. get_processors() depends on coreutils nproc.
 110     DEFAULT_MAKEOPTS=${DEFAULT_MAKEOPTS:--j$(get_processors)}
 111     BUILD_MAKEOPTS=${BUILD_MAKEOPTS:-$DEFAULT_MAKEOPTS}
 112     test "$BUILD_MAKEOPTS" && echo "make will run with option(s) $BUILD_MAKEOPTS"
 113     CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
 114     CMAKE_BUILD_OPTS="-DWITH_GTEST_PARALLEL=ON -DWITH_FIO=ON -DWITH_SEASTAR=ON"
 115     CMAKE_BUILD_OPTS+=$(detect_ceph_dev_pkgs)
 116     cat <<EOM
 117 Note that the binaries produced by this script do not contain correct time
 118 and git version information, which may make them unsuitable for debugging
 119 and production use.
 120 EOM
 121     save_ccache_conf
 122     # remove the entropy generated by the date/time embedded in the build
 123     CMAKE_BUILD_OPTS+=" -DENABLE_GIT_VERSION=OFF"
 124     $DRY_RUN export SOURCE_DATE_EPOCH="946684800"
 125     $DRY_RUN ccache -o sloppiness=time_macros
 126     $DRY_RUN ccache -o run_second_cpp=true
 127     if [ -n "$JENKINS_HOME" ]; then
 128         # Build host has plenty of space available, let's use it to keep
 129         # various versions of the built objects. This could increase the cache hit
 130         # if the same or similar PRs are running several times
 131         $DRY_RUN ccache -o max_size=100G
 132     else
 133         echo "Current ccache max_size setting:"
 134         ccache -p | grep max_size
 135     fi
 136     $DRY_RUN ccache -sz # Reset the ccache statistics and show the current configuration
 137
 138     $DRY_RUN ./do_cmake.sh $CMAKE_BUILD_OPTS $@ || return 1
 139     $DRY_RUN cd build
 140     $DRY_RUN make $BUILD_MAKEOPTS tests || return 1
 141
 142     $DRY_RUN ccache -s # print the ccache statistics to evaluate the efficiency
 143
 144     # to prevent OSD EMFILE death on tests, make sure ulimit >= 1024
 145     $DRY_RUN ulimit -n $(ulimit -Hn)
 146     if [ $(ulimit -n) -lt 1024 ];then
 147         echo "***ulimit -n too small, better bigger than 1024 for test***"
 148         return 1
 149     fi
 150
 151     # increase the aio-max-nr, which is by default 65536. we could reach this
 152     # limit while running seastar tests and bluestore tests.
 153     $DRY_RUN sudo sysctl -q -w fs.aio-max-nr=$((65536 * 16))
 154
 155     if ! $DRY_RUN ctest $CHECK_MAKEOPTS --output-on-failure; then
 156         rm -fr ${TMPDIR:-/tmp}/ceph-asok.*
 157         return 1
 158     fi
 159 }
 160
 161 function main() {
 162     if [[ $EUID -eq 0 ]] ; then
 163         echo "For best results, run this script as a normal user configured"
 164         echo "with the ability to run commands as root via sudo."
 165     fi
 166     echo -n "Checking hostname sanity... "
 167     if $DRY_RUN hostname --fqdn >/dev/null 2>&1 ; then
 168         echo "OK"
 169     else
 170         echo "NOT OK"
 171         echo "Please fix 'hostname --fqdn', otherwise 'make check' will fail"
 172         return 1
 173     fi
 174     run "$@" && echo "make check: successful run on $(git rev-parse HEAD)"
 175 }
 176
 177 main "$@"