ceph/src/pybind/mgr/dashboard/run-backend-api-tests.sh

   1 #!/usr/bin/env bash
   2
   3 set -eo pipefail
   4
   5 if [[ "$1" = "-h" || "$1" = "--help" ]]; then
   6         echo "Usage (run from ./):"
   7         echo -e "\t./run-backend-api-tests.sh"
   8         echo -e "\t./run-backend-api-tests.sh [tests]..."
   9         echo
  10         echo "Example:"
  11         echo -e "\t./run-backend-api-tests.sh tasks.mgr.dashboard.test_pool.DashboardTest"
  12         echo
  13         echo "Or source this script. This allows to re-run tests faster:"
  14         echo -e "\tsource run-backend-api-tests.sh"
  15         echo -e "\trun_teuthology_tests [tests]..."
  16         echo -e "\tcleanup_teuthology"
  17         echo
  18
  19         exit 0
  20 fi
  21
  22 get_cmake_variable() {
  23     local variable=$1
  24     grep "$variable" CMakeCache.txt | cut -d "=" -f 2
  25 }
  26
  27 get_build_py_version() {
  28     CURR_DIR=`pwd`
  29     BUILD_DIR="$CURR_DIR/../../../../build"
  30     cd $BUILD_DIR
  31
  32     CEPH_MGR_PY_VERSION_MAJOR=$(get_cmake_variable MGR_PYTHON_VERSION | cut -d '.' -f1)
  33     if [ -n "$CEPH_MGR_PY_VERSION_MAJOR" ]; then
  34         CEPH_PY_VERSION_MAJOR=${CEPH_MGR_PY_VERSION_MAJOR}
  35     else
  36         if [ $(get_cmake_variable WITH_PYTHON2) = ON ]; then
  37             CEPH_PY_VERSION_MAJOR=2
  38         else
  39             CEPH_PY_VERSION_MAJOR=3
  40         fi
  41     fi
  42 }
  43
  44 setup_teuthology() {
  45     TEMP_DIR=`mktemp -d`
  46     TEUTHOLOGY_PY_REQS="
  47 apache-libcloud==2.2.1
  48 asn1crypto==0.22.0
  49 backports.ssl-match-hostname==3.5.0.1
  50 bcrypt==3.1.4
  51 certifi==2018.1.18
  52 cffi==1.10.0
  53 chardet==3.0.4
  54 configobj==5.0.6
  55 cryptography==2.1.4
  56 enum34==1.1.6
  57 gevent==1.2.2
  58 greenlet==0.4.13
  59 idna==2.5
  60 ipaddress==1.0.18
  61 Jinja2==2.9.6
  62 manhole==1.5.0
  63 MarkupSafe==1.0
  64 netaddr==0.7.19
  65 packaging==16.8
  66 paramiko==2.4.0
  67 pexpect==4.4.0
  68 psutil==5.4.3
  69 ptyprocess==0.5.2
  70 pyasn1==0.2.3
  71 pycparser==2.17
  72 PyNaCl==1.2.1
  73 pyparsing==2.2.0
  74 python-dateutil==2.6.1
  75 PyYAML==3.12
  76 requests==2.18.4
  77 six==1.10.0
  78 urllib3==1.22
  79 "
  80
  81     cd $TEMP_DIR
  82     virtualenv --python=${TEUTHOLOGY_PYTHON_BIN:-/usr/bin/python} venv
  83     source venv/bin/activate
  84     pip install 'setuptools >= 12'
  85     eval pip install $TEUTHOLOGY_PY_REQS
  86     pip install -r $CURR_DIR/requirements.txt
  87
  88     git clone --depth 1 https://github.com/ceph/teuthology.git
  89
  90     deactivate
  91 }
  92
  93 setup_coverage() {
  94     # In CI environment we cannot install coverage in system, so we install it in a dedicated venv
  95     # so only coverage is available when adding this path.
  96     cd $TEMP_DIR
  97     virtualenv --python=${TEUTHOLOGY_PYTHON_BIN:-/usr/bin/python} coverage-venv
  98     source coverage-venv/bin/activate
  99     cd $CURR_DIR
 100     pip install coverage==4.5.2
 101     COVERAGE_PATH=$(python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])")
 102     deactivate
 103 }
 104
 105 on_tests_error() {
 106     if [[ -n "$JENKINS_HOME" ]]; then
 107         CEPH_OUT_DIR=${CEPH_OUT_DIR:-"$BUILD_DIR"/out}
 108         MGR_LOG_FILES=$(find "$CEPH_OUT_DIR" -iname "mgr.*.log" | tr '\n' ' ')
 109         MGR_LOG_FILE_LAST_LINES=60000
 110         for mgr_log_file in ${MGR_LOG_FILES[@]}; do
 111             printf "\n\nDisplaying last ${MGR_LOG_FILE_LAST_LINES} lines of: $mgr_log_file\n\n"
 112             tail -n ${MGR_LOG_FILE_LAST_LINES} $mgr_log_file
 113             printf "\n\nEnd of: $mgr_log_file\n\n"
 114         done
 115         printf "\n\nTEST FAILED.\n\n"
 116     fi
 117 }
 118
 119 run_teuthology_tests() {
 120     trap on_tests_error ERR
 121
 122     cd "$BUILD_DIR"
 123     find ../src/pybind/mgr/dashboard/ -name '*.pyc' -exec rm -f {} \;
 124
 125     OPTIONS=''
 126     TEST_CASES=''
 127     if [[ "$@" == '' || "$@" == '--create-cluster-only' ]]; then
 128       TEST_CASES=`for i in \`ls $BUILD_DIR/../qa/tasks/mgr/dashboard/test_*\`; do F=$(basename $i); M="${F%.*}"; echo -n " tasks.mgr.dashboard.$M"; done`
 129       # Mgr selftest module tests have to be run at the end as they stress the mgr daemon.
 130       TEST_CASES="tasks.mgr.test_dashboard $TEST_CASES tasks.mgr.test_module_selftest"
 131       if [[ "$@" == '--create-cluster-only' ]]; then
 132         OPTIONS="$@"
 133       fi
 134     else
 135       for t in "$@"; do
 136         TEST_CASES="$TEST_CASES $t"
 137       done
 138     fi
 139
 140     export PATH=$BUILD_DIR/bin:$PATH
 141     source $TEMP_DIR/venv/bin/activate # Run after setting PATH as it does the last PATH export.
 142     export LD_LIBRARY_PATH=$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/:$BUILD_DIR/lib
 143     local source_dir=$(dirname "$BUILD_DIR")
 144     local pybind_dir=$source_dir/src/pybind
 145     local python_common_dir=$source_dir/src/python-common
 146     # In CI environment we set python paths inside build (where you find the required frontend build: "dist" dir).
 147     if [[ -n "$JENKINS_HOME" ]]; then
 148         export PYBIND=$BUILD_DIR/src/pybind
 149         pybind_dir=$PYBIND
 150     fi
 151     export PYTHONPATH=$TEMP_DIR/teuthology:$source_dir/qa:$BUILD_DIR/lib/cython_modules/lib.${CEPH_PY_VERSION_MAJOR}/:$pybind_dir:$python_common_dir:${COVERAGE_PATH}
 152     export RGW=${RGW:-1}
 153
 154     export COVERAGE_ENABLED=true
 155     export COVERAGE_FILE=.coverage.mgr.dashboard
 156     find . -iname "*${COVERAGE_FILE}*" -type f -delete
 157
 158     python ../qa/tasks/vstart_runner.py --ignore-missing-binaries $OPTIONS $TEST_CASES
 159
 160     deactivate
 161     cd $CURR_DIR
 162 }
 163
 164 cleanup_teuthology() {
 165     cd "$BUILD_DIR"
 166     killall ceph-mgr
 167     sleep 10
 168     if [[ "$COVERAGE_ENABLED" == 'true' ]]; then
 169         source $TEMP_DIR/coverage-venv/bin/activate
 170         (coverage combine && coverage report) || true
 171         deactivate
 172     fi
 173     ../src/stop.sh
 174     sleep 5
 175
 176     cd $CURR_DIR
 177     rm -rf $TEMP_DIR
 178
 179     unset TEMP_DIR
 180     unset CURR_DIR
 181     unset BUILD_DIR
 182     unset COVERAGE_PATH
 183     unset get_build_py_version
 184     unset setup_teuthology
 185     unset setup_coverage
 186     unset on_tests_error
 187     unset run_teuthology_tests
 188     unset cleanup_teuthology
 189 }
 190
 191 get_build_py_version
 192 setup_teuthology
 193 setup_coverage
 194 run_teuthology_tests --create-cluster-only
 195
 196 # End sourced section. Do not exit shell when the script has been sourced.
 197 set +e
 198 return 2>/dev/null || set -eo pipefail
 199
 200 run_teuthology_tests "$@"
 201 cleanup_teuthology