ceph/src/ceph-disk/tests/ceph-disk.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
   4 # Copyright (C) 2014, 2015, 2016 Red Hat <contact@redhat.com>
   5 #
   6 # Author: Loic Dachary <loic@dachary.org>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU Library Public License as published by
  10 # the Free Software Foundation; either version 2, or (at your option)
  11 # any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Library Public License for more details.
  17 #
  18
  19 # ceph-disk.sh is launched by tox which expects tox.ini in current
  20 # directory. so we cannot run ceph-disk.sh in build directory directly,
  21 # and hence not able to use detect-build-env-vars.sh to set the build
  22 # env vars.
  23 if [ -z "$CEPH_ROOT" ] || [ -z "$CEPH_BIN" ] || [ -z "$CEPH_LIB" ]; then
  24     CEPH_ROOT=`readlink -f $(dirname $0)/../../..`
  25     CEPH_BIN=$CEPH_ROOT
  26     CEPH_LIB=$CEPH_ROOT/.libs
  27 fi
  28 source $CEPH_ROOT/qa/workunits/ceph-helpers.sh
  29
  30 set -x
  31
  32 PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
  33
  34 export PATH=$CEPH_BIN:.:$PATH # make sure program from sources are preferred
  35 export LD_LIBRARY_PATH=$CEPH_LIB
  36 : ${CEPH_DISK:=ceph-disk}
  37 CEPH_DISK_ARGS=
  38 CEPH_DISK_ARGS+=" --verbose"
  39 CEPH_DISK_ARGS+=" --prepend-to-path="
  40 TIMEOUT=360
  41 if [ `uname` != FreeBSD ]; then
  42     PROCDIR=""
  43 else
  44     PROCDIR="/compat/linux"
  45 fi
  46
  47 cat=$(which cat)
  48 timeout=$(which timeout)
  49 diff=$(which diff)
  50 mkdir=$(which mkdir)
  51 rm=$(which rm)
  52 uuidgen=$(which uuidgen)
  53
  54 function setup() {
  55     local dir=$1
  56     teardown $dir
  57     mkdir -p $dir/osd
  58     touch $dir/ceph.conf # so ceph-disk think ceph is the cluster
  59 }
  60
  61 function teardown() {
  62     local dir=$1
  63     if ! test -e $dir ; then
  64         return
  65     fi
  66     kill_daemons $dir
  67     if [ `uname` != FreeBSD ] && \
  68        [ $(stat -f -c '%T' .) == "btrfs" ]; then
  69         rm -fr $dir/*/*db
  70         __teardown_btrfs $dir
  71     fi
  72     grep " $(pwd)/$dir/" < ${PROCDIR}/proc/mounts | while read mounted rest ; do
  73         umount $mounted
  74     done
  75     rm -fr $dir
  76 }
  77
  78 function command_fixture() {
  79     local dir=$1
  80     shift
  81     local command=$1
  82     shift
  83     local fpath=`readlink -f $(which $command)`
  84     [ "$fpath" = `readlink -f $CEPH_BIN/$command` ] || [ "$fpath" = `readlink -f $(pwd)/$command` ] || return 1
  85
  86     cat > $dir/$command <<EOF
  87 #!/bin/bash
  88 touch $dir/used-$command
  89 exec $CEPH_BIN/$command "\$@"
  90 EOF
  91     chmod +x $dir/$command
  92 }
  93
  94 function tweak_path() {
  95     local dir=$1
  96     shift
  97     local tweaker=$1
  98     shift
  99
 100     setup $dir
 101
 102     command_fixture $dir ceph-conf || return 1
 103     command_fixture $dir ceph-osd || return 1
 104
 105     test_activate_dir $dir || return 1
 106
 107     [ ! -f $dir/used-ceph-conf ] || return 1
 108     [ ! -f $dir/used-ceph-osd ] || return 1
 109
 110     teardown $dir
 111
 112     setup $dir
 113
 114     command_fixture $dir ceph-conf || return 1
 115     command_fixture $dir ceph-osd || return 1
 116
 117     $tweaker $dir test_activate_dir || return 1
 118
 119     [ -f $dir/used-ceph-osd ] || return 1
 120
 121     teardown $dir
 122 }
 123
 124 function use_prepend_to_path() {
 125     local dir=$1
 126     shift
 127
 128     local ceph_disk_args
 129     ceph_disk_args+=" --statedir=$dir"
 130     ceph_disk_args+=" --sysconfdir=$dir"
 131     ceph_disk_args+=" --prepend-to-path=$dir"
 132     ceph_disk_args+=" --verbose"
 133     CEPH_DISK_ARGS="$ceph_disk_args" \
 134         "$@" $dir || return 1
 135 }
 136
 137 function test_prepend_to_path() {
 138     local dir=$1
 139     shift
 140     tweak_path $dir use_prepend_to_path || return 1
 141 }
 142
 143 function use_path() {
 144     local dir=$1
 145     shift
 146     PATH="$dir:$PATH" \
 147         "$@" $dir || return 1
 148 }
 149
 150 function test_path() {
 151     local dir=$1
 152     shift
 153     tweak_path $dir use_path || return 1
 154 }
 155
 156 function test_no_path() {
 157     local dir=$1
 158     shift
 159     ( export PATH=../ceph-detect-init/virtualenv/bin:virtualenv/bin:$CEPH_BIN:/usr/bin:/bin:/usr/local/bin ; test_activate_dir $dir) || return 1
 160 }
 161
 162 function test_mark_init() {
 163     local dir=$1
 164     shift
 165
 166     run_mon $dir a
 167
 168     local osd_data=$dir/dir
 169     $mkdir -p $osd_data
 170
 171     local osd_uuid=$($uuidgen)
 172
 173     $mkdir -p $osd_data
 174
 175     ${CEPH_DISK} $CEPH_DISK_ARGS \
 176         prepare --osd-uuid $osd_uuid $osd_data || return 1
 177
 178     ${CEPH_DISK} $CEPH_DISK_ARGS \
 179         --verbose \
 180         activate \
 181         --mark-init=auto \
 182         --no-start-daemon \
 183         $osd_data || return 1
 184
 185     test -f $osd_data/$(ceph-detect-init) || return 1
 186
 187     if test systemd = $(ceph-detect-init) ; then
 188         expected=sysvinit
 189     else
 190         expected=systemd
 191     fi
 192     $timeout $TIMEOUT ${CEPH_DISK} $CEPH_DISK_ARGS \
 193         --verbose \
 194         activate \
 195         --mark-init=$expected \
 196         --no-start-daemon \
 197         $osd_data || return 1
 198
 199     ! test -f $osd_data/$(ceph-detect-init) || return 1
 200     test -f $osd_data/$expected || return 1
 201 }
 202
 203 function test_zap() {
 204     local dir=$1
 205     local osd_data=$dir/dir
 206     $mkdir -p $osd_data
 207
 208     ${CEPH_DISK} $CEPH_DISK_ARGS zap $osd_data 2>&1 | grep -q 'not full block device' || return 1
 209
 210     $rm -fr $osd_data
 211 }
 212
 213 # ceph-disk prepare returns immediately on success if the magic file
 214 # exists in the --osd-data directory.
 215 function test_activate_dir_magic() {
 216     local dir=$1
 217     local uuid=$($uuidgen)
 218     local osd_data=$dir/osd
 219
 220     echo a failure to create the fsid file implies the magic file is not created
 221
 222     mkdir -p $osd_data/fsid
 223     CEPH_ARGS="--fsid $uuid" \
 224      ${CEPH_DISK} $CEPH_DISK_ARGS prepare $osd_data > $dir/out 2>&1
 225     grep --quiet 'Is a directory' $dir/out || return 1
 226     ! [ -f $osd_data/magic ] || return 1
 227     rmdir $osd_data/fsid
 228
 229     echo successfully prepare the OSD
 230
 231     CEPH_ARGS="--fsid $uuid" \
 232      ${CEPH_DISK} $CEPH_DISK_ARGS prepare $osd_data 2>&1 | tee $dir/out
 233     grep --quiet 'Preparing osd data dir' $dir/out || return 1
 234     grep --quiet $uuid $osd_data/ceph_fsid || return 1
 235     [ -f $osd_data/magic ] || return 1
 236
 237     echo will not override an existing OSD
 238
 239     CEPH_ARGS="--fsid $($uuidgen)" \
 240      ${CEPH_DISK} $CEPH_DISK_ARGS prepare $osd_data 2>&1 | tee $dir/out
 241     grep --quiet 'Data dir .* already exists' $dir/out || return 1
 242     grep --quiet $uuid $osd_data/ceph_fsid || return 1
 243 }
 244
 245 function test_pool_read_write() {
 246     local osd_uuid=$1
 247     local TEST_POOL=rbd
 248
 249     $timeout $TIMEOUT ceph osd pool set $TEST_POOL size 1 || return 1
 250
 251     local id=$(ceph osd create $osd_uuid)
 252     local weight=1
 253     ceph osd crush add osd.$id $weight root=default host=localhost || return 1
 254     echo FOO > $dir/BAR
 255     $timeout $TIMEOUT rados --pool $TEST_POOL put BAR $dir/BAR || return 1
 256     $timeout $TIMEOUT rados --pool $TEST_POOL get BAR $dir/BAR.copy || return 1
 257     $diff $dir/BAR $dir/BAR.copy || return 1
 258 }
 259
 260 function test_activate() {
 261     local to_prepare=$1
 262     local to_activate=$2
 263     local osd_uuid=$($uuidgen)
 264     local timeoutcmd
 265
 266     if [ `uname` = FreeBSD ]; then
 267         # for unknown reasons FreeBSD timeout does not return here
 268         # So we run without timeout
 269         timeoutcmd=""
 270     else
 271         timeoutcmd="${timeout} $TIMEOUT"
 272     fi
 273
 274     ${CEPH_DISK} $CEPH_DISK_ARGS \
 275         prepare --osd-uuid $osd_uuid $to_prepare || return 1
 276
 277     $timeoutcmd ${CEPH_DISK} $CEPH_DISK_ARGS \
 278         activate \
 279         --mark-init=none \
 280         $to_activate || return 1
 281
 282     test_pool_read_write $osd_uuid || return 1
 283 }
 284
 285 function test_activate_dir() {
 286     local dir=$1
 287     shift
 288
 289     run_mon $dir a
 290     $@
 291
 292     local osd_data=$dir/dir
 293     $mkdir -p $osd_data
 294     test_activate $osd_data $osd_data || return 1
 295 }
 296
 297 function test_activate_dir_bluestore() {
 298     local dir=$1
 299     run_mon $dir a
 300
 301     local osd_data=$dir/dir
 302     $mkdir -p $osd_data
 303     local to_prepare=$osd_data
 304     local to_activate=$osd_data
 305     local osd_uuid=$($uuidgen)
 306
 307     CEPH_ARGS=" --bluestore-block-size=10737418240 $CEPH_ARGS" \
 308       ${CEPH_DISK} $CEPH_DISK_ARGS \
 309         prepare --bluestore --block-file --osd-uuid $osd_uuid $to_prepare || return 1
 310
 311     CEPH_ARGS=" --osd-objectstore=bluestore --bluestore-fsck-on-mount=true --enable_experimental_unrecoverable_data_corrupting_features=* --bluestore-block-db-size=67108864 --bluestore-block-wal-size=134217728 --bluestore-block-size=10737418240 $CEPH_ARGS" \
 312       $timeout $TIMEOUT ${CEPH_DISK} $CEPH_DISK_ARGS \
 313         activate \
 314         --mark-init=none \
 315         $to_activate || return 1
 316     test_pool_read_write $osd_uuid || return 1
 317 }
 318
 319 function test_find_cluster_by_uuid() {
 320     local dir=$1
 321     test_activate_dir $dir 2>&1 | tee $dir/test_find
 322     ! grep "No cluster conf found in $dir" $dir/test_find || return 1
 323     teardown $dir
 324
 325     setup $dir
 326     test_activate_dir $dir "rm $dir/ceph.conf" > $dir/test_find 2>&1
 327     cp $dir/test_find /tmp
 328     grep --quiet "No cluster conf found in $dir" $dir/test_find || return 1
 329 }
 330
 331 # http://tracker.ceph.com/issues/9653
 332 function test_keyring_path() {
 333     local dir=$1
 334     test_activate_dir $dir 2>&1 | tee $dir/test_keyring
 335     grep --quiet "keyring $dir/bootstrap-osd/ceph.keyring" $dir/test_keyring || return 1
 336 }
 337
 338 # http://tracker.ceph.com/issues/13522
 339 function ceph_osd_fail_once_fixture() {
 340     local dir=$1
 341     local command=ceph-osd
 342     local fpath=`readlink -f $(which $command)`
 343     [ "$fpath" = `readlink -f $CEPH_BIN/$command` ] || [ "$fpath" = `readlink -f $(pwd)/$command` ] || return 1
 344
 345     cat > $dir/$command <<EOF
 346 #!/bin/bash
 347 if echo "\$@" | grep -e --mkfs && ! test -f $dir/used-$command ; then
 348    touch $dir/used-$command
 349    # sleep longer than the first CEPH_OSD_MKFS_DELAYS value (5) below
 350    sleep 600
 351 else
 352    exec $CEPH_BIN/$command "\$@"
 353 fi
 354 EOF
 355     chmod +x $dir/$command
 356 }
 357
 358 function test_ceph_osd_mkfs() {
 359     local dir=$1
 360     ceph_osd_fail_once_fixture $dir || return 1
 361     CEPH_OSD_MKFS_DELAYS='5 300 300' use_path $dir test_activate_dir || return 1
 362     [ -f $dir/used-ceph-osd ] || return 1
 363 }
 364
 365 function test_crush_device_class() {
 366     local dir=$1
 367     shift
 368
 369     run_mon $dir a
 370
 371     local osd_data=$dir/dir
 372     $mkdir -p $osd_data
 373
 374     local osd_uuid=$($uuidgen)
 375
 376     $mkdir -p $osd_data
 377
 378     ${CEPH_DISK} $CEPH_DISK_ARGS \
 379         prepare --osd-uuid $osd_uuid \
 380                 --crush-device-class CRUSH_CLASS \
 381                 $osd_data || return 1
 382     test -f $osd_data/crush_device_class || return 1
 383     test $(cat $osd_data/crush_device_class) = CRUSH_CLASS || return 1
 384
 385     ceph osd crush class create CRUSH_CLASS || return 1
 386
 387     CEPH_ARGS="--crush-location=root=default $CEPH_ARGS" \
 388       ${CEPH_DISK} $CEPH_DISK_ARGS \
 389         --verbose \
 390         activate \
 391         --mark-init=none \
 392         $osd_data || return 1
 393
 394     ok=false
 395     for delay in 2 4 8 16 32 64 128 256 ; do
 396         if ceph osd crush dump | grep --quiet 'CRUSH_CLASS' ; then
 397             ok=true
 398             break
 399         fi
 400         sleep $delay
 401         ceph osd crush dump # for debugging purposes
 402     done
 403     $ok || return 1
 404 }
 405
 406 function run() {
 407     local dir=$1
 408     shift
 409     CEPH_DISK_ARGS+=" --statedir=$dir"
 410     CEPH_DISK_ARGS+=" --sysconfdir=$dir"
 411
 412     export CEPH_MON="127.0.0.1:7451" # git grep '\<7451\>' : there must be only one
 413     export CEPH_ARGS
 414     CEPH_ARGS+=" --fsid=$(uuidgen)"
 415     CEPH_ARGS+=" --auth-supported=none"
 416     CEPH_ARGS+=" --mon-host=$CEPH_MON"
 417     CEPH_ARGS+=" --chdir="
 418     CEPH_ARGS+=" --journal-dio=false"
 419     CEPH_ARGS+=" --erasure-code-dir=$CEPH_LIB"
 420     CEPH_ARGS+=" --plugin-dir=$CEPH_LIB"
 421     CEPH_ARGS+=" --log-file=$dir/\$name.log"
 422     CEPH_ARGS+=" --pid-file=$dir/\$name.pidfile"
 423     CEPH_ARGS+=" --osd-class-dir=$CEPH_LIB"
 424     CEPH_ARGS+=" --run-dir=$dir"
 425     CEPH_ARGS+=" --osd-failsafe-full-ratio=.99"
 426     CEPH_ARGS+=" --osd-journal-size=100"
 427     CEPH_ARGS+=" --debug-osd=20"
 428     CEPH_ARGS+=" --debug-bdev=20"
 429     CEPH_ARGS+=" --debug-bluestore=20"
 430     CEPH_ARGS+=" --osd-max-object-name-len=460"
 431     CEPH_ARGS+=" --osd-max-object-namespace-len=64 "
 432     local default_actions
 433     default_actions+="test_path "
 434     default_actions+="test_no_path "
 435     default_actions+="test_find_cluster_by_uuid "
 436     default_actions+="test_prepend_to_path "
 437     default_actions+="test_activate_dir_magic "
 438     default_actions+="test_activate_dir "
 439     default_actions+="test_keyring_path "
 440     [ `uname` != FreeBSD ] && \
 441       default_actions+="test_mark_init "
 442     default_actions+="test_zap "
 443     [ `uname` != FreeBSD ] && \
 444       default_actions+="test_activate_dir_bluestore "
 445     default_actions+="test_ceph_osd_mkfs "
 446     default_actions+="test_crush_device_class "
 447     local actions=${@:-$default_actions}
 448     for action in $actions  ; do
 449         setup $dir || return 1
 450         set -x
 451         $action $dir || return 1
 452         set +x
 453         teardown $dir || return 1
 454     done
 455 }
 456
 457 main test-ceph-disk "$@"
 458
 459 # Local Variables:
 460 # compile-command: "cd .. ; test/ceph-disk.sh # test_activate_dir"
 461 # End: