ceph/qa/standalone/osd/osd-rep-recov-eio.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2017 Red Hat <contact@redhat.com>
   4 #
   5 #
   6 # Author: Kefu Chai <kchai@redhat.com>
   7 # Author: David Zafman <dzafman@redhat.com>
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU Library Public License as published by
  11 # the Free Software Foundation; either version 2, or (at your option)
  12 # any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Library Public License for more details.
  18 #
  19
  20 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  21
  22 warnings=10
  23
  24 function run() {
  25     local dir=$1
  26     shift
  27
  28     export CEPH_MON="127.0.0.1:7140" # git grep '\<7140\>' : there must be only one
  29     export CEPH_ARGS
  30     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  31     CEPH_ARGS+="--mon-host=$CEPH_MON "
  32     CEPH_ARGS+="--osd-mclock-profile=high_recovery_ops "
  33
  34
  35     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  36     for func in $funcs ; do
  37         setup $dir || return 1
  38         # set warning amount in case default changes
  39         run_mon $dir a --mon_osd_warn_num_repaired=$warnings || return 1
  40         run_mgr $dir x || return 1
  41         ceph osd pool create foo 8 || return 1
  42
  43         $func $dir || return 1
  44         teardown $dir || return 1
  45     done
  46 }
  47
  48 function setup_osds() {
  49     local count=$1
  50     shift
  51     local type=$1
  52
  53     for id in $(seq 0 $(expr $count - 1)) ; do
  54         run_osd${type} $dir $id || return 1
  55     done
  56     wait_for_clean || return 1
  57 }
  58
  59 function get_state() {
  60     local pgid=$1
  61     local sname=state
  62     ceph --format json pg dump pgs 2>/dev/null | \
  63         jq -r ".pg_stats | .[] | select(.pgid==\"$pgid\") | .$sname"
  64 }
  65
  66 function rados_put() {
  67     local dir=$1
  68     local poolname=$2
  69     local objname=${3:-SOMETHING}
  70
  71     for marker in AAA BBB CCCC DDDD ; do
  72         printf "%*s" 1024 $marker
  73     done > $dir/ORIGINAL
  74     #
  75     # get and put an object, compare they are equal
  76     #
  77     rados --pool $poolname put $objname $dir/ORIGINAL || return 1
  78 }
  79
  80 function rados_get() {
  81     local dir=$1
  82     local poolname=$2
  83     local objname=${3:-SOMETHING}
  84     local expect=${4:-ok}
  85
  86     #
  87     # Expect a failure to get object
  88     #
  89     if [ $expect = "fail" ];
  90     then
  91         ! rados --pool $poolname get $objname $dir/COPY
  92         return
  93     fi
  94     #
  95     # Expect hang trying to get object
  96     #
  97     if [ $expect = "hang" ];
  98     then
  99         timeout 5 rados --pool $poolname get $objname $dir/COPY
 100         test "$?" = "124"
 101         return
 102     fi
 103     #
 104     # get an object, compare with $dir/ORIGINAL
 105     #
 106     rados --pool $poolname get $objname $dir/COPY || return 1
 107     diff $dir/ORIGINAL $dir/COPY || return 1
 108     rm $dir/COPY
 109 }
 110
 111 function rados_get_data() {
 112     local inject=$1
 113     shift
 114     local dir=$1
 115
 116     local poolname=pool-rep
 117     local objname=obj-$inject-$$
 118     local pgid=$(get_pg $poolname $objname)
 119
 120     rados_put $dir $poolname $objname || return 1
 121     inject_$inject rep data $poolname $objname $dir 0 || return 1
 122     rados_get $dir $poolname $objname || return 1
 123
 124     wait_for_clean
 125     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 126     test "$COUNT" = "1" || return 1
 127     flush_pg_stats
 128     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 129     test "$COUNT" = "1" || return 1
 130
 131     local object_osds=($(get_osds $poolname $objname))
 132     local primary=${object_osds[0]}
 133     local bad_peer=${object_osds[1]}
 134     inject_$inject rep data $poolname $objname $dir 0 || return 1
 135     inject_$inject rep data $poolname $objname $dir 1 || return 1
 136     # Force primary to pull from the bad peer, so we can repair it too!
 137     set_config osd $primary osd_debug_feed_pullee $bad_peer || return 1
 138     rados_get $dir $poolname $objname || return 1
 139
 140     # Wait until automatic repair of bad peer is done
 141     wait_for_clean || return 1
 142
 143     inject_$inject rep data $poolname $objname $dir 0 || return 1
 144     inject_$inject rep data $poolname $objname $dir 2 || return 1
 145     rados_get $dir $poolname $objname || return 1
 146
 147     wait_for_clean
 148     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 149     test "$COUNT" = "3" || return 1
 150     flush_pg_stats
 151     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 152     test "$COUNT" = "4" || return 1
 153
 154     inject_$inject rep data $poolname $objname $dir 0 || return 1
 155     inject_$inject rep data $poolname $objname $dir 1 || return 1
 156     inject_$inject rep data $poolname $objname $dir 2 || return 1
 157     rados_get $dir $poolname $objname hang || return 1
 158
 159     wait_for_clean
 160     # After hang another repair couldn't happen, so count stays the same
 161     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 162     test "$COUNT" = "3" || return 1
 163     flush_pg_stats
 164     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 165     test "$COUNT" = "4" || return 1
 166 }
 167
 168 function TEST_rados_get_with_eio() {
 169     local dir=$1
 170
 171     setup_osds 4 || return 1
 172
 173     local poolname=pool-rep
 174     create_pool $poolname 1 1 || return 1
 175     wait_for_clean || return 1
 176     rados_get_data eio $dir || return 1
 177
 178     delete_pool $poolname
 179 }
 180
 181 function TEST_rados_repair_warning() {
 182     local dir=$1
 183     local OBJS=$(expr $warnings + 1)
 184
 185     setup_osds 4 || return 1
 186
 187     local poolname=pool-rep
 188     create_pool $poolname 1 1 || return 1
 189     wait_for_clean || return 1
 190
 191     local poolname=pool-rep
 192     local objbase=obj-warn
 193     local inject=eio
 194
 195    for i in $(seq 1 $OBJS)
 196     do
 197       rados_put $dir $poolname ${objbase}-$i || return 1
 198       inject_$inject rep data $poolname ${objbase}-$i $dir 0 || return 1
 199       rados_get $dir $poolname ${objbase}-$i || return 1
 200     done
 201     local pgid=$(get_pg $poolname ${objbase}-1)
 202
 203     local object_osds=($(get_osds $poolname ${objbase}-1))
 204     local primary=${object_osds[0]}
 205     local bad_peer=${object_osds[1]}
 206
 207     wait_for_clean
 208     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 209     test "$COUNT" = "$OBJS" || return 1
 210     flush_pg_stats
 211     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 212     test "$COUNT" = "$OBJS" || return 1
 213
 214     ceph health | grep -q "Too many repaired reads on 1 OSDs" || return 1
 215     ceph health detail | grep -q "osd.$primary had $OBJS reads repaired" || return 1
 216
 217     ceph health mute OSD_TOO_MANY_REPAIRS
 218     set -o pipefail
 219     # Should mute this
 220     ceph health | $(! grep -q "Too many repaired reads on 1 OSDs") || return 1
 221     set +o pipefail
 222
 223     for i in $(seq 1 $OBJS)
 224      do
 225        inject_$inject rep data $poolname ${objbase}-$i $dir 0 || return 1
 226        inject_$inject rep data $poolname ${objbase}-$i $dir 1 || return 1
 227        # Force primary to pull from the bad peer, so we can repair it too!
 228        set_config osd $primary osd_debug_feed_pullee $bad_peer || return 1
 229        rados_get $dir $poolname ${objbase}-$i || return 1
 230     done
 231
 232     wait_for_clean
 233     COUNT=$(ceph pg $pgid query | jq '.info.stats.stat_sum.num_objects_repaired')
 234     test "$COUNT" = "$(expr $OBJS \* 2)" || return 1
 235     flush_pg_stats
 236     COUNT=$(ceph pg dump --format=json-pretty | jq ".pg_map.osd_stats_sum.num_shards_repaired")
 237     test "$COUNT" = "$(expr $OBJS \* 3)" || return 1
 238
 239     # Give mon a chance to notice additional OSD and unmute
 240     # The default tick time is 5 seconds
 241     CHECKTIME=10
 242     LOOPS=0
 243     while(true)
 244     do
 245       sleep 1
 246       if ceph health | grep -q "Too many repaired reads on 2 OSDs"
 247       then
 248               break
 249       fi
 250       LOOPS=$(expr $LOOPS + 1)
 251       if test "$LOOPS" = "$CHECKTIME"
 252       then
 253               echo "Too many repaired reads not seen after $CHECKTIME seconds"
 254               return 1
 255       fi
 256     done
 257     ceph health detail | grep -q "osd.$primary had $(expr $OBJS \* 2) reads repaired" || return 1
 258     ceph health detail | grep -q "osd.$bad_peer had $OBJS reads repaired" || return 1
 259
 260     delete_pool $poolname
 261 }
 262
 263 # Test backfill with unfound object
 264 function TEST_rep_backfill_unfound() {
 265     local dir=$1
 266     local objname=myobject
 267     local lastobj=300
 268     # Must be between 1 and $lastobj
 269     local testobj=obj250
 270
 271     export CEPH_ARGS
 272     CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10'
 273     setup_osds 3 || return 1
 274
 275     local poolname=test-pool
 276     create_pool $poolname 1 1 || return 1
 277     wait_for_clean || return 1
 278
 279     ceph pg dump pgs
 280
 281     rados_put $dir $poolname $objname || return 1
 282
 283     local -a initial_osds=($(get_osds $poolname $objname))
 284     local last_osd=${initial_osds[-1]}
 285     kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
 286     ceph osd down ${last_osd} || return 1
 287     ceph osd out ${last_osd} || return 1
 288
 289     ceph pg dump pgs
 290
 291     dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4
 292     for i in $(seq 1 $lastobj)
 293     do
 294       rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1
 295     done
 296
 297     inject_eio rep data $poolname $testobj $dir 0 || return 1
 298     inject_eio rep data $poolname $testobj $dir 1 || return 1
 299
 300     activate_osd $dir ${last_osd} || return 1
 301     ceph osd in ${last_osd} || return 1
 302
 303     sleep 15
 304
 305     for tmp in $(seq 1 360); do
 306       state=$(get_state 2.0)
 307       echo $state | grep backfill_unfound
 308       if [ "$?" = "0" ]; then
 309         break
 310       fi
 311       echo "$state "
 312       sleep 1
 313     done
 314
 315     ceph pg dump pgs
 316     ceph pg 2.0 list_unfound | grep -q $testobj || return 1
 317
 318     # Command should hang because object is unfound
 319     timeout 5 rados -p $poolname get $testobj $dir/CHECK
 320     test $? = "124" || return 1
 321
 322     ceph pg 2.0 mark_unfound_lost delete
 323
 324     wait_for_clean || return 1
 325
 326     for i in $(seq 1 $lastobj)
 327     do
 328       if [ obj${i} = "$testobj" ]; then
 329         # Doesn't exist anymore
 330         ! rados -p $poolname get $testobj $dir/CHECK || return 1
 331       else
 332         rados --pool $poolname get obj${i} $dir/CHECK || return 1
 333         diff -q $dir/ORIGINAL $dir/CHECK || return 1
 334       fi
 335     done
 336
 337     rm -f ${dir}/ORIGINAL ${dir}/CHECK
 338
 339     delete_pool $poolname
 340 }
 341
 342 # Test recovery with unfound object
 343 function TEST_rep_recovery_unfound() {
 344     local dir=$1
 345     local objname=myobject
 346     local lastobj=100
 347     # Must be between 1 and $lastobj
 348     local testobj=obj75
 349
 350     setup_osds 3 || return 1
 351
 352     local poolname=test-pool
 353     create_pool $poolname 1 1 || return 1
 354     wait_for_clean || return 1
 355
 356     ceph pg dump pgs
 357
 358     rados_put $dir $poolname $objname || return 1
 359
 360     local -a initial_osds=($(get_osds $poolname $objname))
 361     local last_osd=${initial_osds[-1]}
 362     kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
 363     ceph osd down ${last_osd} || return 1
 364     ceph osd out ${last_osd} || return 1
 365
 366     ceph pg dump pgs
 367
 368     dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4
 369     for i in $(seq 1 $lastobj)
 370     do
 371       rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1
 372     done
 373
 374     inject_eio rep data $poolname $testobj $dir 0 || return 1
 375     inject_eio rep data $poolname $testobj $dir 1 || return 1
 376
 377     activate_osd $dir ${last_osd} || return 1
 378     ceph osd in ${last_osd} || return 1
 379
 380     sleep 15
 381
 382     for tmp in $(seq 1 100); do
 383       state=$(get_state 2.0)
 384       echo $state | grep -v recovering
 385       if [ "$?" = "0" ]; then
 386         break
 387       fi
 388       echo "$state "
 389       sleep 1
 390     done
 391
 392     ceph pg dump pgs
 393     ceph pg 2.0 list_unfound | grep -q $testobj || return 1
 394
 395     # Command should hang because object is unfound
 396     timeout 5 rados -p $poolname get $testobj $dir/CHECK
 397     test $? = "124" || return 1
 398
 399     ceph pg 2.0 mark_unfound_lost delete
 400
 401     wait_for_clean || return 1
 402
 403     for i in $(seq 1 $lastobj)
 404     do
 405       if [ obj${i} = "$testobj" ]; then
 406         # Doesn't exist anymore
 407         ! rados -p $poolname get $testobj $dir/CHECK || return 1
 408       else
 409         rados --pool $poolname get obj${i} $dir/CHECK || return 1
 410         diff -q $dir/ORIGINAL $dir/CHECK || return 1
 411       fi
 412     done
 413
 414     rm -f ${dir}/ORIGINAL ${dir}/CHECK
 415
 416     delete_pool $poolname
 417 }
 418
 419 # This is a filestore only test because it requires data digest in object info
 420 function TEST_rep_read_unfound() {
 421     local dir=$1
 422     local objname=myobject
 423
 424     setup_osds 3 _filestore || return 1
 425
 426     ceph osd pool delete foo foo --yes-i-really-really-mean-it || return 1
 427     local poolname=test-pool
 428     create_pool $poolname 1 1 || return 1
 429     ceph osd pool set $poolname size 2
 430     wait_for_clean || return 1
 431
 432     ceph pg dump pgs
 433
 434     dd if=/dev/urandom bs=8k count=1 of=$dir/ORIGINAL
 435     rados -p $poolname put $objname $dir/ORIGINAL
 436
 437     local primary=$(get_primary $poolname $objname)
 438     local other=$(get_not_primary $poolname $objname)
 439
 440     dd if=/dev/urandom bs=8k count=1 of=$dir/CORRUPT
 441     objectstore_tool $dir $primary $objname set-bytes $dir/CORRUPT || return 1
 442     objectstore_tool $dir $other $objname set-bytes $dir/CORRUPT || return 1
 443
 444     timeout 30 rados -p $poolname get $objname $dir/tmp &
 445
 446     sleep 5
 447
 448     flush_pg_stats
 449     ceph --format=json pg dump pgs | jq '.'
 450
 451     if ! ceph --format=json pg dump pgs | jq '.pg_stats | .[0].state' | grep -q recovery_unfound
 452     then
 453       echo "Failure to get to recovery_unfound state"
 454       return 1
 455     fi
 456
 457     objectstore_tool $dir $other $objname set-bytes $dir/ORIGINAL || return 1
 458
 459     wait
 460
 461     if ! cmp $dir/ORIGINAL $dir/tmp
 462     then
 463        echo "Bad data after primary repair"
 464        return 1
 465     fi
 466 }
 467
 468 main osd-rep-recov-eio.sh "$@"
 469
 470 # Local Variables:
 471 # compile-command: "cd ../../../build ; make -j4 && ../qa/run-standalone.sh osd-rep-recov-eio.sh"
 472 # End: