ceph/qa/standalone/osd/osd-rep-recov-eio.sh

   1 #!/usr/bin/env bash
   2 #
   3 # Copyright (C) 2017 Red Hat <contact@redhat.com>
   4 #
   5 #
   6 # Author: Kefu Chai <kchai@redhat.com>
   7 # Author: David Zafman <dzafman@redhat.com>
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU Library Public License as published by
  11 # the Free Software Foundation; either version 2, or (at your option)
  12 # any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU Library Public License for more details.
  18 #
  19
  20 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  21
  22 function run() {
  23     local dir=$1
  24     shift
  25
  26     export CEPH_MON="127.0.0.1:7140" # git grep '\<7140\>' : there must be only one
  27     export CEPH_ARGS
  28     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  29     CEPH_ARGS+="--mon-host=$CEPH_MON "
  30
  31     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  32     for func in $funcs ; do
  33         setup $dir || return 1
  34         run_mon $dir a || return 1
  35         run_mgr $dir x || return 1
  36         create_rbd_pool || return 1
  37
  38         $func $dir || return 1
  39         teardown $dir || return 1
  40     done
  41 }
  42
  43 function setup_osds() {
  44     local count=$1
  45     shift
  46
  47     for id in $(seq 0 $(expr $count - 1)) ; do
  48         run_osd $dir $id || return 1
  49     done
  50     wait_for_clean || return 1
  51 }
  52
  53 function get_state() {
  54     local pgid=$1
  55     local sname=state
  56     ceph --format json pg dump pgs 2>/dev/null | \
  57         jq -r ".[] | select(.pgid==\"$pgid\") | .$sname"
  58 }
  59
  60 function delete_pool() {
  61     local poolname=$1
  62
  63     ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it
  64 }
  65
  66 function rados_put() {
  67     local dir=$1
  68     local poolname=$2
  69     local objname=${3:-SOMETHING}
  70
  71     for marker in AAA BBB CCCC DDDD ; do
  72         printf "%*s" 1024 $marker
  73     done > $dir/ORIGINAL
  74     #
  75     # get and put an object, compare they are equal
  76     #
  77     rados --pool $poolname put $objname $dir/ORIGINAL || return 1
  78 }
  79
  80 function rados_get() {
  81     local dir=$1
  82     local poolname=$2
  83     local objname=${3:-SOMETHING}
  84     local expect=${4:-ok}
  85
  86     #
  87     # Expect a failure to get object
  88     #
  89     if [ $expect = "fail" ];
  90     then
  91         ! rados --pool $poolname get $objname $dir/COPY
  92         return
  93     fi
  94     #
  95     # Expect hang trying to get object
  96     #
  97     if [ $expect = "hang" ];
  98     then
  99         timeout 5 rados --pool $poolname get $objname $dir/COPY
 100         test "$?" = "124"
 101         return
 102     fi
 103     #
 104     # get an object, compare with $dir/ORIGINAL
 105     #
 106     rados --pool $poolname get $objname $dir/COPY || return 1
 107     diff $dir/ORIGINAL $dir/COPY || return 1
 108     rm $dir/COPY
 109 }
 110
 111 function rados_get_data() {
 112     local inject=$1
 113     shift
 114     local dir=$1
 115
 116     local poolname=pool-rep
 117     local objname=obj-$inject-$$
 118     rados_put $dir $poolname $objname || return 1
 119     inject_$inject rep data $poolname $objname $dir 0 || return 1
 120     rados_get $dir $poolname $objname || return 1
 121
 122     inject_$inject rep data $poolname $objname $dir 0 || return 1
 123     inject_$inject rep data $poolname $objname $dir 1 || return 1
 124     rados_get $dir $poolname $objname || return 1
 125
 126     inject_$inject rep data $poolname $objname $dir 0 || return 1
 127     inject_$inject rep data $poolname $objname $dir 1 || return 1
 128     inject_$inject rep data $poolname $objname $dir 2 || return 1
 129     rados_get $dir $poolname $objname hang || return 1
 130 }
 131
 132 function TEST_rados_get_with_eio() {
 133     local dir=$1
 134
 135     setup_osds 4 || return 1
 136
 137     local poolname=pool-rep
 138     create_pool $poolname 1 1 || return 1
 139     wait_for_clean || return 1
 140     rados_get_data eio $dir || return 1
 141
 142     delete_pool $poolname
 143 }
 144
 145 # Test backfill with unfound object
 146 function TEST_rep_backfill_unfound() {
 147     local dir=$1
 148     local objname=myobject
 149     local lastobj=300
 150     # Must be between 1 and $lastobj
 151     local testobj=obj250
 152
 153     export CEPH_ARGS
 154     CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10'
 155     setup_osds 3 || return 1
 156
 157     local poolname=test-pool
 158     create_pool $poolname 1 1 || return 1
 159     wait_for_clean || return 1
 160
 161     ceph pg dump pgs
 162
 163     rados_put $dir $poolname $objname || return 1
 164
 165     local -a initial_osds=($(get_osds $poolname $objname))
 166     local last_osd=${initial_osds[-1]}
 167     kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
 168     ceph osd down ${last_osd} || return 1
 169     ceph osd out ${last_osd} || return 1
 170
 171     ceph pg dump pgs
 172
 173     dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4
 174     for i in $(seq 1 $lastobj)
 175     do
 176       rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1
 177     done
 178
 179     inject_eio rep data $poolname $testobj $dir 0 || return 1
 180     inject_eio rep data $poolname $testobj $dir 1 || return 1
 181
 182     run_osd $dir ${last_osd} || return 1
 183     ceph osd in ${last_osd} || return 1
 184
 185     sleep 15
 186
 187     for tmp in $(seq 1 100); do
 188       state=$(get_state 2.0)
 189       echo $state | grep backfill_unfound
 190       if [ "$?" = "0" ]; then
 191         break
 192       fi
 193       echo "$state "
 194       sleep 1
 195     done
 196
 197     ceph pg dump pgs
 198     ceph pg 2.0 list_missing | grep -q $testobj || return 1
 199
 200     # Command should hang because object is unfound
 201     timeout 5 rados -p $poolname get $testobj $dir/CHECK
 202     test $? = "124" || return 1
 203
 204     ceph pg 2.0 mark_unfound_lost delete
 205
 206     wait_for_clean || return 1
 207
 208     for i in $(seq 1 $lastobj)
 209     do
 210       if [ obj${i} = "$testobj" ]; then
 211         # Doesn't exist anymore
 212         ! rados -p $poolname get $testobj $dir/CHECK || return 1
 213       else
 214         rados --pool $poolname get obj${i} $dir/CHECK || return 1
 215         diff -q $dir/ORIGINAL $dir/CHECK || return 1
 216       fi
 217     done
 218
 219     rm -f ${dir}/ORIGINAL ${dir}/CHECK
 220
 221     delete_pool $poolname
 222 }
 223
 224 # Test recovery with unfound object
 225 function TEST_rep_recovery_unfound() {
 226     local dir=$1
 227     local objname=myobject
 228     local lastobj=100
 229     # Must be between 1 and $lastobj
 230     local testobj=obj75
 231
 232     setup_osds 3 || return 1
 233
 234     local poolname=test-pool
 235     create_pool $poolname 1 1 || return 1
 236     wait_for_clean || return 1
 237
 238     ceph pg dump pgs
 239
 240     rados_put $dir $poolname $objname || return 1
 241
 242     local -a initial_osds=($(get_osds $poolname $objname))
 243     local last_osd=${initial_osds[-1]}
 244     kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
 245     ceph osd down ${last_osd} || return 1
 246     ceph osd out ${last_osd} || return 1
 247
 248     ceph pg dump pgs
 249
 250     dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4
 251     for i in $(seq 1 $lastobj)
 252     do
 253       rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1
 254     done
 255
 256     inject_eio rep data $poolname $testobj $dir 0 || return 1
 257     inject_eio rep data $poolname $testobj $dir 1 || return 1
 258
 259     run_osd $dir ${last_osd} || return 1
 260     ceph osd in ${last_osd} || return 1
 261
 262     sleep 15
 263
 264     for tmp in $(seq 1 100); do
 265       state=$(get_state 2.0)
 266       echo $state | grep -v recovering
 267       if [ "$?" = "0" ]; then
 268         break
 269       fi
 270       echo "$state "
 271       sleep 1
 272     done
 273
 274     ceph pg dump pgs
 275     ceph pg 2.0 list_missing | grep -q $testobj || return 1
 276
 277     # Command should hang because object is unfound
 278     timeout 5 rados -p $poolname get $testobj $dir/CHECK
 279     test $? = "124" || return 1
 280
 281     ceph pg 2.0 mark_unfound_lost delete
 282
 283     wait_for_clean || return 1
 284
 285     for i in $(seq 1 $lastobj)
 286     do
 287       if [ obj${i} = "$testobj" ]; then
 288         # Doesn't exist anymore
 289         ! rados -p $poolname get $testobj $dir/CHECK || return 1
 290       else
 291         rados --pool $poolname get obj${i} $dir/CHECK || return 1
 292         diff -q $dir/ORIGINAL $dir/CHECK || return 1
 293       fi
 294     done
 295
 296     rm -f ${dir}/ORIGINAL ${dir}/CHECK
 297
 298     delete_pool $poolname
 299 }
 300
 301 main osd-rep-recov-eio.sh "$@"
 302
 303 # Local Variables:
 304 # compile-command: "cd ../../../build ; make -j4 && ../qa/run-standalone.sh osd-rep-recov-eio.sh"
 305 # End: