ceph/qa/standalone/erasure-code/test-erasure-eio.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2015 Red Hat <contact@redhat.com>
   4 #
   5 #
   6 # Author: Kefu Chai <kchai@redhat.com>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU Library Public License as published by
  10 # the Free Software Foundation; either version 2, or (at your option)
  11 # any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Library Public License for more details.
  17 #
  18
  19 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
  20
  21 function run() {
  22     local dir=$1
  23     shift
  24
  25     export CEPH_MON="127.0.0.1:7112" # git grep '\<7112\>' : there must be only one
  26     export CEPH_ARGS
  27     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
  28     CEPH_ARGS+="--mon-host=$CEPH_MON "
  29
  30     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
  31     for func in $funcs ; do
  32         setup $dir || return 1
  33         run_mon $dir a || return 1
  34         run_mgr $dir x || return 1
  35         create_rbd_pool || return 1
  36
  37         # check that erasure code plugins are preloaded
  38         CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1
  39         grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1
  40         $func $dir || return 1
  41         teardown $dir || return 1
  42     done
  43 }
  44
  45 function setup_osds() {
  46     for id in $(seq 0 3) ; do
  47         run_osd $dir $id || return 1
  48     done
  49     wait_for_clean || return 1
  50
  51     # check that erasure code plugins are preloaded
  52     CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1
  53     grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1
  54 }
  55
  56 function create_erasure_coded_pool() {
  57     local poolname=$1
  58
  59     ceph osd erasure-code-profile set myprofile \
  60         plugin=jerasure \
  61         k=2 m=1 \
  62         crush-failure-domain=osd || return 1
  63     create_pool $poolname 1 1 erasure myprofile \
  64         || return 1
  65     wait_for_clean || return 1
  66 }
  67
  68 function delete_pool() {
  69     local poolname=$1
  70
  71     ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it
  72     ceph osd erasure-code-profile rm myprofile
  73 }
  74
  75 function rados_put() {
  76     local dir=$1
  77     local poolname=$2
  78     local objname=${3:-SOMETHING}
  79
  80     for marker in AAA BBB CCCC DDDD ; do
  81         printf "%*s" 1024 $marker
  82     done > $dir/ORIGINAL
  83     #
  84     # get and put an object, compare they are equal
  85     #
  86     rados --pool $poolname put $objname $dir/ORIGINAL || return 1
  87 }
  88
  89 function rados_get() {
  90     local dir=$1
  91     local poolname=$2
  92     local objname=${3:-SOMETHING}
  93     local expect=${4:-ok}
  94
  95     #
  96     # Expect a failure to get object
  97     #
  98     if [ $expect = "fail" ];
  99     then
 100         ! rados --pool $poolname get $objname $dir/COPY
 101         return
 102     fi
 103     #
 104     # get an object, compare with $dir/ORIGINAL
 105     #
 106     rados --pool $poolname get $objname $dir/COPY || return 1
 107     diff $dir/ORIGINAL $dir/COPY || return 1
 108     rm $dir/COPY
 109 }
 110
 111 function rados_put_get() {
 112     local dir=$1
 113     local poolname=$2
 114     local objname=${3:-SOMETHING}
 115     local recovery=$4
 116
 117     #
 118     # get and put an object, compare they are equal
 119     #
 120     rados_put $dir $poolname $objname || return 1
 121     # We can read even though caller injected read error on one of the shards
 122     rados_get $dir $poolname $objname || return 1
 123
 124     if [ -n "$recovery" ];
 125     then
 126         #
 127         # take out the last OSD used to store the object,
 128         # bring it back, and check for clean PGs which means
 129         # recovery didn't crash the primary.
 130         #
 131         local -a initial_osds=($(get_osds $poolname $objname))
 132         local last=$((${#initial_osds[@]} - 1))
 133         # Kill OSD
 134         kill_daemons $dir TERM osd.${initial_osds[$last]} >&2 < /dev/null || return 1
 135         ceph osd out ${initial_osds[$last]} || return 1
 136         ! get_osds $poolname $objname | grep '\<'${initial_osds[$last]}'\>' || return 1
 137         ceph osd in ${initial_osds[$last]} || return 1
 138         run_osd $dir ${initial_osds[$last]} || return 1
 139         wait_for_clean || return 1
 140     fi
 141
 142     rm $dir/ORIGINAL
 143 }
 144
 145 function rados_get_data_eio() {
 146     local dir=$1
 147     shift
 148     local shard_id=$1
 149     shift
 150     local recovery=$1
 151     shift
 152
 153     # inject eio to speificied shard
 154     #
 155     local poolname=pool-jerasure
 156     local objname=obj-eio-$$-$shard_id
 157     inject_eio ec data $poolname $objname $dir $shard_id || return 1
 158     rados_put_get $dir $poolname $objname $recovery || return 1
 159
 160     shard_id=$(expr $shard_id + 1)
 161     inject_eio ec data $poolname $objname $dir $shard_id || return 1
 162     # Now 2 out of 3 shards get EIO, so should fail
 163     rados_get $dir $poolname $objname fail || return 1
 164 }
 165
 166 # Change the size of speificied shard
 167 #
 168 function set_size() {
 169     local objname=$1
 170     shift
 171     local dir=$1
 172     shift
 173     local shard_id=$1
 174     shift
 175     local bytes=$1
 176     shift
 177     local mode=${1}
 178
 179     local poolname=pool-jerasure
 180     local -a initial_osds=($(get_osds $poolname $objname))
 181     local osd_id=${initial_osds[$shard_id]}
 182     ceph osd set noout
 183     if [ "$mode" = "add" ];
 184     then
 185       objectstore_tool $dir $osd_id $objname get-bytes $dir/CORRUPT || return 1
 186       dd if=/dev/urandom bs=$bytes count=1 >> $dir/CORRUPT
 187     elif [ "$bytes" = "0" ];
 188     then
 189       touch $dir/CORRUPT
 190     else
 191       dd if=/dev/urandom bs=$bytes count=1 of=$dir/CORRUPT
 192     fi
 193     objectstore_tool $dir $osd_id $objname set-bytes $dir/CORRUPT || return 1
 194     rm -f $dir/CORRUPT
 195     ceph osd unset noout
 196 }
 197
 198 function rados_get_data_bad_size() {
 199     local dir=$1
 200     shift
 201     local shard_id=$1
 202     shift
 203     local bytes=$1
 204     shift
 205     local mode=${1:-set}
 206
 207     local poolname=pool-jerasure
 208     local objname=obj-size-$$-$shard_id-$bytes
 209     rados_put $dir $poolname $objname || return 1
 210
 211     # Change the size of speificied shard
 212     #
 213     set_size $objname $dir $shard_id $bytes $mode || return 1
 214
 215     rados_get $dir $poolname $objname || return 1
 216
 217     # Leave objname and modify another shard
 218     shard_id=$(expr $shard_id + 1)
 219     set_size $objname $dir $shard_id $bytes $mode || return 1
 220     rados_get $dir $poolname $objname fail || return 1
 221 }
 222
 223 #
 224 # These two test cases try to validate the following behavior:
 225 #  For object on EC pool, if there is one shard having read error (
 226 #  either primary or replica), client can still read object.
 227 #
 228 # If 2 shards have read errors the client will get an error.
 229 #
 230 function TEST_rados_get_subread_eio_shard_0() {
 231     local dir=$1
 232     setup_osds || return 1
 233
 234     local poolname=pool-jerasure
 235     create_erasure_coded_pool $poolname || return 1
 236     # inject eio on primary OSD (0) and replica OSD (1)
 237     local shard_id=0
 238     rados_get_data_eio $dir $shard_id || return 1
 239     delete_pool $poolname
 240 }
 241
 242 function TEST_rados_get_subread_eio_shard_1() {
 243     local dir=$1
 244     setup_osds || return 1
 245
 246     local poolname=pool-jerasure
 247     create_erasure_coded_pool $poolname || return 1
 248     # inject eio into replicas OSD (1) and OSD (2)
 249     local shard_id=1
 250     rados_get_data_eio $dir $shard_id || return 1
 251     delete_pool $poolname
 252 }
 253
 254 #
 255 # These two test cases try to validate that following behavior:
 256 #  For object on EC pool, if there is one shard which an incorrect
 257 # size this will cause an internal read error, client can still read object.
 258 #
 259 # If 2 shards have incorrect size the client will get an error.
 260 #
 261 function TEST_rados_get_bad_size_shard_0() {
 262     local dir=$1
 263     setup_osds || return 1
 264
 265     local poolname=pool-jerasure
 266     create_erasure_coded_pool $poolname || return 1
 267     # Set incorrect size into primary OSD (0) and replica OSD (1)
 268     local shard_id=0
 269     rados_get_data_bad_size $dir $shard_id 10 || return 1
 270     rados_get_data_bad_size $dir $shard_id 0 || return 1
 271     rados_get_data_bad_size $dir $shard_id 256 add || return 1
 272     delete_pool $poolname
 273 }
 274
 275 function TEST_rados_get_bad_size_shard_1() {
 276     local dir=$1
 277     setup_osds || return 1
 278
 279     local poolname=pool-jerasure
 280     create_erasure_coded_pool $poolname || return 1
 281     # Set incorrect size into replicas OSD (1) and OSD (2)
 282     local shard_id=1
 283     rados_get_data_bad_size $dir $shard_id 10 || return 1
 284     rados_get_data_bad_size $dir $shard_id 0 || return 1
 285     rados_get_data_bad_size $dir $shard_id 256 add || return 1
 286     delete_pool $poolname
 287 }
 288
 289 function TEST_rados_get_with_subreadall_eio_shard_0() {
 290     local dir=$1
 291     local shard_id=0
 292
 293     setup_osds || return 1
 294
 295     local poolname=pool-jerasure
 296     create_erasure_coded_pool $poolname || return 1
 297     # inject eio on primary OSD (0)
 298     local shard_id=0
 299     rados_get_data_eio $dir $shard_id recovery || return 1
 300
 301     delete_pool $poolname
 302 }
 303
 304 function TEST_rados_get_with_subreadall_eio_shard_1() {
 305     local dir=$1
 306     local shard_id=0
 307
 308     setup_osds || return 1
 309
 310     local poolname=pool-jerasure
 311     create_erasure_coded_pool $poolname || return 1
 312     # inject eio on replica OSD (1)
 313     local shard_id=1
 314     rados_get_data_eio $dir $shard_id recovery || return 1
 315
 316     delete_pool $poolname
 317 }
 318
 319 main test-erasure-eio "$@"
 320
 321 # Local Variables:
 322 # compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-eio.sh"
 323 # End: