ceph/src/test/objectstore/run_seed_to.sh

   1 #!/bin/bash
   2 # vim: ts=8 sw=2 smarttab
   3 #
   4 # run_seed_to.sh - Run ceph_test_filestore_idempotent_sequence up until an
   5 # injection point, generating a sequence of operations based on a
   6 # provided seed.
   7 #
   8 # We also perform three additional tests, focused on assessing if
   9 # replaying a larger chunck of the journal affects the expected store
  10 # behavior. These tests will be performed by increasing the store's
  11 # journal sync interval to a very large value, allowing the store to
  12 # finish execution before the first sync (unless the store runs for
  13 # over 10 hours, case on which the interval variables must be changed
  14 # to an appropriate value). Unless the '--no-journal-test' option is
  15 # specified, we will run the 3 following scenarios:
  16 #
  17 #  1) journal sync'ing for both stores is good as disabled
  18 #     (we call it '00', for store naming purposes)
  19 #  2) journal sync'ing for store A is as good as disabled
  20 #     (we call it '01', for store naming purposes)
  21 #  3) journal sync'ing for store B is as good as disabled
  22 #     (we call it '10', for store naming purposes)
  23 #
  24 # All log files are also appropriately named accordingly (i.e., a.00.fail,
  25 # a.10.recover, or b.01.clean).
  26 #
  27 # By default, the test will not exit on error, although it will show the
  28 # fail message. This behavior is so defined so we run the whole battery of
  29 # tests, and obtain as many mismatches as possible in one go. We may force
  30 # the test to exit on error by specifying the '--exit-on-error' option.
  31 #
  32 #
  33 set -e
  34
  35 test_opts=""
  36
  37 usage() {
  38   echo "usage: $1 [options..] <seed> <kill-at>"
  39   echo
  40   echo "options:"
  41   echo "  -c, --colls <VAL>    # of collections"
  42   echo "  -o, --objs <VAL>     # of objects"
  43   echo "  -b, --btrfs <VAL>    seq number for btrfs stores"
  44   echo "  --no-journal-test    don't perform journal replay tests"
  45   echo "  -e, --exit-on-error  exit with 1 on error"
  46   echo "  -v, --valgrind       run commands through valgrind"
  47   echo
  48   echo "env vars:"
  49   echo "  OPTS_STORE           additional opts for both stores"
  50   echo "  OPTS_STORE_A         additional opts for store A"
  51   echo "  OPTS_STORE_B         additional opts for store B"
  52   echo
  53 }
  54
  55 echo $0 $*
  56
  57 die_on_missing_arg() {
  58   if [[ "$2" == "" ]]; then
  59     echo "$1: missing required parameter"
  60     exit 1
  61   fi
  62 }
  63
  64
  65 required_args=2
  66 obtained_args=0
  67
  68 seed=""
  69 killat=""
  70 on_btrfs=0
  71 on_btrfs_seq=0
  72 journal_test=1
  73 min_sync_interval="36000" # ten hours, yes.
  74 max_sync_interval="36001"
  75 exit_on_error=0
  76 v=""
  77
  78 do_rm() {
  79   if [[ $on_btrfs -eq 0 ]]; then
  80     rm -fr $*
  81   fi
  82 }
  83
  84 set_arg() {
  85   if [[ $1 -eq 1 ]]; then
  86     seed=$2
  87   elif [[ $1 -eq 2 ]]; then
  88     killat=$2
  89   else
  90     echo "error: unknown purpose for '$2'"
  91     usage $0
  92     exit 1
  93   fi
  94 }
  95
  96 while [[ $# -gt 0 ]];
  97 do
  98   case "$1" in
  99     -c | --colls)
 100       die_on_missing_arg "$1" "$2"
 101       test_opts="$test_opts --test-num-colls $2"
 102       shift 2
 103       ;;
 104     -o | --objs)
 105       die_on_missing_arg "$1" "$2"
 106       test_opts="$test_opts --test-num-objs $2"
 107       shift 2
 108       ;;
 109     -h | --help)
 110       usage $0 ;
 111       exit 0
 112       ;;
 113     -b | --btrfs)
 114       die_on_missing_arg "$1" "$2"
 115       on_btrfs=1
 116       on_btrfs_seq=$2
 117       shift 2
 118       ;;
 119     --no-journal-test)
 120       journal_test=0
 121       shift
 122       ;;
 123     -e | --exit-on-error)
 124       exit_on_error=1
 125       shift
 126       ;;
 127     -v | --valgrind)
 128       v="valgrind --leak-check=full"
 129       shift
 130       ;;
 131     --)
 132       shift
 133       break
 134       ;;
 135     -*)
 136       echo "$1: unknown option" >&2
 137       usage $0
 138       exit 1
 139       ;;
 140     *)
 141       obtained_args=$(($obtained_args+1))
 142       set_arg $obtained_args $1
 143       shift
 144       ;;
 145   esac
 146 done
 147
 148 if [[ $obtained_args -ne $required_args ]]; then
 149   echo "error: missing argument"
 150   usage $0 ;
 151   exit 1
 152 fi
 153
 154 if [[ "$OPTS_STORE" != "" ]]; then
 155   test_opts="$test_opts $OPTS_STORE"
 156 fi
 157
 158 test_opts_a="$test_opts"
 159 test_opts_b="$test_opts"
 160
 161 if [[ "$OPTS_STORE_A" != "" ]]; then
 162     test_opts_a="$test_opts_a $OPTS_STORE_A"
 163 fi
 164 if [[ "$OPTS_STORE_B" != "" ]]; then
 165     test_opts_b="$test_opts_b $OPTS_STORE_B"
 166 fi
 167
 168 echo seed $seed
 169 echo kill at $killat
 170
 171 # run forever, until $killat...
 172 to=1000000000
 173
 174 #
 175 # store names
 176 #
 177 # We need these for two reasons:
 178 #  1) if we are running the tests on a btrfs volume, then we need to use
 179 #     a seq number for each run. Being on btrfs means we will fail when
 180 #     removing the store's directories and it's far more simple to just
 181 #     specify differente store names such as 'a.$seq' or 'b.$seq'.
 182 #
 183 #  2) unless the '--no-journal-test' option is specified, we will run
 184 #     three additional tests for each store, and we will reuse the same
 185 #     command for each one of the runs, but varying the store's name and
 186 #     arguments.
 187 #
 188 store_a="a"
 189 store_b="b"
 190
 191 if [[ $on_btrfs -eq 1 ]]; then
 192   store_a="$store_a.$on_btrfs_seq"
 193   store_b="$store_b.$on_btrfs_seq"
 194 fi
 195
 196 total_runs=1
 197
 198 if [[ $journal_test -eq 1 ]]; then
 199   total_runs=$(($total_runs + 3))
 200 fi
 201
 202 num_runs=0
 203
 204 opt_min_sync="--filestore-min-sync-interval $min_sync_interval"
 205 opt_max_sync="--filestore-max-sync-interval $max_sync_interval"
 206
 207 ret=0
 208
 209 while [[ $num_runs -lt $total_runs ]];
 210 do
 211   tmp_name_a=$store_a
 212   tmp_name_b=$store_b
 213   tmp_opts_a=$test_opts_a
 214   tmp_opts_b=$test_opts_b
 215
 216   #
 217   # We have already tested whether there are diffs when both journals
 218   # are properly working. Now let's try on three other scenarios:
 219   #  1) journal sync'ing for both stores is good as disabled
 220   #     (we call it '00')
 221   #  2) journal sync'ing for store A is as good as disabled
 222   #     (we call it '01')
 223   #  3) journal sync'ing for store B is as good as disabled
 224   #     (we call it '10')
 225   #
 226   if [[ $num_runs -gt 0 && $journal_test -eq 1 ]]; then
 227     echo "run #$num_runs"
 228     case $num_runs in
 229       1)
 230         tmp_name_a="$tmp_name_a.00"
 231         tmp_name_b="$tmp_name_b.00"
 232         tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync"
 233         tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync"
 234         ;;
 235       2)
 236         tmp_name_a="$tmp_name_a.01"
 237         tmp_name_b="$tmp_name_b.01"
 238         tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync"
 239         ;;
 240       3)
 241         tmp_name_a="$tmp_name_a.10"
 242         tmp_name_b="$tmp_name_b.10"
 243         tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync"
 244         ;;
 245     esac
 246   fi
 247
 248   do_rm $tmp_name_a $tmp_name_a.fail $tmp_name_a.recover
 249   $v ceph_test_filestore_idempotent_sequence run-sequence-to $to \
 250     $tmp_name_a $tmp_name_a/journal \
 251     --test-seed $seed --osd-journal-size 100 \
 252     --filestore-kill-at $killat $tmp_opts_a \
 253     --log-file $tmp_name_a.fail --debug-filestore 20 --no-log-to-stderr || true
 254
 255   stop_at=`ceph_test_filestore_idempotent_sequence get-last-op \
 256     $tmp_name_a $tmp_name_a/journal \
 257     --log-file $tmp_name_a.recover \
 258     --debug-filestore 20 --debug-journal 20 --no-log-to-stderr`
 259
 260   if [[ "`expr $stop_at - $stop_at 2>/dev/null`" != "0" ]]; then
 261     echo "error: get-last-op returned '$stop_at'"
 262     exit 1
 263   fi
 264
 265   echo stopped at $stop_at
 266
 267   do_rm $tmp_name_b $tmp_name_b.clean
 268   $v ceph_test_filestore_idempotent_sequence run-sequence-to \
 269     $stop_at $tmp_name_b $tmp_name_b/journal \
 270     --test-seed $seed --osd-journal-size 100 \
 271     --log-file $tmp_name_b.clean --debug-filestore 20 --no-log-to-stderr \
 272     $tmp_opts_b
 273
 274   if $v ceph_test_filestore_idempotent_sequence diff \
 275     $tmp_name_a $tmp_name_a/journal $tmp_name_b $tmp_name_b/journal --no-log-to-stderr --log-file $tmp_name_a.diff.log --debug-filestore 20 ; then
 276       echo OK
 277   else
 278     echo "FAIL"
 279     echo " see:"
 280     echo "   $tmp_name_a.fail     -- leading up to failure"
 281     echo "   $tmp_name_a.recover  -- journal replay"
 282     echo "   $tmp_name_b.clean    -- the clean reference"
 283
 284     ret=1
 285     if [[ $exit_on_error -eq 1 ]]; then
 286       exit 1
 287     fi
 288   fi
 289
 290   num_runs=$(($num_runs+1))
 291 done
 292
 293 exit $ret