ceph/qa/workunits/rbd/rbd_mirror_stress.sh

   1 #!/bin/sh -ex
   2 #
   3 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
   4 #
   5 # The following additional environment variables affect the test:
   6 #
   7 #  RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
   8 #                             tool during the many image test
   9 #
  10
  11 IMAGE_COUNT=50
  12 export LOCKDEP=0
  13
  14 . $(dirname $0)/rbd_mirror_helpers.sh
  15
  16 setup
  17
  18 create_snap()
  19 {
  20     local cluster=$1
  21     local pool=$2
  22     local image=$3
  23     local snap_name=$4
  24
  25     rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \
  26         --debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log
  27 }
  28
  29 compare_image_snaps()
  30 {
  31     local pool=$1
  32     local image=$2
  33     local snap_name=$3
  34     local ret=0
  35
  36     local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
  37     local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
  38
  39     rm -f ${rmt_export} ${loc_export}
  40     rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export}
  41     rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export}
  42     if ! cmp ${rmt_export} ${loc_export}
  43     then
  44         show_diff ${rmt_export} ${loc_export}
  45         ret=1
  46     fi
  47     rm -f ${rmt_export} ${loc_export}
  48     return ${ret}
  49 }
  50
  51 wait_for_pool_images()
  52 {
  53     local cluster=$1
  54     local pool=$2
  55     local image_count=$3
  56     local s
  57     local count
  58     local last_count=0
  59
  60     while true; do
  61         for s in `seq 1 40`; do
  62             test $s -ne 1 && sleep 30
  63             count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
  64             test "${count}" = "${image_count}" && return 0
  65
  66             # reset timeout if making forward progress
  67             test $count -ne $last_count && break
  68         done
  69
  70         test $count -eq $last_count && break
  71         last_count=$count
  72     done
  73     rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
  74     return 1
  75 }
  76
  77 wait_for_pool_healthy()
  78 {
  79     local cluster=$1
  80     local pool=$2
  81     local s
  82     local state
  83
  84     for s in `seq 1 40`; do
  85         test $s -ne 1 && sleep 30
  86         state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'image health:' | cut -d' ' -f 3)
  87         test "${state}" = "ERROR" && break
  88         test "${state}" = "OK" && return 0
  89     done
  90     rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
  91     return 1
  92 }
  93
  94 start_mirrors ${CLUSTER1}
  95 start_mirrors ${CLUSTER2}
  96
  97 testlog "TEST: add image and test replay after client crashes"
  98 image=test
  99 create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '512M'
 100 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 101
 102 clean_snap_name=
 103 for i in `seq 1 10`
 104 do
 105   stress_write_image ${CLUSTER2} ${POOL} ${image}
 106
 107   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 108
 109   snap_name="snap${i}"
 110   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
 111   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 112   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 113   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 114
 115   if [ -n "${clean_snap_name}" ]; then
 116       compare_image_snaps ${POOL} ${image} ${clean_snap_name}
 117   fi
 118   compare_image_snaps ${POOL} ${image} ${snap_name}
 119
 120   clean_snap_name="snap${i}-clean"
 121   create_snap ${CLUSTER2} ${POOL} ${image} ${clean_snap_name}
 122 done
 123
 124 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 125 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 126 wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${clean_snap_name}
 127
 128 for i in `seq 1 10`
 129 do
 130   snap_name="snap${i}"
 131   compare_image_snaps ${POOL} ${image} ${snap_name}
 132
 133   snap_name="snap${i}-clean"
 134   compare_image_snaps ${POOL} ${image} ${snap_name}
 135 done
 136
 137 for i in `seq 1 10`
 138 do
 139   snap_name="snap${i}"
 140   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 141
 142   snap_name="snap${i}-clean"
 143   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 144 done
 145
 146 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 147 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 148
 149 testlog "TEST: create many images"
 150 snap_name="snap"
 151 for i in `seq 1 ${IMAGE_COUNT}`
 152 do
 153   image="image_${i}"
 154   create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '128M'
 155   if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
 156     write_image ${CLUSTER2} ${POOL} ${image} 100
 157   else
 158     stress_write_image ${CLUSTER2} ${POOL} ${image}
 159   fi
 160 done
 161
 162 wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
 163 wait_for_pool_healthy ${CLUSTER2} ${POOL}
 164
 165 wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
 166 wait_for_pool_healthy ${CLUSTER1} ${POOL}
 167
 168 testlog "TEST: compare many images"
 169 for i in `seq 1 ${IMAGE_COUNT}`
 170 do
 171   image="image_${i}"
 172   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
 173   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 174   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 175   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 176   compare_image_snaps ${POOL} ${image} ${snap_name}
 177 done
 178
 179 testlog "TEST: delete many images"
 180 for i in `seq 1 ${IMAGE_COUNT}`
 181 do
 182   image="image_${i}"
 183   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 184   remove_image_retry ${CLUSTER2} ${POOL} ${image}
 185 done
 186
 187 testlog "TEST: image deletions should propagate"
 188 wait_for_pool_images ${CLUSTER1} ${POOL} 0
 189 wait_for_pool_healthy ${CLUSTER1} ${POOL} 0
 190 for i in `seq 1 ${IMAGE_COUNT}`
 191 do
 192   image="image_${i}"
 193   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 194 done
 195
 196 testlog "TEST: delete images during bootstrap"
 197 set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image'
 198 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 199
 200 start_mirror ${CLUSTER1}
 201 image=test
 202
 203 for i in `seq 1 10`
 204 do
 205   image="image_${i}"
 206   create_image ${CLUSTER2} ${POOL} ${image} '512M'
 207   enable_mirror ${CLUSTER2} ${POOL} ${image}
 208
 209   stress_write_image ${CLUSTER2} ${POOL} ${image}
 210   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 211
 212   disable_mirror ${CLUSTER2} ${POOL} ${image}
 213   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 214   purge_snapshots ${CLUSTER2} ${POOL} ${image}
 215   remove_image_retry ${CLUSTER2} ${POOL} ${image}
 216 done
 217
 218 testlog "TEST: check if removed images' OMAP are removed"
 219
 220 wait_for_image_in_omap ${CLUSTER1} ${POOL}
 221 wait_for_image_in_omap ${CLUSTER2} ${POOL}