ceph/qa/workunits/rbd/rbd_mirror_stress.sh

   1 #!/bin/sh -ex
   2 #
   3 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
   4 #
   5 # The following additional environment variables affect the test:
   6 #
   7 #  RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
   8 #                             tool during the many image test
   9 #
  10
  11 IMAGE_COUNT=50
  12 export LOCKDEP=0
  13
  14 . $(dirname $0)/rbd_mirror_helpers.sh
  15
  16 setup
  17
  18 create_snap()
  19 {
  20     local cluster=$1
  21     local pool=$2
  22     local image=$3
  23     local snap_name=$4
  24
  25     rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \
  26         --debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log
  27 }
  28
  29 compare_image_snaps()
  30 {
  31     local pool=$1
  32     local image=$2
  33     local snap_name=$3
  34
  35     local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
  36     local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
  37
  38     rm -f ${rmt_export} ${loc_export}
  39     rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export}
  40     rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export}
  41     cmp ${rmt_export} ${loc_export}
  42     rm -f ${rmt_export} ${loc_export}
  43 }
  44
  45 wait_for_pool_images()
  46 {
  47     local cluster=$1
  48     local pool=$2
  49     local image_count=$3
  50     local s
  51     local count
  52     local last_count=0
  53
  54     while true; do
  55         for s in `seq 1 40`; do
  56             test $s -ne 1 && sleep 30
  57             count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
  58             test "${count}" = "${image_count}" && return 0
  59
  60             # reset timeout if making forward progress
  61             test $count -ne $last_count && break
  62         done
  63
  64         test $count -eq $last_count && break
  65         last_count=$count
  66     done
  67     rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
  68     return 1
  69 }
  70
  71 wait_for_pool_healthy()
  72 {
  73     local cluster=$1
  74     local pool=$2
  75     local s
  76     local state
  77
  78     for s in `seq 1 40`; do
  79         test $s -ne 1 && sleep 30
  80         state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'image health:' | cut -d' ' -f 3)
  81         test "${state}" = "ERROR" && break
  82         test "${state}" = "OK" && return 0
  83     done
  84     rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
  85     return 1
  86 }
  87
  88 start_mirrors ${CLUSTER1}
  89 start_mirrors ${CLUSTER2}
  90
  91 testlog "TEST: add image and test replay after client crashes"
  92 image=test
  93 create_image ${CLUSTER2} ${POOL} ${image} '512M'
  94 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  95
  96 for i in `seq 1 10`
  97 do
  98   stress_write_image ${CLUSTER2} ${POOL} ${image}
  99
 100   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 101
 102   snap_name="snap${i}"
 103   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
 104   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 105   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 106   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 107   compare_image_snaps ${POOL} ${image} ${snap_name}
 108 done
 109
 110 for i in `seq 1 10`
 111 do
 112   snap_name="snap${i}"
 113   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 114 done
 115
 116 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 117 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 118
 119 testlog "TEST: create many images"
 120 snap_name="snap"
 121 for i in `seq 1 ${IMAGE_COUNT}`
 122 do
 123   image="image_${i}"
 124   create_image ${CLUSTER2} ${POOL} ${image} '128M'
 125   if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
 126     write_image ${CLUSTER2} ${POOL} ${image} 100
 127   else
 128     stress_write_image ${CLUSTER2} ${POOL} ${image}
 129   fi
 130 done
 131
 132 wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
 133 wait_for_pool_healthy ${CLUSTER2} ${POOL}
 134
 135 wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
 136 wait_for_pool_healthy ${CLUSTER1} ${POOL}
 137
 138 testlog "TEST: compare many images"
 139 for i in `seq 1 ${IMAGE_COUNT}`
 140 do
 141   image="image_${i}"
 142   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
 143   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 144   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 145   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 146   compare_image_snaps ${POOL} ${image} ${snap_name}
 147 done
 148
 149 testlog "TEST: delete many images"
 150 for i in `seq 1 ${IMAGE_COUNT}`
 151 do
 152   image="image_${i}"
 153   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 154   remove_image_retry ${CLUSTER2} ${POOL} ${image}
 155 done
 156
 157 testlog "TEST: image deletions should propagate"
 158 wait_for_pool_images ${CLUSTER1} ${POOL} 0
 159 wait_for_pool_healthy ${CLUSTER1} ${POOL} 0
 160 for i in `seq 1 ${IMAGE_COUNT}`
 161 do
 162   image="image_${i}"
 163   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 164 done
 165
 166 testlog "TEST: delete images during bootstrap"
 167 set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image'
 168 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 169
 170 start_mirror ${CLUSTER1}
 171 image=test
 172
 173 for i in `seq 1 10`
 174 do
 175   image="image_${i}"
 176   create_image ${CLUSTER2} ${POOL} ${image} '512M'
 177   enable_mirror ${CLUSTER2} ${POOL} ${image}
 178
 179   stress_write_image ${CLUSTER2} ${POOL} ${image}
 180   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 181
 182   disable_mirror ${CLUSTER2} ${POOL} ${image}
 183   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 184   purge_snapshots ${CLUSTER2} ${POOL} ${image}
 185   remove_image_retry ${CLUSTER2} ${POOL} ${image}
 186 done