ceph/qa/workunits/rbd/rbd_mirror.sh

   1 #!/bin/sh -ex
   2 #
   3 # rbd_mirror.sh - test rbd-mirror daemon
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 setup
  13
  14 testlog "TEST: add image and test replay"
  15 start_mirrors ${CLUSTER1}
  16 image=test
  17 create_image ${CLUSTER2} ${POOL} ${image}
  18 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
  19 set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
  20 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  21 write_image ${CLUSTER2} ${POOL} ${image} 100
  22 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  23 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  24 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  25   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  26 fi
  27 compare_images ${POOL} ${image}
  28 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
  29 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
  30
  31 testlog "TEST: stop mirror, add image, start mirror and test replay"
  32 stop_mirrors ${CLUSTER1}
  33 image1=test1
  34 create_image ${CLUSTER2} ${POOL} ${image1}
  35 write_image ${CLUSTER2} ${POOL} ${image1} 100
  36 start_mirrors ${CLUSTER1}
  37 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  38 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  39 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
  40 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  41   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  42 fi
  43 compare_images ${POOL} ${image1}
  44
  45 testlog "TEST: test the first image is replaying after restart"
  46 write_image ${CLUSTER2} ${POOL} ${image} 100
  47 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  48 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  49 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  50 compare_images ${POOL} ${image}
  51
  52 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  53   testlog "TEST: stop/start/restart mirror via admin socket"
  54   all_admin_daemons ${CLUSTER1} rbd mirror stop
  55   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  56   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  57   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  58   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  59
  60   all_admin_daemons ${CLUSTER1} rbd mirror start
  61   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  62   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  63   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  64   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  65
  66   all_admin_daemons ${CLUSTER1} rbd mirror restart
  67   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  68   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  69   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  70   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  71
  72   all_admin_daemons ${CLUSTER1} rbd mirror stop
  73   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  74   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  75   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  76   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  77
  78   all_admin_daemons ${CLUSTER1} rbd mirror restart
  79   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  80   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  81   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  82   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  83
  84   all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
  85   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  86   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  87   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  88   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  89
  90   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
  91   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  92   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  93
  94   all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
  95   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  96   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  97
  98   admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  99   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 100   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 101
 102   all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 103   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 104   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 105
 106   all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 107   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 108   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 109   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 110   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 111
 112   all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 113   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 114   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 115   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 116   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 117
 118   flush ${CLUSTER1}
 119   all_admin_daemons ${CLUSTER1} rbd mirror status
 120 fi
 121
 122 testlog "TEST: test image rename"
 123 new_name="${image}_RENAMED"
 124 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 125 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 126 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 127 admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
 128 admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 129 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 130 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 131 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 132 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 133
 134 testlog "TEST: test trash move restore"
 135 image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image})
 136 trash_move ${CLUSTER2} ${POOL} ${image}
 137 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 138 trash_restore ${CLUSTER2} ${POOL} ${image_id}
 139 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 140
 141 testlog "TEST: failover and failback"
 142 start_mirrors ${CLUSTER2}
 143
 144 # demote and promote same cluster
 145 demote_image ${CLUSTER2} ${POOL} ${image}
 146 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 147 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 148 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 149 promote_image ${CLUSTER2} ${POOL} ${image}
 150 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 151 write_image ${CLUSTER2} ${POOL} ${image} 100
 152 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 153 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 154 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 155 compare_images ${POOL} ${image}
 156
 157 # failover (unmodified)
 158 demote_image ${CLUSTER2} ${POOL} ${image}
 159 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 160 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 161 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 162 promote_image ${CLUSTER1} ${POOL} ${image}
 163 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 164
 165 # failback (unmodified)
 166 demote_image ${CLUSTER1} ${POOL} ${image}
 167 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 168 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 169 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 170 promote_image ${CLUSTER2} ${POOL} ${image}
 171 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 172 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 173 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 174 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 175 compare_images ${POOL} ${image}
 176
 177 # failover
 178 demote_image ${CLUSTER2} ${POOL} ${image}
 179 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 180 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 181 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 182 promote_image ${CLUSTER1} ${POOL} ${image}
 183 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 184 write_image ${CLUSTER1} ${POOL} ${image} 100
 185 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 186 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 187 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
 188 compare_images ${POOL} ${image}
 189
 190 # failback
 191 demote_image ${CLUSTER1} ${POOL} ${image}
 192 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 193 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 194 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 195 promote_image ${CLUSTER2} ${POOL} ${image}
 196 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 197 write_image ${CLUSTER2} ${POOL} ${image} 100
 198 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 199 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 200 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 201 compare_images ${POOL} ${image}
 202
 203 # force promote
 204 force_promote_image=test_force_promote
 205 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 206 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 207 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 208 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 209 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 210 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
 211 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 212 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 213 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 214 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 215 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 216 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 217 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 218 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 219
 220 testlog "TEST: cloned images"
 221 testlog " - default"
 222 parent_image=test_parent
 223 parent_snap=snap
 224 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 225 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 226 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 227 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 228
 229 clone_image=test_clone
 230 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 231 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 232
 233 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 234 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 235 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 236 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
 237 compare_images ${PARENT_POOL} ${parent_image}
 238
 239 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 240 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 241 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
 242 compare_images ${POOL} ${clone_image}
 243
 244 testlog " - clone v1"
 245 clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 246
 247 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 248             ${clone_image}_v1 --rbd-default-clone-format 1
 249 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1
 250 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1
 251 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1
 252
 253 testlog " - clone v2"
 254 parent_snap=snap_v2
 255 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 256 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 257             ${clone_image}_v2 --rbd-default-clone-format 2
 258 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2
 259 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2
 260 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2
 261
 262 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 263 test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 264 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 265 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2
 266 wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted'
 267 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 268 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 269
 270 testlog " - clone v2 non-primary"
 271 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 272 wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 273 clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 274             ${clone_image}_v2 --rbd-default-clone-format 2
 275 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 276 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 277 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 278 remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2
 279 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 280
 281 testlog "TEST: data pool"
 282 dp_image=test_data_pool
 283 create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
 284 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
 285 test "${data_pool}" = "${PARENT_POOL}"
 286 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
 287 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
 288 test "${data_pool}" = "${PARENT_POOL}"
 289 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
 290 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 291 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
 292 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 293 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
 294 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'master_position'
 295 compare_images ${POOL} ${dp_image}@snap1
 296 compare_images ${POOL} ${dp_image}@snap2
 297 compare_images ${POOL} ${dp_image}
 298
 299 testlog "TEST: disable mirroring / delete non-primary image"
 300 image2=test2
 301 image3=test3
 302 image4=test4
 303 image5=test5
 304 for i in ${image2} ${image3} ${image4} ${image5}; do
 305   create_image ${CLUSTER2} ${POOL} ${i}
 306   write_image ${CLUSTER2} ${POOL} ${i} 100
 307   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 308   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 309   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 310     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 311     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 312   fi
 313   write_image ${CLUSTER2} ${POOL} ${i} 100
 314   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 315   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 316 done
 317
 318 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 319 for i in ${image2} ${image4}; do
 320   disable_mirror ${CLUSTER2} ${POOL} ${i}
 321 done
 322
 323 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 324 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 325 for i in ${image3} ${image5}; do
 326   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 327   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 328   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 329 done
 330
 331 for i in ${image2} ${image3} ${image4} ${image5}; do
 332   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 333 done
 334
 335 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 336 for i in ${image2} ${image4}; do
 337   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 338   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 339   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 340   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 341   compare_images ${POOL} ${i}
 342 done
 343
 344 testlog "TEST: remove mirroring pool"
 345 pool=pool_to_remove
 346 for cluster in ${CLUSTER1} ${CLUSTER2}; do
 347     CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16
 348     CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool}
 349     rbd --cluster ${cluster} mirror pool enable ${pool} pool
 350 done
 351 rbd --cluster ${CLUSTER1} mirror pool peer add ${pool} ${CLUSTER2}
 352 rbd --cluster ${CLUSTER2} mirror pool peer add ${pool} ${CLUSTER1}
 353 rdp_image=test_remove_data_pool
 354 create_image ${CLUSTER2} ${pool} ${image} 128
 355 create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
 356 write_image ${CLUSTER2} ${pool} ${image} 100
 357 write_image ${CLUSTER2} ${POOL} ${rdp_image} 100
 358 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image}
 359 wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'master_position'
 360 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
 361 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'master_position'
 362 for cluster in ${CLUSTER1} ${CLUSTER2}; do
 363     CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it
 364 done
 365 remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image}
 366 wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
 367 for i in 0 1 2 4 8 8 8 8 16 16; do
 368     sleep $i
 369     admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break
 370 done
 371 admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
 372
 373 testlog "TEST: snapshot rename"
 374 snap_name='snap_rename'
 375 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 376 for i in `seq 1 20`; do
 377   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 378 done
 379 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 380
 381 testlog "TEST: disable mirror while daemon is stopped"
 382 stop_mirrors ${CLUSTER1}
 383 stop_mirrors ${CLUSTER2}
 384 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 385 disable_mirror ${CLUSTER2} ${POOL} ${image}
 386 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 387   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 388 fi
 389 start_mirrors ${CLUSTER1}
 390 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 391 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 392 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 393 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 394
 395 testlog "TEST: simple image resync"
 396 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 397 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 398 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 399 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 400 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 401 compare_images ${POOL} ${image}
 402
 403 testlog "TEST: image resync while replayer is stopped"
 404 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 405   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 406   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 407   request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 408   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 409   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 410   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 411   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 412   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 413   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 414   compare_images ${POOL} ${image}
 415 fi
 416
 417 testlog "TEST: request image resync while daemon is offline"
 418 stop_mirrors ${CLUSTER1}
 419 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 420 start_mirrors ${CLUSTER1}
 421 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 422 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 423 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 424 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 425 compare_images ${POOL} ${image}
 426
 427 testlog "TEST: client disconnect"
 428 image=laggy
 429 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 430 write_image ${CLUSTER2} ${POOL} ${image} 10
 431
 432 testlog " - replay stopped after disconnect"
 433 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 434 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 435 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 436 disconnect_image ${CLUSTER2} ${POOL} ${image}
 437 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 438 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 439 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 440
 441 testlog " - replay started after resync requested"
 442 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 443 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 444 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 445 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 446 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 447 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 448 compare_images ${POOL} ${image}
 449
 450 testlog " - disconnected after max_concurrent_object_sets reached"
 451 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 452   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 453   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 454   test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 455   set_image_meta ${CLUSTER2} ${POOL} ${image} \
 456                  conf_rbd_journal_max_concurrent_object_sets 1
 457   write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 458   write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 459   test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 460   set_image_meta ${CLUSTER2} ${POOL} ${image} \
 461                  conf_rbd_journal_max_concurrent_object_sets 0
 462
 463   testlog " - replay is still stopped (disconnected) after restart"
 464   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 465   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 466   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 467 fi
 468
 469 testlog " - replay started after resync requested"
 470 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 471 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 472 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 473 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 474 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 475 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 476 compare_images ${POOL} ${image}
 477
 478 testlog " - rbd_mirroring_resync_after_disconnect config option"
 479 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 480                conf_rbd_mirroring_resync_after_disconnect true
 481 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 482 image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
 483 disconnect_image ${CLUSTER2} ${POOL} ${image}
 484 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 485 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 486 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 487 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 488 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 489 compare_images ${POOL} ${image}
 490 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 491                conf_rbd_mirroring_resync_after_disconnect false
 492 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 493 disconnect_image ${CLUSTER2} ${POOL} ${image}
 494 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 495 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 496 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 497
 498 testlog "TEST: split-brain"
 499 image=split-brain
 500 create_image ${CLUSTER2} ${POOL} ${image}
 501 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 502 promote_image ${CLUSTER1} ${POOL} ${image} --force
 503 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 504 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 505 write_image ${CLUSTER1} ${POOL} ${image} 10
 506 demote_image ${CLUSTER1} ${POOL} ${image}
 507 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 508 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 509 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 510
 511 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 512   # teuthology will trash the daemon
 513   testlog "TEST: no blacklists"
 514   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 515   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 516 fi