ceph/qa/workunits/rbd/rbd_mirror.sh

   1 #!/bin/sh
   2 #
   3 # rbd_mirror.sh - test rbd-mirror daemon
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 testlog "TEST: add image and test replay"
  13 start_mirror ${CLUSTER1}
  14 image=test
  15 create_image ${CLUSTER2} ${POOL} ${image}
  16 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  17 write_image ${CLUSTER2} ${POOL} ${image} 100
  18 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  19 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  20 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  21   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  22 fi
  23 compare_images ${POOL} ${image}
  24
  25 testlog "TEST: stop mirror, add image, start mirror and test replay"
  26 stop_mirror ${CLUSTER1}
  27 image1=test1
  28 create_image ${CLUSTER2} ${POOL} ${image1}
  29 write_image ${CLUSTER2} ${POOL} ${image1} 100
  30 start_mirror ${CLUSTER1}
  31 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  32 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  33 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
  34 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  35   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  36 fi
  37 compare_images ${POOL} ${image1}
  38
  39 testlog "TEST: test the first image is replaying after restart"
  40 write_image ${CLUSTER2} ${POOL} ${image} 100
  41 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  42 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  43 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  44 compare_images ${POOL} ${image}
  45
  46 testlog "TEST: stop/start/restart mirror via admin socket"
  47 admin_daemon ${CLUSTER1} rbd mirror stop
  48 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  49 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  50 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  51 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  52
  53 admin_daemon ${CLUSTER1} rbd mirror start
  54 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  55 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  56 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  57 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  58
  59 admin_daemon ${CLUSTER1} rbd mirror restart
  60 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  61 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  62 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  63 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  64
  65 admin_daemon ${CLUSTER1} rbd mirror stop
  66 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  67 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  68 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  69 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  70
  71 admin_daemon ${CLUSTER1} rbd mirror restart
  72 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  73 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  74 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  75 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  76
  77 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
  78 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  79 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  80 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  81 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  82
  83 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
  84 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  85 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  86
  87 admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
  88 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  89 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  90
  91 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  92 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  93 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  94
  95 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
  96 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  97 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  98
  99 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
 100 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 101 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 102 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 103 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 104
 105 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 106 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 107 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 108 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 109 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 110
 111 admin_daemon ${CLUSTER1} rbd mirror flush
 112 admin_daemon ${CLUSTER1} rbd mirror status
 113
 114 testlog "TEST: failover and failback"
 115 start_mirror ${CLUSTER2}
 116
 117 # demote and promote same cluster
 118 demote_image ${CLUSTER2} ${POOL} ${image}
 119 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 120 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 121 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 122 promote_image ${CLUSTER2} ${POOL} ${image}
 123 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 124 write_image ${CLUSTER2} ${POOL} ${image} 100
 125 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 126 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 127 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 128 compare_images ${POOL} ${image}
 129
 130 # failover (unmodified)
 131 demote_image ${CLUSTER2} ${POOL} ${image}
 132 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 133 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 134 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 135 promote_image ${CLUSTER1} ${POOL} ${image}
 136 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 137
 138 # failback (unmodified)
 139 demote_image ${CLUSTER1} ${POOL} ${image}
 140 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 141 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 142 promote_image ${CLUSTER2} ${POOL} ${image}
 143 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 144 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 145 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 146 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 147 compare_images ${POOL} ${image}
 148
 149 # failover
 150 demote_image ${CLUSTER2} ${POOL} ${image}
 151 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 152 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 153 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 154 promote_image ${CLUSTER1} ${POOL} ${image}
 155 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 156 write_image ${CLUSTER1} ${POOL} ${image} 100
 157 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 158 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 159 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
 160 compare_images ${POOL} ${image}
 161
 162 # failback
 163 demote_image ${CLUSTER1} ${POOL} ${image}
 164 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 165 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 166 promote_image ${CLUSTER2} ${POOL} ${image}
 167 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 168 write_image ${CLUSTER2} ${POOL} ${image} 100
 169 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 170 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 171 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 172 compare_images ${POOL} ${image}
 173
 174 # force promote
 175 force_promote_image=test_force_promote
 176 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 177 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 178 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 179 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 180 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 181 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
 182 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 183 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 184 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 185 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 186 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 187 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 188 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 189 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 190
 191 testlog "TEST: cloned images"
 192 parent_image=test_parent
 193 parent_snap=snap
 194 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 195 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 196 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 197 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 198
 199 clone_image=test_clone
 200 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 201 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 202
 203 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 204 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 205 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 206 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
 207 compare_images ${PARENT_POOL} ${parent_image}
 208
 209 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 210 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 211 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
 212 compare_images ${POOL} ${clone_image}
 213
 214 expect_failure "is non-primary" clone_image ${CLUSTER1} ${PARENT_POOL} \
 215     ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 216
 217 testlog "TEST: disable mirroring / delete non-primary image"
 218 image2=test2
 219 image3=test3
 220 image4=test4
 221 image5=test5
 222 for i in ${image2} ${image3} ${image4} ${image5}; do
 223   create_image ${CLUSTER2} ${POOL} ${i}
 224   write_image ${CLUSTER2} ${POOL} ${i} 100
 225   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 226   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 227   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 228     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 229     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 230   fi
 231   write_image ${CLUSTER2} ${POOL} ${i} 100
 232   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 233   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 234 done
 235
 236 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 237 for i in ${image2} ${image4}; do
 238   disable_mirror ${CLUSTER2} ${POOL} ${i}
 239 done
 240
 241 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 242 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 243 for i in ${image3} ${image5}; do
 244   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 245   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 246   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 247 done
 248
 249 for i in ${image2} ${image3} ${image4} ${image5}; do
 250   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 251 done
 252
 253 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 254 for i in ${image2} ${image4}; do
 255   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 256   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 257   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 258   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 259   compare_images ${POOL} ${i}
 260 done
 261
 262 testlog "TEST: snapshot rename"
 263 snap_name='snap_rename'
 264 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 265 for i in `seq 1 20`; do
 266   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 267 done
 268 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 269
 270 testlog "TEST: disable mirror while daemon is stopped"
 271 stop_mirror ${CLUSTER1}
 272 stop_mirror ${CLUSTER2}
 273 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 274 disable_mirror ${CLUSTER2} ${POOL} ${image}
 275 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 276   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 277 fi
 278 start_mirror ${CLUSTER1}
 279 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 280 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 281 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 282 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 283
 284 testlog "TEST: simple image resync"
 285 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 286 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 287 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 288 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 289 compare_images ${POOL} ${image}
 290
 291 testlog "TEST: image resync while replayer is stopped"
 292 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 293 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 294 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 295 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 296 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 297 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 298 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 299 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 300 compare_images ${POOL} ${image}
 301
 302 testlog "TEST: request image resync while daemon is offline"
 303 stop_mirror ${CLUSTER1}
 304 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 305 start_mirror ${CLUSTER1}
 306 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 307 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 308 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 309 compare_images ${POOL} ${image}
 310
 311 testlog "TEST: client disconnect"
 312 image=laggy
 313 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 314 write_image ${CLUSTER2} ${POOL} ${image} 10
 315
 316 testlog " - replay stopped after disconnect"
 317 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 318 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 319 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 320 disconnect_image ${CLUSTER2} ${POOL} ${image}
 321 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 322 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 323 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 324
 325 testlog " - replay started after resync requested"
 326 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 327 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 328 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 329 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 330 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 331 compare_images ${POOL} ${image}
 332
 333 testlog " - disconnected after max_concurrent_object_sets reached"
 334 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 335 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 336 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 337 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 338                conf_rbd_journal_max_concurrent_object_sets 1
 339 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 340 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 341 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 342 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 343                conf_rbd_journal_max_concurrent_object_sets 0
 344
 345 testlog " - replay is still stopped (disconnected) after restart"
 346 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 347 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 348 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 349
 350 testlog " - replay started after resync requested"
 351 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 352 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 353 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 354 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 355 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 356 compare_images ${POOL} ${image}
 357
 358 testlog " - rbd_mirroring_resync_after_disconnect config option"
 359 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 360                conf_rbd_mirroring_resync_after_disconnect true
 361 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 362 disconnect_image ${CLUSTER2} ${POOL} ${image}
 363 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 364 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 365 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 366 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 367 compare_images ${POOL} ${image}
 368 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 369                conf_rbd_mirroring_resync_after_disconnect false
 370 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 371 disconnect_image ${CLUSTER2} ${POOL} ${image}
 372 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 373 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 374 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 375
 376 testlog "TEST: split-brain"
 377 image=split-brain
 378 create_image ${CLUSTER2} ${POOL} ${image}
 379 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 380 demote_image ${CLUSTER2} ${POOL} ${image}
 381 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 382 promote_image ${CLUSTER1} ${POOL} ${image}
 383 write_image ${CLUSTER1} ${POOL} ${image} 10
 384 demote_image ${CLUSTER1} ${POOL} ${image}
 385 promote_image ${CLUSTER2} ${POOL} ${image}
 386 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 387 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 388 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 389
 390 testlog "TEST: no blacklists"
 391 ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 392 ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 393
 394 echo OK