ceph/qa/workunits/rbd/rbd_mirror.sh

   1 #!/bin/sh
   2 #
   3 # rbd_mirror.sh - test rbd-mirror daemon
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 testlog "TEST: add image and test replay"
  13 start_mirror ${CLUSTER1}
  14 image=test
  15 create_image ${CLUSTER2} ${POOL} ${image}
  16 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  17 write_image ${CLUSTER2} ${POOL} ${image} 100
  18 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  19 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  20 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  21   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  22 fi
  23 compare_images ${POOL} ${image}
  24
  25 testlog "TEST: stop mirror, add image, start mirror and test replay"
  26 stop_mirror ${CLUSTER1}
  27 image1=test1
  28 create_image ${CLUSTER2} ${POOL} ${image1}
  29 write_image ${CLUSTER2} ${POOL} ${image1} 100
  30 start_mirror ${CLUSTER1}
  31 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  32 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  33 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
  34 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  35   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  36 fi
  37 compare_images ${POOL} ${image1}
  38
  39 testlog "TEST: test the first image is replaying after restart"
  40 write_image ${CLUSTER2} ${POOL} ${image} 100
  41 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  42 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  43 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  44 compare_images ${POOL} ${image}
  45
  46 testlog "TEST: stop/start/restart mirror via admin socket"
  47 admin_daemon ${CLUSTER1} rbd mirror stop
  48 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  49 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  50 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  51 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  52
  53 admin_daemon ${CLUSTER1} rbd mirror start
  54 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  55 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  56 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  57 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  58
  59 admin_daemon ${CLUSTER1} rbd mirror restart
  60 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  61 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  62 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  63 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  64
  65 admin_daemon ${CLUSTER1} rbd mirror stop
  66 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  67 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  68 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  69 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  70
  71 admin_daemon ${CLUSTER1} rbd mirror restart
  72 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  73 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  74 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  75 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  76
  77 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
  78 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  79 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  80 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  81 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  82
  83 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
  84 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  85 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  86
  87 admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
  88 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  89 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  90
  91 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  92 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  93 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  94
  95 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
  96 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  97 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  98
  99 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
 100 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 101 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 102 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 103 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 104
 105 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 106 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 107 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 108 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 109 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 110
 111 admin_daemon ${CLUSTER1} rbd mirror flush
 112 admin_daemon ${CLUSTER1} rbd mirror status
 113
 114 testlog "TEST: failover and failback"
 115 start_mirror ${CLUSTER2}
 116
 117 # demote and promote same cluster
 118 demote_image ${CLUSTER2} ${POOL} ${image}
 119 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 120 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 121 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 122 promote_image ${CLUSTER2} ${POOL} ${image}
 123 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 124 write_image ${CLUSTER2} ${POOL} ${image} 100
 125 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 126 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 127 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 128 compare_images ${POOL} ${image}
 129
 130 # failover (unmodified)
 131 demote_image ${CLUSTER2} ${POOL} ${image}
 132 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 133 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 134 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 135 promote_image ${CLUSTER1} ${POOL} ${image}
 136 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 137
 138 # failback (unmodified)
 139 demote_image ${CLUSTER1} ${POOL} ${image}
 140 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 141 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 142 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 143 promote_image ${CLUSTER2} ${POOL} ${image}
 144 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 145 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 146 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 147 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 148 compare_images ${POOL} ${image}
 149
 150 # failover
 151 demote_image ${CLUSTER2} ${POOL} ${image}
 152 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 153 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 154 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 155 promote_image ${CLUSTER1} ${POOL} ${image}
 156 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 157 write_image ${CLUSTER1} ${POOL} ${image} 100
 158 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 159 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 160 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
 161 compare_images ${POOL} ${image}
 162
 163 # failback
 164 demote_image ${CLUSTER1} ${POOL} ${image}
 165 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 166 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 167 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 168 promote_image ${CLUSTER2} ${POOL} ${image}
 169 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 170 write_image ${CLUSTER2} ${POOL} ${image} 100
 171 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 172 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 173 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 174 compare_images ${POOL} ${image}
 175
 176 # force promote
 177 force_promote_image=test_force_promote
 178 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 179 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 180 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 181 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 182 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 183 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
 184 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 185 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 186 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 187 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 188 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 189 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 190 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 191 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 192
 193 testlog "TEST: cloned images"
 194 parent_image=test_parent
 195 parent_snap=snap
 196 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 197 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 198 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 199 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 200
 201 clone_image=test_clone
 202 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 203 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 204
 205 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 206 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 207 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 208 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
 209 compare_images ${PARENT_POOL} ${parent_image}
 210
 211 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 212 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 213 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
 214 compare_images ${POOL} ${clone_image}
 215
 216 expect_failure "is non-primary" clone_image ${CLUSTER1} ${PARENT_POOL} \
 217     ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 218
 219 testlog "TEST: disable mirroring / delete non-primary image"
 220 image2=test2
 221 image3=test3
 222 image4=test4
 223 image5=test5
 224 for i in ${image2} ${image3} ${image4} ${image5}; do
 225   create_image ${CLUSTER2} ${POOL} ${i}
 226   write_image ${CLUSTER2} ${POOL} ${i} 100
 227   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 228   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 229   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 230     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 231     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 232   fi
 233   write_image ${CLUSTER2} ${POOL} ${i} 100
 234   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 235   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 236 done
 237
 238 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 239 for i in ${image2} ${image4}; do
 240   disable_mirror ${CLUSTER2} ${POOL} ${i}
 241 done
 242
 243 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 244 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 245 for i in ${image3} ${image5}; do
 246   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 247   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 248   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 249 done
 250
 251 for i in ${image2} ${image3} ${image4} ${image5}; do
 252   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 253 done
 254
 255 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 256 for i in ${image2} ${image4}; do
 257   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 258   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 259   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 260   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 261   compare_images ${POOL} ${i}
 262 done
 263
 264 testlog "TEST: snapshot rename"
 265 snap_name='snap_rename'
 266 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 267 for i in `seq 1 20`; do
 268   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 269 done
 270 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 271
 272 testlog "TEST: disable mirror while daemon is stopped"
 273 stop_mirror ${CLUSTER1}
 274 stop_mirror ${CLUSTER2}
 275 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 276 disable_mirror ${CLUSTER2} ${POOL} ${image}
 277 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 278   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 279 fi
 280 start_mirror ${CLUSTER1}
 281 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 282 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 283 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 284 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 285
 286 testlog "TEST: simple image resync"
 287 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 288 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 289 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 290 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 291 compare_images ${POOL} ${image}
 292
 293 testlog "TEST: image resync while replayer is stopped"
 294 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 295 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 296 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 297 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 298 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 299 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 300 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 301 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 302 compare_images ${POOL} ${image}
 303
 304 testlog "TEST: request image resync while daemon is offline"
 305 stop_mirror ${CLUSTER1}
 306 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 307 start_mirror ${CLUSTER1}
 308 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 309 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 310 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 311 compare_images ${POOL} ${image}
 312
 313 testlog "TEST: client disconnect"
 314 image=laggy
 315 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 316 write_image ${CLUSTER2} ${POOL} ${image} 10
 317
 318 testlog " - replay stopped after disconnect"
 319 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 320 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 321 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 322 disconnect_image ${CLUSTER2} ${POOL} ${image}
 323 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 324 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 325 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 326
 327 testlog " - replay started after resync requested"
 328 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 329 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 330 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 331 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 332 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 333 compare_images ${POOL} ${image}
 334
 335 testlog " - disconnected after max_concurrent_object_sets reached"
 336 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 337 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 338 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 339 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 340                conf_rbd_journal_max_concurrent_object_sets 1
 341 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 342 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 343 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 344 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 345                conf_rbd_journal_max_concurrent_object_sets 0
 346
 347 testlog " - replay is still stopped (disconnected) after restart"
 348 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 349 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 350 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 351
 352 testlog " - replay started after resync requested"
 353 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 354 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 355 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 356 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 357 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 358 compare_images ${POOL} ${image}
 359
 360 testlog " - rbd_mirroring_resync_after_disconnect config option"
 361 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 362                conf_rbd_mirroring_resync_after_disconnect true
 363 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 364 disconnect_image ${CLUSTER2} ${POOL} ${image}
 365 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 366 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 367 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 368 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 369 compare_images ${POOL} ${image}
 370 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 371                conf_rbd_mirroring_resync_after_disconnect false
 372 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 373 disconnect_image ${CLUSTER2} ${POOL} ${image}
 374 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 375 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 376 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 377
 378 testlog "TEST: split-brain"
 379 image=split-brain
 380 create_image ${CLUSTER2} ${POOL} ${image}
 381 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 382 demote_image ${CLUSTER2} ${POOL} ${image}
 383 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 384 promote_image ${CLUSTER1} ${POOL} ${image}
 385 write_image ${CLUSTER1} ${POOL} ${image} 10
 386 demote_image ${CLUSTER1} ${POOL} ${image}
 387 promote_image ${CLUSTER2} ${POOL} ${image}
 388 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 389 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 390 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 391
 392 testlog "TEST: no blacklists"
 393 CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 394 CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 395
 396 echo OK