ceph/qa/workunits/rbd/rbd_mirror.sh

   1 #!/bin/sh
   2 #
   3 # rbd_mirror.sh - test rbd-mirror daemon
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 testlog "TEST: add image and test replay"
  13 start_mirror ${CLUSTER1}
  14 image=test
  15 create_image ${CLUSTER2} ${POOL} ${image}
  16 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
  17 set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
  18 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  19 write_image ${CLUSTER2} ${POOL} ${image} 100
  20 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  21 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  22 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  23   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  24 fi
  25 compare_images ${POOL} ${image}
  26 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
  27 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
  28
  29 testlog "TEST: stop mirror, add image, start mirror and test replay"
  30 stop_mirror ${CLUSTER1}
  31 image1=test1
  32 create_image ${CLUSTER2} ${POOL} ${image1}
  33 write_image ${CLUSTER2} ${POOL} ${image1} 100
  34 start_mirror ${CLUSTER1}
  35 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  36 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  37 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
  38 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  39   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  40 fi
  41 compare_images ${POOL} ${image1}
  42
  43 testlog "TEST: test the first image is replaying after restart"
  44 write_image ${CLUSTER2} ${POOL} ${image} 100
  45 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  46 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  47 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  48 compare_images ${POOL} ${image}
  49
  50 testlog "TEST: stop/start/restart mirror via admin socket"
  51 admin_daemon ${CLUSTER1} rbd mirror stop
  52 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  53 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  54 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  55 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  56
  57 admin_daemon ${CLUSTER1} rbd mirror start
  58 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  59 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  60 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  61 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  62
  63 admin_daemon ${CLUSTER1} rbd mirror restart
  64 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  65 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  66 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  67 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  68
  69 admin_daemon ${CLUSTER1} rbd mirror stop
  70 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  71 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  72 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  73 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  74
  75 admin_daemon ${CLUSTER1} rbd mirror restart
  76 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  77 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  78 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  79 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  80
  81 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
  82 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  83 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  84 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  85 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  86
  87 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
  88 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  89 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  90
  91 admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
  92 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  93 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  94
  95 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  96 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  97 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  98
  99 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 100 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 101 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 102
 103 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
 104 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 105 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 106 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 107 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 108
 109 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 110 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 111 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 112 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 113 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 114
 115 admin_daemon ${CLUSTER1} rbd mirror flush
 116 admin_daemon ${CLUSTER1} rbd mirror status
 117
 118 testlog "TEST: test image rename"
 119 new_name="${image}_RENAMED"
 120 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 121 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 122 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 123 admin_daemon ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
 124 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 125 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 126 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 127 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 128 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 129
 130 testlog "TEST: failover and failback"
 131 start_mirror ${CLUSTER2}
 132
 133 # demote and promote same cluster
 134 demote_image ${CLUSTER2} ${POOL} ${image}
 135 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 136 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 137 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 138 promote_image ${CLUSTER2} ${POOL} ${image}
 139 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 140 write_image ${CLUSTER2} ${POOL} ${image} 100
 141 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 142 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 143 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 144 compare_images ${POOL} ${image}
 145
 146 # failover (unmodified)
 147 demote_image ${CLUSTER2} ${POOL} ${image}
 148 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 149 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 150 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 151 promote_image ${CLUSTER1} ${POOL} ${image}
 152 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 153
 154 # failback (unmodified)
 155 demote_image ${CLUSTER1} ${POOL} ${image}
 156 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 157 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 158 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 159 promote_image ${CLUSTER2} ${POOL} ${image}
 160 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 161 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 162 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 163 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 164 compare_images ${POOL} ${image}
 165
 166 # failover
 167 demote_image ${CLUSTER2} ${POOL} ${image}
 168 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 169 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 170 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 171 promote_image ${CLUSTER1} ${POOL} ${image}
 172 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 173 write_image ${CLUSTER1} ${POOL} ${image} 100
 174 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 175 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 176 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
 177 compare_images ${POOL} ${image}
 178
 179 # failback
 180 demote_image ${CLUSTER1} ${POOL} ${image}
 181 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 182 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 183 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 184 promote_image ${CLUSTER2} ${POOL} ${image}
 185 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 186 write_image ${CLUSTER2} ${POOL} ${image} 100
 187 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 188 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 189 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 190 compare_images ${POOL} ${image}
 191
 192 # force promote
 193 force_promote_image=test_force_promote
 194 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 195 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 196 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 197 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 198 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 199 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
 200 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 201 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 202 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 203 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 204 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 205 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 206 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 207 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 208
 209 testlog "TEST: cloned images"
 210 parent_image=test_parent
 211 parent_snap=snap
 212 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 213 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 214 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 215 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 216
 217 clone_image=test_clone
 218 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 219 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 220
 221 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 222 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 223 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 224 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
 225 compare_images ${PARENT_POOL} ${parent_image}
 226
 227 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 228 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 229 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
 230 compare_images ${POOL} ${clone_image}
 231
 232 expect_failure "is non-primary" clone_image ${CLUSTER1} ${PARENT_POOL} \
 233     ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 234
 235 testlog "TEST: data pool"
 236 dp_image=test_data_pool
 237 create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
 238 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
 239 test "${data_pool}" = "${PARENT_POOL}"
 240 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
 241 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
 242 test "${data_pool}" = "${PARENT_POOL}"
 243 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
 244 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 245 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
 246 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 247 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
 248 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'master_position'
 249 compare_images ${POOL} ${dp_image}@snap1
 250 compare_images ${POOL} ${dp_image}@snap2
 251 compare_images ${POOL} ${dp_image}
 252
 253 testlog "TEST: disable mirroring / delete non-primary image"
 254 image2=test2
 255 image3=test3
 256 image4=test4
 257 image5=test5
 258 for i in ${image2} ${image3} ${image4} ${image5}; do
 259   create_image ${CLUSTER2} ${POOL} ${i}
 260   write_image ${CLUSTER2} ${POOL} ${i} 100
 261   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 262   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 263   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 264     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 265     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 266   fi
 267   write_image ${CLUSTER2} ${POOL} ${i} 100
 268   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 269   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 270 done
 271
 272 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 273 for i in ${image2} ${image4}; do
 274   disable_mirror ${CLUSTER2} ${POOL} ${i}
 275 done
 276
 277 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 278 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 279 for i in ${image3} ${image5}; do
 280   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 281   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 282   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 283 done
 284
 285 for i in ${image2} ${image3} ${image4} ${image5}; do
 286   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 287 done
 288
 289 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 290 for i in ${image2} ${image4}; do
 291   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 292   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 293   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 294   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 295   compare_images ${POOL} ${i}
 296 done
 297
 298 testlog "TEST: snapshot rename"
 299 snap_name='snap_rename'
 300 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 301 for i in `seq 1 20`; do
 302   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 303 done
 304 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 305
 306 testlog "TEST: disable mirror while daemon is stopped"
 307 stop_mirror ${CLUSTER1}
 308 stop_mirror ${CLUSTER2}
 309 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 310 disable_mirror ${CLUSTER2} ${POOL} ${image}
 311 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 312   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 313 fi
 314 start_mirror ${CLUSTER1}
 315 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 316 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 317 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 318 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 319
 320 testlog "TEST: simple image resync"
 321 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 322 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 323 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 324 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 325 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 326 compare_images ${POOL} ${image}
 327
 328 testlog "TEST: image resync while replayer is stopped"
 329 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 330 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 331 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 332 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 333 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 334 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 335 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 336 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 337 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 338 compare_images ${POOL} ${image}
 339
 340 testlog "TEST: request image resync while daemon is offline"
 341 stop_mirror ${CLUSTER1}
 342 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 343 start_mirror ${CLUSTER1}
 344 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 345 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 346 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 347 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 348 compare_images ${POOL} ${image}
 349
 350 testlog "TEST: client disconnect"
 351 image=laggy
 352 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 353 write_image ${CLUSTER2} ${POOL} ${image} 10
 354
 355 testlog " - replay stopped after disconnect"
 356 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 357 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 358 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 359 disconnect_image ${CLUSTER2} ${POOL} ${image}
 360 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 361 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 362 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 363
 364 testlog " - replay started after resync requested"
 365 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 366 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 367 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 368 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 369 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 370 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 371 compare_images ${POOL} ${image}
 372
 373 testlog " - disconnected after max_concurrent_object_sets reached"
 374 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 375 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 376 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 377 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 378                conf_rbd_journal_max_concurrent_object_sets 1
 379 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 380 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 381 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 382 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 383                conf_rbd_journal_max_concurrent_object_sets 0
 384
 385 testlog " - replay is still stopped (disconnected) after restart"
 386 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 387 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 388 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 389
 390 testlog " - replay started after resync requested"
 391 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 392 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 393 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 394 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 395 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 396 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 397 compare_images ${POOL} ${image}
 398
 399 testlog " - rbd_mirroring_resync_after_disconnect config option"
 400 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 401                conf_rbd_mirroring_resync_after_disconnect true
 402 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 403 image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
 404 disconnect_image ${CLUSTER2} ${POOL} ${image}
 405 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 406 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 407 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 408 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 409 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 410 compare_images ${POOL} ${image}
 411 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 412                conf_rbd_mirroring_resync_after_disconnect false
 413 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 414 disconnect_image ${CLUSTER2} ${POOL} ${image}
 415 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 416 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 417 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 418
 419 testlog "TEST: split-brain"
 420 image=split-brain
 421 create_image ${CLUSTER2} ${POOL} ${image}
 422 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 423 promote_image ${CLUSTER1} ${POOL} ${image} --force
 424 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 425 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 426 write_image ${CLUSTER1} ${POOL} ${image} 10
 427 demote_image ${CLUSTER1} ${POOL} ${image}
 428 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 429 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 430 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 431
 432 testlog "TEST: no blacklists"
 433 CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 434 CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 435
 436 echo OK