ceph/qa/workunits/rbd/rbd_mirror_journal.sh

   1 #!/bin/sh -ex
   2 #
   3 # rbd_mirror_journal.sh - test rbd-mirror daemon in journal-based mirroring mode
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 setup
  13
  14 testlog "TEST: add image and test replay"
  15 start_mirrors ${CLUSTER1}
  16 image=test
  17 create_image ${CLUSTER2} ${POOL} ${image}
  18 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
  19 set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
  20 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  21 write_image ${CLUSTER2} ${POOL} ${image} 100
  22 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  23 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  24 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  25   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  26 fi
  27 compare_images ${POOL} ${image}
  28 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
  29 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
  30
  31 testlog "TEST: stop mirror, add image, start mirror and test replay"
  32 stop_mirrors ${CLUSTER1}
  33 image1=test1
  34 create_image ${CLUSTER2} ${POOL} ${image1}
  35 write_image ${CLUSTER2} ${POOL} ${image1} 100
  36 start_mirrors ${CLUSTER1}
  37 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  38 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  39 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
  40 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  41   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  42 fi
  43 compare_images ${POOL} ${image1}
  44
  45 testlog "TEST: test the first image is replaying after restart"
  46 write_image ${CLUSTER2} ${POOL} ${image} 100
  47 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  48 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  49 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  50 compare_images ${POOL} ${image}
  51
  52 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  53   testlog "TEST: stop/start/restart mirror via admin socket"
  54   all_admin_daemons ${CLUSTER1} rbd mirror stop
  55   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  56   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  57   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  58   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  59
  60   all_admin_daemons ${CLUSTER1} rbd mirror start
  61   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  62   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  63   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  64   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  65
  66   all_admin_daemons ${CLUSTER1} rbd mirror restart
  67   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  68   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  69   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  70   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  71
  72   all_admin_daemons ${CLUSTER1} rbd mirror stop
  73   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  74   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  75   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  76   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  77
  78   all_admin_daemons ${CLUSTER1} rbd mirror restart
  79   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  80   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  81   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  82   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  83
  84   all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
  85   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  86   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  87   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  88   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  89
  90   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
  91   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  92   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  93
  94   all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
  95   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  96   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  97
  98   admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  99   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 100   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 101
 102   all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 103   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 104   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 105
 106   all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 107   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 108   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 109   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 110   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 111
 112   all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 113   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 114   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 115   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 116   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 117
 118   flush ${CLUSTER1}
 119   all_admin_daemons ${CLUSTER1} rbd mirror status
 120 fi
 121
 122 testlog "TEST: test image rename"
 123 new_name="${image}_RENAMED"
 124 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 125 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 126 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 127 admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
 128 admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 129 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 130 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 131 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 132 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 133
 134 testlog "TEST: test trash move restore"
 135 image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image})
 136 trash_move ${CLUSTER2} ${POOL} ${image}
 137 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 138 trash_restore ${CLUSTER2} ${POOL} ${image_id}
 139 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 140
 141 testlog "TEST: failover and failback"
 142 start_mirrors ${CLUSTER2}
 143
 144 # demote and promote same cluster
 145 demote_image ${CLUSTER2} ${POOL} ${image}
 146 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 147 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 148 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 149 promote_image ${CLUSTER2} ${POOL} ${image}
 150 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 151 write_image ${CLUSTER2} ${POOL} ${image} 100
 152 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 153 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 154 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 155 compare_images ${POOL} ${image}
 156
 157 # failover (unmodified)
 158 demote_image ${CLUSTER2} ${POOL} ${image}
 159 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 160 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 161 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 162 promote_image ${CLUSTER1} ${POOL} ${image}
 163 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 164
 165 # failback (unmodified)
 166 demote_image ${CLUSTER1} ${POOL} ${image}
 167 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 168 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 169 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 170 promote_image ${CLUSTER2} ${POOL} ${image}
 171 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 172 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 173 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 174 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 175 compare_images ${POOL} ${image}
 176
 177 # failover
 178 demote_image ${CLUSTER2} ${POOL} ${image}
 179 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 180 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 181 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 182 promote_image ${CLUSTER1} ${POOL} ${image}
 183 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 184 write_image ${CLUSTER1} ${POOL} ${image} 100
 185 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 186 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 187 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
 188 compare_images ${POOL} ${image}
 189
 190 # failback
 191 demote_image ${CLUSTER1} ${POOL} ${image}
 192 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 193 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 194 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 195 promote_image ${CLUSTER2} ${POOL} ${image}
 196 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 197 write_image ${CLUSTER2} ${POOL} ${image} 100
 198 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 199 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 200 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 201 compare_images ${POOL} ${image}
 202
 203 # force promote
 204 force_promote_image=test_force_promote
 205 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 206 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 207 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 208 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 209 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 210 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
 211 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 212 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 213 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 214 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 215 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 216 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 217 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 218 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 219
 220 testlog "TEST: cloned images"
 221 testlog " - default"
 222 parent_image=test_parent
 223 parent_snap=snap
 224 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 225 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 226 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 227 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 228
 229 clone_image=test_clone
 230 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 231 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 232
 233 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} journal
 234 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 235 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 236 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
 237 compare_images ${PARENT_POOL} ${parent_image}
 238
 239 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 240 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 241 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
 242 compare_images ${POOL} ${clone_image}
 243
 244 testlog " - clone v1"
 245 clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 246
 247 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 248             ${clone_image}_v1 --rbd-default-clone-format 1
 249 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1
 250 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1
 251 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1
 252
 253 testlog " - clone v2"
 254 parent_snap=snap_v2
 255 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 256 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 257             ${clone_image}_v2 --rbd-default-clone-format 2
 258 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2
 259 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2
 260 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2
 261
 262 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 263 test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 264 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 265 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2
 266 wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted'
 267 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 268 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 269
 270 testlog " - clone v2 non-primary"
 271 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 272 wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 273 clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 274             ${clone_image}_v2 --rbd-default-clone-format 2
 275 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 276 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 277 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 278 remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2
 279 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 280
 281 testlog "TEST: data pool"
 282 dp_image=test_data_pool
 283 create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
 284 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
 285 test "${data_pool}" = "${PARENT_POOL}"
 286 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
 287 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
 288 test "${data_pool}" = "${PARENT_POOL}"
 289 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
 290 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 291 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
 292 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 293 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
 294 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'master_position'
 295 compare_images ${POOL} ${dp_image}@snap1
 296 compare_images ${POOL} ${dp_image}@snap2
 297 compare_images ${POOL} ${dp_image}
 298
 299 testlog "TEST: disable mirroring / delete non-primary image"
 300 image2=test2
 301 image3=test3
 302 image4=test4
 303 image5=test5
 304 for i in ${image2} ${image3} ${image4} ${image5}; do
 305   create_image ${CLUSTER2} ${POOL} ${i}
 306   write_image ${CLUSTER2} ${POOL} ${i} 100
 307   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 308   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 309   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 310     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 311     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 312   fi
 313   write_image ${CLUSTER2} ${POOL} ${i} 100
 314   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 315   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 316 done
 317
 318 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 319 for i in ${image2} ${image4}; do
 320   disable_mirror ${CLUSTER2} ${POOL} ${i}
 321 done
 322
 323 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 324 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 325 for i in ${image3} ${image5}; do
 326   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 327   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 328   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 329 done
 330
 331 for i in ${image2} ${image3} ${image4} ${image5}; do
 332   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 333 done
 334
 335 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 336 for i in ${image2} ${image4}; do
 337   enable_journaling ${CLUSTER2} ${POOL} ${i}
 338   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 339   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 340   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 341   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 342   compare_images ${POOL} ${i}
 343 done
 344
 345 testlog "TEST: remove mirroring pool"
 346 pool=pool_to_remove
 347 for cluster in ${CLUSTER1} ${CLUSTER2}; do
 348     CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16
 349     CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool}
 350     rbd --cluster ${cluster} mirror pool enable ${pool} pool
 351 done
 352 rbd --cluster ${CLUSTER1} mirror pool peer add ${pool} ${CLUSTER2}
 353 rbd --cluster ${CLUSTER2} mirror pool peer add ${pool} ${CLUSTER1}
 354 rdp_image=test_remove_data_pool
 355 create_image ${CLUSTER2} ${pool} ${image} 128
 356 create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
 357 write_image ${CLUSTER2} ${pool} ${image} 100
 358 write_image ${CLUSTER2} ${POOL} ${rdp_image} 100
 359 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image}
 360 wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'master_position'
 361 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
 362 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'master_position'
 363 for cluster in ${CLUSTER1} ${CLUSTER2}; do
 364     CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it
 365 done
 366 remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image}
 367 wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
 368 for i in 0 1 2 4 8 8 8 8 16 16; do
 369     sleep $i
 370     admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break
 371 done
 372 admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
 373
 374 testlog "TEST: snapshot rename"
 375 snap_name='snap_rename'
 376 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 377 for i in `seq 1 20`; do
 378   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 379 done
 380 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 381
 382 testlog "TEST: disable mirror while daemon is stopped"
 383 stop_mirrors ${CLUSTER1}
 384 stop_mirrors ${CLUSTER2}
 385 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 386 disable_mirror ${CLUSTER2} ${POOL} ${image}
 387 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 388   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 389 fi
 390 start_mirrors ${CLUSTER1}
 391 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 392 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 393 enable_journaling ${CLUSTER2} ${POOL} ${image}
 394 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 395 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 396
 397 testlog "TEST: non-default namespace image mirroring"
 398 testlog " - replay"
 399 create_image ${CLUSTER2} ${POOL}/${NS1} ${image}
 400 create_image ${CLUSTER2} ${POOL}/${NS2} ${image}
 401 enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} journal
 402 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image}
 403 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image}
 404 write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100
 405 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100
 406 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image}
 407 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image}
 408 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'master_position'
 409 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'master_position'
 410 compare_images ${POOL}/${NS1} ${image}
 411 compare_images ${POOL}/${NS2} ${image}
 412
 413 testlog " - disable mirroring / delete image"
 414 remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image}
 415 disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image}
 416 wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted'
 417 wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted'
 418
 419 testlog " - data pool"
 420 dp_image=test_data_pool
 421 create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL}
 422 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image})
 423 test "${data_pool}" = "${PARENT_POOL}"
 424 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image}
 425 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image})
 426 test "${data_pool}" = "${PARENT_POOL}"
 427 write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100
 428 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
 429 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'master_position'
 430 compare_images ${POOL}/${NS1} ${dp_image}
 431
 432 testlog "TEST: simple image resync"
 433 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 434 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 435 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 436 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 437 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 438 compare_images ${POOL} ${image}
 439
 440 testlog "TEST: image resync while replayer is stopped"
 441 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 442   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 443   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 444   request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 445   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 446   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 447   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 448   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 449   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 450   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 451   compare_images ${POOL} ${image}
 452 fi
 453
 454 testlog "TEST: request image resync while daemon is offline"
 455 stop_mirrors ${CLUSTER1}
 456 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 457 start_mirrors ${CLUSTER1}
 458 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 459 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 460 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 461 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 462 compare_images ${POOL} ${image}
 463
 464 testlog "TEST: client disconnect"
 465 image=laggy
 466 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 467 write_image ${CLUSTER2} ${POOL} ${image} 10
 468
 469 testlog " - replay stopped after disconnect"
 470 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 471 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 472 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 473 disconnect_image ${CLUSTER2} ${POOL} ${image}
 474 test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 475 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 476 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 477
 478 testlog " - replay started after resync requested"
 479 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 480 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 481 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 482 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 483 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 484 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 485 compare_images ${POOL} ${image}
 486
 487 testlog " - disconnected after max_concurrent_object_sets reached"
 488 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 489   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 490   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 491   test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 492   set_image_meta ${CLUSTER2} ${POOL} ${image} \
 493                  conf_rbd_journal_max_concurrent_object_sets 1
 494   write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 495   write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 496   test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 497   set_image_meta ${CLUSTER2} ${POOL} ${image} \
 498                  conf_rbd_journal_max_concurrent_object_sets 0
 499
 500   testlog " - replay is still stopped (disconnected) after restart"
 501   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 502   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 503   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 504 fi
 505
 506 testlog " - replay started after resync requested"
 507 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 508 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 509 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 510 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 511 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 512 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 513 compare_images ${POOL} ${image}
 514
 515 testlog " - rbd_mirroring_resync_after_disconnect config option"
 516 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 517                conf_rbd_mirroring_resync_after_disconnect true
 518 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 519 image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
 520 disconnect_image ${CLUSTER2} ${POOL} ${image}
 521 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 522 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 523 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 524 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 525 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 526 compare_images ${POOL} ${image}
 527 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 528                conf_rbd_mirroring_resync_after_disconnect false
 529 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 530 disconnect_image ${CLUSTER2} ${POOL} ${image}
 531 test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 532 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 533 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 534
 535 testlog "TEST: split-brain"
 536 image=split-brain
 537 create_image ${CLUSTER2} ${POOL} ${image}
 538 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 539 promote_image ${CLUSTER1} ${POOL} ${image} --force
 540 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 541 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 542 write_image ${CLUSTER1} ${POOL} ${image} 10
 543 demote_image ${CLUSTER1} ${POOL} ${image}
 544 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 545 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 546 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 547
 548 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 549   # teuthology will trash the daemon
 550   testlog "TEST: no blacklists"
 551   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 552   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 553 fi