ceph/qa/workunits/rbd/rbd_mirror_journal.sh

   1 #!/bin/sh -ex
   2 #
   3 # rbd_mirror_journal.sh - test rbd-mirror daemon in journal-based mirroring mode
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 setup
  13
  14 testlog "TEST: add image and test replay"
  15 start_mirrors ${CLUSTER1}
  16 image=test
  17 create_image ${CLUSTER2} ${POOL} ${image}
  18 set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
  19 set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
  20 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  21 write_image ${CLUSTER2} ${POOL} ${image} 100
  22 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  23 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
  24 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  25   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  26 fi
  27 compare_images ${POOL} ${image}
  28 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
  29 compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
  30
  31 testlog "TEST: stop mirror, add image, start mirror and test replay"
  32 stop_mirrors ${CLUSTER1}
  33 image1=test1
  34 create_image ${CLUSTER2} ${POOL} ${image1}
  35 write_image ${CLUSTER2} ${POOL} ${image1} 100
  36 start_mirrors ${CLUSTER1}
  37 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  38 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  39 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'primary_position'
  40 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  41   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  42 fi
  43 compare_images ${POOL} ${image1}
  44
  45 testlog "TEST: test the first image is replaying after restart"
  46 write_image ${CLUSTER2} ${POOL} ${image} 100
  47 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  48 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  49 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
  50 compare_images ${POOL} ${image}
  51
  52 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  53   testlog "TEST: stop/start/restart mirror via admin socket"
  54   all_admin_daemons ${CLUSTER1} rbd mirror stop
  55   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  56   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  57   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  58   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  59
  60   all_admin_daemons ${CLUSTER1} rbd mirror start
  61   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  62   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  63   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  64   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  65
  66   all_admin_daemons ${CLUSTER1} rbd mirror restart
  67   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  68   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  69   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  70   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  71
  72   all_admin_daemons ${CLUSTER1} rbd mirror stop
  73   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  74   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  75   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  76   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  77
  78   all_admin_daemons ${CLUSTER1} rbd mirror restart
  79   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  80   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  81   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  82   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  83
  84   all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
  85   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  86   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  87   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  88   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  89
  90   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
  91   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  92   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  93
  94   all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
  95   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  96   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  97
  98   admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  99   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 100   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 101
 102   all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 103   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 104   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 105
 106   all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 107   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 108   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 109   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 110   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 111
 112   all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
 113   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 114   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 115   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 116   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 117
 118   flush ${CLUSTER1}
 119   all_admin_daemons ${CLUSTER1} rbd mirror status
 120 fi
 121
 122 remove_image_retry ${CLUSTER2} ${POOL} ${image1}
 123
 124 testlog "TEST: test image rename"
 125 new_name="${image}_RENAMED"
 126 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 127 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 128 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 129 admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
 130 admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 131 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 132 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 133 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 134 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 135
 136 testlog "TEST: test trash move restore"
 137 image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image})
 138 trash_move ${CLUSTER2} ${POOL} ${image}
 139 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 140 trash_restore ${CLUSTER2} ${POOL} ${image_id}
 141 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 142
 143 testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)"
 144 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 145
 146 wait_for_image_in_omap ${CLUSTER1} ${POOL}
 147 wait_for_image_in_omap ${CLUSTER2} ${POOL}
 148
 149 create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image}
 150 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 151 write_image ${CLUSTER2} ${POOL} ${image} 100
 152 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 153 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 154
 155 testlog "TEST: failover and failback"
 156 start_mirrors ${CLUSTER2}
 157
 158 # demote and promote same cluster
 159 demote_image ${CLUSTER2} ${POOL} ${image}
 160 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 161 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 162 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 163 promote_image ${CLUSTER2} ${POOL} ${image}
 164 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 165 write_image ${CLUSTER2} ${POOL} ${image} 100
 166 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 167 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 168 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 169 compare_images ${POOL} ${image}
 170
 171 # failover (unmodified)
 172 demote_image ${CLUSTER2} ${POOL} ${image}
 173 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 174 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 175 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 176 promote_image ${CLUSTER1} ${POOL} ${image}
 177 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 178
 179 # failback (unmodified)
 180 demote_image ${CLUSTER1} ${POOL} ${image}
 181 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 182 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 183 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 184 promote_image ${CLUSTER2} ${POOL} ${image}
 185 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 186 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 187 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 188 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 189 compare_images ${POOL} ${image}
 190
 191 # failover
 192 demote_image ${CLUSTER2} ${POOL} ${image}
 193 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 194 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 195 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 196 promote_image ${CLUSTER1} ${POOL} ${image}
 197 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 198 write_image ${CLUSTER1} ${POOL} ${image} 100
 199 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 200 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 201 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'primary_position'
 202 compare_images ${POOL} ${image}
 203
 204 # failback
 205 demote_image ${CLUSTER1} ${POOL} ${image}
 206 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 207 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 208 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 209 promote_image ${CLUSTER2} ${POOL} ${image}
 210 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 211 write_image ${CLUSTER2} ${POOL} ${image} 100
 212 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 213 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 214 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 215 compare_images ${POOL} ${image}
 216
 217 testlog "TEST: failover / failback loop"
 218 for i in `seq 1 20`; do
 219   demote_image ${CLUSTER2} ${POOL} ${image}
 220   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 221   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 222   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 223   promote_image ${CLUSTER1} ${POOL} ${image}
 224   wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 225   wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 226   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 227   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
 228   demote_image ${CLUSTER1} ${POOL} ${image}
 229   wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 230   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 231   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 232   promote_image ${CLUSTER2} ${POOL} ${image}
 233   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 234   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 235   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 236   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 237 done
 238
 239 testlog "TEST: force promote"
 240 force_promote_image=test_force_promote
 241 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 242 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 243 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 244 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 245 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 246 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'primary_position'
 247 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 248 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 249 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 250 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 251 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 252 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 253 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 254 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 255 remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image}
 256 remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image}
 257
 258 testlog "TEST: cloned images"
 259 testlog " - default"
 260 parent_image=test_parent
 261 parent_snap=snap
 262 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 263 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 264 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 265 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 266
 267 clone_image=test_clone
 268 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 269 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 270
 271 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} journal
 272 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 273 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 274 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'primary_position'
 275 compare_images ${PARENT_POOL} ${parent_image}
 276
 277 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 278 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 279 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position'
 280 compare_images ${POOL} ${clone_image}
 281 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}
 282
 283 testlog " - clone v1"
 284 clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 285
 286 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 287             ${clone_image}_v1 --rbd-default-clone-format 1
 288 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1
 289 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1
 290 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1
 291 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1
 292 remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1
 293 unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 294 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 295
 296 testlog " - clone v2"
 297 parent_snap=snap_v2
 298 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 299 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 300             ${clone_image}_v2 --rbd-default-clone-format 2
 301 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2
 302 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2
 303 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2
 304
 305 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 306 test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 307 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 308 remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2
 309 wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted'
 310 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 311 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 312
 313 testlog " - clone v2 non-primary"
 314 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 315 wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 316 clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
 317             ${clone_image}_v2 --rbd-default-clone-format 2
 318 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 319 test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 320 wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 321 remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2
 322 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
 323 remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 324
 325 testlog "TEST: data pool"
 326 dp_image=test_data_pool
 327 create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
 328 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
 329 test "${data_pool}" = "${PARENT_POOL}"
 330 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
 331 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
 332 test "${data_pool}" = "${PARENT_POOL}"
 333 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
 334 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 335 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
 336 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 337 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
 338 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'primary_position'
 339 compare_images ${POOL} ${dp_image}@snap1
 340 compare_images ${POOL} ${dp_image}@snap2
 341 compare_images ${POOL} ${dp_image}
 342 remove_image_retry ${CLUSTER2} ${POOL} ${dp_image}
 343
 344 testlog "TEST: disable mirroring / delete non-primary image"
 345 image2=test2
 346 image3=test3
 347 image4=test4
 348 image5=test5
 349 for i in ${image2} ${image3} ${image4} ${image5}; do
 350   create_image ${CLUSTER2} ${POOL} ${i}
 351   write_image ${CLUSTER2} ${POOL} ${i} 100
 352   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 353   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 354   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 355     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 356     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 357   fi
 358   write_image ${CLUSTER2} ${POOL} ${i} 100
 359   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 360   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 361 done
 362
 363 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 364 for i in ${image2} ${image4}; do
 365   disable_mirror ${CLUSTER2} ${POOL} ${i}
 366 done
 367
 368 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 369 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 370 for i in ${image3} ${image5}; do
 371   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 372   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 373   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 374 done
 375
 376 for i in ${image2} ${image3} ${image4} ${image5}; do
 377   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 378 done
 379
 380 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 381 for i in ${image2} ${image4}; do
 382   enable_journaling ${CLUSTER2} ${POOL} ${i}
 383   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 384   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 385   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 386   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 387   compare_images ${POOL} ${i}
 388 done
 389
 390 testlog "TEST: remove mirroring pool"
 391 pool=pool_to_remove
 392 for cluster in ${CLUSTER1} ${CLUSTER2}; do
 393     CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16
 394     CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool}
 395     rbd --cluster ${cluster} mirror pool enable ${pool} pool
 396 done
 397 peer_add ${CLUSTER1} ${pool} ${CLUSTER2}
 398 peer_add ${CLUSTER2} ${pool} ${CLUSTER1}
 399 rdp_image=test_remove_data_pool
 400 create_image ${CLUSTER2} ${pool} ${image} 128
 401 create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
 402 write_image ${CLUSTER2} ${pool} ${image} 100
 403 write_image ${CLUSTER2} ${POOL} ${rdp_image} 100
 404 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image}
 405 wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'primary_position'
 406 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
 407 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'primary_position'
 408 for cluster in ${CLUSTER1} ${CLUSTER2}; do
 409     CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it
 410 done
 411 remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image}
 412 wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
 413 for i in 0 1 2 4 8 8 8 8 16 16; do
 414     sleep $i
 415     admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break
 416 done
 417 admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
 418
 419 testlog "TEST: snapshot rename"
 420 snap_name='snap_rename'
 421 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 422 for i in `seq 1 20`; do
 423   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 424 done
 425 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 426
 427 unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1'
 428 unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2'
 429 for i in ${image2} ${image4}; do
 430     remove_image_retry ${CLUSTER2} ${POOL} ${i}
 431 done
 432
 433 testlog "TEST: disable mirror while daemon is stopped"
 434 stop_mirrors ${CLUSTER1}
 435 stop_mirrors ${CLUSTER2}
 436 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 437 disable_mirror ${CLUSTER2} ${POOL} ${image}
 438 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 439   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 440 fi
 441 start_mirrors ${CLUSTER1}
 442 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 443 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 444 enable_journaling ${CLUSTER2} ${POOL} ${image}
 445 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 446 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 447
 448 testlog "TEST: non-default namespace image mirroring"
 449 testlog " - replay"
 450 create_image ${CLUSTER2} ${POOL}/${NS1} ${image}
 451 create_image ${CLUSTER2} ${POOL}/${NS2} ${image}
 452 enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} journal
 453 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image}
 454 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image}
 455 write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100
 456 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100
 457 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image}
 458 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image}
 459 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'primary_position'
 460 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'primary_position'
 461 compare_images ${POOL}/${NS1} ${image}
 462 compare_images ${POOL}/${NS2} ${image}
 463
 464 testlog " - disable mirroring / delete image"
 465 remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image}
 466 disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image}
 467 wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted'
 468 wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted'
 469 remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image}
 470
 471 testlog " - data pool"
 472 dp_image=test_data_pool
 473 create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL}
 474 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image})
 475 test "${data_pool}" = "${PARENT_POOL}"
 476 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image}
 477 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image})
 478 test "${data_pool}" = "${PARENT_POOL}"
 479 write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100
 480 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
 481 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position'
 482 compare_images ${POOL}/${NS1} ${dp_image}
 483 remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
 484
 485 testlog "TEST: simple image resync"
 486 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 487 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 488 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 489 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 490 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 491 compare_images ${POOL} ${image}
 492
 493 testlog "TEST: image resync while replayer is stopped"
 494 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 495   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 496   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 497   request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 498   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 499   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 500   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 501   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 502   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 503   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 504   compare_images ${POOL} ${image}
 505 fi
 506
 507 testlog "TEST: request image resync while daemon is offline"
 508 stop_mirrors ${CLUSTER1}
 509 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 510 start_mirrors ${CLUSTER1}
 511 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 512 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 513 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 514 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 515 compare_images ${POOL} ${image}
 516 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 517
 518 testlog "TEST: client disconnect"
 519 image=laggy
 520 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 521 write_image ${CLUSTER2} ${POOL} ${image} 10
 522
 523 testlog " - replay stopped after disconnect"
 524 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 525 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 526 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 527 disconnect_image ${CLUSTER2} ${POOL} ${image}
 528 test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 529 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 530 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 531
 532 testlog " - replay started after resync requested"
 533 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 534 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 535 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 536 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 537 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 538 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 539 compare_images ${POOL} ${image}
 540
 541 testlog " - disconnected after max_concurrent_object_sets reached"
 542 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 543   admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 544   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 545   test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 546   set_image_meta ${CLUSTER2} ${POOL} ${image} \
 547                  conf_rbd_journal_max_concurrent_object_sets 1
 548   write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 549   write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 550   test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 551   set_image_meta ${CLUSTER2} ${POOL} ${image} \
 552                  conf_rbd_journal_max_concurrent_object_sets 0
 553
 554   testlog " - replay is still stopped (disconnected) after restart"
 555   admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
 556   wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 557   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 558 fi
 559
 560 testlog " - replay started after resync requested"
 561 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 562 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 563 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 564 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 565 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 566 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 567 compare_images ${POOL} ${image}
 568
 569 testlog " - rbd_mirroring_resync_after_disconnect config option"
 570 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 571                conf_rbd_mirroring_resync_after_disconnect true
 572 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 573 image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
 574 disconnect_image ${CLUSTER2} ${POOL} ${image}
 575 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 576 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 577 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 578 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 579 test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 580 compare_images ${POOL} ${image}
 581 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 582                conf_rbd_mirroring_resync_after_disconnect false
 583 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 584 disconnect_image ${CLUSTER2} ${POOL} ${image}
 585 test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})"
 586 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 587 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 588 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 589
 590 testlog "TEST: split-brain"
 591 image=split-brain
 592 create_image ${CLUSTER2} ${POOL} ${image}
 593 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 594 promote_image ${CLUSTER1} ${POOL} ${image} --force
 595 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 596 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 597 write_image ${CLUSTER1} ${POOL} ${image} 10
 598 demote_image ${CLUSTER1} ${POOL} ${image}
 599 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 600 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 601 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
 602 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 603
 604 testlog "TEST: check if removed images' OMAP are removed"
 605 start_mirrors ${CLUSTER2}
 606 wait_for_image_in_omap ${CLUSTER1} ${POOL}
 607 wait_for_image_in_omap ${CLUSTER2} ${POOL}
 608
 609 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 610   # teuthology will trash the daemon
 611   testlog "TEST: no blocklists"
 612   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
 613   CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
 614 fi