[ceph.git] / ceph / qa / workunits / rbd / rbd_mirror.sh

#!/bin/sh
#
# rbd_mirror.sh - test rbd-mirror daemon
#
# The scripts starts two ("local" and "remote") clusters using mstart.sh script,
# creates a temporary directory, used for cluster configs, daemon logs, admin
# socket, temporary files, and launches rbd-mirror daemon.
#

. $(dirname $0)/rbd_mirror_helpers.sh

testlog "TEST: add image and test replay"
start_mirror ${CLUSTER1}
image=test
create_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
write_image ${CLUSTER2} ${POOL} ${image} 100
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
fi
compare_images ${POOL} ${image}

testlog "TEST: stop mirror, add image, start mirror and test replay"
stop_mirror ${CLUSTER1}
image1=test1
create_image ${CLUSTER2} ${POOL} ${image1}
write_image ${CLUSTER2} ${POOL} ${image1} 100
start_mirror ${CLUSTER1}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
fi
compare_images ${POOL} ${image1}

testlog "TEST: test the first image is replaying after restart"
write_image ${CLUSTER2} ${POOL} ${image} 100
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
compare_images ${POOL} ${image}

testlog "TEST: stop/start/restart mirror via admin socket"
admin_daemon ${CLUSTER1} rbd mirror stop
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'

admin_daemon ${CLUSTER1} rbd mirror start
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror restart
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror stop
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'

admin_daemon ${CLUSTER1} rbd mirror restart
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'

admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}

admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'

admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'

admin_daemon ${CLUSTER1} rbd mirror flush
admin_daemon ${CLUSTER1} rbd mirror status

testlog "TEST: failover and failback"
start_mirror ${CLUSTER2}

# demote and promote same cluster
demote_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
promote_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
write_image ${CLUSTER2} ${POOL} ${image} 100
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
compare_images ${POOL} ${image}

# failover (unmodified)
demote_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
promote_image ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}

# failback (unmodified)
demote_image ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
promote_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
compare_images ${POOL} ${image}

# failover
demote_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
promote_image ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
write_image ${CLUSTER1} ${POOL} ${image} 100
wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
compare_images ${POOL} ${image}

# failback
demote_image ${CLUSTER1} ${POOL} ${image}
wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
promote_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
write_image ${CLUSTER2} ${POOL} ${image} 100
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
compare_images ${POOL} ${image}

# force promote
force_promote_image=test_force_promote
create_image ${CLUSTER2} ${POOL} ${force_promote_image}
write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100

testlog "TEST: cloned images"
parent_image=test_parent
parent_snap=snap
create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}

clone_image=test_clone
clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
write_image ${CLUSTER2} ${POOL} ${clone_image} 100

enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
compare_images ${PARENT_POOL} ${parent_image}

wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
compare_images ${POOL} ${clone_image}

expect_failure "is non-primary" clone_image ${CLUSTER1} ${PARENT_POOL} \
    ${parent_image} ${parent_snap} ${POOL} ${clone_image}1

testlog "TEST: disable mirroring / delete non-primary image"
image2=test2
image3=test3
image4=test4
image5=test5
for i in ${image2} ${image3} ${image4} ${image5}; do
  create_image ${CLUSTER2} ${POOL} ${i}
  write_image ${CLUSTER2} ${POOL} ${i} 100
  create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
  create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
  if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
    protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
    protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
  fi
  write_image ${CLUSTER2} ${POOL} ${i} 100
  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
  wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
done

set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
for i in ${image2} ${image4}; do
  disable_mirror ${CLUSTER2} ${POOL} ${i}
done

unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
for i in ${image3} ${image5}; do
  remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
  remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
  remove_image_retry ${CLUSTER2} ${POOL} ${i}
done

for i in ${image2} ${image3} ${image4} ${image5}; do
  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
done

set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
for i in ${image2} ${image4}; do
  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
  wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
  compare_images ${POOL} ${i}
done

testlog "TEST: snapshot rename"
snap_name='snap_rename'
create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
for i in `seq 1 20`; do
  rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
done
wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"

testlog "TEST: disable mirror while daemon is stopped"
stop_mirror ${CLUSTER1}
stop_mirror ${CLUSTER2}
set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
disable_mirror ${CLUSTER2} ${POOL} ${image}
if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
fi
start_mirror ${CLUSTER1}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}

testlog "TEST: simple image resync"
request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
compare_images ${POOL} ${image}

testlog "TEST: image resync while replayer is stopped"
admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
compare_images ${POOL} ${image}

testlog "TEST: request image resync while daemon is offline"
stop_mirror ${CLUSTER1}
request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
start_mirror ${CLUSTER1}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
compare_images ${POOL} ${image}

testlog "TEST: client disconnect"
image=laggy
create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
write_image ${CLUSTER2} ${POOL} ${image} 10

testlog " - replay stopped after disconnect"
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
disconnect_image ${CLUSTER2} ${POOL} ${image}
test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'

testlog " - replay started after resync requested"
request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
compare_images ${POOL} ${image}

testlog " - disconnected after max_concurrent_object_sets reached"
admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
set_image_meta ${CLUSTER2} ${POOL} ${image} \
	       conf_rbd_journal_max_concurrent_object_sets 1
write_image ${CLUSTER2} ${POOL} ${image} 20 16384
write_image ${CLUSTER2} ${POOL} ${image} 20 16384
test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
set_image_meta ${CLUSTER2} ${POOL} ${image} \
	       conf_rbd_journal_max_concurrent_object_sets 0

testlog " - replay is still stopped (disconnected) after restart"
admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'

testlog " - replay started after resync requested"
request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
compare_images ${POOL} ${image}

testlog " - rbd_mirroring_resync_after_disconnect config option"
set_image_meta ${CLUSTER2} ${POOL} ${image} \
	       conf_rbd_mirroring_resync_after_disconnect true
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
disconnect_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
compare_images ${POOL} ${image}
set_image_meta ${CLUSTER2} ${POOL} ${image} \
	       conf_rbd_mirroring_resync_after_disconnect false
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
disconnect_image ${CLUSTER2} ${POOL} ${image}
test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'

testlog "TEST: split-brain"
image=split-brain
create_image ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
demote_image ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
promote_image ${CLUSTER1} ${POOL} ${image}
write_image ${CLUSTER1} ${POOL} ${image} 10
demote_image ${CLUSTER1} ${POOL} ${image}
promote_image ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'

testlog "TEST: no blacklists"
CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"

echo OK
Commit	Line	Data
7c673cae FG	1	#!/bin/sh
	2	#
	3	# rbd_mirror.sh - test rbd-mirror daemon
	4	#
	5	# The scripts starts two ("local" and "remote") clusters using mstart.sh script,
	6	# creates a temporary directory, used for cluster configs, daemon logs, admin
	7	# socket, temporary files, and launches rbd-mirror daemon.
	8	#
	9
	10	. $(dirname $0)/rbd_mirror_helpers.sh
	11
	12	testlog "TEST: add image and test replay"
	13	start_mirror ${CLUSTER1}
	14	image=test
	15	create_image ${CLUSTER2} ${POOL} ${image}
	16	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	17	write_image ${CLUSTER2} ${POOL} ${image} 100
	18	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
	19	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
	20	if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
	21	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
	22	fi
	23	compare_images ${POOL} ${image}
	24
	25	testlog "TEST: stop mirror, add image, start mirror and test replay"
	26	stop_mirror ${CLUSTER1}
	27	image1=test1
	28	create_image ${CLUSTER2} ${POOL} ${image1}
	29	write_image ${CLUSTER2} ${POOL} ${image1} 100
	30	start_mirror ${CLUSTER1}
	31	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
	32	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
	33	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
	34	if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
	35	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
	36	fi
	37	compare_images ${POOL} ${image1}
	38
	39	testlog "TEST: test the first image is replaying after restart"
	40	write_image ${CLUSTER2} ${POOL} ${image} 100
	41	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	42	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
	43	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
	44	compare_images ${POOL} ${image}
	45
	46	testlog "TEST: stop/start/restart mirror via admin socket"
	47	admin_daemon ${CLUSTER1} rbd mirror stop
	48	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
	49	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
	50	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
	51	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
	52
	53	admin_daemon ${CLUSTER1} rbd mirror start
	54	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	55	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
	56	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
	57	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
	58
	59	admin_daemon ${CLUSTER1} rbd mirror restart
	60	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	61	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
	62	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
	63	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
	64
65	admin_daemon ${CLUSTER1} rbd mirror stop
66	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
67	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
68	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
69	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
70
71	admin_daemon ${CLUSTER1} rbd mirror restart
72	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
73	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
74	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
75	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
76
77	admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
78	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
79	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
80	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
81	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
82
83	admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
84	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
85	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
86
87	admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
88	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
89	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
90
91	admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
92	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
93	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
94
95	admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
96	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
97	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
98
99	admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
100	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
101	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
102	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
103	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
104
105	admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
106	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
107	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
108	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
109	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
110
111	admin_daemon ${CLUSTER1} rbd mirror flush
112	admin_daemon ${CLUSTER1} rbd mirror status
113
114	testlog "TEST: failover and failback"
115	start_mirror ${CLUSTER2}
116
117	# demote and promote same cluster
118	demote_image ${CLUSTER2} ${POOL} ${image}
119	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
c07f9fc5 FG	120	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
c07f9fc5 FG	121	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
7c673cae FG	122	promote_image ${CLUSTER2} ${POOL} ${image}
	123	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	124	write_image ${CLUSTER2} ${POOL} ${image} 100
	125	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
	126	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
	127	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
	128	compare_images ${POOL} ${image}
	129
	130	# failover (unmodified)
	131	demote_image ${CLUSTER2} ${POOL} ${image}
	132	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
c07f9fc5 FG	133	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
c07f9fc5 FG	134	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
7c673cae FG	135	promote_image ${CLUSTER1} ${POOL} ${image}
	136	wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
	137
	138	# failback (unmodified)
	139	demote_image ${CLUSTER1} ${POOL} ${image}
	140	wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
c07f9fc5 FG	141	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
c07f9fc5 FG	142	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
7c673cae FG	143	promote_image ${CLUSTER2} ${POOL} ${image}
	144	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	145	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
	146	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
	147	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
	148	compare_images ${POOL} ${image}
	149
	150	# failover
	151	demote_image ${CLUSTER2} ${POOL} ${image}
	152	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
c07f9fc5 FG	153	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
c07f9fc5 FG	154	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
7c673cae FG	155	promote_image ${CLUSTER1} ${POOL} ${image}
	156	wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
	157	write_image ${CLUSTER1} ${POOL} ${image} 100
	158	wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
	159	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
	160	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
	161	compare_images ${POOL} ${image}
	162
	163	# failback
	164	demote_image ${CLUSTER1} ${POOL} ${image}
	165	wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
c07f9fc5 FG	166	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
c07f9fc5 FG	167	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
7c673cae FG	168	promote_image ${CLUSTER2} ${POOL} ${image}
	169	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
	170	write_image ${CLUSTER2} ${POOL} ${image} 100
	171	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
	172	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
	173	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
	174	compare_images ${POOL} ${image}
	175
	176	# force promote
	177	force_promote_image=test_force_promote
	178	create_image ${CLUSTER2} ${POOL} ${force_promote_image}
	179	write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
	180	wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
	181	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
	182	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
	183	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
	184	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
	185	promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
	186	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
	187	wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
	188	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
	189	wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
	190	write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
	191	write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
	192
	193	testlog "TEST: cloned images"
	194	parent_image=test_parent
	195	parent_snap=snap
	196	create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
	197	write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
	198	create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
	199	protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
	200
	201	clone_image=test_clone
	202	clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
	203	write_image ${CLUSTER2} ${POOL} ${clone_image} 100
	204
	205	enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
	206	wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
	207	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
	208	wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
	209	compare_images ${PARENT_POOL} ${parent_image}
	210
	211	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
	212	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
	213	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
	214	compare_images ${POOL} ${clone_image}
	215
	216	expect_failure "is non-primary" clone_image ${CLUSTER1} ${PARENT_POOL} \
	217	${parent_image} ${parent_snap} ${POOL} ${clone_image}1
	218
	219	testlog "TEST: disable mirroring / delete non-primary image"
	220	image2=test2
	221	image3=test3
	222	image4=test4
	223	image5=test5
	224	for i in ${image2} ${image3} ${image4} ${image5}; do
	225	create_image ${CLUSTER2} ${POOL} ${i}
	226	write_image ${CLUSTER2} ${POOL} ${i} 100
	227	create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
	228	create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
	229	if [ "${i}" = "${image4}" ] \|\| [ "${i}" = "${image5}" ]; then
	230	protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
	231	protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
232	fi
233	write_image ${CLUSTER2} ${POOL} ${i} 100
234	wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
235	wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
236	done
237
238	set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
239	for i in ${image2} ${image4}; do
240	disable_mirror ${CLUSTER2} ${POOL} ${i}
241	done
242
243	unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
244	unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
245	for i in ${image3} ${image5}; do
246	remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
247	remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
248	remove_image_retry ${CLUSTER2} ${POOL} ${i}
249	done
250
251	for i in ${image2} ${image3} ${image4} ${image5}; do
252	wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
253	done
254
255	set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
256	for i in ${image2} ${image4}; do
257	wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
258	wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
259	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
260	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
261	compare_images ${POOL} ${i}
262	done
263
264	testlog "TEST: snapshot rename"
265	snap_name='snap_rename'
266	create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
267	for i in `seq 1 20`; do
268	rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
269	done
270	wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
271
272	testlog "TEST: disable mirror while daemon is stopped"
273	stop_mirror ${CLUSTER1}
274	stop_mirror ${CLUSTER2}
275	set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
276	disable_mirror ${CLUSTER2} ${POOL} ${image}
277	if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
278	test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
279	fi
280	start_mirror ${CLUSTER1}
281	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
282	set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
283	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
284	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
285
286	testlog "TEST: simple image resync"
287	request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
288	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
289	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
290	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
291	compare_images ${POOL} ${image}
292
293	testlog "TEST: image resync while replayer is stopped"
294	admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
295	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
296	request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
297	admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
298	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
299	admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
300	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
301	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
302	compare_images ${POOL} ${image}
303
304	testlog "TEST: request image resync while daemon is offline"
305	stop_mirror ${CLUSTER1}
306	request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
307	start_mirror ${CLUSTER1}
308	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
309	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
310	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
311	compare_images ${POOL} ${image}
312
313	testlog "TEST: client disconnect"
314	image=laggy
315	create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
316	write_image ${CLUSTER2} ${POOL} ${image} 10
317
318	testlog " - replay stopped after disconnect"
319	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
320	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
321	test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
322	disconnect_image ${CLUSTER2} ${POOL} ${image}
323	test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
324	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
325	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
326
327	testlog " - replay started after resync requested"
328	request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
329	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
330	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
331	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
332	test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
333	compare_images ${POOL} ${image}
334
335	testlog " - disconnected after max_concurrent_object_sets reached"
336	admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
337	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
338	test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
339	set_image_meta ${CLUSTER2} ${POOL} ${image} \
340	conf_rbd_journal_max_concurrent_object_sets 1
341	write_image ${CLUSTER2} ${POOL} ${image} 20 16384
342	write_image ${CLUSTER2} ${POOL} ${image} 20 16384
343	test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
344	set_image_meta ${CLUSTER2} ${POOL} ${image} \
345	conf_rbd_journal_max_concurrent_object_sets 0
346
347	testlog " - replay is still stopped (disconnected) after restart"
348	admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
349	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
350	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
351
352	testlog " - replay started after resync requested"
353	request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
354	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
355	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
356	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
357	test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
358	compare_images ${POOL} ${image}
359
360	testlog " - rbd_mirroring_resync_after_disconnect config option"
361	set_image_meta ${CLUSTER2} ${POOL} ${image} \
362	conf_rbd_mirroring_resync_after_disconnect true
363	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
364	disconnect_image ${CLUSTER2} ${POOL} ${image}
365	wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
366	wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
367	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
368	test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
369	compare_images ${POOL} ${image}
370	set_image_meta ${CLUSTER2} ${POOL} ${image} \
371	conf_rbd_mirroring_resync_after_disconnect false
372	wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
373	disconnect_image ${CLUSTER2} ${POOL} ${image}
374	test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
375	wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
376	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
377
378	testlog "TEST: split-brain"
379	image=split-brain
380	create_image ${CLUSTER2} ${POOL} ${image}
381	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
382	demote_image ${CLUSTER2} ${POOL} ${image}
c07f9fc5	383	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
7c673cae FG	384	promote_image ${CLUSTER1} ${POOL} ${image}
	385	write_image ${CLUSTER1} ${POOL} ${image} 10
	386	demote_image ${CLUSTER1} ${POOL} ${image}
	387	promote_image ${CLUSTER2} ${POOL} ${image}
	388	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
	389	request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
	390	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
	391
31f18b77	392	testlog "TEST: no blacklists"
c07f9fc5 FG	393	CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 \| grep -q "listed 0 entries"
c07f9fc5 FG	394	CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 \| grep -q "listed 0 entries"
31f18b77	395
7c673cae	396	echo OK