3 # rbd_mirror_journal.sh - test rbd-mirror daemon in journal-based mirroring mode
5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
6 # creates a temporary directory, used for cluster configs, daemon logs, admin
7 # socket, temporary files, and launches rbd-mirror daemon.
10 . $
(dirname $0)/rbd_mirror_helpers.sh
14 testlog
"TEST: add image and test replay"
15 start_mirrors
${CLUSTER1}
17 create_image
${CLUSTER2} ${POOL} ${image}
18 set_image_meta
${CLUSTER2} ${POOL} ${image} "key1" "value1"
19 set_image_meta
${CLUSTER2} ${POOL} ${image} "key2" "value2"
20 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
21 write_image
${CLUSTER2} ${POOL} ${image} 100
22 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
23 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
24 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
25 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'down+unknown'
27 compare_images
${POOL} ${image}
28 compare_image_meta
${CLUSTER1} ${POOL} ${image} "key1" "value1"
29 compare_image_meta
${CLUSTER1} ${POOL} ${image} "key2" "value2"
31 testlog
"TEST: stop mirror, add image, start mirror and test replay"
32 stop_mirrors
${CLUSTER1}
34 create_image
${CLUSTER2} ${POOL} ${image1}
35 write_image
${CLUSTER2} ${POOL} ${image1} 100
36 start_mirrors
${CLUSTER1}
37 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
38 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
39 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'primary_position'
40 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
41 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image1} 'down+unknown'
43 compare_images
${POOL} ${image1}
45 testlog
"TEST: test the first image is replaying after restart"
46 write_image
${CLUSTER2} ${POOL} ${image} 100
47 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
48 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
49 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
50 compare_images
${POOL} ${image}
52 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
53 testlog
"TEST: stop/start/restart mirror via admin socket"
54 all_admin_daemons
${CLUSTER1} rbd mirror stop
55 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
56 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image1}
57 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+stopped'
58 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+stopped'
60 all_admin_daemons
${CLUSTER1} rbd mirror start
61 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
62 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
63 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
64 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+replaying'
66 all_admin_daemons
${CLUSTER1} rbd mirror restart
67 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
68 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
69 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
70 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+replaying'
72 all_admin_daemons
${CLUSTER1} rbd mirror stop
73 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
74 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image1}
75 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+stopped'
76 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+stopped'
78 all_admin_daemons
${CLUSTER1} rbd mirror restart
79 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
80 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
81 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
82 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+replaying'
84 all_admin_daemons
${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
85 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
86 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image1}
87 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+stopped'
88 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+stopped'
90 admin_daemons
${CLUSTER1} rbd mirror start ${POOL}/${image}
91 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
92 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
94 all_admin_daemons
${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
95 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
96 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+replaying'
98 admin_daemons
${CLUSTER1} rbd mirror restart ${POOL}/${image}
99 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
100 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
102 all_admin_daemons
${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
103 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
104 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
106 all_admin_daemons
${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
107 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
108 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image1}
109 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+stopped'
110 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+stopped'
112 all_admin_daemons
${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
113 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
114 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image1}
115 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
116 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image1} 'up+replaying'
119 all_admin_daemons
${CLUSTER1} rbd mirror status
122 remove_image_retry
${CLUSTER2} ${POOL} ${image1}
124 testlog
"TEST: test image rename"
125 new_name
="${image}_RENAMED"
126 rename_image
${CLUSTER2} ${POOL} ${image} ${new_name}
127 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${new_name}
128 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
129 admin_daemons
${CLUSTER1} rbd mirror status ${POOL}/${new_name}
130 admin_daemons
${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
131 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${new_name}
132 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
133 rename_image
${CLUSTER2} ${POOL} ${new_name} ${image}
134 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
136 testlog
"TEST: test trash move restore"
137 image_id
=$
(get_image_id
${CLUSTER2} ${POOL} ${image})
138 trash_move
${CLUSTER2} ${POOL} ${image}
139 wait_for_image_present
${CLUSTER1} ${POOL} ${image} 'deleted'
140 trash_restore
${CLUSTER2} ${POOL} ${image_id}
141 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
143 testlog
"TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)"
144 remove_image_retry
${CLUSTER2} ${POOL} ${image}
146 wait_for_image_in_omap
${CLUSTER1} ${POOL}
147 wait_for_image_in_omap
${CLUSTER2} ${POOL}
149 create_image_and_enable_mirror
${CLUSTER2} ${POOL} ${image}
150 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
151 write_image
${CLUSTER2} ${POOL} ${image} 100
152 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
153 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
155 testlog
"TEST: failover and failback"
156 start_mirrors
${CLUSTER2}
158 # demote and promote same cluster
159 demote_image
${CLUSTER2} ${POOL} ${image}
160 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
161 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
162 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
163 promote_image
${CLUSTER2} ${POOL} ${image}
164 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
165 write_image
${CLUSTER2} ${POOL} ${image} 100
166 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
167 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+stopped'
168 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
169 compare_images
${POOL} ${image}
171 # failover (unmodified)
172 demote_image
${CLUSTER2} ${POOL} ${image}
173 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
174 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
175 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
176 promote_image
${CLUSTER1} ${POOL} ${image}
177 wait_for_image_replay_started
${CLUSTER2} ${POOL} ${image}
179 # failback (unmodified)
180 demote_image
${CLUSTER1} ${POOL} ${image}
181 wait_for_image_replay_stopped
${CLUSTER2} ${POOL} ${image}
182 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
183 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
184 promote_image
${CLUSTER2} ${POOL} ${image}
185 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
186 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
187 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
188 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+stopped'
189 compare_images
${POOL} ${image}
192 demote_image
${CLUSTER2} ${POOL} ${image}
193 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
194 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
195 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
196 promote_image
${CLUSTER1} ${POOL} ${image}
197 wait_for_image_replay_started
${CLUSTER2} ${POOL} ${image}
198 write_image
${CLUSTER1} ${POOL} ${image} 100
199 wait_for_replay_complete
${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
200 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+stopped'
201 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+replaying' 'primary_position'
202 compare_images
${POOL} ${image}
205 demote_image
${CLUSTER1} ${POOL} ${image}
206 wait_for_image_replay_stopped
${CLUSTER2} ${POOL} ${image}
207 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
208 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
209 promote_image
${CLUSTER2} ${POOL} ${image}
210 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
211 write_image
${CLUSTER2} ${POOL} ${image} 100
212 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
213 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
214 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+stopped'
215 compare_images
${POOL} ${image}
217 testlog
"TEST: failover / failback loop"
218 for i
in `seq 1 20`; do
219 demote_image
${CLUSTER2} ${POOL} ${image}
220 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${image}
221 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
222 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
223 promote_image
${CLUSTER1} ${POOL} ${image}
224 wait_for_image_replay_started
${CLUSTER2} ${POOL} ${image}
225 wait_for_replay_complete
${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
226 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+stopped'
227 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+replaying'
228 demote_image
${CLUSTER1} ${POOL} ${image}
229 wait_for_image_replay_stopped
${CLUSTER2} ${POOL} ${image}
230 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+unknown'
231 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+unknown'
232 promote_image
${CLUSTER2} ${POOL} ${image}
233 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${image}
234 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
235 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${image} 'up+stopped'
236 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${image} 'up+replaying'
239 testlog
"TEST: force promote"
240 force_promote_image
=test_force_promote
241 create_image
${CLUSTER2} ${POOL} ${force_promote_image}
242 write_image
${CLUSTER2} ${POOL} ${force_promote_image} 100
243 wait_for_image_replay_stopped
${CLUSTER2} ${POOL} ${force_promote_image}
244 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${force_promote_image}
245 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
246 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'primary_position'
247 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
248 promote_image
${CLUSTER1} ${POOL} ${force_promote_image} '--force'
249 wait_for_image_replay_stopped
${CLUSTER1} ${POOL} ${force_promote_image}
250 wait_for_image_replay_stopped
${CLUSTER2} ${POOL} ${force_promote_image}
251 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
252 wait_for_status_in_pool_dir
${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
253 write_image
${CLUSTER1} ${POOL} ${force_promote_image} 100
254 write_image
${CLUSTER2} ${POOL} ${force_promote_image} 100
255 remove_image_retry
${CLUSTER1} ${POOL} ${force_promote_image}
256 remove_image_retry
${CLUSTER2} ${POOL} ${force_promote_image}
258 testlog
"TEST: cloned images"
260 parent_image
=test_parent
262 create_image
${CLUSTER2} ${PARENT_POOL} ${parent_image}
263 write_image
${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
264 create_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
265 protect_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
267 clone_image
=test_clone
268 clone_image
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
269 write_image
${CLUSTER2} ${POOL} ${clone_image} 100
271 enable_mirror
${CLUSTER2} ${PARENT_POOL} ${parent_image} journal
272 wait_for_image_replay_started
${CLUSTER1} ${PARENT_POOL} ${parent_image}
273 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
274 wait_for_status_in_pool_dir
${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'primary_position'
275 compare_images
${PARENT_POOL} ${parent_image}
277 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${clone_image}
278 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
279 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position'
280 compare_images
${POOL} ${clone_image}
281 remove_image_retry
${CLUSTER2} ${POOL} ${clone_image}
283 testlog
" - clone v1"
284 clone_image
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
286 clone_image
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
287 ${clone_image}_v1
--rbd-default-clone-format 1
288 test $
(get_clone_format
${CLUSTER2} ${POOL} ${clone_image}_v1
) = 1
289 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${clone_image}_v1
290 test $
(get_clone_format
${CLUSTER1} ${POOL} ${clone_image}_v1
) = 1
291 remove_image_retry
${CLUSTER2} ${POOL} ${clone_image}_v1
292 remove_image_retry
${CLUSTER1} ${POOL} ${clone_image}1
293 unprotect_snapshot_retry
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
294 remove_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
296 testlog
" - clone v2"
298 create_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
299 clone_image
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
300 ${clone_image}_v2
--rbd-default-clone-format 2
301 test $
(get_clone_format
${CLUSTER2} ${POOL} ${clone_image}_v2
) = 2
302 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${clone_image}_v2
303 test $
(get_clone_format
${CLUSTER1} ${POOL} ${clone_image}_v2
) = 2
305 remove_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
306 test_snap_moved_to_trash
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
307 wait_for_snap_moved_to_trash
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
308 remove_image_retry
${CLUSTER2} ${POOL} ${clone_image}_v2
309 wait_for_image_present
${CLUSTER1} ${POOL} ${clone_image}_v2
'deleted'
310 test_snap_removed_from_trash
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
311 wait_for_snap_removed_from_trash
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
313 testlog
" - clone v2 non-primary"
314 create_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
315 wait_for_snap_present
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
316 clone_image
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
317 ${clone_image}_v2
--rbd-default-clone-format 2
318 remove_snapshot
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
319 test_snap_removed_from_trash
${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
320 wait_for_snap_moved_to_trash
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
321 remove_image_retry
${CLUSTER1} ${POOL} ${clone_image}_v2
322 wait_for_snap_removed_from_trash
${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
323 remove_image_retry
${CLUSTER2} ${PARENT_POOL} ${parent_image}
325 testlog
"TEST: data pool"
326 dp_image
=test_data_pool
327 create_image
${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
328 data_pool
=$
(get_image_data_pool
${CLUSTER2} ${POOL} ${dp_image})
329 test "${data_pool}" = "${PARENT_POOL}"
330 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${dp_image}
331 data_pool
=$
(get_image_data_pool
${CLUSTER1} ${POOL} ${dp_image})
332 test "${data_pool}" = "${PARENT_POOL}"
333 create_snapshot
${CLUSTER2} ${POOL} ${dp_image} 'snap1'
334 write_image
${CLUSTER2} ${POOL} ${dp_image} 100
335 create_snapshot
${CLUSTER2} ${POOL} ${dp_image} 'snap2'
336 write_image
${CLUSTER2} ${POOL} ${dp_image} 100
337 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
338 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'primary_position'
339 compare_images
${POOL} ${dp_image}@snap1
340 compare_images
${POOL} ${dp_image}@snap2
341 compare_images
${POOL} ${dp_image}
342 remove_image_retry
${CLUSTER2} ${POOL} ${dp_image}
344 testlog
"TEST: disable mirroring / delete non-primary image"
349 for i
in ${image2} ${image3} ${image4} ${image5}; do
350 create_image
${CLUSTER2} ${POOL} ${i}
351 write_image
${CLUSTER2} ${POOL} ${i} 100
352 create_snapshot
${CLUSTER2} ${POOL} ${i} 'snap1'
353 create_snapshot
${CLUSTER2} ${POOL} ${i} 'snap2'
354 if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
355 protect_snapshot
${CLUSTER2} ${POOL} ${i} 'snap1'
356 protect_snapshot
${CLUSTER2} ${POOL} ${i} 'snap2'
358 write_image
${CLUSTER2} ${POOL} ${i} 100
359 wait_for_image_present
${CLUSTER1} ${POOL} ${i} 'present'
360 wait_for_snap_present
${CLUSTER1} ${POOL} ${i} 'snap2'
363 set_pool_mirror_mode
${CLUSTER2} ${POOL} 'image'
364 for i
in ${image2} ${image4}; do
365 disable_mirror
${CLUSTER2} ${POOL} ${i}
368 unprotect_snapshot
${CLUSTER2} ${POOL} ${image5} 'snap1'
369 unprotect_snapshot
${CLUSTER2} ${POOL} ${image5} 'snap2'
370 for i
in ${image3} ${image5}; do
371 remove_snapshot
${CLUSTER2} ${POOL} ${i} 'snap1'
372 remove_snapshot
${CLUSTER2} ${POOL} ${i} 'snap2'
373 remove_image_retry
${CLUSTER2} ${POOL} ${i}
376 for i
in ${image2} ${image3} ${image4} ${image5}; do
377 wait_for_image_present
${CLUSTER1} ${POOL} ${i} 'deleted'
380 set_pool_mirror_mode
${CLUSTER2} ${POOL} 'pool'
381 for i
in ${image2} ${image4}; do
382 enable_journaling
${CLUSTER2} ${POOL} ${i}
383 wait_for_image_present
${CLUSTER1} ${POOL} ${i} 'present'
384 wait_for_snap_present
${CLUSTER1} ${POOL} ${i} 'snap2'
385 wait_for_image_replay_started
${CLUSTER1} ${POOL} ${i}
386 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
387 compare_images
${POOL} ${i}
390 testlog
"TEST: remove mirroring pool"
392 for cluster
in ${CLUSTER1} ${CLUSTER2}; do
393 CEPH_ARGS
='' ceph
--cluster ${cluster} osd pool create
${pool} 16 16
394 CEPH_ARGS
='' rbd
--cluster ${cluster} pool init
${pool}
395 rbd
--cluster ${cluster} mirror pool
enable ${pool} pool
397 peer_add
${CLUSTER1} ${pool} ${CLUSTER2}
398 peer_add
${CLUSTER2} ${pool} ${CLUSTER1}
399 rdp_image
=test_remove_data_pool
400 create_image
${CLUSTER2} ${pool} ${image} 128
401 create_image
${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
402 write_image
${CLUSTER2} ${pool} ${image} 100
403 write_image
${CLUSTER2} ${POOL} ${rdp_image} 100
404 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${pool} ${image}
405 wait_for_status_in_pool_dir
${CLUSTER1} ${pool} ${image} 'up+replaying' 'primary_position'
406 wait_for_replay_complete
${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
407 wait_for_status_in_pool_dir
${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'primary_position'
408 for cluster
in ${CLUSTER1} ${CLUSTER2}; do
409 CEPH_ARGS
='' ceph
--cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it
411 remove_image_retry
${CLUSTER2} ${POOL} ${rdp_image}
412 wait_for_image_present
${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
413 for i
in 0 1 2 4 8 8 8 8 16 16; do
415 admin_daemons
"${CLUSTER2}" rbd mirror status ${pool}/${image} || break
417 admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
419 testlog
"TEST: snapshot rename"
420 snap_name
='snap_rename'
421 create_snapshot
${CLUSTER2} ${POOL} ${image2} "${snap_name}_0
"
422 for i in `seq 1 20`; do
423 rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
425 wait_for_snap_present
${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
427 unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1'
428 unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2'
429 for i in ${image2} ${image4}; do
430 remove_image_retry ${CLUSTER2} ${POOL} ${i}
433 testlog "TEST
: disable mirror
while daemon is stopped
"
434 stop_mirrors ${CLUSTER1}
435 stop_mirrors ${CLUSTER2}
436 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
437 disable_mirror ${CLUSTER2} ${POOL} ${image}
438 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
439 test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
441 start_mirrors ${CLUSTER1}
442 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
443 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
444 enable_journaling ${CLUSTER2} ${POOL} ${image}
445 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
446 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
448 testlog "TEST
: non-default namespace image mirroring
"
450 create_image ${CLUSTER2} ${POOL}/${NS1} ${image}
451 create_image ${CLUSTER2} ${POOL}/${NS2} ${image}
452 enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} journal
453 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image}
454 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image}
455 write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100
456 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100
457 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image}
458 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image}
459 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'primary_position'
460 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'primary_position'
461 compare_images ${POOL}/${NS1} ${image}
462 compare_images ${POOL}/${NS2} ${image}
464 testlog " - disable mirroring
/ delete image
"
465 remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image}
466 disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image}
467 wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted'
468 wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted'
469 remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image}
471 testlog " - data pool
"
472 dp_image=test_data_pool
473 create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL}
474 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image})
475 test "${data_pool}" = "${PARENT_POOL}"
476 wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image}
477 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image})
478 test "${data_pool}" = "${PARENT_POOL}"
479 write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100
480 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
481 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position'
482 compare_images ${POOL}/${NS1} ${dp_image}
483 remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image}
485 testlog "TEST
: simple image resync
"
486 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
487 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
488 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
489 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
490 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
491 compare_images ${POOL} ${image}
493 testlog "TEST
: image resync
while replayer is stopped
"
494 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
495 admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
496 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
497 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
498 admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
499 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
500 admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
501 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
502 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
503 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
504 compare_images ${POOL} ${image}
507 testlog "TEST
: request image resync
while daemon is offline
"
508 stop_mirrors ${CLUSTER1}
509 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
510 start_mirrors ${CLUSTER1}
511 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
512 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
513 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
514 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
515 compare_images ${POOL} ${image}
516 remove_image_retry ${CLUSTER2} ${POOL} ${image}
518 testlog "TEST
: client disconnect
"
520 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
521 write_image ${CLUSTER2} ${POOL} ${image} 10
523 testlog " - replay stopped after disconnect
"
524 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
525 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
526 test -n "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
527 disconnect_image ${CLUSTER2} ${POOL} ${image}
528 test -z "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
529 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
530 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
532 testlog " - replay started after resync requested
"
533 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
534 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
535 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
536 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
537 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
538 test -n "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
539 compare_images ${POOL} ${image}
541 testlog " - disconnected after max_concurrent_object_sets reached
"
542 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
543 admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
544 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
545 test -n "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
546 set_image_meta ${CLUSTER2} ${POOL} ${image} \
547 conf_rbd_journal_max_concurrent_object_sets 1
548 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
549 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
550 test -z "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
551 set_image_meta ${CLUSTER2} ${POOL} ${image} \
552 conf_rbd_journal_max_concurrent_object_sets 0
554 testlog " - replay is still stopped
(disconnected
) after restart
"
555 admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
556 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
557 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
560 testlog " - replay started after resync requested
"
561 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
562 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
563 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
564 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
565 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
566 test -n "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
567 compare_images ${POOL} ${image}
569 testlog " - rbd_mirroring_resync_after_disconnect config option
"
570 set_image_meta ${CLUSTER2} ${POOL} ${image} \
571 conf_rbd_mirroring_resync_after_disconnect true
572 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
573 image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
574 disconnect_image ${CLUSTER2} ${POOL} ${image}
575 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
576 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
577 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
578 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
579 test -n "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
580 compare_images ${POOL} ${image}
581 set_image_meta ${CLUSTER2} ${POOL} ${image} \
582 conf_rbd_mirroring_resync_after_disconnect false
583 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
584 disconnect_image ${CLUSTER2} ${POOL} ${image}
585 test -z "$
(get_mirror_journal_position
${CLUSTER2} ${POOL} ${image})"
586 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
587 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
588 remove_image_retry ${CLUSTER2} ${POOL} ${image}
590 testlog "TEST
: split-brain
"
592 create_image ${CLUSTER2} ${POOL} ${image}
593 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
594 promote_image ${CLUSTER1} ${POOL} ${image} --force
595 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
596 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
597 write_image ${CLUSTER1} ${POOL} ${image} 10
598 demote_image ${CLUSTER1} ${POOL} ${image}
599 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
600 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
601 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position'
602 remove_image_retry ${CLUSTER2} ${POOL} ${image}
604 testlog "TEST
: check
if removed images
' OMAP are removed"
605 start_mirrors ${CLUSTER2}
606 wait_for_image_in_omap ${CLUSTER1} ${POOL}
607 wait_for_image_in_omap ${CLUSTER2} ${POOL}
609 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
610 # teuthology will trash the daemon
611 testlog "TEST: no blocklists"
612 CEPH_ARGS='--id admin
' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries"
613 CEPH_ARGS='--id admin
' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries"