]>
Commit | Line | Data |
---|---|---|
f38dd50b | 1 | #!/usr/bin/env bash |
7c673cae FG |
2 | # |
3 | # rbd_mirror_stress.sh - stress test rbd-mirror daemon | |
4 | # | |
5 | # The following additional environment variables affect the test: | |
6 | # | |
7 | # RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write | |
8 | # tool during the many image test | |
9 | # | |
10 | ||
f38dd50b TL |
11 | set -ex |
12 | ||
7c673cae FG |
13 | IMAGE_COUNT=50 |
14 | export LOCKDEP=0 | |
15 | ||
16 | . $(dirname $0)/rbd_mirror_helpers.sh | |
17 | ||
11fdf7f2 TL |
18 | setup |
19 | ||
7c673cae FG |
20 | create_snap() |
21 | { | |
22 | local cluster=$1 | |
23 | local pool=$2 | |
24 | local image=$3 | |
25 | local snap_name=$4 | |
26 | ||
27 | rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \ | |
28 | --debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log | |
29 | } | |
30 | ||
31 | compare_image_snaps() | |
32 | { | |
33 | local pool=$1 | |
34 | local image=$2 | |
35 | local snap_name=$3 | |
cd265ab1 | 36 | local ret=0 |
7c673cae FG |
37 | |
38 | local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export | |
39 | local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export | |
40 | ||
41 | rm -f ${rmt_export} ${loc_export} | |
42 | rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export} | |
43 | rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export} | |
cd265ab1 TL |
44 | if ! cmp ${rmt_export} ${loc_export} |
45 | then | |
46 | show_diff ${rmt_export} ${loc_export} | |
47 | ret=1 | |
48 | fi | |
7c673cae | 49 | rm -f ${rmt_export} ${loc_export} |
cd265ab1 | 50 | return ${ret} |
7c673cae FG |
51 | } |
52 | ||
53 | wait_for_pool_images() | |
54 | { | |
55 | local cluster=$1 | |
56 | local pool=$2 | |
57 | local image_count=$3 | |
58 | local s | |
59 | local count | |
60 | local last_count=0 | |
61 | ||
62 | while true; do | |
63 | for s in `seq 1 40`; do | |
64 | test $s -ne 1 && sleep 30 | |
65 | count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2) | |
66 | test "${count}" = "${image_count}" && return 0 | |
67 | ||
68 | # reset timeout if making forward progress | |
69 | test $count -ne $last_count && break | |
70 | done | |
71 | ||
72 | test $count -eq $last_count && break | |
73 | last_count=$count | |
74 | done | |
75 | rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2 | |
76 | return 1 | |
77 | } | |
78 | ||
79 | wait_for_pool_healthy() | |
80 | { | |
81 | local cluster=$1 | |
82 | local pool=$2 | |
83 | local s | |
84 | local state | |
85 | ||
86 | for s in `seq 1 40`; do | |
87 | test $s -ne 1 && sleep 30 | |
9f95a23c | 88 | state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'image health:' | cut -d' ' -f 3) |
7c673cae FG |
89 | test "${state}" = "ERROR" && break |
90 | test "${state}" = "OK" && return 0 | |
91 | done | |
92 | rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2 | |
93 | return 1 | |
94 | } | |
95 | ||
11fdf7f2 TL |
96 | start_mirrors ${CLUSTER1} |
97 | start_mirrors ${CLUSTER2} | |
7c673cae FG |
98 | |
99 | testlog "TEST: add image and test replay after client crashes" | |
100 | image=test | |
cd265ab1 | 101 | create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '512M' |
7c673cae FG |
102 | wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} |
103 | ||
cd265ab1 | 104 | clean_snap_name= |
7c673cae FG |
105 | for i in `seq 1 10` |
106 | do | |
107 | stress_write_image ${CLUSTER2} ${POOL} ${image} | |
108 | ||
9f95a23c | 109 | wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' |
7c673cae FG |
110 | |
111 | snap_name="snap${i}" | |
112 | create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} | |
113 | wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} | |
114 | wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} | |
115 | wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} | |
cd265ab1 TL |
116 | |
117 | if [ -n "${clean_snap_name}" ]; then | |
118 | compare_image_snaps ${POOL} ${image} ${clean_snap_name} | |
119 | fi | |
7c673cae | 120 | compare_image_snaps ${POOL} ${image} ${snap_name} |
cd265ab1 TL |
121 | |
122 | clean_snap_name="snap${i}-clean" | |
123 | create_snap ${CLUSTER2} ${POOL} ${image} ${clean_snap_name} | |
7c673cae FG |
124 | done |
125 | ||
cd265ab1 TL |
126 | wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} |
127 | wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} | |
128 | wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${clean_snap_name} | |
129 | ||
7c673cae FG |
130 | for i in `seq 1 10` |
131 | do | |
132 | snap_name="snap${i}" | |
cd265ab1 TL |
133 | compare_image_snaps ${POOL} ${image} ${snap_name} |
134 | ||
135 | snap_name="snap${i}-clean" | |
136 | compare_image_snaps ${POOL} ${image} ${snap_name} | |
137 | done | |
138 | ||
139 | for i in `seq 1 10` | |
140 | do | |
141 | snap_name="snap${i}" | |
142 | remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} | |
143 | ||
144 | snap_name="snap${i}-clean" | |
7c673cae FG |
145 | remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} |
146 | done | |
147 | ||
148 | remove_image_retry ${CLUSTER2} ${POOL} ${image} | |
149 | wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' | |
150 | ||
151 | testlog "TEST: create many images" | |
152 | snap_name="snap" | |
153 | for i in `seq 1 ${IMAGE_COUNT}` | |
154 | do | |
155 | image="image_${i}" | |
cd265ab1 | 156 | create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '128M' |
7c673cae FG |
157 | if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then |
158 | write_image ${CLUSTER2} ${POOL} ${image} 100 | |
159 | else | |
160 | stress_write_image ${CLUSTER2} ${POOL} ${image} | |
161 | fi | |
162 | done | |
163 | ||
164 | wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT} | |
165 | wait_for_pool_healthy ${CLUSTER2} ${POOL} | |
166 | ||
167 | wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT} | |
168 | wait_for_pool_healthy ${CLUSTER1} ${POOL} | |
169 | ||
170 | testlog "TEST: compare many images" | |
171 | for i in `seq 1 ${IMAGE_COUNT}` | |
172 | do | |
173 | image="image_${i}" | |
174 | create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} | |
175 | wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} | |
176 | wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} | |
177 | wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} | |
178 | compare_image_snaps ${POOL} ${image} ${snap_name} | |
179 | done | |
180 | ||
181 | testlog "TEST: delete many images" | |
182 | for i in `seq 1 ${IMAGE_COUNT}` | |
183 | do | |
184 | image="image_${i}" | |
185 | remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} | |
186 | remove_image_retry ${CLUSTER2} ${POOL} ${image} | |
187 | done | |
188 | ||
189 | testlog "TEST: image deletions should propagate" | |
190 | wait_for_pool_images ${CLUSTER1} ${POOL} 0 | |
191 | wait_for_pool_healthy ${CLUSTER1} ${POOL} 0 | |
192 | for i in `seq 1 ${IMAGE_COUNT}` | |
193 | do | |
194 | image="image_${i}" | |
195 | wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' | |
196 | done | |
197 | ||
198 | testlog "TEST: delete images during bootstrap" | |
199 | set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image' | |
200 | set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' | |
201 | ||
202 | start_mirror ${CLUSTER1} | |
203 | image=test | |
204 | ||
205 | for i in `seq 1 10` | |
206 | do | |
207 | image="image_${i}" | |
208 | create_image ${CLUSTER2} ${POOL} ${image} '512M' | |
209 | enable_mirror ${CLUSTER2} ${POOL} ${image} | |
210 | ||
211 | stress_write_image ${CLUSTER2} ${POOL} ${image} | |
212 | wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' | |
213 | ||
214 | disable_mirror ${CLUSTER2} ${POOL} ${image} | |
215 | wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' | |
216 | purge_snapshots ${CLUSTER2} ${POOL} ${image} | |
217 | remove_image_retry ${CLUSTER2} ${POOL} ${image} | |
218 | done | |
a4b75251 TL |
219 | |
220 | testlog "TEST: check if removed images' OMAP are removed" | |
221 | ||
222 | wait_for_image_in_omap ${CLUSTER1} ${POOL} | |
223 | wait_for_image_in_omap ${CLUSTER2} ${POOL} |