]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/sh |
2 | # | |
3 | # rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions | |
4 | # | |
5 | # The scripts starts two ("local" and "remote") clusters using mstart.sh script, | |
6 | # creates a temporary directory, used for cluster configs, daemon logs, admin | |
7 | # socket, temporary files, and launches rbd-mirror daemon. | |
8 | # | |
9 | # There are several env variables useful when troubleshooting a test failure: | |
10 | # | |
11fdf7f2 TL |
11 | # RBD_MIRROR_NOCLEANUP - if not empty, don't run the cleanup (stop processes, |
12 | # destroy the clusters and remove the temp directory) | |
13 | # on exit, so it is possible to check the test state | |
14 | # after failure. | |
15 | # RBD_MIRROR_TEMDIR - use this path when creating the temporary directory | |
16 | # (should not exist) instead of running mktemp(1). | |
17 | # RBD_MIRROR_ARGS - use this to pass additional arguments to started | |
18 | # rbd-mirror daemons. | |
19 | # RBD_MIRROR_VARGS - use this to pass additional arguments to vstart.sh | |
20 | # when starting clusters. | |
21 | # RBD_MIRROR_INSTANCES - number of daemons to start per cluster | |
22 | # RBD_MIRROR_CONFIG_KEY - if not empty, use config-key for remote cluster | |
23 | # secrets | |
7c673cae FG |
24 | # The cleanup can be done as a separate step, running the script with |
25 | # `cleanup ${RBD_MIRROR_TEMDIR}' arguments. | |
26 | # | |
9f95a23c | 27 | # Note, as other workunits tests, rbd_mirror_journal.sh expects to find ceph binaries |
7c673cae FG |
28 | # in PATH. |
29 | # | |
30 | # Thus a typical troubleshooting session: | |
31 | # | |
32 | # From Ceph src dir (CEPH_SRC_PATH), start the test in NOCLEANUP mode and with | |
33 | # TEMPDIR pointing to a known location: | |
34 | # | |
35 | # cd $CEPH_SRC_PATH | |
36 | # PATH=$CEPH_SRC_PATH:$PATH | |
37 | # RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ | |
9f95a23c | 38 | # ../qa/workunits/rbd/rbd_mirror_journal.sh |
7c673cae FG |
39 | # |
40 | # After the test failure cd to TEMPDIR and check the current state: | |
41 | # | |
42 | # cd /tmp/tmp.rbd_mirror | |
43 | # ls | |
44 | # less rbd-mirror.cluster1_daemon.$pid.log | |
45 | # ceph --cluster cluster1 -s | |
46 | # ceph --cluster cluster1 -s | |
47 | # rbd --cluster cluster2 -p mirror ls | |
48 | # rbd --cluster cluster2 -p mirror journal status --image test | |
49 | # ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help | |
50 | # ... | |
51 | # | |
52 | # Also you can execute commands (functions) from the script: | |
53 | # | |
54 | # cd $CEPH_SRC_PATH | |
55 | # export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror | |
9f95a23c TL |
56 | # ../qa/workunits/rbd/rbd_mirror_journal.sh status |
57 | # ../qa/workunits/rbd/rbd_mirror_journal.sh stop_mirror cluster1 | |
58 | # ../qa/workunits/rbd/rbd_mirror_journal.sh start_mirror cluster2 | |
59 | # ../qa/workunits/rbd/rbd_mirror_journal.sh flush cluster2 | |
7c673cae FG |
60 | # ... |
61 | # | |
62 | # Eventually, run the cleanup: | |
63 | # | |
64 | # cd $CEPH_SRC_PATH | |
65 | # RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ | |
9f95a23c | 66 | # ../qa/workunits/rbd/rbd_mirror_journal.sh cleanup |
7c673cae FG |
67 | # |
68 | ||
11fdf7f2 TL |
69 | if type xmlstarlet > /dev/null 2>&1; then |
70 | XMLSTARLET=xmlstarlet | |
71 | elif type xml > /dev/null 2>&1; then | |
72 | XMLSTARLET=xml | |
73 | else | |
74 | echo "Missing xmlstarlet binary!" | |
75 | exit 1 | |
76 | fi | |
77 | ||
78 | RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2} | |
79 | ||
7c673cae FG |
80 | CLUSTER1=cluster1 |
81 | CLUSTER2=cluster2 | |
11fdf7f2 | 82 | PEER_CLUSTER_SUFFIX= |
7c673cae FG |
83 | POOL=mirror |
84 | PARENT_POOL=mirror_parent | |
9f95a23c TL |
85 | NS1=ns1 |
86 | NS2=ns2 | |
7c673cae | 87 | TEMPDIR= |
11fdf7f2 | 88 | CEPH_ID=${CEPH_ID:-mirror} |
9f95a23c | 89 | RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling} |
11fdf7f2 | 90 | MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} |
9f95a23c TL |
91 | MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool} |
92 | MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal} | |
93 | ||
11fdf7f2 TL |
94 | export CEPH_ARGS="--id ${CEPH_ID}" |
95 | ||
96 | LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1)) | |
7c673cae FG |
97 | |
98 | CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src) | |
99 | CEPH_BIN=. | |
100 | CEPH_SRC=. | |
101 | if [ -e CMakeCache.txt ]; then | |
102 | CEPH_SRC=${CEPH_ROOT} | |
103 | CEPH_ROOT=${PWD} | |
104 | CEPH_BIN=./bin | |
105 | ||
106 | # needed for ceph CLI under cmake | |
107 | export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH} | |
9f95a23c | 108 | export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3 |
7c673cae FG |
109 | fi |
110 | ||
111 | # These vars facilitate running this script in an environment with | |
112 | # ceph installed from packages, like teuthology. These are not defined | |
113 | # by default. | |
114 | # | |
115 | # RBD_MIRROR_USE_EXISTING_CLUSTER - if set, do not start and stop ceph clusters | |
116 | # RBD_MIRROR_USE_RBD_MIRROR - if set, use an existing instance of rbd-mirror | |
117 | # running as ceph client $CEPH_ID. If empty, | |
118 | # this script will start and stop rbd-mirror | |
119 | ||
120 | # | |
121 | # Functions | |
122 | # | |
123 | ||
124 | # Parse a value in format cluster[:instance] and set cluster and instance vars. | |
125 | set_cluster_instance() | |
126 | { | |
127 | local val=$1 | |
128 | local cluster_var_name=$2 | |
129 | local instance_var_name=$3 | |
130 | ||
131 | cluster=${val%:*} | |
132 | instance=${val##*:} | |
133 | ||
134 | if [ "${instance}" = "${val}" ]; then | |
9f95a23c TL |
135 | # instance was not specified, use default |
136 | instance=0 | |
7c673cae FG |
137 | fi |
138 | ||
139 | eval ${cluster_var_name}=${cluster} | |
140 | eval ${instance_var_name}=${instance} | |
141 | } | |
142 | ||
143 | daemon_asok_file() | |
144 | { | |
145 | local local_cluster=$1 | |
146 | local cluster=$2 | |
147 | local instance | |
148 | ||
149 | set_cluster_instance "${local_cluster}" local_cluster instance | |
150 | ||
11fdf7f2 | 151 | echo $(ceph-conf --cluster $local_cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'admin socket') |
7c673cae FG |
152 | } |
153 | ||
154 | daemon_pid_file() | |
155 | { | |
156 | local cluster=$1 | |
157 | local instance | |
158 | ||
159 | set_cluster_instance "${cluster}" cluster instance | |
160 | ||
11fdf7f2 | 161 | echo $(ceph-conf --cluster $cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'pid file') |
7c673cae FG |
162 | } |
163 | ||
164 | testlog() | |
165 | { | |
166 | echo $(date '+%F %T') $@ | tee -a "${TEMPDIR}/rbd-mirror.test.log" >&2 | |
167 | } | |
168 | ||
169 | expect_failure() | |
170 | { | |
171 | local expected="$1" ; shift | |
172 | local out=${TEMPDIR}/expect_failure.out | |
173 | ||
174 | if "$@" > ${out} 2>&1 ; then | |
175 | cat ${out} >&2 | |
176 | return 1 | |
177 | fi | |
178 | ||
179 | if [ -z "${expected}" ]; then | |
9f95a23c | 180 | return 0 |
7c673cae FG |
181 | fi |
182 | ||
183 | if ! grep -q "${expected}" ${out} ; then | |
184 | cat ${out} >&2 | |
185 | return 1 | |
186 | fi | |
187 | ||
188 | return 0 | |
189 | } | |
190 | ||
9f95a23c TL |
191 | mkfname() |
192 | { | |
193 | echo "$@" | sed -e 's|[/ ]|_|g' | |
194 | } | |
195 | ||
11fdf7f2 | 196 | create_users() |
7c673cae | 197 | { |
11fdf7f2 TL |
198 | local cluster=$1 |
199 | ||
200 | CEPH_ARGS='' ceph --cluster "${cluster}" \ | |
201 | auth get-or-create client.${CEPH_ID} \ | |
9f95a23c | 202 | mon 'profile rbd' osd 'profile rbd' mgr 'profile rbd' >> \ |
11fdf7f2 TL |
203 | ${CEPH_ROOT}/run/${cluster}/keyring |
204 | for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do | |
205 | CEPH_ARGS='' ceph --cluster "${cluster}" \ | |
206 | auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \ | |
9f95a23c | 207 | mon 'profile rbd-mirror' osd 'profile rbd' mgr 'profile rbd' >> \ |
11fdf7f2 TL |
208 | ${CEPH_ROOT}/run/${cluster}/keyring |
209 | done | |
210 | } | |
211 | ||
11fdf7f2 TL |
212 | setup_cluster() |
213 | { | |
214 | local cluster=$1 | |
215 | ||
216 | CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS} | |
217 | ||
218 | cd ${CEPH_ROOT} | |
219 | rm -f ${TEMPDIR}/${cluster}.conf | |
220 | ln -s $(readlink -f run/${cluster}/ceph.conf) \ | |
221 | ${TEMPDIR}/${cluster}.conf | |
222 | ||
223 | cd ${TEMPDIR} | |
224 | create_users "${cluster}" | |
225 | ||
226 | for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do | |
227 | cat<<EOF >> ${TEMPDIR}/${cluster}.conf | |
228 | [client.${MIRROR_USER_ID_PREFIX}${instance}] | |
229 | admin socket = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.asok | |
230 | pid file = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.pid | |
231 | log file = ${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log | |
232 | EOF | |
233 | done | |
234 | } | |
235 | ||
236 | setup_pools() | |
237 | { | |
238 | local cluster=$1 | |
239 | local remote_cluster=$2 | |
240 | local mon_map_file | |
241 | local mon_addr | |
242 | local admin_key_file | |
243 | local uuid | |
244 | ||
245 | CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64 | |
246 | CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64 | |
247 | ||
248 | CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL} | |
249 | CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL} | |
7c673cae | 250 | |
9f95a23c TL |
251 | if [ -n "${RBD_MIRROR_CONFIG_KEY}" ]; then |
252 | PEER_CLUSTER_SUFFIX=-DNE | |
253 | fi | |
254 | ||
255 | CEPH_ARGS='' rbd --cluster ${cluster} mirror pool enable \ | |
256 | --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} ${MIRROR_POOL_MODE} | |
11fdf7f2 TL |
257 | rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image |
258 | ||
9f95a23c TL |
259 | rbd --cluster ${cluster} namespace create ${POOL}/${NS1} |
260 | rbd --cluster ${cluster} namespace create ${POOL}/${NS2} | |
261 | ||
262 | rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} | |
263 | rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS2} image | |
264 | ||
eafe8130 TL |
265 | if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then |
266 | if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then | |
267 | rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster} | |
268 | rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster} | |
269 | else | |
270 | mon_map_file=${TEMPDIR}/${remote_cluster}.monmap | |
9f95a23c | 271 | CEPH_ARGS='' ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file} |
eafe8130 TL |
272 | mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' | |
273 | head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g') | |
274 | ||
275 | admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key | |
276 | CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file} | |
277 | ||
9f95a23c TL |
278 | CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer add ${POOL} \ |
279 | client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} \ | |
eafe8130 TL |
280 | --remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file} |
281 | ||
9f95a23c TL |
282 | uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} \ |
283 | client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX}) | |
284 | CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr} | |
285 | CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file} | |
eafe8130 | 286 | fi |
11fdf7f2 TL |
287 | fi |
288 | } | |
289 | ||
290 | setup_tempdir() | |
291 | { | |
7c673cae | 292 | if [ -n "${RBD_MIRROR_TEMDIR}" ]; then |
9f95a23c TL |
293 | test -d "${RBD_MIRROR_TEMDIR}" || |
294 | mkdir "${RBD_MIRROR_TEMDIR}" | |
295 | TEMPDIR="${RBD_MIRROR_TEMDIR}" | |
296 | cd ${TEMPDIR} | |
7c673cae | 297 | else |
9f95a23c | 298 | TEMPDIR=`mktemp -d` |
7c673cae | 299 | fi |
11fdf7f2 TL |
300 | } |
301 | ||
302 | setup() | |
303 | { | |
304 | local c | |
305 | trap 'cleanup $?' INT TERM EXIT | |
7c673cae | 306 | |
11fdf7f2 | 307 | setup_tempdir |
7c673cae | 308 | if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then |
9f95a23c TL |
309 | setup_cluster "${CLUSTER1}" |
310 | setup_cluster "${CLUSTER2}" | |
7c673cae FG |
311 | fi |
312 | ||
11fdf7f2 TL |
313 | setup_pools "${CLUSTER1}" "${CLUSTER2}" |
314 | setup_pools "${CLUSTER2}" "${CLUSTER1}" | |
9f95a23c TL |
315 | |
316 | if [ -n "${RBD_MIRROR_MIN_COMPAT_CLIENT}" ]; then | |
317 | CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd \ | |
318 | set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT} | |
319 | CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd \ | |
320 | set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT} | |
321 | fi | |
7c673cae FG |
322 | } |
323 | ||
324 | cleanup() | |
325 | { | |
11fdf7f2 | 326 | local error_code=$1 |
7c673cae FG |
327 | |
328 | set +e | |
329 | ||
11fdf7f2 TL |
330 | if [ "${error_code}" -ne 0 ]; then |
331 | status | |
332 | fi | |
333 | ||
334 | if [ -z "${RBD_MIRROR_NOCLEANUP}" ]; then | |
335 | local cluster instance | |
7c673cae | 336 | |
c07f9fc5 FG |
337 | CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it |
338 | CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it | |
339 | CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it | |
340 | CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it | |
11fdf7f2 TL |
341 | |
342 | for cluster in "${CLUSTER1}" "${CLUSTER2}"; do | |
9f95a23c | 343 | stop_mirrors "${cluster}" |
11fdf7f2 TL |
344 | done |
345 | ||
346 | if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then | |
347 | cd ${CEPH_ROOT} | |
348 | CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER1} | |
349 | CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER2} | |
350 | fi | |
351 | test "${RBD_MIRROR_TEMDIR}" = "${TEMPDIR}" || rm -Rf ${TEMPDIR} | |
352 | fi | |
353 | ||
354 | if [ "${error_code}" -eq 0 ]; then | |
355 | echo "OK" | |
356 | else | |
357 | echo "FAIL" | |
7c673cae | 358 | fi |
11fdf7f2 TL |
359 | |
360 | exit ${error_code} | |
7c673cae FG |
361 | } |
362 | ||
363 | start_mirror() | |
364 | { | |
365 | local cluster=$1 | |
366 | local instance | |
367 | ||
368 | set_cluster_instance "${cluster}" cluster instance | |
369 | ||
370 | test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return | |
371 | ||
372 | rbd-mirror \ | |
9f95a23c | 373 | --cluster ${cluster} \ |
11fdf7f2 | 374 | --id ${MIRROR_USER_ID_PREFIX}${instance} \ |
9f95a23c TL |
375 | --rbd-mirror-delete-retry-interval=5 \ |
376 | --rbd-mirror-image-state-check-interval=5 \ | |
377 | --rbd-mirror-journal-poll-age=1 \ | |
378 | --rbd-mirror-pool-replayers-refresh-interval=5 \ | |
379 | --debug-rbd=30 --debug-journaler=30 \ | |
380 | --debug-rbd_mirror=30 \ | |
381 | --daemonize=true \ | |
382 | ${RBD_MIRROR_ARGS} | |
7c673cae FG |
383 | } |
384 | ||
11fdf7f2 TL |
385 | start_mirrors() |
386 | { | |
387 | local cluster=$1 | |
388 | ||
389 | for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do | |
390 | start_mirror "${cluster}:${instance}" | |
391 | done | |
392 | } | |
393 | ||
7c673cae FG |
394 | stop_mirror() |
395 | { | |
396 | local cluster=$1 | |
397 | local sig=$2 | |
398 | ||
399 | test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return | |
400 | ||
401 | local pid | |
402 | pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || : | |
403 | if [ -n "${pid}" ] | |
404 | then | |
9f95a23c TL |
405 | kill ${sig} ${pid} |
406 | for s in 1 2 4 8 16 32; do | |
407 | sleep $s | |
408 | ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break | |
409 | done | |
410 | ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' | |
7c673cae FG |
411 | fi |
412 | rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}") | |
413 | rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}") | |
414 | rm -f $(daemon_pid_file "${cluster}") | |
415 | } | |
416 | ||
11fdf7f2 TL |
417 | stop_mirrors() |
418 | { | |
419 | local cluster=$1 | |
420 | local sig=$2 | |
421 | ||
422 | for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do | |
423 | stop_mirror "${cluster}:${instance}" "${sig}" | |
424 | done | |
425 | } | |
426 | ||
7c673cae FG |
427 | admin_daemon() |
428 | { | |
429 | local cluster=$1 ; shift | |
430 | local instance | |
431 | ||
432 | set_cluster_instance "${cluster}" cluster instance | |
433 | ||
434 | local asok_file=$(daemon_asok_file "${cluster}:${instance}" "${cluster}") | |
435 | test -S "${asok_file}" | |
436 | ||
437 | ceph --admin-daemon ${asok_file} $@ | |
438 | } | |
439 | ||
11fdf7f2 TL |
440 | admin_daemons() |
441 | { | |
442 | local cluster_instance=$1 ; shift | |
443 | local cluster="${cluster_instance%:*}" | |
444 | local instance="${cluster_instance##*:}" | |
445 | local loop_instance | |
446 | ||
447 | for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16; do | |
9f95a23c TL |
448 | sleep ${s} |
449 | if [ "${instance}" != "${cluster_instance}" ]; then | |
450 | admin_daemon "${cluster}:${instance}" $@ && return 0 | |
451 | else | |
452 | for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do | |
453 | admin_daemon "${cluster}:${loop_instance}" $@ && return 0 | |
454 | done | |
455 | fi | |
11fdf7f2 TL |
456 | done |
457 | return 1 | |
458 | } | |
459 | ||
460 | all_admin_daemons() | |
461 | { | |
462 | local cluster=$1 ; shift | |
463 | ||
464 | for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do | |
9f95a23c | 465 | admin_daemon "${cluster}:${instance}" $@ |
11fdf7f2 TL |
466 | done |
467 | } | |
468 | ||
7c673cae FG |
469 | status() |
470 | { | |
9f95a23c | 471 | local cluster daemon image_pool image_ns image |
7c673cae FG |
472 | |
473 | for cluster in ${CLUSTER1} ${CLUSTER2} | |
474 | do | |
9f95a23c TL |
475 | echo "${cluster} status" |
476 | CEPH_ARGS='' ceph --cluster ${cluster} -s | |
477 | CEPH_ARGS='' ceph --cluster ${cluster} service dump | |
478 | CEPH_ARGS='' ceph --cluster ${cluster} service status | |
479 | echo | |
480 | ||
481 | for image_pool in ${POOL} ${PARENT_POOL} | |
482 | do | |
483 | for image_ns in "" "${NS1}" "${NS2}" | |
484 | do | |
485 | echo "${cluster} ${image_pool} ${image_ns} images" | |
486 | rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls -l | |
487 | echo | |
488 | ||
489 | echo "${cluster} ${image_pool}${image_ns} mirror pool info" | |
490 | rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool info | |
491 | echo | |
492 | ||
493 | echo "${cluster} ${image_pool}${image_ns} mirror pool status" | |
494 | CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool status --verbose | |
495 | echo | |
496 | ||
497 | for image in `rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls 2>/dev/null` | |
498 | do | |
499 | echo "image ${image} info" | |
500 | rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" info ${image} | |
501 | echo | |
502 | echo "image ${image} journal status" | |
503 | rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" journal status --image ${image} | |
504 | echo | |
505 | done | |
506 | ||
507 | echo "${cluster} ${image_pool} ${image_ns} rbd_mirroring omap vals" | |
508 | rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirroring | |
509 | echo | |
510 | done | |
511 | done | |
7c673cae FG |
512 | done |
513 | ||
514 | local ret | |
515 | ||
516 | for cluster in "${CLUSTER1}" "${CLUSTER2}" | |
517 | do | |
9f95a23c TL |
518 | for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do |
519 | local pid_file=$(daemon_pid_file ${cluster}:${instance}) | |
520 | if [ ! -e ${pid_file} ] | |
521 | then | |
522 | echo "${cluster} rbd-mirror not running or unknown" \ | |
523 | "(${pid_file} not exist)" | |
524 | continue | |
525 | fi | |
526 | ||
527 | local pid | |
528 | pid=$(cat ${pid_file} 2>/dev/null) || : | |
529 | if [ -z "${pid}" ] | |
530 | then | |
531 | echo "${cluster} rbd-mirror not running or unknown" \ | |
532 | "(can't find pid using ${pid_file})" | |
533 | ret=1 | |
534 | continue | |
535 | fi | |
536 | ||
537 | echo "${daemon} rbd-mirror process in ps output:" | |
538 | if ps auxww | | |
539 | awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' | |
540 | then | |
541 | echo | |
542 | echo "${cluster} rbd-mirror not running" \ | |
543 | "(can't find pid $pid in ps output)" | |
544 | ret=1 | |
545 | continue | |
546 | fi | |
547 | echo | |
548 | ||
549 | local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster}) | |
550 | if [ ! -S "${asok_file}" ] | |
551 | then | |
552 | echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" | |
553 | ret=1 | |
554 | continue | |
555 | fi | |
556 | ||
557 | echo "${cluster} rbd-mirror status" | |
558 | ceph --admin-daemon ${asok_file} rbd mirror status | |
559 | echo | |
560 | done | |
7c673cae FG |
561 | done |
562 | ||
563 | return ${ret} | |
564 | } | |
565 | ||
566 | flush() | |
567 | { | |
568 | local cluster=$1 | |
569 | local pool=$2 | |
570 | local image=$3 | |
571 | local cmd="rbd mirror flush" | |
572 | ||
573 | if [ -n "${image}" ] | |
574 | then | |
575 | cmd="${cmd} ${pool}/${image}" | |
576 | fi | |
577 | ||
11fdf7f2 | 578 | admin_daemons "${cluster}" ${cmd} |
7c673cae FG |
579 | } |
580 | ||
581 | test_image_replay_state() | |
582 | { | |
583 | local cluster=$1 | |
584 | local pool=$2 | |
585 | local image=$3 | |
586 | local test_state=$4 | |
11fdf7f2 | 587 | local status_result |
7c673cae FG |
588 | local current_state=stopped |
589 | ||
11fdf7f2 TL |
590 | status_result=$(admin_daemons "${cluster}" rbd mirror status ${pool}/${image} | grep -i 'state') || return 1 |
591 | echo "${status_result}" | grep -i 'Replaying' && current_state=started | |
7c673cae FG |
592 | test "${test_state}" = "${current_state}" |
593 | } | |
594 | ||
595 | wait_for_image_replay_state() | |
596 | { | |
597 | local cluster=$1 | |
598 | local pool=$2 | |
599 | local image=$3 | |
600 | local state=$4 | |
601 | local s | |
602 | ||
603 | # TODO: add a way to force rbd-mirror to update replayers | |
604 | for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do | |
9f95a23c TL |
605 | sleep ${s} |
606 | test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0 | |
7c673cae FG |
607 | done |
608 | return 1 | |
609 | } | |
610 | ||
611 | wait_for_image_replay_started() | |
612 | { | |
613 | local cluster=$1 | |
614 | local pool=$2 | |
615 | local image=$3 | |
616 | ||
617 | wait_for_image_replay_state "${cluster}" "${pool}" "${image}" started | |
618 | } | |
619 | ||
620 | wait_for_image_replay_stopped() | |
621 | { | |
622 | local cluster=$1 | |
623 | local pool=$2 | |
624 | local image=$3 | |
625 | ||
626 | wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped | |
627 | } | |
628 | ||
9f95a23c | 629 | get_journal_position() |
7c673cae FG |
630 | { |
631 | local cluster=$1 | |
632 | local pool=$2 | |
633 | local image=$3 | |
634 | local id_regexp=$4 | |
635 | ||
636 | # Parse line like below, looking for the first position | |
637 | # [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]] | |
638 | ||
9f95a23c TL |
639 | local status_log=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.status) |
640 | rbd --cluster ${cluster} journal status --image ${pool}/${image} | | |
641 | tee ${status_log} >&2 | |
7c673cae | 642 | sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \ |
9f95a23c | 643 | ${status_log} |
7c673cae FG |
644 | } |
645 | ||
9f95a23c | 646 | get_master_journal_position() |
7c673cae FG |
647 | { |
648 | local cluster=$1 | |
649 | local pool=$2 | |
650 | local image=$3 | |
651 | ||
9f95a23c | 652 | get_journal_position "${cluster}" "${pool}" "${image}" '' |
7c673cae FG |
653 | } |
654 | ||
9f95a23c | 655 | get_mirror_journal_position() |
7c673cae FG |
656 | { |
657 | local cluster=$1 | |
658 | local pool=$2 | |
659 | local image=$3 | |
660 | ||
9f95a23c | 661 | get_journal_position "${cluster}" "${pool}" "${image}" '..*' |
7c673cae FG |
662 | } |
663 | ||
9f95a23c | 664 | wait_for_journal_replay_complete() |
7c673cae FG |
665 | { |
666 | local local_cluster=$1 | |
667 | local cluster=$2 | |
668 | local pool=$3 | |
669 | local image=$4 | |
670 | local s master_pos mirror_pos last_mirror_pos | |
671 | local master_tag master_entry mirror_tag mirror_entry | |
672 | ||
673 | while true; do | |
674 | for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do | |
9f95a23c TL |
675 | sleep ${s} |
676 | flush "${local_cluster}" "${pool}" "${image}" | |
677 | master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}") | |
678 | mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}") | |
679 | test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 | |
7c673cae FG |
680 | test "${mirror_pos}" != "${last_mirror_pos}" && break |
681 | done | |
682 | ||
683 | test "${mirror_pos}" = "${last_mirror_pos}" && return 1 | |
684 | last_mirror_pos="${mirror_pos}" | |
685 | ||
686 | # handle the case where the mirror is ahead of the master | |
687 | master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) | |
688 | mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) | |
689 | master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) | |
690 | mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) | |
691 | test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0 | |
692 | done | |
693 | return 1 | |
694 | } | |
695 | ||
9f95a23c TL |
696 | mirror_image_snapshot() |
697 | { | |
698 | local cluster=$1 | |
699 | local pool=$2 | |
700 | local image=$3 | |
701 | ||
702 | rbd --cluster "${cluster}" mirror image snapshot "${pool}/${image}" | |
703 | } | |
704 | ||
705 | get_newest_mirror_snapshot() | |
706 | { | |
707 | local cluster=$1 | |
708 | local pool=$2 | |
709 | local image=$3 | |
710 | local log=$4 | |
711 | ||
712 | rbd --cluster "${cluster}" snap list --all "${pool}/${image}" --format xml | \ | |
713 | xmlstarlet sel -t -c "//snapshots/snapshot[namespace/complete='true' and position()=last()]" > \ | |
714 | ${log} || true | |
715 | } | |
716 | ||
717 | wait_for_snapshot_sync_complete() | |
718 | { | |
719 | local local_cluster=$1 | |
720 | local cluster=$2 | |
721 | local pool=$3 | |
722 | local image=$4 | |
723 | ||
724 | local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status) | |
725 | local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status) | |
726 | ||
727 | mirror_image_snapshot "${cluster}" "${pool}" "${image}" | |
728 | get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}" | |
729 | local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log}) | |
730 | ||
731 | while true; do | |
732 | for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do | |
733 | sleep ${s} | |
734 | ||
735 | get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}" | |
736 | local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log}) | |
737 | ||
738 | test "${snapshot_id}" = "${primary_snapshot_id}" && return 0 | |
739 | done | |
740 | ||
741 | return 1 | |
742 | done | |
743 | return 1 | |
744 | } | |
745 | ||
746 | wait_for_replay_complete() | |
747 | { | |
748 | local local_cluster=$1 | |
749 | local cluster=$2 | |
750 | local pool=$3 | |
751 | local image=$4 | |
752 | ||
753 | if [ "${MIRROR_IMAGE_MODE}" = "journal" ]; then | |
754 | wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image} | |
755 | elif [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then | |
756 | wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image} | |
757 | else | |
758 | return 1 | |
759 | fi | |
760 | } | |
761 | ||
762 | ||
7c673cae FG |
763 | test_status_in_pool_dir() |
764 | { | |
765 | local cluster=$1 | |
766 | local pool=$2 | |
767 | local image=$3 | |
11fdf7f2 TL |
768 | local state_pattern="$4" |
769 | local description_pattern="$5" | |
770 | local service_pattern="$6" | |
7c673cae | 771 | |
9f95a23c TL |
772 | local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status) |
773 | CEPH_ARGS='' rbd --cluster ${cluster} mirror image status ${pool}/${image} | | |
774 | tee ${status_log} >&2 | |
775 | grep "^ state: .*${state_pattern}" ${status_log} || return 1 | |
776 | grep "^ description: .*${description_pattern}" ${status_log} || return 1 | |
11fdf7f2 TL |
777 | |
778 | if [ -n "${service_pattern}" ]; then | |
779 | grep "service: *${service_pattern}" ${status_log} || return 1 | |
780 | elif echo ${state_pattern} | grep '^up+'; then | |
781 | grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1 | |
782 | else | |
783 | grep "service: " ${status_log} && return 1 | |
784 | fi | |
785 | ||
eafe8130 TL |
786 | # recheck using `mirror pool status` command to stress test it. |
787 | ||
9f95a23c | 788 | local last_update="$(sed -nEe 's/^ last_update: *(.*) *$/\1/p' ${status_log})" |
eafe8130 TL |
789 | test_mirror_pool_status_verbose \ |
790 | ${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" && | |
11fdf7f2 | 791 | return 0 |
eafe8130 TL |
792 | |
793 | echo "'mirror pool status' test failed" >&2 | |
794 | exit 1 | |
795 | } | |
796 | ||
797 | test_mirror_pool_status_verbose() | |
798 | { | |
799 | local cluster=$1 | |
800 | local pool=$2 | |
801 | local image=$3 | |
802 | local state_pattern="$4" | |
803 | local prev_last_update="$5" | |
804 | ||
9f95a23c | 805 | local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}.mirror_status) |
eafe8130 TL |
806 | |
807 | rbd --cluster ${cluster} mirror pool status ${pool} --verbose --format xml \ | |
808 | > ${status_log} | |
809 | ||
810 | local last_update state | |
811 | last_update=$($XMLSTARLET sel -t -v \ | |
812 | "//images/image[name='${image}']/last_update" < ${status_log}) | |
813 | state=$($XMLSTARLET sel -t -v \ | |
814 | "//images/image[name='${image}']/state" < ${status_log}) | |
815 | ||
816 | echo "${state}" | grep "${state_pattern}" || | |
817 | test "${last_update}" '>' "${prev_last_update}" | |
7c673cae FG |
818 | } |
819 | ||
820 | wait_for_status_in_pool_dir() | |
821 | { | |
822 | local cluster=$1 | |
823 | local pool=$2 | |
824 | local image=$3 | |
11fdf7f2 TL |
825 | local state_pattern="$4" |
826 | local description_pattern="$5" | |
827 | local service_pattern="$6" | |
7c673cae FG |
828 | |
829 | for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do | |
9f95a23c TL |
830 | sleep ${s} |
831 | test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \ | |
11fdf7f2 TL |
832 | "${description_pattern}" "${service_pattern}" && |
833 | return 0 | |
7c673cae FG |
834 | done |
835 | return 1 | |
836 | } | |
837 | ||
838 | create_image() | |
839 | { | |
840 | local cluster=$1 ; shift | |
841 | local pool=$1 ; shift | |
842 | local image=$1 ; shift | |
843 | local size=128 | |
844 | ||
845 | if [ -n "$1" ]; then | |
9f95a23c TL |
846 | size=$1 |
847 | shift | |
7c673cae FG |
848 | fi |
849 | ||
9f95a23c TL |
850 | rbd --cluster ${cluster} create --size ${size} \ |
851 | --image-feature "${RBD_IMAGE_FEATURES}" $@ ${pool}/${image} | |
852 | } | |
853 | ||
854 | create_image_and_enable_mirror() | |
855 | { | |
856 | local cluster=$1 ; shift | |
857 | local pool=$1 ; shift | |
858 | local image=$1 ; shift | |
859 | local mode=${1:-${MIRROR_IMAGE_MODE}} | |
860 | if [ -n "$1" ]; then | |
861 | shift | |
862 | fi | |
863 | ||
864 | create_image ${cluster} ${pool} ${image} $@ | |
865 | enable_mirror ${cluster} ${pool} ${image} ${mode} | |
866 | } | |
867 | ||
868 | enable_journaling() | |
869 | { | |
870 | local cluster=$1 | |
871 | local pool=$2 | |
872 | local image=$3 | |
873 | ||
874 | rbd --cluster ${cluster} feature enable ${pool}/${image} journaling | |
7c673cae FG |
875 | } |
876 | ||
877 | set_image_meta() | |
878 | { | |
879 | local cluster=$1 | |
880 | local pool=$2 | |
881 | local image=$3 | |
882 | local key=$4 | |
883 | local val=$5 | |
884 | ||
9f95a23c | 885 | rbd --cluster ${cluster} image-meta set ${pool}/${image} $key $val |
7c673cae FG |
886 | } |
887 | ||
b32b8144 FG |
888 | compare_image_meta() |
889 | { | |
890 | local cluster=$1 | |
891 | local pool=$2 | |
892 | local image=$3 | |
893 | local key=$4 | |
894 | local value=$5 | |
895 | ||
9f95a23c | 896 | test `rbd --cluster ${cluster} image-meta get ${pool}/${image} ${key}` = "${value}" |
b32b8144 FG |
897 | } |
898 | ||
3efd9988 FG |
899 | rename_image() |
900 | { | |
901 | local cluster=$1 | |
902 | local pool=$2 | |
903 | local image=$3 | |
904 | local new_name=$4 | |
905 | ||
9f95a23c | 906 | rbd --cluster=${cluster} rename ${pool}/${image} ${pool}/${new_name} |
3efd9988 FG |
907 | } |
908 | ||
7c673cae FG |
909 | remove_image() |
910 | { | |
911 | local cluster=$1 | |
912 | local pool=$2 | |
913 | local image=$3 | |
914 | ||
9f95a23c TL |
915 | rbd --cluster=${cluster} snap purge ${pool}/${image} |
916 | rbd --cluster=${cluster} rm ${pool}/${image} | |
7c673cae FG |
917 | } |
918 | ||
919 | remove_image_retry() | |
920 | { | |
921 | local cluster=$1 | |
922 | local pool=$2 | |
923 | local image=$3 | |
924 | ||
11fdf7f2 | 925 | for s in 0 1 2 4 8 16 32; do |
7c673cae | 926 | sleep ${s} |
11fdf7f2 | 927 | remove_image ${cluster} ${pool} ${image} && return 0 |
7c673cae FG |
928 | done |
929 | return 1 | |
930 | } | |
931 | ||
11fdf7f2 TL |
932 | trash_move() { |
933 | local cluster=$1 | |
934 | local pool=$2 | |
935 | local image=$3 | |
936 | ||
9f95a23c | 937 | rbd --cluster=${cluster} trash move ${pool}/${image} |
11fdf7f2 TL |
938 | } |
939 | ||
940 | trash_restore() { | |
941 | local cluster=$1 | |
942 | local pool=$2 | |
943 | local image_id=$3 | |
944 | ||
9f95a23c | 945 | rbd --cluster=${cluster} trash restore ${pool}/${image_id} |
11fdf7f2 TL |
946 | } |
947 | ||
7c673cae FG |
948 | clone_image() |
949 | { | |
950 | local cluster=$1 | |
951 | local parent_pool=$2 | |
952 | local parent_image=$3 | |
953 | local parent_snap=$4 | |
954 | local clone_pool=$5 | |
955 | local clone_image=$6 | |
956 | ||
92f5a8d4 TL |
957 | shift 6 |
958 | ||
959 | rbd --cluster ${cluster} clone \ | |
960 | ${parent_pool}/${parent_image}@${parent_snap} \ | |
9f95a23c TL |
961 | ${clone_pool}/${clone_image} --image-feature "${RBD_IMAGE_FEATURES}" $@ |
962 | } | |
963 | ||
964 | clone_image_and_enable_mirror() | |
965 | { | |
966 | local cluster=$1 | |
967 | local parent_pool=$2 | |
968 | local parent_image=$3 | |
969 | local parent_snap=$4 | |
970 | local clone_pool=$5 | |
971 | local clone_image=$6 | |
972 | shift 6 | |
973 | ||
974 | local mode=${1:-${MIRROR_IMAGE_MODE}} | |
975 | if [ -n "$1" ]; then | |
976 | shift | |
977 | fi | |
978 | ||
979 | clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@ | |
980 | enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode} | |
7c673cae FG |
981 | } |
982 | ||
983 | disconnect_image() | |
984 | { | |
985 | local cluster=$1 | |
986 | local pool=$2 | |
987 | local image=$3 | |
988 | ||
9f95a23c TL |
989 | rbd --cluster ${cluster} journal client disconnect \ |
990 | --image ${pool}/${image} | |
7c673cae FG |
991 | } |
992 | ||
993 | create_snapshot() | |
994 | { | |
995 | local cluster=$1 | |
996 | local pool=$2 | |
997 | local image=$3 | |
998 | local snap=$4 | |
999 | ||
9f95a23c | 1000 | rbd --cluster ${cluster} snap create ${pool}/${image}@${snap} |
7c673cae FG |
1001 | } |
1002 | ||
1003 | remove_snapshot() | |
1004 | { | |
1005 | local cluster=$1 | |
1006 | local pool=$2 | |
1007 | local image=$3 | |
1008 | local snap=$4 | |
1009 | ||
9f95a23c | 1010 | rbd --cluster ${cluster} snap rm ${pool}/${image}@${snap} |
7c673cae FG |
1011 | } |
1012 | ||
1013 | rename_snapshot() | |
1014 | { | |
1015 | local cluster=$1 | |
1016 | local pool=$2 | |
1017 | local image=$3 | |
1018 | local snap=$4 | |
1019 | local new_snap=$5 | |
1020 | ||
9f95a23c TL |
1021 | rbd --cluster ${cluster} snap rename ${pool}/${image}@${snap} \ |
1022 | ${pool}/${image}@${new_snap} | |
7c673cae FG |
1023 | } |
1024 | ||
1025 | purge_snapshots() | |
1026 | { | |
1027 | local cluster=$1 | |
1028 | local pool=$2 | |
1029 | local image=$3 | |
1030 | ||
9f95a23c | 1031 | rbd --cluster ${cluster} snap purge ${pool}/${image} |
7c673cae FG |
1032 | } |
1033 | ||
1034 | protect_snapshot() | |
1035 | { | |
1036 | local cluster=$1 | |
1037 | local pool=$2 | |
1038 | local image=$3 | |
1039 | local snap=$4 | |
1040 | ||
9f95a23c | 1041 | rbd --cluster ${cluster} snap protect ${pool}/${image}@${snap} |
7c673cae FG |
1042 | } |
1043 | ||
1044 | unprotect_snapshot() | |
1045 | { | |
1046 | local cluster=$1 | |
1047 | local pool=$2 | |
1048 | local image=$3 | |
1049 | local snap=$4 | |
1050 | ||
9f95a23c | 1051 | rbd --cluster ${cluster} snap unprotect ${pool}/${image}@${snap} |
7c673cae FG |
1052 | } |
1053 | ||
1054 | wait_for_snap_present() | |
1055 | { | |
1056 | local cluster=$1 | |
1057 | local pool=$2 | |
1058 | local image=$3 | |
1059 | local snap_name=$4 | |
1060 | local s | |
1061 | ||
1062 | for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do | |
9f95a23c TL |
1063 | sleep ${s} |
1064 | rbd --cluster ${cluster} info ${pool}/${image}@${snap_name} || continue | |
7c673cae FG |
1065 | return 0 |
1066 | done | |
1067 | return 1 | |
1068 | } | |
1069 | ||
92f5a8d4 TL |
1070 | test_snap_moved_to_trash() |
1071 | { | |
1072 | local cluster=$1 | |
1073 | local pool=$2 | |
1074 | local image=$3 | |
1075 | local snap_name=$4 | |
1076 | ||
1077 | rbd --cluster ${cluster} snap ls ${pool}/${image} --all | | |
1078 | grep -F " trash (${snap_name})" | |
1079 | } | |
1080 | ||
1081 | wait_for_snap_moved_to_trash() | |
1082 | { | |
1083 | local s | |
1084 | ||
1085 | for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do | |
9f95a23c | 1086 | sleep ${s} |
92f5a8d4 TL |
1087 | test_snap_moved_to_trash $@ || continue |
1088 | return 0 | |
1089 | done | |
1090 | return 1 | |
1091 | } | |
1092 | ||
1093 | test_snap_removed_from_trash() | |
1094 | { | |
1095 | test_snap_moved_to_trash $@ && return 1 | |
1096 | return 0 | |
1097 | } | |
1098 | ||
1099 | wait_for_snap_removed_from_trash() | |
1100 | { | |
1101 | local s | |
1102 | ||
1103 | for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do | |
9f95a23c | 1104 | sleep ${s} |
92f5a8d4 TL |
1105 | test_snap_removed_from_trash $@ || continue |
1106 | return 0 | |
1107 | done | |
1108 | return 1 | |
1109 | } | |
1110 | ||
7c673cae FG |
1111 | write_image() |
1112 | { | |
1113 | local cluster=$1 | |
1114 | local pool=$2 | |
1115 | local image=$3 | |
1116 | local count=$4 | |
1117 | local size=$5 | |
1118 | ||
1119 | test -n "${size}" || size=4096 | |
1120 | ||
9f95a23c TL |
1121 | rbd --cluster ${cluster} bench ${pool}/${image} --io-type write \ |
1122 | --io-size ${size} --io-threads 1 --io-total $((size * count)) \ | |
1123 | --io-pattern rand | |
7c673cae FG |
1124 | } |
1125 | ||
1126 | stress_write_image() | |
1127 | { | |
1128 | local cluster=$1 | |
1129 | local pool=$2 | |
1130 | local image=$3 | |
1131 | local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}') | |
1132 | ||
1133 | timeout ${duration}s ceph_test_rbd_mirror_random_write \ | |
9f95a23c TL |
1134 | --cluster ${cluster} ${pool} ${image} \ |
1135 | --debug-rbd=20 --debug-journaler=20 \ | |
1136 | 2> ${TEMPDIR}/rbd-mirror-random-write.log || true | |
1137 | } | |
1138 | ||
1139 | show_diff() | |
1140 | { | |
1141 | local file1=$1 | |
1142 | local file2=$2 | |
1143 | ||
1144 | xxd ${file1} > ${file1}.xxd | |
1145 | xxd ${file2} > ${file2}.xxd | |
1146 | sdiff -s ${file1}.xxd ${file2}.xxd | head -n 64 | |
1147 | rm -f ${file1}.xxd ${file2}.xxd | |
7c673cae FG |
1148 | } |
1149 | ||
1150 | compare_images() | |
1151 | { | |
1152 | local pool=$1 | |
1153 | local image=$2 | |
9f95a23c | 1154 | local ret=0 |
7c673cae | 1155 | |
9f95a23c TL |
1156 | local rmt_export=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.export) |
1157 | local loc_export=${TEMPDIR}/$(mkfname ${CLUSTER1}-${pool}-${image}.export) | |
7c673cae FG |
1158 | |
1159 | rm -f ${rmt_export} ${loc_export} | |
9f95a23c TL |
1160 | rbd --cluster ${CLUSTER2} export ${pool}/${image} ${rmt_export} |
1161 | rbd --cluster ${CLUSTER1} export ${pool}/${image} ${loc_export} | |
1162 | if ! cmp ${rmt_export} ${loc_export} | |
1163 | then | |
1164 | show_diff ${rmt_export} ${loc_export} | |
1165 | ret=1 | |
1166 | fi | |
11fdf7f2 | 1167 | rm -f ${rmt_export} ${loc_export} |
9f95a23c | 1168 | return ${ret} |
11fdf7f2 TL |
1169 | } |
1170 | ||
1171 | compare_image_snapshots() | |
1172 | { | |
1173 | local pool=$1 | |
1174 | local image=$2 | |
9f95a23c | 1175 | local ret=0 |
11fdf7f2 TL |
1176 | |
1177 | local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export | |
1178 | local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export | |
1179 | ||
9f95a23c TL |
1180 | for snap_name in $(rbd --cluster ${CLUSTER1} --format xml \ |
1181 | snap list ${pool}/${image} | \ | |
11fdf7f2 TL |
1182 | $XMLSTARLET sel -t -v "//snapshot/name" | \ |
1183 | grep -E -v "^\.rbd-mirror\."); do | |
1184 | rm -f ${rmt_export} ${loc_export} | |
9f95a23c TL |
1185 | rbd --cluster ${CLUSTER2} export ${pool}/${image}@${snap_name} ${rmt_export} |
1186 | rbd --cluster ${CLUSTER1} export ${pool}/${image}@${snap_name} ${loc_export} | |
1187 | if ! cmp ${rmt_export} ${loc_export} | |
1188 | then | |
1189 | show_diff ${rmt_export} ${loc_export} | |
1190 | ret=1 | |
1191 | fi | |
11fdf7f2 | 1192 | done |
7c673cae | 1193 | rm -f ${rmt_export} ${loc_export} |
9f95a23c | 1194 | return ${ret} |
7c673cae FG |
1195 | } |
1196 | ||
1197 | demote_image() | |
1198 | { | |
1199 | local cluster=$1 | |
1200 | local pool=$2 | |
1201 | local image=$3 | |
1202 | ||
1203 | rbd --cluster=${cluster} mirror image demote ${pool}/${image} | |
1204 | } | |
1205 | ||
1206 | promote_image() | |
1207 | { | |
1208 | local cluster=$1 | |
1209 | local pool=$2 | |
1210 | local image=$3 | |
1211 | local force=$4 | |
1212 | ||
1213 | rbd --cluster=${cluster} mirror image promote ${pool}/${image} ${force} | |
1214 | } | |
1215 | ||
1216 | set_pool_mirror_mode() | |
1217 | { | |
1218 | local cluster=$1 | |
1219 | local pool=$2 | |
9f95a23c | 1220 | local mode=${3:-${MIRROR_POOL_MODE}} |
7c673cae | 1221 | |
9f95a23c | 1222 | rbd --cluster=${cluster} mirror pool enable ${pool} ${mode} |
7c673cae FG |
1223 | } |
1224 | ||
1225 | disable_mirror() | |
1226 | { | |
1227 | local cluster=$1 | |
1228 | local pool=$2 | |
1229 | local image=$3 | |
1230 | ||
1231 | rbd --cluster=${cluster} mirror image disable ${pool}/${image} | |
1232 | } | |
1233 | ||
1234 | enable_mirror() | |
1235 | { | |
1236 | local cluster=$1 | |
1237 | local pool=$2 | |
1238 | local image=$3 | |
9f95a23c | 1239 | local mode=${4:-${MIRROR_IMAGE_MODE}} |
7c673cae | 1240 | |
9f95a23c | 1241 | rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode} |
7c673cae FG |
1242 | } |
1243 | ||
1244 | test_image_present() | |
1245 | { | |
1246 | local cluster=$1 | |
1247 | local pool=$2 | |
1248 | local image=$3 | |
1249 | local test_state=$4 | |
1250 | local image_id=$5 | |
1251 | local current_state=deleted | |
1252 | local current_image_id | |
1253 | ||
1254 | current_image_id=$(get_image_id ${cluster} ${pool} ${image}) | |
1255 | test -n "${current_image_id}" && | |
1256 | test -z "${image_id}" -o "${image_id}" = "${current_image_id}" && | |
1257 | current_state=present | |
1258 | ||
1259 | test "${test_state}" = "${current_state}" | |
1260 | } | |
1261 | ||
1262 | wait_for_image_present() | |
1263 | { | |
1264 | local cluster=$1 | |
1265 | local pool=$2 | |
1266 | local image=$3 | |
1267 | local state=$4 | |
1268 | local image_id=$5 | |
1269 | local s | |
1270 | ||
1271 | test -n "${image_id}" || | |
1272 | image_id=$(get_image_id ${cluster} ${pool} ${image}) | |
1273 | ||
1274 | # TODO: add a way to force rbd-mirror to update replayers | |
1275 | for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do | |
9f95a23c TL |
1276 | sleep ${s} |
1277 | test_image_present \ | |
7c673cae FG |
1278 | "${cluster}" "${pool}" "${image}" "${state}" "${image_id}" && |
1279 | return 0 | |
1280 | done | |
1281 | return 1 | |
1282 | } | |
1283 | ||
1284 | get_image_id() | |
1285 | { | |
1286 | local cluster=$1 | |
1287 | local pool=$2 | |
1288 | local image=$3 | |
1289 | ||
9f95a23c TL |
1290 | rbd --cluster=${cluster} info ${pool}/${image} | |
1291 | sed -ne 's/^.*block_name_prefix: rbd_data\.//p' | |
7c673cae FG |
1292 | } |
1293 | ||
1294 | request_resync_image() | |
1295 | { | |
1296 | local cluster=$1 | |
1297 | local pool=$2 | |
1298 | local image=$3 | |
b32b8144 | 1299 | local image_id_var_name=$4 |
7c673cae FG |
1300 | |
1301 | eval "${image_id_var_name}='$(get_image_id ${cluster} ${pool} ${image})'" | |
1302 | eval 'test -n "$'${image_id_var_name}'"' | |
1303 | ||
9f95a23c | 1304 | rbd --cluster=${cluster} mirror image resync ${pool}/${image} |
7c673cae FG |
1305 | } |
1306 | ||
b5b8bbf5 FG |
1307 | get_image_data_pool() |
1308 | { | |
1309 | local cluster=$1 | |
1310 | local pool=$2 | |
1311 | local image=$3 | |
1312 | ||
9f95a23c | 1313 | rbd --cluster ${cluster} info ${pool}/${image} | |
b5b8bbf5 FG |
1314 | awk '$1 == "data_pool:" {print $2}' |
1315 | } | |
1316 | ||
92f5a8d4 TL |
1317 | get_clone_format() |
1318 | { | |
1319 | local cluster=$1 | |
1320 | local pool=$2 | |
1321 | local image=$3 | |
1322 | ||
1323 | rbd --cluster ${cluster} info ${pool}/${image} | | |
1324 | awk 'BEGIN { | |
1325 | format = 1 | |
1326 | } | |
1327 | $1 == "parent:" { | |
1328 | parent = $2 | |
1329 | } | |
1330 | /op_features: .*clone-child/ { | |
1331 | format = 2 | |
1332 | } | |
1333 | END { | |
1334 | if (!parent) exit 1 | |
1335 | print format | |
1336 | }' | |
1337 | } | |
1338 | ||
7c673cae FG |
1339 | # |
1340 | # Main | |
1341 | # | |
1342 | ||
1343 | if [ "$#" -gt 0 ] | |
1344 | then | |
1345 | if [ -z "${RBD_MIRROR_TEMDIR}" ] | |
1346 | then | |
1347 | echo "RBD_MIRROR_TEMDIR is not set" >&2 | |
1348 | exit 1 | |
1349 | fi | |
1350 | ||
1351 | TEMPDIR="${RBD_MIRROR_TEMDIR}" | |
1352 | cd ${TEMPDIR} | |
1353 | $@ | |
1354 | exit $? | |
1355 | fi |