ceph/qa/workunits/rbd/rbd_mirror_helpers.sh

   1 #!/bin/sh
   2 #
   3 # rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9 # There are several env variables useful when troubleshooting a test failure:
  10 #
  11 #  RBD_MIRROR_NOCLEANUP - if not empty, don't run the cleanup (stop processes,
  12 #                         destroy the clusters and remove the temp directory)
  13 #                         on exit, so it is possible to check the test state
  14 #                         after failure.
  15 #  RBD_MIRROR_TEMDIR    - use this path when creating the temporary directory
  16 #                         (should not exist) instead of running mktemp(1).
  17 #  RBD_MIRROR_ARGS      - use this to pass additional arguments to started
  18 #                         rbd-mirror daemons.
  19 #  RBD_MIRROR_VARGS     - use this to pass additional arguments to vstart.sh
  20 #                         when starting clusters.
  21 #
  22 # The cleanup can be done as a separate step, running the script with
  23 # `cleanup ${RBD_MIRROR_TEMDIR}' arguments.
  24 #
  25 # Note, as other workunits tests, rbd_mirror.sh expects to find ceph binaries
  26 # in PATH.
  27 #
  28 # Thus a typical troubleshooting session:
  29 #
  30 # From Ceph src dir (CEPH_SRC_PATH), start the test in NOCLEANUP mode and with
  31 # TEMPDIR pointing to a known location:
  32 #
  33 #   cd $CEPH_SRC_PATH
  34 #   PATH=$CEPH_SRC_PATH:$PATH
  35 #   RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
  36 #     ../qa/workunits/rbd/rbd_mirror.sh
  37 #
  38 # After the test failure cd to TEMPDIR and check the current state:
  39 #
  40 #   cd /tmp/tmp.rbd_mirror
  41 #   ls
  42 #   less rbd-mirror.cluster1_daemon.$pid.log
  43 #   ceph --cluster cluster1 -s
  44 #   ceph --cluster cluster1 -s
  45 #   rbd --cluster cluster2 -p mirror ls
  46 #   rbd --cluster cluster2 -p mirror journal status --image test
  47 #   ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help
  48 #   ...
  49 #
  50 # Also you can execute commands (functions) from the script:
  51 #
  52 #   cd $CEPH_SRC_PATH
  53 #   export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
  54 #   ../qa/workunits/rbd/rbd_mirror.sh status
  55 #   ../qa/workunits/rbd/rbd_mirror.sh stop_mirror cluster1
  56 #   ../qa/workunits/rbd/rbd_mirror.sh start_mirror cluster2
  57 #   ../qa/workunits/rbd/rbd_mirror.sh flush cluster2
  58 #   ...
  59 #
  60 # Eventually, run the cleanup:
  61 #
  62 #   cd $CEPH_SRC_PATH
  63 #   RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
  64 #     ../qa/workunits/rbd/rbd_mirror.sh cleanup
  65 #
  66
  67 CLUSTER1=cluster1
  68 CLUSTER2=cluster2
  69 POOL=mirror
  70 PARENT_POOL=mirror_parent
  71 TEMPDIR=
  72 USER_ID=mirror
  73 export CEPH_ARGS="--id ${USER_ID}"
  74
  75 CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src)
  76 CEPH_BIN=.
  77 CEPH_SRC=.
  78 if [ -e CMakeCache.txt ]; then
  79     CEPH_SRC=${CEPH_ROOT}
  80     CEPH_ROOT=${PWD}
  81     CEPH_BIN=./bin
  82
  83     # needed for ceph CLI under cmake
  84     export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH}
  85     export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind
  86     for x in ${CEPH_ROOT}/lib/cython_modules/lib* ; do
  87         export PYTHONPATH="${PYTHONPATH}:${x}"
  88     done
  89 fi
  90
  91 # These vars facilitate running this script in an environment with
  92 # ceph installed from packages, like teuthology. These are not defined
  93 # by default.
  94 #
  95 # RBD_MIRROR_USE_EXISTING_CLUSTER - if set, do not start and stop ceph clusters
  96 # RBD_MIRROR_USE_RBD_MIRROR - if set, use an existing instance of rbd-mirror
  97 #                             running as ceph client $CEPH_ID. If empty,
  98 #                             this script will start and stop rbd-mirror
  99
 100 #
 101 # Functions
 102 #
 103
 104 # Parse a value in format cluster[:instance] and set cluster and instance vars.
 105 set_cluster_instance()
 106 {
 107     local val=$1
 108     local cluster_var_name=$2
 109     local instance_var_name=$3
 110
 111     cluster=${val%:*}
 112     instance=${val##*:}
 113
 114     if [ "${instance}" =  "${val}" ]; then
 115         # instance was not specified, use default
 116         instance=0
 117     fi
 118
 119     eval ${cluster_var_name}=${cluster}
 120     eval ${instance_var_name}=${instance}
 121 }
 122
 123 daemon_asok_file()
 124 {
 125     local local_cluster=$1
 126     local cluster=$2
 127     local instance
 128
 129     set_cluster_instance "${local_cluster}" local_cluster instance
 130
 131     if [ -n "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 132         echo $(ceph-conf --cluster $local_cluster --name "client.${CEPH_ID}" 'admin socket')
 133     else
 134         echo "${TEMPDIR}/rbd-mirror.${local_cluster}_daemon.${instance}.${cluster}.asok"
 135     fi
 136 }
 137
 138 daemon_pid_file()
 139 {
 140     local cluster=$1
 141     local instance
 142
 143     set_cluster_instance "${cluster}" cluster instance
 144
 145     if [ -n "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 146         echo $(ceph-conf --cluster $cluster --name "client.${CEPH_ID}" 'pid file')
 147     else
 148         echo "${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.pid"
 149     fi
 150 }
 151
 152 testlog()
 153 {
 154     echo $(date '+%F %T') $@ | tee -a "${TEMPDIR}/rbd-mirror.test.log" >&2
 155 }
 156
 157 expect_failure()
 158 {
 159     local expected="$1" ; shift
 160     local out=${TEMPDIR}/expect_failure.out
 161
 162     if "$@" > ${out} 2>&1 ; then
 163         cat ${out} >&2
 164         return 1
 165     fi
 166
 167     if [ -z "${expected}" ]; then
 168         return 0
 169     fi
 170
 171     if ! grep -q "${expected}" ${out} ; then
 172         cat ${out} >&2
 173         return 1
 174     fi
 175
 176     return 0
 177 }
 178
 179 setup()
 180 {
 181     local c
 182     trap cleanup INT TERM EXIT
 183
 184     if [ -n "${RBD_MIRROR_TEMDIR}" ]; then
 185         test -d "${RBD_MIRROR_TEMDIR}" ||
 186         mkdir "${RBD_MIRROR_TEMDIR}"
 187         TEMPDIR="${RBD_MIRROR_TEMDIR}"
 188         cd ${TEMPDIR}
 189     else
 190         TEMPDIR=`mktemp -d`
 191     fi
 192
 193     if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
 194         cd ${CEPH_ROOT}
 195         CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${CLUSTER1} -n ${RBD_MIRROR_VARGS}
 196         CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${CLUSTER2} -n ${RBD_MIRROR_VARGS}
 197
 198         CEPH_ARGS='' ceph --conf run/${CLUSTER1}/ceph.conf \
 199             auth get-or-create client.${USER_ID} mon 'profile rbd' osd 'profile rbd' >> \
 200             run/${CLUSTER1}/keyring
 201         CEPH_ARGS='' ceph --conf run/${CLUSTER2}/ceph.conf \
 202             auth get-or-create client.${USER_ID} mon 'profile rbd' osd 'profile rbd' >> \
 203             run/${CLUSTER2}/keyring
 204
 205         rm -f ${TEMPDIR}/${CLUSTER1}.conf
 206         ln -s $(readlink -f run/${CLUSTER1}/ceph.conf) \
 207            ${TEMPDIR}/${CLUSTER1}.conf
 208         rm -f ${TEMPDIR}/${CLUSTER2}.conf
 209         ln -s $(readlink -f run/${CLUSTER2}/ceph.conf) \
 210            ${TEMPDIR}/${CLUSTER2}.conf
 211
 212         cd ${TEMPDIR}
 213     fi
 214
 215     CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool create ${POOL} 64 64
 216     CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool create ${PARENT_POOL} 64 64
 217     CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool create ${PARENT_POOL} 64 64
 218     CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool create ${POOL} 64 64
 219
 220     rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} pool
 221     rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} pool
 222     rbd --cluster ${CLUSTER1} mirror pool enable ${PARENT_POOL} image
 223     rbd --cluster ${CLUSTER2} mirror pool enable ${PARENT_POOL} image
 224
 225     rbd --cluster ${CLUSTER1} mirror pool peer add ${POOL} ${CLUSTER2}
 226     rbd --cluster ${CLUSTER2} mirror pool peer add ${POOL} ${CLUSTER1}
 227     rbd --cluster ${CLUSTER1} mirror pool peer add ${PARENT_POOL} ${CLUSTER2}
 228     rbd --cluster ${CLUSTER2} mirror pool peer add ${PARENT_POOL} ${CLUSTER1}
 229 }
 230
 231 cleanup()
 232 {
 233     test  -n "${RBD_MIRROR_NOCLEANUP}" && return
 234     local cluster instance
 235
 236     set +e
 237
 238     for cluster in "${CLUSTER1}" "${CLUSTER2}"; do
 239         for instance in `seq 0 9`; do
 240             stop_mirror "${cluster}:${instance}"
 241         done
 242     done
 243
 244     if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
 245         cd ${CEPH_ROOT}
 246         CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER1}
 247         CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER2}
 248     else
 249         CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
 250         CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
 251         CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
 252         CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
 253     fi
 254     test "${RBD_MIRROR_TEMDIR}" = "${TEMPDIR}" ||
 255     rm -Rf ${TEMPDIR}
 256 }
 257
 258 start_mirror()
 259 {
 260     local cluster=$1
 261     local instance
 262
 263     set_cluster_instance "${cluster}" cluster instance
 264
 265     test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return
 266
 267     rbd-mirror \
 268         --cluster ${cluster} \
 269         --id mirror \
 270         --pid-file=$(daemon_pid_file "${cluster}:${instance}") \
 271         --log-file=${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log \
 272         --admin-socket=${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.\$cluster.asok \
 273         --rbd-mirror-delete-retry-interval=5 \
 274         --rbd-mirror-image-state-check-interval=5 \
 275         --rbd-mirror-journal-poll-age=1 \
 276         --rbd-mirror-pool-replayers-refresh-interval=5 \
 277         --debug-rbd=30 --debug-journaler=30 \
 278         --debug-rbd_mirror=30 \
 279         --daemonize=true \
 280         ${RBD_MIRROR_ARGS}
 281 }
 282
 283 stop_mirror()
 284 {
 285     local cluster=$1
 286     local sig=$2
 287
 288     test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return
 289
 290     local pid
 291     pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || :
 292     if [ -n "${pid}" ]
 293     then
 294         kill ${sig} ${pid}
 295         for s in 1 2 4 8 16 32; do
 296             sleep $s
 297             ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break
 298         done
 299         ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}'
 300     fi
 301     rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}")
 302     rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}")
 303     rm -f $(daemon_pid_file "${cluster}")
 304 }
 305
 306 admin_daemon()
 307 {
 308     local cluster=$1 ; shift
 309     local instance
 310
 311     set_cluster_instance "${cluster}" cluster instance
 312
 313     local asok_file=$(daemon_asok_file "${cluster}:${instance}" "${cluster}")
 314     test -S "${asok_file}"
 315
 316     ceph --admin-daemon ${asok_file} $@
 317 }
 318
 319 status()
 320 {
 321     local cluster daemon image_pool image
 322
 323     for cluster in ${CLUSTER1} ${CLUSTER2}
 324     do
 325         echo "${cluster} status"
 326         ceph --cluster ${cluster} -s
 327         echo
 328
 329         for image_pool in ${POOL} ${PARENT_POOL}
 330         do
 331             echo "${cluster} ${image_pool} images"
 332             rbd --cluster ${cluster} -p ${image_pool} ls
 333             echo
 334
 335             echo "${cluster} ${image_pool} mirror pool status"
 336             rbd --cluster ${cluster} -p ${image_pool} mirror pool status --verbose
 337             echo
 338
 339             for image in `rbd --cluster ${cluster} -p ${image_pool} ls 2>/dev/null`
 340             do
 341                 echo "image ${image} info"
 342                 rbd --cluster ${cluster} -p ${image_pool} info ${image}
 343                 echo
 344                 echo "image ${image} journal status"
 345                 rbd --cluster ${cluster} -p ${image_pool} journal status --image ${image}
 346                 echo
 347             done
 348         done
 349     done
 350
 351     local ret
 352
 353     for cluster in "${CLUSTER1}" "${CLUSTER2}"
 354     do
 355         local pid_file=$(daemon_pid_file ${cluster} )
 356         if [ ! -e ${pid_file} ]
 357         then
 358             echo "${cluster} rbd-mirror not running or unknown" \
 359                  "(${pid_file} not exist)"
 360             continue
 361         fi
 362
 363         local pid
 364         pid=$(cat ${pid_file} 2>/dev/null) || :
 365         if [ -z "${pid}" ]
 366         then
 367             echo "${cluster} rbd-mirror not running or unknown" \
 368                  "(can't find pid using ${pid_file})"
 369             ret=1
 370             continue
 371         fi
 372
 373         echo "${daemon} rbd-mirror process in ps output:"
 374         if ps auxww |
 375                 awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}'
 376         then
 377             echo
 378             echo "${cluster} rbd-mirror not running" \
 379                  "(can't find pid $pid in ps output)"
 380             ret=1
 381             continue
 382         fi
 383         echo
 384
 385         local asok_file=$(daemon_asok_file ${cluster} ${cluster})
 386         if [ ! -S "${asok_file}" ]
 387         then
 388             echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)"
 389             ret=1
 390             continue
 391         fi
 392
 393         echo "${cluster} rbd-mirror status"
 394         ceph --admin-daemon ${asok_file} rbd mirror status
 395         echo
 396     done
 397
 398     return ${ret}
 399 }
 400
 401 flush()
 402 {
 403     local cluster=$1
 404     local pool=$2
 405     local image=$3
 406     local cmd="rbd mirror flush"
 407
 408     if [ -n "${image}" ]
 409     then
 410        cmd="${cmd} ${pool}/${image}"
 411     fi
 412
 413     admin_daemon "${cluster}" ${cmd}
 414 }
 415
 416 test_image_replay_state()
 417 {
 418     local cluster=$1
 419     local pool=$2
 420     local image=$3
 421     local test_state=$4
 422     local current_state=stopped
 423
 424     admin_daemon "${cluster}" help |
 425         fgrep "\"rbd mirror status ${pool}/${image}\"" &&
 426     admin_daemon "${cluster}" rbd mirror status ${pool}/${image} |
 427         grep -i 'state.*Replaying' &&
 428     current_state=started
 429
 430     test "${test_state}" = "${current_state}"
 431 }
 432
 433 wait_for_image_replay_state()
 434 {
 435     local cluster=$1
 436     local pool=$2
 437     local image=$3
 438     local state=$4
 439     local s
 440
 441     # TODO: add a way to force rbd-mirror to update replayers
 442     for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
 443         sleep ${s}
 444         test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0
 445     done
 446     return 1
 447 }
 448
 449 wait_for_image_replay_started()
 450 {
 451     local cluster=$1
 452     local pool=$2
 453     local image=$3
 454
 455     wait_for_image_replay_state "${cluster}" "${pool}" "${image}" started
 456 }
 457
 458 wait_for_image_replay_stopped()
 459 {
 460     local cluster=$1
 461     local pool=$2
 462     local image=$3
 463
 464     wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped
 465 }
 466
 467 get_position()
 468 {
 469     local cluster=$1
 470     local pool=$2
 471     local image=$3
 472     local id_regexp=$4
 473
 474     # Parse line like below, looking for the first position
 475     # [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]]
 476
 477     local status_log=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.status
 478     rbd --cluster ${cluster} -p ${pool} journal status --image ${image} |
 479         tee ${status_log} >&2
 480     sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \
 481         ${status_log}
 482 }
 483
 484 get_master_position()
 485 {
 486     local cluster=$1
 487     local pool=$2
 488     local image=$3
 489
 490     get_position "${cluster}" "${pool}" "${image}" ''
 491 }
 492
 493 get_mirror_position()
 494 {
 495     local cluster=$1
 496     local pool=$2
 497     local image=$3
 498
 499     get_position "${cluster}" "${pool}" "${image}" '..*'
 500 }
 501
 502 wait_for_replay_complete()
 503 {
 504     local local_cluster=$1
 505     local cluster=$2
 506     local pool=$3
 507     local image=$4
 508     local s master_pos mirror_pos last_mirror_pos
 509     local master_tag master_entry mirror_tag mirror_entry
 510
 511     while true; do
 512         for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
 513             sleep ${s}
 514             flush "${local_cluster}" "${pool}" "${image}"
 515             master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
 516             mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
 517             test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
 518             test "${mirror_pos}" != "${last_mirror_pos}" && break
 519         done
 520
 521         test "${mirror_pos}" = "${last_mirror_pos}" && return 1
 522         last_mirror_pos="${mirror_pos}"
 523
 524         # handle the case where the mirror is ahead of the master
 525         master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
 526         mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
 527         master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
 528         mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
 529         test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0
 530     done
 531     return 1
 532 }
 533
 534 test_status_in_pool_dir()
 535 {
 536     local cluster=$1
 537     local pool=$2
 538     local image=$3
 539     local state_pattern=$4
 540     local description_pattern=$5
 541
 542     local status_log=${TEMPDIR}/${cluster}-${image}.mirror_status
 543     rbd --cluster ${cluster} -p ${pool} mirror image status ${image} |
 544         tee ${status_log} >&2
 545     grep "state: .*${state_pattern}" ${status_log} || return 1
 546     grep "description: .*${description_pattern}" ${status_log} || return 1
 547 }
 548
 549 wait_for_status_in_pool_dir()
 550 {
 551     local cluster=$1
 552     local pool=$2
 553     local image=$3
 554     local state_pattern=$4
 555     local description_pattern=$5
 556
 557     for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
 558         sleep ${s}
 559         test_status_in_pool_dir ${cluster} ${pool} ${image} ${state_pattern} ${description_pattern} && return 0
 560     done
 561     return 1
 562 }
 563
 564 create_image()
 565 {
 566     local cluster=$1 ; shift
 567     local pool=$1 ; shift
 568     local image=$1 ; shift
 569     local size=128
 570
 571     if [ -n "$1" ]; then
 572         size=$1
 573         shift
 574     fi
 575
 576     rbd --cluster ${cluster} -p ${pool} create --size ${size} \
 577         --image-feature layering,exclusive-lock,journaling $@ ${image}
 578 }
 579
 580 set_image_meta()
 581 {
 582     local cluster=$1
 583     local pool=$2
 584     local image=$3
 585     local key=$4
 586     local val=$5
 587
 588     rbd --cluster ${cluster} -p ${pool} image-meta set ${image} $key $val
 589 }
 590
 591 remove_image()
 592 {
 593     local cluster=$1
 594     local pool=$2
 595     local image=$3
 596
 597     rbd --cluster=${cluster} -p ${pool} snap purge ${image}
 598     rbd --cluster=${cluster} -p ${pool} rm ${image}
 599 }
 600
 601 remove_image_retry()
 602 {
 603     local cluster=$1
 604     local pool=$2
 605     local image=$3
 606
 607     for s in 1 2 4 8 16 32; do
 608         remove_image ${cluster} ${pool} ${image} && return 0
 609         sleep ${s}
 610     done
 611     return 1
 612 }
 613
 614 clone_image()
 615 {
 616     local cluster=$1
 617     local parent_pool=$2
 618     local parent_image=$3
 619     local parent_snap=$4
 620     local clone_pool=$5
 621     local clone_image=$6
 622
 623     rbd --cluster ${cluster} clone ${parent_pool}/${parent_image}@${parent_snap} \
 624         ${clone_pool}/${clone_image} --image-feature layering,exclusive-lock,journaling
 625 }
 626
 627 disconnect_image()
 628 {
 629     local cluster=$1
 630     local pool=$2
 631     local image=$3
 632
 633     rbd --cluster ${cluster} -p ${pool} journal client disconnect \
 634         --image ${image}
 635 }
 636
 637 create_snapshot()
 638 {
 639     local cluster=$1
 640     local pool=$2
 641     local image=$3
 642     local snap=$4
 643
 644     rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap}
 645 }
 646
 647 remove_snapshot()
 648 {
 649     local cluster=$1
 650     local pool=$2
 651     local image=$3
 652     local snap=$4
 653
 654     rbd --cluster ${cluster} -p ${pool} snap rm ${image}@${snap}
 655 }
 656
 657 rename_snapshot()
 658 {
 659     local cluster=$1
 660     local pool=$2
 661     local image=$3
 662     local snap=$4
 663     local new_snap=$5
 664
 665     rbd --cluster ${cluster} -p ${pool} snap rename ${image}@${snap} ${image}@${new_snap}
 666 }
 667
 668 purge_snapshots()
 669 {
 670     local cluster=$1
 671     local pool=$2
 672     local image=$3
 673
 674     rbd --cluster ${cluster} -p ${pool} snap purge ${image}
 675 }
 676
 677 protect_snapshot()
 678 {
 679     local cluster=$1
 680     local pool=$2
 681     local image=$3
 682     local snap=$4
 683
 684     rbd --cluster ${cluster} -p ${pool} snap protect ${image}@${snap}
 685 }
 686
 687 unprotect_snapshot()
 688 {
 689     local cluster=$1
 690     local pool=$2
 691     local image=$3
 692     local snap=$4
 693
 694     rbd --cluster ${cluster} -p ${pool} snap unprotect ${image}@${snap}
 695 }
 696
 697 wait_for_snap_present()
 698 {
 699     local cluster=$1
 700     local pool=$2
 701     local image=$3
 702     local snap_name=$4
 703     local s
 704
 705     for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
 706         sleep ${s}
 707         rbd --cluster ${cluster} -p ${pool} info ${image}@${snap_name} || continue
 708         return 0
 709     done
 710     return 1
 711 }
 712
 713 write_image()
 714 {
 715     local cluster=$1
 716     local pool=$2
 717     local image=$3
 718     local count=$4
 719     local size=$5
 720
 721     test -n "${size}" || size=4096
 722
 723     rbd --cluster ${cluster} -p ${pool} bench ${image} --io-type write \
 724         --io-size ${size} --io-threads 1 --io-total $((size * count)) \
 725         --io-pattern rand
 726 }
 727
 728 stress_write_image()
 729 {
 730     local cluster=$1
 731     local pool=$2
 732     local image=$3
 733     local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}')
 734
 735     timeout ${duration}s ceph_test_rbd_mirror_random_write \
 736         --cluster ${cluster} ${pool} ${image} \
 737         --debug-rbd=20 --debug-journaler=20 \
 738         2> ${TEMPDIR}/rbd-mirror-random-write.log || true
 739 }
 740
 741 compare_images()
 742 {
 743     local pool=$1
 744     local image=$2
 745
 746     local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
 747     local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
 748
 749     rm -f ${rmt_export} ${loc_export}
 750     rbd --cluster ${CLUSTER2} -p ${pool} export ${image} ${rmt_export}
 751     rbd --cluster ${CLUSTER1} -p ${pool} export ${image} ${loc_export}
 752     cmp ${rmt_export} ${loc_export}
 753     rm -f ${rmt_export} ${loc_export}
 754 }
 755
 756 demote_image()
 757 {
 758     local cluster=$1
 759     local pool=$2
 760     local image=$3
 761
 762     rbd --cluster=${cluster} mirror image demote ${pool}/${image}
 763 }
 764
 765 promote_image()
 766 {
 767     local cluster=$1
 768     local pool=$2
 769     local image=$3
 770     local force=$4
 771
 772     rbd --cluster=${cluster} mirror image promote ${pool}/${image} ${force}
 773 }
 774
 775 set_pool_mirror_mode()
 776 {
 777     local cluster=$1
 778     local pool=$2
 779     local mode=$3
 780
 781     rbd --cluster=${cluster} -p ${pool} mirror pool enable ${mode}
 782 }
 783
 784 disable_mirror()
 785 {
 786     local cluster=$1
 787     local pool=$2
 788     local image=$3
 789
 790     rbd --cluster=${cluster} mirror image disable ${pool}/${image}
 791 }
 792
 793 enable_mirror()
 794 {
 795     local cluster=$1
 796     local pool=$2
 797     local image=$3
 798
 799     rbd --cluster=${cluster} mirror image enable ${pool}/${image}
 800 }
 801
 802 test_image_present()
 803 {
 804     local cluster=$1
 805     local pool=$2
 806     local image=$3
 807     local test_state=$4
 808     local image_id=$5
 809     local current_state=deleted
 810     local current_image_id
 811
 812     current_image_id=$(get_image_id ${cluster} ${pool} ${image})
 813     test -n "${current_image_id}" &&
 814     test -z "${image_id}" -o "${image_id}" = "${current_image_id}" &&
 815     current_state=present
 816
 817     test "${test_state}" = "${current_state}"
 818 }
 819
 820 wait_for_image_present()
 821 {
 822     local cluster=$1
 823     local pool=$2
 824     local image=$3
 825     local state=$4
 826     local image_id=$5
 827     local s
 828
 829     test -n "${image_id}" ||
 830     image_id=$(get_image_id ${cluster} ${pool} ${image})
 831
 832     # TODO: add a way to force rbd-mirror to update replayers
 833     for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
 834         sleep ${s}
 835         test_image_present \
 836             "${cluster}" "${pool}" "${image}" "${state}" "${image_id}" &&
 837         return 0
 838     done
 839     return 1
 840 }
 841
 842 get_image_id()
 843 {
 844     local cluster=$1
 845     local pool=$2
 846     local image=$3
 847
 848     rbd --cluster=${cluster} -p ${pool} info ${image} |
 849         sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
 850 }
 851
 852 request_resync_image()
 853 {
 854     local cluster=$1
 855     local pool=$2
 856     local image=$3
 857     local image_id_var_name=$1
 858
 859     eval "${image_id_var_name}='$(get_image_id ${cluster} ${pool} ${image})'"
 860     eval 'test -n "$'${image_id_var_name}'"'
 861
 862     rbd --cluster=${cluster} -p ${pool} mirror image resync ${image}
 863 }
 864
 865 #
 866 # Main
 867 #
 868
 869 if [ "$#" -gt 0 ]
 870 then
 871     if [ -z "${RBD_MIRROR_TEMDIR}" ]
 872     then
 873        echo "RBD_MIRROR_TEMDIR is not set" >&2
 874        exit 1
 875     fi
 876
 877     TEMPDIR="${RBD_MIRROR_TEMDIR}"
 878     cd ${TEMPDIR}
 879     $@
 880     exit $?
 881 fi
 882
 883 set -xe
 884
 885 setup