# The cleanup can be done as a separate step, running the script with
# `cleanup ${RBD_MIRROR_TEMDIR}' arguments.
#
-# Note, as other workunits tests, rbd_mirror.sh expects to find ceph binaries
+# Note, as other workunits tests, rbd_mirror_journal.sh expects to find ceph binaries
# in PATH.
#
# Thus a typical troubleshooting session:
# cd $CEPH_SRC_PATH
# PATH=$CEPH_SRC_PATH:$PATH
# RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
-# ../qa/workunits/rbd/rbd_mirror.sh
+# ../qa/workunits/rbd/rbd_mirror_journal.sh
#
# After the test failure cd to TEMPDIR and check the current state:
#
#
# cd $CEPH_SRC_PATH
# export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
-# ../qa/workunits/rbd/rbd_mirror.sh status
-# ../qa/workunits/rbd/rbd_mirror.sh stop_mirror cluster1
-# ../qa/workunits/rbd/rbd_mirror.sh start_mirror cluster2
-# ../qa/workunits/rbd/rbd_mirror.sh flush cluster2
+# ../qa/workunits/rbd/rbd_mirror_journal.sh status
+# ../qa/workunits/rbd/rbd_mirror_journal.sh stop_mirror cluster1
+# ../qa/workunits/rbd/rbd_mirror_journal.sh start_mirror cluster2
+# ../qa/workunits/rbd/rbd_mirror_journal.sh flush cluster2
# ...
#
# Eventually, run the cleanup:
#
# cd $CEPH_SRC_PATH
# RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
-# ../qa/workunits/rbd/rbd_mirror.sh cleanup
+# ../qa/workunits/rbd/rbd_mirror_journal.sh cleanup
#
if type xmlstarlet > /dev/null 2>&1; then
PEER_CLUSTER_SUFFIX=
POOL=mirror
PARENT_POOL=mirror_parent
+NS1=ns1
+NS2=ns2
TEMPDIR=
CEPH_ID=${CEPH_ID:-mirror}
+RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling}
MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.}
+MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool}
+MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal}
+
export CEPH_ARGS="--id ${CEPH_ID}"
LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1))
# needed for ceph CLI under cmake
export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH}
- export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind
- for x in ${CEPH_ROOT}/lib/cython_modules/lib* ; do
- export PYTHONPATH="${PYTHONPATH}:${x}"
- done
+ export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3
fi
# These vars facilitate running this script in an environment with
instance=${val##*:}
if [ "${instance}" = "${val}" ]; then
- # instance was not specified, use default
- instance=0
+ # instance was not specified, use default
+ instance=0
fi
eval ${cluster_var_name}=${cluster}
fi
if [ -z "${expected}" ]; then
- return 0
+ return 0
fi
if ! grep -q "${expected}" ${out} ; then
return 0
}
+mkfname()
+{
+ echo "$@" | sed -e 's|[/ ]|_|g'
+}
+
create_users()
{
local cluster=$1
CEPH_ARGS='' ceph --cluster "${cluster}" \
auth get-or-create client.${CEPH_ID} \
- mon 'profile rbd' osd 'profile rbd' >> \
+ mon 'profile rbd' osd 'profile rbd' mgr 'profile rbd' >> \
${CEPH_ROOT}/run/${cluster}/keyring
for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
CEPH_ARGS='' ceph --cluster "${cluster}" \
auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \
- mon 'profile rbd-mirror' osd 'profile rbd' >> \
+ mon 'profile rbd-mirror' osd 'profile rbd' mgr 'profile rbd' >> \
${CEPH_ROOT}/run/${cluster}/keyring
done
}
-update_users()
-{
- local cluster=$1
-
- CEPH_ARGS='' ceph --cluster "${cluster}" \
- auth caps client.${CEPH_ID} \
- mon 'profile rbd' osd 'profile rbd'
- for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
- CEPH_ARGS='' ceph --cluster "${cluster}" \
- auth caps client.${MIRROR_USER_ID_PREFIX}${instance} \
- mon 'profile rbd-mirror' osd 'profile rbd'
- done
-}
-
setup_cluster()
{
local cluster=$1
CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL}
CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL}
- rbd --cluster ${cluster} mirror pool enable ${POOL} pool
+ if [ -n "${RBD_MIRROR_CONFIG_KEY}" ]; then
+ PEER_CLUSTER_SUFFIX=-DNE
+ fi
+
+ CEPH_ARGS='' rbd --cluster ${cluster} mirror pool enable \
+ --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} ${MIRROR_POOL_MODE}
rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image
+ rbd --cluster ${cluster} namespace create ${POOL}/${NS1}
+ rbd --cluster ${cluster} namespace create ${POOL}/${NS2}
+
+ rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE}
+ rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS2} image
+
if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then
if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then
rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster}
rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster}
else
mon_map_file=${TEMPDIR}/${remote_cluster}.monmap
- ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file}
+ CEPH_ARGS='' ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file}
mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' |
head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g')
admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key
CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file}
- rbd --cluster ${cluster} mirror pool peer add ${POOL} client.${CEPH_ID}@${remote_cluster}-DNE \
+ CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer add ${POOL} \
+ client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} \
--remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file}
- uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}-DNE)
- rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr}
- rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file}
-
- PEER_CLUSTER_SUFFIX=-DNE
+ uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} \
+ client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX})
+ CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr}
+ CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file}
fi
fi
}
setup_tempdir()
{
if [ -n "${RBD_MIRROR_TEMDIR}" ]; then
- test -d "${RBD_MIRROR_TEMDIR}" ||
- mkdir "${RBD_MIRROR_TEMDIR}"
- TEMPDIR="${RBD_MIRROR_TEMDIR}"
- cd ${TEMPDIR}
+ test -d "${RBD_MIRROR_TEMDIR}" ||
+ mkdir "${RBD_MIRROR_TEMDIR}"
+ TEMPDIR="${RBD_MIRROR_TEMDIR}"
+ cd ${TEMPDIR}
else
- TEMPDIR=`mktemp -d`
+ TEMPDIR=`mktemp -d`
fi
}
setup_tempdir
if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
- setup_cluster "${CLUSTER1}"
- setup_cluster "${CLUSTER2}"
- else
- update_users "${CLUSTER1}"
- update_users "${CLUSTER2}"
+ setup_cluster "${CLUSTER1}"
+ setup_cluster "${CLUSTER2}"
fi
setup_pools "${CLUSTER1}" "${CLUSTER2}"
setup_pools "${CLUSTER2}" "${CLUSTER1}"
+
+ if [ -n "${RBD_MIRROR_MIN_COMPAT_CLIENT}" ]; then
+ CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd \
+ set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT}
+ CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd \
+ set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT}
+ fi
}
cleanup()
CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
for cluster in "${CLUSTER1}" "${CLUSTER2}"; do
- stop_mirrors "${cluster}"
+ stop_mirrors "${cluster}"
done
if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return
rbd-mirror \
- --cluster ${cluster} \
+ --cluster ${cluster} \
--id ${MIRROR_USER_ID_PREFIX}${instance} \
- --rbd-mirror-delete-retry-interval=5 \
- --rbd-mirror-image-state-check-interval=5 \
- --rbd-mirror-journal-poll-age=1 \
- --rbd-mirror-pool-replayers-refresh-interval=5 \
- --debug-rbd=30 --debug-journaler=30 \
- --debug-rbd_mirror=30 \
- --daemonize=true \
- ${RBD_MIRROR_ARGS}
+ --rbd-mirror-delete-retry-interval=5 \
+ --rbd-mirror-image-state-check-interval=5 \
+ --rbd-mirror-journal-poll-age=1 \
+ --rbd-mirror-pool-replayers-refresh-interval=5 \
+ --debug-rbd=30 --debug-journaler=30 \
+ --debug-rbd_mirror=30 \
+ --daemonize=true \
+ ${RBD_MIRROR_ARGS}
}
start_mirrors()
pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || :
if [ -n "${pid}" ]
then
- kill ${sig} ${pid}
- for s in 1 2 4 8 16 32; do
- sleep $s
- ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break
- done
- ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}'
+ kill ${sig} ${pid}
+ for s in 1 2 4 8 16 32; do
+ sleep $s
+ ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break
+ done
+ ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}'
fi
rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}")
rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}")
local loop_instance
for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16; do
- sleep ${s}
- if [ "${instance}" != "${cluster_instance}" ]; then
- admin_daemon "${cluster}:${instance}" $@ && return 0
- else
- for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
- admin_daemon "${cluster}:${loop_instance}" $@ && return 0
- done
- fi
+ sleep ${s}
+ if [ "${instance}" != "${cluster_instance}" ]; then
+ admin_daemon "${cluster}:${instance}" $@ && return 0
+ else
+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+ admin_daemon "${cluster}:${loop_instance}" $@ && return 0
+ done
+ fi
done
return 1
}
local cluster=$1 ; shift
for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
- admin_daemon "${cluster}:${instance}" $@
+ admin_daemon "${cluster}:${instance}" $@
done
}
status()
{
- local cluster daemon image_pool image
+ local cluster daemon image_pool image_ns image
for cluster in ${CLUSTER1} ${CLUSTER2}
do
- echo "${cluster} status"
- ceph --cluster ${cluster} -s
- ceph --cluster ${cluster} service dump
- ceph --cluster ${cluster} service status
- echo
-
- for image_pool in ${POOL} ${PARENT_POOL}
- do
- echo "${cluster} ${image_pool} images"
- rbd --cluster ${cluster} -p ${image_pool} ls -l
- echo
-
- echo "${cluster} ${image_pool} mirror pool status"
- rbd --cluster ${cluster} -p ${image_pool} mirror pool status --verbose
- echo
-
- for image in `rbd --cluster ${cluster} -p ${image_pool} ls 2>/dev/null`
- do
- echo "image ${image} info"
- rbd --cluster ${cluster} -p ${image_pool} info ${image}
- echo
- echo "image ${image} journal status"
- rbd --cluster ${cluster} -p ${image_pool} journal status --image ${image}
- echo
- done
- done
+ echo "${cluster} status"
+ CEPH_ARGS='' ceph --cluster ${cluster} -s
+ CEPH_ARGS='' ceph --cluster ${cluster} service dump
+ CEPH_ARGS='' ceph --cluster ${cluster} service status
+ echo
+
+ for image_pool in ${POOL} ${PARENT_POOL}
+ do
+ for image_ns in "" "${NS1}" "${NS2}"
+ do
+ echo "${cluster} ${image_pool} ${image_ns} images"
+ rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls -l
+ echo
+
+ echo "${cluster} ${image_pool}${image_ns} mirror pool info"
+ rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool info
+ echo
+
+ echo "${cluster} ${image_pool}${image_ns} mirror pool status"
+ CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool status --verbose
+ echo
+
+ for image in `rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls 2>/dev/null`
+ do
+ echo "image ${image} info"
+ rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" info ${image}
+ echo
+ echo "image ${image} journal status"
+ rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" journal status --image ${image}
+ echo
+ done
+
+ echo "${cluster} ${image_pool} ${image_ns} rbd_mirroring omap vals"
+ rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirroring
+ echo
+ done
+ done
done
local ret
for cluster in "${CLUSTER1}" "${CLUSTER2}"
do
- for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
- local pid_file=$(daemon_pid_file ${cluster}:${instance})
- if [ ! -e ${pid_file} ]
- then
- echo "${cluster} rbd-mirror not running or unknown" \
- "(${pid_file} not exist)"
- continue
- fi
-
- local pid
- pid=$(cat ${pid_file} 2>/dev/null) || :
- if [ -z "${pid}" ]
- then
- echo "${cluster} rbd-mirror not running or unknown" \
- "(can't find pid using ${pid_file})"
- ret=1
- continue
- fi
-
- echo "${daemon} rbd-mirror process in ps output:"
- if ps auxww |
- awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}'
- then
- echo
- echo "${cluster} rbd-mirror not running" \
- "(can't find pid $pid in ps output)"
- ret=1
- continue
- fi
- echo
-
- local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster})
- if [ ! -S "${asok_file}" ]
- then
- echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)"
- ret=1
- continue
- fi
-
- echo "${cluster} rbd-mirror status"
- ceph --admin-daemon ${asok_file} rbd mirror status
- echo
- done
+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+ local pid_file=$(daemon_pid_file ${cluster}:${instance})
+ if [ ! -e ${pid_file} ]
+ then
+ echo "${cluster} rbd-mirror not running or unknown" \
+ "(${pid_file} not exist)"
+ continue
+ fi
+
+ local pid
+ pid=$(cat ${pid_file} 2>/dev/null) || :
+ if [ -z "${pid}" ]
+ then
+ echo "${cluster} rbd-mirror not running or unknown" \
+ "(can't find pid using ${pid_file})"
+ ret=1
+ continue
+ fi
+
+ echo "${daemon} rbd-mirror process in ps output:"
+ if ps auxww |
+ awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}'
+ then
+ echo
+ echo "${cluster} rbd-mirror not running" \
+ "(can't find pid $pid in ps output)"
+ ret=1
+ continue
+ fi
+ echo
+
+ local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster})
+ if [ ! -S "${asok_file}" ]
+ then
+ echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)"
+ ret=1
+ continue
+ fi
+
+ echo "${cluster} rbd-mirror status"
+ ceph --admin-daemon ${asok_file} rbd mirror status
+ echo
+ done
done
return ${ret}
# TODO: add a way to force rbd-mirror to update replayers
for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
- sleep ${s}
- test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0
+ sleep ${s}
+ test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0
done
return 1
}
wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped
}
-get_position()
+get_journal_position()
{
local cluster=$1
local pool=$2
# Parse line like below, looking for the first position
# [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]]
- local status_log=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.status
- rbd --cluster ${cluster} -p ${pool} journal status --image ${image} |
- tee ${status_log} >&2
+ local status_log=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.status)
+ rbd --cluster ${cluster} journal status --image ${pool}/${image} |
+ tee ${status_log} >&2
sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \
- ${status_log}
+ ${status_log}
}
-get_master_position()
+get_master_journal_position()
{
local cluster=$1
local pool=$2
local image=$3
- get_position "${cluster}" "${pool}" "${image}" ''
+ get_journal_position "${cluster}" "${pool}" "${image}" ''
}
-get_mirror_position()
+get_mirror_journal_position()
{
local cluster=$1
local pool=$2
local image=$3
- get_position "${cluster}" "${pool}" "${image}" '..*'
+ get_journal_position "${cluster}" "${pool}" "${image}" '..*'
}
-wait_for_replay_complete()
+wait_for_journal_replay_complete()
{
local local_cluster=$1
local cluster=$2
while true; do
for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
- sleep ${s}
- flush "${local_cluster}" "${pool}" "${image}"
- master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
- mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
- test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
+ sleep ${s}
+ flush "${local_cluster}" "${pool}" "${image}"
+ master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}")
+ mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}")
+ test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
test "${mirror_pos}" != "${last_mirror_pos}" && break
done
return 1
}
+mirror_image_snapshot()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+
+ rbd --cluster "${cluster}" mirror image snapshot "${pool}/${image}"
+}
+
+get_newest_mirror_snapshot()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+ local log=$4
+
+ rbd --cluster "${cluster}" snap list --all "${pool}/${image}" --format xml | \
+ xmlstarlet sel -t -c "//snapshots/snapshot[namespace/complete='true' and position()=last()]" > \
+ ${log} || true
+}
+
+wait_for_snapshot_sync_complete()
+{
+ local local_cluster=$1
+ local cluster=$2
+ local pool=$3
+ local image=$4
+
+ local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status)
+ local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status)
+
+ mirror_image_snapshot "${cluster}" "${pool}" "${image}"
+ get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}"
+ local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log})
+
+ while true; do
+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
+ sleep ${s}
+
+ get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}"
+ local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log})
+
+ test "${snapshot_id}" = "${primary_snapshot_id}" && return 0
+ done
+
+ return 1
+ done
+ return 1
+}
+
+wait_for_replay_complete()
+{
+ local local_cluster=$1
+ local cluster=$2
+ local pool=$3
+ local image=$4
+
+ if [ "${MIRROR_IMAGE_MODE}" = "journal" ]; then
+ wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image}
+ elif [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then
+ wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image}
+ else
+ return 1
+ fi
+}
+
+
test_status_in_pool_dir()
{
local cluster=$1
local description_pattern="$5"
local service_pattern="$6"
- local status_log=${TEMPDIR}/${cluster}-${pool}-${image}.mirror_status
- rbd --cluster ${cluster} -p ${pool} mirror image status ${image} |
- tee ${status_log} >&2
- grep "state: .*${state_pattern}" ${status_log} || return 1
- grep "description: .*${description_pattern}" ${status_log} || return 1
+ local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status)
+ CEPH_ARGS='' rbd --cluster ${cluster} mirror image status ${pool}/${image} |
+ tee ${status_log} >&2
+ grep "^ state: .*${state_pattern}" ${status_log} || return 1
+ grep "^ description: .*${description_pattern}" ${status_log} || return 1
if [ -n "${service_pattern}" ]; then
grep "service: *${service_pattern}" ${status_log} || return 1
# recheck using `mirror pool status` command to stress test it.
- local last_update="$(sed -nEe 's/^ *last_update: *(.*) *$/\1/p' ${status_log})"
+ local last_update="$(sed -nEe 's/^ last_update: *(.*) *$/\1/p' ${status_log})"
test_mirror_pool_status_verbose \
${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" &&
return 0
local state_pattern="$4"
local prev_last_update="$5"
- local status_log=${TEMPDIR}/${cluster}-${pool}.mirror_status
+ local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}.mirror_status)
rbd --cluster ${cluster} mirror pool status ${pool} --verbose --format xml \
> ${status_log}
local service_pattern="$6"
for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
- sleep ${s}
- test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \
+ sleep ${s}
+ test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \
"${description_pattern}" "${service_pattern}" &&
return 0
done
local size=128
if [ -n "$1" ]; then
- size=$1
- shift
+ size=$1
+ shift
fi
- rbd --cluster ${cluster} -p ${pool} create --size ${size} \
- --image-feature layering,exclusive-lock,journaling $@ ${image}
+ rbd --cluster ${cluster} create --size ${size} \
+ --image-feature "${RBD_IMAGE_FEATURES}" $@ ${pool}/${image}
+}
+
+create_image_and_enable_mirror()
+{
+ local cluster=$1 ; shift
+ local pool=$1 ; shift
+ local image=$1 ; shift
+ local mode=${1:-${MIRROR_IMAGE_MODE}}
+ if [ -n "$1" ]; then
+ shift
+ fi
+
+ create_image ${cluster} ${pool} ${image} $@
+ enable_mirror ${cluster} ${pool} ${image} ${mode}
+}
+
+enable_journaling()
+{
+ local cluster=$1
+ local pool=$2
+ local image=$3
+
+ rbd --cluster ${cluster} feature enable ${pool}/${image} journaling
}
set_image_meta()
local key=$4
local val=$5
- rbd --cluster ${cluster} -p ${pool} image-meta set ${image} $key $val
+ rbd --cluster ${cluster} image-meta set ${pool}/${image} $key $val
}
compare_image_meta()
local key=$4
local value=$5
- test `rbd --cluster ${cluster} -p ${pool} image-meta get ${image} ${key}` = "${value}"
+ test `rbd --cluster ${cluster} image-meta get ${pool}/${image} ${key}` = "${value}"
}
rename_image()
local image=$3
local new_name=$4
- rbd --cluster=${cluster} -p ${pool} rename ${image} ${new_name}
+ rbd --cluster=${cluster} rename ${pool}/${image} ${pool}/${new_name}
}
remove_image()
local pool=$2
local image=$3
- rbd --cluster=${cluster} -p ${pool} snap purge ${image}
- rbd --cluster=${cluster} -p ${pool} rm ${image}
+ rbd --cluster=${cluster} snap purge ${pool}/${image}
+ rbd --cluster=${cluster} rm ${pool}/${image}
}
remove_image_retry()
local pool=$2
local image=$3
- rbd --cluster=${cluster} -p ${pool} trash move ${image}
+ rbd --cluster=${cluster} trash move ${pool}/${image}
}
trash_restore() {
local pool=$2
local image_id=$3
- rbd --cluster=${cluster} -p ${pool} trash restore ${image_id}
+ rbd --cluster=${cluster} trash restore ${pool}/${image_id}
}
clone_image()
rbd --cluster ${cluster} clone \
${parent_pool}/${parent_image}@${parent_snap} \
- ${clone_pool}/${clone_image} \
- --image-feature layering,exclusive-lock,journaling $@
+ ${clone_pool}/${clone_image} --image-feature "${RBD_IMAGE_FEATURES}" $@
+}
+
+clone_image_and_enable_mirror()
+{
+ local cluster=$1
+ local parent_pool=$2
+ local parent_image=$3
+ local parent_snap=$4
+ local clone_pool=$5
+ local clone_image=$6
+ shift 6
+
+ local mode=${1:-${MIRROR_IMAGE_MODE}}
+ if [ -n "$1" ]; then
+ shift
+ fi
+
+ clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@
+ enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode}
}
disconnect_image()
local pool=$2
local image=$3
- rbd --cluster ${cluster} -p ${pool} journal client disconnect \
- --image ${image}
+ rbd --cluster ${cluster} journal client disconnect \
+ --image ${pool}/${image}
}
create_snapshot()
local image=$3
local snap=$4
- rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap}
+ rbd --cluster ${cluster} snap create ${pool}/${image}@${snap}
}
remove_snapshot()
local image=$3
local snap=$4
- rbd --cluster ${cluster} -p ${pool} snap rm ${image}@${snap}
+ rbd --cluster ${cluster} snap rm ${pool}/${image}@${snap}
}
rename_snapshot()
local snap=$4
local new_snap=$5
- rbd --cluster ${cluster} -p ${pool} snap rename ${image}@${snap} ${image}@${new_snap}
+ rbd --cluster ${cluster} snap rename ${pool}/${image}@${snap} \
+ ${pool}/${image}@${new_snap}
}
purge_snapshots()
local pool=$2
local image=$3
- rbd --cluster ${cluster} -p ${pool} snap purge ${image}
+ rbd --cluster ${cluster} snap purge ${pool}/${image}
}
protect_snapshot()
local image=$3
local snap=$4
- rbd --cluster ${cluster} -p ${pool} snap protect ${image}@${snap}
+ rbd --cluster ${cluster} snap protect ${pool}/${image}@${snap}
}
unprotect_snapshot()
local image=$3
local snap=$4
- rbd --cluster ${cluster} -p ${pool} snap unprotect ${image}@${snap}
+ rbd --cluster ${cluster} snap unprotect ${pool}/${image}@${snap}
}
wait_for_snap_present()
local s
for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
- sleep ${s}
- rbd --cluster ${cluster} -p ${pool} info ${image}@${snap_name} || continue
+ sleep ${s}
+ rbd --cluster ${cluster} info ${pool}/${image}@${snap_name} || continue
return 0
done
return 1
local s
for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
- sleep ${s}
+ sleep ${s}
test_snap_moved_to_trash $@ || continue
return 0
done
local s
for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
- sleep ${s}
+ sleep ${s}
test_snap_removed_from_trash $@ || continue
return 0
done
test -n "${size}" || size=4096
- rbd --cluster ${cluster} -p ${pool} bench ${image} --io-type write \
- --io-size ${size} --io-threads 1 --io-total $((size * count)) \
- --io-pattern rand
+ rbd --cluster ${cluster} bench ${pool}/${image} --io-type write \
+ --io-size ${size} --io-threads 1 --io-total $((size * count)) \
+ --io-pattern rand
}
stress_write_image()
local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}')
timeout ${duration}s ceph_test_rbd_mirror_random_write \
- --cluster ${cluster} ${pool} ${image} \
- --debug-rbd=20 --debug-journaler=20 \
- 2> ${TEMPDIR}/rbd-mirror-random-write.log || true
+ --cluster ${cluster} ${pool} ${image} \
+ --debug-rbd=20 --debug-journaler=20 \
+ 2> ${TEMPDIR}/rbd-mirror-random-write.log || true
+}
+
+show_diff()
+{
+ local file1=$1
+ local file2=$2
+
+ xxd ${file1} > ${file1}.xxd
+ xxd ${file2} > ${file2}.xxd
+ sdiff -s ${file1}.xxd ${file2}.xxd | head -n 64
+ rm -f ${file1}.xxd ${file2}.xxd
}
compare_images()
{
local pool=$1
local image=$2
+ local ret=0
- local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
- local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
+ local rmt_export=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.export)
+ local loc_export=${TEMPDIR}/$(mkfname ${CLUSTER1}-${pool}-${image}.export)
rm -f ${rmt_export} ${loc_export}
- rbd --cluster ${CLUSTER2} -p ${pool} export ${image} - | xxd > ${rmt_export}
- rbd --cluster ${CLUSTER1} -p ${pool} export ${image} - | xxd > ${loc_export}
- sdiff -s ${rmt_export} ${loc_export} | head -n 64
+ rbd --cluster ${CLUSTER2} export ${pool}/${image} ${rmt_export}
+ rbd --cluster ${CLUSTER1} export ${pool}/${image} ${loc_export}
+ if ! cmp ${rmt_export} ${loc_export}
+ then
+ show_diff ${rmt_export} ${loc_export}
+ ret=1
+ fi
rm -f ${rmt_export} ${loc_export}
+ return ${ret}
}
compare_image_snapshots()
{
local pool=$1
local image=$2
+ local ret=0
local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
- for snap_name in $(rbd --cluster ${CLUSTER1} -p ${pool} --format xml \
- snap list ${image} | \
+ for snap_name in $(rbd --cluster ${CLUSTER1} --format xml \
+ snap list ${pool}/${image} | \
$XMLSTARLET sel -t -v "//snapshot/name" | \
grep -E -v "^\.rbd-mirror\."); do
rm -f ${rmt_export} ${loc_export}
- rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} - | xxd > ${rmt_export}
- rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} - | xxd > ${loc_export}
- sdiff -s ${rmt_export} ${loc_export} | head -n 64
+ rbd --cluster ${CLUSTER2} export ${pool}/${image}@${snap_name} ${rmt_export}
+ rbd --cluster ${CLUSTER1} export ${pool}/${image}@${snap_name} ${loc_export}
+ if ! cmp ${rmt_export} ${loc_export}
+ then
+ show_diff ${rmt_export} ${loc_export}
+ ret=1
+ fi
done
rm -f ${rmt_export} ${loc_export}
+ return ${ret}
}
demote_image()
{
local cluster=$1
local pool=$2
- local mode=$3
+ local mode=${3:-${MIRROR_POOL_MODE}}
- rbd --cluster=${cluster} -p ${pool} mirror pool enable ${mode}
+ rbd --cluster=${cluster} mirror pool enable ${pool} ${mode}
}
disable_mirror()
local cluster=$1
local pool=$2
local image=$3
+ local mode=${4:-${MIRROR_IMAGE_MODE}}
- rbd --cluster=${cluster} mirror image enable ${pool}/${image}
+ rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode}
}
test_image_present()
# TODO: add a way to force rbd-mirror to update replayers
for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
- sleep ${s}
- test_image_present \
+ sleep ${s}
+ test_image_present \
"${cluster}" "${pool}" "${image}" "${state}" "${image_id}" &&
return 0
done
local pool=$2
local image=$3
- rbd --cluster=${cluster} -p ${pool} info ${image} |
- sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
+ rbd --cluster=${cluster} info ${pool}/${image} |
+ sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
}
request_resync_image()
eval "${image_id_var_name}='$(get_image_id ${cluster} ${pool} ${image})'"
eval 'test -n "$'${image_id_var_name}'"'
- rbd --cluster=${cluster} -p ${pool} mirror image resync ${image}
+ rbd --cluster=${cluster} mirror image resync ${pool}/${image}
}
get_image_data_pool()
local pool=$2
local image=$3
- rbd --cluster ${cluster} -p ${pool} info ${image} |
+ rbd --cluster ${cluster} info ${pool}/${image} |
awk '$1 == "data_pool:" {print $2}'
}
projects / ceph.git / blobdiff