update ceph source to reef 18.1.2

[ceph.git] / ceph / qa / workunits / rbd / diff_continuous.sh

diff --git a/ceph/qa/workunits/rbd/diff_continuous.sh b/ceph/qa/workunits/rbd/diff_continuous.sh
index b8f7e8b7e137570c6c9d324f0286ef374a7e5dbb..fd1785e07c4b418e0d3b01365ad4eae0f9737e6e 100755 (executable)
--- a/ceph/qa/workunits/rbd/diff_continuous.sh
+++ b/ceph/qa/workunits/rbd/diff_continuous.sh
@@ -1,60 +1,106 @@
 #!/usr/bin/env bash
+
 set -ex
+set -o pipefail
+
+function untar_workload() {
+    local i
+    for ((i = 0; i < 10; i++)); do
+        pv -L 10M linux-5.4.tar.gz > "${MOUNT}/linux-5.4.tar.gz"
+        tar -C "${MOUNT}" -xzf "${MOUNT}/linux-5.4.tar.gz"
+        sync "${MOUNT}"
+        rm -rf "${MOUNT}"/linux-5.4*
+    done
+}
+
+function check_object_map() {
+    local spec="$1"
+
+    rbd object-map check "${spec}"
 
-max=20
-size=1500
-
-iosize=16384
-iototal=16384000
-iothreads=16
-
-parent=`uuidgen`"-parent"
-src=`uuidgen`"-src";
-dst=`uuidgen`"-dst";
-
-function cleanup() {
-    rbd snap purge $src || :
-    rbd rm $src || :
-    rbd snap purge $dst || :
-    rbd rm $dst || :
-    rbd snap unprotect $parent --snap parent || :
-    rbd snap purge $parent || :
-    rbd rm $parent || :
+    local flags
+    flags="$(rbd info "${spec}" | grep 'flags: ')"
+    if [[ "${flags}" =~ object\ map\ invalid ]]; then
+        echo "Object map invalid at ${spec}"
+        exit 1
+    fi
+    if [[ "${flags}" =~ fast\ diff\ invalid ]]; then
+        echo "Fast diff invalid at ${spec}"
+        exit 1
+    fi
 }
-trap cleanup EXIT
-
-# start from a clone
-rbd create $parent --size $size --image-format 2 --stripe-count 8 --stripe-unit 65536
-rbd bench --io-type write $parent --io-size $iosize --io-threads $iothreads --io-total $iototal --io-pattern rand
-rbd snap create $parent --snap parent
-rbd snap protect $parent --snap parent
-rbd clone $parent@parent $src --stripe-count 4 --stripe-unit 262144
-rbd create $dst --size $size --image-format 2 --order 19
-
-# mirror for a while
-for s in `seq 1 $max`; do
-    rbd snap create $src --snap=snap$s
-    rbd export-diff $src@snap$s - $lastsnap | rbd import-diff - $dst  &
-    rbd bench --io-type write $src --io-size $iosize --io-threads $iothreads --io-total $iototal --io-pattern rand  &
-    wait
-    lastsnap="--from-snap snap$s"
+
+# RBD_DEVICE_TYPE is intended to be set from yaml, default to krbd
+readonly DEVICE_TYPE="${RBD_DEVICE_TYPE:-krbd}"
+
+BASE_UUID="$(uuidgen)"
+readonly BASE_UUID
+
+readonly SIZE="2G"
+readonly SRC="${BASE_UUID}-src"
+readonly DST="${BASE_UUID}-dst"
+readonly MOUNT="${BASE_UUID}-mnt"
+
+rbd create -s "${SIZE}" --stripe-unit 64K --stripe-count 8 \
+    --image-feature exclusive-lock,object-map,fast-diff "${SRC}"
+rbd create -s "${SIZE}" --object-size 512K "${DST}"
+
+dev="$(sudo rbd device map -t "${DEVICE_TYPE}" "${SRC}")"
+sudo mkfs.ext4 "${dev}"
+mkdir "${MOUNT}"
+sudo mount "${dev}" "${MOUNT}"
+sudo chown "$(whoami)" "${MOUNT}"
+
+# start untar in the background
+wget https://download.ceph.com/qa/linux-5.4.tar.gz
+untar_workload &
+untar_pid=$!
+
+# export initial incremental
+snap_num=1
+rbd snap create "${SRC}@snap${snap_num}"
+rbd export-diff "${SRC}@snap${snap_num}" "${BASE_UUID}@snap${snap_num}.diff"
+
+# keep exporting successive incrementals while untar is running
+while kill -0 "${untar_pid}"; do
+    snap_num=$((snap_num + 1))
+    rbd snap create "${SRC}@snap${snap_num}"
+    sleep $((RANDOM % 4 + 1))
+    rbd export-diff --whole-object --from-snap "snap$((snap_num - 1))" \
+        "${SRC}@snap${snap_num}" "${BASE_UUID}@snap${snap_num}.diff"
 done
 
-#trap "" EXIT
-#exit 0
+sudo umount "${MOUNT}"
+sudo rbd device unmap -t "${DEVICE_TYPE}" "${dev}"
+
+if ! wait "${untar_pid}"; then
+    echo "untar_workload failed"
+    exit 1
+fi
+
+echo "Exported ${snap_num} incrementals"
+if ((snap_num < 30)); then
+    echo "Too few incrementals"
+    exit 1
+fi
 
 # validate
-for s in `seq 1 $max`; do
-    ssum=`rbd export $src@snap$s - | md5sum`
-    dsum=`rbd export $dst@snap$s - | md5sum`
-    if [ "$ssum" != "$dsum" ]; then
-       echo different sum at snap$s
-       exit 1
+for ((i = 1; i <= snap_num; i++)); do
+    rbd import-diff "${BASE_UUID}@snap${i}.diff" "${DST}"
+    src_sum="$(rbd export "${SRC}@snap${i}" - | md5sum | awk '{print $1}')"
+    dst_sum="$(rbd export "${DST}@snap${i}" - | md5sum | awk '{print $1}')"
+    if [[ "${src_sum}" != "${dst_sum}" ]]; then
+        echo "Mismatch at snap${i}: ${src_sum} != ${dst_sum}"
+        exit 1
     fi
+    check_object_map "${SRC}@snap${i}"
+    # FIXME: this reproduces http://tracker.ceph.com/issues/37876
+    # there is no fstrim involved but "rbd import-diff" can produce
+    # write-zeroes requests which turn into discards under the hood
+    # actual: EXISTS, expected: EXISTS_CLEAN inconsistency is harmless
+    # from a data integrity POV and data is validated above regardless,
+    # so just waive it for now
+    #check_object_map "${DST}@snap${i}"
 done
 
-cleanup
-trap "" EXIT
-
 echo OK
-