ceph/qa/workunits/rbd/rbd_support_module_recovery.sh

   1 #!/bin/bash
   2 set -ex
   3
   4 POOL=rbd
   5 IMAGE_PREFIX=image
   6 NUM_IMAGES=20
   7 RUN_TIME=3600
   8
   9 rbd mirror pool enable ${POOL} image
  10 rbd mirror pool peer add ${POOL} dummy
  11
  12 # Create images and schedule their mirror snapshots
  13 for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
  14     rbd create -s 1G --image-feature exclusive-lock ${POOL}/${IMAGE_PREFIX}$i
  15     rbd mirror image enable ${POOL}/${IMAGE_PREFIX}$i snapshot
  16     rbd mirror snapshot schedule add -p ${POOL} --image ${IMAGE_PREFIX}$i 1m
  17 done
  18
  19 # Run fio workloads on images via kclient
  20 # Test the recovery of the rbd_support module and its scheduler from their
  21 # librbd client being blocklisted while a exclusive lock gets passed around
  22 # between their librbd client and a kclient trying to take mirror snapshots
  23 # and perform I/O on the same image.
  24 for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
  25     DEVS[$i]=$(sudo rbd device map ${POOL}/${IMAGE_PREFIX}$i)
  26     fio --name=fiotest --filename=${DEVS[$i]} --rw=randrw --bs=4K --direct=1 \
  27         --ioengine=libaio --iodepth=2 --runtime=43200 --time_based \
  28         &> /dev/null &
  29 done
  30
  31 # Repeatedly blocklist rbd_support module's client ~10s after the module
  32 # recovers from previous blocklisting
  33 CURRENT_TIME=$(date +%s)
  34 END_TIME=$((CURRENT_TIME + RUN_TIME))
  35 PREV_CLIENT_ADDR=""
  36 CLIENT_ADDR=""
  37 while ((CURRENT_TIME <= END_TIME)); do
  38     if [[ -n "${CLIENT_ADDR}" ]] &&
  39        [[ "${CLIENT_ADDR}" != "${PREV_CLIENT_ADDR}" ]]; then
  40             ceph osd blocklist add ${CLIENT_ADDR}
  41             # Confirm rbd_support module's client is blocklisted
  42             ceph osd blocklist ls | grep -q ${CLIENT_ADDR}
  43             PREV_CLIENT_ADDR=${CLIENT_ADDR}
  44     fi
  45     sleep 10
  46     CLIENT_ADDR=$(ceph mgr dump |
  47         jq .active_clients[] |
  48         jq 'select(.name == "rbd_support")' |
  49         jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add')
  50     CURRENT_TIME=$(date +%s)
  51 done
  52
  53 # Confirm that rbd_support module recovered from repeated blocklisting
  54 # Check that you can add a mirror snapshot schedule after a few retries
  55 for ((i = 1; i <= 24; i++)); do
  56     rbd mirror snapshot schedule add -p ${POOL} \
  57         --image ${IMAGE_PREFIX}1 2m && break
  58     sleep 10
  59 done
  60 rbd mirror snapshot schedule ls -p ${POOL} --image ${IMAGE_PREFIX}1 |
  61     grep 'every 2m'
  62 # Verify that the schedule present before client blocklisting is preserved
  63 rbd mirror snapshot schedule ls -p ${POOL} --image ${IMAGE_PREFIX}1 |
  64     grep 'every 1m'
  65 rbd mirror snapshot schedule rm -p ${POOL} --image ${IMAGE_PREFIX}1 2m
  66 for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
  67     rbd mirror snapshot schedule rm -p ${POOL} --image ${IMAGE_PREFIX}$i 1m
  68 done
  69
  70 # cleanup
  71 killall fio || true
  72 wait
  73 for ((i = 1; i <= ${NUM_IMAGES}; i++)); do
  74     sudo rbd device unmap ${DEVS[$i]}
  75 done
  76
  77 echo OK