ceph/qa/workunits/rbd/rbd-nbd.sh

   1 #!/usr/bin/env bash
   2 set -ex
   3
   4 . $(dirname $0)/../../standalone/ceph-helpers.sh
   5
   6 POOL=rbd
   7 ANOTHER_POOL=new_default_pool$$
   8 NS=ns
   9 IMAGE=testrbdnbd$$
  10 SIZE=64
  11 DATA=
  12 DEV=
  13
  14 _sudo()
  15 {
  16     local cmd
  17
  18     if [ `id -u` -eq 0 ]
  19     then
  20         "$@"
  21         return $?
  22     fi
  23
  24     # Look for the command in the user path. If it fails run it as is,
  25     # supposing it is in sudo path.
  26     cmd=`which $1 2>/dev/null` || cmd=$1
  27     shift
  28     sudo -nE "${cmd}" "$@"
  29 }
  30
  31 setup()
  32 {
  33     local ns x
  34
  35     if [ -e CMakeCache.txt ]; then
  36         # running under cmake build dir
  37
  38         CEPH_SRC=$(readlink -f $(dirname $0)/../../../src)
  39         CEPH_ROOT=${PWD}
  40         CEPH_BIN=${CEPH_ROOT}/bin
  41
  42         export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH}
  43         export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3
  44         PATH=${CEPH_BIN}:${PATH}
  45     fi
  46
  47     _sudo echo test sudo
  48
  49     trap cleanup INT TERM EXIT
  50     TEMPDIR=`mktemp -d`
  51     DATA=${TEMPDIR}/data
  52     dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE}
  53
  54     rbd namespace create ${POOL}/${NS}
  55
  56     for ns in '' ${NS}; do
  57         rbd --dest-pool ${POOL} --dest-namespace "${ns}" --no-progress import \
  58             ${DATA} ${IMAGE}
  59     done
  60
  61     # create another pool
  62     ceph osd pool create ${ANOTHER_POOL} 8
  63     rbd pool init ${ANOTHER_POOL}
  64 }
  65
  66 function cleanup()
  67 {
  68     local ns s
  69
  70     set +e
  71
  72     mount | fgrep ${TEMPDIR}/mnt && _sudo umount -f ${TEMPDIR}/mnt
  73
  74     rm -Rf ${TEMPDIR}
  75     if [ -n "${DEV}" ]
  76     then
  77         _sudo rbd device --device-type nbd unmap ${DEV}
  78     fi
  79
  80     for ns in '' ${NS}; do
  81         if rbd -p ${POOL} --namespace "${ns}" status ${IMAGE} 2>/dev/null; then
  82             for s in 0.5 1 2 4 8 16 32; do
  83                 sleep $s
  84                 rbd -p ${POOL} --namespace "${ns}" status ${IMAGE} |
  85                     grep 'Watchers: none' && break
  86             done
  87             rbd -p ${POOL} --namespace "${ns}" snap purge ${IMAGE}
  88             rbd -p ${POOL} --namespace "${ns}" remove ${IMAGE}
  89         fi
  90     done
  91     rbd namespace remove ${POOL}/${NS}
  92
  93     # cleanup/reset default pool
  94     rbd config global rm global rbd_default_pool
  95     ceph osd pool delete ${ANOTHER_POOL} ${ANOTHER_POOL} --yes-i-really-really-mean-it
  96 }
  97
  98 function expect_false()
  99 {
 100   if "$@"; then return 1; else return 0; fi
 101 }
 102
 103 function get_pid()
 104 {
 105     local pool=$1
 106     local ns=$2
 107
 108     PID=$(rbd device --device-type nbd --format xml list | $XMLSTARLET sel -t -v \
 109       "//devices/device[pool='${pool}'][namespace='${ns}'][image='${IMAGE}'][device='${DEV}']/id")
 110     test -n "${PID}" || return 1
 111     ps -p ${PID} -C rbd-nbd
 112 }
 113
 114 unmap_device()
 115 {
 116     local dev=$1
 117     local pid=$2
 118
 119     _sudo rbd device --device-type nbd unmap ${dev}
 120     rbd device --device-type nbd list | expect_false grep "^${pid}\\b" || return 1
 121     ps -C rbd-nbd | expect_false grep "^ *${pid}\\b" || return 1
 122
 123     # workaround possible race between unmap and following map
 124     sleep 0.5
 125 }
 126
 127 #
 128 # main
 129 #
 130
 131 setup
 132
 133 # exit status test
 134 expect_false rbd-nbd
 135 expect_false rbd-nbd INVALIDCMD
 136 if [ `id -u` -ne 0 ]
 137 then
 138     expect_false rbd device --device-type nbd map ${IMAGE}
 139 fi
 140 expect_false _sudo rbd device --device-type nbd map INVALIDIMAGE
 141 expect_false _sudo rbd-nbd --device INVALIDDEV map ${IMAGE}
 142
 143 # list format test
 144 expect_false rbd device --device-type nbd --format INVALID list
 145 rbd device --device-type nbd --format json --pretty-format list
 146 rbd device --device-type nbd --format xml list
 147
 148 # map test using the first unused device
 149 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
 150 get_pid ${POOL}
 151 # map test specifying the device
 152 expect_false _sudo rbd-nbd --device ${DEV} map ${POOL}/${IMAGE}
 153 dev1=${DEV}
 154 unmap_device ${DEV} ${PID}
 155 DEV=
 156 # XXX: race possible when the device is reused by other process
 157 DEV=`_sudo rbd-nbd --device ${dev1} map ${POOL}/${IMAGE}`
 158 [ "${DEV}" = "${dev1}" ]
 159 rbd device --device-type nbd list | grep "${IMAGE}"
 160 get_pid ${POOL}
 161
 162 # read test
 163 [ "`dd if=${DATA} bs=1M | md5sum`" = "`_sudo dd if=${DEV} bs=1M | md5sum`" ]
 164
 165 # write test
 166 dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE}
 167 _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct
 168 [ "`dd if=${DATA} bs=1M | md5sum`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5sum`" ]
 169 unmap_device ${DEV} ${PID}
 170
 171 # notrim test
 172 DEV=`_sudo rbd device --device-type nbd --options notrim map ${POOL}/${IMAGE}`
 173 get_pid ${POOL}
 174 provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
 175   $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
 176 used=`rbd -p ${POOL} --format xml du ${IMAGE} |
 177   $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
 178 [ "${used}" -eq "${provisioned}" ]
 179 # should fail discard as at time of mapping notrim was used
 180 expect_false _sudo blkdiscard ${DEV}
 181 sync
 182 provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
 183   $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
 184 used=`rbd -p ${POOL} --format xml du ${IMAGE} |
 185   $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
 186 [ "${used}" -eq "${provisioned}" ]
 187 unmap_device ${DEV} ${PID}
 188
 189 # trim test
 190 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
 191 get_pid ${POOL}
 192 provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
 193   $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
 194 used=`rbd -p ${POOL} --format xml du ${IMAGE} |
 195   $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
 196 [ "${used}" -eq "${provisioned}" ]
 197 # should honor discard as at time of mapping trim was considered by default
 198 _sudo blkdiscard ${DEV}
 199 sync
 200 provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
 201   $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
 202 used=`rbd -p ${POOL} --format xml du ${IMAGE} |
 203   $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
 204 [ "${used}" -lt "${provisioned}" ]
 205
 206 # resize test
 207 devname=$(basename ${DEV})
 208 blocks=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
 209 test -n "${blocks}"
 210 rbd resize ${POOL}/${IMAGE} --size $((SIZE * 2))M
 211 rbd info ${POOL}/${IMAGE}
 212 blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
 213 test -n "${blocks2}"
 214 test ${blocks2} -eq $((blocks * 2))
 215 rbd resize ${POOL}/${IMAGE} --allow-shrink --size ${SIZE}M
 216 blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
 217 test -n "${blocks2}"
 218 test ${blocks2} -eq ${blocks}
 219
 220 # read-only option test
 221 unmap_device ${DEV} ${PID}
 222 DEV=`_sudo rbd --device-type nbd map --read-only ${POOL}/${IMAGE}`
 223 PID=$(rbd device --device-type nbd list | awk -v pool=${POOL} -v img=${IMAGE} -v dev=${DEV} \
 224     '$2 == pool && $3 == img && $5 == dev {print $1}')
 225 test -n "${PID}"
 226 ps -p ${PID} -C rbd-nbd
 227
 228 _sudo dd if=${DEV} of=/dev/null bs=1M
 229 expect_false _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct
 230 unmap_device ${DEV} ${PID}
 231
 232 # exclusive option test
 233 DEV=`_sudo rbd --device-type nbd map --exclusive ${POOL}/${IMAGE}`
 234 get_pid ${POOL}
 235
 236 _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct
 237 expect_false timeout 10 \
 238         rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024
 239 unmap_device ${DEV} ${PID}
 240 DEV=
 241 rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024
 242
 243 # unmap by image name test
 244 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
 245 get_pid ${POOL}
 246 unmap_device ${IMAGE} ${PID}
 247 DEV=
 248
 249 # map/unmap snap test
 250 rbd snap create ${POOL}/${IMAGE}@snap
 251 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}@snap`
 252 get_pid ${POOL}
 253 unmap_device "${IMAGE}@snap" ${PID}
 254 DEV=
 255
 256 # map/unmap namespace test
 257 rbd snap create ${POOL}/${NS}/${IMAGE}@snap
 258 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${NS}/${IMAGE}@snap`
 259 get_pid ${POOL} ${NS}
 260 unmap_device "${POOL}/${NS}/${IMAGE}@snap" ${PID}
 261 DEV=
 262
 263 # unmap by image name test 2
 264 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
 265 get_pid ${POOL}
 266 pid=$PID
 267 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${NS}/${IMAGE}`
 268 get_pid ${POOL} ${NS}
 269 unmap_device ${POOL}/${NS}/${IMAGE} ${PID}
 270 DEV=
 271 unmap_device ${POOL}/${IMAGE} ${pid}
 272
 273 # map/unmap test with just image name and expect image to come from default pool
 274 if [ "${POOL}" = "rbd" ];then
 275     DEV=`_sudo rbd device --device-type nbd map ${IMAGE}`
 276     get_pid ${POOL}
 277     unmap_device ${IMAGE} ${PID}
 278     DEV=
 279 fi
 280
 281 # map/unmap test with just image name after changing default pool
 282 rbd config global set global rbd_default_pool ${ANOTHER_POOL}
 283 rbd create --size 10M ${IMAGE}
 284 DEV=`_sudo rbd device --device-type nbd map ${IMAGE}`
 285 get_pid ${ANOTHER_POOL}
 286 unmap_device ${IMAGE} ${PID}
 287 DEV=
 288
 289 # reset
 290 rbd config global rm global rbd_default_pool
 291
 292 # auto unmap test
 293 DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}`
 294 get_pid ${POOL}
 295 _sudo kill ${PID}
 296 for i in `seq 10`; do
 297   rbd device --device-type nbd list | expect_false grep "^${PID} *${POOL} *${IMAGE}" && break
 298   sleep 1
 299 done
 300 rbd device --device-type nbd list | expect_false grep "^${PID} *${POOL} *${IMAGE}"
 301
 302 # quiesce test
 303 QUIESCE_HOOK=${TEMPDIR}/quiesce.sh
 304 DEV=`_sudo rbd device --device-type nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} ${POOL}/${IMAGE}`
 305 get_pid ${POOL}
 306
 307 # test it fails if the hook does not exists
 308 test ! -e ${QUIESCE_HOOK}
 309 expect_false rbd snap create ${POOL}/${IMAGE}@quiesce1
 310 _sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
 311
 312 # test the hook is executed
 313 touch ${QUIESCE_HOOK}
 314 chmod +x ${QUIESCE_HOOK}
 315 cat > ${QUIESCE_HOOK} <<EOF
 316 #/bin/sh
 317 echo "test the hook is executed" >&2
 318 echo \$1 > ${TEMPDIR}/\$2
 319 EOF
 320 rbd snap create ${POOL}/${IMAGE}@quiesce1
 321 _sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
 322 test "$(cat ${TEMPDIR}/quiesce)" = ${DEV}
 323 test "$(cat ${TEMPDIR}/unquiesce)" = ${DEV}
 324
 325 # test snap create fails if the hook fails
 326 touch ${QUIESCE_HOOK}
 327 chmod +x ${QUIESCE_HOOK}
 328 cat > ${QUIESCE_HOOK} <<EOF
 329 #/bin/sh
 330 echo "test snap create fails if the hook fails" >&2
 331 exit 22
 332 EOF
 333 expect_false rbd snap create ${POOL}/${IMAGE}@quiesce2
 334 _sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
 335
 336 # test the hook is slow
 337 cat > ${QUIESCE_HOOK} <<EOF
 338 #/bin/sh
 339 echo "test the hook is slow" >&2
 340 sleep 7
 341 EOF
 342 rbd snap create ${POOL}/${IMAGE}@quiesce2
 343 _sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct
 344
 345 # test rbd-nbd_quiesce hook that comes with distribution
 346 unmap_device ${DEV} ${PID}
 347 LOG_FILE=${TEMPDIR}/rbd-nbd.log
 348 if [ -n "${CEPH_SRC}" ]; then
 349     QUIESCE_HOOK=${CEPH_SRC}/tools/rbd_nbd/rbd-nbd_quiesce
 350     DEV=`_sudo rbd device --device-type nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} \
 351                ${POOL}/${IMAGE} --log-file=${LOG_FILE}`
 352 else
 353     DEV=`_sudo rbd device --device-type nbd map --quiesce ${POOL}/${IMAGE} --log-file=${LOG_FILE}`
 354 fi
 355 get_pid ${POOL}
 356 _sudo mkfs ${DEV}
 357 mkdir ${TEMPDIR}/mnt
 358 _sudo mount ${DEV} ${TEMPDIR}/mnt
 359 rbd snap create ${POOL}/${IMAGE}@quiesce3
 360 _sudo dd if=${DATA} of=${TEMPDIR}/mnt/test bs=1M count=1 oflag=direct
 361 _sudo umount ${TEMPDIR}/mnt
 362 unmap_device ${DEV} ${PID}
 363 DEV=
 364 cat ${LOG_FILE}
 365 expect_false grep 'quiesce failed' ${LOG_FILE}
 366
 367 # test detach/attach
 368 OUT=`_sudo rbd device --device-type nbd --options try-netlink,show-cookie map ${POOL}/${IMAGE}`
 369 read DEV COOKIE <<< "${OUT}"
 370 get_pid ${POOL}
 371 _sudo mount ${DEV} ${TEMPDIR}/mnt
 372 _sudo rbd device detach ${POOL}/${IMAGE} --device-type nbd
 373 expect_false get_pid ${POOL}
 374 expect_false _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd
 375 if [ -n "${COOKIE}" ]; then
 376     _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} ${POOL}/${IMAGE} --device-type nbd
 377 else
 378     _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force
 379 fi
 380 get_pid ${POOL}
 381 _sudo rbd device detach ${DEV} --device-type nbd
 382 expect_false get_pid ${POOL}
 383 if [ -n "${COOKIE}" ]; then
 384     _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} ${POOL}/${IMAGE} --device-type nbd
 385 else
 386     _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force
 387 fi
 388 get_pid ${POOL}
 389 ls ${TEMPDIR}/mnt/
 390 dd if=${TEMPDIR}/mnt/test of=/dev/null bs=1M count=1
 391 _sudo dd if=${DATA} of=${TEMPDIR}/mnt/test1 bs=1M count=1 oflag=direct
 392 _sudo umount ${TEMPDIR}/mnt
 393 unmap_device ${DEV} ${PID}
 394 # if kernel supports cookies
 395 if [ -n "${COOKIE}" ]; then
 396     OUT=`_sudo rbd device --device-type nbd --show-cookie --cookie "abc de" --options try-netlink map ${POOL}/${IMAGE}`
 397     read DEV ANOTHER_COOKIE <<< "${OUT}"
 398     get_pid ${POOL}
 399     test "${ANOTHER_COOKIE}" = "abc de"
 400     unmap_device ${DEV} ${PID}
 401 fi
 402
 403 echo OK