ceph/qa/workunits/rbd/krbd_exclusive_option.sh

   1 #!/usr/bin/env bash
   2
   3 set -ex
   4
   5 function expect_false() {
   6     if "$@"; then return 1; else return 0; fi
   7 }
   8
   9 function assert_locked() {
  10     local dev_id="${1#/dev/rbd}"
  11
  12     local client_addr
  13     client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)"
  14
  15     local client_id
  16     client_id="$(< $SYSFS_DIR/$dev_id/client_id)"
  17     # client4324 -> client.4324
  18     client_id="client.${client_id#client}"
  19
  20     local watch_cookie
  21     watch_cookie="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID |
  22         grep $client_id | cut -d ' ' -f 3 | cut -d '=' -f 2)"
  23     [[ $(echo -n "$watch_cookie" | grep -c '^') -eq 1 ]]
  24
  25     local actual
  26     actual="$(rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock |
  27         python -m json.tool)"
  28
  29     local expected
  30     expected="$(cat <<EOF | python -m json.tool
  31 {
  32     "lockers": [
  33         {
  34             "addr": "$client_addr",
  35             "cookie": "auto $watch_cookie",
  36             "description": "",
  37             "expiration": "0.000000",
  38             "name": "$client_id"
  39         }
  40     ],
  41     "name": "rbd_lock",
  42     "tag": "internal",
  43     "type": "exclusive"
  44 }
  45 EOF
  46     )"
  47
  48     [ "$actual" = "$expected" ]
  49 }
  50
  51 function assert_unlocked() {
  52     rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock |
  53         grep '"lockers":\[\]'
  54 }
  55
  56 function blacklist_add() {
  57     local dev_id="${1#/dev/rbd}"
  58
  59     local client_addr
  60     client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)"
  61
  62     ceph osd blacklist add $client_addr
  63 }
  64
  65 SYSFS_DIR="/sys/bus/rbd/devices"
  66 IMAGE_NAME="exclusive-option-test"
  67
  68 rbd create --size 1 --image-feature '' $IMAGE_NAME
  69
  70 IMAGE_ID="$(rbd info --format=json $IMAGE_NAME |
  71     python -c "import sys, json; print json.load(sys.stdin)['block_name_prefix'].split('.')[1]")"
  72
  73 DEV=$(sudo rbd map $IMAGE_NAME)
  74 assert_unlocked
  75 sudo rbd unmap $DEV
  76 assert_unlocked
  77
  78 expect_false sudo rbd map -o exclusive $IMAGE_NAME
  79 assert_unlocked
  80
  81 expect_false sudo rbd map -o lock_on_read $IMAGE_NAME
  82 assert_unlocked
  83
  84 rbd feature enable $IMAGE_NAME exclusive-lock
  85 rbd snap create $IMAGE_NAME@snap
  86
  87 DEV=$(sudo rbd map $IMAGE_NAME)
  88 assert_locked $DEV
  89 [[ $(blockdev --getro $DEV) -eq 0 ]]
  90 sudo rbd unmap $DEV
  91 assert_unlocked
  92
  93 DEV=$(sudo rbd map $IMAGE_NAME@snap)
  94 assert_unlocked
  95 [[ $(blockdev --getro $DEV) -eq 1 ]]
  96 sudo rbd unmap $DEV
  97 assert_unlocked
  98
  99 DEV=$(sudo rbd map -o ro $IMAGE_NAME)
 100 assert_unlocked
 101 [[ $(blockdev --getro $DEV) -eq 1 ]]
 102 sudo rbd unmap $DEV
 103 assert_unlocked
 104
 105 DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
 106 assert_locked $DEV
 107 [[ $(blockdev --getro $DEV) -eq 0 ]]
 108 sudo rbd unmap $DEV
 109 assert_unlocked
 110
 111 DEV=$(sudo rbd map -o exclusive $IMAGE_NAME@snap)
 112 assert_unlocked
 113 [[ $(blockdev --getro $DEV) -eq 1 ]]
 114 sudo rbd unmap $DEV
 115 assert_unlocked
 116
 117 DEV=$(sudo rbd map -o exclusive,ro $IMAGE_NAME)
 118 assert_unlocked
 119 [[ $(blockdev --getro $DEV) -eq 1 ]]
 120 sudo rbd unmap $DEV
 121 assert_unlocked
 122
 123 # alternate syntax
 124 DEV=$(sudo rbd map --exclusive --read-only $IMAGE_NAME)
 125 assert_unlocked
 126 [[ $(blockdev --getro $DEV) -eq 1 ]]
 127 sudo rbd unmap $DEV
 128 assert_unlocked
 129
 130 DEV=$(sudo rbd map $IMAGE_NAME)
 131 assert_locked $DEV
 132 OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME)
 133 assert_locked $OTHER_DEV
 134 dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
 135 assert_locked $DEV
 136 dd if=/dev/urandom of=$OTHER_DEV bs=4k count=10 oflag=direct
 137 assert_locked $OTHER_DEV
 138 sudo rbd unmap $DEV
 139 sudo rbd unmap $OTHER_DEV
 140 assert_unlocked
 141
 142 DEV=$(sudo rbd map $IMAGE_NAME)
 143 assert_locked $DEV
 144 OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME)
 145 assert_locked $OTHER_DEV
 146 dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
 147 expect_false dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
 148 assert_locked $OTHER_DEV
 149 sudo rbd unmap $OTHER_DEV
 150 assert_unlocked
 151 dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
 152 assert_unlocked
 153 dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
 154 assert_locked $DEV
 155 sudo rbd unmap $DEV
 156 assert_unlocked
 157
 158 DEV=$(sudo rbd map -o lock_on_read $IMAGE_NAME)
 159 assert_locked $DEV
 160 OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME)
 161 assert_locked $OTHER_DEV
 162 expect_false dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
 163 expect_false dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
 164 sudo udevadm settle
 165 assert_locked $OTHER_DEV
 166 sudo rbd unmap $OTHER_DEV
 167 assert_unlocked
 168 dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
 169 assert_locked $DEV
 170 dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
 171 assert_locked $DEV
 172 sudo rbd unmap $DEV
 173 assert_unlocked
 174
 175 DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
 176 assert_locked $DEV
 177 expect_false sudo rbd map -o noshare $IMAGE_NAME
 178 assert_locked $DEV
 179 sudo rbd unmap $DEV
 180 assert_unlocked
 181
 182 DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
 183 assert_locked $DEV
 184 expect_false sudo rbd map -o noshare,exclusive $IMAGE_NAME
 185 assert_locked $DEV
 186 sudo rbd unmap $DEV
 187 assert_unlocked
 188
 189 DEV=$(sudo rbd map $IMAGE_NAME)
 190 assert_locked $DEV
 191 rbd resize --size 1G $IMAGE_NAME
 192 assert_unlocked
 193 sudo rbd unmap $DEV
 194 assert_unlocked
 195
 196 DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
 197 assert_locked $DEV
 198 expect_false rbd resize --size 2G $IMAGE_NAME
 199 assert_locked $DEV
 200 sudo rbd unmap $DEV
 201 assert_unlocked
 202
 203 DEV=$(sudo rbd map $IMAGE_NAME)
 204 assert_locked $DEV
 205 dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
 206 { sleep 10; blacklist_add $DEV; } &
 207 PID=$!
 208 expect_false dd if=/dev/urandom of=$DEV bs=4k count=200000 oflag=direct
 209 wait $PID
 210 # break lock
 211 OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME)
 212 assert_locked $OTHER_DEV
 213 sudo rbd unmap $DEV
 214 assert_locked $OTHER_DEV
 215 sudo rbd unmap $OTHER_DEV
 216 assert_unlocked
 217
 218 # induce a watch error after 30 seconds
 219 DEV=$(sudo rbd map -o exclusive,osdkeepalive=60 $IMAGE_NAME)
 220 assert_locked $DEV
 221 OLD_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)"
 222 sleep 40
 223 assert_locked $DEV
 224 NEW_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)"
 225 # same client_id, old cookie < new cookie
 226 [ "$(echo "$OLD_WATCHER" | cut -d ' ' -f 2)" = \
 227     "$(echo "$NEW_WATCHER" | cut -d ' ' -f 2)" ]
 228 [[ $(echo "$OLD_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) -lt \
 229     $(echo "$NEW_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) ]]
 230 sudo rbd unmap $DEV
 231 assert_unlocked
 232
 233 echo OK