ceph/qa/workunits/rbd/krbd_data_pool.sh

   1 #!/bin/bash
   2
   3 set -ex
   4
   5 function fill_image() {
   6     local spec=$1
   7
   8     local dev
   9     dev=$(sudo rbd map $spec)
  10     xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 -W 0 $IMAGE_SIZE" $dev
  11     sudo rbd unmap $dev
  12 }
  13
  14 function create_clones() {
  15     local spec=$1
  16
  17     rbd snap create $spec@snap
  18     rbd snap protect $spec@snap
  19
  20     local pool=${spec%/*}  # pool/image is assumed
  21     local image=${spec#*/}
  22     local child_pool
  23     for child_pool in $pool clonesonly; do
  24         rbd clone $spec@snap $child_pool/$pool-$image-clone1
  25         rbd clone $spec@snap --data-pool repdata $child_pool/$pool-$image-clone2
  26         rbd clone $spec@snap --data-pool ecdata $child_pool/$pool-$image-clone3
  27     done
  28 }
  29
  30 function trigger_copyup() {
  31     local spec=$1
  32
  33     local dev
  34     dev=$(sudo rbd map $spec)
  35     local i
  36     {
  37         for ((i = 0; i < $NUM_OBJECTS; i++)); do
  38             echo pwrite -b $OBJECT_SIZE -S 0x59 $((i * OBJECT_SIZE + OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))
  39         done
  40         echo fsync
  41         echo quit
  42     } | xfs_io $dev
  43     sudo rbd unmap $dev
  44 }
  45
  46 function compare() {
  47     local spec=$1
  48     local object=$2
  49
  50     local dev
  51     dev=$(sudo rbd map $spec)
  52     local i
  53     for ((i = 0; i < $NUM_OBJECTS; i++)); do
  54         dd if=$dev bs=$OBJECT_SIZE count=1 skip=$i | cmp $object -
  55     done
  56     sudo rbd unmap $dev
  57 }
  58
  59 function mkfs_and_mount() {
  60     local spec=$1
  61
  62     local dev
  63     dev=$(sudo rbd map $spec)
  64     mkfs.ext4 -q -E discard $dev
  65     sudo mount $dev /mnt
  66     sudo umount /mnt
  67     sudo rbd unmap $dev
  68 }
  69
  70 function list_HEADs() {
  71     local pool=$1
  72
  73     rados -p $pool ls | while read obj; do
  74         if rados -p $pool stat $obj >/dev/null 2>&1; then
  75             echo $obj
  76         fi
  77     done
  78 }
  79
  80 function count_data_objects() {
  81     local spec=$1
  82
  83     local pool
  84     pool=$(rbd info $spec | grep 'data_pool: ' | awk '{ print $NF }')
  85     if [[ -z $pool ]]; then
  86         pool=${spec%/*}  # pool/image is assumed
  87     fi
  88
  89     local prefix
  90     prefix=$(rbd info $spec | grep 'block_name_prefix: ' | awk '{ print $NF }')
  91     rados -p $pool ls | grep -c $prefix
  92 }
  93
  94 function get_num_clones() {
  95     local pool=$1
  96
  97     rados -p $pool --format=json df |
  98         python -c 'import sys, json; print json.load(sys.stdin)["pools"][0]["num_object_clones"]'
  99 }
 100
 101 ceph osd pool create repdata 24 24
 102 rbd pool init repdata
 103 ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
 104 ceph osd pool create ecdata 24 24 erasure teuthologyprofile
 105 rbd pool init ecdata
 106 ceph osd pool set ecdata allow_ec_overwrites true
 107 ceph osd pool create rbdnonzero 24 24
 108 rbd pool init rbdnonzero
 109 ceph osd pool create clonesonly 24 24
 110 rbd pool init clonesonly
 111
 112 for pool in rbd rbdnonzero; do
 113     rbd create --size 200 --image-format 1 $pool/img0
 114     rbd create --size 200 $pool/img1
 115     rbd create --size 200 --data-pool repdata $pool/img2
 116     rbd create --size 200 --data-pool ecdata $pool/img3
 117 done
 118
 119 IMAGE_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["size"]')
 120 OBJECT_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["object_size"]')
 121 NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
 122 [[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
 123
 124 OBJECT_X=$(mktemp)   # xxxx
 125 xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $OBJECT_SIZE" $OBJECT_X
 126
 127 OBJECT_XY=$(mktemp)  # xxYY
 128 xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $((OBJECT_SIZE / 2))" \
 129        -c "pwrite -b $OBJECT_SIZE -S 0x59 $((OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))" \
 130        $OBJECT_XY
 131
 132 for pool in rbd rbdnonzero; do
 133     for i in {0..3}; do
 134         fill_image $pool/img$i
 135         if [[ $i -ne 0 ]]; then
 136             create_clones $pool/img$i
 137             for child_pool in $pool clonesonly; do
 138                 for j in {1..3}; do
 139                     trigger_copyup $child_pool/$pool-img$i-clone$j
 140                 done
 141             done
 142         fi
 143     done
 144 done
 145
 146 # rbd_directory, rbd_children, rbd_info + img0 header + ...
 147 NUM_META_RBDS=$((3 + 1 + 3 * (1*2 + 3*2)))
 148 # rbd_directory, rbd_children, rbd_info + ...
 149 NUM_META_CLONESONLY=$((3 + 2 * 3 * (3*2)))
 150
 151 [[ $(rados -p rbd ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 152 [[ $(rados -p repdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
 153 [[ $(rados -p ecdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
 154 [[ $(rados -p rbdnonzero ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 155 [[ $(rados -p clonesonly ls | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
 156
 157 for pool in rbd rbdnonzero; do
 158     for i in {0..3}; do
 159         [[ $(count_data_objects $pool/img$i) -eq $NUM_OBJECTS ]]
 160         if [[ $i -ne 0 ]]; then
 161             for child_pool in $pool clonesonly; do
 162                 for j in {1..3}; do
 163                     [[ $(count_data_objects $child_pool/$pool-img$i-clone$j) -eq $NUM_OBJECTS ]]
 164                 done
 165             done
 166         fi
 167     done
 168 done
 169
 170 [[ $(get_num_clones rbd) -eq 0 ]]
 171 [[ $(get_num_clones repdata) -eq 0 ]]
 172 [[ $(get_num_clones ecdata) -eq 0 ]]
 173 [[ $(get_num_clones rbdnonzero) -eq 0 ]]
 174 [[ $(get_num_clones clonesonly) -eq 0 ]]
 175
 176 for pool in rbd rbdnonzero; do
 177     for i in {0..3}; do
 178         compare $pool/img$i $OBJECT_X
 179         mkfs_and_mount $pool/img$i
 180         if [[ $i -ne 0 ]]; then
 181             for child_pool in $pool clonesonly; do
 182                 for j in {1..3}; do
 183                     compare $child_pool/$pool-img$i-clone$j $OBJECT_XY
 184                 done
 185             done
 186         fi
 187     done
 188 done
 189
 190 # mkfs should discard some objects everywhere but in clonesonly
 191 [[ $(list_HEADs rbd | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 192 [[ $(list_HEADs repdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
 193 [[ $(list_HEADs ecdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
 194 [[ $(list_HEADs rbdnonzero | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 195 [[ $(list_HEADs clonesonly | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
 196
 197 [[ $(get_num_clones rbd) -eq $NUM_OBJECTS ]]
 198 [[ $(get_num_clones repdata) -eq $((2 * NUM_OBJECTS)) ]]
 199 [[ $(get_num_clones ecdata) -eq $((2 * NUM_OBJECTS)) ]]
 200 [[ $(get_num_clones rbdnonzero) -eq $NUM_OBJECTS ]]
 201 [[ $(get_num_clones clonesonly) -eq 0 ]]
 202
 203 echo OK