ceph/qa/workunits/rbd/krbd_data_pool.sh

   1 #!/bin/bash
   2
   3 set -ex
   4
   5 function fill_image() {
   6     local spec=$1
   7
   8     local dev
   9     dev=$(sudo rbd map $spec)
  10     xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 -W 0 $IMAGE_SIZE" $dev
  11     sudo rbd unmap $dev
  12 }
  13
  14 function create_clones() {
  15     local spec=$1
  16
  17     rbd snap create $spec@snap
  18     rbd snap protect $spec@snap
  19
  20     local pool=${spec%/*}  # pool/image is assumed
  21     local image=${spec#*/}
  22     local child_pool
  23     for child_pool in $pool clonesonly; do
  24         rbd clone $spec@snap $child_pool/$pool-$image-clone1
  25         rbd clone $spec@snap --data-pool repdata $child_pool/$pool-$image-clone2
  26         rbd clone $spec@snap --data-pool ecdata $child_pool/$pool-$image-clone3
  27     done
  28 }
  29
  30 function trigger_copyup() {
  31     local spec=$1
  32
  33     local dev
  34     dev=$(sudo rbd map $spec)
  35     local i
  36     {
  37         for ((i = 0; i < $NUM_OBJECTS; i++)); do
  38             echo pwrite -b $OBJECT_SIZE -S 0x59 $((i * OBJECT_SIZE + OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))
  39         done
  40         echo fsync
  41         echo quit
  42     } | xfs_io $dev
  43     sudo rbd unmap $dev
  44 }
  45
  46 function compare() {
  47     local spec=$1
  48     local object=$2
  49
  50     local dev
  51     dev=$(sudo rbd map $spec)
  52     local i
  53     for ((i = 0; i < $NUM_OBJECTS; i++)); do
  54         dd if=$dev bs=$OBJECT_SIZE count=1 skip=$i | cmp $object -
  55     done
  56     sudo rbd unmap $dev
  57 }
  58
  59 function mkfs_and_mount() {
  60     local spec=$1
  61
  62     local dev
  63     dev=$(sudo rbd map $spec)
  64     mkfs.ext4 -q -E discard $dev
  65     sudo mount $dev /mnt
  66     sudo umount /mnt
  67     sudo rbd unmap $dev
  68 }
  69
  70 function list_HEADs() {
  71     local pool=$1
  72
  73     rados -p $pool ls | while read obj; do
  74         if rados -p $pool stat $obj >/dev/null 2>&1; then
  75             echo $obj
  76         fi
  77     done
  78 }
  79
  80 function count_data_objects() {
  81     local spec=$1
  82
  83     local pool
  84     pool=$(rbd info $spec | grep 'data_pool: ' | awk '{ print $NF }')
  85     if [[ -z $pool ]]; then
  86         pool=${spec%/*}  # pool/image is assumed
  87     fi
  88
  89     local prefix
  90     prefix=$(rbd info $spec | grep 'block_name_prefix: ' | awk '{ print $NF }')
  91     rados -p $pool ls | grep -c $prefix
  92 }
  93
  94 function get_num_clones() {
  95     local pool=$1
  96
  97     rados -p $pool --format=json df |
  98         python -c 'import sys, json; print json.load(sys.stdin)["pools"][0]["num_object_clones"]'
  99 }
 100
 101 ceph osd pool create repdata 24 24
 102 ceph osd erasure-code-profile set teuthologyprofile ruleset-failure-domain=osd m=1 k=2
 103 ceph osd pool create ecdata 24 24 erasure teuthologyprofile
 104 ceph osd pool set ecdata allow_ec_overwrites true
 105 ceph osd pool create rbdnonzero 24 24
 106 ceph osd pool create clonesonly 24 24
 107
 108 for pool in rbd rbdnonzero; do
 109     rbd create --size 200 --image-format 1 $pool/img0
 110     rbd create --size 200 $pool/img1
 111     rbd create --size 200 --data-pool repdata $pool/img2
 112     rbd create --size 200 --data-pool ecdata $pool/img3
 113 done
 114
 115 IMAGE_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["size"]')
 116 OBJECT_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["object_size"]')
 117 NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
 118 [[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
 119
 120 OBJECT_X=$(mktemp)   # xxxx
 121 xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $OBJECT_SIZE" $OBJECT_X
 122
 123 OBJECT_XY=$(mktemp)  # xxYY
 124 xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $((OBJECT_SIZE / 2))" \
 125        -c "pwrite -b $OBJECT_SIZE -S 0x59 $((OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))" \
 126        $OBJECT_XY
 127
 128 for pool in rbd rbdnonzero; do
 129     for i in {0..3}; do
 130         fill_image $pool/img$i
 131         if [[ $i -ne 0 ]]; then
 132             create_clones $pool/img$i
 133             for child_pool in $pool clonesonly; do
 134                 for j in {1..3}; do
 135                     trigger_copyup $child_pool/$pool-img$i-clone$j
 136                 done
 137             done
 138         fi
 139     done
 140 done
 141
 142 # rbd_directory, rbd_children, rbd_info + img0 header + ...
 143 NUM_META_RBDS=$((3 + 1 + 3 * (1*2 + 3*2)))
 144 # rbd_directory, rbd_children, rbd_info + ...
 145 NUM_META_CLONESONLY=$((3 + 2 * 3 * (3*2)))
 146
 147 [[ $(rados -p rbd ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 148 [[ $(rados -p repdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
 149 [[ $(rados -p ecdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
 150 [[ $(rados -p rbdnonzero ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 151 [[ $(rados -p clonesonly ls | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
 152
 153 for pool in rbd rbdnonzero; do
 154     for i in {0..3}; do
 155         [[ $(count_data_objects $pool/img$i) -eq $NUM_OBJECTS ]]
 156         if [[ $i -ne 0 ]]; then
 157             for child_pool in $pool clonesonly; do
 158                 for j in {1..3}; do
 159                     [[ $(count_data_objects $child_pool/$pool-img$i-clone$j) -eq $NUM_OBJECTS ]]
 160                 done
 161             done
 162         fi
 163     done
 164 done
 165
 166 [[ $(get_num_clones rbd) -eq 0 ]]
 167 [[ $(get_num_clones repdata) -eq 0 ]]
 168 [[ $(get_num_clones ecdata) -eq 0 ]]
 169 [[ $(get_num_clones rbdnonzero) -eq 0 ]]
 170 [[ $(get_num_clones clonesonly) -eq 0 ]]
 171
 172 for pool in rbd rbdnonzero; do
 173     for i in {0..3}; do
 174         compare $pool/img$i $OBJECT_X
 175         mkfs_and_mount $pool/img$i
 176         if [[ $i -ne 0 ]]; then
 177             for child_pool in $pool clonesonly; do
 178                 for j in {1..3}; do
 179                     compare $child_pool/$pool-img$i-clone$j $OBJECT_XY
 180                 done
 181             done
 182         fi
 183     done
 184 done
 185
 186 # mkfs should discard some objects everywhere but in clonesonly
 187 [[ $(list_HEADs rbd | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 188 [[ $(list_HEADs repdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
 189 [[ $(list_HEADs ecdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
 190 [[ $(list_HEADs rbdnonzero | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 191 [[ $(list_HEADs clonesonly | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
 192
 193 [[ $(get_num_clones rbd) -eq $NUM_OBJECTS ]]
 194 [[ $(get_num_clones repdata) -eq $((2 * NUM_OBJECTS)) ]]
 195 [[ $(get_num_clones ecdata) -eq $((2 * NUM_OBJECTS)) ]]
 196 [[ $(get_num_clones rbdnonzero) -eq $NUM_OBJECTS ]]
 197 [[ $(get_num_clones clonesonly) -eq 0 ]]
 198
 199 echo OK