ceph/qa/workunits/rbd/krbd_data_pool.sh

   1 #!/usr/bin/env bash
   2
   3 set -ex
   4
   5 export RBD_FORCE_ALLOW_V1=1
   6
   7 function fill_image() {
   8     local spec=$1
   9
  10     local dev
  11     dev=$(sudo rbd map $spec)
  12     xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 -W 0 $IMAGE_SIZE" $dev
  13     sudo rbd unmap $dev
  14 }
  15
  16 function create_clones() {
  17     local spec=$1
  18
  19     rbd snap create $spec@snap
  20     rbd snap protect $spec@snap
  21
  22     local pool=${spec%/*}  # pool/image is assumed
  23     local image=${spec#*/}
  24     local child_pool
  25     for child_pool in $pool clonesonly; do
  26         rbd clone $spec@snap $child_pool/$pool-$image-clone1
  27         rbd clone $spec@snap --data-pool repdata $child_pool/$pool-$image-clone2
  28         rbd clone $spec@snap --data-pool ecdata $child_pool/$pool-$image-clone3
  29     done
  30 }
  31
  32 function trigger_copyup() {
  33     local spec=$1
  34
  35     local dev
  36     dev=$(sudo rbd map $spec)
  37     local i
  38     {
  39         for ((i = 0; i < $NUM_OBJECTS; i++)); do
  40             echo pwrite -b $OBJECT_SIZE -S 0x59 $((i * OBJECT_SIZE + OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))
  41         done
  42         echo fsync
  43         echo quit
  44     } | xfs_io $dev
  45     sudo rbd unmap $dev
  46 }
  47
  48 function compare() {
  49     local spec=$1
  50     local object=$2
  51
  52     local dev
  53     dev=$(sudo rbd map $spec)
  54     local i
  55     for ((i = 0; i < $NUM_OBJECTS; i++)); do
  56         dd if=$dev bs=$OBJECT_SIZE count=1 skip=$i | cmp $object -
  57     done
  58     sudo rbd unmap $dev
  59 }
  60
  61 function mkfs_and_mount() {
  62     local spec=$1
  63
  64     local dev
  65     dev=$(sudo rbd map $spec)
  66     blkdiscard $dev
  67     mkfs.ext4 -q -E nodiscard $dev
  68     sudo mount $dev /mnt
  69     sudo umount /mnt
  70     sudo rbd unmap $dev
  71 }
  72
  73 function list_HEADs() {
  74     local pool=$1
  75
  76     rados -p $pool ls | while read obj; do
  77         if rados -p $pool stat $obj >/dev/null 2>&1; then
  78             echo $obj
  79         fi
  80     done
  81 }
  82
  83 function count_data_objects() {
  84     local spec=$1
  85
  86     local pool
  87     pool=$(rbd info $spec | grep 'data_pool: ' | awk '{ print $NF }')
  88     if [[ -z $pool ]]; then
  89         pool=${spec%/*}  # pool/image is assumed
  90     fi
  91
  92     local prefix
  93     prefix=$(rbd info $spec | grep 'block_name_prefix: ' | awk '{ print $NF }')
  94     rados -p $pool ls | grep -c $prefix
  95 }
  96
  97 function get_num_clones() {
  98     local pool=$1
  99
 100     rados -p $pool --format=json df |
 101         python3 -c 'import sys, json; print(json.load(sys.stdin)["pools"][0]["num_object_clones"])'
 102 }
 103
 104 ceph osd pool create repdata 24 24
 105 rbd pool init repdata
 106 ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
 107 ceph osd pool create ecdata 24 24 erasure teuthologyprofile
 108 rbd pool init ecdata
 109 ceph osd pool set ecdata allow_ec_overwrites true
 110 ceph osd pool create rbdnonzero 24 24
 111 rbd pool init rbdnonzero
 112 ceph osd pool create clonesonly 24 24
 113 rbd pool init clonesonly
 114
 115 for pool in rbd rbdnonzero; do
 116     rbd create --size 200 --image-format 1 $pool/img0
 117     rbd create --size 200 $pool/img1
 118     rbd create --size 200 --data-pool repdata $pool/img2
 119     rbd create --size 200 --data-pool ecdata $pool/img3
 120 done
 121
 122 IMAGE_SIZE=$(rbd info --format=json img1 | python3 -c 'import sys, json; print(json.load(sys.stdin)["size"])')
 123 OBJECT_SIZE=$(rbd info --format=json img1 | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_size"])')
 124 NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
 125 [[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
 126
 127 OBJECT_X=$(mktemp)   # xxxx
 128 xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $OBJECT_SIZE" $OBJECT_X
 129
 130 OBJECT_XY=$(mktemp)  # xxYY
 131 xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $((OBJECT_SIZE / 2))" \
 132        -c "pwrite -b $OBJECT_SIZE -S 0x59 $((OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))" \
 133        $OBJECT_XY
 134
 135 for pool in rbd rbdnonzero; do
 136     for i in {0..3}; do
 137         fill_image $pool/img$i
 138         if [[ $i -ne 0 ]]; then
 139             create_clones $pool/img$i
 140             for child_pool in $pool clonesonly; do
 141                 for j in {1..3}; do
 142                     trigger_copyup $child_pool/$pool-img$i-clone$j
 143                 done
 144             done
 145         fi
 146     done
 147 done
 148
 149 # rbd_directory, rbd_children, rbd_info + img0 header + ...
 150 NUM_META_RBDS=$((3 + 1 + 3 * (1*2 + 3*2)))
 151 # rbd_directory, rbd_children, rbd_info + ...
 152 NUM_META_CLONESONLY=$((3 + 2 * 3 * (3*2)))
 153
 154 [[ $(rados -p rbd ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 155 [[ $(rados -p repdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
 156 [[ $(rados -p ecdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
 157 [[ $(rados -p rbdnonzero ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 158 [[ $(rados -p clonesonly ls | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
 159
 160 for pool in rbd rbdnonzero; do
 161     for i in {0..3}; do
 162         [[ $(count_data_objects $pool/img$i) -eq $NUM_OBJECTS ]]
 163         if [[ $i -ne 0 ]]; then
 164             for child_pool in $pool clonesonly; do
 165                 for j in {1..3}; do
 166                     [[ $(count_data_objects $child_pool/$pool-img$i-clone$j) -eq $NUM_OBJECTS ]]
 167                 done
 168             done
 169         fi
 170     done
 171 done
 172
 173 [[ $(get_num_clones rbd) -eq 0 ]]
 174 [[ $(get_num_clones repdata) -eq 0 ]]
 175 [[ $(get_num_clones ecdata) -eq 0 ]]
 176 [[ $(get_num_clones rbdnonzero) -eq 0 ]]
 177 [[ $(get_num_clones clonesonly) -eq 0 ]]
 178
 179 for pool in rbd rbdnonzero; do
 180     for i in {0..3}; do
 181         compare $pool/img$i $OBJECT_X
 182         mkfs_and_mount $pool/img$i
 183         if [[ $i -ne 0 ]]; then
 184             for child_pool in $pool clonesonly; do
 185                 for j in {1..3}; do
 186                     compare $child_pool/$pool-img$i-clone$j $OBJECT_XY
 187                 done
 188             done
 189         fi
 190     done
 191 done
 192
 193 # mkfs_and_mount should discard some objects everywhere but in clonesonly
 194 [[ $(list_HEADs rbd | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 195 [[ $(list_HEADs repdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
 196 [[ $(list_HEADs ecdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
 197 [[ $(list_HEADs rbdnonzero | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
 198 [[ $(list_HEADs clonesonly | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
 199
 200 [[ $(get_num_clones rbd) -eq $NUM_OBJECTS ]]
 201 [[ $(get_num_clones repdata) -eq $((2 * NUM_OBJECTS)) ]]
 202 [[ $(get_num_clones ecdata) -eq $((2 * NUM_OBJECTS)) ]]
 203 [[ $(get_num_clones rbdnonzero) -eq $NUM_OBJECTS ]]
 204 [[ $(get_num_clones clonesonly) -eq 0 ]]
 205
 206 echo OK