ceph/qa/workunits/rados/test_dedup_tool.sh

   1 #!/usr/bin/env bash
   2
   3 set -x
   4
   5 die() {
   6     echo "$@"
   7     exit 1
   8 }
   9
  10 do_run() {
  11     if [ "$1" == "--tee" ]; then
  12       shift
  13       tee_out="$1"
  14       shift
  15       "$@" | tee $tee_out
  16     else
  17       "$@"
  18     fi
  19 }
  20
  21 run_expect_succ() {
  22     echo "RUN_EXPECT_SUCC: " "$@"
  23     do_run "$@"
  24     [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@"
  25 }
  26
  27 run() {
  28     echo "RUN: " $@
  29     do_run "$@"
  30 }
  31
  32 if [ -n "$CEPH_BIN" ] ; then
  33    # CMake env
  34    RADOS_TOOL="$CEPH_BIN/rados"
  35    CEPH_TOOL="$CEPH_BIN/ceph"
  36    DEDUP_TOOL="$CEPH_BIN/ceph-dedup-tool"
  37 else
  38    # executables should be installed by the QA env
  39    RADOS_TOOL=$(which rados)
  40    CEPH_TOOL=$(which ceph)
  41    DEDUP_TOOL=$(which ceph-dedup-tool)
  42 fi
  43
  44 POOL=dedup_pool
  45 OBJ=test_rados_obj
  46
  47 [ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test"
  48 [ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test"
  49
  50 run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
  51
  52 function test_dedup_ratio_fixed()
  53 {
  54   # case 1
  55   dd if=/dev/urandom of=dedup_object_1k bs=1K count=1
  56   dd if=dedup_object_1k of=dedup_object_100k bs=1K count=100
  57
  58   $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k
  59   RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024  --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
  60   if [ 1024 -ne $RESULT ];
  61   then
  62     die "Estimate failed expecting 1024 result $RESULT"
  63   fi
  64
  65   # case 2
  66   dd if=/dev/zero of=dedup_object_10m bs=10M count=1
  67
  68   $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m
  69   RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096  --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
  70   if [ 4096 -ne $RESULT ];
  71   then
  72     die "Estimate failed expecting 4096 result $RESULT"
  73   fi
  74
  75   # case 3 max_thread
  76   for num in `seq 0 20`
  77   do
  78     dd if=/dev/zero of=dedup_object_$num bs=4M count=1
  79     $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num
  80   done
  81
  82   RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096  --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}')
  83
  84   if [ 98566144 -ne $RESULT ];
  85   then
  86     die "Estimate failed expecting 98566144 result $RESULT"
  87   fi
  88
  89   rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m
  90   for num in `seq 0 20`
  91   do
  92     rm -rf ./dedup_object_$num
  93   done
  94   $RADOS_TOOL -p $POOL rm $OBJ
  95   for num in `seq 0 20`
  96   do
  97     $RADOS_TOOL -p $POOL rm dedup_object_$num
  98   done
  99 }
 100
 101 function test_dedup_chunk_scrub()
 102 {
 103
 104   CHUNK_POOL=dedup_chunk_pool
 105   run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8
 106
 107   echo "hi there" > foo
 108
 109   echo "hi there" > bar
 110
 111   echo "there" > foo-chunk
 112
 113   echo "CHUNK" > bar-chunk
 114
 115   $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it
 116
 117   $RADOS_TOOL -p $POOL put foo ./foo
 118   $RADOS_TOOL -p $POOL put bar ./bar
 119
 120   $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk
 121   $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk
 122
 123   $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference
 124   $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference
 125
 126   echo "There hi" > test_obj
 127   # dirty
 128   $RADOS_TOOL -p $POOL put foo ./test_obj
 129   # flush
 130   $RADOS_TOOL -p $POOL put foo ./test_obj
 131   sleep 2
 132
 133   rados ls -p $CHUNK_POOL
 134   CHUNK_OID=$(echo -n "There hi" | sha1sum)
 135
 136   POOL_ID=$(ceph osd pool ls detail | grep $POOL |  awk '{print$2}')
 137   $DEDUP_TOOL --op add-chunk-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref bar --target-ref-pool-id $POOL_ID
 138   RESULT=$($DEDUP_TOOL --op get-chunk-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID)
 139
 140   $DEDUP_TOOL --op chunk-scrub --chunk-pool $CHUNK_POOL
 141
 142   RESULT=$($DEDUP_TOOL --op get-chunk-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar)
 143   if [ -n "$RESULT" ] ; then
 144     $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
 145     $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
 146     die "Scrub failed expecting bar is removed"
 147   fi
 148
 149   $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
 150
 151   rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj
 152   $RADOS_TOOL -p $POOL rm foo
 153   $RADOS_TOOL -p $POOL rm bar
 154 }
 155
 156 function test_dedup_ratio_rabin()
 157 {
 158   # case 1
 159   echo "abcdefghijklmnop" >> dedup_16
 160   for num in `seq 0 63`
 161   do
 162     dd if=./dedup_16  bs=16 count=1 >> dedup_object_1k
 163   done
 164
 165   for num in `seq 0 11`
 166   do
 167     dd if=dedup_object_1k bs=1K count=1 >> test_rabin_object
 168   done
 169   $RADOS_TOOL -p $POOL put $OBJ ./test_rabin_object
 170   RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --min-chunk 1015  --chunk-algorithm rabin --fingerprint-algorithm rabin --debug | grep result -a | awk '{print$4}')
 171   if [ 4096 -ne $RESULT ];
 172   then
 173     die "Estimate failed expecting 4096 result $RESULT"
 174   fi
 175
 176   echo "a" >> test_rabin_object_2
 177   dd if=./test_rabin_object bs=8K count=1 >> test_rabin_object_2
 178   $RADOS_TOOL -p $POOL put $OBJ"_2" ./test_rabin_object_2
 179   RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --min-chunk 1012 --chunk-algorithm rabin --fingerprint-algorithm rabin --debug | grep result -a | awk '{print$4}')
 180   if [ 11259 -ne $RESULT ];
 181   then
 182     die "Estimate failed expecting 11259 result $RESULT"
 183   fi
 184
 185   RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --min-chunk 1024 --chunk-mask-bit 3  --chunk-algorithm rabin --fingerprint-algorithm rabin --debug | grep result -a | awk '{print$4}')
 186   if [ 7170 -ne $RESULT ];
 187   then
 188     die "Estimate failed expecting 7170 result $RESULT"
 189   fi
 190
 191   rm -rf ./dedup_object_1k ./test_rabin_object ./test_rabin_object_2 ./dedup_16
 192
 193 }
 194
 195 test_dedup_ratio_fixed
 196 test_dedup_chunk_scrub
 197 test_dedup_ratio_rabin
 198
 199 $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
 200
 201 echo "SUCCESS!"
 202 exit 0
 203
 204