ceph/src/rocksdb/tools/check_format_compatible.sh

   1 #!/usr/bin/env bash
   2 #
   3 # A shell script to load some pre generated data file to a DB using ldb tool
   4 # ./ldb needs to be avaible to be executed.
   5 #
   6 # Usage: <SCRIPT> [checkout]
   7 # `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous branches (or tags for very old versions without branch) can be opened by it.
   8 # Return value 0 means all regression tests pass. 1 if not pass.
   9
  10 scriptpath=`dirname $BASH_SOURCE`
  11 test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check"
  12 script_copy_dir=$test_dir"/script_copy"
  13 input_data_path=$test_dir"/test_data_input/"
  14
  15 mkdir $test_dir || true
  16 mkdir $input_data_path || true
  17 rm -rf $script_copy_dir
  18 cp $scriptpath $script_copy_dir -rf
  19
  20 # Generate random files.
  21 for i in {1..6}
  22 do
  23   input_data[$i]=$input_data_path/data$i
  24   echo == Generating random input file ${input_data[$i]}
  25   python - <<EOF
  26 import random
  27 random.seed($i)
  28 symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
  29 with open('${input_data[$i]}', 'w') as f:
  30   for i in range(1,1024):
  31     k = ""
  32     for j in range(1, random.randint(1,32)):
  33       k=k + symbols[random.randint(0, len(symbols) - 1)]
  34     vb = ""
  35     for j in range(1, random.randint(0,128)):
  36       vb = vb + symbols[random.randint(0, len(symbols) - 1)]
  37     v = ""
  38     for j in range(1, random.randint(1, 5)):
  39       v = v + vb
  40     print >> f, k + " ==> " + v
  41 EOF
  42 done
  43
  44 # Generate file(s) with sorted keys.
  45 sorted_input_data=$input_data_path/sorted_data
  46 echo == Generating file with sorted keys ${sorted_input_data}
  47 python - <<EOF
  48 with open('${sorted_input_data}', 'w') as f:
  49   for i in range(0,10):
  50     k = str(i)
  51     v = "value" + k
  52     print >> f, k + " ==> " + v
  53 EOF
  54
  55 declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb")
  56 declare -a forward_compatible_checkout_objs=("3.10.fb" "3.11.fb" "3.12.fb" "3.13.fb" "4.0.fb" "4.1.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb")
  57 declare -a forward_compatible_with_options_checkout_objs=("5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb")
  58 declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]})
  59 declare -a extern_sst_ingestion_compatible_checkout_objs=("5.14.fb" "5.15.fb")
  60
  61 generate_db()
  62 {
  63     set +e
  64     $script_copy_dir/generate_random_db.sh $1 $2
  65     if [ $? -ne 0 ]; then
  66         echo ==== Error loading data from $2 to $1 ====
  67         exit 1
  68     fi
  69     set -e
  70 }
  71
  72 compare_db()
  73 {
  74     set +e
  75     $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 $5
  76     if [ $? -ne 0 ]; then
  77         echo ==== Read different content from $1 and $2 or error happened. ====
  78         exit 1
  79     fi
  80     set -e
  81 }
  82
  83 write_external_sst()
  84 {
  85     set +e
  86     $script_copy_dir/write_external_sst.sh $1 $2 $3
  87     if [ $? -ne 0 ]; then
  88         echo ==== Error writing external SST file using data from $1 to $3 ====
  89         exit 1
  90     fi
  91     set -e
  92 }
  93
  94 ingest_external_sst()
  95 {
  96     set +e
  97     $script_copy_dir/ingest_external_sst.sh $1 $2
  98     if [ $? -ne 0 ]; then
  99         echo ==== Error ingesting external SST in $2 to DB at $1 ====
 100         exit 1
 101     fi
 102     set -e
 103 }
 104
 105 # Sandcastle sets us up with a remote that is just another directory on the same
 106 # machine and doesn't have our branches. Need to fetch them so checkout works.
 107 # Remote add may fail if added previously (we don't cleanup).
 108 git remote add github_origin "https://github.com/facebook/rocksdb.git"
 109 set -e
 110 https_proxy="fwdproxy:8080" git fetch github_origin
 111
 112 # Compatibility test for external SST file ingestion
 113 for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}"
 114 do
 115   echo == Generating DB with extern SST file in "$checkout_obj" ...
 116   https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj
 117   make clean
 118   make ldb -j32
 119   write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj
 120   ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj
 121 done
 122
 123 checkout_flag=${1:-"master"}
 124
 125 echo == Building $checkout_flag debug
 126 https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag
 127 make clean
 128 make ldb -j32
 129 compare_base_db_dir=$test_dir"/base_db_dir"
 130 write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir
 131 ingest_external_sst $compare_base_db_dir $compare_base_db_dir
 132
 133 for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}"
 134 do
 135   echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag
 136   git checkout $checkout_obj
 137   make clean
 138   make ldb -j32
 139   compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1
 140   git checkout tmp-$checkout_flag
 141   # Clean up
 142   git branch -D $checkout_obj
 143 done
 144
 145 echo == Finish compatibility test for SST ingestion.
 146
 147 for checkout_obj in "${checkout_objs[@]}"
 148 do
 149    echo == Generating DB from "$checkout_obj" ...
 150    https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj
 151    make clean
 152    make ldb -j32
 153    generate_db $input_data_path $test_dir/$checkout_obj
 154 done
 155
 156 checkout_flag=${1:-"master"}
 157
 158 echo == Building $checkout_flag debug
 159 git checkout tmp-$checkout_flag
 160 make clean
 161 make ldb -j32
 162 compare_base_db_dir=$test_dir"/base_db_dir"
 163 echo == Generate compare base DB to $compare_base_db_dir
 164 generate_db $input_data_path $compare_base_db_dir
 165
 166 for checkout_obj in "${checkout_objs[@]}"
 167 do
 168    echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ...
 169    compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 0
 170 done
 171
 172 for checkout_obj in "${forward_compatible_checkout_objs[@]}"
 173 do
 174    echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag...
 175    git checkout $checkout_obj
 176    make clean
 177    make ldb -j32
 178    compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0
 179 done
 180
 181 for checkout_obj in "${forward_compatible_with_options_checkout_objs[@]}"
 182 do
 183    echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag with its options...
 184    git checkout $checkout_obj
 185    make clean
 186    make ldb -j32
 187    compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 1 1
 188 done
 189
 190 echo ==== Compatibility Test PASSED ====