ceph/src/rocksdb/tools/check_format_compatible.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
   3 #
   4 # A shell script to load some pre generated data file to a DB using ldb tool
   5 # ./ldb needs to be avaible to be executed.
   6 #
   7 # Usage: <SCRIPT> [checkout]
   8 # `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous branches (or tags for very old versions without branch) can be opened by it.
   9 # Return value 0 means all regression tests pass. 1 if not pass.
  10
  11 scriptpath=`dirname $BASH_SOURCE`
  12 test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check"
  13 script_copy_dir=$test_dir"/script_copy"
  14 input_data_path=$test_dir"/test_data_input/"
  15
  16 mkdir $test_dir || true
  17 mkdir $input_data_path || true
  18 rm -rf $script_copy_dir
  19 cp $scriptpath $script_copy_dir -rf
  20
  21 # Generate random files.
  22 for i in {1..6}
  23 do
  24   input_data[$i]=$input_data_path/data$i
  25   echo == Generating random input file ${input_data[$i]}
  26   python - <<EOF
  27 import random
  28 random.seed($i)
  29 symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
  30 with open('${input_data[$i]}', 'w') as f:
  31   for i in range(1,1024):
  32     k = ""
  33     for j in range(1, random.randint(1,32)):
  34       k=k + symbols[random.randint(0, len(symbols) - 1)]
  35     vb = ""
  36     for j in range(1, random.randint(0,128)):
  37       vb = vb + symbols[random.randint(0, len(symbols) - 1)]
  38     v = ""
  39     for j in range(1, random.randint(1, 5)):
  40       v = v + vb
  41     print >> f, k + " ==> " + v
  42 EOF
  43 done
  44
  45 # Generate file(s) with sorted keys.
  46 sorted_input_data=$input_data_path/sorted_data
  47 echo == Generating file with sorted keys ${sorted_input_data}
  48 python - <<EOF
  49 with open('${sorted_input_data}', 'w') as f:
  50   for i in range(0,10):
  51     k = str(i)
  52     v = "value" + k
  53     print >> f, k + " ==> " + v
  54 EOF
  55
  56 declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb")
  57 declare -a forward_compatible_checkout_objs=("4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb")
  58 declare -a forward_compatible_with_options_checkout_objs=("5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb" "5.15.fb" "5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb")
  59 declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]})
  60 declare -a extern_sst_ingestion_compatible_checkout_objs=("5.14.fb" "5.15.fb" "5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb")
  61
  62 generate_db()
  63 {
  64     set +e
  65     $script_copy_dir/generate_random_db.sh $1 $2
  66     if [ $? -ne 0 ]; then
  67         echo ==== Error loading data from $2 to $1 ====
  68         exit 1
  69     fi
  70     set -e
  71 }
  72
  73 compare_db()
  74 {
  75     set +e
  76     $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 $5
  77     if [ $? -ne 0 ]; then
  78         echo ==== Read different content from $1 and $2 or error happened. ====
  79         exit 1
  80     fi
  81     set -e
  82 }
  83
  84 write_external_sst()
  85 {
  86     set +e
  87     $script_copy_dir/write_external_sst.sh $1 $2 $3
  88     if [ $? -ne 0 ]; then
  89         echo ==== Error writing external SST file using data from $1 to $3 ====
  90         exit 1
  91     fi
  92     set -e
  93 }
  94
  95 ingest_external_sst()
  96 {
  97     set +e
  98     $script_copy_dir/ingest_external_sst.sh $1 $2
  99     if [ $? -ne 0 ]; then
 100         echo ==== Error ingesting external SST in $2 to DB at $1 ====
 101         exit 1
 102     fi
 103     set -e
 104 }
 105
 106 # Sandcastle sets us up with a remote that is just another directory on the same
 107 # machine and doesn't have our branches. Need to fetch them so checkout works.
 108 # Remote add may fail if added previously (we don't cleanup).
 109 git remote add github_origin "https://github.com/facebook/rocksdb.git"
 110 set -e
 111 https_proxy="fwdproxy:8080" git fetch github_origin
 112
 113 # Compatibility test for external SST file ingestion
 114 for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}"
 115 do
 116   echo == Generating DB with extern SST file in "$checkout_obj" ...
 117   https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj
 118   make clean
 119   make ldb -j32
 120   write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj
 121   ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj
 122 done
 123
 124 checkout_flag=${1:-"master"}
 125
 126 echo == Building $checkout_flag debug
 127 https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag
 128 make clean
 129 make ldb -j32
 130 compare_base_db_dir=$test_dir"/base_db_dir"
 131 write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir
 132 ingest_external_sst $compare_base_db_dir $compare_base_db_dir
 133
 134 for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}"
 135 do
 136   echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag
 137   git checkout $checkout_obj
 138   make clean
 139   make ldb -j32
 140   compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1
 141   git checkout tmp-$checkout_flag
 142   # Clean up
 143   git branch -D $checkout_obj
 144 done
 145
 146 echo == Finish compatibility test for SST ingestion.
 147
 148 for checkout_obj in "${checkout_objs[@]}"
 149 do
 150    echo == Generating DB from "$checkout_obj" ...
 151    https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj
 152    make clean
 153    make ldb -j32
 154    generate_db $input_data_path $test_dir/$checkout_obj
 155 done
 156
 157 checkout_flag=${1:-"master"}
 158
 159 echo == Building $checkout_flag debug
 160 git checkout tmp-$checkout_flag
 161 make clean
 162 make ldb -j32
 163 compare_base_db_dir=$test_dir"/base_db_dir"
 164 echo == Generate compare base DB to $compare_base_db_dir
 165 generate_db $input_data_path $compare_base_db_dir
 166
 167 for checkout_obj in "${checkout_objs[@]}"
 168 do
 169    echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ...
 170    compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 0
 171 done
 172
 173 for checkout_obj in "${forward_compatible_checkout_objs[@]}"
 174 do
 175    echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag...
 176    git checkout $checkout_obj
 177    make clean
 178    make ldb -j32
 179    compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0
 180 done
 181
 182 for checkout_obj in "${forward_compatible_with_options_checkout_objs[@]}"
 183 do
 184    echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag with its options...
 185    git checkout $checkout_obj
 186    make clean
 187    make ldb -j32
 188    compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 1 1
 189 done
 190
 191 echo ==== Compatibility Test PASSED ====