]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | #!/usr/bin/env bash |
f67539c2 | 2 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. |
7c673cae FG |
3 | # |
4 | # A shell script to load some pre generated data file to a DB using ldb tool | |
5 | # ./ldb needs to be avaible to be executed. | |
6 | # | |
7 | # Usage: <SCRIPT> [checkout] | |
8 | # `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous branches (or tags for very old versions without branch) can be opened by it. | |
9 | # Return value 0 means all regression tests pass. 1 if not pass. | |
10 | ||
11 | scriptpath=`dirname $BASH_SOURCE` | |
12 | test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check" | |
13 | script_copy_dir=$test_dir"/script_copy" | |
14 | input_data_path=$test_dir"/test_data_input/" | |
15 | ||
20effc67 TL |
16 | python_bin=$(which python3 || which python || echo python3) |
17 | ||
7c673cae FG |
18 | mkdir $test_dir || true |
19 | mkdir $input_data_path || true | |
20 | rm -rf $script_copy_dir | |
21 | cp $scriptpath $script_copy_dir -rf | |
22 | ||
11fdf7f2 | 23 | # Generate random files. |
7c673cae FG |
24 | for i in {1..6} |
25 | do | |
26 | input_data[$i]=$input_data_path/data$i | |
27 | echo == Generating random input file ${input_data[$i]} | |
20effc67 | 28 | $python_bin - <<EOF |
7c673cae FG |
29 | import random |
30 | random.seed($i) | |
31 | symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] | |
32 | with open('${input_data[$i]}', 'w') as f: | |
33 | for i in range(1,1024): | |
34 | k = "" | |
35 | for j in range(1, random.randint(1,32)): | |
36 | k=k + symbols[random.randint(0, len(symbols) - 1)] | |
37 | vb = "" | |
38 | for j in range(1, random.randint(0,128)): | |
39 | vb = vb + symbols[random.randint(0, len(symbols) - 1)] | |
40 | v = "" | |
41 | for j in range(1, random.randint(1, 5)): | |
42 | v = v + vb | |
20effc67 | 43 | print(k + " ==> " + v, file=f) |
7c673cae FG |
44 | EOF |
45 | done | |
46 | ||
11fdf7f2 TL |
47 | # Generate file(s) with sorted keys. |
48 | sorted_input_data=$input_data_path/sorted_data | |
49 | echo == Generating file with sorted keys ${sorted_input_data} | |
20effc67 | 50 | $python_bin - <<EOF |
11fdf7f2 TL |
51 | with open('${sorted_input_data}', 'w') as f: |
52 | for i in range(0,10): | |
53 | k = str(i) | |
54 | v = "value" + k | |
20effc67 | 55 | print(k + " ==> " + v, file=f) |
11fdf7f2 TL |
56 | EOF |
57 | ||
20effc67 TL |
58 | declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb" "5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb" "5.15.fb") |
59 | declare -a forward_compatible_checkout_objs=() # N/A at the moment | |
60 | declare -a forward_compatible_with_options_checkout_objs=("5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb" "6.6.fb" "6.7.fb" "6.8.fb" "6.9.fb" "6.10.fb" "6.11.fb" "6.12.fb" "6.13.fb" "6.14.fb") | |
11fdf7f2 | 61 | declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]}) |
20effc67 | 62 | declare -a extern_sst_ingestion_compatible_checkout_objs=("5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb" "6.6.fb" "6.7.fb" "6.8.fb" "6.9.fb" "6.10.fb" "6.11.fb" "6.12.fb" "6.13.fb" "6.14.fb") |
7c673cae FG |
63 | |
64 | generate_db() | |
65 | { | |
66 | set +e | |
67 | $script_copy_dir/generate_random_db.sh $1 $2 | |
68 | if [ $? -ne 0 ]; then | |
69 | echo ==== Error loading data from $2 to $1 ==== | |
70 | exit 1 | |
71 | fi | |
72 | set -e | |
73 | } | |
74 | ||
75 | compare_db() | |
76 | { | |
77 | set +e | |
11fdf7f2 | 78 | $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 $5 |
7c673cae FG |
79 | if [ $? -ne 0 ]; then |
80 | echo ==== Read different content from $1 and $2 or error happened. ==== | |
81 | exit 1 | |
82 | fi | |
83 | set -e | |
84 | } | |
85 | ||
11fdf7f2 TL |
86 | write_external_sst() |
87 | { | |
88 | set +e | |
89 | $script_copy_dir/write_external_sst.sh $1 $2 $3 | |
90 | if [ $? -ne 0 ]; then | |
91 | echo ==== Error writing external SST file using data from $1 to $3 ==== | |
92 | exit 1 | |
93 | fi | |
94 | set -e | |
95 | } | |
96 | ||
97 | ingest_external_sst() | |
98 | { | |
99 | set +e | |
100 | $script_copy_dir/ingest_external_sst.sh $1 $2 | |
101 | if [ $? -ne 0 ]; then | |
102 | echo ==== Error ingesting external SST in $2 to DB at $1 ==== | |
103 | exit 1 | |
104 | fi | |
105 | set -e | |
106 | } | |
107 | ||
7c673cae FG |
108 | # Sandcastle sets us up with a remote that is just another directory on the same |
109 | # machine and doesn't have our branches. Need to fetch them so checkout works. | |
110 | # Remote add may fail if added previously (we don't cleanup). | |
111 | git remote add github_origin "https://github.com/facebook/rocksdb.git" | |
112 | set -e | |
20effc67 | 113 | git fetch github_origin |
7c673cae | 114 | |
11fdf7f2 TL |
115 | # Compatibility test for external SST file ingestion |
116 | for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" | |
117 | do | |
118 | echo == Generating DB with extern SST file in "$checkout_obj" ... | |
20effc67 | 119 | git checkout github_origin/$checkout_obj -b $checkout_obj |
11fdf7f2 | 120 | make clean |
20effc67 | 121 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
11fdf7f2 TL |
122 | write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj |
123 | ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj | |
124 | done | |
125 | ||
126 | checkout_flag=${1:-"master"} | |
127 | ||
128 | echo == Building $checkout_flag debug | |
20effc67 | 129 | git checkout github_origin/$checkout_flag -b tmp-$checkout_flag |
11fdf7f2 | 130 | make clean |
20effc67 | 131 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
11fdf7f2 TL |
132 | compare_base_db_dir=$test_dir"/base_db_dir" |
133 | write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir | |
134 | ingest_external_sst $compare_base_db_dir $compare_base_db_dir | |
135 | ||
136 | for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" | |
137 | do | |
138 | echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag | |
139 | git checkout $checkout_obj | |
140 | make clean | |
20effc67 | 141 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
11fdf7f2 TL |
142 | compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1 |
143 | git checkout tmp-$checkout_flag | |
144 | # Clean up | |
145 | git branch -D $checkout_obj | |
146 | done | |
147 | ||
148 | echo == Finish compatibility test for SST ingestion. | |
149 | ||
7c673cae FG |
150 | for checkout_obj in "${checkout_objs[@]}" |
151 | do | |
152 | echo == Generating DB from "$checkout_obj" ... | |
20effc67 | 153 | git checkout github_origin/$checkout_obj -b $checkout_obj |
7c673cae | 154 | make clean |
20effc67 | 155 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
7c673cae FG |
156 | generate_db $input_data_path $test_dir/$checkout_obj |
157 | done | |
158 | ||
159 | checkout_flag=${1:-"master"} | |
160 | ||
161 | echo == Building $checkout_flag debug | |
11fdf7f2 | 162 | git checkout tmp-$checkout_flag |
7c673cae | 163 | make clean |
20effc67 | 164 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
7c673cae FG |
165 | compare_base_db_dir=$test_dir"/base_db_dir" |
166 | echo == Generate compare base DB to $compare_base_db_dir | |
167 | generate_db $input_data_path $compare_base_db_dir | |
168 | ||
169 | for checkout_obj in "${checkout_objs[@]}" | |
170 | do | |
171 | echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ... | |
11fdf7f2 | 172 | compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 0 |
7c673cae FG |
173 | done |
174 | ||
175 | for checkout_obj in "${forward_compatible_checkout_objs[@]}" | |
176 | do | |
177 | echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag... | |
178 | git checkout $checkout_obj | |
179 | make clean | |
20effc67 | 180 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
7c673cae FG |
181 | compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0 |
182 | done | |
183 | ||
11fdf7f2 TL |
184 | for checkout_obj in "${forward_compatible_with_options_checkout_objs[@]}" |
185 | do | |
186 | echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag with its options... | |
187 | git checkout $checkout_obj | |
188 | make clean | |
20effc67 | 189 | DISABLE_WARNING_AS_ERROR=1 make ldb -j32 |
11fdf7f2 TL |
190 | compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 1 1 |
191 | done | |
192 | ||
7c673cae | 193 | echo ==== Compatibility Test PASSED ==== |