]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | #!/usr/bin/env bash |
7c673cae FG |
2 | # |
3 | # A shell script to load some pre generated data file to a DB using ldb tool | |
4 | # ./ldb needs to be avaible to be executed. | |
5 | # | |
6 | # Usage: <SCRIPT> [checkout] | |
7 | # `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous branches (or tags for very old versions without branch) can be opened by it. | |
8 | # Return value 0 means all regression tests pass. 1 if not pass. | |
9 | ||
10 | scriptpath=`dirname $BASH_SOURCE` | |
11 | test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check" | |
12 | script_copy_dir=$test_dir"/script_copy" | |
13 | input_data_path=$test_dir"/test_data_input/" | |
14 | ||
15 | mkdir $test_dir || true | |
16 | mkdir $input_data_path || true | |
17 | rm -rf $script_copy_dir | |
18 | cp $scriptpath $script_copy_dir -rf | |
19 | ||
11fdf7f2 | 20 | # Generate random files. |
7c673cae FG |
21 | for i in {1..6} |
22 | do | |
23 | input_data[$i]=$input_data_path/data$i | |
24 | echo == Generating random input file ${input_data[$i]} | |
25 | python - <<EOF | |
26 | import random | |
27 | random.seed($i) | |
28 | symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] | |
29 | with open('${input_data[$i]}', 'w') as f: | |
30 | for i in range(1,1024): | |
31 | k = "" | |
32 | for j in range(1, random.randint(1,32)): | |
33 | k=k + symbols[random.randint(0, len(symbols) - 1)] | |
34 | vb = "" | |
35 | for j in range(1, random.randint(0,128)): | |
36 | vb = vb + symbols[random.randint(0, len(symbols) - 1)] | |
37 | v = "" | |
38 | for j in range(1, random.randint(1, 5)): | |
39 | v = v + vb | |
40 | print >> f, k + " ==> " + v | |
41 | EOF | |
42 | done | |
43 | ||
11fdf7f2 TL |
44 | # Generate file(s) with sorted keys. |
45 | sorted_input_data=$input_data_path/sorted_data | |
46 | echo == Generating file with sorted keys ${sorted_input_data} | |
47 | python - <<EOF | |
48 | with open('${sorted_input_data}', 'w') as f: | |
49 | for i in range(0,10): | |
50 | k = str(i) | |
51 | v = "value" + k | |
52 | print >> f, k + " ==> " + v | |
53 | EOF | |
54 | ||
55 | declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb") | |
56 | declare -a forward_compatible_checkout_objs=("3.10.fb" "3.11.fb" "3.12.fb" "3.13.fb" "4.0.fb" "4.1.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb") | |
57 | declare -a forward_compatible_with_options_checkout_objs=("5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb") | |
58 | declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]}) | |
494da23a | 59 | declare -a extern_sst_ingestion_compatible_checkout_objs=("5.14.fb" "5.15.fb" "5.16.fb" "5.17.fb" "5.18.fb") |
7c673cae FG |
60 | |
61 | generate_db() | |
62 | { | |
63 | set +e | |
64 | $script_copy_dir/generate_random_db.sh $1 $2 | |
65 | if [ $? -ne 0 ]; then | |
66 | echo ==== Error loading data from $2 to $1 ==== | |
67 | exit 1 | |
68 | fi | |
69 | set -e | |
70 | } | |
71 | ||
72 | compare_db() | |
73 | { | |
74 | set +e | |
11fdf7f2 | 75 | $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 $5 |
7c673cae FG |
76 | if [ $? -ne 0 ]; then |
77 | echo ==== Read different content from $1 and $2 or error happened. ==== | |
78 | exit 1 | |
79 | fi | |
80 | set -e | |
81 | } | |
82 | ||
11fdf7f2 TL |
83 | write_external_sst() |
84 | { | |
85 | set +e | |
86 | $script_copy_dir/write_external_sst.sh $1 $2 $3 | |
87 | if [ $? -ne 0 ]; then | |
88 | echo ==== Error writing external SST file using data from $1 to $3 ==== | |
89 | exit 1 | |
90 | fi | |
91 | set -e | |
92 | } | |
93 | ||
94 | ingest_external_sst() | |
95 | { | |
96 | set +e | |
97 | $script_copy_dir/ingest_external_sst.sh $1 $2 | |
98 | if [ $? -ne 0 ]; then | |
99 | echo ==== Error ingesting external SST in $2 to DB at $1 ==== | |
100 | exit 1 | |
101 | fi | |
102 | set -e | |
103 | } | |
104 | ||
7c673cae FG |
105 | # Sandcastle sets us up with a remote that is just another directory on the same |
106 | # machine and doesn't have our branches. Need to fetch them so checkout works. | |
107 | # Remote add may fail if added previously (we don't cleanup). | |
108 | git remote add github_origin "https://github.com/facebook/rocksdb.git" | |
109 | set -e | |
110 | https_proxy="fwdproxy:8080" git fetch github_origin | |
111 | ||
11fdf7f2 TL |
112 | # Compatibility test for external SST file ingestion |
113 | for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" | |
114 | do | |
115 | echo == Generating DB with extern SST file in "$checkout_obj" ... | |
116 | https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj | |
117 | make clean | |
118 | make ldb -j32 | |
119 | write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj | |
120 | ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj | |
121 | done | |
122 | ||
123 | checkout_flag=${1:-"master"} | |
124 | ||
125 | echo == Building $checkout_flag debug | |
126 | https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag | |
127 | make clean | |
128 | make ldb -j32 | |
129 | compare_base_db_dir=$test_dir"/base_db_dir" | |
130 | write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir | |
131 | ingest_external_sst $compare_base_db_dir $compare_base_db_dir | |
132 | ||
133 | for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" | |
134 | do | |
135 | echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag | |
136 | git checkout $checkout_obj | |
137 | make clean | |
138 | make ldb -j32 | |
139 | compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1 | |
140 | git checkout tmp-$checkout_flag | |
141 | # Clean up | |
142 | git branch -D $checkout_obj | |
143 | done | |
144 | ||
145 | echo == Finish compatibility test for SST ingestion. | |
146 | ||
7c673cae FG |
147 | for checkout_obj in "${checkout_objs[@]}" |
148 | do | |
149 | echo == Generating DB from "$checkout_obj" ... | |
11fdf7f2 | 150 | https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj |
7c673cae FG |
151 | make clean |
152 | make ldb -j32 | |
153 | generate_db $input_data_path $test_dir/$checkout_obj | |
154 | done | |
155 | ||
156 | checkout_flag=${1:-"master"} | |
157 | ||
158 | echo == Building $checkout_flag debug | |
11fdf7f2 | 159 | git checkout tmp-$checkout_flag |
7c673cae FG |
160 | make clean |
161 | make ldb -j32 | |
162 | compare_base_db_dir=$test_dir"/base_db_dir" | |
163 | echo == Generate compare base DB to $compare_base_db_dir | |
164 | generate_db $input_data_path $compare_base_db_dir | |
165 | ||
166 | for checkout_obj in "${checkout_objs[@]}" | |
167 | do | |
168 | echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ... | |
11fdf7f2 | 169 | compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 0 |
7c673cae FG |
170 | done |
171 | ||
172 | for checkout_obj in "${forward_compatible_checkout_objs[@]}" | |
173 | do | |
174 | echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag... | |
175 | git checkout $checkout_obj | |
176 | make clean | |
177 | make ldb -j32 | |
178 | compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0 | |
179 | done | |
180 | ||
11fdf7f2 TL |
181 | for checkout_obj in "${forward_compatible_with_options_checkout_objs[@]}" |
182 | do | |
183 | echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag with its options... | |
184 | git checkout $checkout_obj | |
185 | make clean | |
186 | make ldb -j32 | |
187 | compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 1 1 | |
188 | done | |
189 | ||
7c673cae | 190 | echo ==== Compatibility Test PASSED ==== |