]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | #!/usr/bin/env bash |
7c673cae FG |
2 | # vim: ts=8 sw=2 smarttab |
3 | # | |
4 | # run_seed_to.sh - Run ceph_test_filestore_idempotent_sequence up until an | |
5 | # injection point, generating a sequence of operations based on a | |
6 | # provided seed. | |
7 | # | |
8 | # We also perform three additional tests, focused on assessing if | |
9 | # replaying a larger chunck of the journal affects the expected store | |
10 | # behavior. These tests will be performed by increasing the store's | |
11 | # journal sync interval to a very large value, allowing the store to | |
12 | # finish execution before the first sync (unless the store runs for | |
13 | # over 10 hours, case on which the interval variables must be changed | |
14 | # to an appropriate value). Unless the '--no-journal-test' option is | |
15 | # specified, we will run the 3 following scenarios: | |
16 | # | |
17 | # 1) journal sync'ing for both stores is good as disabled | |
18 | # (we call it '00', for store naming purposes) | |
19 | # 2) journal sync'ing for store A is as good as disabled | |
20 | # (we call it '01', for store naming purposes) | |
21 | # 3) journal sync'ing for store B is as good as disabled | |
22 | # (we call it '10', for store naming purposes) | |
23 | # | |
24 | # All log files are also appropriately named accordingly (i.e., a.00.fail, | |
25 | # a.10.recover, or b.01.clean). | |
26 | # | |
27 | # By default, the test will not exit on error, although it will show the | |
28 | # fail message. This behavior is so defined so we run the whole battery of | |
29 | # tests, and obtain as many mismatches as possible in one go. We may force | |
30 | # the test to exit on error by specifying the '--exit-on-error' option. | |
31 | # | |
32 | # | |
33 | set -e | |
34 | ||
35 | test_opts="" | |
36 | ||
37 | usage() { | |
38 | echo "usage: $1 [options..] <seed> <kill-at>" | |
39 | echo | |
40 | echo "options:" | |
41 | echo " -c, --colls <VAL> # of collections" | |
42 | echo " -o, --objs <VAL> # of objects" | |
43 | echo " -b, --btrfs <VAL> seq number for btrfs stores" | |
44 | echo " --no-journal-test don't perform journal replay tests" | |
45 | echo " -e, --exit-on-error exit with 1 on error" | |
46 | echo " -v, --valgrind run commands through valgrind" | |
47 | echo | |
48 | echo "env vars:" | |
49 | echo " OPTS_STORE additional opts for both stores" | |
50 | echo " OPTS_STORE_A additional opts for store A" | |
51 | echo " OPTS_STORE_B additional opts for store B" | |
52 | echo | |
53 | } | |
54 | ||
28e407b8 AA |
55 | echo $0 $* |
56 | ||
7c673cae FG |
57 | die_on_missing_arg() { |
58 | if [[ "$2" == "" ]]; then | |
59 | echo "$1: missing required parameter" | |
60 | exit 1 | |
61 | fi | |
62 | } | |
63 | ||
64 | ||
65 | required_args=2 | |
66 | obtained_args=0 | |
67 | ||
68 | seed="" | |
69 | killat="" | |
70 | on_btrfs=0 | |
71 | on_btrfs_seq=0 | |
72 | journal_test=1 | |
73 | min_sync_interval="36000" # ten hours, yes. | |
74 | max_sync_interval="36001" | |
75 | exit_on_error=0 | |
76 | v="" | |
77 | ||
78 | do_rm() { | |
79 | if [[ $on_btrfs -eq 0 ]]; then | |
80 | rm -fr $* | |
81 | fi | |
82 | } | |
83 | ||
84 | set_arg() { | |
85 | if [[ $1 -eq 1 ]]; then | |
86 | seed=$2 | |
87 | elif [[ $1 -eq 2 ]]; then | |
88 | killat=$2 | |
89 | else | |
90 | echo "error: unknown purpose for '$2'" | |
91 | usage $0 | |
92 | exit 1 | |
93 | fi | |
94 | } | |
95 | ||
96 | while [[ $# -gt 0 ]]; | |
97 | do | |
98 | case "$1" in | |
99 | -c | --colls) | |
100 | die_on_missing_arg "$1" "$2" | |
101 | test_opts="$test_opts --test-num-colls $2" | |
102 | shift 2 | |
103 | ;; | |
104 | -o | --objs) | |
105 | die_on_missing_arg "$1" "$2" | |
106 | test_opts="$test_opts --test-num-objs $2" | |
107 | shift 2 | |
108 | ;; | |
109 | -h | --help) | |
110 | usage $0 ; | |
111 | exit 0 | |
112 | ;; | |
113 | -b | --btrfs) | |
114 | die_on_missing_arg "$1" "$2" | |
115 | on_btrfs=1 | |
116 | on_btrfs_seq=$2 | |
117 | shift 2 | |
118 | ;; | |
119 | --no-journal-test) | |
120 | journal_test=0 | |
121 | shift | |
122 | ;; | |
123 | -e | --exit-on-error) | |
124 | exit_on_error=1 | |
125 | shift | |
126 | ;; | |
127 | -v | --valgrind) | |
128 | v="valgrind --leak-check=full" | |
129 | shift | |
130 | ;; | |
131 | --) | |
132 | shift | |
133 | break | |
134 | ;; | |
135 | -*) | |
136 | echo "$1: unknown option" >&2 | |
137 | usage $0 | |
138 | exit 1 | |
139 | ;; | |
140 | *) | |
141 | obtained_args=$(($obtained_args+1)) | |
142 | set_arg $obtained_args $1 | |
143 | shift | |
144 | ;; | |
145 | esac | |
146 | done | |
147 | ||
148 | if [[ $obtained_args -ne $required_args ]]; then | |
149 | echo "error: missing argument" | |
150 | usage $0 ; | |
151 | exit 1 | |
152 | fi | |
153 | ||
154 | if [[ "$OPTS_STORE" != "" ]]; then | |
155 | test_opts="$test_opts $OPTS_STORE" | |
156 | fi | |
157 | ||
158 | test_opts_a="$test_opts" | |
159 | test_opts_b="$test_opts" | |
160 | ||
161 | if [[ "$OPTS_STORE_A" != "" ]]; then | |
162 | test_opts_a="$test_opts_a $OPTS_STORE_A" | |
163 | fi | |
164 | if [[ "$OPTS_STORE_B" != "" ]]; then | |
165 | test_opts_b="$test_opts_b $OPTS_STORE_B" | |
166 | fi | |
167 | ||
168 | echo seed $seed | |
169 | echo kill at $killat | |
170 | ||
171 | # run forever, until $killat... | |
172 | to=1000000000 | |
173 | ||
174 | # | |
175 | # store names | |
176 | # | |
177 | # We need these for two reasons: | |
178 | # 1) if we are running the tests on a btrfs volume, then we need to use | |
179 | # a seq number for each run. Being on btrfs means we will fail when | |
180 | # removing the store's directories and it's far more simple to just | |
181 | # specify differente store names such as 'a.$seq' or 'b.$seq'. | |
182 | # | |
183 | # 2) unless the '--no-journal-test' option is specified, we will run | |
184 | # three additional tests for each store, and we will reuse the same | |
185 | # command for each one of the runs, but varying the store's name and | |
186 | # arguments. | |
187 | # | |
188 | store_a="a" | |
189 | store_b="b" | |
190 | ||
191 | if [[ $on_btrfs -eq 1 ]]; then | |
192 | store_a="$store_a.$on_btrfs_seq" | |
193 | store_b="$store_b.$on_btrfs_seq" | |
194 | fi | |
195 | ||
196 | total_runs=1 | |
197 | ||
198 | if [[ $journal_test -eq 1 ]]; then | |
199 | total_runs=$(($total_runs + 3)) | |
200 | fi | |
201 | ||
202 | num_runs=0 | |
203 | ||
204 | opt_min_sync="--filestore-min-sync-interval $min_sync_interval" | |
205 | opt_max_sync="--filestore-max-sync-interval $max_sync_interval" | |
206 | ||
207 | ret=0 | |
208 | ||
209 | while [[ $num_runs -lt $total_runs ]]; | |
210 | do | |
211 | tmp_name_a=$store_a | |
212 | tmp_name_b=$store_b | |
213 | tmp_opts_a=$test_opts_a | |
214 | tmp_opts_b=$test_opts_b | |
215 | ||
216 | # | |
217 | # We have already tested whether there are diffs when both journals | |
218 | # are properly working. Now let's try on three other scenarios: | |
219 | # 1) journal sync'ing for both stores is good as disabled | |
220 | # (we call it '00') | |
221 | # 2) journal sync'ing for store A is as good as disabled | |
222 | # (we call it '01') | |
223 | # 3) journal sync'ing for store B is as good as disabled | |
224 | # (we call it '10') | |
225 | # | |
226 | if [[ $num_runs -gt 0 && $journal_test -eq 1 ]]; then | |
227 | echo "run #$num_runs" | |
228 | case $num_runs in | |
229 | 1) | |
230 | tmp_name_a="$tmp_name_a.00" | |
231 | tmp_name_b="$tmp_name_b.00" | |
232 | tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync" | |
233 | tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync" | |
234 | ;; | |
235 | 2) | |
236 | tmp_name_a="$tmp_name_a.01" | |
237 | tmp_name_b="$tmp_name_b.01" | |
238 | tmp_opts_a="$tmp_opts_a $opt_min_sync $opt_max_sync" | |
239 | ;; | |
240 | 3) | |
241 | tmp_name_a="$tmp_name_a.10" | |
242 | tmp_name_b="$tmp_name_b.10" | |
243 | tmp_opts_b="$tmp_opts_b $opt_min_sync $opt_max_sync" | |
244 | ;; | |
245 | esac | |
246 | fi | |
247 | ||
248 | do_rm $tmp_name_a $tmp_name_a.fail $tmp_name_a.recover | |
249 | $v ceph_test_filestore_idempotent_sequence run-sequence-to $to \ | |
250 | $tmp_name_a $tmp_name_a/journal \ | |
251 | --test-seed $seed --osd-journal-size 100 \ | |
252 | --filestore-kill-at $killat $tmp_opts_a \ | |
28e407b8 | 253 | --log-file $tmp_name_a.fail --debug-filestore 20 --no-log-to-stderr || true |
7c673cae FG |
254 | |
255 | stop_at=`ceph_test_filestore_idempotent_sequence get-last-op \ | |
256 | $tmp_name_a $tmp_name_a/journal \ | |
257 | --log-file $tmp_name_a.recover \ | |
28e407b8 | 258 | --debug-filestore 20 --debug-journal 20 --no-log-to-stderr` |
7c673cae FG |
259 | |
260 | if [[ "`expr $stop_at - $stop_at 2>/dev/null`" != "0" ]]; then | |
261 | echo "error: get-last-op returned '$stop_at'" | |
262 | exit 1 | |
263 | fi | |
264 | ||
265 | echo stopped at $stop_at | |
266 | ||
267 | do_rm $tmp_name_b $tmp_name_b.clean | |
268 | $v ceph_test_filestore_idempotent_sequence run-sequence-to \ | |
269 | $stop_at $tmp_name_b $tmp_name_b/journal \ | |
270 | --test-seed $seed --osd-journal-size 100 \ | |
28e407b8 AA |
271 | --log-file $tmp_name_b.clean --debug-filestore 20 --no-log-to-stderr \ |
272 | $tmp_opts_b | |
7c673cae FG |
273 | |
274 | if $v ceph_test_filestore_idempotent_sequence diff \ | |
28e407b8 | 275 | $tmp_name_a $tmp_name_a/journal $tmp_name_b $tmp_name_b/journal --no-log-to-stderr --log-file $tmp_name_a.diff.log --debug-filestore 20 ; then |
7c673cae FG |
276 | echo OK |
277 | else | |
278 | echo "FAIL" | |
279 | echo " see:" | |
280 | echo " $tmp_name_a.fail -- leading up to failure" | |
281 | echo " $tmp_name_a.recover -- journal replay" | |
282 | echo " $tmp_name_b.clean -- the clean reference" | |
283 | ||
284 | ret=1 | |
285 | if [[ $exit_on_error -eq 1 ]]; then | |
286 | exit 1 | |
287 | fi | |
288 | fi | |
289 | ||
290 | num_runs=$(($num_runs+1)) | |
291 | done | |
292 | ||
293 | exit $ret |