]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | #!/usr/bin/env bash |
2 | set -e | |
7c673cae FG |
3 | |
4 | source $(dirname $0)/../detect-build-env-vars.sh | |
5 | ||
6 | [ -z "$CEPH_ROOT" ] && CEPH_ROOT=.. | |
7 | ||
8 | dir=$CEPH_ROOT/ceph-object-corpus | |
9 | ||
7c673cae FG |
10 | failed=0 |
11 | numtests=0 | |
12 | pids="" | |
13 | ||
14 | if [ -x ./ceph-dencoder ]; then | |
15 | CEPH_DENCODER=./ceph-dencoder | |
16 | else | |
17 | CEPH_DENCODER=ceph-dencoder | |
18 | fi | |
19 | ||
20 | myversion=`$CEPH_DENCODER version` | |
21 | DEBUG=0 | |
22 | WAITALL_DELAY=.1 | |
23 | debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi } | |
24 | ||
25 | test_object() { | |
26 | local type=$1 | |
27 | local output_file=$2 | |
28 | local failed=0 | |
29 | local numtests=0 | |
30 | ||
d2e6a577 FG |
31 | tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX` |
32 | tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX` | |
7c673cae FG |
33 | |
34 | rm -f $output_file | |
35 | if $CEPH_DENCODER type $type 2>/dev/null; then | |
36 | #echo "type $type"; | |
37 | echo " $vdir/objects/$type" | |
38 | ||
39 | # is there a fwd incompat change between $arversion and $version? | |
40 | incompat="" | |
41 | incompat_paths="" | |
42 | sawarversion=0 | |
43 | for iv in `ls $dir/archive | sort -n`; do | |
44 | if [ "$iv" = "$arversion" ]; then | |
45 | sawarversion=1 | |
46 | fi | |
47 | ||
48 | if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then | |
49 | incompat="$iv" | |
50 | ||
51 | # Check if we'll be ignoring only specified objects, not whole type. If so, remember | |
52 | # all paths for this type into variable. Assuming that this path won't contain any | |
53 | # whitechars (implication of above for loop). | |
54 | if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then | |
55 | if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then | |
56 | incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type" | |
57 | else | |
58 | echo "type $type directory empty, ignoring whole type instead of single objects" | |
59 | fi; | |
60 | fi | |
61 | fi | |
62 | ||
63 | if [ "$iv" = "$version" ]; then | |
d2e6a577 | 64 | rm -rf $tmp1 $tmp2 |
7c673cae FG |
65 | break |
66 | fi | |
67 | done | |
68 | ||
69 | if [ -n "$incompat" ]; then | |
70 | if [ -z "$incompat_paths" ]; then | |
71 | echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion" | |
d2e6a577 | 72 | rm -rf $tmp1 $tmp2 |
7c673cae FG |
73 | return |
74 | else | |
75 | # If we are ignoring not whole type, but objects that are in $incompat_path, | |
76 | # we don't skip here, just give info. | |
77 | echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion" | |
78 | fi; | |
79 | fi | |
80 | ||
81 | for f in `ls $vdir/objects/$type`; do | |
82 | ||
83 | skip=0; | |
84 | # Check if processed object $f of $type should be skipped (postponed skip) | |
85 | if [ -n "$incompat_paths" ]; then | |
86 | for i_path in $incompat_paths; do | |
87 | # Check if $f is a symbolic link and if it's pointing to existing target | |
88 | if [ -L "$i_path/$f" ]; then | |
89 | echo "skipping object $f of type $type" | |
90 | skip=1 | |
91 | break | |
92 | fi; | |
93 | done; | |
94 | fi; | |
95 | ||
96 | if [ $skip -ne 0 ]; then | |
97 | continue | |
98 | fi; | |
99 | ||
100 | $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 & | |
101 | pid1="$!" | |
102 | $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 & | |
103 | pid2="$!" | |
104 | #echo "\t$vdir/$type/$f" | |
105 | if ! wait $pid1; then | |
106 | echo "**** failed to decode $vdir/objects/$type/$f ****" | |
107 | failed=$(($failed + 1)) | |
108 | rm -f $tmp1 $tmp2 | |
109 | continue | |
110 | fi | |
111 | if ! wait $pid2; then | |
112 | echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****" | |
113 | failed=$(($failed + 1)) | |
114 | rm -f $tmp1 $tmp2 | |
115 | continue | |
116 | fi | |
117 | ||
118 | # nondeterministic classes may dump | |
119 | # nondeterministically. compare the sorted json | |
120 | # output. this is a weaker test, but is better than | |
121 | # nothing. | |
122 | if ! $CEPH_DENCODER type $type is_deterministic; then | |
123 | echo " sorting json output for nondeterministic object" | |
124 | for f in $tmp1 $tmp2; do | |
125 | sort $f | sed 's/,$//' > $f.new | |
126 | mv $f.new $f | |
127 | done | |
128 | fi | |
129 | ||
130 | if ! cmp $tmp1 $tmp2; then | |
131 | echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****" | |
132 | diff $tmp1 $tmp2 | |
133 | failed=$(($failed + 1)) | |
134 | fi | |
135 | numtests=$(($numtests + 1)) | |
d2e6a577 | 136 | rm -f $tmp1 $tmp2 |
7c673cae FG |
137 | done |
138 | else | |
139 | echo "skipping unrecognized type $type" | |
d2e6a577 | 140 | rm -f $tmp1 $tmp2 |
7c673cae FG |
141 | fi |
142 | ||
143 | echo "failed=$failed" > $output_file | |
144 | echo "numtests=$numtests" >> $output_file | |
7c673cae FG |
145 | } |
146 | ||
147 | waitall() { # PID... | |
148 | ## Wait for children to exit and indicate whether all exited with 0 status. | |
149 | local errors=0 | |
150 | while :; do | |
151 | debug "Processes remaining: $*" | |
152 | for pid in "$@"; do | |
153 | shift | |
154 | if kill -0 "$pid" 2>/dev/null; then | |
155 | debug "$pid is still alive." | |
156 | set -- "$@" "$pid" | |
157 | elif wait "$pid"; then | |
158 | debug "$pid exited with zero exit status." | |
159 | else | |
160 | debug "$pid exited with non-zero exit status." | |
161 | errors=$(($errors + 1)) | |
162 | fi | |
163 | done | |
164 | [ $# -eq 0 ] && break | |
165 | sleep ${WAITALL_DELAY:-1} | |
166 | done | |
167 | [ $errors -eq 0 ] | |
168 | } | |
169 | ||
170 | ###### | |
171 | # MAIN | |
172 | ###### | |
173 | ||
174 | do_join() { | |
175 | waitall $pids | |
176 | pids="" | |
177 | # Reading the output of jobs to compute failed & numtests | |
178 | # Tests are run in parallel but sum should be done sequentialy to avoid | |
179 | # races between threads | |
180 | while [ "$running_jobs" -ge 0 ]; do | |
181 | if [ -f $output_file.$running_jobs ]; then | |
182 | read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2) | |
183 | read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2) | |
184 | rm -f $output_file.$running_jobs | |
185 | failed=$(($failed + $read_failed)) | |
186 | numtests=$(($numtests + $read_numtests)) | |
187 | fi | |
188 | running_jobs=$(($running_jobs - 1)) | |
189 | done | |
190 | running_jobs=0 | |
191 | } | |
192 | ||
193 | # Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical | |
194 | # processors | |
11fdf7f2 | 195 | if [ `uname` == FreeBSD -o `uname` == Darwin ]; then |
7c673cae FG |
196 | NPROC=`sysctl -n hw.ncpu` |
197 | max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}} | |
198 | else | |
199 | max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)} | |
200 | fi | |
201 | ||
d2e6a577 | 202 | output_file=`mktemp /tmp/output_file-XXXXXXXXX` |
7c673cae FG |
203 | running_jobs=0 |
204 | ||
205 | for arversion in `ls $dir/archive | sort -n`; do | |
206 | vdir="$dir/archive/$arversion" | |
207 | #echo $vdir | |
208 | ||
209 | if [ ! -d "$vdir/objects" ]; then | |
210 | continue; | |
211 | fi | |
212 | ||
213 | for type in `ls $vdir/objects`; do | |
214 | test_object $type $output_file.$running_jobs & | |
215 | pids="$pids $!" | |
216 | running_jobs=$(($running_jobs + 1)) | |
217 | ||
218 | # Once we spawned enough jobs, let's wait them to complete | |
219 | # Every spawned job have almost the same execution time so | |
220 | # it's not a big deal having them not ending at the same time | |
221 | if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then | |
222 | do_join | |
223 | fi | |
d2e6a577 | 224 | rm -f ${output_file}* |
7c673cae FG |
225 | done |
226 | done | |
227 | ||
228 | do_join | |
229 | ||
230 | if [ $failed -gt 0 ]; then | |
231 | echo "FAILED $failed / $numtests tests." | |
232 | exit 1 | |
233 | fi | |
f64942e4 AA |
234 | |
235 | if [ $numtests -eq 0 ]; then | |
236 | echo "FAILED: no tests found to run!" | |
237 | exit 1 | |
238 | fi | |
239 | ||
7c673cae FG |
240 | echo "passed $numtests tests." |
241 |