]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/bash -e |
2 | ||
3 | source $(dirname $0)/../detect-build-env-vars.sh | |
4 | ||
5 | [ -z "$CEPH_ROOT" ] && CEPH_ROOT=.. | |
6 | ||
7 | dir=$CEPH_ROOT/ceph-object-corpus | |
8 | ||
9 | set -e | |
10 | ||
11 | failed=0 | |
12 | numtests=0 | |
13 | pids="" | |
14 | ||
15 | if [ -x ./ceph-dencoder ]; then | |
16 | CEPH_DENCODER=./ceph-dencoder | |
17 | else | |
18 | CEPH_DENCODER=ceph-dencoder | |
19 | fi | |
20 | ||
21 | myversion=`$CEPH_DENCODER version` | |
22 | DEBUG=0 | |
23 | WAITALL_DELAY=.1 | |
24 | debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi } | |
25 | ||
26 | test_object() { | |
27 | local type=$1 | |
28 | local output_file=$2 | |
29 | local failed=0 | |
30 | local numtests=0 | |
31 | ||
d2e6a577 FG |
32 | tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX` |
33 | tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX` | |
7c673cae FG |
34 | |
35 | rm -f $output_file | |
36 | if $CEPH_DENCODER type $type 2>/dev/null; then | |
37 | #echo "type $type"; | |
38 | echo " $vdir/objects/$type" | |
39 | ||
40 | # is there a fwd incompat change between $arversion and $version? | |
41 | incompat="" | |
42 | incompat_paths="" | |
43 | sawarversion=0 | |
44 | for iv in `ls $dir/archive | sort -n`; do | |
45 | if [ "$iv" = "$arversion" ]; then | |
46 | sawarversion=1 | |
47 | fi | |
48 | ||
49 | if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then | |
50 | incompat="$iv" | |
51 | ||
52 | # Check if we'll be ignoring only specified objects, not whole type. If so, remember | |
53 | # all paths for this type into variable. Assuming that this path won't contain any | |
54 | # whitechars (implication of above for loop). | |
55 | if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then | |
56 | if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then | |
57 | incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type" | |
58 | else | |
59 | echo "type $type directory empty, ignoring whole type instead of single objects" | |
60 | fi; | |
61 | fi | |
62 | fi | |
63 | ||
64 | if [ "$iv" = "$version" ]; then | |
d2e6a577 | 65 | rm -rf $tmp1 $tmp2 |
7c673cae FG |
66 | break |
67 | fi | |
68 | done | |
69 | ||
70 | if [ -n "$incompat" ]; then | |
71 | if [ -z "$incompat_paths" ]; then | |
72 | echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion" | |
d2e6a577 | 73 | rm -rf $tmp1 $tmp2 |
7c673cae FG |
74 | return |
75 | else | |
76 | # If we are ignoring not whole type, but objects that are in $incompat_path, | |
77 | # we don't skip here, just give info. | |
78 | echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion" | |
79 | fi; | |
80 | fi | |
81 | ||
82 | for f in `ls $vdir/objects/$type`; do | |
83 | ||
84 | skip=0; | |
85 | # Check if processed object $f of $type should be skipped (postponed skip) | |
86 | if [ -n "$incompat_paths" ]; then | |
87 | for i_path in $incompat_paths; do | |
88 | # Check if $f is a symbolic link and if it's pointing to existing target | |
89 | if [ -L "$i_path/$f" ]; then | |
90 | echo "skipping object $f of type $type" | |
91 | skip=1 | |
92 | break | |
93 | fi; | |
94 | done; | |
95 | fi; | |
96 | ||
97 | if [ $skip -ne 0 ]; then | |
98 | continue | |
99 | fi; | |
100 | ||
101 | $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 & | |
102 | pid1="$!" | |
103 | $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 & | |
104 | pid2="$!" | |
105 | #echo "\t$vdir/$type/$f" | |
106 | if ! wait $pid1; then | |
107 | echo "**** failed to decode $vdir/objects/$type/$f ****" | |
108 | failed=$(($failed + 1)) | |
109 | rm -f $tmp1 $tmp2 | |
110 | continue | |
111 | fi | |
112 | if ! wait $pid2; then | |
113 | echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****" | |
114 | failed=$(($failed + 1)) | |
115 | rm -f $tmp1 $tmp2 | |
116 | continue | |
117 | fi | |
118 | ||
119 | # nondeterministic classes may dump | |
120 | # nondeterministically. compare the sorted json | |
121 | # output. this is a weaker test, but is better than | |
122 | # nothing. | |
123 | if ! $CEPH_DENCODER type $type is_deterministic; then | |
124 | echo " sorting json output for nondeterministic object" | |
125 | for f in $tmp1 $tmp2; do | |
126 | sort $f | sed 's/,$//' > $f.new | |
127 | mv $f.new $f | |
128 | done | |
129 | fi | |
130 | ||
131 | if ! cmp $tmp1 $tmp2; then | |
132 | echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****" | |
133 | diff $tmp1 $tmp2 | |
134 | failed=$(($failed + 1)) | |
135 | fi | |
136 | numtests=$(($numtests + 1)) | |
d2e6a577 | 137 | rm -f $tmp1 $tmp2 |
7c673cae FG |
138 | done |
139 | else | |
140 | echo "skipping unrecognized type $type" | |
d2e6a577 | 141 | rm -f $tmp1 $tmp2 |
7c673cae FG |
142 | fi |
143 | ||
144 | echo "failed=$failed" > $output_file | |
145 | echo "numtests=$numtests" >> $output_file | |
7c673cae FG |
146 | } |
147 | ||
148 | waitall() { # PID... | |
149 | ## Wait for children to exit and indicate whether all exited with 0 status. | |
150 | local errors=0 | |
151 | while :; do | |
152 | debug "Processes remaining: $*" | |
153 | for pid in "$@"; do | |
154 | shift | |
155 | if kill -0 "$pid" 2>/dev/null; then | |
156 | debug "$pid is still alive." | |
157 | set -- "$@" "$pid" | |
158 | elif wait "$pid"; then | |
159 | debug "$pid exited with zero exit status." | |
160 | else | |
161 | debug "$pid exited with non-zero exit status." | |
162 | errors=$(($errors + 1)) | |
163 | fi | |
164 | done | |
165 | [ $# -eq 0 ] && break | |
166 | sleep ${WAITALL_DELAY:-1} | |
167 | done | |
168 | [ $errors -eq 0 ] | |
169 | } | |
170 | ||
171 | ###### | |
172 | # MAIN | |
173 | ###### | |
174 | ||
175 | do_join() { | |
176 | waitall $pids | |
177 | pids="" | |
178 | # Reading the output of jobs to compute failed & numtests | |
179 | # Tests are run in parallel but sum should be done sequentialy to avoid | |
180 | # races between threads | |
181 | while [ "$running_jobs" -ge 0 ]; do | |
182 | if [ -f $output_file.$running_jobs ]; then | |
183 | read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2) | |
184 | read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2) | |
185 | rm -f $output_file.$running_jobs | |
186 | failed=$(($failed + $read_failed)) | |
187 | numtests=$(($numtests + $read_numtests)) | |
188 | fi | |
189 | running_jobs=$(($running_jobs - 1)) | |
190 | done | |
191 | running_jobs=0 | |
192 | } | |
193 | ||
194 | # Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical | |
195 | # processors | |
196 | if [ `uname` == FreeBSD ]; then | |
197 | NPROC=`sysctl -n hw.ncpu` | |
198 | max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}} | |
199 | else | |
200 | max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)} | |
201 | fi | |
202 | ||
d2e6a577 | 203 | output_file=`mktemp /tmp/output_file-XXXXXXXXX` |
7c673cae FG |
204 | running_jobs=0 |
205 | ||
206 | for arversion in `ls $dir/archive | sort -n`; do | |
207 | vdir="$dir/archive/$arversion" | |
208 | #echo $vdir | |
209 | ||
210 | if [ ! -d "$vdir/objects" ]; then | |
211 | continue; | |
212 | fi | |
213 | ||
214 | for type in `ls $vdir/objects`; do | |
215 | test_object $type $output_file.$running_jobs & | |
216 | pids="$pids $!" | |
217 | running_jobs=$(($running_jobs + 1)) | |
218 | ||
219 | # Once we spawned enough jobs, let's wait them to complete | |
220 | # Every spawned job have almost the same execution time so | |
221 | # it's not a big deal having them not ending at the same time | |
222 | if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then | |
223 | do_join | |
224 | fi | |
d2e6a577 | 225 | rm -f ${output_file}* |
7c673cae FG |
226 | done |
227 | done | |
228 | ||
229 | do_join | |
230 | ||
231 | if [ $failed -gt 0 ]; then | |
232 | echo "FAILED $failed / $numtests tests." | |
233 | exit 1 | |
234 | fi | |
235 | echo "passed $numtests tests." | |
236 |