ceph/src/test/encoding/readable.sh

   1 #!/usr/bin/env bash
   2 set -e
   3
   4 source $(dirname $0)/../detect-build-env-vars.sh
   5
   6 [ -z "$CEPH_ROOT" ] && CEPH_ROOT=..
   7
   8 dir=$CEPH_ROOT/ceph-object-corpus
   9
  10 failed=0
  11 numtests=0
  12 pids=""
  13
  14 if [ -x ./ceph-dencoder ]; then
  15   CEPH_DENCODER=./ceph-dencoder
  16 else
  17   CEPH_DENCODER=ceph-dencoder
  18 fi
  19
  20 myversion=`$CEPH_DENCODER version`
  21 DEBUG=0
  22 WAITALL_DELAY=.1
  23 debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi }
  24
  25 test_object() {
  26     local type=$1
  27     local output_file=$2
  28     local failed=0
  29     local numtests=0
  30
  31     tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX`
  32     tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX`
  33
  34     rm -f $output_file
  35     if $CEPH_DENCODER type $type 2>/dev/null; then
  36       #echo "type $type";
  37       echo "        $vdir/objects/$type"
  38
  39       # is there a fwd incompat change between $arversion and $version?
  40       incompat=""
  41       incompat_paths=""
  42       sawarversion=0
  43       for iv in `ls $dir/archive | sort -n`; do
  44         if [ "$iv" = "$arversion" ]; then
  45           sawarversion=1
  46         fi
  47
  48         if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then
  49           incompat="$iv"
  50
  51           # Check if we'll be ignoring only specified objects, not whole type. If so, remember
  52           # all paths for this type into variable. Assuming that this path won't contain any
  53           # whitechars (implication of above for loop).
  54           if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then
  55             if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then
  56               incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type"
  57             else
  58               echo "type $type directory empty, ignoring whole type instead of single objects"
  59             fi;
  60           fi
  61         fi
  62
  63         if [ "$iv" = "$version" ]; then
  64           rm -rf $tmp1 $tmp2
  65           break
  66         fi
  67       done
  68
  69       if [ -n "$incompat" ]; then
  70         if [ -z "$incompat_paths" ]; then
  71           echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion"
  72           rm -rf $tmp1 $tmp2
  73           return
  74         else
  75           # If we are ignoring not whole type, but objects that are in $incompat_path,
  76           # we don't skip here, just give info.
  77           echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion"
  78         fi;
  79       fi
  80
  81       for f in `ls $vdir/objects/$type`; do
  82
  83         skip=0;
  84         # Check if processed object $f of $type should be skipped (postponed skip)
  85         if [ -n "$incompat_paths" ]; then
  86             for i_path in $incompat_paths; do
  87               # Check if $f is a symbolic link and if it's pointing to existing target
  88               if [ -L "$i_path/$f" ]; then
  89                 echo "skipping object $f of type $type"
  90                 skip=1
  91                 break
  92               fi;
  93             done;
  94         fi;
  95
  96         if [ $skip -ne 0 ]; then
  97           continue
  98         fi;
  99
 100         $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 &
 101         pid1="$!"
 102         $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 &
 103         pid2="$!"
 104         #echo "\t$vdir/$type/$f"
 105         if ! wait $pid1; then
 106           echo "**** failed to decode $vdir/objects/$type/$f ****"
 107           failed=$(($failed + 1))
 108           rm -f $tmp1 $tmp2
 109           continue
 110         fi
 111         if ! wait $pid2; then
 112           echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****"
 113           failed=$(($failed + 1))
 114           rm -f $tmp1 $tmp2
 115           continue
 116         fi
 117
 118         # nondeterministic classes may dump
 119         # nondeterministically.  compare the sorted json
 120         # output.  this is a weaker test, but is better than
 121         # nothing.
 122         if ! $CEPH_DENCODER type $type is_deterministic; then
 123           echo "  sorting json output for nondeterministic object"
 124           for f in $tmp1 $tmp2; do
 125             sort $f | sed 's/,$//' > $f.new
 126             mv $f.new $f
 127           done
 128         fi
 129
 130         if ! cmp $tmp1 $tmp2; then
 131           echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****"
 132           diff $tmp1 $tmp2
 133           failed=$(($failed + 1))
 134         fi
 135         numtests=$(($numtests + 1))
 136         rm -f $tmp1 $tmp2
 137       done
 138     else
 139       echo "skipping unrecognized type $type"
 140       rm -f $tmp1 $tmp2
 141     fi
 142
 143     echo "failed=$failed" > $output_file
 144     echo "numtests=$numtests" >> $output_file
 145 }
 146
 147 waitall() { # PID...
 148    ## Wait for children to exit and indicate whether all exited with 0 status.
 149    local errors=0
 150    while :; do
 151      debug "Processes remaining: $*"
 152      for pid in "$@"; do
 153        shift
 154        if kill -0 "$pid" 2>/dev/null; then
 155          debug "$pid is still alive."
 156          set -- "$@" "$pid"
 157        elif wait "$pid"; then
 158          debug "$pid exited with zero exit status."
 159        else
 160          debug "$pid exited with non-zero exit status."
 161          errors=$(($errors + 1))
 162        fi
 163      done
 164      [ $# -eq 0 ] && break
 165      sleep ${WAITALL_DELAY:-1}
 166     done
 167    [ $errors -eq 0 ]
 168 }
 169
 170 ######
 171 # MAIN
 172 ######
 173
 174 do_join() {
 175         waitall $pids
 176         pids=""
 177         # Reading the output of jobs to compute failed & numtests
 178         # Tests are run in parallel but sum should be done sequentialy to avoid
 179         # races between threads
 180         while [ "$running_jobs" -ge 0 ]; do
 181             if [ -f $output_file.$running_jobs ]; then
 182                 read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2)
 183                 read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2)
 184                 rm -f $output_file.$running_jobs
 185                 failed=$(($failed + $read_failed))
 186                 numtests=$(($numtests + $read_numtests))
 187             fi
 188             running_jobs=$(($running_jobs - 1))
 189         done
 190         running_jobs=0
 191 }
 192
 193 # Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical
 194 # processors
 195 if [ `uname` == FreeBSD -o `uname` == Darwin ]; then
 196   NPROC=`sysctl -n hw.ncpu`
 197   max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
 198 else
 199   max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)}
 200 fi
 201
 202 output_file=`mktemp /tmp/output_file-XXXXXXXXX`
 203 running_jobs=0
 204
 205 for arversion in `ls $dir/archive | sort -n`; do
 206   vdir="$dir/archive/$arversion"
 207   #echo $vdir
 208
 209   if [ ! -d "$vdir/objects" ]; then
 210     continue;
 211   fi
 212
 213   for type in `ls $vdir/objects`; do
 214     test_object $type $output_file.$running_jobs &
 215     pids="$pids $!"
 216     running_jobs=$(($running_jobs + 1))
 217
 218     # Once we spawned enough jobs, let's wait them to complete
 219     # Every spawned job have almost the same execution time so
 220     # it's not a big deal having them not ending at the same time
 221     if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then
 222         do_join
 223     fi
 224     rm -f ${output_file}*
 225   done
 226 done
 227
 228 do_join
 229
 230 if [ $failed -gt 0 ]; then
 231   echo "FAILED $failed / $numtests tests."
 232   exit 1
 233 fi
 234
 235 if [ $numtests -eq 0 ]; then
 236   echo "FAILED: no tests found to run!"
 237   exit 1
 238 fi
 239
 240 echo "passed $numtests tests."
 241