ceph/src/test/encoding/readable.sh

   1 #!/bin/bash -e
   2
   3 source $(dirname $0)/../detect-build-env-vars.sh
   4
   5 [ -z "$CEPH_ROOT" ] && CEPH_ROOT=..
   6
   7 dir=$CEPH_ROOT/ceph-object-corpus
   8
   9 set -e
  10
  11 failed=0
  12 numtests=0
  13 pids=""
  14
  15 if [ -x ./ceph-dencoder ]; then
  16   CEPH_DENCODER=./ceph-dencoder
  17 else
  18   CEPH_DENCODER=ceph-dencoder
  19 fi
  20
  21 myversion=`$CEPH_DENCODER version`
  22 DEBUG=0
  23 WAITALL_DELAY=.1
  24 debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi }
  25
  26 test_object() {
  27     local type=$1
  28     local output_file=$2
  29     local failed=0
  30     local numtests=0
  31
  32     tmp1=`mktemp /tmp/typ-XXXXXXXXX`
  33     tmp2=`mktemp /tmp/typ-XXXXXXXXX`
  34
  35     rm -f $output_file
  36     if $CEPH_DENCODER type $type 2>/dev/null; then
  37       #echo "type $type";
  38       echo "        $vdir/objects/$type"
  39
  40       # is there a fwd incompat change between $arversion and $version?
  41       incompat=""
  42       incompat_paths=""
  43       sawarversion=0
  44       for iv in `ls $dir/archive | sort -n`; do
  45         if [ "$iv" = "$arversion" ]; then
  46           sawarversion=1
  47         fi
  48
  49         if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then
  50           incompat="$iv"
  51
  52           # Check if we'll be ignoring only specified objects, not whole type. If so, remember
  53           # all paths for this type into variable. Assuming that this path won't contain any
  54           # whitechars (implication of above for loop).
  55           if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then
  56             if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then
  57               incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type"
  58             else
  59               echo "type $type directory empty, ignoring whole type instead of single objects"
  60             fi;
  61           fi
  62         fi
  63
  64         if [ "$iv" = "$version" ]; then
  65           break
  66         fi
  67       done
  68
  69       if [ -n "$incompat" ]; then
  70         if [ -z "$incompat_paths" ]; then
  71           echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion"
  72           return
  73         else
  74           # If we are ignoring not whole type, but objects that are in $incompat_path,
  75           # we don't skip here, just give info.
  76           echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion"
  77         fi;
  78       fi
  79
  80       for f in `ls $vdir/objects/$type`; do
  81
  82         skip=0;
  83         # Check if processed object $f of $type should be skipped (postponed skip)
  84         if [ -n "$incompat_paths" ]; then
  85             for i_path in $incompat_paths; do
  86               # Check if $f is a symbolic link and if it's pointing to existing target
  87               if [ -L "$i_path/$f" ]; then
  88                 echo "skipping object $f of type $type"
  89                 skip=1
  90                 break
  91               fi;
  92             done;
  93         fi;
  94
  95         if [ $skip -ne 0 ]; then
  96           continue
  97         fi;
  98
  99         $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 &
 100         pid1="$!"
 101         $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 &
 102         pid2="$!"
 103         #echo "\t$vdir/$type/$f"
 104         if ! wait $pid1; then
 105           echo "**** failed to decode $vdir/objects/$type/$f ****"
 106           failed=$(($failed + 1))
 107           rm -f $tmp1 $tmp2
 108           continue
 109         fi
 110         if ! wait $pid2; then
 111           echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****"
 112           failed=$(($failed + 1))
 113           rm -f $tmp1 $tmp2
 114           continue
 115         fi
 116
 117         # nondeterministic classes may dump
 118         # nondeterministically.  compare the sorted json
 119         # output.  this is a weaker test, but is better than
 120         # nothing.
 121         if ! $CEPH_DENCODER type $type is_deterministic; then
 122           echo "  sorting json output for nondeterministic object"
 123           for f in $tmp1 $tmp2; do
 124             sort $f | sed 's/,$//' > $f.new
 125             mv $f.new $f
 126           done
 127         fi
 128
 129         if ! cmp $tmp1 $tmp2; then
 130           echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****"
 131           diff $tmp1 $tmp2
 132           failed=$(($failed + 1))
 133         fi
 134         numtests=$(($numtests + 1))
 135       done
 136     else
 137       echo "skipping unrecognized type $type"
 138     fi
 139
 140     echo "failed=$failed" > $output_file
 141     echo "numtests=$numtests" >> $output_file
 142     rm -f $tmp1 $tmp2
 143 }
 144
 145 waitall() { # PID...
 146    ## Wait for children to exit and indicate whether all exited with 0 status.
 147    local errors=0
 148    while :; do
 149      debug "Processes remaining: $*"
 150      for pid in "$@"; do
 151        shift
 152        if kill -0 "$pid" 2>/dev/null; then
 153          debug "$pid is still alive."
 154          set -- "$@" "$pid"
 155        elif wait "$pid"; then
 156          debug "$pid exited with zero exit status."
 157        else
 158          debug "$pid exited with non-zero exit status."
 159          errors=$(($errors + 1))
 160        fi
 161      done
 162      [ $# -eq 0 ] && break
 163      sleep ${WAITALL_DELAY:-1}
 164     done
 165    [ $errors -eq 0 ]
 166 }
 167
 168 ######
 169 # MAIN
 170 ######
 171
 172 do_join() {
 173         waitall $pids
 174         pids=""
 175         # Reading the output of jobs to compute failed & numtests
 176         # Tests are run in parallel but sum should be done sequentialy to avoid
 177         # races between threads
 178         while [ "$running_jobs" -ge 0 ]; do
 179             if [ -f $output_file.$running_jobs ]; then
 180                 read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2)
 181                 read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2)
 182                 rm -f $output_file.$running_jobs
 183                 failed=$(($failed + $read_failed))
 184                 numtests=$(($numtests + $read_numtests))
 185             fi
 186             running_jobs=$(($running_jobs - 1))
 187         done
 188         running_jobs=0
 189 }
 190
 191 # Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical
 192 # processors
 193 if [ `uname` == FreeBSD ]; then
 194   NPROC=`sysctl -n hw.ncpu`
 195   max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
 196 else
 197   max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)}
 198 fi
 199
 200 output_file=`mktemp /tmp/typ-XXXXXXXXX`
 201 running_jobs=0
 202
 203 for arversion in `ls $dir/archive | sort -n`; do
 204   vdir="$dir/archive/$arversion"
 205   #echo $vdir
 206
 207   if [ ! -d "$vdir/objects" ]; then
 208     continue;
 209   fi
 210
 211   for type in `ls $vdir/objects`; do
 212     test_object $type $output_file.$running_jobs &
 213     pids="$pids $!"
 214     running_jobs=$(($running_jobs + 1))
 215
 216     # Once we spawned enough jobs, let's wait them to complete
 217     # Every spawned job have almost the same execution time so
 218     # it's not a big deal having them not ending at the same time
 219     if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then
 220         do_join
 221     fi
 222   done
 223 done
 224
 225 do_join
 226
 227 if [ $failed -gt 0 ]; then
 228   echo "FAILED $failed / $numtests tests."
 229   exit 1
 230 fi
 231 echo "passed $numtests tests."
 232