ceph/src/test/encoding/readable.sh

   1 #!/bin/bash -e
   2
   3 source $(dirname $0)/../detect-build-env-vars.sh
   4
   5 [ -z "$CEPH_ROOT" ] && CEPH_ROOT=..
   6
   7 dir=$CEPH_ROOT/ceph-object-corpus
   8
   9 set -e
  10
  11 failed=0
  12 numtests=0
  13 pids=""
  14
  15 if [ -x ./ceph-dencoder ]; then
  16   CEPH_DENCODER=./ceph-dencoder
  17 else
  18   CEPH_DENCODER=ceph-dencoder
  19 fi
  20
  21 myversion=`$CEPH_DENCODER version`
  22 DEBUG=0
  23 WAITALL_DELAY=.1
  24 debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi }
  25
  26 test_object() {
  27     local type=$1
  28     local output_file=$2
  29     local failed=0
  30     local numtests=0
  31
  32     tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX`
  33     tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX`
  34
  35     rm -f $output_file
  36     if $CEPH_DENCODER type $type 2>/dev/null; then
  37       #echo "type $type";
  38       echo "        $vdir/objects/$type"
  39
  40       # is there a fwd incompat change between $arversion and $version?
  41       incompat=""
  42       incompat_paths=""
  43       sawarversion=0
  44       for iv in `ls $dir/archive | sort -n`; do
  45         if [ "$iv" = "$arversion" ]; then
  46           sawarversion=1
  47         fi
  48
  49         if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then
  50           incompat="$iv"
  51
  52           # Check if we'll be ignoring only specified objects, not whole type. If so, remember
  53           # all paths for this type into variable. Assuming that this path won't contain any
  54           # whitechars (implication of above for loop).
  55           if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then
  56             if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then
  57               incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type"
  58             else
  59               echo "type $type directory empty, ignoring whole type instead of single objects"
  60             fi;
  61           fi
  62         fi
  63
  64         if [ "$iv" = "$version" ]; then
  65           rm -rf $tmp1 $tmp2
  66           break
  67         fi
  68       done
  69
  70       if [ -n "$incompat" ]; then
  71         if [ -z "$incompat_paths" ]; then
  72           echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion"
  73           rm -rf $tmp1 $tmp2
  74           return
  75         else
  76           # If we are ignoring not whole type, but objects that are in $incompat_path,
  77           # we don't skip here, just give info.
  78           echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion"
  79         fi;
  80       fi
  81
  82       for f in `ls $vdir/objects/$type`; do
  83
  84         skip=0;
  85         # Check if processed object $f of $type should be skipped (postponed skip)
  86         if [ -n "$incompat_paths" ]; then
  87             for i_path in $incompat_paths; do
  88               # Check if $f is a symbolic link and if it's pointing to existing target
  89               if [ -L "$i_path/$f" ]; then
  90                 echo "skipping object $f of type $type"
  91                 skip=1
  92                 break
  93               fi;
  94             done;
  95         fi;
  96
  97         if [ $skip -ne 0 ]; then
  98           continue
  99         fi;
 100
 101         $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 &
 102         pid1="$!"
 103         $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 &
 104         pid2="$!"
 105         #echo "\t$vdir/$type/$f"
 106         if ! wait $pid1; then
 107           echo "**** failed to decode $vdir/objects/$type/$f ****"
 108           failed=$(($failed + 1))
 109           rm -f $tmp1 $tmp2
 110           continue
 111         fi
 112         if ! wait $pid2; then
 113           echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****"
 114           failed=$(($failed + 1))
 115           rm -f $tmp1 $tmp2
 116           continue
 117         fi
 118
 119         # nondeterministic classes may dump
 120         # nondeterministically.  compare the sorted json
 121         # output.  this is a weaker test, but is better than
 122         # nothing.
 123         if ! $CEPH_DENCODER type $type is_deterministic; then
 124           echo "  sorting json output for nondeterministic object"
 125           for f in $tmp1 $tmp2; do
 126             sort $f | sed 's/,$//' > $f.new
 127             mv $f.new $f
 128           done
 129         fi
 130
 131         if ! cmp $tmp1 $tmp2; then
 132           echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****"
 133           diff $tmp1 $tmp2
 134           failed=$(($failed + 1))
 135         fi
 136         numtests=$(($numtests + 1))
 137         rm -f $tmp1 $tmp2
 138       done
 139     else
 140       echo "skipping unrecognized type $type"
 141       rm -f $tmp1 $tmp2
 142     fi
 143
 144     echo "failed=$failed" > $output_file
 145     echo "numtests=$numtests" >> $output_file
 146 }
 147
 148 waitall() { # PID...
 149    ## Wait for children to exit and indicate whether all exited with 0 status.
 150    local errors=0
 151    while :; do
 152      debug "Processes remaining: $*"
 153      for pid in "$@"; do
 154        shift
 155        if kill -0 "$pid" 2>/dev/null; then
 156          debug "$pid is still alive."
 157          set -- "$@" "$pid"
 158        elif wait "$pid"; then
 159          debug "$pid exited with zero exit status."
 160        else
 161          debug "$pid exited with non-zero exit status."
 162          errors=$(($errors + 1))
 163        fi
 164      done
 165      [ $# -eq 0 ] && break
 166      sleep ${WAITALL_DELAY:-1}
 167     done
 168    [ $errors -eq 0 ]
 169 }
 170
 171 ######
 172 # MAIN
 173 ######
 174
 175 do_join() {
 176         waitall $pids
 177         pids=""
 178         # Reading the output of jobs to compute failed & numtests
 179         # Tests are run in parallel but sum should be done sequentialy to avoid
 180         # races between threads
 181         while [ "$running_jobs" -ge 0 ]; do
 182             if [ -f $output_file.$running_jobs ]; then
 183                 read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2)
 184                 read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2)
 185                 rm -f $output_file.$running_jobs
 186                 failed=$(($failed + $read_failed))
 187                 numtests=$(($numtests + $read_numtests))
 188             fi
 189             running_jobs=$(($running_jobs - 1))
 190         done
 191         running_jobs=0
 192 }
 193
 194 # Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical
 195 # processors
 196 if [ `uname` == FreeBSD ]; then
 197   NPROC=`sysctl -n hw.ncpu`
 198   max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
 199 else
 200   max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)}
 201 fi
 202
 203 output_file=`mktemp /tmp/output_file-XXXXXXXXX`
 204 running_jobs=0
 205
 206 for arversion in `ls $dir/archive | sort -n`; do
 207   vdir="$dir/archive/$arversion"
 208   #echo $vdir
 209
 210   if [ ! -d "$vdir/objects" ]; then
 211     continue;
 212   fi
 213
 214   for type in `ls $vdir/objects`; do
 215     test_object $type $output_file.$running_jobs &
 216     pids="$pids $!"
 217     running_jobs=$(($running_jobs + 1))
 218
 219     # Once we spawned enough jobs, let's wait them to complete
 220     # Every spawned job have almost the same execution time so
 221     # it's not a big deal having them not ending at the same time
 222     if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then
 223         do_join
 224     fi
 225     rm -f ${output_file}*
 226   done
 227 done
 228
 229 do_join
 230
 231 if [ $failed -gt 0 ]; then
 232   echo "FAILED $failed / $numtests tests."
 233   exit 1
 234 fi
 235 echo "passed $numtests tests."
 236