src/binaryen/auto_update_tests.py

   1 #!/usr/bin/env python
   2
   3 import os, sys, subprocess, difflib
   4
   5 from scripts.test.support import run_command, split_wast
   6 from scripts.test.shared import (
   7     ASM2WASM, MOZJS, S2WASM, WASM_SHELL, WASM_OPT, WASM_AS, WASM_DIS,
   8     WASM_CTOR_EVAL, WASM_MERGE, WASM_REDUCE, WASM2ASM,
   9     BINARYEN_INSTALL_DIR, has_shell_timeout)
  10 from scripts.test.wasm2asm import tests, spec_tests, extra_tests, assert_tests
  11
  12
  13 print '[ processing and updating testcases... ]\n'
  14
  15 for asm in sorted(os.listdir('test')):
  16   if asm.endswith('.asm.js'):
  17     for precise in [0, 1, 2]:
  18       for opts in [1, 0]:
  19         cmd = ASM2WASM + [os.path.join('test', asm), '--enable-threads']
  20         wasm = asm.replace('.asm.js', '.fromasm')
  21         if not precise:
  22           cmd += ['--trap-mode=allow', '--ignore-implicit-traps']
  23           wasm += '.imprecise'
  24         elif precise == 2:
  25           cmd += ['--trap-mode=clamp']
  26           wasm += '.clamp'
  27         if not opts:
  28           wasm += '.no-opts'
  29           if precise:
  30             cmd += ['-O0'] # test that -O0 does nothing
  31         else:
  32           cmd += ['-O']
  33         if 'debugInfo' in asm:
  34           cmd += ['-g']
  35         if 'noffi' in asm:
  36           cmd += ['--no-legalize-javascript-ffi']
  37         if precise and opts:
  38           # test mem init importing
  39           open('a.mem', 'wb').write(asm)
  40           cmd += ['--mem-init=a.mem']
  41           if asm[0] == 'e':
  42             cmd += ['--mem-base=1024']
  43         if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm:
  44           cmd += ['--wasm-only']
  45         print ' '.join(cmd)
  46         actual = run_command(cmd)
  47         with open(os.path.join('test', wasm), 'w') as o: o.write(actual)
  48         if 'debugInfo' in asm:
  49           cmd += ['--source-map', os.path.join('test', wasm + '.map'), '-o', 'a.wasm']
  50           run_command(cmd)
  51
  52 extension_arg_map = {
  53   '.wast': [],
  54   '.clamp.wast': ['--trap-mode=clamp'],
  55   '.js.wast': ['--trap-mode=js'],
  56 }
  57 for dot_s_dir in ['dot_s', 'llvm_autogenerated']:
  58   for s in sorted(os.listdir(os.path.join('test', dot_s_dir))):
  59     if not s.endswith('.s'): continue
  60     print '..', s
  61     for ext, ext_args in extension_arg_map.iteritems():
  62       wasm = s.replace('.s', ext)
  63       expected_file = os.path.join('test', dot_s_dir, wasm)
  64       if ext != '.wast' and not os.path.exists(expected_file):
  65         continue
  66
  67       full = os.path.join('test', dot_s_dir, s)
  68       stack_alloc = ['--allocate-stack=1024'] if dot_s_dir == 'llvm_autogenerated' else []
  69       cmd = S2WASM + [full, '--emscripten-glue'] + stack_alloc + ext_args
  70       if s.startswith('start_'):
  71         cmd.append('--start')
  72       actual = run_command(cmd, stderr=subprocess.PIPE, expected_err='')
  73
  74       with open(expected_file, 'w') as o: o.write(actual)
  75
  76 for t in sorted(os.listdir(os.path.join('test', 'print'))):
  77   if t.endswith('.wast'):
  78     print '..', t
  79     wasm = os.path.basename(t).replace('.wast', '')
  80     cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print']
  81     print '    ', ' '.join(cmd)
  82     actual = subprocess.check_output(cmd)
  83     print cmd, actual
  84     with open(os.path.join('test', 'print', wasm + '.txt'), 'w') as o: o.write(actual)
  85     cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print-minified']
  86     print '    ', ' '.join(cmd)
  87     actual = subprocess.check_output(cmd)
  88     with open(os.path.join('test', 'print', wasm + '.minified.txt'), 'w') as o: o.write(actual)
  89
  90 for t in sorted(os.listdir(os.path.join('test', 'passes'))):
  91   if t.endswith(('.wast', '.wasm')):
  92     print '..', t
  93     binary = '.wasm' in t
  94     passname = os.path.basename(t).replace('.wast', '').replace('.wasm', '')
  95     opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')]
  96     t = os.path.join('test', 'passes', t)
  97     actual = ''
  98     for module, asserts in split_wast(t):
  99       assert len(asserts) == 0
 100       with open('split.wast', 'w') as o: o.write(module)
 101       cmd = WASM_OPT + opts + ['split.wast', '--print']
 102       actual += run_command(cmd)
 103     with open(os.path.join('test', 'passes', passname + ('.bin' if binary else '') + '.txt'), 'w') as o: o.write(actual)
 104     if 'emit-js-wrapper' in t:
 105       with open('a.js') as i:
 106         with open(t + '.js', 'w') as o:
 107           o.write(i.read())
 108     if 'emit-spec-wrapper' in t:
 109       with open('a.wat') as i:
 110         with open(t + '.wat', 'w') as o:
 111           o.write(i.read())
 112
 113 print '\n[ checking wasm-opt -o notation... ]\n'
 114
 115 wast = os.path.join('test', 'hello_world.wast')
 116 cmd = WASM_OPT + [wast, '-o', 'a.wast', '-S']
 117 run_command(cmd)
 118 open(wast, 'w').write(open('a.wast').read())
 119
 120 print '\n[ checking binary format testcases... ]\n'
 121
 122 for wast in sorted(os.listdir('test')):
 123   if wast.endswith('.wast') and not wast in []: # blacklist some known failures
 124     for debug_info in [0, 1]:
 125       cmd = WASM_AS + [os.path.join('test', wast), '-o', 'a.wasm']
 126       if debug_info: cmd += ['-g']
 127       print ' '.join(cmd)
 128       if os.path.exists('a.wasm'): os.unlink('a.wasm')
 129       subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 130       assert os.path.exists('a.wasm')
 131
 132       cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast']
 133       print ' '.join(cmd)
 134       if os.path.exists('a.wast'): os.unlink('a.wast')
 135       subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 136       assert os.path.exists('a.wast')
 137       actual = open('a.wast').read()
 138       binary_name = wast + '.fromBinary'
 139       if not debug_info: binary_name += '.noDebugInfo'
 140       with open(os.path.join('test', binary_name), 'w') as o: o.write(actual)
 141
 142 print '\n[ checking example testcases... ]\n'
 143
 144 for t in sorted(os.listdir(os.path.join('test', 'example'))):
 145   output_file = os.path.join('bin', 'example')
 146   libdir = os.path.join(BINARYEN_INSTALL_DIR, 'lib')
 147   cmd = ['-Isrc', '-g', '-lasmjs', '-lsupport', '-L' + libdir, '-pthread', '-o', output_file]
 148   if t.endswith('.txt'):
 149     # check if there is a trace in the file, if so, we should build it
 150     out = subprocess.Popen([os.path.join('scripts', 'clean_c_api_trace.py'), os.path.join('test', 'example', t)], stdout=subprocess.PIPE).communicate()[0]
 151     if len(out) == 0:
 152       print '  (no trace in ', t, ')'
 153       continue
 154     print '  (will check trace in ', t, ')'
 155     src = 'trace.cpp'
 156     with open(src, 'w') as o: o.write(out)
 157     expected = os.path.join('test', 'example', t + '.txt')
 158   else:
 159     src = os.path.join('test', 'example', t)
 160     expected = os.path.join('test', 'example', '.'.join(t.split('.')[:-1]) + '.txt')
 161   if not src.endswith(('.c', '.cpp')):
 162     continue
 163   # build the C file separately
 164   extra = [os.environ.get('CC') or 'gcc',
 165            src, '-c', '-o', 'example.o',
 166            '-Isrc', '-g', '-L' + libdir, '-pthread']
 167   print 'build: ', ' '.join(extra)
 168   print os.getcwd()
 169   subprocess.check_call(extra)
 170   # Link against the binaryen C library DSO, using rpath
 171   cmd = ['example.o', '-lbinaryen', '-Wl,-rpath=' + os.path.abspath(libdir)] + cmd
 172   print '  ', t, src, expected
 173   if os.environ.get('COMPILER_FLAGS'):
 174     for f in os.environ.get('COMPILER_FLAGS').split(' '):
 175       cmd.append(f)
 176   cmd = [os.environ.get('CXX') or 'g++', '-std=c++11'] + cmd
 177   try:
 178     print 'link: ', ' '.join(cmd)
 179     subprocess.check_call(cmd)
 180     print 'run...', output_file
 181     proc = subprocess.Popen([output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 182     actual, err = proc.communicate()
 183     assert proc.returncode == 0, [proc.returncode, actual, err]
 184     with open(expected, 'w') as o: o.write(actual)
 185   finally:
 186     os.remove(output_file)
 187     if sys.platform == 'darwin':
 188       # Also removes debug directory produced on Mac OS
 189       shutil.rmtree(output_file + '.dSYM')
 190
 191 print '\n[ checking wasm-opt testcases... ]\n'
 192
 193 for t in os.listdir('test'):
 194   if t.endswith('.wast') and not t.startswith('spec'):
 195     print '..', t
 196     t = os.path.join('test', t)
 197     f = t + '.from-wast'
 198     cmd = WASM_OPT + [t, '--print']
 199     actual = run_command(cmd)
 200     actual = actual.replace('printing before:\n', '')
 201     open(f, 'w').write(actual)
 202
 203 print '\n[ checking wasm-dis on provided binaries... ]\n'
 204
 205 for t in os.listdir('test'):
 206   if t.endswith('.wasm') and not t.startswith('spec'):
 207     print '..', t
 208     t = os.path.join('test', t)
 209     cmd = WASM_DIS + [t]
 210     if os.path.isfile(t + '.map'): cmd += ['--source-map', t + '.map']
 211     actual = run_command(cmd)
 212
 213     open(t + '.fromBinary', 'w').write(actual)
 214
 215 print '\n[ checking wasm-merge... ]\n'
 216
 217 for t in os.listdir(os.path.join('test', 'merge')):
 218   if t.endswith(('.wast', '.wasm')):
 219     print '..', t
 220     t = os.path.join('test', 'merge', t)
 221     u = t + '.toMerge'
 222     for finalize in [0, 1]:
 223       for opt in [0, 1]:
 224         cmd = WASM_MERGE + [t, u, '-o', 'a.wast', '-S', '--verbose']
 225         if finalize: cmd += ['--finalize-memory-base=1024', '--finalize-table-base=8']
 226         if opt: cmd += ['-O']
 227         stdout = run_command(cmd)
 228         actual = open('a.wast').read()
 229         out = t + '.combined'
 230         if finalize: out += '.finalized'
 231         if opt: out += '.opt'
 232         with open(out, 'w') as o: o.write(actual)
 233         with open(out + '.stdout', 'w') as o: o.write(stdout)
 234
 235 if MOZJS:
 236   print '\n[ checking binaryen.js testcases... ]\n'
 237
 238   for s in sorted(os.listdir(os.path.join('test', 'binaryen.js'))):
 239     if not s.endswith('.js'): continue
 240     print s
 241     f = open('a.js', 'w')
 242     f.write(open(os.path.join('bin', 'binaryen.js')).read())
 243     f.write(open(os.path.join('test', 'binaryen.js', s)).read())
 244     f.close()
 245     cmd = [MOZJS, 'a.js']
 246     out = run_command(cmd, stderr=subprocess.STDOUT)
 247     with open(os.path.join('test', 'binaryen.js', s + '.txt'), 'w') as o: o.write(out)
 248
 249 print '\n[ checking wasm-ctor-eval... ]\n'
 250
 251 for t in os.listdir(os.path.join('test', 'ctor-eval')):
 252   if t.endswith(('.wast', '.wasm')):
 253     print '..', t
 254     t = os.path.join('test', 'ctor-eval', t)
 255     ctors = open(t + '.ctors').read().strip()
 256     cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors]
 257     stdout = run_command(cmd)
 258     actual = open('a.wast').read()
 259     out = t + '.out'
 260     with open(out, 'w') as o: o.write(actual)
 261
 262 print '\n[ checking wasm2asm ]\n'
 263
 264 for wasm in tests + spec_tests + extra_tests:
 265   if not wasm.endswith('.wast'):
 266     continue
 267
 268   asm = os.path.basename(wasm).replace('.wast', '.2asm.js')
 269   expected_file = os.path.join('test', asm)
 270
 271   if not os.path.exists(expected_file):
 272     continue
 273
 274   print '..', wasm
 275
 276   cmd = WASM2ASM + [os.path.join('test', wasm)]
 277   out = run_command(cmd)
 278   with open(expected_file, 'w') as o: o.write(out)
 279
 280 for wasm in assert_tests:
 281   print '..', wasm
 282
 283   asserts = os.path.basename(wasm).replace('.wast.asserts', '.asserts.js')
 284   traps = os.path.basename(wasm).replace('.wast.asserts', '.traps.js')
 285   asserts_expected_file = os.path.join('test', asserts)
 286   traps_expected_file = os.path.join('test', traps)
 287
 288   cmd = WASM2ASM + [os.path.join('test', wasm), '--allow-asserts']
 289   out = run_command(cmd)
 290   with open(asserts_expected_file, 'w') as o: o.write(out)
 291
 292   cmd += ['--pedantic']
 293   out = run_command(cmd)
 294   with open(traps_expected_file, 'w') as o: o.write(out)
 295
 296 if has_shell_timeout():
 297   print '\n[ checking wasm-reduce ]\n'
 298
 299   for t in os.listdir(os.path.join('test', 'reduce')):
 300     if t.endswith('.wast'):
 301       print '..', t
 302       t = os.path.join('test', 'reduce', t)
 303       # convert to wasm
 304       run_command(WASM_AS + [t, '-o', 'a.wasm'])
 305       print run_command(WASM_REDUCE + ['a.wasm', '--command=bin/wasm-opt b.wasm --fuzz-exec', '-t', 'b.wasm', '-w', 'c.wasm'])
 306       expected = t + '.txt'
 307       run_command(WASM_DIS + ['c.wasm', '-o', expected])
 308
 309 print '\n[ success! ]'