test/Makefile

   1 # Build the `libc-test` tests as Wasm programs and run them with the selected
   2 # engine. Contributors beware! This Makefile follows the style of the
   3 # `libc-test` Makefile and uses some more exotic features of `make`.
   4 #
   5 # The top-level `test` target is composed of a chain of several phony
   6 # sub-targets:
   7 # - `download`: retrieve the `libc-test` source from a Git `$(MIRROR)`
   8 # - `build`: construct Wasm modules for a subset of the `libc-test` tests
   9 # - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
  10 #   Wasmtime)
  11
  12 test: run
  13
  14 # Unlike the `libc-test` directory, we will output all artifacts to the `build`
  15 # directory (keeping with the `wasi-libc` conventions).
  16 OBJDIR ?= $(CURDIR)/build
  17 DOWNDIR ?= $(CURDIR)/download
  18
  19 ##### DOWNLOAD #################################################################
  20
  21 LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
  22 LIBC_TEST = $(DOWNDIR)/libc-test
  23 LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-16/libclang_rt.builtins-wasm32-wasi-16.0.tar.gz
  24 LIBRT = $(DOWNDIR)/lib/wasi/libclang_rt.builtins-wasm32.a
  25 WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v3.0.0/wasmtime-v3.0.0-x86_64-linux.tar.xz
  26 WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
  27
  28 download: $(LIBC_TEST) $(LIBRT) $(WASMTIME)
  29
  30 $(DOWNDIR):
  31         mkdir -p download
  32
  33 $(LIBC_TEST): | $(DOWNDIR)
  34         git clone --depth 1 $(LIBC_TEST_URL) $@
  35
  36 # TODO install target to place into...
  37 $(LIBRT): | $(DOWNDIR)
  38         wget --no-clobber --directory-prefix=$(DOWNDIR) $(LIBRT_URL)
  39         tar --extract --file=$(DOWNDIR)/$(shell basename $(LIBRT_URL)) --directory=$(DOWNDIR)/
  40
  41 $(WASMTIME): | $(DOWNDIR)
  42         wget --no-clobber --directory-prefix=$(DOWNDIR) $(WASMTIME_URL)
  43         tar --extract --file=$(DOWNDIR)/$(shell basename $(WASMTIME_URL)) --directory=$(DOWNDIR)/
  44
  45 clean::
  46         rm -rf download
  47
  48 ##### BUILD ####################################################################
  49
  50 # For now, we list out the tests that we can currently build and run. This is
  51 # heavily focused on the functional tests; in the future it would be good to
  52 # fill out the missing tests and also include some `src/api` and `src/math`
  53 # tests (TODO).
  54 TESTS := \
  55         $(LIBC_TEST)/src/functional/argv.c \
  56         $(LIBC_TEST)/src/functional/basename.c \
  57         $(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
  58         $(LIBC_TEST)/src/functional/clock_gettime.c \
  59         $(LIBC_TEST)/src/functional/crypt.c \
  60         $(LIBC_TEST)/src/functional/dirname.c \
  61         $(LIBC_TEST)/src/functional/env.c \
  62         $(LIBC_TEST)/src/functional/fnmatch.c \
  63         $(LIBC_TEST)/src/functional/iconv_open.c \
  64         $(LIBC_TEST)/src/functional/mbc.c \
  65         $(LIBC_TEST)/src/functional/memstream.c \
  66         $(LIBC_TEST)/src/functional/qsort.c \
  67         $(LIBC_TEST)/src/functional/random.c \
  68         $(LIBC_TEST)/src/functional/search_hsearch.c \
  69         $(LIBC_TEST)/src/functional/search_insque.c \
  70         $(LIBC_TEST)/src/functional/search_lsearch.c \
  71         $(LIBC_TEST)/src/functional/search_tsearch.c \
  72         $(LIBC_TEST)/src/functional/snprintf.c \
  73         $(LIBC_TEST)/src/functional/sscanf.c \
  74         $(LIBC_TEST)/src/functional/strftime.c \
  75         $(LIBC_TEST)/src/functional/string.c \
  76         $(LIBC_TEST)/src/functional/string_memcpy.c \
  77         $(LIBC_TEST)/src/functional/string_memmem.c \
  78         $(LIBC_TEST)/src/functional/string_memset.c \
  79         $(LIBC_TEST)/src/functional/string_strchr.c \
  80         $(LIBC_TEST)/src/functional/string_strcspn.c \
  81         $(LIBC_TEST)/src/functional/string_strstr.c \
  82         $(LIBC_TEST)/src/functional/strtod.c \
  83         $(LIBC_TEST)/src/functional/strtod_long.c \
  84         $(LIBC_TEST)/src/functional/strtod_simple.c \
  85         $(LIBC_TEST)/src/functional/strtof.c \
  86         $(LIBC_TEST)/src/functional/strtol.c \
  87         $(LIBC_TEST)/src/functional/swprintf.c \
  88         $(LIBC_TEST)/src/functional/tgmath.c \
  89         $(LIBC_TEST)/src/functional/udiv.c \
  90         $(LIBC_TEST)/src/functional/wcsstr.c \
  91         $(LIBC_TEST)/src/functional/wcstol.c
  92
  93 # Part of the problem including more tests is that the `libc-test`
  94 # infrastructure code is not all Wasm-compilable. As we include more tests
  95 # above, this list will also likely need to grow.
  96 COMMON_TEST_INFRA = \
  97         $(LIBC_TEST)/src/common/path.c \
  98         $(LIBC_TEST)/src/common/print.c \
  99         $(LIBC_TEST)/src/common/rand.c \
 100         $(LIBC_TEST)/src/common/utf8.c
 101
 102 # Create various lists containing the various artifacts to be built: mainly,
 103 # $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
 104 # the $(WASMS) tests.
 105 NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
 106 WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm)
 107 WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
 108 INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
 109 WASM_OBJS += $(INFRA_WASM_OBJS)
 110 DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
 111 OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
 112
 113 # Allow $(CC) to be set from the command line; ?= doesn't work for CC because
 114 # make has a default value for it.
 115 ifeq ($(origin CC), default)
 116 CC := clang
 117 endif
 118 LDFLAGS ?=
 119 CFLAGS ?= --target=wasm32-wasi --sysroot=../sysroot
 120 # Always include the `libc-test` infrastructure headers.
 121 CFLAGS += -I$(LIBC_TEST)/src/common
 122
 123 # Compile each selected test using Clang. Note that failures here are likely
 124 # due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
 125 # This location is system-dependent, but could be fixed by something like:
 126 #  $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
 127 #  $ sudo cp download/lib/wasi/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
 128 build: download $(WASMS)
 129
 130 $(WASMS): | $(OBJDIRS)
 131 $(OBJDIR)/%.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
 132         $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
 133
 134 $(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
 135 $(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
 136         $(CC) $(CFLAGS) -c -o $@ $<
 137
 138 $(OBJDIRS):
 139         mkdir -p $@
 140
 141 clean::
 142         rm -rf $(OBJDIR)
 143
 144 ##### RUN ######################################################################
 145
 146 ENGINE ?= $(WASMTIME) run
 147 ERRS:=$(WASMS:%.wasm=%.wasm.err)
 148
 149 # Use the provided Wasm engine to execute each test, emitting its output into
 150 # a `.err` file.
 151 run: build $(ERRS)
 152         @echo "Tests passed"
 153
 154 $(ERRS): | $(OBJDIRS)
 155
 156 %.wasm.err: %.wasm
 157         $(ENGINE) $< >$@
 158
 159 clean::
 160         rm -rf $(OBJDIR)/*/*.err
 161
 162 ##### MISC #####################################################################
 163
 164 # Note: the `clean` target has been built up by all of the previous sections.
 165
 166 debug:
 167         @echo NAMES $(NAMES)
 168         @echo TESTS $(TESTS)
 169         @echo WASMS $(WASMS)
 170         @echo WASM_OBJS $(WASM_OBJS)
 171         @echo ERRS $(ERRS)
 172         @echo DIRS $(DIRS)
 173         @echo OBJDIRS $(OBJDIRS)
 174
 175 .PHONY: test download build run clean