From 8098d8621ef6eeaa27f71bc4a0efb9dd51ea8f5e Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Fri, 2 Dec 2022 18:27:42 -0800 Subject: [PATCH] test: run a subset of tests using `libc-test` (#346) * test: run a subset of tests from `libc-test` This change introduces a `test` directory that retrieves the `libc-test` suite, compiles a subset of the tests using `wasi-libc`, and runs them with Wasmtime. * ci: run tests during CI This change includes some fixups to the filesystem to place Clang's runtime library for `wasm32-wasi` in the right location. Note that this CI action is limited to a single OS--Linux. --- .github/workflows/main.yml | 17 ++++ Makefile | 2 +- test/.gitignore | 3 + test/Makefile | 175 +++++++++++++++++++++++++++++++++++++ test/README.md | 6 ++ 5 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 test/.gitignore create mode 100644 test/Makefile create mode 100644 test/README.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 17ef735..83c2751 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,7 @@ jobs: curl -sSfL https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.clang_version }}/clang+llvm-${{ matrix.clang_version }}-x86_64-linux-gnu-ubuntu-18.04.tar.xz | tar xJf - export CLANG_DIR=`pwd`/clang+llvm-${{ matrix.clang_version }}-x86_64-linux-gnu-ubuntu-18.04/bin echo "$CLANG_DIR" >> $GITHUB_PATH + echo "CLANG_DIR=$CLANG_DIR" >> $GITHUB_ENV echo "CC=$CLANG_DIR/clang" >> $GITHUB_ENV echo "AR=$CLANG_DIR/llvm-ar" >> $GITHUB_ENV echo "NM=$CLANG_DIR/llvm-nm" >> $GITHUB_ENV @@ -67,6 +68,22 @@ jobs: shell: bash run: make -j4 + - name: Test + shell: bash + # For Clang linking to work correctly, we need to place Clang's runtime + # library for `wasm32-wasi` in the right location (i.e., the `mkdir` and + # `cp` below). + run: | + cd test + make download + export WASI_DIR=$(realpath $CLANG_DIR/../lib/clang/${{ matrix.clang_version }}/lib/wasi/) + mkdir -p $WASI_DIR + cp download/lib/wasi/libclang_rt.builtins-wasm32.a $WASI_DIR + make test + # The older version of Clang does not provide the expected symbol for the + # test entrypoints: `undefined symbol: __main_argc_argv`. + if: matrix.os == 'ubuntu-latest' && matrix.clang_version != '10.0.0' + - uses: actions/upload-artifact@v1 with: # Upload the sysroot folder. Give it a name according to the OS it was built for. diff --git a/Makefile b/Makefile index d6a20f5..aa12cf1 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ THREAD_MODEL ?= single MALLOC_IMPL ?= dlmalloc # yes or no BUILD_LIBC_TOP_HALF ?= yes -# The directory where we're store intermediate artifacts. +# The directory where we will store intermediate artifacts. OBJDIR ?= $(CURDIR)/build # When the length is no larger than this threshold, we consider the diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..f5fa56e --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,3 @@ +build +download +run diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..61e1756 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,175 @@ +# Build the `libc-test` tests as Wasm programs and run them with the selected +# engine. Contributors beware! This Makefile follows the style of the +# `libc-test` Makefile and uses some more exotic features of `make`. +# +# The top-level `test` target is composed of a chain of several phony +# sub-targets: +# - `download`: retrieve the `libc-test` source from a Git `$(MIRROR)` +# - `build`: construct Wasm modules for a subset of the `libc-test` tests +# - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g., +# Wasmtime) + +test: run + +# Unlike the `libc-test` directory, we will output all artifacts to the `build` +# directory (keeping with the `wasi-libc` conventions). +OBJDIR ?= $(CURDIR)/build +DOWNDIR ?= $(CURDIR)/download + +##### DOWNLOAD ################################################################# + +LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test +LIBC_TEST = $(DOWNDIR)/libc-test +LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-16/libclang_rt.builtins-wasm32-wasi-16.0.tar.gz +LIBRT = $(DOWNDIR)/lib/wasi/libclang_rt.builtins-wasm32.a +WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v3.0.0/wasmtime-v3.0.0-x86_64-linux.tar.xz +WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime + +download: $(LIBC_TEST) $(LIBRT) $(WASMTIME) + +$(DOWNDIR): + mkdir -p download + +$(LIBC_TEST): | $(DOWNDIR) + git clone --depth 1 $(LIBC_TEST_URL) $@ + +# TODO install target to place into... +$(LIBRT): | $(DOWNDIR) + wget --no-clobber --directory-prefix=$(DOWNDIR) $(LIBRT_URL) + tar --extract --file=$(DOWNDIR)/$(shell basename $(LIBRT_URL)) --directory=$(DOWNDIR)/ + +$(WASMTIME): | $(DOWNDIR) + wget --no-clobber --directory-prefix=$(DOWNDIR) $(WASMTIME_URL) + tar --extract --file=$(DOWNDIR)/$(shell basename $(WASMTIME_URL)) --directory=$(DOWNDIR)/ + +clean:: + rm -rf download + +##### BUILD #################################################################### + +# For now, we list out the tests that we can currently build and run. This is +# heavily focused on the functional tests; in the future it would be good to +# fill out the missing tests and also include some `src/api` and `src/math` +# tests (TODO). +TESTS := \ + $(LIBC_TEST)/src/functional/argv.c \ + $(LIBC_TEST)/src/functional/basename.c \ + $(LIBC_TEST)/src/functional/clocale_mbfuncs.c \ + $(LIBC_TEST)/src/functional/clock_gettime.c \ + $(LIBC_TEST)/src/functional/crypt.c \ + $(LIBC_TEST)/src/functional/dirname.c \ + $(LIBC_TEST)/src/functional/env.c \ + $(LIBC_TEST)/src/functional/fnmatch.c \ + $(LIBC_TEST)/src/functional/iconv_open.c \ + $(LIBC_TEST)/src/functional/mbc.c \ + $(LIBC_TEST)/src/functional/memstream.c \ + $(LIBC_TEST)/src/functional/qsort.c \ + $(LIBC_TEST)/src/functional/random.c \ + $(LIBC_TEST)/src/functional/search_hsearch.c \ + $(LIBC_TEST)/src/functional/search_insque.c \ + $(LIBC_TEST)/src/functional/search_lsearch.c \ + $(LIBC_TEST)/src/functional/search_tsearch.c \ + $(LIBC_TEST)/src/functional/snprintf.c \ + $(LIBC_TEST)/src/functional/sscanf.c \ + $(LIBC_TEST)/src/functional/strftime.c \ + $(LIBC_TEST)/src/functional/string.c \ + $(LIBC_TEST)/src/functional/string_memcpy.c \ + $(LIBC_TEST)/src/functional/string_memmem.c \ + $(LIBC_TEST)/src/functional/string_memset.c \ + $(LIBC_TEST)/src/functional/string_strchr.c \ + $(LIBC_TEST)/src/functional/string_strcspn.c \ + $(LIBC_TEST)/src/functional/string_strstr.c \ + $(LIBC_TEST)/src/functional/strtod.c \ + $(LIBC_TEST)/src/functional/strtod_long.c \ + $(LIBC_TEST)/src/functional/strtod_simple.c \ + $(LIBC_TEST)/src/functional/strtof.c \ + $(LIBC_TEST)/src/functional/strtol.c \ + $(LIBC_TEST)/src/functional/swprintf.c \ + $(LIBC_TEST)/src/functional/tgmath.c \ + $(LIBC_TEST)/src/functional/udiv.c \ + $(LIBC_TEST)/src/functional/wcsstr.c \ + $(LIBC_TEST)/src/functional/wcstol.c + +# Part of the problem including more tests is that the `libc-test` +# infrastructure code is not all Wasm-compilable. As we include more tests +# above, this list will also likely need to grow. +COMMON_TEST_INFRA = \ + $(LIBC_TEST)/src/common/path.c \ + $(LIBC_TEST)/src/common/print.c \ + $(LIBC_TEST)/src/common/rand.c \ + $(LIBC_TEST)/src/common/utf8.c + +# Create various lists containing the various artifacts to be built: mainly, +# $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form +# the $(WASMS) tests. +NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%) +WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm) +WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o) +INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o) +WASM_OBJS += $(INFRA_WASM_OBJS) +DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS)))) +OBJDIRS := $(DIRS:%=$(OBJDIR)/%) + +# Allow $(CC) to be set from the command line; ?= doesn't work for CC because +# make has a default value for it. +ifeq ($(origin CC), default) +CC := clang +endif +LDFLAGS ?= +CFLAGS ?= --target=wasm32-wasi --sysroot=../sysroot +# Always include the `libc-test` infrastructure headers. +CFLAGS += -I$(LIBC_TEST)/src/common + +# Compile each selected test using Clang. Note that failures here are likely +# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory. +# This location is system-dependent, but could be fixed by something like: +# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi +# $ sudo cp download/lib/wasi/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/ +build: download $(WASMS) + +$(WASMS): | $(OBJDIRS) +$(OBJDIR)/%.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ + +$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS) +$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c + $(CC) $(CFLAGS) -c -o $@ $< + +$(OBJDIRS): + mkdir -p $@ + +clean:: + rm -rf $(OBJDIR) + +##### RUN ###################################################################### + +ENGINE ?= $(WASMTIME) run +ERRS:=$(WASMS:%.wasm=%.wasm.err) + +# Use the provided Wasm engine to execute each test, emitting its output into +# a `.err` file. +run: build $(ERRS) + @echo "Tests passed" + +$(ERRS): | $(OBJDIRS) + +%.wasm.err: %.wasm + $(ENGINE) $< >$@ + +clean:: + rm -rf $(OBJDIR)/*/*.err + +##### MISC ##################################################################### + +# Note: the `clean` target has been built up by all of the previous sections. + +debug: + @echo NAMES $(NAMES) + @echo TESTS $(TESTS) + @echo WASMS $(WASMS) + @echo WASM_OBJS $(WASM_OBJS) + @echo ERRS $(ERRS) + @echo DIRS $(DIRS) + @echo OBJDIRS $(OBJDIRS) + +.PHONY: test download build run clean diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..590c2d4 --- /dev/null +++ b/test/README.md @@ -0,0 +1,6 @@ +# Test + +This directory runs a subset of libc-test using the sysroot produced by +`wasi-libc`. + +[libc-test]: https://wiki.musl-libc.org/libc-test.html -- 2.39.2