1 # needs-profiler-support
4 # FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
5 # properly. Since we only have GCC on the CI ignore the test for now.
7 -include ..
/coverage
/coverage_tools.mk
9 BASEDIR
=..
/coverage-reports
12 # The `llvm-cov show` flag `--debug`, used to generate the `counters` output files, is only
13 # enabled if LLVM assertions are enabled. This requires Rust config `llvm/optimize` and not
14 # `llvm/release_debuginfo`. Note that some CI builds disable debug assertions (by setting
15 # `NO_LLVM_ASSERTIONS=1`), so the tests must still pass even if the `--debug` flag is
16 # not supported. (Note that `counters` files are only produced in the `$(TMPDIR)`
17 # directory, for inspection and debugging support. They are *not* copied to `expected_*`
18 # files when `--bless`ed.)
19 LLVM_COV_DEBUG
:= $(shell \
20 "$(LLVM_BIN_DIR)"/llvm-cov show
--debug
2>&1 | \
21 grep
-q
"Unknown command line argument '--debug'"; \
23 ifeq ($(LLVM_COV_DEBUG
), 1)
27 # FIXME(richkadel): I'm adding `--ignore-filename-regex=` line(s) for specific test(s) that produce
28 # `llvm-cov` results for multiple files (for example `uses_crate.rs` and `used_crate/mod.rs`) as a
29 # workaround for two problems causing tests to fail on Windows:
31 # 1. When multiple files appear in the `llvm-cov show` results, each file's coverage results can
32 # appear in different a different order. Whether this is random or, somehow, platform-specific,
33 # the Windows output flips the order of the files, compared to Linux. In the `uses_crate.rs`
34 # test, the only test-unique (interesting) results we care about are the results for only one
35 # of the two files, `mod/uses_crate.rs`, so the workaround is to ignore all but this one file.
36 # In the future, we may want a more sophisticated solution that splits apart `llvm-cov show`
37 # results into separate results files for each result (taking care not to create new file
38 # paths that might be too long for Windows MAX_PATH limits when creating these new sub-results,
40 # 2. When multiple files appear in the `llvm-cov show` results, the results for each file are
41 # prefixed with their filename, including platform-specific path separators (`\` for Windows,
42 # and `/` everywhere else). This could be filtered or normalized of course, but by ignoring
43 # coverage results for all but one of the file, the filenames are no longer included anyway.
44 # If this changes (if/when we decide to support `llvm-cov show` results for multiple files),
45 # the file path separator differences may need to be addressed.
47 # Since this is only a workaround, I decided to implement the override by adding an option for
48 # each file to be ignored, using a `--ignore-filename-regex=` entry for each one, rather than
49 # implement some more sophisticated solution with a new custom test directive in the test file
50 # itself (similar to `expect-exit-status`) because that would add a lot of complexity and still
51 # be a workaround, with the same result, with no benefit.
53 # Yes these `--ignore-filename-regex=` options are included in all invocations of `llvm-cov show`
54 # for now, but it is effectively ignored for all tests that don't include this file anyway.
56 # (Note that it's also possible the `_counters.<test>.txt` and `<test>.json` files (if generated)
57 # may order results from multiple files inconsistently, which might also have to be accomodated
58 # if and when we allow `llvm-cov` to produce results for multiple files. Note, the path separators
59 # appear to be normalized to `/` in those files, thankfully.)
60 LLVM_COV_IGNORE_FILES
=\
61 --ignore-filename-regex
='(uses_crate.rs|uses_inline_crate.rs)'
63 all: $(patsubst $(SOURCEDIR
)/lib
/%.rs
,%,$(wildcard $(SOURCEDIR
)/lib
/*.rs
)) $(patsubst $(SOURCEDIR
)/%.rs
,%,$(wildcard $(SOURCEDIR
)/*.rs
))
65 # Ensure there are no `expected` results for tests that may have been removed or renamed
66 .PHONY
: clear_expected_if_blessed
67 clear_expected_if_blessed
:
68 ifdef RUSTC_BLESS_TEST
72 -include clear_expected_if_blessed
74 %: $(SOURCEDIR
)/lib
/%.rs
75 # Compile the test library with coverage instrumentation
76 $(RUSTC
) $(SOURCEDIR
)/lib
/$@.rs \
77 $$( sed
-n
's/^\/\/ compile-flags: \([^#]*\).*/\1/p' $(SOURCEDIR
)/lib
/$@.rs
) \
78 --crate-type rlib
-Zinstrument-coverage
81 # Compile the test program with coverage instrumentation
82 $(RUSTC
) $(SOURCEDIR
)/$@.rs \
83 $$( sed
-n
's/^\/\/ compile-flags: \([^#]*\).*/\1/p' $(SOURCEDIR
)/$@.rs
) \
84 -L
"$(TMPDIR)" -Zinstrument-coverage
86 # Run it in order to generate some profiling data,
87 # with `LLVM_PROFILE_FILE=<profdata_file>` environment variable set to
88 # output the coverage stats for this run.
89 LLVM_PROFILE_FILE
="$(TMPDIR)"/$@.profraw \
93 grep
-q
"^\/\/ expect-exit-status-$$status" $(SOURCEDIR
)/$@.rs || \
94 ( >&2 echo
"program exited with an unexpected exit status: $$status"; \
99 # Run it through rustdoc as well to cover doctests.
100 # `%p` is the pid, and `%m` the binary signature. We suspect that the pid alone
101 # might result in overwritten files and failed tests, as rustdoc spawns each
102 # doctest as its own process, so make sure the filename is as unique as possible.
103 LLVM_PROFILE_FILE
="$(TMPDIR)"/$@
-%p-
%m.profraw \
104 $(RUSTDOC
) --crate-name workaround_for_79771
--test $(SOURCEDIR
)/$@.rs \
105 $$( sed
-n
's/^\/\/ compile-flags: \([^#]*\).*/\1/p' $(SOURCEDIR
)/$@.rs
) \
106 -L
"$(TMPDIR)" -Zinstrument-coverage \
107 -Z unstable-options
--persist-doctests
=$(TMPDIR
)/rustdoc-
$@
109 # Postprocess the profiling data so it can be used by the llvm-cov tool
110 "$(LLVM_BIN_DIR)"/llvm-profdata merge
--sparse \
111 "$(TMPDIR)"/$@
*.profraw \
112 -o
"$(TMPDIR)"/$@.profdata
114 # Generate a coverage report using `llvm-cov show`.
115 "$(LLVM_BIN_DIR)"/llvm-cov show \
117 $(LLVM_COV_IGNORE_FILES
) \
118 --Xdemangler
="$(RUST_DEMANGLER)" \
119 --show-line-counts-or-regions \
120 --instr-profile
="$(TMPDIR)"/$@.profdata \
121 $(call BIN
,"$(TMPDIR)"/$@
) \
123 for file in
$(TMPDIR
)/rustdoc-
$@
/*/rust_out
; do \
124 [ -x
"$$file" ] && printf
"%s %s " -object
$$file; \
127 2> "$(TMPDIR)"/show_coverage_stderr.
$@.txt \
128 |
"$(PYTHON)" $(BASEDIR
)/normalize_paths.py \
129 > "$(TMPDIR)"/actual_show_coverage.
$@.txt || \
131 >&2 cat
"$(TMPDIR)"/show_coverage_stderr.
$@.txt
; \
136 # The first line (beginning with "Args:" contains hard-coded, build-specific
137 # file paths. Strip that line and keep the remaining lines with counter debug
139 tail
-n
+2 "$(TMPDIR)"/show_coverage_stderr.
$@.txt \
140 > "$(TMPDIR)"/actual_show_coverage_counters.
$@.txt
143 ifdef RUSTC_BLESS_TEST
144 cp
"$(TMPDIR)"/actual_show_coverage.
$@.txt \
145 expected_show_coverage.
$@.txt
147 # Compare the show coverage output (`--bless` refreshes `typical` files).
149 # FIXME(richkadel): None of the Rust test source samples have the
150 # `// ignore-llvm-cov-show-diffs` anymore. This directive exists to work around a limitation
151 # with `llvm-cov show`. When reporting coverage for multiple instantiations of a generic function,
152 # with different type substitutions, `llvm-cov show` prints these in a non-deterministic order,
153 # breaking the `diff` comparision.
155 # A partial workaround is implemented below, with `diff --ignore-matching-lines=RE`
156 # to ignore each line prefixing each generic instantiation coverage code region.
158 # This workaround only works if the coverage counts are identical across all reported
159 # instantiations. If there is no way to ensure this, you may need to apply the
160 # `// ignore-llvm-cov-show-diffs` directive, and check for differences using the
161 # `.json` files to validate that results have not changed. (Until then, the JSON
162 # files are redundant, so there is no need to generate `expected_*.json` files or
163 # compare actual JSON results.)
165 $(DIFF
) --ignore-matching-lines
='^ | .*::<.*>.*:$$' --ignore-matching-lines
='^ | <.*>::.*:$$' \
166 expected_show_coverage.
$@.txt
"$(TMPDIR)"/actual_show_coverage.
$@.txt || \
167 ( grep
-q
'^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR
)/$@.rs
&& \
168 >&2 echo
'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
170 ( >&2 echo
'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \