1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
16 // N.B., this is also specified in this crate's Cargo.toml, but librustc_ast contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
23 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
24 #![feature(rustc_private)]
26 #![feature(bool_to_option)]
27 #![feature(set_stdio)]
28 #![feature(panic_unwind)]
29 #![feature(staged_api)]
30 #![feature(termination_trait_lib)]
32 #![feature(total_cmp)]
35 pub use self::bench
::{black_box, Bencher}
;
36 pub use self::console
::run_tests_console
;
37 pub use self::options
::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic}
;
38 pub use self::types
::TestName
::*;
39 pub use self::types
::*;
40 pub use self::ColorConfig
::*;
41 pub use cli
::TestOpts
;
43 // Module to be used by rustc to compile tests in libtest
48 cli
::{parse_opts, TestOpts}
,
50 helpers
::metrics
::{Metric, MetricMap}
,
51 options
::{Options, RunIgnored, RunStrategy, ShouldPanic}
,
52 run_test
, test_main
, test_main_static
,
53 test_result
::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk}
,
54 time
::{TestExecTime, TestTimeOptions}
,
56 DynTestFn
, DynTestName
, StaticBenchFn
, StaticTestFn
, StaticTestName
, TestDesc
,
57 TestDescAndFn
, TestName
, TestType
,
65 panic
::{self, catch_unwind, AssertUnwindSafe, PanicInfo}
,
66 process
::{self, Command, Termination}
,
67 sync
::mpsc
::{channel, Sender}
,
70 time
::{Duration, Instant}
,
88 use event
::{CompletedTest, TestEvent}
;
89 use helpers
::concurrency
::get_concurrency
;
90 use helpers
::exit_code
::get_exit_code
;
91 use helpers
::sink
::Sink
;
92 use options
::{Concurrent, RunStrategy}
;
94 use time
::TestExecTime
;
96 // Process exit code to be used to indicate test failures.
97 const ERROR_EXIT_CODE
: i32 = 101;
99 const SECONDARY_TEST_INVOKER_VAR
: &str = "__RUST_TEST_INVOKE";
101 // The default console test runner. It accepts the command line
102 // arguments and a vector of test_descs.
103 pub fn test_main(args
: &[String
], tests
: Vec
<TestDescAndFn
>, options
: Option
<Options
>) {
104 let mut opts
= match cli
::parse_opts(args
) {
107 eprintln
!("error: {}", msg
);
108 process
::exit(ERROR_EXIT_CODE
);
112 if let Some(options
) = options
{
113 opts
.options
= options
;
116 if let Err(e
) = console
::list_tests_console(&opts
, tests
) {
117 eprintln
!("error: io error when listing tests: {:?}", e
);
118 process
::exit(ERROR_EXIT_CODE
);
121 match console
::run_tests_console(&opts
, tests
) {
123 Ok(false) => process
::exit(ERROR_EXIT_CODE
),
125 eprintln
!("error: io error when listing tests: {:?}", e
);
126 process
::exit(ERROR_EXIT_CODE
);
132 /// A variant optimized for invocation with a static test vector.
133 /// This will panic (intentionally) when fed any dynamic tests.
135 /// This is the entry point for the main function generated by `rustc --test`
136 /// when panic=unwind.
137 pub fn test_main_static(tests
: &[&TestDescAndFn
]) {
138 let args
= env
::args().collect
::<Vec
<_
>>();
139 let owned_tests
: Vec
<_
> = tests
.iter().map(make_owned_test
).collect();
140 test_main(&args
, owned_tests
, None
)
143 /// A variant optimized for invocation with a static test vector.
144 /// This will panic (intentionally) when fed any dynamic tests.
146 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
149 /// This is the entry point for the main function generated by `rustc --test`
150 /// when panic=abort.
151 pub fn test_main_static_abort(tests
: &[&TestDescAndFn
]) {
152 // If we're being run in SpawnedSecondary mode, run the test here. run_test
153 // will then exit the process.
154 if let Ok(name
) = env
::var(SECONDARY_TEST_INVOKER_VAR
) {
155 env
::remove_var(SECONDARY_TEST_INVOKER_VAR
);
158 .filter(|test
| test
.desc
.name
.as_slice() == name
)
159 .map(make_owned_test
)
161 .unwrap_or_else(|| panic
!("couldn't find a test with the provided name '{}'", name
));
162 let TestDescAndFn { desc, testfn }
= test
;
163 let testfn
= match testfn
{
164 StaticTestFn(f
) => f
,
165 _
=> panic
!("only static tests are supported"),
167 run_test_in_spawned_subprocess(desc
, Box
::new(testfn
));
170 let args
= env
::args().collect
::<Vec
<_
>>();
171 let owned_tests
: Vec
<_
> = tests
.iter().map(make_owned_test
).collect();
172 test_main(&args
, owned_tests
, Some(Options
::new().panic_abort(true)))
175 /// Clones static values for putting into a dynamic vector, which test_main()
176 /// needs to hand out ownership of tests to parallel test runners.
178 /// This will panic when fed any dynamic tests, because they cannot be cloned.
179 fn make_owned_test(test
: &&TestDescAndFn
) -> TestDescAndFn
{
181 StaticTestFn(f
) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() }
,
182 StaticBenchFn(f
) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() }
,
183 _
=> panic
!("non-static tests passed to test::test_main_static"),
187 /// Invoked when unit tests terminate. Should panic if the unit
188 /// Tests is considered a failure. By default, invokes `report()`
189 /// and checks for a `0` result.
190 pub fn assert_test_result
<T
: Termination
>(result
: T
) {
191 let code
= result
.report();
194 "the test returned a termination value with a non-zero status code ({}) \
195 which indicates a failure",
202 tests
: Vec
<TestDescAndFn
>,
203 mut notify_about_test_event
: F
,
206 F
: FnMut(TestEvent
) -> io
::Result
<()>,
208 use std
::collections
::{self, HashMap}
;
209 use std
::hash
::BuildHasherDefault
;
210 use std
::sync
::mpsc
::RecvTimeoutError
;
211 // Use a deterministic hasher
213 HashMap
<TestDesc
, Instant
, BuildHasherDefault
<collections
::hash_map
::DefaultHasher
>>;
215 let tests_len
= tests
.len();
217 let mut filtered_tests
= filter_tests(opts
, tests
);
218 if !opts
.bench_benchmarks
{
219 filtered_tests
= convert_benchmarks_to_tests(filtered_tests
);
222 let filtered_tests
= {
223 let mut filtered_tests
= filtered_tests
;
224 for test
in filtered_tests
.iter_mut() {
225 test
.desc
.name
= test
.desc
.name
.with_padding(test
.testfn
.padding());
231 let filtered_out
= tests_len
- filtered_tests
.len();
232 let event
= TestEvent
::TeFilteredOut(filtered_out
);
233 notify_about_test_event(event
)?
;
235 let filtered_descs
= filtered_tests
.iter().map(|t
| t
.desc
.clone()).collect();
237 let event
= TestEvent
::TeFiltered(filtered_descs
);
238 notify_about_test_event(event
)?
;
240 let (filtered_tests
, filtered_benchs
): (Vec
<_
>, _
) = filtered_tests
242 .partition(|e
| matches
!(e
.testfn
, StaticTestFn(_
) | DynTestFn(_
)));
244 let concurrency
= opts
.test_threads
.unwrap_or_else(get_concurrency
);
246 let mut remaining
= filtered_tests
;
250 let (tx
, rx
) = channel
::<CompletedTest
>();
251 let run_strategy
= if opts
.options
.panic_abort
&& !opts
.force_run_in_process
{
252 RunStrategy
::SpawnPrimary
254 RunStrategy
::InProcess
257 let mut running_tests
: TestMap
= HashMap
::default();
259 fn get_timed_out_tests(running_tests
: &mut TestMap
) -> Vec
<TestDesc
> {
260 let now
= Instant
::now();
261 let timed_out
= running_tests
263 .filter_map(|(desc
, timeout
)| if &now
>= timeout { Some(desc.clone()) }
else { None }
)
265 for test
in &timed_out
{
266 running_tests
.remove(test
);
271 fn calc_timeout(running_tests
: &TestMap
) -> Option
<Duration
> {
272 running_tests
.values().min().map(|next_timeout
| {
273 let now
= Instant
::now();
274 if *next_timeout
>= now { *next_timeout - now }
else { Duration::new(0, 0) }
278 if concurrency
== 1 {
279 while !remaining
.is_empty() {
280 let test
= remaining
.pop().unwrap();
281 let event
= TestEvent
::TeWait(test
.desc
.clone());
282 notify_about_test_event(event
)?
;
283 run_test(opts
, !opts
.run_tests
, test
, run_strategy
, tx
.clone(), Concurrent
::No
);
284 let completed_test
= rx
.recv().unwrap();
286 let event
= TestEvent
::TeResult(completed_test
);
287 notify_about_test_event(event
)?
;
290 while pending
> 0 || !remaining
.is_empty() {
291 while pending
< concurrency
&& !remaining
.is_empty() {
292 let test
= remaining
.pop().unwrap();
293 let timeout
= time
::get_default_test_timeout();
294 running_tests
.insert(test
.desc
.clone(), timeout
);
296 let event
= TestEvent
::TeWait(test
.desc
.clone());
297 notify_about_test_event(event
)?
; //here no pad
298 run_test(opts
, !opts
.run_tests
, test
, run_strategy
, tx
.clone(), Concurrent
::Yes
);
304 if let Some(timeout
) = calc_timeout(&running_tests
) {
305 res
= rx
.recv_timeout(timeout
);
306 for test
in get_timed_out_tests(&mut running_tests
) {
307 let event
= TestEvent
::TeTimeout(test
);
308 notify_about_test_event(event
)?
;
312 Err(RecvTimeoutError
::Timeout
) => {
313 // Result is not yet ready, continue waiting.
316 // We've got a result, stop the loop.
321 res
= rx
.recv().map_err(|_
| RecvTimeoutError
::Disconnected
);
326 let completed_test
= res
.unwrap();
327 running_tests
.remove(&completed_test
.desc
);
329 let event
= TestEvent
::TeResult(completed_test
);
330 notify_about_test_event(event
)?
;
335 if opts
.bench_benchmarks
{
336 // All benchmarks run at the end, in serial.
337 for b
in filtered_benchs
{
338 let event
= TestEvent
::TeWait(b
.desc
.clone());
339 notify_about_test_event(event
)?
;
340 run_test(opts
, false, b
, run_strategy
, tx
.clone(), Concurrent
::No
);
341 let completed_test
= rx
.recv().unwrap();
343 let event
= TestEvent
::TeResult(completed_test
);
344 notify_about_test_event(event
)?
;
350 pub fn filter_tests(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
351 let mut filtered
= tests
;
352 let matches_filter
= |test
: &TestDescAndFn
, filter
: &str| {
353 let test_name
= test
.desc
.name
.as_slice();
355 match opts
.filter_exact
{
356 true => test_name
== filter
,
357 false => test_name
.contains(filter
),
361 // Remove tests that don't match the test filter
362 if let Some(ref filter
) = opts
.filter
{
363 filtered
.retain(|test
| matches_filter(test
, filter
));
366 // Skip tests that match any of the skip filters
367 filtered
.retain(|test
| !opts
.skip
.iter().any(|sf
| matches_filter(test
, sf
)));
369 // Excludes #[should_panic] tests
370 if opts
.exclude_should_panic
{
371 filtered
.retain(|test
| test
.desc
.should_panic
== ShouldPanic
::No
);
374 // maybe unignore tests
375 match opts
.run_ignored
{
377 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
379 RunIgnored
::Only
=> {
380 filtered
.retain(|test
| test
.desc
.ignore
);
381 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
386 // Sort the tests alphabetically
387 filtered
.sort_by(|t1
, t2
| t1
.desc
.name
.as_slice().cmp(t2
.desc
.name
.as_slice()));
392 pub fn convert_benchmarks_to_tests(tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
393 // convert benchmarks to tests, if we're not benchmarking them
397 let testfn
= match x
.testfn
{
398 DynBenchFn(bench
) => DynTestFn(Box
::new(move || {
399 bench
::run_once(|b
| __rust_begin_short_backtrace(|| bench
.run(b
)))
401 StaticBenchFn(benchfn
) => DynTestFn(Box
::new(move || {
402 bench
::run_once(|b
| __rust_begin_short_backtrace(|| benchfn(b
)))
406 TestDescAndFn { desc: x.desc, testfn }
415 strategy
: RunStrategy
,
416 monitor_ch
: Sender
<CompletedTest
>,
417 concurrency
: Concurrent
,
419 let TestDescAndFn { desc, testfn }
= test
;
421 // Emscripten can catch panics but other wasm targets cannot
422 let ignore_because_no_process_support
= desc
.should_panic
!= ShouldPanic
::No
423 && cfg
!(target_arch
= "wasm32")
424 && !cfg
!(target_os
= "emscripten");
426 if force_ignore
|| desc
.ignore
|| ignore_because_no_process_support
{
427 let message
= CompletedTest
::new(desc
, TrIgnored
, None
, Vec
::new());
428 monitor_ch
.send(message
).unwrap();
433 pub strategy
: RunStrategy
,
435 pub concurrency
: Concurrent
,
436 pub time
: Option
<time
::TestTimeOptions
>,
441 monitor_ch
: Sender
<CompletedTest
>,
442 testfn
: Box
<dyn FnOnce() + Send
>,
445 let concurrency
= opts
.concurrency
;
446 let name
= desc
.name
.clone();
448 let runtest
= move || match opts
.strategy
{
449 RunStrategy
::InProcess
=> run_test_in_process(
457 RunStrategy
::SpawnPrimary
=> spawn_test_subprocess(
466 // If the platform is single-threaded we're just going to run
467 // the test synchronously, regardless of the concurrency
469 let supports_threads
= !cfg
!(target_os
= "emscripten") && !cfg
!(target_arch
= "wasm32");
470 if concurrency
== Concurrent
::Yes
&& supports_threads
{
471 let cfg
= thread
::Builder
::new().name(name
.as_slice().to_owned());
472 cfg
.spawn(runtest
).unwrap();
479 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options }
;
482 DynBenchFn(bencher
) => {
483 // Benchmarks aren't expected to panic, so we run them all in-process.
484 crate::bench
::benchmark(desc
, monitor_ch
, opts
.nocapture
, |harness
| {
488 StaticBenchFn(benchfn
) => {
489 // Benchmarks aren't expected to panic, so we run them all in-process.
490 crate::bench
::benchmark(desc
, monitor_ch
, opts
.nocapture
, benchfn
);
494 RunStrategy
::InProcess
=> (),
495 _
=> panic
!("Cannot run dynamic test fn out-of-process"),
500 Box
::new(move || __rust_begin_short_backtrace(f
)),
504 StaticTestFn(f
) => run_test_inner(
507 Box
::new(move || __rust_begin_short_backtrace(f
)),
513 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
515 fn __rust_begin_short_backtrace
<F
: FnOnce()>(f
: F
) {
518 // prevent this frame from being tail-call optimised away
522 fn run_test_in_process(
526 testfn
: Box
<dyn FnOnce() + Send
>,
527 monitor_ch
: Sender
<CompletedTest
>,
528 time_opts
: Option
<time
::TestTimeOptions
>,
530 // Buffer for capturing standard I/O
531 let data
= Arc
::new(Mutex
::new(Vec
::new()));
533 let oldio
= if !nocapture
{
535 io
::set_print(Some(Sink
::new_boxed(&data
))),
536 io
::set_panic(Some(Sink
::new_boxed(&data
))),
542 let start
= report_time
.then(Instant
::now
);
543 let result
= catch_unwind(AssertUnwindSafe(testfn
));
544 let exec_time
= start
.map(|start
| {
545 let duration
= start
.elapsed();
546 TestExecTime(duration
)
549 if let Some((printio
, panicio
)) = oldio
{
550 io
::set_print(printio
);
551 io
::set_panic(panicio
);
554 let test_result
= match result
{
555 Ok(()) => calc_result(&desc
, Ok(()), &time_opts
, &exec_time
),
556 Err(e
) => calc_result(&desc
, Err(e
.as_ref()), &time_opts
, &exec_time
),
558 let stdout
= data
.lock().unwrap().to_vec();
559 let message
= CompletedTest
::new(desc
, test_result
, exec_time
, stdout
);
560 monitor_ch
.send(message
).unwrap();
563 fn spawn_test_subprocess(
567 monitor_ch
: Sender
<CompletedTest
>,
568 time_opts
: Option
<time
::TestTimeOptions
>,
570 let (result
, test_output
, exec_time
) = (|| {
571 let args
= env
::args().collect
::<Vec
<_
>>();
572 let current_exe
= &args
[0];
574 let mut command
= Command
::new(current_exe
);
575 command
.env(SECONDARY_TEST_INVOKER_VAR
, desc
.name
.as_slice());
577 command
.stdout(process
::Stdio
::inherit());
578 command
.stderr(process
::Stdio
::inherit());
581 let start
= report_time
.then(Instant
::now
);
582 let output
= match command
.output() {
585 let err
= format
!("Failed to spawn {} as child for test: {:?}", args
[0], e
);
586 return (TrFailed
, err
.into_bytes(), None
);
589 let exec_time
= start
.map(|start
| {
590 let duration
= start
.elapsed();
591 TestExecTime(duration
)
594 let std
::process
::Output { stdout, stderr, status }
= output
;
595 let mut test_output
= stdout
;
596 formatters
::write_stderr_delimiter(&mut test_output
, &desc
.name
);
597 test_output
.extend_from_slice(&stderr
);
599 let result
= match (|| -> Result
<TestResult
, String
> {
600 let exit_code
= get_exit_code(status
)?
;
601 Ok(get_result_from_exit_code(&desc
, exit_code
, &time_opts
, &exec_time
))
605 write
!(&mut test_output
, "Unexpected error: {}", e
).unwrap();
610 (result
, test_output
, exec_time
)
613 let message
= CompletedTest
::new(desc
, result
, exec_time
, test_output
);
614 monitor_ch
.send(message
).unwrap();
617 fn run_test_in_spawned_subprocess(desc
: TestDesc
, testfn
: Box
<dyn FnOnce() + Send
>) -> ! {
618 let builtin_panic_hook
= panic
::take_hook();
619 let record_result
= Arc
::new(move |panic_info
: Option
<&'_ PanicInfo
<'_
>>| {
620 let test_result
= match panic_info
{
621 Some(info
) => calc_result(&desc
, Err(info
.payload()), &None
, &None
),
622 None
=> calc_result(&desc
, Ok(()), &None
, &None
),
625 // We don't support serializing TrFailedMsg, so just
626 // print the message out to stderr.
627 if let TrFailedMsg(msg
) = &test_result
{
628 eprintln
!("{}", msg
);
631 if let Some(info
) = panic_info
{
632 builtin_panic_hook(info
);
635 if let TrOk
= test_result
{
636 process
::exit(test_result
::TR_OK
);
638 process
::exit(test_result
::TR_FAILED
);
641 let record_result2
= record_result
.clone();
642 panic
::set_hook(Box
::new(move |info
| record_result2(Some(&info
))));
645 unreachable
!("panic=abort callback should have exited the process")