1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
16 // N.B., this is also specified in this crate's Cargo.toml, but librustc_ast contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(test(attr(deny(warnings))))]
24 #![feature(rustc_private)]
26 #![feature(available_parallelism)]
27 #![feature(bench_black_box)]
28 #![feature(internal_output_capture)]
29 #![feature(panic_unwind)]
30 #![feature(staged_api)]
31 #![feature(termination_trait_lib)]
33 #![feature(total_cmp)]
36 pub use self::bench
::{black_box, Bencher}
;
37 pub use self::console
::run_tests_console
;
38 pub use self::options
::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic}
;
39 pub use self::types
::TestName
::*;
40 pub use self::types
::*;
41 pub use self::ColorConfig
::*;
42 pub use cli
::TestOpts
;
44 // Module to be used by rustc to compile tests in libtest
49 cli
::{parse_opts, TestOpts}
,
51 helpers
::metrics
::{Metric, MetricMap}
,
52 options
::{Concurrent, Options, RunIgnored, RunStrategy, ShouldPanic}
,
53 run_test
, test_main
, test_main_static
,
54 test_result
::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk}
,
55 time
::{TestExecTime, TestTimeOptions}
,
57 DynTestFn
, DynTestName
, StaticBenchFn
, StaticTestFn
, StaticTestName
, TestDesc
,
58 TestDescAndFn
, TestId
, TestName
, TestType
,
64 collections
::VecDeque
,
67 panic
::{self, catch_unwind, AssertUnwindSafe, PanicInfo}
,
68 process
::{self, Command, Termination}
,
69 sync
::mpsc
::{channel, Sender}
,
72 time
::{Duration, Instant}
,
91 use event
::{CompletedTest, TestEvent}
;
92 use helpers
::concurrency
::get_concurrency
;
93 use helpers
::exit_code
::get_exit_code
;
94 use helpers
::shuffle
::{get_shuffle_seed, shuffle_tests}
;
95 use options
::{Concurrent, RunStrategy}
;
97 use time
::TestExecTime
;
99 // Process exit code to be used to indicate test failures.
100 const ERROR_EXIT_CODE
: i32 = 101;
102 const SECONDARY_TEST_INVOKER_VAR
: &str = "__RUST_TEST_INVOKE";
104 // The default console test runner. It accepts the command line
105 // arguments and a vector of test_descs.
106 pub fn test_main(args
: &[String
], tests
: Vec
<TestDescAndFn
>, options
: Option
<Options
>) {
107 let mut opts
= match cli
::parse_opts(args
) {
110 eprintln
!("error: {}", msg
);
111 process
::exit(ERROR_EXIT_CODE
);
115 if let Some(options
) = options
{
116 opts
.options
= options
;
119 if let Err(e
) = console
::list_tests_console(&opts
, tests
) {
120 eprintln
!("error: io error when listing tests: {:?}", e
);
121 process
::exit(ERROR_EXIT_CODE
);
124 match console
::run_tests_console(&opts
, tests
) {
126 Ok(false) => process
::exit(ERROR_EXIT_CODE
),
128 eprintln
!("error: io error when listing tests: {:?}", e
);
129 process
::exit(ERROR_EXIT_CODE
);
135 /// A variant optimized for invocation with a static test vector.
136 /// This will panic (intentionally) when fed any dynamic tests.
138 /// This is the entry point for the main function generated by `rustc --test`
139 /// when panic=unwind.
140 pub fn test_main_static(tests
: &[&TestDescAndFn
]) {
141 let args
= env
::args().collect
::<Vec
<_
>>();
142 let owned_tests
: Vec
<_
> = tests
.iter().map(make_owned_test
).collect();
143 test_main(&args
, owned_tests
, None
)
146 /// A variant optimized for invocation with a static test vector.
147 /// This will panic (intentionally) when fed any dynamic tests.
149 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
152 /// This is the entry point for the main function generated by `rustc --test`
153 /// when panic=abort.
154 pub fn test_main_static_abort(tests
: &[&TestDescAndFn
]) {
155 // If we're being run in SpawnedSecondary mode, run the test here. run_test
156 // will then exit the process.
157 if let Ok(name
) = env
::var(SECONDARY_TEST_INVOKER_VAR
) {
158 env
::remove_var(SECONDARY_TEST_INVOKER_VAR
);
161 .filter(|test
| test
.desc
.name
.as_slice() == name
)
162 .map(make_owned_test
)
164 .unwrap_or_else(|| panic
!("couldn't find a test with the provided name '{}'", name
));
165 let TestDescAndFn { desc, testfn }
= test
;
166 let testfn
= match testfn
{
167 StaticTestFn(f
) => f
,
168 _
=> panic
!("only static tests are supported"),
170 run_test_in_spawned_subprocess(desc
, Box
::new(testfn
));
173 let args
= env
::args().collect
::<Vec
<_
>>();
174 let owned_tests
: Vec
<_
> = tests
.iter().map(make_owned_test
).collect();
175 test_main(&args
, owned_tests
, Some(Options
::new().panic_abort(true)))
178 /// Clones static values for putting into a dynamic vector, which test_main()
179 /// needs to hand out ownership of tests to parallel test runners.
181 /// This will panic when fed any dynamic tests, because they cannot be cloned.
182 fn make_owned_test(test
: &&TestDescAndFn
) -> TestDescAndFn
{
184 StaticTestFn(f
) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() }
,
185 StaticBenchFn(f
) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() }
,
186 _
=> panic
!("non-static tests passed to test::test_main_static"),
190 /// Invoked when unit tests terminate. Should panic if the unit
191 /// Tests is considered a failure. By default, invokes `report()`
192 /// and checks for a `0` result.
193 pub fn assert_test_result
<T
: Termination
>(result
: T
) {
194 let code
= result
.report();
197 "the test returned a termination value with a non-zero status code ({}) \
198 which indicates a failure",
205 tests
: Vec
<TestDescAndFn
>,
206 mut notify_about_test_event
: F
,
209 F
: FnMut(TestEvent
) -> io
::Result
<()>,
211 use std
::collections
::{self, HashMap}
;
212 use std
::hash
::BuildHasherDefault
;
213 use std
::sync
::mpsc
::RecvTimeoutError
;
216 join_handle
: Option
<thread
::JoinHandle
<()>>,
219 // Use a deterministic hasher
221 HashMap
<TestId
, RunningTest
, BuildHasherDefault
<collections
::hash_map
::DefaultHasher
>>;
223 struct TimeoutEntry
{
229 let tests_len
= tests
.len();
231 let mut filtered_tests
= filter_tests(opts
, tests
);
232 if !opts
.bench_benchmarks
{
233 filtered_tests
= convert_benchmarks_to_tests(filtered_tests
);
236 let filtered_tests
= {
237 let mut filtered_tests
= filtered_tests
;
238 for test
in filtered_tests
.iter_mut() {
239 test
.desc
.name
= test
.desc
.name
.with_padding(test
.testfn
.padding());
245 let filtered_out
= tests_len
- filtered_tests
.len();
246 let event
= TestEvent
::TeFilteredOut(filtered_out
);
247 notify_about_test_event(event
)?
;
249 let filtered_descs
= filtered_tests
.iter().map(|t
| t
.desc
.clone()).collect();
251 let shuffle_seed
= get_shuffle_seed(opts
);
253 let event
= TestEvent
::TeFiltered(filtered_descs
, shuffle_seed
);
254 notify_about_test_event(event
)?
;
256 let (filtered_tests
, filtered_benchs
): (Vec
<_
>, _
) = filtered_tests
259 .map(|(i
, e
)| (TestId(i
), e
))
260 .partition(|(_
, e
)| matches
!(e
.testfn
, StaticTestFn(_
) | DynTestFn(_
)));
262 let concurrency
= opts
.test_threads
.unwrap_or_else(get_concurrency
);
264 let mut remaining
= filtered_tests
;
265 if let Some(shuffle_seed
) = shuffle_seed
{
266 shuffle_tests(shuffle_seed
, &mut remaining
);
272 let (tx
, rx
) = channel
::<CompletedTest
>();
273 let run_strategy
= if opts
.options
.panic_abort
&& !opts
.force_run_in_process
{
274 RunStrategy
::SpawnPrimary
276 RunStrategy
::InProcess
279 let mut running_tests
: TestMap
= HashMap
::default();
280 let mut timeout_queue
: VecDeque
<TimeoutEntry
> = VecDeque
::new();
282 fn get_timed_out_tests(
283 running_tests
: &TestMap
,
284 timeout_queue
: &mut VecDeque
<TimeoutEntry
>,
286 let now
= Instant
::now();
287 let mut timed_out
= Vec
::new();
288 while let Some(timeout_entry
) = timeout_queue
.front() {
289 if now
< timeout_entry
.timeout
{
292 let timeout_entry
= timeout_queue
.pop_front().unwrap();
293 if running_tests
.contains_key(&timeout_entry
.id
) {
294 timed_out
.push(timeout_entry
.desc
);
300 fn calc_timeout(timeout_queue
: &VecDeque
<TimeoutEntry
>) -> Option
<Duration
> {
301 timeout_queue
.front().map(|&TimeoutEntry { timeout: next_timeout, .. }
| {
302 let now
= Instant
::now();
303 if next_timeout
>= now { next_timeout - now }
else { Duration::new(0, 0) }
307 if concurrency
== 1 {
308 while !remaining
.is_empty() {
309 let (id
, test
) = remaining
.pop().unwrap();
310 let event
= TestEvent
::TeWait(test
.desc
.clone());
311 notify_about_test_event(event
)?
;
313 run_test(opts
, !opts
.run_tests
, id
, test
, run_strategy
, tx
.clone(), Concurrent
::No
);
314 assert
!(join_handle
.is_none());
315 let completed_test
= rx
.recv().unwrap();
317 let event
= TestEvent
::TeResult(completed_test
);
318 notify_about_test_event(event
)?
;
321 while pending
> 0 || !remaining
.is_empty() {
322 while pending
< concurrency
&& !remaining
.is_empty() {
323 let (id
, test
) = remaining
.pop().unwrap();
324 let timeout
= time
::get_default_test_timeout();
325 let desc
= test
.desc
.clone();
327 let event
= TestEvent
::TeWait(desc
.clone());
328 notify_about_test_event(event
)?
; //here no pad
329 let join_handle
= run_test(
338 running_tests
.insert(id
, RunningTest { join_handle }
);
339 timeout_queue
.push_back(TimeoutEntry { id, desc, timeout }
);
345 if let Some(timeout
) = calc_timeout(&timeout_queue
) {
346 res
= rx
.recv_timeout(timeout
);
347 for test
in get_timed_out_tests(&running_tests
, &mut timeout_queue
) {
348 let event
= TestEvent
::TeTimeout(test
);
349 notify_about_test_event(event
)?
;
353 Err(RecvTimeoutError
::Timeout
) => {
354 // Result is not yet ready, continue waiting.
357 // We've got a result, stop the loop.
362 res
= rx
.recv().map_err(|_
| RecvTimeoutError
::Disconnected
);
367 let mut completed_test
= res
.unwrap();
368 let running_test
= running_tests
.remove(&completed_test
.id
).unwrap();
369 if let Some(join_handle
) = running_test
.join_handle
{
370 if let Err(_
) = join_handle
.join() {
371 if let TrOk
= completed_test
.result
{
372 completed_test
.result
=
373 TrFailedMsg("panicked after reporting success".to_string());
378 let event
= TestEvent
::TeResult(completed_test
);
379 notify_about_test_event(event
)?
;
384 if opts
.bench_benchmarks
{
385 // All benchmarks run at the end, in serial.
386 for (id
, b
) in filtered_benchs
{
387 let event
= TestEvent
::TeWait(b
.desc
.clone());
388 notify_about_test_event(event
)?
;
389 run_test(opts
, false, id
, b
, run_strategy
, tx
.clone(), Concurrent
::No
);
390 let completed_test
= rx
.recv().unwrap();
392 let event
= TestEvent
::TeResult(completed_test
);
393 notify_about_test_event(event
)?
;
399 pub fn filter_tests(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
400 let mut filtered
= tests
;
401 let matches_filter
= |test
: &TestDescAndFn
, filter
: &str| {
402 let test_name
= test
.desc
.name
.as_slice();
404 match opts
.filter_exact
{
405 true => test_name
== filter
,
406 false => test_name
.contains(filter
),
410 // Remove tests that don't match the test filter
411 if !opts
.filters
.is_empty() {
412 filtered
.retain(|test
| opts
.filters
.iter().any(|filter
| matches_filter(test
, filter
)));
415 // Skip tests that match any of the skip filters
416 filtered
.retain(|test
| !opts
.skip
.iter().any(|sf
| matches_filter(test
, sf
)));
418 // Excludes #[should_panic] tests
419 if opts
.exclude_should_panic
{
420 filtered
.retain(|test
| test
.desc
.should_panic
== ShouldPanic
::No
);
423 // maybe unignore tests
424 match opts
.run_ignored
{
426 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
428 RunIgnored
::Only
=> {
429 filtered
.retain(|test
| test
.desc
.ignore
);
430 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
435 // Sort the tests alphabetically
436 filtered
.sort_by(|t1
, t2
| t1
.desc
.name
.as_slice().cmp(t2
.desc
.name
.as_slice()));
441 pub fn convert_benchmarks_to_tests(tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
442 // convert benchmarks to tests, if we're not benchmarking them
446 let testfn
= match x
.testfn
{
447 DynBenchFn(bench
) => DynTestFn(Box
::new(move || {
448 bench
::run_once(|b
| __rust_begin_short_backtrace(|| bench
.run(b
)))
450 StaticBenchFn(benchfn
) => DynTestFn(Box
::new(move || {
451 bench
::run_once(|b
| __rust_begin_short_backtrace(|| benchfn(b
)))
455 TestDescAndFn { desc: x.desc, testfn }
465 strategy
: RunStrategy
,
466 monitor_ch
: Sender
<CompletedTest
>,
467 concurrency
: Concurrent
,
468 ) -> Option
<thread
::JoinHandle
<()>> {
469 let TestDescAndFn { desc, testfn }
= test
;
471 // Emscripten can catch panics but other wasm targets cannot
472 let ignore_because_no_process_support
= desc
.should_panic
!= ShouldPanic
::No
473 && cfg
!(target_arch
= "wasm32")
474 && !cfg
!(target_os
= "emscripten");
476 if force_ignore
|| desc
.ignore
|| ignore_because_no_process_support
{
477 let message
= CompletedTest
::new(id
, desc
, TrIgnored
, None
, Vec
::new());
478 monitor_ch
.send(message
).unwrap();
483 pub strategy
: RunStrategy
,
485 pub concurrency
: Concurrent
,
486 pub time
: Option
<time
::TestTimeOptions
>,
492 monitor_ch
: Sender
<CompletedTest
>,
493 testfn
: Box
<dyn FnOnce() + Send
>,
495 ) -> Option
<thread
::JoinHandle
<()>> {
496 let concurrency
= opts
.concurrency
;
497 let name
= desc
.name
.clone();
499 let runtest
= move || match opts
.strategy
{
500 RunStrategy
::InProcess
=> run_test_in_process(
509 RunStrategy
::SpawnPrimary
=> spawn_test_subprocess(
519 // If the platform is single-threaded we're just going to run
520 // the test synchronously, regardless of the concurrency
522 let supports_threads
= !cfg
!(target_os
= "emscripten") && !cfg
!(target_arch
= "wasm32");
523 if concurrency
== Concurrent
::Yes
&& supports_threads
{
524 let cfg
= thread
::Builder
::new().name(name
.as_slice().to_owned());
525 let mut runtest
= Arc
::new(Mutex
::new(Some(runtest
)));
526 let runtest2
= runtest
.clone();
527 match cfg
.spawn(move || runtest2
.lock().unwrap().take().unwrap()()) {
528 Ok(handle
) => Some(handle
),
529 Err(e
) if e
.kind() == io
::ErrorKind
::WouldBlock
=> {
530 // `ErrorKind::WouldBlock` means hitting the thread limit on some
531 // platforms, so run the test synchronously here instead.
532 Arc
::get_mut(&mut runtest
).unwrap().get_mut().unwrap().take().unwrap()();
535 Err(e
) => panic
!("failed to spawn thread to run test: {}", e
),
544 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options }
;
547 DynBenchFn(bencher
) => {
548 // Benchmarks aren't expected to panic, so we run them all in-process.
549 crate::bench
::benchmark(id
, desc
, monitor_ch
, opts
.nocapture
, |harness
| {
554 StaticBenchFn(benchfn
) => {
555 // Benchmarks aren't expected to panic, so we run them all in-process.
556 crate::bench
::benchmark(id
, desc
, monitor_ch
, opts
.nocapture
, benchfn
);
561 RunStrategy
::InProcess
=> (),
562 _
=> panic
!("Cannot run dynamic test fn out-of-process"),
568 Box
::new(move || __rust_begin_short_backtrace(f
)),
572 StaticTestFn(f
) => run_test_inner(
576 Box
::new(move || __rust_begin_short_backtrace(f
)),
582 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
584 fn __rust_begin_short_backtrace
<F
: FnOnce()>(f
: F
) {
587 // prevent this frame from being tail-call optimised away
591 fn run_test_in_process(
596 testfn
: Box
<dyn FnOnce() + Send
>,
597 monitor_ch
: Sender
<CompletedTest
>,
598 time_opts
: Option
<time
::TestTimeOptions
>,
600 // Buffer for capturing standard I/O
601 let data
= Arc
::new(Mutex
::new(Vec
::new()));
604 io
::set_output_capture(Some(data
.clone()));
607 let start
= report_time
.then(Instant
::now
);
608 let result
= catch_unwind(AssertUnwindSafe(testfn
));
609 let exec_time
= start
.map(|start
| {
610 let duration
= start
.elapsed();
611 TestExecTime(duration
)
614 io
::set_output_capture(None
);
616 let test_result
= match result
{
617 Ok(()) => calc_result(&desc
, Ok(()), &time_opts
, &exec_time
),
618 Err(e
) => calc_result(&desc
, Err(e
.as_ref()), &time_opts
, &exec_time
),
620 let stdout
= data
.lock().unwrap_or_else(|e
| e
.into_inner()).to_vec();
621 let message
= CompletedTest
::new(id
, desc
, test_result
, exec_time
, stdout
);
622 monitor_ch
.send(message
).unwrap();
625 fn spawn_test_subprocess(
630 monitor_ch
: Sender
<CompletedTest
>,
631 time_opts
: Option
<time
::TestTimeOptions
>,
633 let (result
, test_output
, exec_time
) = (|| {
634 let args
= env
::args().collect
::<Vec
<_
>>();
635 let current_exe
= &args
[0];
637 let mut command
= Command
::new(current_exe
);
638 command
.env(SECONDARY_TEST_INVOKER_VAR
, desc
.name
.as_slice());
640 command
.stdout(process
::Stdio
::inherit());
641 command
.stderr(process
::Stdio
::inherit());
644 let start
= report_time
.then(Instant
::now
);
645 let output
= match command
.output() {
648 let err
= format
!("Failed to spawn {} as child for test: {:?}", args
[0], e
);
649 return (TrFailed
, err
.into_bytes(), None
);
652 let exec_time
= start
.map(|start
| {
653 let duration
= start
.elapsed();
654 TestExecTime(duration
)
657 let std
::process
::Output { stdout, stderr, status }
= output
;
658 let mut test_output
= stdout
;
659 formatters
::write_stderr_delimiter(&mut test_output
, &desc
.name
);
660 test_output
.extend_from_slice(&stderr
);
662 let result
= match (|| -> Result
<TestResult
, String
> {
663 let exit_code
= get_exit_code(status
)?
;
664 Ok(get_result_from_exit_code(&desc
, exit_code
, &time_opts
, &exec_time
))
668 write
!(&mut test_output
, "Unexpected error: {}", e
).unwrap();
673 (result
, test_output
, exec_time
)
676 let message
= CompletedTest
::new(id
, desc
, result
, exec_time
, test_output
);
677 monitor_ch
.send(message
).unwrap();
680 fn run_test_in_spawned_subprocess(desc
: TestDesc
, testfn
: Box
<dyn FnOnce() + Send
>) -> ! {
681 let builtin_panic_hook
= panic
::take_hook();
682 let record_result
= Arc
::new(move |panic_info
: Option
<&'_ PanicInfo
<'_
>>| {
683 let test_result
= match panic_info
{
684 Some(info
) => calc_result(&desc
, Err(info
.payload()), &None
, &None
),
685 None
=> calc_result(&desc
, Ok(()), &None
, &None
),
688 // We don't support serializing TrFailedMsg, so just
689 // print the message out to stderr.
690 if let TrFailedMsg(msg
) = &test_result
{
691 eprintln
!("{}", msg
);
694 if let Some(info
) = panic_info
{
695 builtin_panic_hook(info
);
698 if let TrOk
= test_result
{
699 process
::exit(test_result
::TR_OK
);
701 process
::exit(test_result
::TR_FAILED
);
704 let record_result2
= record_result
.clone();
705 panic
::set_hook(Box
::new(move |info
| record_result2(Some(&info
))));
708 unreachable
!("panic=abort callback should have exited the process")