1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
16 // N.B., this is also specified in this crate's Cargo.toml, but librustc_ast contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(test(attr(deny(warnings))))]
23 #![cfg_attr(unix, feature(libc))]
24 #![feature(rustc_private)]
26 #![feature(available_concurrency)]
27 #![feature(bench_black_box)]
28 #![feature(internal_output_capture)]
29 #![feature(panic_unwind)]
30 #![feature(staged_api)]
31 #![feature(termination_trait_lib)]
33 #![feature(total_cmp)]
36 pub use self::bench
::{black_box, Bencher}
;
37 pub use self::console
::run_tests_console
;
38 pub use self::options
::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic}
;
39 pub use self::types
::TestName
::*;
40 pub use self::types
::*;
41 pub use self::ColorConfig
::*;
42 pub use cli
::TestOpts
;
44 // Module to be used by rustc to compile tests in libtest
49 cli
::{parse_opts, TestOpts}
,
51 helpers
::metrics
::{Metric, MetricMap}
,
52 options
::{Concurrent, Options, RunIgnored, RunStrategy, ShouldPanic}
,
53 run_test
, test_main
, test_main_static
,
54 test_result
::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk}
,
55 time
::{TestExecTime, TestTimeOptions}
,
57 DynTestFn
, DynTestName
, StaticBenchFn
, StaticTestFn
, StaticTestName
, TestDesc
,
58 TestDescAndFn
, TestId
, TestName
, TestType
,
64 collections
::VecDeque
,
67 panic
::{self, catch_unwind, AssertUnwindSafe, PanicInfo}
,
68 process
::{self, Command, Termination}
,
69 sync
::mpsc
::{channel, Sender}
,
72 time
::{Duration, Instant}
,
90 use event
::{CompletedTest, TestEvent}
;
91 use helpers
::concurrency
::get_concurrency
;
92 use helpers
::exit_code
::get_exit_code
;
93 use options
::{Concurrent, RunStrategy}
;
95 use time
::TestExecTime
;
97 // Process exit code to be used to indicate test failures.
98 const ERROR_EXIT_CODE
: i32 = 101;
100 const SECONDARY_TEST_INVOKER_VAR
: &str = "__RUST_TEST_INVOKE";
102 // The default console test runner. It accepts the command line
103 // arguments and a vector of test_descs.
104 pub fn test_main(args
: &[String
], tests
: Vec
<TestDescAndFn
>, options
: Option
<Options
>) {
105 let mut opts
= match cli
::parse_opts(args
) {
108 eprintln
!("error: {}", msg
);
109 process
::exit(ERROR_EXIT_CODE
);
113 if let Some(options
) = options
{
114 opts
.options
= options
;
117 if let Err(e
) = console
::list_tests_console(&opts
, tests
) {
118 eprintln
!("error: io error when listing tests: {:?}", e
);
119 process
::exit(ERROR_EXIT_CODE
);
122 match console
::run_tests_console(&opts
, tests
) {
124 Ok(false) => process
::exit(ERROR_EXIT_CODE
),
126 eprintln
!("error: io error when listing tests: {:?}", e
);
127 process
::exit(ERROR_EXIT_CODE
);
133 /// A variant optimized for invocation with a static test vector.
134 /// This will panic (intentionally) when fed any dynamic tests.
136 /// This is the entry point for the main function generated by `rustc --test`
137 /// when panic=unwind.
138 pub fn test_main_static(tests
: &[&TestDescAndFn
]) {
139 let args
= env
::args().collect
::<Vec
<_
>>();
140 let owned_tests
: Vec
<_
> = tests
.iter().map(make_owned_test
).collect();
141 test_main(&args
, owned_tests
, None
)
144 /// A variant optimized for invocation with a static test vector.
145 /// This will panic (intentionally) when fed any dynamic tests.
147 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
150 /// This is the entry point for the main function generated by `rustc --test`
151 /// when panic=abort.
152 pub fn test_main_static_abort(tests
: &[&TestDescAndFn
]) {
153 // If we're being run in SpawnedSecondary mode, run the test here. run_test
154 // will then exit the process.
155 if let Ok(name
) = env
::var(SECONDARY_TEST_INVOKER_VAR
) {
156 env
::remove_var(SECONDARY_TEST_INVOKER_VAR
);
159 .filter(|test
| test
.desc
.name
.as_slice() == name
)
160 .map(make_owned_test
)
162 .unwrap_or_else(|| panic
!("couldn't find a test with the provided name '{}'", name
));
163 let TestDescAndFn { desc, testfn }
= test
;
164 let testfn
= match testfn
{
165 StaticTestFn(f
) => f
,
166 _
=> panic
!("only static tests are supported"),
168 run_test_in_spawned_subprocess(desc
, Box
::new(testfn
));
171 let args
= env
::args().collect
::<Vec
<_
>>();
172 let owned_tests
: Vec
<_
> = tests
.iter().map(make_owned_test
).collect();
173 test_main(&args
, owned_tests
, Some(Options
::new().panic_abort(true)))
176 /// Clones static values for putting into a dynamic vector, which test_main()
177 /// needs to hand out ownership of tests to parallel test runners.
179 /// This will panic when fed any dynamic tests, because they cannot be cloned.
180 fn make_owned_test(test
: &&TestDescAndFn
) -> TestDescAndFn
{
182 StaticTestFn(f
) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() }
,
183 StaticBenchFn(f
) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() }
,
184 _
=> panic
!("non-static tests passed to test::test_main_static"),
188 /// Invoked when unit tests terminate. Should panic if the unit
189 /// Tests is considered a failure. By default, invokes `report()`
190 /// and checks for a `0` result.
191 pub fn assert_test_result
<T
: Termination
>(result
: T
) {
192 let code
= result
.report();
195 "the test returned a termination value with a non-zero status code ({}) \
196 which indicates a failure",
203 tests
: Vec
<TestDescAndFn
>,
204 mut notify_about_test_event
: F
,
207 F
: FnMut(TestEvent
) -> io
::Result
<()>,
209 use std
::collections
::{self, HashMap}
;
210 use std
::hash
::BuildHasherDefault
;
211 use std
::sync
::mpsc
::RecvTimeoutError
;
214 join_handle
: Option
<thread
::JoinHandle
<()>>,
217 // Use a deterministic hasher
219 HashMap
<TestId
, RunningTest
, BuildHasherDefault
<collections
::hash_map
::DefaultHasher
>>;
221 struct TimeoutEntry
{
227 let tests_len
= tests
.len();
229 let mut filtered_tests
= filter_tests(opts
, tests
);
230 if !opts
.bench_benchmarks
{
231 filtered_tests
= convert_benchmarks_to_tests(filtered_tests
);
234 let filtered_tests
= {
235 let mut filtered_tests
= filtered_tests
;
236 for test
in filtered_tests
.iter_mut() {
237 test
.desc
.name
= test
.desc
.name
.with_padding(test
.testfn
.padding());
243 let filtered_out
= tests_len
- filtered_tests
.len();
244 let event
= TestEvent
::TeFilteredOut(filtered_out
);
245 notify_about_test_event(event
)?
;
247 let filtered_descs
= filtered_tests
.iter().map(|t
| t
.desc
.clone()).collect();
249 let event
= TestEvent
::TeFiltered(filtered_descs
);
250 notify_about_test_event(event
)?
;
252 let (filtered_tests
, filtered_benchs
): (Vec
<_
>, _
) = filtered_tests
255 .map(|(i
, e
)| (TestId(i
), e
))
256 .partition(|(_
, e
)| matches
!(e
.testfn
, StaticTestFn(_
) | DynTestFn(_
)));
258 let concurrency
= opts
.test_threads
.unwrap_or_else(get_concurrency
);
260 let mut remaining
= filtered_tests
;
264 let (tx
, rx
) = channel
::<CompletedTest
>();
265 let run_strategy
= if opts
.options
.panic_abort
&& !opts
.force_run_in_process
{
266 RunStrategy
::SpawnPrimary
268 RunStrategy
::InProcess
271 let mut running_tests
: TestMap
= HashMap
::default();
272 let mut timeout_queue
: VecDeque
<TimeoutEntry
> = VecDeque
::new();
274 fn get_timed_out_tests(
275 running_tests
: &TestMap
,
276 timeout_queue
: &mut VecDeque
<TimeoutEntry
>,
278 let now
= Instant
::now();
279 let mut timed_out
= Vec
::new();
280 while let Some(timeout_entry
) = timeout_queue
.front() {
281 if now
< timeout_entry
.timeout
{
284 let timeout_entry
= timeout_queue
.pop_front().unwrap();
285 if running_tests
.contains_key(&timeout_entry
.id
) {
286 timed_out
.push(timeout_entry
.desc
);
292 fn calc_timeout(timeout_queue
: &VecDeque
<TimeoutEntry
>) -> Option
<Duration
> {
293 timeout_queue
.front().map(|&TimeoutEntry { timeout: next_timeout, .. }
| {
294 let now
= Instant
::now();
295 if next_timeout
>= now { next_timeout - now }
else { Duration::new(0, 0) }
299 if concurrency
== 1 {
300 while !remaining
.is_empty() {
301 let (id
, test
) = remaining
.pop().unwrap();
302 let event
= TestEvent
::TeWait(test
.desc
.clone());
303 notify_about_test_event(event
)?
;
305 run_test(opts
, !opts
.run_tests
, id
, test
, run_strategy
, tx
.clone(), Concurrent
::No
);
306 assert
!(join_handle
.is_none());
307 let completed_test
= rx
.recv().unwrap();
309 let event
= TestEvent
::TeResult(completed_test
);
310 notify_about_test_event(event
)?
;
313 while pending
> 0 || !remaining
.is_empty() {
314 while pending
< concurrency
&& !remaining
.is_empty() {
315 let (id
, test
) = remaining
.pop().unwrap();
316 let timeout
= time
::get_default_test_timeout();
317 let desc
= test
.desc
.clone();
319 let event
= TestEvent
::TeWait(desc
.clone());
320 notify_about_test_event(event
)?
; //here no pad
321 let join_handle
= run_test(
330 running_tests
.insert(id
, RunningTest { join_handle }
);
331 timeout_queue
.push_back(TimeoutEntry { id, desc, timeout }
);
337 if let Some(timeout
) = calc_timeout(&timeout_queue
) {
338 res
= rx
.recv_timeout(timeout
);
339 for test
in get_timed_out_tests(&running_tests
, &mut timeout_queue
) {
340 let event
= TestEvent
::TeTimeout(test
);
341 notify_about_test_event(event
)?
;
345 Err(RecvTimeoutError
::Timeout
) => {
346 // Result is not yet ready, continue waiting.
349 // We've got a result, stop the loop.
354 res
= rx
.recv().map_err(|_
| RecvTimeoutError
::Disconnected
);
359 let mut completed_test
= res
.unwrap();
360 let running_test
= running_tests
.remove(&completed_test
.id
).unwrap();
361 if let Some(join_handle
) = running_test
.join_handle
{
362 if let Err(_
) = join_handle
.join() {
363 if let TrOk
= completed_test
.result
{
364 completed_test
.result
=
365 TrFailedMsg("panicked after reporting success".to_string());
370 let event
= TestEvent
::TeResult(completed_test
);
371 notify_about_test_event(event
)?
;
376 if opts
.bench_benchmarks
{
377 // All benchmarks run at the end, in serial.
378 for (id
, b
) in filtered_benchs
{
379 let event
= TestEvent
::TeWait(b
.desc
.clone());
380 notify_about_test_event(event
)?
;
381 run_test(opts
, false, id
, b
, run_strategy
, tx
.clone(), Concurrent
::No
);
382 let completed_test
= rx
.recv().unwrap();
384 let event
= TestEvent
::TeResult(completed_test
);
385 notify_about_test_event(event
)?
;
391 pub fn filter_tests(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
392 let mut filtered
= tests
;
393 let matches_filter
= |test
: &TestDescAndFn
, filter
: &str| {
394 let test_name
= test
.desc
.name
.as_slice();
396 match opts
.filter_exact
{
397 true => test_name
== filter
,
398 false => test_name
.contains(filter
),
402 // Remove tests that don't match the test filter
403 if !opts
.filters
.is_empty() {
404 filtered
.retain(|test
| opts
.filters
.iter().any(|filter
| matches_filter(test
, filter
)));
407 // Skip tests that match any of the skip filters
408 filtered
.retain(|test
| !opts
.skip
.iter().any(|sf
| matches_filter(test
, sf
)));
410 // Excludes #[should_panic] tests
411 if opts
.exclude_should_panic
{
412 filtered
.retain(|test
| test
.desc
.should_panic
== ShouldPanic
::No
);
415 // maybe unignore tests
416 match opts
.run_ignored
{
418 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
420 RunIgnored
::Only
=> {
421 filtered
.retain(|test
| test
.desc
.ignore
);
422 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
427 // Sort the tests alphabetically
428 filtered
.sort_by(|t1
, t2
| t1
.desc
.name
.as_slice().cmp(t2
.desc
.name
.as_slice()));
433 pub fn convert_benchmarks_to_tests(tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
434 // convert benchmarks to tests, if we're not benchmarking them
438 let testfn
= match x
.testfn
{
439 DynBenchFn(bench
) => DynTestFn(Box
::new(move || {
440 bench
::run_once(|b
| __rust_begin_short_backtrace(|| bench
.run(b
)))
442 StaticBenchFn(benchfn
) => DynTestFn(Box
::new(move || {
443 bench
::run_once(|b
| __rust_begin_short_backtrace(|| benchfn(b
)))
447 TestDescAndFn { desc: x.desc, testfn }
457 strategy
: RunStrategy
,
458 monitor_ch
: Sender
<CompletedTest
>,
459 concurrency
: Concurrent
,
460 ) -> Option
<thread
::JoinHandle
<()>> {
461 let TestDescAndFn { desc, testfn }
= test
;
463 // Emscripten can catch panics but other wasm targets cannot
464 let ignore_because_no_process_support
= desc
.should_panic
!= ShouldPanic
::No
465 && cfg
!(target_arch
= "wasm32")
466 && !cfg
!(target_os
= "emscripten");
468 if force_ignore
|| desc
.ignore
|| ignore_because_no_process_support
{
469 let message
= CompletedTest
::new(id
, desc
, TrIgnored
, None
, Vec
::new());
470 monitor_ch
.send(message
).unwrap();
475 pub strategy
: RunStrategy
,
477 pub concurrency
: Concurrent
,
478 pub time
: Option
<time
::TestTimeOptions
>,
484 monitor_ch
: Sender
<CompletedTest
>,
485 testfn
: Box
<dyn FnOnce() + Send
>,
487 ) -> Option
<thread
::JoinHandle
<()>> {
488 let concurrency
= opts
.concurrency
;
489 let name
= desc
.name
.clone();
491 let runtest
= move || match opts
.strategy
{
492 RunStrategy
::InProcess
=> run_test_in_process(
501 RunStrategy
::SpawnPrimary
=> spawn_test_subprocess(
511 // If the platform is single-threaded we're just going to run
512 // the test synchronously, regardless of the concurrency
514 let supports_threads
= !cfg
!(target_os
= "emscripten") && !cfg
!(target_arch
= "wasm32");
515 if concurrency
== Concurrent
::Yes
&& supports_threads
{
516 let cfg
= thread
::Builder
::new().name(name
.as_slice().to_owned());
517 let mut runtest
= Arc
::new(Mutex
::new(Some(runtest
)));
518 let runtest2
= runtest
.clone();
519 match cfg
.spawn(move || runtest2
.lock().unwrap().take().unwrap()()) {
520 Ok(handle
) => Some(handle
),
521 Err(e
) if e
.kind() == io
::ErrorKind
::WouldBlock
=> {
522 // `ErrorKind::WouldBlock` means hitting the thread limit on some
523 // platforms, so run the test synchronously here instead.
524 Arc
::get_mut(&mut runtest
).unwrap().get_mut().unwrap().take().unwrap()();
527 Err(e
) => panic
!("failed to spawn thread to run test: {}", e
),
536 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options }
;
539 DynBenchFn(bencher
) => {
540 // Benchmarks aren't expected to panic, so we run them all in-process.
541 crate::bench
::benchmark(id
, desc
, monitor_ch
, opts
.nocapture
, |harness
| {
546 StaticBenchFn(benchfn
) => {
547 // Benchmarks aren't expected to panic, so we run them all in-process.
548 crate::bench
::benchmark(id
, desc
, monitor_ch
, opts
.nocapture
, benchfn
);
553 RunStrategy
::InProcess
=> (),
554 _
=> panic
!("Cannot run dynamic test fn out-of-process"),
560 Box
::new(move || __rust_begin_short_backtrace(f
)),
564 StaticTestFn(f
) => run_test_inner(
568 Box
::new(move || __rust_begin_short_backtrace(f
)),
574 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
576 fn __rust_begin_short_backtrace
<F
: FnOnce()>(f
: F
) {
579 // prevent this frame from being tail-call optimised away
583 fn run_test_in_process(
588 testfn
: Box
<dyn FnOnce() + Send
>,
589 monitor_ch
: Sender
<CompletedTest
>,
590 time_opts
: Option
<time
::TestTimeOptions
>,
592 // Buffer for capturing standard I/O
593 let data
= Arc
::new(Mutex
::new(Vec
::new()));
596 io
::set_output_capture(Some(data
.clone()));
599 let start
= report_time
.then(Instant
::now
);
600 let result
= catch_unwind(AssertUnwindSafe(testfn
));
601 let exec_time
= start
.map(|start
| {
602 let duration
= start
.elapsed();
603 TestExecTime(duration
)
606 io
::set_output_capture(None
);
608 let test_result
= match result
{
609 Ok(()) => calc_result(&desc
, Ok(()), &time_opts
, &exec_time
),
610 Err(e
) => calc_result(&desc
, Err(e
.as_ref()), &time_opts
, &exec_time
),
612 let stdout
= data
.lock().unwrap_or_else(|e
| e
.into_inner()).to_vec();
613 let message
= CompletedTest
::new(id
, desc
, test_result
, exec_time
, stdout
);
614 monitor_ch
.send(message
).unwrap();
617 fn spawn_test_subprocess(
622 monitor_ch
: Sender
<CompletedTest
>,
623 time_opts
: Option
<time
::TestTimeOptions
>,
625 let (result
, test_output
, exec_time
) = (|| {
626 let args
= env
::args().collect
::<Vec
<_
>>();
627 let current_exe
= &args
[0];
629 let mut command
= Command
::new(current_exe
);
630 command
.env(SECONDARY_TEST_INVOKER_VAR
, desc
.name
.as_slice());
632 command
.stdout(process
::Stdio
::inherit());
633 command
.stderr(process
::Stdio
::inherit());
636 let start
= report_time
.then(Instant
::now
);
637 let output
= match command
.output() {
640 let err
= format
!("Failed to spawn {} as child for test: {:?}", args
[0], e
);
641 return (TrFailed
, err
.into_bytes(), None
);
644 let exec_time
= start
.map(|start
| {
645 let duration
= start
.elapsed();
646 TestExecTime(duration
)
649 let std
::process
::Output { stdout, stderr, status }
= output
;
650 let mut test_output
= stdout
;
651 formatters
::write_stderr_delimiter(&mut test_output
, &desc
.name
);
652 test_output
.extend_from_slice(&stderr
);
654 let result
= match (|| -> Result
<TestResult
, String
> {
655 let exit_code
= get_exit_code(status
)?
;
656 Ok(get_result_from_exit_code(&desc
, exit_code
, &time_opts
, &exec_time
))
660 write
!(&mut test_output
, "Unexpected error: {}", e
).unwrap();
665 (result
, test_output
, exec_time
)
668 let message
= CompletedTest
::new(id
, desc
, result
, exec_time
, test_output
);
669 monitor_ch
.send(message
).unwrap();
672 fn run_test_in_spawned_subprocess(desc
: TestDesc
, testfn
: Box
<dyn FnOnce() + Send
>) -> ! {
673 let builtin_panic_hook
= panic
::take_hook();
674 let record_result
= Arc
::new(move |panic_info
: Option
<&'_ PanicInfo
<'_
>>| {
675 let test_result
= match panic_info
{
676 Some(info
) => calc_result(&desc
, Err(info
.payload()), &None
, &None
),
677 None
=> calc_result(&desc
, Ok(()), &None
, &None
),
680 // We don't support serializing TrFailedMsg, so just
681 // print the message out to stderr.
682 if let TrFailedMsg(msg
) = &test_result
{
683 eprintln
!("{}", msg
);
686 if let Some(info
) = panic_info
{
687 builtin_panic_hook(info
);
690 if let TrOk
= test_result
{
691 process
::exit(test_result
::TR_OK
);
693 process
::exit(test_result
::TR_FAILED
);
696 let record_result2
= record_result
.clone();
697 panic
::set_hook(Box
::new(move |info
| record_result2(Some(&info
))));
700 unreachable
!("panic=abort callback should have exited the process")