1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/first-edition/testing.html) of the book for more details.
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
16 // N.B., this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "27812")]
22 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
23 html_favicon_url
= "https://doc.rust-lang.org/favicon.ico",
24 html_root_url
= "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings
))))]
26 #![cfg_attr(stage0, feature(cfg_target_vendor))]
28 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
30 #![feature(set_stdio)]
31 #![feature(panic_unwind)]
32 #![feature(staged_api)]
33 #![feature(termination_trait_lib)]
37 #[cfg(any(unix, target_os = "cloudabi"))]
41 // FIXME(#54291): rustc and/or LLVM don't yet support building with panic-unwind
42 // on aarch64-pc-windows-msvc, so we don't link libtest against
43 // libunwind (for the time being), even though it means that
44 // libtest won't be fully functional on this platform.
46 // See also: https://github.com/rust-lang/rust/issues/54190#issuecomment-422904437
47 #[cfg(not(all(windows, target_arch = "aarch64")))]
48 extern crate panic_unwind
;
50 pub use self::TestFn
::*;
51 pub use self::ColorConfig
::*;
52 pub use self::TestResult
::*;
53 pub use self::TestName
::*;
54 use self::TestEvent
::*;
55 use self::NamePadding
::*;
56 use self::OutputLocation
::*;
58 use std
::panic
::{catch_unwind, AssertUnwindSafe}
;
60 use std
::boxed
::FnBox
;
62 use std
::collections
::BTreeMap
;
66 use std
::io
::prelude
::*;
68 use std
::path
::PathBuf
;
69 use std
::process
::Termination
;
70 use std
::sync
::mpsc
::{channel, Sender}
;
71 use std
::sync
::{Arc, Mutex}
;
73 use std
::time
::{Duration, Instant}
;
77 const TEST_WARN_TIMEOUT_S
: u64 = 60;
78 const QUIET_MODE_MAX_COLUMN
: usize = 100; // insert a '\n' after 100 tests in quiet mode
80 // to be used by rustc to compile tests in libtest
82 pub use {assert_test_result
, filter_tests
, parse_opts
, run_test
, test_main
, test_main_static
,
83 Bencher
, DynTestFn
, DynTestName
, Metric
, MetricMap
, Options
, RunIgnored
, ShouldPanic
,
84 StaticBenchFn
, StaticTestFn
, StaticTestName
, TestDesc
, TestDescAndFn
, TestName
,
85 TestOpts
, TestResult
, TrFailed
, TrFailedMsg
, TrIgnored
, TrOk
};
91 use formatters
::{JsonFormatter, OutputFormatter, PrettyFormatter, TerseFormatter}
;
93 /// Whether to execute tests concurrently or not
94 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
95 pub enum Concurrent { Yes, No }
97 // The name of a test. By convention this follows the rules for rust
98 // paths; i.e., it should be a series of identifiers separated by double
99 // colons. This way if some test runner wants to arrange the tests
100 // hierarchically it may.
102 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
104 StaticTestName(&'
static str),
106 AlignedTestName(Cow
<'
static, str>, NamePadding
),
109 fn as_slice(&self) -> &str {
111 StaticTestName(s
) => s
,
112 DynTestName(ref s
) => s
,
113 AlignedTestName(ref s
, _
) => &*s
,
117 fn padding(&self) -> NamePadding
{
119 &AlignedTestName(_
, p
) => p
,
124 fn with_padding(&self, padding
: NamePadding
) -> TestName
{
125 let name
= match self {
126 &TestName
::StaticTestName(name
) => Cow
::Borrowed(name
),
127 &TestName
::DynTestName(ref name
) => Cow
::Owned(name
.clone()),
128 &TestName
::AlignedTestName(ref name
, _
) => name
.clone(),
131 TestName
::AlignedTestName(name
, padding
)
134 impl fmt
::Display
for TestName
{
135 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
136 fmt
::Display
::fmt(self.as_slice(), f
)
140 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
141 pub enum NamePadding
{
147 fn padded_name(&self, column_count
: usize, align
: NamePadding
) -> String
{
148 let mut name
= String
::from(self.name
.as_slice());
149 let fill
= column_count
.saturating_sub(name
.len());
150 let pad
= " ".repeat(fill
);
161 /// Represents a benchmark function.
162 pub trait TDynBenchFn
: Send
{
163 fn run(&self, harness
: &mut Bencher
);
166 // A function that runs a test. If the function returns successfully,
167 // the test succeeds; if the function panics then the test fails. We
168 // may need to come up with a more clever definition of test in order
169 // to support isolation of tests into threads.
172 StaticBenchFn(fn(&mut Bencher
)),
173 DynTestFn(Box
<dyn FnBox() + Send
>),
174 DynBenchFn(Box
<dyn TDynBenchFn
+ '
static>),
178 fn padding(&self) -> NamePadding
{
180 StaticTestFn(..) => PadNone
,
181 StaticBenchFn(..) => PadOnRight
,
182 DynTestFn(..) => PadNone
,
183 DynBenchFn(..) => PadOnRight
,
188 impl fmt
::Debug
for TestFn
{
189 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
190 f
.write_str(match *self {
191 StaticTestFn(..) => "StaticTestFn(..)",
192 StaticBenchFn(..) => "StaticBenchFn(..)",
193 DynTestFn(..) => "DynTestFn(..)",
194 DynBenchFn(..) => "DynBenchFn(..)",
199 /// Manager of the benchmarking runs.
201 /// This is fed into functions marked with `#[bench]` to allow for
202 /// set-up & tear-down before running a piece of code repeatedly via a
207 summary
: Option
<stats
::Summary
>,
211 #[derive(Clone, PartialEq, Eq)]
217 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
218 pub enum ShouldPanic
{
221 YesWithMessage(&'
static str),
224 // The definition of a single test. A test runner will run a list of
226 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
227 pub struct TestDesc
{
230 pub should_panic
: ShouldPanic
,
231 pub allow_fail
: bool
,
235 pub struct TestDescAndFn
{
240 #[derive(Clone, PartialEq, Debug, Copy)]
247 pub fn new(value
: f64, noise
: f64) -> Metric
{
248 Metric { value, noise }
252 /// In case we want to add other options as well, just add them in this struct.
253 #[derive(Copy, Clone, Debug)]
255 display_output
: bool
,
259 pub fn new() -> Options
{
261 display_output
: false,
265 pub fn display_output(mut self, display_output
: bool
) -> Options
{
266 self.display_output
= display_output
;
271 // The default console test runner. It accepts the command line
272 // arguments and a vector of test_descs.
273 pub fn test_main(args
: &[String
], tests
: Vec
<TestDescAndFn
>, options
: Options
) {
274 let mut opts
= match parse_opts(args
) {
277 eprintln
!("error: {}", msg
);
283 opts
.options
= options
;
285 if let Err(e
) = list_tests_console(&opts
, tests
) {
286 eprintln
!("error: io error when listing tests: {:?}", e
);
290 match run_tests_console(&opts
, tests
) {
292 Ok(false) => process
::exit(101),
294 eprintln
!("error: io error when listing tests: {:?}", e
);
301 // A variant optimized for invocation with a static test vector.
302 // This will panic (intentionally) when fed any dynamic tests, because
303 // it is copying the static values out into a dynamic vector and cannot
304 // copy dynamic values. It is doing this because from this point on
305 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
306 // semantics into parallel test runners, which in turn requires a Vec<>
307 // rather than a &[].
308 pub fn test_main_static(tests
: &[&TestDescAndFn
]) {
309 let args
= env
::args().collect
::<Vec
<_
>>();
310 let owned_tests
= tests
312 .map(|t
| match t
.testfn
{
313 StaticTestFn(f
) => TestDescAndFn
{
314 testfn
: StaticTestFn(f
),
315 desc
: t
.desc
.clone(),
317 StaticBenchFn(f
) => TestDescAndFn
{
318 testfn
: StaticBenchFn(f
),
319 desc
: t
.desc
.clone(),
321 _
=> panic
!("non-static tests passed to test::test_main_static"),
324 test_main(&args
, owned_tests
, Options
::new())
327 /// Invoked when unit tests terminate. Should panic if the unit
328 /// test is considered a failure. By default, invokes `report()`
329 /// and checks for a `0` result.
330 pub fn assert_test_result
<T
: Termination
>(result
: T
) {
331 let code
= result
.report();
335 "the test returned a termination value with a non-zero status code ({}) \
336 which indicates a failure",
341 #[derive(Copy, Clone, Debug)]
342 pub enum ColorConfig
{
348 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
349 pub enum OutputFormat
{
355 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
356 pub enum RunIgnored
{
363 pub struct TestOpts
{
365 pub filter
: Option
<String
>,
366 pub filter_exact
: bool
,
367 pub run_ignored
: RunIgnored
,
369 pub bench_benchmarks
: bool
,
370 pub logfile
: Option
<PathBuf
>,
372 pub color
: ColorConfig
,
373 pub format
: OutputFormat
,
374 pub test_threads
: Option
<usize>,
375 pub skip
: Vec
<String
>,
376 pub options
: Options
,
381 fn new() -> TestOpts
{
386 run_ignored
: RunIgnored
::No
,
388 bench_benchmarks
: false,
392 format
: OutputFormat
::Pretty
,
395 options
: Options
::new(),
400 /// Result of parsing the options.
401 pub type OptRes
= Result
<TestOpts
, String
>;
403 fn optgroups() -> getopts
::Options
{
404 let mut opts
= getopts
::Options
::new();
405 opts
.optflag("", "include-ignored", "Run ignored and not ignored tests")
406 .optflag("", "ignored", "Run only ignored tests")
407 .optflag("", "test", "Run tests and not benchmarks")
408 .optflag("", "bench", "Run benchmarks instead of tests")
409 .optflag("", "list", "List all tests and benchmarks")
410 .optflag("h", "help", "Display this message (longer with --help)")
414 "Write logs to the specified file instead \
421 "don't capture stdout/stderr of each \
422 task, allow printing directly",
427 "Number of threads used for running tests \
434 "Skip tests whose names contain FILTER (this flag can \
435 be used multiple times)",
441 "Display one character per test instead of one line. \
442 Alias to --format=terse",
447 "Exactly match filters rather than by substring",
452 "Configure coloring of output:
453 auto = colorize if stdout is a tty and tests are run on serially (default);
454 always = always colorize output;
455 never = never colorize output;",
461 "Configure formatting of output:
462 pretty = Print verbose output;
463 terse = Display one character per test;
464 json = Output a json document",
470 "Enable nightly-only flags:
471 unstable-options = Allow use of experimental features",
477 fn usage(binary
: &str, options
: &getopts
::Options
) {
478 let message
= format
!("Usage: {} [OPTIONS] [FILTER]", binary
);
482 The FILTER string is tested against the name of all tests, and only those
483 tests whose names contain the filter are run.
485 By default, all tests are run in parallel. This can be altered with the
486 --test-threads flag or the RUST_TEST_THREADS environment variable when running
489 All tests have their standard output and standard error captured by default.
490 This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE
491 environment variable to a value other than "0". Logging is not captured by default.
495 #[test] - Indicates a function is a test to be run. This function
497 #[bench] - Indicates a function is a benchmark to be run. This
498 function takes one argument (test::Bencher).
499 #[should_panic] - This function (also labeled with #[test]) will only pass if
500 the code causes a panic (an assertion failure or panic!)
501 A message may be provided, which the failure string must
502 contain: #[should_panic(expected = "foo")].
503 #[ignore] - When applied to a function which is already attributed as a
504 test, then the test runner will ignore these tests during
505 normal test runs. Running with --ignored or --include-ignored will run
507 usage
= options
.usage(&message
)
511 // FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566
512 fn is_nightly() -> bool
{
513 // Whether this is a feature-staged build, i.e., on the beta or stable channel
514 let disable_unstable_features
= option_env
!("CFG_DISABLE_UNSTABLE_FEATURES").is_some();
515 // Whether we should enable unstable features for bootstrapping
516 let bootstrap
= env
::var("RUSTC_BOOTSTRAP").is_ok();
518 bootstrap
|| !disable_unstable_features
521 // Parses command line arguments into test options
522 pub fn parse_opts(args
: &[String
]) -> Option
<OptRes
> {
523 let mut allow_unstable
= false;
524 let opts
= optgroups();
525 let args
= args
.get(1..).unwrap_or(args
);
526 let matches
= match opts
.parse(args
) {
528 Err(f
) => return Some(Err(f
.to_string())),
531 if let Some(opt
) = matches
.opt_str("Z") {
534 "the option `Z` is only accepted on the nightly compiler".into(),
539 "unstable-options" => {
540 allow_unstable
= true;
543 return Some(Err("Unrecognized option to `Z`".into()));
548 if matches
.opt_present("h") {
549 usage(&args
[0], &opts
);
553 let filter
= if !matches
.free
.is_empty() {
554 Some(matches
.free
[0].clone())
559 let include_ignored
= matches
.opt_present("include-ignored");
560 if !allow_unstable
&& include_ignored
{
562 "The \"include-ignored\" flag is only accepted on the nightly compiler".into()
566 let run_ignored
= match (include_ignored
, matches
.opt_present("ignored")) {
567 (true, true) => return Some(Err(
568 "the options --include-ignored and --ignored are mutually exclusive".into()
570 (true, false) => RunIgnored
::Yes
,
571 (false, true) => RunIgnored
::Only
,
572 (false, false) => RunIgnored
::No
,
574 let quiet
= matches
.opt_present("quiet");
575 let exact
= matches
.opt_present("exact");
576 let list
= matches
.opt_present("list");
578 let logfile
= matches
.opt_str("logfile");
579 let logfile
= logfile
.map(|s
| PathBuf
::from(&s
));
581 let bench_benchmarks
= matches
.opt_present("bench");
582 let run_tests
= !bench_benchmarks
|| matches
.opt_present("test");
584 let mut nocapture
= matches
.opt_present("nocapture");
586 nocapture
= match env
::var("RUST_TEST_NOCAPTURE") {
587 Ok(val
) => &val
!= "0",
592 let test_threads
= match matches
.opt_str("test-threads") {
593 Some(n_str
) => match n_str
.parse
::<usize>() {
594 Ok(0) => return Some(Err("argument for --test-threads must not be 0".to_string())),
597 return Some(Err(format
!(
598 "argument for --test-threads must be a number > 0 \
607 let color
= match matches
.opt_str("color").as_ref().map(|s
| &**s
) {
608 Some("auto") | None
=> AutoColor
,
609 Some("always") => AlwaysColor
,
610 Some("never") => NeverColor
,
613 return Some(Err(format
!(
614 "argument for --color must be auto, always, or never (was \
621 let format
= match matches
.opt_str("format").as_ref().map(|s
| &**s
) {
622 None
if quiet
=> OutputFormat
::Terse
,
623 Some("pretty") | None
=> OutputFormat
::Pretty
,
624 Some("terse") => OutputFormat
::Terse
,
628 "The \"json\" format is only accepted on the nightly compiler".into(),
635 return Some(Err(format
!(
636 "argument for --format must be pretty, terse, or json (was \
643 let test_opts
= TestOpts
{
655 skip
: matches
.opt_strs("skip"),
656 options
: Options
::new(),
662 #[derive(Clone, PartialEq)]
663 pub struct BenchSamples
{
664 ns_iter_summ
: stats
::Summary
,
668 #[derive(Clone, PartialEq)]
669 pub enum TestResult
{
675 TrBench(BenchSamples
),
678 unsafe impl Send
for TestResult {}
680 enum OutputLocation
<T
> {
681 Pretty(Box
<term
::StdoutTerminal
>),
685 impl<T
: Write
> Write
for OutputLocation
<T
> {
686 fn write(&mut self, buf
: &[u8]) -> io
::Result
<usize> {
688 Pretty(ref mut term
) => term
.write(buf
),
689 Raw(ref mut stdout
) => stdout
.write(buf
),
693 fn flush(&mut self) -> io
::Result
<()> {
695 Pretty(ref mut term
) => term
.flush(),
696 Raw(ref mut stdout
) => stdout
.flush(),
701 struct ConsoleTestState
{
702 log_out
: Option
<File
>,
711 failures
: Vec
<(TestDesc
, Vec
<u8>)>,
712 not_failures
: Vec
<(TestDesc
, Vec
<u8>)>,
716 impl ConsoleTestState
{
717 pub fn new(opts
: &TestOpts
) -> io
::Result
<ConsoleTestState
> {
718 let log_out
= match opts
.logfile
{
719 Some(ref path
) => Some(File
::create(path
)?
),
723 Ok(ConsoleTestState
{
732 metrics
: MetricMap
::new(),
733 failures
: Vec
::new(),
734 not_failures
: Vec
::new(),
735 options
: opts
.options
,
739 pub fn write_log
<S
: AsRef
<str>>(&mut self, msg
: S
) -> io
::Result
<()> {
740 let msg
= msg
.as_ref();
743 Some(ref mut o
) => o
.write_all(msg
.as_bytes()),
747 pub fn write_log_result(&mut self, test
: &TestDesc
, result
: &TestResult
) -> io
::Result
<()> {
748 self.write_log(format
!(
751 TrOk
=> "ok".to_owned(),
752 TrFailed
=> "failed".to_owned(),
753 TrFailedMsg(ref msg
) => format
!("failed: {}", msg
),
754 TrIgnored
=> "ignored".to_owned(),
755 TrAllowedFail
=> "failed (allowed)".to_owned(),
756 TrBench(ref bs
) => fmt_bench_samples(bs
),
762 fn current_test_count(&self) -> usize {
763 self.passed
+ self.failed
+ self.ignored
+ self.measured
+ self.allowed_fail
767 // Format a number with thousands separators
768 fn fmt_thousands_sep(mut n
: usize, sep
: char) -> String
{
770 let mut output
= String
::new();
771 let mut trailing
= false;
772 for &pow
in &[9, 6, 3, 0] {
773 let base
= 10_usize
.pow(pow
);
774 if pow
== 0 || trailing
|| n
/ base
!= 0 {
776 output
.write_fmt(format_args
!("{}", n
/ base
)).unwrap();
778 output
.write_fmt(format_args
!("{:03}", n
/ base
)).unwrap();
791 pub fn fmt_bench_samples(bs
: &BenchSamples
) -> String
{
793 let mut output
= String
::new();
795 let median
= bs
.ns_iter_summ
.median
as usize;
796 let deviation
= (bs
.ns_iter_summ
.max
- bs
.ns_iter_summ
.min
) as usize;
799 .write_fmt(format_args
!(
800 "{:>11} ns/iter (+/- {})",
801 fmt_thousands_sep(median
, '
,'
),
802 fmt_thousands_sep(deviation
, '
,'
)
807 .write_fmt(format_args
!(" = {} MB/s", bs
.mb_s
))
813 // List the tests to console, and optionally to logfile. Filters are honored.
814 pub fn list_tests_console(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>) -> io
::Result
<()> {
815 let mut output
= match term
::stdout() {
816 None
=> Raw(io
::stdout()),
817 Some(t
) => Pretty(t
),
820 let quiet
= opts
.format
== OutputFormat
::Terse
;
821 let mut st
= ConsoleTestState
::new(opts
)?
;
826 for test
in filter_tests(&opts
, tests
) {
830 desc
: TestDesc { name, .. }
,
834 let fntype
= match testfn
{
835 StaticTestFn(..) | DynTestFn(..) => {
839 StaticBenchFn(..) | DynBenchFn(..) => {
845 writeln
!(output
, "{}: {}", name
, fntype
)?
;
846 st
.write_log(format
!("{} {}\n", fntype
, name
))?
;
849 fn plural(count
: u32, s
: &str) -> String
{
851 1 => format
!("{} {}", 1, s
),
852 n
=> format
!("{} {}s", n
, s
),
857 if ntest
!= 0 || nbench
!= 0 {
858 writeln
!(output
, "")?
;
864 plural(ntest
, "test"),
865 plural(nbench
, "benchmark")
872 // A simple console test runner
873 pub fn run_tests_console(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>) -> io
::Result
<bool
> {
876 st
: &mut ConsoleTestState
,
877 out
: &mut dyn OutputFormatter
,
878 ) -> io
::Result
<()> {
879 match (*event
).clone() {
880 TeFiltered(ref filtered_tests
) => {
881 st
.total
= filtered_tests
.len();
882 out
.write_run_start(filtered_tests
.len())
884 TeFilteredOut(filtered_out
) => Ok(st
.filtered_out
= filtered_out
),
885 TeWait(ref test
) => out
.write_test_start(test
),
886 TeTimeout(ref test
) => out
.write_timeout(test
),
887 TeResult(test
, result
, stdout
) => {
888 st
.write_log_result(&test
, &result
)?
;
889 out
.write_result(&test
, &result
, &*stdout
)?
;
893 st
.not_failures
.push((test
, stdout
));
895 TrIgnored
=> st
.ignored
+= 1,
896 TrAllowedFail
=> st
.allowed_fail
+= 1,
898 st
.metrics
.insert_metric(
899 test
.name
.as_slice(),
900 bs
.ns_iter_summ
.median
,
901 bs
.ns_iter_summ
.max
- bs
.ns_iter_summ
.min
,
907 st
.failures
.push((test
, stdout
));
909 TrFailedMsg(msg
) => {
911 let mut stdout
= stdout
;
912 stdout
.extend_from_slice(format
!("note: {}", msg
).as_bytes());
913 st
.failures
.push((test
, stdout
));
921 let output
= match term
::stdout() {
922 None
=> Raw(io
::stdout()),
923 Some(t
) => Pretty(t
),
926 let max_name_len
= tests
928 .max_by_key(|t
| len_if_padded(*t
))
929 .map(|t
| t
.desc
.name
.as_slice().len())
932 let is_multithreaded
= opts
.test_threads
.unwrap_or_else(get_concurrency
) > 1;
934 let mut out
: Box
<dyn OutputFormatter
> = match opts
.format
{
935 OutputFormat
::Pretty
=> Box
::new(PrettyFormatter
::new(
941 OutputFormat
::Terse
=> Box
::new(TerseFormatter
::new(
947 OutputFormat
::Json
=> Box
::new(JsonFormatter
::new(output
)),
949 let mut st
= ConsoleTestState
::new(opts
)?
;
950 fn len_if_padded(t
: &TestDescAndFn
) -> usize {
951 match t
.testfn
.padding() {
953 PadOnRight
=> t
.desc
.name
.as_slice().len(),
957 run_tests(opts
, tests
, |x
| callback(&x
, &mut st
, &mut *out
))?
;
959 assert
!(st
.current_test_count() == st
.total
);
961 return out
.write_run_finish(&st
);
965 fn should_sort_failures_before_printing_them() {
966 let test_a
= TestDesc
{
967 name
: StaticTestName("a"),
969 should_panic
: ShouldPanic
::No
,
973 let test_b
= TestDesc
{
974 name
: StaticTestName("b"),
976 should_panic
: ShouldPanic
::No
,
980 let mut out
= PrettyFormatter
::new(Raw(Vec
::new()), false, 10, false);
982 let st
= ConsoleTestState
{
991 metrics
: MetricMap
::new(),
992 failures
: vec
![(test_b
, Vec
::new()), (test_a
, Vec
::new())],
993 options
: Options
::new(),
994 not_failures
: Vec
::new(),
997 out
.write_failures(&st
).unwrap();
998 let s
= match out
.output_location() {
999 &Raw(ref m
) => String
::from_utf8_lossy(&m
[..]),
1000 &Pretty(_
) => unreachable
!(),
1003 let apos
= s
.find("a").unwrap();
1004 let bpos
= s
.find("b").unwrap();
1005 assert
!(apos
< bpos
);
1008 fn use_color(opts
: &TestOpts
) -> bool
{
1010 AutoColor
=> !opts
.nocapture
&& stdout_isatty(),
1011 AlwaysColor
=> true,
1012 NeverColor
=> false,
1016 #[cfg(any(target_os = "cloudabi",
1017 target_os
= "redox",
1018 all(target_arch
= "wasm32", not(target_os
= "emscripten")),
1019 all(target_vendor
= "fortanix", target_env
= "sgx")))]
1020 fn stdout_isatty() -> bool
{
1021 // FIXME: Implement isatty on Redox and SGX
1025 fn stdout_isatty() -> bool
{
1026 unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
1029 fn stdout_isatty() -> bool
{
1032 type HANDLE
= *mut u8;
1033 type LPDWORD
= *mut u32;
1034 const STD_OUTPUT_HANDLE
: DWORD
= -11i32 as DWORD
;
1036 fn GetStdHandle(which
: DWORD
) -> HANDLE
;
1037 fn GetConsoleMode(hConsoleHandle
: HANDLE
, lpMode
: LPDWORD
) -> BOOL
;
1040 let handle
= GetStdHandle(STD_OUTPUT_HANDLE
);
1042 GetConsoleMode(handle
, &mut out
) != 0
1047 pub enum TestEvent
{
1048 TeFiltered(Vec
<TestDesc
>),
1050 TeResult(TestDesc
, TestResult
, Vec
<u8>),
1051 TeTimeout(TestDesc
),
1052 TeFilteredOut(usize),
1055 pub type MonitorMsg
= (TestDesc
, TestResult
, Vec
<u8>);
1057 struct Sink(Arc
<Mutex
<Vec
<u8>>>);
1058 impl Write
for Sink
{
1059 fn write(&mut self, data
: &[u8]) -> io
::Result
<usize> {
1060 Write
::write(&mut *self.0.lock().unwrap(), data
)
1062 fn flush(&mut self) -> io
::Result
<()> {
1067 pub fn run_tests
<F
>(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>, mut callback
: F
) -> io
::Result
<()>
1069 F
: FnMut(TestEvent
) -> io
::Result
<()>,
1071 use std
::collections
::{self, HashMap}
;
1072 use std
::hash
::BuildHasherDefault
;
1073 use std
::sync
::mpsc
::RecvTimeoutError
;
1074 // Use a deterministic hasher
1076 HashMap
<TestDesc
, Instant
, BuildHasherDefault
<collections
::hash_map
::DefaultHasher
>>;
1078 let tests_len
= tests
.len();
1080 let mut filtered_tests
= filter_tests(opts
, tests
);
1081 if !opts
.bench_benchmarks
{
1082 filtered_tests
= convert_benchmarks_to_tests(filtered_tests
);
1085 let filtered_tests
= {
1086 let mut filtered_tests
= filtered_tests
;
1087 for test
in filtered_tests
.iter_mut() {
1088 test
.desc
.name
= test
.desc
.name
.with_padding(test
.testfn
.padding());
1094 let filtered_out
= tests_len
- filtered_tests
.len();
1095 callback(TeFilteredOut(filtered_out
))?
;
1097 let filtered_descs
= filtered_tests
.iter().map(|t
| t
.desc
.clone()).collect();
1099 callback(TeFiltered(filtered_descs
))?
;
1101 let (filtered_tests
, filtered_benchs
): (Vec
<_
>, _
) =
1102 filtered_tests
.into_iter().partition(|e
| match e
.testfn
{
1103 StaticTestFn(_
) | DynTestFn(_
) => true,
1107 let concurrency
= opts
.test_threads
.unwrap_or_else(get_concurrency
);
1109 let mut remaining
= filtered_tests
;
1110 remaining
.reverse();
1111 let mut pending
= 0;
1113 let (tx
, rx
) = channel
::<MonitorMsg
>();
1115 let mut running_tests
: TestMap
= HashMap
::default();
1117 fn get_timed_out_tests(running_tests
: &mut TestMap
) -> Vec
<TestDesc
> {
1118 let now
= Instant
::now();
1119 let timed_out
= running_tests
1121 .filter_map(|(desc
, timeout
)| {
1122 if &now
>= timeout
{
1129 for test
in &timed_out
{
1130 running_tests
.remove(test
);
1135 fn calc_timeout(running_tests
: &TestMap
) -> Option
<Duration
> {
1136 running_tests
.values().min().map(|next_timeout
| {
1137 let now
= Instant
::now();
1138 if *next_timeout
>= now
{
1146 if concurrency
== 1 {
1147 while !remaining
.is_empty() {
1148 let test
= remaining
.pop().unwrap();
1149 callback(TeWait(test
.desc
.clone()))?
;
1150 run_test(opts
, !opts
.run_tests
, test
, tx
.clone(), Concurrent
::No
);
1151 let (test
, result
, stdout
) = rx
.recv().unwrap();
1152 callback(TeResult(test
, result
, stdout
))?
;
1155 while pending
> 0 || !remaining
.is_empty() {
1156 while pending
< concurrency
&& !remaining
.is_empty() {
1157 let test
= remaining
.pop().unwrap();
1158 let timeout
= Instant
::now() + Duration
::from_secs(TEST_WARN_TIMEOUT_S
);
1159 running_tests
.insert(test
.desc
.clone(), timeout
);
1160 callback(TeWait(test
.desc
.clone()))?
; //here no pad
1161 run_test(opts
, !opts
.run_tests
, test
, tx
.clone(), Concurrent
::Yes
);
1167 if let Some(timeout
) = calc_timeout(&running_tests
) {
1168 res
= rx
.recv_timeout(timeout
);
1169 for test
in get_timed_out_tests(&mut running_tests
) {
1170 callback(TeTimeout(test
))?
;
1172 if res
!= Err(RecvTimeoutError
::Timeout
) {
1176 res
= rx
.recv().map_err(|_
| RecvTimeoutError
::Disconnected
);
1181 let (desc
, result
, stdout
) = res
.unwrap();
1182 running_tests
.remove(&desc
);
1184 callback(TeResult(desc
, result
, stdout
))?
;
1189 if opts
.bench_benchmarks
{
1190 // All benchmarks run at the end, in serial.
1191 for b
in filtered_benchs
{
1192 callback(TeWait(b
.desc
.clone()))?
;
1193 run_test(opts
, false, b
, tx
.clone(), Concurrent
::No
);
1194 let (test
, result
, stdout
) = rx
.recv().unwrap();
1195 callback(TeResult(test
, result
, stdout
))?
;
1201 #[allow(deprecated)]
1202 fn get_concurrency() -> usize {
1203 return match env
::var("RUST_TEST_THREADS") {
1205 let opt_n
: Option
<usize> = s
.parse().ok();
1207 Some(n
) if n
> 0 => n
,
1209 "RUST_TEST_THREADS is `{}`, should be a positive integer.",
1214 Err(..) => num_cpus(),
1218 #[allow(nonstandard_style)]
1219 fn num_cpus() -> usize {
1221 struct SYSTEM_INFO
{
1222 wProcessorArchitecture
: u16,
1225 lpMinimumApplicationAddress
: *mut u8,
1226 lpMaximumApplicationAddress
: *mut u8,
1227 dwActiveProcessorMask
: *mut u8,
1228 dwNumberOfProcessors
: u32,
1229 dwProcessorType
: u32,
1230 dwAllocationGranularity
: u32,
1231 wProcessorLevel
: u16,
1232 wProcessorRevision
: u16,
1235 fn GetSystemInfo(info
: *mut SYSTEM_INFO
) -> i32;
1238 let mut sysinfo
= std
::mem
::zeroed();
1239 GetSystemInfo(&mut sysinfo
);
1240 sysinfo
.dwNumberOfProcessors
as usize
1244 #[cfg(target_os = "redox")]
1245 fn num_cpus() -> usize {
1246 // FIXME: Implement num_cpus on Redox
1250 #[cfg(any(all(target_arch = "wasm32", not(target_os = "emscripten")),
1251 all(target_vendor
= "fortanix", target_env
= "sgx")))]
1252 fn num_cpus() -> usize {
1256 #[cfg(any(target_os = "android", target_os = "cloudabi", target_os = "emscripten",
1257 target_os
= "fuchsia", target_os
= "ios", target_os
= "linux",
1258 target_os
= "macos", target_os
= "solaris"))]
1259 fn num_cpus() -> usize {
1260 unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
1263 #[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "bitrig",
1264 target_os
= "netbsd"))]
1265 fn num_cpus() -> usize {
1268 let mut cpus
: libc
::c_uint
= 0;
1269 let mut cpus_size
= std
::mem
::size_of_val(&cpus
);
1272 cpus
= libc
::sysconf(libc
::_SC_NPROCESSORS_ONLN
) as libc
::c_uint
;
1275 let mut mib
= [libc
::CTL_HW
, libc
::HW_NCPU
, 0, 0];
1280 &mut cpus
as *mut _
as *mut _
,
1281 &mut cpus_size
as *mut _
as *mut _
,
1293 #[cfg(target_os = "openbsd")]
1294 fn num_cpus() -> usize {
1297 let mut cpus
: libc
::c_uint
= 0;
1298 let mut cpus_size
= std
::mem
::size_of_val(&cpus
);
1299 let mut mib
= [libc
::CTL_HW
, libc
::HW_NCPU
, 0, 0];
1305 &mut cpus
as *mut _
as *mut _
,
1306 &mut cpus_size
as *mut _
as *mut _
,
1317 #[cfg(target_os = "haiku")]
1318 fn num_cpus() -> usize {
1323 #[cfg(target_os = "l4re")]
1324 fn num_cpus() -> usize {
1330 pub fn filter_tests(opts
: &TestOpts
, tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
1331 let mut filtered
= tests
;
1332 let matches_filter
= |test
: &TestDescAndFn
, filter
: &str| {
1333 let test_name
= test
.desc
.name
.as_slice();
1335 match opts
.filter_exact
{
1336 true => test_name
== filter
,
1337 false => test_name
.contains(filter
),
1341 // Remove tests that don't match the test filter
1342 if let Some(ref filter
) = opts
.filter
{
1343 filtered
.retain(|test
| matches_filter(test
, filter
));
1346 // Skip tests that match any of the skip filters
1347 filtered
.retain(|test
| {
1348 !opts
.skip
.iter().any(|sf
| matches_filter(test
, sf
))
1351 // maybe unignore tests
1352 match opts
.run_ignored
{
1353 RunIgnored
::Yes
=> {
1354 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
1356 RunIgnored
::Only
=> {
1357 filtered
.retain(|test
| test
.desc
.ignore
);
1358 filtered
.iter_mut().for_each(|test
| test
.desc
.ignore
= false);
1360 RunIgnored
::No
=> {}
1363 // Sort the tests alphabetically
1364 filtered
.sort_by(|t1
, t2
| t1
.desc
.name
.as_slice().cmp(t2
.desc
.name
.as_slice()));
1369 pub fn convert_benchmarks_to_tests(tests
: Vec
<TestDescAndFn
>) -> Vec
<TestDescAndFn
> {
1370 // convert benchmarks to tests, if we're not benchmarking them
1374 let testfn
= match x
.testfn
{
1375 DynBenchFn(bench
) => DynTestFn(Box
::new(move || {
1376 bench
::run_once(|b
| __rust_begin_short_backtrace(|| bench
.run(b
)))
1378 StaticBenchFn(benchfn
) => DynTestFn(Box
::new(move || {
1379 bench
::run_once(|b
| __rust_begin_short_backtrace(|| benchfn(b
)))
1394 test
: TestDescAndFn
,
1395 monitor_ch
: Sender
<MonitorMsg
>,
1396 concurrency
: Concurrent
,
1398 let TestDescAndFn { desc, testfn }
= test
;
1400 let ignore_because_panic_abort
= cfg
!(target_arch
= "wasm32") && !cfg
!(target_os
= "emscripten")
1401 && desc
.should_panic
!= ShouldPanic
::No
;
1403 if force_ignore
|| desc
.ignore
|| ignore_because_panic_abort
{
1404 monitor_ch
.send((desc
, TrIgnored
, Vec
::new())).unwrap();
1410 monitor_ch
: Sender
<MonitorMsg
>,
1412 testfn
: Box
<dyn FnBox() + Send
>,
1413 concurrency
: Concurrent
,
1415 // Buffer for capturing standard I/O
1416 let data
= Arc
::new(Mutex
::new(Vec
::new()));
1417 let data2
= data
.clone();
1419 let name
= desc
.name
.clone();
1420 let runtest
= move || {
1421 let oldio
= if !nocapture
{
1423 io
::set_print(Some(Box
::new(Sink(data2
.clone())))),
1424 io
::set_panic(Some(Box
::new(Sink(data2
)))),
1430 let result
= catch_unwind(AssertUnwindSafe(testfn
));
1432 if let Some((printio
, panicio
)) = oldio
{
1433 io
::set_print(printio
);
1434 io
::set_panic(panicio
);
1437 let test_result
= calc_result(&desc
, result
);
1438 let stdout
= data
.lock().unwrap().to_vec();
1440 .send((desc
.clone(), test_result
, stdout
))
1444 // If the platform is single-threaded we're just going to run
1445 // the test synchronously, regardless of the concurrency
1447 let supports_threads
= !cfg
!(target_os
= "emscripten") && !cfg
!(target_arch
= "wasm32");
1448 if concurrency
== Concurrent
::Yes
&& supports_threads
{
1449 let cfg
= thread
::Builder
::new().name(name
.as_slice().to_owned());
1450 cfg
.spawn(runtest
).unwrap();
1457 DynBenchFn(bencher
) => {
1458 ::bench
::benchmark(desc
, monitor_ch
, opts
.nocapture
, |harness
| {
1459 bencher
.run(harness
)
1462 StaticBenchFn(benchfn
) => {
1463 ::bench
::benchmark(desc
, monitor_ch
, opts
.nocapture
, |harness
| {
1464 (benchfn
.clone())(harness
)
1468 let cb
= move || __rust_begin_short_backtrace(f
);
1469 run_test_inner(desc
, monitor_ch
, opts
.nocapture
, Box
::new(cb
), concurrency
)
1471 StaticTestFn(f
) => run_test_inner(
1475 Box
::new(move || __rust_begin_short_backtrace(f
)),
1481 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
1483 fn __rust_begin_short_backtrace
<F
: FnOnce()>(f
: F
) {
1487 fn calc_result(desc
: &TestDesc
, task_result
: Result
<(), Box
<dyn Any
+ Send
>>) -> TestResult
{
1488 match (&desc
.should_panic
, task_result
) {
1489 (&ShouldPanic
::No
, Ok(())) | (&ShouldPanic
::Yes
, Err(_
)) => TrOk
,
1490 (&ShouldPanic
::YesWithMessage(msg
), Err(ref err
)) => {
1491 if err
.downcast_ref
::<String
>()
1493 .or_else(|| err
.downcast_ref
::<&'
static str>().map(|e
| *e
))
1494 .map(|e
| e
.contains(msg
))
1499 if desc
.allow_fail
{
1502 TrFailedMsg(format
!("Panic did not include expected string '{}'", msg
))
1506 _
if desc
.allow_fail
=> TrAllowedFail
,
1511 #[derive(Clone, PartialEq)]
1512 pub struct MetricMap(BTreeMap
<String
, Metric
>);
1515 pub fn new() -> MetricMap
{
1516 MetricMap(BTreeMap
::new())
1519 /// Insert a named `value` (+/- `noise`) metric into the map. The value
1520 /// must be non-negative. The `noise` indicates the uncertainty of the
1521 /// metric, which doubles as the "noise range" of acceptable
1522 /// pairwise-regressions on this named value, when comparing from one
1523 /// metric to the next using `compare_to_old`.
1525 /// If `noise` is positive, then it means this metric is of a value
1526 /// you want to see grow smaller, so a change larger than `noise` in the
1527 /// positive direction represents a regression.
1529 /// If `noise` is negative, then it means this metric is of a value
1530 /// you want to see grow larger, so a change larger than `noise` in the
1531 /// negative direction represents a regression.
1532 pub fn insert_metric(&mut self, name
: &str, value
: f64, noise
: f64) {
1533 let m
= Metric { value, noise }
;
1534 self.0.insert
(name
.to_owned(), m
);
1537 pub fn fmt_metrics(&self) -> String
{
1540 .map(|(k
, v
)| format
!("{}: {} (+/- {})", *k
, v
.value
, v
.noise
))
1541 .collect
::<Vec
<_
>>();
1548 /// A function that is opaque to the optimizer, to allow benchmarks to
1549 /// pretend to use outputs to assist in avoiding dead-code
1552 /// This function is a no-op, and does not even read from `dummy`.
1553 #[cfg(not(any(target_arch = "asmjs", target_arch = "wasm32")))]
1554 pub fn black_box
<T
>(dummy
: T
) -> T
{
1555 // we need to "use" the argument in some way LLVM can't
1557 unsafe { asm!("" : : "r"(&dummy)) }
1560 #[cfg(any(target_arch = "asmjs", target_arch = "wasm32"))]
1562 pub fn black_box
<T
>(dummy
: T
) -> T
{
1567 /// Callback for benchmark functions to run in their body.
1568 pub fn iter
<T
, F
>(&mut self, mut inner
: F
)
1572 if self.mode
== BenchMode
::Single
{
1573 ns_iter_inner(&mut inner
, 1);
1577 self.summary
= Some(iter(&mut inner
));
1580 pub fn bench
<F
>(&mut self, mut f
: F
) -> Option
<stats
::Summary
>
1582 F
: FnMut(&mut Bencher
),
1585 return self.summary
;
1589 fn ns_from_dur(dur
: Duration
) -> u64 {
1590 dur
.as_secs() * 1_000_000_000 + (dur
.subsec_nanos() as u64)
1593 fn ns_iter_inner
<T
, F
>(inner
: &mut F
, k
: u64) -> u64
1597 let start
= Instant
::now();
1601 return ns_from_dur(start
.elapsed());
1604 pub fn iter
<T
, F
>(inner
: &mut F
) -> stats
::Summary
1608 // Initial bench run to get ballpark figure.
1609 let ns_single
= ns_iter_inner(inner
, 1);
1611 // Try to estimate iter count for 1ms falling back to 1m
1612 // iterations if first run took < 1ns.
1613 let ns_target_total
= 1_000_000; // 1ms
1614 let mut n
= ns_target_total
/ cmp
::max(1, ns_single
);
1616 // if the first run took more than 1ms we don't want to just
1617 // be left doing 0 iterations on every loop. The unfortunate
1618 // side effect of not being able to do as many runs is
1619 // automatically handled by the statistical analysis below
1620 // (i.e., larger error bars).
1623 let mut total_run
= Duration
::new(0, 0);
1624 let samples
: &mut [f64] = &mut [0.0_f64; 50];
1626 let loop_start
= Instant
::now();
1628 for p
in &mut *samples
{
1629 *p
= ns_iter_inner(inner
, n
) as f64 / n
as f64;
1632 stats
::winsorize(samples
, 5.0);
1633 let summ
= stats
::Summary
::new(samples
);
1635 for p
in &mut *samples
{
1636 let ns
= ns_iter_inner(inner
, 5 * n
);
1637 *p
= ns
as f64 / (5 * n
) as f64;
1640 stats
::winsorize(samples
, 5.0);
1641 let summ5
= stats
::Summary
::new(samples
);
1643 let loop_run
= loop_start
.elapsed();
1645 // If we've run for 100ms and seem to have converged to a
1647 if loop_run
> Duration
::from_millis(100) && summ
.median_abs_dev_pct
< 1.0
1648 && summ
.median
- summ5
.median
< summ5
.median_abs_dev
1653 total_run
= total_run
+ loop_run
;
1654 // Longest we ever run for is 3s.
1655 if total_run
> Duration
::from_secs(3) {
1659 // If we overflow here just return the results so far. We check a
1660 // multiplier of 10 because we're about to multiply by 2 and the
1661 // next iteration of the loop will also multiply by 5 (to calculate
1662 // the summ5 result)
1663 n
= match n
.checked_mul(10) {
1673 use std
::panic
::{catch_unwind, AssertUnwindSafe}
;
1676 use std
::sync
::{Arc, Mutex}
;
1678 use super::{BenchMode, BenchSamples, Bencher, MonitorMsg, Sender, Sink, TestDesc, TestResult}
;
1680 pub fn benchmark
<F
>(desc
: TestDesc
, monitor_ch
: Sender
<MonitorMsg
>, nocapture
: bool
, f
: F
)
1682 F
: FnMut(&mut Bencher
),
1684 let mut bs
= Bencher
{
1685 mode
: BenchMode
::Auto
,
1690 let data
= Arc
::new(Mutex
::new(Vec
::new()));
1691 let data2
= data
.clone();
1693 let oldio
= if !nocapture
{
1695 io
::set_print(Some(Box
::new(Sink(data2
.clone())))),
1696 io
::set_panic(Some(Box
::new(Sink(data2
)))),
1702 let result
= catch_unwind(AssertUnwindSafe(|| bs
.bench(f
)));
1704 if let Some((printio
, panicio
)) = oldio
{
1705 io
::set_print(printio
);
1706 io
::set_panic(panicio
);
1709 let test_result
= match result
{
1711 Ok(Some(ns_iter_summ
)) => {
1712 let ns_iter
= cmp
::max(ns_iter_summ
.median
as u64, 1);
1713 let mb_s
= bs
.bytes
* 1000 / ns_iter
;
1715 let bs
= BenchSamples
{
1717 mb_s
: mb_s
as usize,
1719 TestResult
::TrBench(bs
)
1722 // iter not called, so no data.
1723 // FIXME: error in this case?
1724 let samples
: &mut [f64] = &mut [0.0_f64; 1];
1725 let bs
= BenchSamples
{
1726 ns_iter_summ
: stats
::Summary
::new(samples
),
1729 TestResult
::TrBench(bs
)
1731 Err(_
) => TestResult
::TrFailed
,
1734 let stdout
= data
.lock().unwrap().to_vec();
1735 monitor_ch
.send((desc
, test_result
, stdout
)).unwrap();
1738 pub fn run_once
<F
>(f
: F
)
1740 F
: FnMut(&mut Bencher
),
1742 let mut bs
= Bencher
{
1743 mode
: BenchMode
::Single
,
1753 use test
::{filter_tests
, parse_opts
, run_test
, DynTestFn
, DynTestName
, MetricMap
, RunIgnored
,
1754 ShouldPanic
, StaticTestName
, TestDesc
, TestDescAndFn
, TestOpts
, TrFailed
,
1755 TrFailedMsg
, TrIgnored
, TrOk
};
1756 use std
::sync
::mpsc
::channel
;
1762 fn one_ignored_one_unignored_test() -> Vec
<TestDescAndFn
> {
1766 name
: StaticTestName("1"),
1768 should_panic
: ShouldPanic
::No
,
1771 testfn
: DynTestFn(Box
::new(move || {}
)),
1775 name
: StaticTestName("2"),
1777 should_panic
: ShouldPanic
::No
,
1780 testfn
: DynTestFn(Box
::new(move || {}
)),
1786 pub fn do_not_run_ignored_tests() {
1790 let desc
= TestDescAndFn
{
1792 name
: StaticTestName("whatever"),
1794 should_panic
: ShouldPanic
::No
,
1797 testfn
: DynTestFn(Box
::new(f
)),
1799 let (tx
, rx
) = channel();
1800 run_test(&TestOpts
::new(), false, desc
, tx
, Concurrent
::No
);
1801 let (_
, res
, _
) = rx
.recv().unwrap();
1802 assert
!(res
!= TrOk
);
1806 pub fn ignored_tests_result_in_ignored() {
1808 let desc
= TestDescAndFn
{
1810 name
: StaticTestName("whatever"),
1812 should_panic
: ShouldPanic
::No
,
1815 testfn
: DynTestFn(Box
::new(f
)),
1817 let (tx
, rx
) = channel();
1818 run_test(&TestOpts
::new(), false, desc
, tx
, Concurrent
::No
);
1819 let (_
, res
, _
) = rx
.recv().unwrap();
1820 assert
!(res
== TrIgnored
);
1824 fn test_should_panic() {
1828 let desc
= TestDescAndFn
{
1830 name
: StaticTestName("whatever"),
1832 should_panic
: ShouldPanic
::Yes
,
1835 testfn
: DynTestFn(Box
::new(f
)),
1837 let (tx
, rx
) = channel();
1838 run_test(&TestOpts
::new(), false, desc
, tx
, Concurrent
::No
);
1839 let (_
, res
, _
) = rx
.recv().unwrap();
1840 assert
!(res
== TrOk
);
1844 fn test_should_panic_good_message() {
1846 panic
!("an error message");
1848 let desc
= TestDescAndFn
{
1850 name
: StaticTestName("whatever"),
1852 should_panic
: ShouldPanic
::YesWithMessage("error message"),
1855 testfn
: DynTestFn(Box
::new(f
)),
1857 let (tx
, rx
) = channel();
1858 run_test(&TestOpts
::new(), false, desc
, tx
, Concurrent
::No
);
1859 let (_
, res
, _
) = rx
.recv().unwrap();
1860 assert
!(res
== TrOk
);
1864 fn test_should_panic_bad_message() {
1866 panic
!("an error message");
1868 let expected
= "foobar";
1869 let failed_msg
= "Panic did not include expected string";
1870 let desc
= TestDescAndFn
{
1872 name
: StaticTestName("whatever"),
1874 should_panic
: ShouldPanic
::YesWithMessage(expected
),
1877 testfn
: DynTestFn(Box
::new(f
)),
1879 let (tx
, rx
) = channel();
1880 run_test(&TestOpts
::new(), false, desc
, tx
, Concurrent
::No
);
1881 let (_
, res
, _
) = rx
.recv().unwrap();
1882 assert
!(res
== TrFailedMsg(format
!("{} '{}'", failed_msg
, expected
)));
1886 fn test_should_panic_but_succeeds() {
1888 let desc
= TestDescAndFn
{
1890 name
: StaticTestName("whatever"),
1892 should_panic
: ShouldPanic
::Yes
,
1895 testfn
: DynTestFn(Box
::new(f
)),
1897 let (tx
, rx
) = channel();
1898 run_test(&TestOpts
::new(), false, desc
, tx
, Concurrent
::No
);
1899 let (_
, res
, _
) = rx
.recv().unwrap();
1900 assert
!(res
== TrFailed
);
1904 fn parse_ignored_flag() {
1906 "progname".to_string(),
1907 "filter".to_string(),
1908 "--ignored".to_string(),
1910 let opts
= parse_opts(&args
).unwrap().unwrap();
1911 assert_eq
!(opts
.run_ignored
, RunIgnored
::Only
);
1915 fn parse_include_ignored_flag() {
1917 "progname".to_string(),
1918 "filter".to_string(),
1919 "-Zunstable-options".to_string(),
1920 "--include-ignored".to_string(),
1922 let opts
= parse_opts(&args
).unwrap().unwrap();
1923 assert_eq
!(opts
.run_ignored
, RunIgnored
::Yes
);
1927 pub fn filter_for_ignored_option() {
1928 // When we run ignored tests the test filter should filter out all the
1929 // unignored tests and flip the ignore flag on the rest to false
1931 let mut opts
= TestOpts
::new();
1932 opts
.run_tests
= true;
1933 opts
.run_ignored
= RunIgnored
::Only
;
1935 let tests
= one_ignored_one_unignored_test();
1936 let filtered
= filter_tests(&opts
, tests
);
1938 assert_eq
!(filtered
.len(), 1);
1939 assert_eq
!(filtered
[0].desc
.name
.to_string(), "1");
1940 assert
!(!filtered
[0].desc
.ignore
);
1944 pub fn run_include_ignored_option() {
1945 // When we "--include-ignored" tests, the ignore flag should be set to false on
1946 // all tests and no test filtered out
1948 let mut opts
= TestOpts
::new();
1949 opts
.run_tests
= true;
1950 opts
.run_ignored
= RunIgnored
::Yes
;
1952 let tests
= one_ignored_one_unignored_test();
1953 let filtered
= filter_tests(&opts
, tests
);
1955 assert_eq
!(filtered
.len(), 2);
1956 assert
!(!filtered
[0].desc
.ignore
);
1957 assert
!(!filtered
[1].desc
.ignore
);
1961 pub fn exact_filter_match() {
1962 fn tests() -> Vec
<TestDescAndFn
> {
1963 vec
!["base", "base::test", "base::test1", "base::test2"]
1965 .map(|name
| TestDescAndFn
{
1967 name
: StaticTestName(name
),
1969 should_panic
: ShouldPanic
::No
,
1972 testfn
: DynTestFn(Box
::new(move || {}
)),
1977 let substr
= filter_tests(
1979 filter
: Some("base".into()),
1984 assert_eq
!(substr
.len(), 4);
1986 let substr
= filter_tests(
1988 filter
: Some("bas".into()),
1993 assert_eq
!(substr
.len(), 4);
1995 let substr
= filter_tests(
1997 filter
: Some("::test".into()),
2002 assert_eq
!(substr
.len(), 3);
2004 let substr
= filter_tests(
2006 filter
: Some("base::test".into()),
2011 assert_eq
!(substr
.len(), 3);
2013 let exact
= filter_tests(
2015 filter
: Some("base".into()),
2021 assert_eq
!(exact
.len(), 1);
2023 let exact
= filter_tests(
2025 filter
: Some("bas".into()),
2031 assert_eq
!(exact
.len(), 0);
2033 let exact
= filter_tests(
2035 filter
: Some("::test".into()),
2041 assert_eq
!(exact
.len(), 0);
2043 let exact
= filter_tests(
2045 filter
: Some("base::test".into()),
2051 assert_eq
!(exact
.len(), 1);
2055 pub fn sort_tests() {
2056 let mut opts
= TestOpts
::new();
2057 opts
.run_tests
= true;
2060 "sha1::test".to_string(),
2061 "isize::test_to_str".to_string(),
2062 "isize::test_pow".to_string(),
2063 "test::do_not_run_ignored_tests".to_string(),
2064 "test::ignored_tests_result_in_ignored".to_string(),
2065 "test::first_free_arg_should_be_a_filter".to_string(),
2066 "test::parse_ignored_flag".to_string(),
2067 "test::parse_include_ignored_flag".to_string(),
2068 "test::filter_for_ignored_option".to_string(),
2069 "test::run_include_ignored_option".to_string(),
2070 "test::sort_tests".to_string(),
2074 let mut tests
= Vec
::new();
2075 for name
in &names
{
2076 let test
= TestDescAndFn
{
2078 name
: DynTestName((*name
).clone()),
2080 should_panic
: ShouldPanic
::No
,
2083 testfn
: DynTestFn(Box
::new(testfn
)),
2089 let filtered
= filter_tests(&opts
, tests
);
2091 let expected
= vec
![
2092 "isize::test_pow".to_string(),
2093 "isize::test_to_str".to_string(),
2094 "sha1::test".to_string(),
2095 "test::do_not_run_ignored_tests".to_string(),
2096 "test::filter_for_ignored_option".to_string(),
2097 "test::first_free_arg_should_be_a_filter".to_string(),
2098 "test::ignored_tests_result_in_ignored".to_string(),
2099 "test::parse_ignored_flag".to_string(),
2100 "test::parse_include_ignored_flag".to_string(),
2101 "test::run_include_ignored_option".to_string(),
2102 "test::sort_tests".to_string(),
2105 for (a
, b
) in expected
.iter().zip(filtered
) {
2106 assert
!(*a
== b
.desc
.name
.to_string());
2111 pub fn test_metricmap_compare() {
2112 let mut m1
= MetricMap
::new();
2113 let mut m2
= MetricMap
::new();
2114 m1
.insert_metric("in-both-noise", 1000.0, 200.0);
2115 m2
.insert_metric("in-both-noise", 1100.0, 200.0);
2117 m1
.insert_metric("in-first-noise", 1000.0, 2.0);
2118 m2
.insert_metric("in-second-noise", 1000.0, 2.0);
2120 m1
.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
2121 m2
.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
2123 m1
.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
2124 m2
.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
2126 m1
.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
2127 m2
.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
2129 m1
.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
2130 m2
.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
2134 pub fn test_bench_once_no_iter() {
2135 fn f(_
: &mut Bencher
) {}
2140 pub fn test_bench_once_iter() {
2141 fn f(b
: &mut Bencher
) {
2148 pub fn test_bench_no_iter() {
2149 fn f(_
: &mut Bencher
) {}
2151 let (tx
, rx
) = channel();
2153 let desc
= TestDesc
{
2154 name
: StaticTestName("f"),
2156 should_panic
: ShouldPanic
::No
,
2160 ::bench
::benchmark(desc
, tx
, true, f
);
2165 pub fn test_bench_iter() {
2166 fn f(b
: &mut Bencher
) {
2170 let (tx
, rx
) = channel();
2172 let desc
= TestDesc
{
2173 name
: StaticTestName("f"),
2175 should_panic
: ShouldPanic
::No
,
2179 ::bench
::benchmark(desc
, tx
, true, f
);