]>
Commit | Line | Data |
---|---|---|
1a4d82fc JJ |
1 | // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT |
2 | // file at the top-level directory of this distribution and at | |
3 | // http://rust-lang.org/COPYRIGHT. | |
4 | // | |
5 | // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | // option. This file may not be copied, modified, or distributed | |
9 | // except according to those terms. | |
10 | ||
11 | //! Support code for rustc's built in unit-test and micro-benchmarking | |
12 | //! framework. | |
13 | //! | |
14 | //! Almost all user code will only be interested in `Bencher` and | |
15 | //! `black_box`. All other interactions (such as writing tests and | |
16 | //! benchmarks themselves) should be done via the `#[test]` and | |
17 | //! `#[bench]` attributes. | |
18 | //! | |
85aaf69f | 19 | //! See the [Testing Chapter](../book/testing.html) of the book for more details. |
1a4d82fc JJ |
20 | |
21 | // Currently, not much of this is meant for users. It is intended to | |
22 | // support the simplest interface possible for representing and | |
23 | // running tests while providing a base that other test frameworks may | |
24 | // build off of. | |
25 | ||
26 | #![crate_name = "test"] | |
e9174d1e | 27 | #![unstable(feature = "test", issue = "27812")] |
1a4d82fc JJ |
28 | #![crate_type = "rlib"] |
29 | #![crate_type = "dylib"] | |
e9174d1e | 30 | #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png", |
62682a34 | 31 | html_favicon_url = "https://doc.rust-lang.org/favicon.ico", |
92a42be0 SL |
32 | html_root_url = "https://doc.rust-lang.org/nightly/", |
33 | test(attr(deny(warnings))))] | |
32a655c1 | 34 | #![deny(warnings)] |
85aaf69f SL |
35 | |
36 | #![feature(asm)] | |
62682a34 | 37 | #![feature(libc)] |
62682a34 SL |
38 | #![feature(rustc_private)] |
39 | #![feature(set_stdio)] | |
62682a34 | 40 | #![feature(staged_api)] |
a7813a04 | 41 | #![feature(panic_unwind)] |
1a4d82fc JJ |
42 | |
43 | extern crate getopts; | |
1a4d82fc | 44 | extern crate term; |
c34b1796 | 45 | extern crate libc; |
a7813a04 | 46 | extern crate panic_unwind; |
1a4d82fc JJ |
47 | |
48 | pub use self::TestFn::*; | |
1a4d82fc JJ |
49 | pub use self::ColorConfig::*; |
50 | pub use self::TestResult::*; | |
51 | pub use self::TestName::*; | |
52 | use self::TestEvent::*; | |
53 | use self::NamePadding::*; | |
54 | use self::OutputLocation::*; | |
55 | ||
c30ab7b3 | 56 | use std::panic::{catch_unwind, AssertUnwindSafe}; |
1a4d82fc JJ |
57 | use std::any::Any; |
58 | use std::cmp; | |
59 | use std::collections::BTreeMap; | |
c34b1796 | 60 | use std::env; |
1a4d82fc | 61 | use std::fmt; |
c34b1796 AL |
62 | use std::fs::File; |
63 | use std::io::prelude::*; | |
64 | use std::io; | |
1a4d82fc | 65 | use std::iter::repeat; |
c34b1796 | 66 | use std::path::PathBuf; |
1a4d82fc | 67 | use std::sync::mpsc::{channel, Sender}; |
c34b1796 | 68 | use std::sync::{Arc, Mutex}; |
85aaf69f | 69 | use std::thread; |
92a42be0 | 70 | use std::time::{Instant, Duration}; |
1a4d82fc | 71 | |
5bcae85e SL |
72 | const TEST_WARN_TIMEOUT_S: u64 = 60; |
73 | ||
1a4d82fc JJ |
74 | // to be used by rustc to compile tests in libtest |
75 | pub mod test { | |
9cc50fc6 | 76 | pub use {Bencher, TestName, TestResult, TestDesc, TestDescAndFn, TestOpts, TrFailed, |
476ff2be SL |
77 | TrFailedMsg, TrIgnored, TrOk, Metric, MetricMap, StaticTestFn, StaticTestName, |
78 | DynTestName, DynTestFn, run_test, test_main, test_main_static, filter_tests, | |
7cac9316 | 79 | parse_opts, StaticBenchFn, ShouldPanic, Options}; |
1a4d82fc JJ |
80 | } |
81 | ||
82 | pub mod stats; | |
83 | ||
84 | // The name of a test. By convention this follows the rules for rust | |
85 | // paths; i.e. it should be a series of identifiers separated by double | |
86 | // colons. This way if some test runner wants to arrange the tests | |
87 | // hierarchically it may. | |
88 | ||
85aaf69f | 89 | #[derive(Clone, PartialEq, Eq, Hash, Debug)] |
1a4d82fc JJ |
90 | pub enum TestName { |
91 | StaticTestName(&'static str), | |
9cc50fc6 | 92 | DynTestName(String), |
1a4d82fc JJ |
93 | } |
94 | impl TestName { | |
e9174d1e | 95 | fn as_slice(&self) -> &str { |
1a4d82fc JJ |
96 | match *self { |
97 | StaticTestName(s) => s, | |
9cc50fc6 | 98 | DynTestName(ref s) => s, |
1a4d82fc JJ |
99 | } |
100 | } | |
101 | } | |
85aaf69f | 102 | impl fmt::Display for TestName { |
1a4d82fc | 103 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
85aaf69f | 104 | fmt::Display::fmt(self.as_slice(), f) |
1a4d82fc JJ |
105 | } |
106 | } | |
107 | ||
54a0048b | 108 | #[derive(Clone, Copy, PartialEq, Eq)] |
8bb4bdeb | 109 | pub enum NamePadding { |
1a4d82fc | 110 | PadNone, |
1a4d82fc JJ |
111 | PadOnRight, |
112 | } | |
113 | ||
114 | impl TestDesc { | |
c34b1796 | 115 | fn padded_name(&self, column_count: usize, align: NamePadding) -> String { |
62682a34 | 116 | let mut name = String::from(self.name.as_slice()); |
1a4d82fc | 117 | let fill = column_count.saturating_sub(name.len()); |
d9579d0f | 118 | let pad = repeat(" ").take(fill).collect::<String>(); |
1a4d82fc JJ |
119 | match align { |
120 | PadNone => name, | |
1a4d82fc | 121 | PadOnRight => { |
85aaf69f | 122 | name.push_str(&pad); |
1a4d82fc JJ |
123 | name |
124 | } | |
125 | } | |
126 | } | |
127 | } | |
128 | ||
129 | /// Represents a benchmark function. | |
d9579d0f | 130 | pub trait TDynBenchFn: Send { |
1a4d82fc JJ |
131 | fn run(&self, harness: &mut Bencher); |
132 | } | |
133 | ||
c30ab7b3 SL |
134 | pub trait FnBox<T>: Send + 'static { |
135 | fn call_box(self: Box<Self>, t: T); | |
136 | } | |
137 | ||
138 | impl<T, F: FnOnce(T) + Send + 'static> FnBox<T> for F { | |
139 | fn call_box(self: Box<F>, t: T) { | |
140 | (*self)(t) | |
141 | } | |
142 | } | |
143 | ||
1a4d82fc JJ |
144 | // A function that runs a test. If the function returns successfully, |
145 | // the test succeeds; if the function panics then the test fails. We | |
146 | // may need to come up with a more clever definition of test in order | |
bd371182 | 147 | // to support isolation of tests into threads. |
1a4d82fc JJ |
148 | pub enum TestFn { |
149 | StaticTestFn(fn()), | |
150 | StaticBenchFn(fn(&mut Bencher)), | |
151 | StaticMetricFn(fn(&mut MetricMap)), | |
c30ab7b3 SL |
152 | DynTestFn(Box<FnBox<()>>), |
153 | DynMetricFn(Box<for<'a> FnBox<&'a mut MetricMap>>), | |
9cc50fc6 | 154 | DynBenchFn(Box<TDynBenchFn + 'static>), |
1a4d82fc JJ |
155 | } |
156 | ||
157 | impl TestFn { | |
158 | fn padding(&self) -> NamePadding { | |
e9174d1e | 159 | match *self { |
9cc50fc6 SL |
160 | StaticTestFn(..) => PadNone, |
161 | StaticBenchFn(..) => PadOnRight, | |
e9174d1e | 162 | StaticMetricFn(..) => PadOnRight, |
9cc50fc6 SL |
163 | DynTestFn(..) => PadNone, |
164 | DynMetricFn(..) => PadOnRight, | |
165 | DynBenchFn(..) => PadOnRight, | |
1a4d82fc JJ |
166 | } |
167 | } | |
168 | } | |
169 | ||
85aaf69f | 170 | impl fmt::Debug for TestFn { |
1a4d82fc JJ |
171 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
172 | f.write_str(match *self { | |
173 | StaticTestFn(..) => "StaticTestFn(..)", | |
174 | StaticBenchFn(..) => "StaticBenchFn(..)", | |
175 | StaticMetricFn(..) => "StaticMetricFn(..)", | |
176 | DynTestFn(..) => "DynTestFn(..)", | |
177 | DynMetricFn(..) => "DynMetricFn(..)", | |
9cc50fc6 | 178 | DynBenchFn(..) => "DynBenchFn(..)", |
1a4d82fc JJ |
179 | }) |
180 | } | |
181 | } | |
182 | ||
183 | /// Manager of the benchmarking runs. | |
184 | /// | |
85aaf69f | 185 | /// This is fed into functions marked with `#[bench]` to allow for |
1a4d82fc JJ |
186 | /// set-up & tear-down before running a piece of code repeatedly via a |
187 | /// call to `iter`. | |
32a655c1 | 188 | #[derive(Clone)] |
1a4d82fc | 189 | pub struct Bencher { |
32a655c1 SL |
190 | mode: BenchMode, |
191 | summary: Option<stats::Summary>, | |
1a4d82fc JJ |
192 | pub bytes: u64, |
193 | } | |
194 | ||
32a655c1 SL |
195 | #[derive(Clone, PartialEq, Eq)] |
196 | pub enum BenchMode { | |
197 | Auto, | |
198 | Single, | |
199 | } | |
200 | ||
85aaf69f | 201 | #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] |
c34b1796 | 202 | pub enum ShouldPanic { |
1a4d82fc | 203 | No, |
e9174d1e | 204 | Yes, |
9cc50fc6 | 205 | YesWithMessage(&'static str), |
1a4d82fc JJ |
206 | } |
207 | ||
208 | // The definition of a single test. A test runner will run a list of | |
209 | // these. | |
85aaf69f | 210 | #[derive(Clone, Debug, PartialEq, Eq, Hash)] |
1a4d82fc JJ |
211 | pub struct TestDesc { |
212 | pub name: TestName, | |
213 | pub ignore: bool, | |
c34b1796 | 214 | pub should_panic: ShouldPanic, |
1a4d82fc JJ |
215 | } |
216 | ||
7453a54e SL |
217 | #[derive(Clone)] |
218 | pub struct TestPaths { | |
219 | pub file: PathBuf, // e.g., compile-test/foo/bar/baz.rs | |
220 | pub base: PathBuf, // e.g., compile-test, auxiliary | |
221 | pub relative_dir: PathBuf, // e.g., foo/bar | |
222 | } | |
85aaf69f SL |
223 | |
224 | #[derive(Debug)] | |
1a4d82fc JJ |
225 | pub struct TestDescAndFn { |
226 | pub desc: TestDesc, | |
227 | pub testfn: TestFn, | |
228 | } | |
229 | ||
54a0048b | 230 | #[derive(Clone, PartialEq, Debug, Copy)] |
1a4d82fc JJ |
231 | pub struct Metric { |
232 | value: f64, | |
9cc50fc6 | 233 | noise: f64, |
1a4d82fc JJ |
234 | } |
235 | ||
236 | impl Metric { | |
237 | pub fn new(value: f64, noise: f64) -> Metric { | |
9cc50fc6 SL |
238 | Metric { |
239 | value: value, | |
240 | noise: noise, | |
241 | } | |
1a4d82fc JJ |
242 | } |
243 | } | |
244 | ||
245 | #[derive(PartialEq)] | |
9cc50fc6 | 246 | pub struct MetricMap(BTreeMap<String, Metric>); |
1a4d82fc JJ |
247 | |
248 | impl Clone for MetricMap { | |
249 | fn clone(&self) -> MetricMap { | |
250 | let MetricMap(ref map) = *self; | |
251 | MetricMap(map.clone()) | |
252 | } | |
253 | } | |
254 | ||
7cac9316 XL |
255 | /// In case we want to add other options as well, just add them in this struct. |
256 | #[derive(Copy, Clone, Debug)] | |
257 | pub struct Options { | |
258 | display_output: bool, | |
259 | } | |
260 | ||
261 | impl Options { | |
262 | pub fn new() -> Options { | |
263 | Options { | |
264 | display_output: false, | |
265 | } | |
266 | } | |
267 | ||
268 | pub fn display_output(mut self, display_output: bool) -> Options { | |
269 | self.display_output = display_output; | |
270 | self | |
271 | } | |
272 | } | |
273 | ||
1a4d82fc JJ |
274 | // The default console test runner. It accepts the command line |
275 | // arguments and a vector of test_descs. | |
7cac9316 XL |
276 | pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) { |
277 | let mut opts = match parse_opts(args) { | |
9cc50fc6 SL |
278 | Some(Ok(o)) => o, |
279 | Some(Err(msg)) => panic!("{:?}", msg), | |
280 | None => return, | |
281 | }; | |
7cac9316 | 282 | opts.options = options; |
476ff2be SL |
283 | if opts.list { |
284 | if let Err(e) = list_tests_console(&opts, tests) { | |
285 | panic!("io error when listing tests: {:?}", e); | |
286 | } | |
287 | } else { | |
288 | match run_tests_console(&opts, tests) { | |
289 | Ok(true) => {} | |
290 | Ok(false) => std::process::exit(101), | |
291 | Err(e) => panic!("io error when running tests: {:?}", e), | |
292 | } | |
1a4d82fc JJ |
293 | } |
294 | } | |
295 | ||
296 | // A variant optimized for invocation with a static test vector. | |
297 | // This will panic (intentionally) when fed any dynamic tests, because | |
298 | // it is copying the static values out into a dynamic vector and cannot | |
299 | // copy dynamic values. It is doing this because from this point on | |
d9579d0f AL |
300 | // a Vec<TestDescAndFn> is used in order to effect ownership-transfer |
301 | // semantics into parallel test runners, which in turn requires a Vec<> | |
1a4d82fc | 302 | // rather than a &[]. |
e9174d1e SL |
303 | pub fn test_main_static(tests: &[TestDescAndFn]) { |
304 | let args = env::args().collect::<Vec<_>>(); | |
9cc50fc6 SL |
305 | let owned_tests = tests.iter() |
306 | .map(|t| { | |
307 | match t.testfn { | |
308 | StaticTestFn(f) => { | |
309 | TestDescAndFn { | |
310 | testfn: StaticTestFn(f), | |
311 | desc: t.desc.clone(), | |
312 | } | |
313 | } | |
314 | StaticBenchFn(f) => { | |
315 | TestDescAndFn { | |
316 | testfn: StaticBenchFn(f), | |
317 | desc: t.desc.clone(), | |
318 | } | |
319 | } | |
320 | _ => panic!("non-static tests passed to test::test_main_static"), | |
321 | } | |
322 | }) | |
323 | .collect(); | |
7cac9316 | 324 | test_main(&args, owned_tests, Options::new()) |
1a4d82fc JJ |
325 | } |
326 | ||
7cac9316 | 327 | #[derive(Copy, Clone, Debug)] |
1a4d82fc JJ |
328 | pub enum ColorConfig { |
329 | AutoColor, | |
330 | AlwaysColor, | |
331 | NeverColor, | |
332 | } | |
333 | ||
7cac9316 | 334 | #[derive(Debug)] |
1a4d82fc | 335 | pub struct TestOpts { |
476ff2be | 336 | pub list: bool, |
85aaf69f | 337 | pub filter: Option<String>, |
476ff2be | 338 | pub filter_exact: bool, |
1a4d82fc JJ |
339 | pub run_ignored: bool, |
340 | pub run_tests: bool, | |
d9579d0f | 341 | pub bench_benchmarks: bool, |
c34b1796 | 342 | pub logfile: Option<PathBuf>, |
1a4d82fc JJ |
343 | pub nocapture: bool, |
344 | pub color: ColorConfig, | |
54a0048b | 345 | pub quiet: bool, |
5bcae85e | 346 | pub test_threads: Option<usize>, |
c30ab7b3 | 347 | pub skip: Vec<String>, |
7cac9316 | 348 | pub options: Options, |
1a4d82fc JJ |
349 | } |
350 | ||
351 | impl TestOpts { | |
352 | #[cfg(test)] | |
353 | fn new() -> TestOpts { | |
354 | TestOpts { | |
476ff2be | 355 | list: false, |
1a4d82fc | 356 | filter: None, |
476ff2be | 357 | filter_exact: false, |
1a4d82fc JJ |
358 | run_ignored: false, |
359 | run_tests: false, | |
d9579d0f | 360 | bench_benchmarks: false, |
1a4d82fc JJ |
361 | logfile: None, |
362 | nocapture: false, | |
363 | color: AutoColor, | |
54a0048b | 364 | quiet: false, |
5bcae85e | 365 | test_threads: None, |
c30ab7b3 | 366 | skip: vec![], |
7cac9316 | 367 | options: Options::new(), |
1a4d82fc JJ |
368 | } |
369 | } | |
370 | } | |
371 | ||
372 | /// Result of parsing the options. | |
373 | pub type OptRes = Result<TestOpts, String>; | |
374 | ||
9cc50fc6 | 375 | #[cfg_attr(rustfmt, rustfmt_skip)] |
1a4d82fc | 376 | fn optgroups() -> Vec<getopts::OptGroup> { |
c30ab7b3 | 377 | vec![getopts::optflag("", "ignored", "Run ignored tests"), |
1a4d82fc JJ |
378 | getopts::optflag("", "test", "Run tests and not benchmarks"), |
379 | getopts::optflag("", "bench", "Run benchmarks instead of tests"), | |
476ff2be | 380 | getopts::optflag("", "list", "List all tests and benchmarks"), |
1a4d82fc | 381 | getopts::optflag("h", "help", "Display this message (longer with --help)"), |
1a4d82fc JJ |
382 | getopts::optopt("", "logfile", "Write logs to the specified file instead \ |
383 | of stdout", "PATH"), | |
1a4d82fc JJ |
384 | getopts::optflag("", "nocapture", "don't capture stdout/stderr of each \ |
385 | task, allow printing directly"), | |
5bcae85e SL |
386 | getopts::optopt("", "test-threads", "Number of threads used for running tests \ |
387 | in parallel", "n_threads"), | |
c30ab7b3 SL |
388 | getopts::optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \ |
389 | be used multiple times)","FILTER"), | |
54a0048b | 390 | getopts::optflag("q", "quiet", "Display one character per test instead of one line"), |
476ff2be | 391 | getopts::optflag("", "exact", "Exactly match filters rather than by substring"), |
1a4d82fc JJ |
392 | getopts::optopt("", "color", "Configure coloring of output: |
393 | auto = colorize if stdout is a tty and tests are run on serially (default); | |
394 | always = always colorize output; | |
c30ab7b3 | 395 | never = never colorize output;", "auto|always|never")] |
1a4d82fc JJ |
396 | } |
397 | ||
398 | fn usage(binary: &str) { | |
399 | let message = format!("Usage: {} [OPTIONS] [FILTER]", binary); | |
400 | println!(r#"{usage} | |
401 | ||
9cc50fc6 SL |
402 | The FILTER string is tested against the name of all tests, and only those |
403 | tests whose names contain the filter are run. | |
1a4d82fc JJ |
404 | |
405 | By default, all tests are run in parallel. This can be altered with the | |
5bcae85e SL |
406 | --test-threads flag or the RUST_TEST_THREADS environment variable when running |
407 | tests (set it to 1). | |
1a4d82fc JJ |
408 | |
409 | All tests have their standard output and standard error captured by default. | |
54a0048b SL |
410 | This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE |
411 | environment variable to a value other than "0". Logging is not captured by default. | |
1a4d82fc JJ |
412 | |
413 | Test Attributes: | |
414 | ||
415 | #[test] - Indicates a function is a test to be run. This function | |
416 | takes no arguments. | |
417 | #[bench] - Indicates a function is a benchmark to be run. This | |
418 | function takes one argument (test::Bencher). | |
c34b1796 AL |
419 | #[should_panic] - This function (also labeled with #[test]) will only pass if |
420 | the code causes a panic (an assertion failure or panic!) | |
1a4d82fc | 421 | A message may be provided, which the failure string must |
c34b1796 | 422 | contain: #[should_panic(expected = "foo")]. |
1a4d82fc JJ |
423 | #[ignore] - When applied to a function which is already attributed as a |
424 | test, then the test runner will ignore these tests during | |
425 | normal test runs. Running with --ignored will run these | |
426 | tests."#, | |
85aaf69f | 427 | usage = getopts::usage(&message, &optgroups())); |
1a4d82fc JJ |
428 | } |
429 | ||
430 | // Parses command line arguments into test options | |
431 | pub fn parse_opts(args: &[String]) -> Option<OptRes> { | |
c1a9b12d | 432 | let args_ = &args[1..]; |
9cc50fc6 SL |
433 | let matches = match getopts::getopts(args_, &optgroups()) { |
434 | Ok(m) => m, | |
435 | Err(f) => return Some(Err(f.to_string())), | |
436 | }; | |
1a4d82fc | 437 | |
9cc50fc6 SL |
438 | if matches.opt_present("h") { |
439 | usage(&args[0]); | |
440 | return None; | |
441 | } | |
1a4d82fc | 442 | |
9346a6ac | 443 | let filter = if !matches.free.is_empty() { |
85aaf69f | 444 | Some(matches.free[0].clone()) |
1a4d82fc JJ |
445 | } else { |
446 | None | |
447 | }; | |
448 | ||
449 | let run_ignored = matches.opt_present("ignored"); | |
54a0048b | 450 | let quiet = matches.opt_present("quiet"); |
476ff2be SL |
451 | let exact = matches.opt_present("exact"); |
452 | let list = matches.opt_present("list"); | |
1a4d82fc JJ |
453 | |
454 | let logfile = matches.opt_str("logfile"); | |
c34b1796 | 455 | let logfile = logfile.map(|s| PathBuf::from(&s)); |
1a4d82fc | 456 | |
d9579d0f | 457 | let bench_benchmarks = matches.opt_present("bench"); |
9cc50fc6 | 458 | let run_tests = !bench_benchmarks || matches.opt_present("test"); |
1a4d82fc | 459 | |
1a4d82fc JJ |
460 | let mut nocapture = matches.opt_present("nocapture"); |
461 | if !nocapture { | |
54a0048b SL |
462 | nocapture = match env::var("RUST_TEST_NOCAPTURE") { |
463 | Ok(val) => &val != "0", | |
464 | Err(_) => false | |
465 | }; | |
1a4d82fc JJ |
466 | } |
467 | ||
5bcae85e SL |
468 | let test_threads = match matches.opt_str("test-threads") { |
469 | Some(n_str) => | |
470 | match n_str.parse::<usize>() { | |
8bb4bdeb XL |
471 | Ok(0) => |
472 | return Some(Err(format!("argument for --test-threads must not be 0"))), | |
5bcae85e SL |
473 | Ok(n) => Some(n), |
474 | Err(e) => | |
475 | return Some(Err(format!("argument for --test-threads must be a number > 0 \ | |
476 | (error: {})", e))) | |
477 | }, | |
478 | None => | |
479 | None, | |
480 | }; | |
481 | ||
85aaf69f | 482 | let color = match matches.opt_str("color").as_ref().map(|s| &**s) { |
1a4d82fc JJ |
483 | Some("auto") | None => AutoColor, |
484 | Some("always") => AlwaysColor, | |
485 | Some("never") => NeverColor, | |
486 | ||
9cc50fc6 SL |
487 | Some(v) => { |
488 | return Some(Err(format!("argument for --color must be auto, always, or never (was \ | |
489 | {})", | |
490 | v))) | |
491 | } | |
1a4d82fc JJ |
492 | }; |
493 | ||
1a4d82fc | 494 | let test_opts = TestOpts { |
476ff2be | 495 | list: list, |
1a4d82fc | 496 | filter: filter, |
476ff2be | 497 | filter_exact: exact, |
1a4d82fc JJ |
498 | run_ignored: run_ignored, |
499 | run_tests: run_tests, | |
d9579d0f | 500 | bench_benchmarks: bench_benchmarks, |
1a4d82fc JJ |
501 | logfile: logfile, |
502 | nocapture: nocapture, | |
503 | color: color, | |
54a0048b | 504 | quiet: quiet, |
5bcae85e | 505 | test_threads: test_threads, |
c30ab7b3 | 506 | skip: matches.opt_strs("skip"), |
7cac9316 | 507 | options: Options::new(), |
1a4d82fc JJ |
508 | }; |
509 | ||
510 | Some(Ok(test_opts)) | |
511 | } | |
512 | ||
1a4d82fc JJ |
513 | #[derive(Clone, PartialEq)] |
514 | pub struct BenchSamples { | |
9346a6ac | 515 | ns_iter_summ: stats::Summary, |
c34b1796 | 516 | mb_s: usize, |
1a4d82fc JJ |
517 | } |
518 | ||
519 | #[derive(Clone, PartialEq)] | |
520 | pub enum TestResult { | |
521 | TrOk, | |
522 | TrFailed, | |
476ff2be | 523 | TrFailedMsg(String), |
1a4d82fc JJ |
524 | TrIgnored, |
525 | TrMetrics(MetricMap), | |
526 | TrBench(BenchSamples), | |
527 | } | |
528 | ||
85aaf69f SL |
529 | unsafe impl Send for TestResult {} |
530 | ||
1a4d82fc | 531 | enum OutputLocation<T> { |
92a42be0 | 532 | Pretty(Box<term::StdoutTerminal>), |
1a4d82fc JJ |
533 | Raw(T), |
534 | } | |
535 | ||
536 | struct ConsoleTestState<T> { | |
537 | log_out: Option<File>, | |
538 | out: OutputLocation<T>, | |
539 | use_color: bool, | |
54a0048b | 540 | quiet: bool, |
c34b1796 AL |
541 | total: usize, |
542 | passed: usize, | |
543 | failed: usize, | |
544 | ignored: usize, | |
7cac9316 | 545 | filtered_out: usize, |
c34b1796 | 546 | measured: usize, |
1a4d82fc | 547 | metrics: MetricMap, |
9cc50fc6 | 548 | failures: Vec<(TestDesc, Vec<u8>)>, |
7cac9316 | 549 | not_failures: Vec<(TestDesc, Vec<u8>)>, |
c34b1796 | 550 | max_name_len: usize, // number of columns to fill when aligning names |
7cac9316 | 551 | options: Options, |
1a4d82fc JJ |
552 | } |
553 | ||
c34b1796 | 554 | impl<T: Write> ConsoleTestState<T> { |
9cc50fc6 | 555 | pub fn new(opts: &TestOpts, _: Option<T>) -> io::Result<ConsoleTestState<io::Stdout>> { |
1a4d82fc | 556 | let log_out = match opts.logfile { |
54a0048b | 557 | Some(ref path) => Some(File::create(path)?), |
9cc50fc6 | 558 | None => None, |
1a4d82fc JJ |
559 | }; |
560 | let out = match term::stdout() { | |
c34b1796 | 561 | None => Raw(io::stdout()), |
9cc50fc6 | 562 | Some(t) => Pretty(t), |
1a4d82fc JJ |
563 | }; |
564 | ||
565 | Ok(ConsoleTestState { | |
566 | out: out, | |
567 | log_out: log_out, | |
568 | use_color: use_color(opts), | |
54a0048b | 569 | quiet: opts.quiet, |
85aaf69f SL |
570 | total: 0, |
571 | passed: 0, | |
572 | failed: 0, | |
573 | ignored: 0, | |
7cac9316 | 574 | filtered_out: 0, |
85aaf69f | 575 | measured: 0, |
1a4d82fc JJ |
576 | metrics: MetricMap::new(), |
577 | failures: Vec::new(), | |
7cac9316 | 578 | not_failures: Vec::new(), |
85aaf69f | 579 | max_name_len: 0, |
7cac9316 | 580 | options: opts.options, |
1a4d82fc JJ |
581 | }) |
582 | } | |
583 | ||
c34b1796 | 584 | pub fn write_ok(&mut self) -> io::Result<()> { |
54a0048b | 585 | self.write_short_result("ok", ".", term::color::GREEN) |
1a4d82fc JJ |
586 | } |
587 | ||
c34b1796 | 588 | pub fn write_failed(&mut self) -> io::Result<()> { |
54a0048b | 589 | self.write_short_result("FAILED", "F", term::color::RED) |
1a4d82fc JJ |
590 | } |
591 | ||
c34b1796 | 592 | pub fn write_ignored(&mut self) -> io::Result<()> { |
54a0048b | 593 | self.write_short_result("ignored", "i", term::color::YELLOW) |
1a4d82fc JJ |
594 | } |
595 | ||
c34b1796 | 596 | pub fn write_metric(&mut self) -> io::Result<()> { |
1a4d82fc JJ |
597 | self.write_pretty("metric", term::color::CYAN) |
598 | } | |
599 | ||
c34b1796 | 600 | pub fn write_bench(&mut self) -> io::Result<()> { |
1a4d82fc JJ |
601 | self.write_pretty("bench", term::color::CYAN) |
602 | } | |
603 | ||
54a0048b SL |
604 | pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color) |
605 | -> io::Result<()> { | |
606 | if self.quiet { | |
607 | self.write_pretty(quiet, color) | |
608 | } else { | |
609 | self.write_pretty(verbose, color)?; | |
610 | self.write_plain("\n") | |
611 | } | |
612 | } | |
613 | ||
9cc50fc6 | 614 | pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { |
1a4d82fc JJ |
615 | match self.out { |
616 | Pretty(ref mut term) => { | |
617 | if self.use_color { | |
54a0048b | 618 | term.fg(color)?; |
1a4d82fc | 619 | } |
54a0048b | 620 | term.write_all(word.as_bytes())?; |
1a4d82fc | 621 | if self.use_color { |
54a0048b | 622 | term.reset()?; |
1a4d82fc | 623 | } |
c34b1796 AL |
624 | term.flush() |
625 | } | |
626 | Raw(ref mut stdout) => { | |
54a0048b | 627 | stdout.write_all(word.as_bytes())?; |
c34b1796 | 628 | stdout.flush() |
1a4d82fc | 629 | } |
1a4d82fc JJ |
630 | } |
631 | } | |
632 | ||
476ff2be SL |
633 | pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> { |
634 | let s = s.as_ref(); | |
1a4d82fc | 635 | match self.out { |
c34b1796 | 636 | Pretty(ref mut term) => { |
54a0048b | 637 | term.write_all(s.as_bytes())?; |
c34b1796 | 638 | term.flush() |
9cc50fc6 | 639 | } |
c34b1796 | 640 | Raw(ref mut stdout) => { |
54a0048b | 641 | stdout.write_all(s.as_bytes())?; |
c34b1796 | 642 | stdout.flush() |
9cc50fc6 | 643 | } |
1a4d82fc JJ |
644 | } |
645 | } | |
646 | ||
c34b1796 | 647 | pub fn write_run_start(&mut self, len: usize) -> io::Result<()> { |
1a4d82fc | 648 | self.total = len; |
9cc50fc6 SL |
649 | let noun = if len != 1 { |
650 | "tests" | |
651 | } else { | |
652 | "test" | |
653 | }; | |
85aaf69f | 654 | self.write_plain(&format!("\nrunning {} {}\n", len, noun)) |
1a4d82fc JJ |
655 | } |
656 | ||
9cc50fc6 | 657 | pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> { |
54a0048b SL |
658 | if self.quiet && align != PadOnRight { |
659 | Ok(()) | |
660 | } else { | |
661 | let name = test.padded_name(self.max_name_len, align); | |
662 | self.write_plain(&format!("test {} ... ", name)) | |
663 | } | |
1a4d82fc JJ |
664 | } |
665 | ||
c34b1796 | 666 | pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> { |
54a0048b | 667 | match *result { |
1a4d82fc | 668 | TrOk => self.write_ok(), |
476ff2be | 669 | TrFailed | TrFailedMsg(_) => self.write_failed(), |
1a4d82fc JJ |
670 | TrIgnored => self.write_ignored(), |
671 | TrMetrics(ref mm) => { | |
54a0048b SL |
672 | self.write_metric()?; |
673 | self.write_plain(&format!(": {}\n", mm.fmt_metrics())) | |
1a4d82fc JJ |
674 | } |
675 | TrBench(ref bs) => { | |
54a0048b SL |
676 | self.write_bench()?; |
677 | self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) | |
1a4d82fc | 678 | } |
54a0048b | 679 | } |
1a4d82fc JJ |
680 | } |
681 | ||
5bcae85e SL |
682 | pub fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { |
683 | self.write_plain(&format!("test {} has been running for over {} seconds\n", | |
684 | desc.name, | |
685 | TEST_WARN_TIMEOUT_S)) | |
686 | } | |
687 | ||
476ff2be SL |
688 | pub fn write_log<S: AsRef<str>>(&mut self, msg: S) -> io::Result<()> { |
689 | let msg = msg.as_ref(); | |
1a4d82fc JJ |
690 | match self.log_out { |
691 | None => Ok(()), | |
476ff2be | 692 | Some(ref mut o) => o.write_all(msg.as_bytes()), |
1a4d82fc JJ |
693 | } |
694 | } | |
695 | ||
476ff2be SL |
696 | pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> { |
697 | self.write_log( | |
698 | format!("{} {}\n", | |
699 | match *result { | |
700 | TrOk => "ok".to_owned(), | |
701 | TrFailed => "failed".to_owned(), | |
702 | TrFailedMsg(ref msg) => format!("failed: {}", msg), | |
703 | TrIgnored => "ignored".to_owned(), | |
704 | TrMetrics(ref mm) => mm.fmt_metrics(), | |
705 | TrBench(ref bs) => fmt_bench_samples(bs), | |
706 | }, | |
707 | test.name)) | |
708 | } | |
709 | ||
c34b1796 | 710 | pub fn write_failures(&mut self) -> io::Result<()> { |
54a0048b | 711 | self.write_plain("\nfailures:\n")?; |
1a4d82fc JJ |
712 | let mut failures = Vec::new(); |
713 | let mut fail_out = String::new(); | |
85aaf69f | 714 | for &(ref f, ref stdout) in &self.failures { |
1a4d82fc | 715 | failures.push(f.name.to_string()); |
9346a6ac | 716 | if !stdout.is_empty() { |
85aaf69f SL |
717 | fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); |
718 | let output = String::from_utf8_lossy(stdout); | |
719 | fail_out.push_str(&output); | |
1a4d82fc JJ |
720 | fail_out.push_str("\n"); |
721 | } | |
722 | } | |
9346a6ac | 723 | if !fail_out.is_empty() { |
54a0048b SL |
724 | self.write_plain("\n")?; |
725 | self.write_plain(&fail_out)?; | |
1a4d82fc JJ |
726 | } |
727 | ||
54a0048b | 728 | self.write_plain("\nfailures:\n")?; |
1a4d82fc | 729 | failures.sort(); |
85aaf69f | 730 | for name in &failures { |
54a0048b | 731 | self.write_plain(&format!(" {}\n", name))?; |
1a4d82fc JJ |
732 | } |
733 | Ok(()) | |
734 | } | |
735 | ||
7cac9316 XL |
736 | pub fn write_outputs(&mut self) -> io::Result<()> { |
737 | self.write_plain("\nsuccesses:\n")?; | |
738 | let mut successes = Vec::new(); | |
739 | let mut stdouts = String::new(); | |
740 | for &(ref f, ref stdout) in &self.not_failures { | |
741 | successes.push(f.name.to_string()); | |
742 | if !stdout.is_empty() { | |
743 | stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); | |
744 | let output = String::from_utf8_lossy(stdout); | |
745 | stdouts.push_str(&output); | |
746 | stdouts.push_str("\n"); | |
747 | } | |
748 | } | |
749 | if !stdouts.is_empty() { | |
750 | self.write_plain("\n")?; | |
751 | self.write_plain(&stdouts)?; | |
752 | } | |
753 | ||
754 | self.write_plain("\nsuccesses:\n")?; | |
755 | successes.sort(); | |
756 | for name in &successes { | |
757 | self.write_plain(&format!(" {}\n", name))?; | |
758 | } | |
759 | Ok(()) | |
760 | } | |
761 | ||
c34b1796 | 762 | pub fn write_run_finish(&mut self) -> io::Result<bool> { |
1a4d82fc JJ |
763 | assert!(self.passed + self.failed + self.ignored + self.measured == self.total); |
764 | ||
7cac9316 XL |
765 | if self.options.display_output { |
766 | self.write_outputs()?; | |
767 | } | |
85aaf69f SL |
768 | let success = self.failed == 0; |
769 | if !success { | |
54a0048b | 770 | self.write_failures()?; |
1a4d82fc JJ |
771 | } |
772 | ||
54a0048b | 773 | self.write_plain("\ntest result: ")?; |
1a4d82fc JJ |
774 | if success { |
775 | // There's no parallelism at this point so it's safe to use color | |
54a0048b | 776 | self.write_pretty("ok", term::color::GREEN)?; |
1a4d82fc | 777 | } else { |
54a0048b | 778 | self.write_pretty("FAILED", term::color::RED)?; |
1a4d82fc | 779 | } |
7cac9316 | 780 | let s = format!(". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", |
9cc50fc6 SL |
781 | self.passed, |
782 | self.failed, | |
783 | self.ignored, | |
7cac9316 XL |
784 | self.measured, |
785 | self.filtered_out); | |
54a0048b | 786 | self.write_plain(&s)?; |
1a4d82fc JJ |
787 | return Ok(success); |
788 | } | |
789 | } | |
790 | ||
62682a34 SL |
791 | // Format a number with thousands separators |
792 | fn fmt_thousands_sep(mut n: usize, sep: char) -> String { | |
793 | use std::fmt::Write; | |
794 | let mut output = String::new(); | |
795 | let mut trailing = false; | |
796 | for &pow in &[9, 6, 3, 0] { | |
797 | let base = 10_usize.pow(pow); | |
798 | if pow == 0 || trailing || n / base != 0 { | |
799 | if !trailing { | |
800 | output.write_fmt(format_args!("{}", n / base)).unwrap(); | |
801 | } else { | |
802 | output.write_fmt(format_args!("{:03}", n / base)).unwrap(); | |
803 | } | |
804 | if pow != 0 { | |
805 | output.push(sep); | |
806 | } | |
807 | trailing = true; | |
808 | } | |
809 | n %= base; | |
810 | } | |
811 | ||
812 | output | |
813 | } | |
814 | ||
1a4d82fc | 815 | pub fn fmt_bench_samples(bs: &BenchSamples) -> String { |
62682a34 SL |
816 | use std::fmt::Write; |
817 | let mut output = String::new(); | |
818 | ||
819 | let median = bs.ns_iter_summ.median as usize; | |
820 | let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; | |
821 | ||
822 | output.write_fmt(format_args!("{:>11} ns/iter (+/- {})", | |
9cc50fc6 SL |
823 | fmt_thousands_sep(median, ','), |
824 | fmt_thousands_sep(deviation, ','))) | |
825 | .unwrap(); | |
1a4d82fc | 826 | if bs.mb_s != 0 { |
62682a34 | 827 | output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap(); |
1a4d82fc | 828 | } |
62682a34 | 829 | output |
1a4d82fc JJ |
830 | } |
831 | ||
476ff2be SL |
832 | // List the tests to console, and optionally to logfile. Filters are honored. |
833 | pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> { | |
834 | let mut st = ConsoleTestState::new(opts, None::<io::Stdout>)?; | |
835 | ||
836 | let mut ntest = 0; | |
837 | let mut nbench = 0; | |
838 | let mut nmetric = 0; | |
839 | ||
840 | for test in filter_tests(&opts, tests) { | |
841 | use TestFn::*; | |
842 | ||
843 | let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test; | |
844 | ||
845 | let fntype = match testfn { | |
846 | StaticTestFn(..) | DynTestFn(..) => { ntest += 1; "test" }, | |
847 | StaticBenchFn(..) | DynBenchFn(..) => { nbench += 1; "benchmark" }, | |
848 | StaticMetricFn(..) | DynMetricFn(..) => { nmetric += 1; "metric" }, | |
849 | }; | |
850 | ||
851 | st.write_plain(format!("{}: {}\n", name, fntype))?; | |
852 | st.write_log(format!("{} {}\n", fntype, name))?; | |
853 | } | |
854 | ||
855 | fn plural(count: u32, s: &str) -> String { | |
856 | match count { | |
857 | 1 => format!("{} {}", 1, s), | |
858 | n => format!("{} {}s", n, s), | |
859 | } | |
860 | } | |
861 | ||
862 | if !opts.quiet { | |
863 | if ntest != 0 || nbench != 0 || nmetric != 0 { | |
864 | st.write_plain("\n")?; | |
865 | } | |
866 | st.write_plain(format!("{}, {}, {}\n", | |
867 | plural(ntest, "test"), | |
868 | plural(nbench, "benchmark"), | |
869 | plural(nmetric, "metric")))?; | |
870 | } | |
871 | ||
872 | Ok(()) | |
873 | } | |
874 | ||
1a4d82fc | 875 | // A simple console test runner |
9cc50fc6 | 876 | pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> { |
1a4d82fc | 877 | |
9cc50fc6 | 878 | fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> { |
1a4d82fc JJ |
879 | match (*event).clone() { |
880 | TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()), | |
7cac9316 | 881 | TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), |
1a4d82fc | 882 | TeWait(ref test, padding) => st.write_test_start(test, padding), |
5bcae85e | 883 | TeTimeout(ref test) => st.write_timeout(test), |
1a4d82fc | 884 | TeResult(test, result, stdout) => { |
476ff2be | 885 | st.write_log_result(&test, &result)?; |
54a0048b | 886 | st.write_result(&result)?; |
1a4d82fc | 887 | match result { |
7cac9316 XL |
888 | TrOk => { |
889 | st.passed += 1; | |
890 | st.not_failures.push((test, stdout)); | |
891 | } | |
1a4d82fc JJ |
892 | TrIgnored => st.ignored += 1, |
893 | TrMetrics(mm) => { | |
85aaf69f | 894 | let tname = test.name; |
1a4d82fc | 895 | let MetricMap(mm) = mm; |
9cc50fc6 | 896 | for (k, v) in &mm { |
1a4d82fc | 897 | st.metrics |
9cc50fc6 | 898 | .insert_metric(&format!("{}.{}", tname, k), v.value, v.noise); |
1a4d82fc JJ |
899 | } |
900 | st.measured += 1 | |
901 | } | |
902 | TrBench(bs) => { | |
903 | st.metrics.insert_metric(test.name.as_slice(), | |
904 | bs.ns_iter_summ.median, | |
905 | bs.ns_iter_summ.max - bs.ns_iter_summ.min); | |
906 | st.measured += 1 | |
907 | } | |
908 | TrFailed => { | |
909 | st.failed += 1; | |
910 | st.failures.push((test, stdout)); | |
911 | } | |
476ff2be SL |
912 | TrFailedMsg(msg) => { |
913 | st.failed += 1; | |
914 | let mut stdout = stdout; | |
915 | stdout.extend_from_slice( | |
916 | format!("note: {}", msg).as_bytes() | |
917 | ); | |
918 | st.failures.push((test, stdout)); | |
919 | } | |
1a4d82fc JJ |
920 | } |
921 | Ok(()) | |
922 | } | |
923 | } | |
924 | } | |
925 | ||
54a0048b | 926 | let mut st = ConsoleTestState::new(opts, None::<io::Stdout>)?; |
c34b1796 | 927 | fn len_if_padded(t: &TestDescAndFn) -> usize { |
1a4d82fc | 928 | match t.testfn.padding() { |
85aaf69f | 929 | PadNone => 0, |
d9579d0f | 930 | PadOnRight => t.desc.name.as_slice().len(), |
1a4d82fc JJ |
931 | } |
932 | } | |
3157f602 XL |
933 | if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { |
934 | let n = t.desc.name.as_slice(); | |
935 | st.max_name_len = n.len(); | |
1a4d82fc | 936 | } |
54a0048b | 937 | run_tests(opts, tests, |x| callback(&x, &mut st))?; |
85aaf69f | 938 | return st.write_run_finish(); |
1a4d82fc JJ |
939 | } |
940 | ||
941 | #[test] | |
942 | fn should_sort_failures_before_printing_them() { | |
943 | let test_a = TestDesc { | |
944 | name: StaticTestName("a"), | |
945 | ignore: false, | |
9cc50fc6 | 946 | should_panic: ShouldPanic::No, |
1a4d82fc JJ |
947 | }; |
948 | ||
949 | let test_b = TestDesc { | |
950 | name: StaticTestName("b"), | |
951 | ignore: false, | |
9cc50fc6 | 952 | should_panic: ShouldPanic::No, |
1a4d82fc JJ |
953 | }; |
954 | ||
955 | let mut st = ConsoleTestState { | |
956 | log_out: None, | |
957 | out: Raw(Vec::new()), | |
958 | use_color: false, | |
54a0048b | 959 | quiet: false, |
85aaf69f SL |
960 | total: 0, |
961 | passed: 0, | |
962 | failed: 0, | |
963 | ignored: 0, | |
7cac9316 | 964 | filtered_out: 0, |
85aaf69f SL |
965 | measured: 0, |
966 | max_name_len: 10, | |
1a4d82fc | 967 | metrics: MetricMap::new(), |
9cc50fc6 | 968 | failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], |
7cac9316 XL |
969 | options: Options::new(), |
970 | not_failures: Vec::new(), | |
1a4d82fc JJ |
971 | }; |
972 | ||
973 | st.write_failures().unwrap(); | |
974 | let s = match st.out { | |
85aaf69f | 975 | Raw(ref m) => String::from_utf8_lossy(&m[..]), |
9cc50fc6 | 976 | Pretty(_) => unreachable!(), |
1a4d82fc JJ |
977 | }; |
978 | ||
c34b1796 AL |
979 | let apos = s.find("a").unwrap(); |
980 | let bpos = s.find("b").unwrap(); | |
1a4d82fc JJ |
981 | assert!(apos < bpos); |
982 | } | |
983 | ||
984 | fn use_color(opts: &TestOpts) -> bool { | |
985 | match opts.color { | |
62682a34 | 986 | AutoColor => !opts.nocapture && stdout_isatty(), |
1a4d82fc JJ |
987 | AlwaysColor => true, |
988 | NeverColor => false, | |
989 | } | |
990 | } | |
991 | ||
32a655c1 SL |
992 | #[cfg(target_os = "redox")] |
993 | fn stdout_isatty() -> bool { | |
994 | // FIXME: Implement isatty on Redox | |
995 | false | |
996 | } | |
c34b1796 AL |
997 | #[cfg(unix)] |
998 | fn stdout_isatty() -> bool { | |
999 | unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 } | |
1000 | } | |
1001 | #[cfg(windows)] | |
1002 | fn stdout_isatty() -> bool { | |
92a42be0 SL |
1003 | type DWORD = u32; |
1004 | type BOOL = i32; | |
1005 | type HANDLE = *mut u8; | |
1006 | type LPDWORD = *mut u32; | |
1007 | const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD; | |
c34b1796 | 1008 | extern "system" { |
92a42be0 SL |
1009 | fn GetStdHandle(which: DWORD) -> HANDLE; |
1010 | fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL; | |
c34b1796 AL |
1011 | } |
1012 | unsafe { | |
1013 | let handle = GetStdHandle(STD_OUTPUT_HANDLE); | |
1014 | let mut out = 0; | |
1015 | GetConsoleMode(handle, &mut out) != 0 | |
1016 | } | |
1017 | } | |
1018 | ||
1a4d82fc | 1019 | #[derive(Clone)] |
8bb4bdeb | 1020 | pub enum TestEvent { |
9cc50fc6 | 1021 | TeFiltered(Vec<TestDesc>), |
1a4d82fc | 1022 | TeWait(TestDesc, NamePadding), |
9cc50fc6 | 1023 | TeResult(TestDesc, TestResult, Vec<u8>), |
5bcae85e | 1024 | TeTimeout(TestDesc), |
7cac9316 | 1025 | TeFilteredOut(usize), |
1a4d82fc JJ |
1026 | } |
1027 | ||
9cc50fc6 | 1028 | pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>); |
1a4d82fc | 1029 | |
1a4d82fc | 1030 | |
8bb4bdeb | 1031 | pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()> |
9cc50fc6 | 1032 | where F: FnMut(TestEvent) -> io::Result<()> |
1a4d82fc | 1033 | { |
5bcae85e SL |
1034 | use std::collections::HashMap; |
1035 | use std::sync::mpsc::RecvTimeoutError; | |
1036 | ||
7cac9316 XL |
1037 | let tests_len = tests.len(); |
1038 | ||
d9579d0f AL |
1039 | let mut filtered_tests = filter_tests(opts, tests); |
1040 | if !opts.bench_benchmarks { | |
1041 | filtered_tests = convert_benchmarks_to_tests(filtered_tests); | |
1042 | } | |
1043 | ||
7cac9316 XL |
1044 | let filtered_out = tests_len - filtered_tests.len(); |
1045 | callback(TeFilteredOut(filtered_out))?; | |
1046 | ||
1a4d82fc JJ |
1047 | let filtered_descs = filtered_tests.iter() |
1048 | .map(|t| t.desc.clone()) | |
1049 | .collect(); | |
1050 | ||
54a0048b | 1051 | callback(TeFiltered(filtered_descs))?; |
1a4d82fc JJ |
1052 | |
1053 | let (filtered_tests, filtered_benchs_and_metrics): (Vec<_>, _) = | |
1054 | filtered_tests.into_iter().partition(|e| { | |
1055 | match e.testfn { | |
1056 | StaticTestFn(_) | DynTestFn(_) => true, | |
9cc50fc6 | 1057 | _ => false, |
1a4d82fc JJ |
1058 | } |
1059 | }); | |
1060 | ||
5bcae85e SL |
1061 | let concurrency = match opts.test_threads { |
1062 | Some(n) => n, | |
1063 | None => get_concurrency(), | |
1064 | }; | |
1a4d82fc JJ |
1065 | |
1066 | let mut remaining = filtered_tests; | |
1067 | remaining.reverse(); | |
1068 | let mut pending = 0; | |
1069 | ||
1070 | let (tx, rx) = channel::<MonitorMsg>(); | |
1071 | ||
5bcae85e SL |
1072 | let mut running_tests: HashMap<TestDesc, Instant> = HashMap::new(); |
1073 | ||
1074 | fn get_timed_out_tests(running_tests: &mut HashMap<TestDesc, Instant>) -> Vec<TestDesc> { | |
1075 | let now = Instant::now(); | |
1076 | let timed_out = running_tests.iter() | |
1077 | .filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone())} else { None }) | |
1078 | .collect(); | |
1079 | for test in &timed_out { | |
1080 | running_tests.remove(test); | |
1081 | } | |
1082 | timed_out | |
1083 | }; | |
1084 | ||
1085 | fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration> { | |
1086 | running_tests.values().min().map(|next_timeout| { | |
1087 | let now = Instant::now(); | |
1088 | if *next_timeout >= now { | |
1089 | *next_timeout - now | |
1090 | } else { | |
1091 | Duration::new(0, 0) | |
1092 | }}) | |
1093 | }; | |
1094 | ||
1a4d82fc JJ |
1095 | while pending > 0 || !remaining.is_empty() { |
1096 | while pending < concurrency && !remaining.is_empty() { | |
1097 | let test = remaining.pop().unwrap(); | |
1098 | if concurrency == 1 { | |
1099 | // We are doing one test at a time so we can print the name | |
1100 | // of the test before we run it. Useful for debugging tests | |
1101 | // that hang forever. | |
54a0048b | 1102 | callback(TeWait(test.desc.clone(), test.testfn.padding()))?; |
1a4d82fc | 1103 | } |
5bcae85e SL |
1104 | let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); |
1105 | running_tests.insert(test.desc.clone(), timeout); | |
1a4d82fc JJ |
1106 | run_test(opts, !opts.run_tests, test, tx.clone()); |
1107 | pending += 1; | |
1108 | } | |
1109 | ||
5bcae85e SL |
1110 | let mut res; |
1111 | loop { | |
1112 | if let Some(timeout) = calc_timeout(&running_tests) { | |
1113 | res = rx.recv_timeout(timeout); | |
1114 | for test in get_timed_out_tests(&mut running_tests) { | |
1115 | callback(TeTimeout(test))?; | |
1116 | } | |
1117 | if res != Err(RecvTimeoutError::Timeout) { | |
1118 | break; | |
1119 | } | |
1120 | } else { | |
1121 | res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected); | |
1122 | break; | |
1123 | } | |
1124 | } | |
1125 | ||
1126 | let (desc, result, stdout) = res.unwrap(); | |
1127 | running_tests.remove(&desc); | |
1128 | ||
1a4d82fc | 1129 | if concurrency != 1 { |
54a0048b | 1130 | callback(TeWait(desc.clone(), PadNone))?; |
1a4d82fc | 1131 | } |
54a0048b | 1132 | callback(TeResult(desc, result, stdout))?; |
1a4d82fc JJ |
1133 | pending -= 1; |
1134 | } | |
1135 | ||
d9579d0f AL |
1136 | if opts.bench_benchmarks { |
1137 | // All benchmarks run at the end, in serial. | |
1138 | // (this includes metric fns) | |
1139 | for b in filtered_benchs_and_metrics { | |
54a0048b | 1140 | callback(TeWait(b.desc.clone(), b.testfn.padding()))?; |
d9579d0f AL |
1141 | run_test(opts, false, b, tx.clone()); |
1142 | let (test, result, stdout) = rx.recv().unwrap(); | |
54a0048b | 1143 | callback(TeResult(test, result, stdout))?; |
d9579d0f | 1144 | } |
1a4d82fc JJ |
1145 | } |
1146 | Ok(()) | |
1147 | } | |
1148 | ||
c34b1796 AL |
1149 | #[allow(deprecated)] |
1150 | fn get_concurrency() -> usize { | |
c1a9b12d | 1151 | return match env::var("RUST_TEST_THREADS") { |
85aaf69f | 1152 | Ok(s) => { |
c34b1796 | 1153 | let opt_n: Option<usize> = s.parse().ok(); |
1a4d82fc JJ |
1154 | match opt_n { |
1155 | Some(n) if n > 0 => n, | |
9cc50fc6 SL |
1156 | _ => { |
1157 | panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.", | |
1158 | s) | |
1159 | } | |
1a4d82fc JJ |
1160 | } |
1161 | } | |
c1a9b12d SL |
1162 | Err(..) => num_cpus(), |
1163 | }; | |
1164 | ||
1165 | #[cfg(windows)] | |
92a42be0 | 1166 | #[allow(bad_style)] |
c1a9b12d | 1167 | fn num_cpus() -> usize { |
92a42be0 SL |
1168 | #[repr(C)] |
1169 | struct SYSTEM_INFO { | |
1170 | wProcessorArchitecture: u16, | |
1171 | wReserved: u16, | |
1172 | dwPageSize: u32, | |
1173 | lpMinimumApplicationAddress: *mut u8, | |
1174 | lpMaximumApplicationAddress: *mut u8, | |
1175 | dwActiveProcessorMask: *mut u8, | |
1176 | dwNumberOfProcessors: u32, | |
1177 | dwProcessorType: u32, | |
1178 | dwAllocationGranularity: u32, | |
1179 | wProcessorLevel: u16, | |
1180 | wProcessorRevision: u16, | |
1181 | } | |
1182 | extern "system" { | |
1183 | fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32; | |
1184 | } | |
c1a9b12d SL |
1185 | unsafe { |
1186 | let mut sysinfo = std::mem::zeroed(); | |
92a42be0 | 1187 | GetSystemInfo(&mut sysinfo); |
c1a9b12d | 1188 | sysinfo.dwNumberOfProcessors as usize |
1a4d82fc JJ |
1189 | } |
1190 | } | |
c1a9b12d | 1191 | |
32a655c1 SL |
1192 | #[cfg(target_os = "redox")] |
1193 | fn num_cpus() -> usize { | |
1194 | // FIXME: Implement num_cpus on Redox | |
1195 | 1 | |
1196 | } | |
1197 | ||
9cc50fc6 SL |
1198 | #[cfg(any(target_os = "linux", |
1199 | target_os = "macos", | |
1200 | target_os = "ios", | |
7453a54e SL |
1201 | target_os = "android", |
1202 | target_os = "solaris", | |
c30ab7b3 SL |
1203 | target_os = "emscripten", |
1204 | target_os = "fuchsia"))] | |
9cc50fc6 SL |
1205 | fn num_cpus() -> usize { |
1206 | unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize } | |
1207 | } | |
1208 | ||
1209 | #[cfg(any(target_os = "freebsd", | |
1210 | target_os = "dragonfly", | |
1211 | target_os = "bitrig", | |
1212 | target_os = "netbsd"))] | |
c1a9b12d | 1213 | fn num_cpus() -> usize { |
5bcae85e SL |
1214 | use std::ptr; |
1215 | ||
9cc50fc6 SL |
1216 | let mut cpus: libc::c_uint = 0; |
1217 | let mut cpus_size = std::mem::size_of_val(&cpus); | |
9cc50fc6 SL |
1218 | |
1219 | unsafe { | |
7453a54e | 1220 | cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint; |
9cc50fc6 SL |
1221 | } |
1222 | if cpus < 1 { | |
7453a54e | 1223 | let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; |
9cc50fc6 SL |
1224 | unsafe { |
1225 | libc::sysctl(mib.as_mut_ptr(), | |
1226 | 2, | |
1227 | &mut cpus as *mut _ as *mut _, | |
1228 | &mut cpus_size as *mut _ as *mut _, | |
5bcae85e | 1229 | ptr::null_mut(), |
9cc50fc6 SL |
1230 | 0); |
1231 | } | |
1232 | if cpus < 1 { | |
1233 | cpus = 1; | |
1234 | } | |
1235 | } | |
1236 | cpus as usize | |
1237 | } | |
1238 | ||
1239 | #[cfg(target_os = "openbsd")] | |
1240 | fn num_cpus() -> usize { | |
5bcae85e SL |
1241 | use std::ptr; |
1242 | ||
9cc50fc6 SL |
1243 | let mut cpus: libc::c_uint = 0; |
1244 | let mut cpus_size = std::mem::size_of_val(&cpus); | |
1245 | let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; | |
1246 | ||
1247 | unsafe { | |
1248 | libc::sysctl(mib.as_mut_ptr(), | |
1249 | 2, | |
1250 | &mut cpus as *mut _ as *mut _, | |
1251 | &mut cpus_size as *mut _ as *mut _, | |
5bcae85e | 1252 | ptr::null_mut(), |
9cc50fc6 SL |
1253 | 0); |
1254 | } | |
1255 | if cpus < 1 { | |
1256 | cpus = 1; | |
1257 | } | |
1258 | cpus as usize | |
c1a9b12d | 1259 | } |
9e0c209e SL |
1260 | |
1261 | #[cfg(target_os = "haiku")] | |
1262 | fn num_cpus() -> usize { | |
1263 | // FIXME: implement | |
1264 | 1 | |
1265 | } | |
1a4d82fc JJ |
1266 | } |
1267 | ||
1268 | pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> { | |
1269 | let mut filtered = tests; | |
1270 | ||
1271 | // Remove tests that don't match the test filter | |
1272 | filtered = match opts.filter { | |
1273 | None => filtered, | |
85aaf69f | 1274 | Some(ref filter) => { |
9cc50fc6 | 1275 | filtered.into_iter() |
476ff2be SL |
1276 | .filter(|test| { |
1277 | if opts.filter_exact { | |
1278 | test.desc.name.as_slice() == &filter[..] | |
1279 | } else { | |
1280 | test.desc.name.as_slice().contains(&filter[..]) | |
1281 | } | |
1282 | }) | |
9cc50fc6 | 1283 | .collect() |
1a4d82fc JJ |
1284 | } |
1285 | }; | |
1286 | ||
c30ab7b3 SL |
1287 | // Skip tests that match any of the skip filters |
1288 | filtered = filtered.into_iter() | |
476ff2be SL |
1289 | .filter(|t| !opts.skip.iter().any(|sf| { |
1290 | if opts.filter_exact { | |
1291 | t.desc.name.as_slice() == &sf[..] | |
1292 | } else { | |
1293 | t.desc.name.as_slice().contains(&sf[..]) | |
1294 | } | |
1295 | })) | |
c30ab7b3 SL |
1296 | .collect(); |
1297 | ||
1a4d82fc JJ |
1298 | // Maybe pull out the ignored test and unignore them |
1299 | filtered = if !opts.run_ignored { | |
1300 | filtered | |
1301 | } else { | |
1302 | fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> { | |
1303 | if test.desc.ignore { | |
1304 | let TestDescAndFn {desc, testfn} = test; | |
1305 | Some(TestDescAndFn { | |
9cc50fc6 SL |
1306 | desc: TestDesc { ignore: false, ..desc }, |
1307 | testfn: testfn, | |
1a4d82fc JJ |
1308 | }) |
1309 | } else { | |
1310 | None | |
1311 | } | |
b039eaaf | 1312 | } |
e9174d1e | 1313 | filtered.into_iter().filter_map(filter).collect() |
1a4d82fc JJ |
1314 | }; |
1315 | ||
1316 | // Sort the tests alphabetically | |
1317 | filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice())); | |
1318 | ||
85aaf69f | 1319 | filtered |
1a4d82fc JJ |
1320 | } |
1321 | ||
d9579d0f AL |
1322 | pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> { |
1323 | // convert benchmarks to tests, if we're not benchmarking them | |
c30ab7b3 SL |
1324 | tests.into_iter().map(|x| { |
1325 | let testfn = match x.testfn { | |
1326 | DynBenchFn(bench) => { | |
1327 | DynTestFn(Box::new(move |()| { | |
7cac9316 XL |
1328 | bench::run_once(|b| { |
1329 | __rust_begin_short_backtrace(|| bench.run(b)) | |
1330 | }) | |
c30ab7b3 SL |
1331 | })) |
1332 | } | |
1333 | StaticBenchFn(benchfn) => { | |
1334 | DynTestFn(Box::new(move |()| { | |
7cac9316 XL |
1335 | bench::run_once(|b| { |
1336 | __rust_begin_short_backtrace(|| benchfn(b)) | |
1337 | }) | |
c30ab7b3 SL |
1338 | })) |
1339 | } | |
1340 | f => f, | |
1341 | }; | |
1342 | TestDescAndFn { | |
1343 | desc: x.desc, | |
1344 | testfn: testfn, | |
1345 | } | |
1346 | }).collect() | |
d9579d0f AL |
1347 | } |
1348 | ||
1a4d82fc JJ |
1349 | pub fn run_test(opts: &TestOpts, |
1350 | force_ignore: bool, | |
1351 | test: TestDescAndFn, | |
1352 | monitor_ch: Sender<MonitorMsg>) { | |
1353 | ||
1354 | let TestDescAndFn {desc, testfn} = test; | |
1355 | ||
1356 | if force_ignore || desc.ignore { | |
1357 | monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap(); | |
1358 | return; | |
1359 | } | |
1360 | ||
1361 | fn run_test_inner(desc: TestDesc, | |
1362 | monitor_ch: Sender<MonitorMsg>, | |
1363 | nocapture: bool, | |
c30ab7b3 | 1364 | testfn: Box<FnBox<()>>) { |
c34b1796 AL |
1365 | struct Sink(Arc<Mutex<Vec<u8>>>); |
1366 | impl Write for Sink { | |
1367 | fn write(&mut self, data: &[u8]) -> io::Result<usize> { | |
1368 | Write::write(&mut *self.0.lock().unwrap(), data) | |
1369 | } | |
9cc50fc6 SL |
1370 | fn flush(&mut self) -> io::Result<()> { |
1371 | Ok(()) | |
1372 | } | |
c34b1796 AL |
1373 | } |
1374 | ||
c30ab7b3 SL |
1375 | // Buffer for capturing standard I/O |
1376 | let data = Arc::new(Mutex::new(Vec::new())); | |
1377 | let data2 = data.clone(); | |
1378 | ||
1379 | let name = desc.name.clone(); | |
1380 | let runtest = move || { | |
1381 | let oldio = if !nocapture { | |
1382 | Some(( | |
1383 | io::set_print(Some(Box::new(Sink(data2.clone())))), | |
1384 | io::set_panic(Some(Box::new(Sink(data2)))) | |
1385 | )) | |
1386 | } else { | |
1387 | None | |
1388 | }; | |
1a4d82fc | 1389 | |
c30ab7b3 SL |
1390 | let result = catch_unwind(AssertUnwindSafe(|| { |
1391 | testfn.call_box(()) | |
1392 | })); | |
1393 | ||
1394 | if let Some((printio, panicio)) = oldio { | |
1395 | io::set_print(printio); | |
1396 | io::set_panic(panicio); | |
1397 | }; | |
1398 | ||
1399 | let test_result = calc_result(&desc, result); | |
c34b1796 | 1400 | let stdout = data.lock().unwrap().to_vec(); |
1a4d82fc | 1401 | monitor_ch.send((desc.clone(), test_result, stdout)).unwrap(); |
c30ab7b3 SL |
1402 | }; |
1403 | ||
1404 | ||
1405 | // If the platform is single-threaded we're just going to run | |
1406 | // the test synchronously, regardless of the concurrency | |
1407 | // level. | |
1408 | let supports_threads = !cfg!(target_os = "emscripten"); | |
1409 | if supports_threads { | |
1410 | let cfg = thread::Builder::new().name(match name { | |
1411 | DynTestName(ref name) => name.clone(), | |
1412 | StaticTestName(name) => name.to_owned(), | |
1413 | }); | |
1414 | cfg.spawn(runtest).unwrap(); | |
1415 | } else { | |
1416 | runtest(); | |
1417 | } | |
1a4d82fc JJ |
1418 | } |
1419 | ||
1420 | match testfn { | |
1421 | DynBenchFn(bencher) => { | |
1422 | let bs = ::bench::benchmark(|harness| bencher.run(harness)); | |
1423 | monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap(); | |
1424 | return; | |
1425 | } | |
1426 | StaticBenchFn(benchfn) => { | |
1427 | let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness)); | |
1428 | monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap(); | |
1429 | return; | |
1430 | } | |
1431 | DynMetricFn(f) => { | |
1432 | let mut mm = MetricMap::new(); | |
c30ab7b3 | 1433 | f.call_box(&mut mm); |
1a4d82fc JJ |
1434 | monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap(); |
1435 | return; | |
1436 | } | |
1437 | StaticMetricFn(f) => { | |
1438 | let mut mm = MetricMap::new(); | |
1439 | f(&mut mm); | |
1440 | monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap(); | |
1441 | return; | |
1442 | } | |
7cac9316 XL |
1443 | DynTestFn(f) => { |
1444 | let cb = move |()| { | |
1445 | __rust_begin_short_backtrace(|| f.call_box(())) | |
1446 | }; | |
1447 | run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb)) | |
1448 | } | |
1449 | StaticTestFn(f) => | |
1450 | run_test_inner(desc, monitor_ch, opts.nocapture, | |
1451 | Box::new(move |()| __rust_begin_short_backtrace(f))), | |
1a4d82fc JJ |
1452 | } |
1453 | } | |
1454 | ||
7cac9316 XL |
1455 | /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. |
1456 | #[inline(never)] | |
1457 | fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) { | |
1458 | f() | |
1459 | } | |
1460 | ||
9cc50fc6 | 1461 | fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> TestResult { |
c34b1796 AL |
1462 | match (&desc.should_panic, task_result) { |
1463 | (&ShouldPanic::No, Ok(())) | | |
e9174d1e | 1464 | (&ShouldPanic::Yes, Err(_)) => TrOk, |
476ff2be | 1465 | (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => |
1a4d82fc | 1466 | if err.downcast_ref::<String>() |
476ff2be SL |
1467 | .map(|e| &**e) |
1468 | .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) | |
1469 | .map(|e| e.contains(msg)) | |
1470 | .unwrap_or(false) { | |
1471 | TrOk | |
1472 | } else { | |
1473 | TrFailedMsg(format!("Panic did not include expected string '{}'", msg)) | |
1474 | }, | |
1a4d82fc JJ |
1475 | _ => TrFailed, |
1476 | } | |
1477 | } | |
1478 | ||
1479 | impl MetricMap { | |
1a4d82fc JJ |
1480 | pub fn new() -> MetricMap { |
1481 | MetricMap(BTreeMap::new()) | |
1482 | } | |
1483 | ||
1a4d82fc JJ |
1484 | /// Insert a named `value` (+/- `noise`) metric into the map. The value |
1485 | /// must be non-negative. The `noise` indicates the uncertainty of the | |
1486 | /// metric, which doubles as the "noise range" of acceptable | |
1487 | /// pairwise-regressions on this named value, when comparing from one | |
1488 | /// metric to the next using `compare_to_old`. | |
1489 | /// | |
1490 | /// If `noise` is positive, then it means this metric is of a value | |
1491 | /// you want to see grow smaller, so a change larger than `noise` in the | |
1492 | /// positive direction represents a regression. | |
1493 | /// | |
1494 | /// If `noise` is negative, then it means this metric is of a value | |
1495 | /// you want to see grow larger, so a change larger than `noise` in the | |
1496 | /// negative direction represents a regression. | |
1497 | pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) { | |
1498 | let m = Metric { | |
1499 | value: value, | |
9cc50fc6 | 1500 | noise: noise, |
1a4d82fc JJ |
1501 | }; |
1502 | let MetricMap(ref mut map) = *self; | |
e9174d1e | 1503 | map.insert(name.to_owned(), m); |
1a4d82fc JJ |
1504 | } |
1505 | ||
85aaf69f SL |
1506 | pub fn fmt_metrics(&self) -> String { |
1507 | let MetricMap(ref mm) = *self; | |
9cc50fc6 SL |
1508 | let v: Vec<String> = mm.iter() |
1509 | .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) | |
1510 | .collect(); | |
c1a9b12d | 1511 | v.join(", ") |
1a4d82fc JJ |
1512 | } |
1513 | } | |
1514 | ||
1515 | ||
1516 | // Benchmarking | |
1517 | ||
1518 | /// A function that is opaque to the optimizer, to allow benchmarks to | |
1519 | /// pretend to use outputs to assist in avoiding dead-code | |
1520 | /// elimination. | |
1521 | /// | |
1522 | /// This function is a no-op, and does not even read from `dummy`. | |
7453a54e | 1523 | #[cfg(not(any(all(target_os = "nacl", target_arch = "le32"), |
c30ab7b3 | 1524 | target_arch = "asmjs", target_arch = "wasm32")))] |
85aaf69f | 1525 | pub fn black_box<T>(dummy: T) -> T { |
1a4d82fc JJ |
1526 | // we need to "use" the argument in some way LLVM can't |
1527 | // introspect. | |
9cc50fc6 | 1528 | unsafe { asm!("" : : "r"(&dummy)) } |
85aaf69f | 1529 | dummy |
1a4d82fc | 1530 | } |
7453a54e | 1531 | #[cfg(any(all(target_os = "nacl", target_arch = "le32"), |
c30ab7b3 | 1532 | target_arch = "asmjs", target_arch = "wasm32"))] |
92a42be0 | 1533 | #[inline(never)] |
9cc50fc6 SL |
1534 | pub fn black_box<T>(dummy: T) -> T { |
1535 | dummy | |
1536 | } | |
1a4d82fc JJ |
1537 | |
1538 | ||
1539 | impl Bencher { | |
1540 | /// Callback for benchmark functions to run in their body. | |
9cc50fc6 SL |
1541 | pub fn iter<T, F>(&mut self, mut inner: F) |
1542 | where F: FnMut() -> T | |
1543 | { | |
32a655c1 SL |
1544 | if self.mode == BenchMode::Single { |
1545 | ns_iter_inner(&mut inner, 1); | |
1546 | return; | |
92a42be0 | 1547 | } |
1a4d82fc | 1548 | |
32a655c1 | 1549 | self.summary = Some(iter(&mut inner)); |
1a4d82fc JJ |
1550 | } |
1551 | ||
32a655c1 SL |
1552 | pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary> |
1553 | where F: FnMut(&mut Bencher) | |
9cc50fc6 | 1554 | { |
1a4d82fc | 1555 | f(self); |
32a655c1 | 1556 | return self.summary; |
1a4d82fc | 1557 | } |
32a655c1 | 1558 | } |
1a4d82fc | 1559 | |
32a655c1 SL |
1560 | fn ns_from_dur(dur: Duration) -> u64 { |
1561 | dur.as_secs() * 1_000_000_000 + (dur.subsec_nanos() as u64) | |
1562 | } | |
1563 | ||
1564 | fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64 | |
1565 | where F: FnMut() -> T | |
1566 | { | |
1567 | let start = Instant::now(); | |
1568 | for _ in 0..k { | |
1569 | black_box(inner()); | |
1570 | } | |
1571 | return ns_from_dur(start.elapsed()); | |
1572 | } | |
1573 | ||
1574 | ||
1575 | pub fn iter<T, F>(inner: &mut F) -> stats::Summary | |
1576 | where F: FnMut() -> T | |
1577 | { | |
1578 | // Initial bench run to get ballpark figure. | |
1579 | let ns_single = ns_iter_inner(inner, 1); | |
1580 | ||
1581 | // Try to estimate iter count for 1ms falling back to 1m | |
1582 | // iterations if first run took < 1ns. | |
1583 | let ns_target_total = 1_000_000; // 1ms | |
1584 | let mut n = ns_target_total / cmp::max(1, ns_single); | |
1585 | ||
1586 | // if the first run took more than 1ms we don't want to just | |
1587 | // be left doing 0 iterations on every loop. The unfortunate | |
1588 | // side effect of not being able to do as many runs is | |
1589 | // automatically handled by the statistical analysis below | |
1590 | // (i.e. larger error bars). | |
1591 | n = cmp::max(1, n); | |
1592 | ||
1593 | let mut total_run = Duration::new(0, 0); | |
1594 | let samples: &mut [f64] = &mut [0.0_f64; 50]; | |
1595 | loop { | |
1596 | let loop_start = Instant::now(); | |
1597 | ||
1598 | for p in &mut *samples { | |
1599 | *p = ns_iter_inner(inner, n) as f64 / n as f64; | |
9cc50fc6 | 1600 | } |
1a4d82fc | 1601 | |
32a655c1 SL |
1602 | stats::winsorize(samples, 5.0); |
1603 | let summ = stats::Summary::new(samples); | |
1a4d82fc | 1604 | |
32a655c1 SL |
1605 | for p in &mut *samples { |
1606 | let ns = ns_iter_inner(inner, 5 * n); | |
1607 | *p = ns as f64 / (5 * n) as f64; | |
1608 | } | |
1a4d82fc | 1609 | |
32a655c1 SL |
1610 | stats::winsorize(samples, 5.0); |
1611 | let summ5 = stats::Summary::new(samples); | |
1a4d82fc | 1612 | |
32a655c1 | 1613 | let loop_run = loop_start.elapsed(); |
1a4d82fc | 1614 | |
32a655c1 SL |
1615 | // If we've run for 100ms and seem to have converged to a |
1616 | // stable median. | |
1617 | if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 && | |
1618 | summ.median - summ5.median < summ5.median_abs_dev { | |
1619 | return summ5; | |
1620 | } | |
1a4d82fc | 1621 | |
32a655c1 SL |
1622 | total_run = total_run + loop_run; |
1623 | // Longest we ever run for is 3s. | |
1624 | if total_run > Duration::from_secs(3) { | |
1625 | return summ5; | |
1626 | } | |
1a4d82fc | 1627 | |
32a655c1 SL |
1628 | // If we overflow here just return the results so far. We check a |
1629 | // multiplier of 10 because we're about to multiply by 2 and the | |
1630 | // next iteration of the loop will also multiply by 5 (to calculate | |
1631 | // the summ5 result) | |
1632 | n = match n.checked_mul(10) { | |
1633 | Some(_) => n * 2, | |
1634 | None => { | |
1a4d82fc JJ |
1635 | return summ5; |
1636 | } | |
32a655c1 | 1637 | }; |
1a4d82fc JJ |
1638 | } |
1639 | } | |
1640 | ||
1641 | pub mod bench { | |
1642 | use std::cmp; | |
32a655c1 SL |
1643 | use stats; |
1644 | use super::{Bencher, BenchSamples, BenchMode}; | |
1a4d82fc | 1645 | |
9cc50fc6 SL |
1646 | pub fn benchmark<F>(f: F) -> BenchSamples |
1647 | where F: FnMut(&mut Bencher) | |
1648 | { | |
1a4d82fc | 1649 | let mut bs = Bencher { |
32a655c1 SL |
1650 | mode: BenchMode::Auto, |
1651 | summary: None, | |
9cc50fc6 | 1652 | bytes: 0, |
1a4d82fc JJ |
1653 | }; |
1654 | ||
32a655c1 SL |
1655 | return match bs.bench(f) { |
1656 | Some(ns_iter_summ) => { | |
1657 | let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); | |
1658 | let mb_s = bs.bytes * 1000 / ns_iter; | |
1a4d82fc | 1659 | |
32a655c1 SL |
1660 | BenchSamples { |
1661 | ns_iter_summ: ns_iter_summ, | |
1662 | mb_s: mb_s as usize, | |
1663 | } | |
1664 | } | |
1665 | None => { | |
1666 | // iter not called, so no data. | |
1667 | // FIXME: error in this case? | |
1668 | let samples: &mut [f64] = &mut [0.0_f64; 1]; | |
1669 | BenchSamples { | |
1670 | ns_iter_summ: stats::Summary::new(samples), | |
1671 | mb_s: 0, | |
1672 | } | |
1673 | } | |
1674 | }; | |
1a4d82fc | 1675 | } |
d9579d0f | 1676 | |
9cc50fc6 | 1677 | pub fn run_once<F>(f: F) |
32a655c1 | 1678 | where F: FnMut(&mut Bencher) |
9cc50fc6 | 1679 | { |
d9579d0f | 1680 | let mut bs = Bencher { |
32a655c1 SL |
1681 | mode: BenchMode::Single, |
1682 | summary: None, | |
9cc50fc6 | 1683 | bytes: 0, |
d9579d0f | 1684 | }; |
32a655c1 | 1685 | bs.bench(f); |
d9579d0f | 1686 | } |
1a4d82fc JJ |
1687 | } |
1688 | ||
1689 | #[cfg(test)] | |
1690 | mod tests { | |
476ff2be SL |
1691 | use test::{TrFailed, TrFailedMsg, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc, |
1692 | TestDescAndFn, TestOpts, run_test, MetricMap, StaticTestName, DynTestName, | |
1693 | DynTestFn, ShouldPanic}; | |
1a4d82fc | 1694 | use std::sync::mpsc::channel; |
32a655c1 SL |
1695 | use bench; |
1696 | use Bencher; | |
1a4d82fc JJ |
1697 | |
1698 | #[test] | |
1699 | pub fn do_not_run_ignored_tests() { | |
9cc50fc6 SL |
1700 | fn f() { |
1701 | panic!(); | |
1702 | } | |
1a4d82fc JJ |
1703 | let desc = TestDescAndFn { |
1704 | desc: TestDesc { | |
1705 | name: StaticTestName("whatever"), | |
1706 | ignore: true, | |
c34b1796 | 1707 | should_panic: ShouldPanic::No, |
1a4d82fc | 1708 | }, |
c30ab7b3 | 1709 | testfn: DynTestFn(Box::new(move |()| f())), |
1a4d82fc JJ |
1710 | }; |
1711 | let (tx, rx) = channel(); | |
1712 | run_test(&TestOpts::new(), false, desc, tx); | |
1713 | let (_, res, _) = rx.recv().unwrap(); | |
1714 | assert!(res != TrOk); | |
1715 | } | |
1716 | ||
1717 | #[test] | |
1718 | pub fn ignored_tests_result_in_ignored() { | |
9cc50fc6 | 1719 | fn f() {} |
1a4d82fc JJ |
1720 | let desc = TestDescAndFn { |
1721 | desc: TestDesc { | |
1722 | name: StaticTestName("whatever"), | |
1723 | ignore: true, | |
c34b1796 | 1724 | should_panic: ShouldPanic::No, |
1a4d82fc | 1725 | }, |
c30ab7b3 | 1726 | testfn: DynTestFn(Box::new(move |()| f())), |
1a4d82fc JJ |
1727 | }; |
1728 | let (tx, rx) = channel(); | |
1729 | run_test(&TestOpts::new(), false, desc, tx); | |
1730 | let (_, res, _) = rx.recv().unwrap(); | |
1731 | assert!(res == TrIgnored); | |
1732 | } | |
1733 | ||
1734 | #[test] | |
c34b1796 | 1735 | fn test_should_panic() { |
9cc50fc6 SL |
1736 | fn f() { |
1737 | panic!(); | |
1738 | } | |
1a4d82fc JJ |
1739 | let desc = TestDescAndFn { |
1740 | desc: TestDesc { | |
1741 | name: StaticTestName("whatever"), | |
1742 | ignore: false, | |
e9174d1e | 1743 | should_panic: ShouldPanic::Yes, |
1a4d82fc | 1744 | }, |
c30ab7b3 | 1745 | testfn: DynTestFn(Box::new(move |()| f())), |
1a4d82fc JJ |
1746 | }; |
1747 | let (tx, rx) = channel(); | |
1748 | run_test(&TestOpts::new(), false, desc, tx); | |
1749 | let (_, res, _) = rx.recv().unwrap(); | |
1750 | assert!(res == TrOk); | |
1751 | } | |
1752 | ||
1753 | #[test] | |
c34b1796 | 1754 | fn test_should_panic_good_message() { |
9cc50fc6 SL |
1755 | fn f() { |
1756 | panic!("an error message"); | |
1757 | } | |
1a4d82fc JJ |
1758 | let desc = TestDescAndFn { |
1759 | desc: TestDesc { | |
1760 | name: StaticTestName("whatever"), | |
1761 | ignore: false, | |
e9174d1e | 1762 | should_panic: ShouldPanic::YesWithMessage("error message"), |
1a4d82fc | 1763 | }, |
c30ab7b3 | 1764 | testfn: DynTestFn(Box::new(move |()| f())), |
1a4d82fc JJ |
1765 | }; |
1766 | let (tx, rx) = channel(); | |
1767 | run_test(&TestOpts::new(), false, desc, tx); | |
1768 | let (_, res, _) = rx.recv().unwrap(); | |
1769 | assert!(res == TrOk); | |
1770 | } | |
1771 | ||
1772 | #[test] | |
c34b1796 | 1773 | fn test_should_panic_bad_message() { |
9cc50fc6 SL |
1774 | fn f() { |
1775 | panic!("an error message"); | |
1776 | } | |
476ff2be SL |
1777 | let expected = "foobar"; |
1778 | let failed_msg = "Panic did not include expected string"; | |
1a4d82fc JJ |
1779 | let desc = TestDescAndFn { |
1780 | desc: TestDesc { | |
1781 | name: StaticTestName("whatever"), | |
1782 | ignore: false, | |
476ff2be | 1783 | should_panic: ShouldPanic::YesWithMessage(expected), |
1a4d82fc | 1784 | }, |
c30ab7b3 | 1785 | testfn: DynTestFn(Box::new(move |()| f())), |
1a4d82fc JJ |
1786 | }; |
1787 | let (tx, rx) = channel(); | |
1788 | run_test(&TestOpts::new(), false, desc, tx); | |
1789 | let (_, res, _) = rx.recv().unwrap(); | |
476ff2be | 1790 | assert!(res == TrFailedMsg(format!("{} '{}'", failed_msg, expected))); |
1a4d82fc JJ |
1791 | } |
1792 | ||
1793 | #[test] | |
c34b1796 | 1794 | fn test_should_panic_but_succeeds() { |
9cc50fc6 | 1795 | fn f() {} |
1a4d82fc JJ |
1796 | let desc = TestDescAndFn { |
1797 | desc: TestDesc { | |
1798 | name: StaticTestName("whatever"), | |
1799 | ignore: false, | |
e9174d1e | 1800 | should_panic: ShouldPanic::Yes, |
1a4d82fc | 1801 | }, |
c30ab7b3 | 1802 | testfn: DynTestFn(Box::new(move |()| f())), |
1a4d82fc JJ |
1803 | }; |
1804 | let (tx, rx) = channel(); | |
1805 | run_test(&TestOpts::new(), false, desc, tx); | |
1806 | let (_, res, _) = rx.recv().unwrap(); | |
1807 | assert!(res == TrFailed); | |
1808 | } | |
1809 | ||
1a4d82fc JJ |
1810 | #[test] |
1811 | fn parse_ignored_flag() { | |
9cc50fc6 | 1812 | let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()]; |
85aaf69f | 1813 | let opts = match parse_opts(&args) { |
1a4d82fc | 1814 | Some(Ok(o)) => o, |
9cc50fc6 | 1815 | _ => panic!("Malformed arg in parse_ignored_flag"), |
1a4d82fc JJ |
1816 | }; |
1817 | assert!((opts.run_ignored)); | |
1818 | } | |
1819 | ||
1820 | #[test] | |
1821 | pub fn filter_for_ignored_option() { | |
1822 | // When we run ignored tests the test filter should filter out all the | |
1823 | // unignored tests and flip the ignore flag on the rest to false | |
1824 | ||
1825 | let mut opts = TestOpts::new(); | |
1826 | opts.run_tests = true; | |
1827 | opts.run_ignored = true; | |
1828 | ||
9cc50fc6 SL |
1829 | let tests = vec![TestDescAndFn { |
1830 | desc: TestDesc { | |
1831 | name: StaticTestName("1"), | |
1832 | ignore: true, | |
1833 | should_panic: ShouldPanic::No, | |
1834 | }, | |
c30ab7b3 | 1835 | testfn: DynTestFn(Box::new(move |()| {})), |
9cc50fc6 SL |
1836 | }, |
1837 | TestDescAndFn { | |
1838 | desc: TestDesc { | |
1839 | name: StaticTestName("2"), | |
1840 | ignore: false, | |
1841 | should_panic: ShouldPanic::No, | |
1842 | }, | |
c30ab7b3 | 1843 | testfn: DynTestFn(Box::new(move |()| {})), |
9cc50fc6 | 1844 | }]; |
1a4d82fc JJ |
1845 | let filtered = filter_tests(&opts, tests); |
1846 | ||
1847 | assert_eq!(filtered.len(), 1); | |
9cc50fc6 | 1848 | assert_eq!(filtered[0].desc.name.to_string(), "1"); |
54a0048b | 1849 | assert!(!filtered[0].desc.ignore); |
1a4d82fc JJ |
1850 | } |
1851 | ||
476ff2be SL |
1852 | #[test] |
1853 | pub fn exact_filter_match() { | |
1854 | fn tests() -> Vec<TestDescAndFn> { | |
1855 | vec!["base", | |
1856 | "base::test", | |
1857 | "base::test1", | |
1858 | "base::test2", | |
1859 | ].into_iter() | |
1860 | .map(|name| TestDescAndFn { | |
1861 | desc: TestDesc { | |
1862 | name: StaticTestName(name), | |
1863 | ignore: false, | |
1864 | should_panic: ShouldPanic::No, | |
1865 | }, | |
1866 | testfn: DynTestFn(Box::new(move |()| {})) | |
1867 | }) | |
1868 | .collect() | |
1869 | } | |
1870 | ||
1871 | let substr = filter_tests(&TestOpts { | |
1872 | filter: Some("base".into()), | |
1873 | ..TestOpts::new() | |
1874 | }, tests()); | |
1875 | assert_eq!(substr.len(), 4); | |
1876 | ||
1877 | let substr = filter_tests(&TestOpts { | |
1878 | filter: Some("bas".into()), | |
1879 | ..TestOpts::new() | |
1880 | }, tests()); | |
1881 | assert_eq!(substr.len(), 4); | |
1882 | ||
1883 | let substr = filter_tests(&TestOpts { | |
1884 | filter: Some("::test".into()), | |
1885 | ..TestOpts::new() | |
1886 | }, tests()); | |
1887 | assert_eq!(substr.len(), 3); | |
1888 | ||
1889 | let substr = filter_tests(&TestOpts { | |
1890 | filter: Some("base::test".into()), | |
1891 | ..TestOpts::new() | |
1892 | }, tests()); | |
1893 | assert_eq!(substr.len(), 3); | |
1894 | ||
1895 | let exact = filter_tests(&TestOpts { | |
1896 | filter: Some("base".into()), | |
1897 | filter_exact: true, ..TestOpts::new() | |
1898 | }, tests()); | |
1899 | assert_eq!(exact.len(), 1); | |
1900 | ||
1901 | let exact = filter_tests(&TestOpts { | |
1902 | filter: Some("bas".into()), | |
1903 | filter_exact: true, | |
1904 | ..TestOpts::new() | |
1905 | }, tests()); | |
1906 | assert_eq!(exact.len(), 0); | |
1907 | ||
1908 | let exact = filter_tests(&TestOpts { | |
1909 | filter: Some("::test".into()), | |
1910 | filter_exact: true, | |
1911 | ..TestOpts::new() | |
1912 | }, tests()); | |
1913 | assert_eq!(exact.len(), 0); | |
1914 | ||
1915 | let exact = filter_tests(&TestOpts { | |
1916 | filter: Some("base::test".into()), | |
1917 | filter_exact: true, | |
1918 | ..TestOpts::new() | |
1919 | }, tests()); | |
1920 | assert_eq!(exact.len(), 1); | |
1921 | } | |
1922 | ||
1a4d82fc JJ |
1923 | #[test] |
1924 | pub fn sort_tests() { | |
1925 | let mut opts = TestOpts::new(); | |
1926 | opts.run_tests = true; | |
1927 | ||
9cc50fc6 SL |
1928 | let names = vec!["sha1::test".to_string(), |
1929 | "isize::test_to_str".to_string(), | |
1930 | "isize::test_pow".to_string(), | |
1931 | "test::do_not_run_ignored_tests".to_string(), | |
1932 | "test::ignored_tests_result_in_ignored".to_string(), | |
1933 | "test::first_free_arg_should_be_a_filter".to_string(), | |
1934 | "test::parse_ignored_flag".to_string(), | |
1935 | "test::filter_for_ignored_option".to_string(), | |
1936 | "test::sort_tests".to_string()]; | |
1937 | let tests = { | |
1938 | fn testfn() {} | |
1a4d82fc | 1939 | let mut tests = Vec::new(); |
85aaf69f | 1940 | for name in &names { |
1a4d82fc JJ |
1941 | let test = TestDescAndFn { |
1942 | desc: TestDesc { | |
1943 | name: DynTestName((*name).clone()), | |
1944 | ignore: false, | |
c34b1796 | 1945 | should_panic: ShouldPanic::No, |
1a4d82fc | 1946 | }, |
c30ab7b3 | 1947 | testfn: DynTestFn(Box::new(move |()| testfn())), |
1a4d82fc JJ |
1948 | }; |
1949 | tests.push(test); | |
1950 | } | |
1951 | tests | |
1952 | }; | |
1953 | let filtered = filter_tests(&opts, tests); | |
1954 | ||
9cc50fc6 SL |
1955 | let expected = vec!["isize::test_pow".to_string(), |
1956 | "isize::test_to_str".to_string(), | |
1957 | "sha1::test".to_string(), | |
1958 | "test::do_not_run_ignored_tests".to_string(), | |
1959 | "test::filter_for_ignored_option".to_string(), | |
1960 | "test::first_free_arg_should_be_a_filter".to_string(), | |
1961 | "test::ignored_tests_result_in_ignored".to_string(), | |
1962 | "test::parse_ignored_flag".to_string(), | |
1963 | "test::sort_tests".to_string()]; | |
1a4d82fc | 1964 | |
62682a34 | 1965 | for (a, b) in expected.iter().zip(filtered) { |
1a4d82fc JJ |
1966 | assert!(*a == b.desc.name.to_string()); |
1967 | } | |
1968 | } | |
1969 | ||
1a4d82fc JJ |
1970 | #[test] |
1971 | pub fn test_metricmap_compare() { | |
1972 | let mut m1 = MetricMap::new(); | |
1973 | let mut m2 = MetricMap::new(); | |
1974 | m1.insert_metric("in-both-noise", 1000.0, 200.0); | |
1975 | m2.insert_metric("in-both-noise", 1100.0, 200.0); | |
1976 | ||
1977 | m1.insert_metric("in-first-noise", 1000.0, 2.0); | |
1978 | m2.insert_metric("in-second-noise", 1000.0, 2.0); | |
1979 | ||
1980 | m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0); | |
1981 | m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0); | |
1982 | ||
1983 | m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0); | |
1984 | m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0); | |
1985 | ||
1986 | m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0); | |
1987 | m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0); | |
1988 | ||
1989 | m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0); | |
1990 | m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0); | |
1a4d82fc | 1991 | } |
32a655c1 SL |
1992 | |
1993 | #[test] | |
1994 | pub fn test_bench_once_no_iter() { | |
1995 | fn f(_: &mut Bencher) {} | |
1996 | bench::run_once(f); | |
1997 | } | |
1998 | ||
1999 | #[test] | |
2000 | pub fn test_bench_once_iter() { | |
2001 | fn f(b: &mut Bencher) { | |
2002 | b.iter(|| { | |
2003 | }) | |
2004 | } | |
2005 | bench::run_once(f); | |
2006 | } | |
2007 | ||
2008 | #[test] | |
2009 | pub fn test_bench_no_iter() { | |
2010 | fn f(_: &mut Bencher) {} | |
2011 | bench::benchmark(f); | |
2012 | } | |
2013 | ||
2014 | #[test] | |
2015 | pub fn test_bench_iter() { | |
2016 | fn f(b: &mut Bencher) { | |
2017 | b.iter(|| { | |
2018 | }) | |
2019 | } | |
2020 | bench::benchmark(f); | |
2021 | } | |
1a4d82fc | 2022 | } |