]>
Commit | Line | Data |
---|---|---|
416331ca XL |
1 | use super::*; |
2 | ||
e74abb32 | 3 | use crate::{ |
e74abb32 | 4 | console::OutputLocation, |
e74abb32 XL |
5 | formatters::PrettyFormatter, |
6 | test::{ | |
dfeec247 | 7 | parse_opts, |
dfeec247 | 8 | MetricMap, |
e74abb32 XL |
9 | // FIXME (introduced by #65251) |
10 | // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions, | |
11 | // TestType, TrFailedMsg, TrIgnored, TrOk, | |
12 | }, | |
dfeec247 | 13 | time::{TestTimeOptions, TimeThreshold}, |
dc9dc135 | 14 | }; |
dc9dc135 | 15 | |
416331ca XL |
16 | impl TestOpts { |
17 | fn new() -> TestOpts { | |
18 | TestOpts { | |
19 | list: false, | |
6a06907d | 20 | filters: vec![], |
416331ca | 21 | filter_exact: false, |
60c5eb7d | 22 | force_run_in_process: false, |
416331ca XL |
23 | exclude_should_panic: false, |
24 | run_ignored: RunIgnored::No, | |
25 | run_tests: false, | |
26 | bench_benchmarks: false, | |
27 | logfile: None, | |
28 | nocapture: false, | |
29 | color: AutoColor, | |
30 | format: OutputFormat::Pretty, | |
c295e0f8 XL |
31 | shuffle: false, |
32 | shuffle_seed: None, | |
416331ca XL |
33 | test_threads: None, |
34 | skip: vec![], | |
e74abb32 | 35 | time_options: None, |
416331ca | 36 | options: Options::new(), |
487cf647 | 37 | fail_fast: false, |
416331ca XL |
38 | } |
39 | } | |
40 | } | |
41 | ||
dc9dc135 XL |
42 | fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> { |
43 | vec![ | |
44 | TestDescAndFn { | |
45 | desc: TestDesc { | |
46 | name: StaticTestName("1"), | |
47 | ignore: true, | |
5e7ed085 | 48 | ignore_message: None, |
353b0b11 | 49 | source_file: "", |
353b0b11 | 50 | start_line: 0, |
353b0b11 | 51 | start_col: 0, |
353b0b11 | 52 | end_line: 0, |
353b0b11 | 53 | end_col: 0, |
dc9dc135 | 54 | should_panic: ShouldPanic::No, |
17df50a5 | 55 | compile_fail: false, |
17df50a5 | 56 | no_run: false, |
e74abb32 | 57 | test_type: TestType::Unknown, |
dc9dc135 | 58 | }, |
2b03887a | 59 | testfn: DynTestFn(Box::new(move || Ok(()))), |
dc9dc135 XL |
60 | }, |
61 | TestDescAndFn { | |
62 | desc: TestDesc { | |
63 | name: StaticTestName("2"), | |
64 | ignore: false, | |
5e7ed085 | 65 | ignore_message: None, |
353b0b11 | 66 | source_file: "", |
353b0b11 | 67 | start_line: 0, |
353b0b11 | 68 | start_col: 0, |
353b0b11 | 69 | end_line: 0, |
353b0b11 | 70 | end_col: 0, |
dc9dc135 | 71 | should_panic: ShouldPanic::No, |
17df50a5 | 72 | compile_fail: false, |
17df50a5 | 73 | no_run: false, |
e74abb32 | 74 | test_type: TestType::Unknown, |
dc9dc135 | 75 | }, |
2b03887a | 76 | testfn: DynTestFn(Box::new(move || Ok(()))), |
dc9dc135 XL |
77 | }, |
78 | ] | |
79 | } | |
80 | ||
81 | #[test] | |
82 | pub fn do_not_run_ignored_tests() { | |
2b03887a | 83 | fn f() -> Result<(), String> { |
dc9dc135 XL |
84 | panic!(); |
85 | } | |
86 | let desc = TestDescAndFn { | |
87 | desc: TestDesc { | |
88 | name: StaticTestName("whatever"), | |
89 | ignore: true, | |
5e7ed085 | 90 | ignore_message: None, |
353b0b11 | 91 | source_file: "", |
353b0b11 | 92 | start_line: 0, |
353b0b11 | 93 | start_col: 0, |
353b0b11 | 94 | end_line: 0, |
353b0b11 | 95 | end_col: 0, |
dc9dc135 | 96 | should_panic: ShouldPanic::No, |
17df50a5 | 97 | compile_fail: false, |
17df50a5 | 98 | no_run: false, |
e74abb32 | 99 | test_type: TestType::Unknown, |
dc9dc135 XL |
100 | }, |
101 | testfn: DynTestFn(Box::new(f)), | |
102 | }; | |
103 | let (tx, rx) = channel(); | |
487cf647 | 104 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 | 105 | let result = rx.recv().unwrap().result; |
60c5eb7d | 106 | assert_ne!(result, TrOk); |
dc9dc135 XL |
107 | } |
108 | ||
109 | #[test] | |
110 | pub fn ignored_tests_result_in_ignored() { | |
2b03887a FG |
111 | fn f() -> Result<(), String> { |
112 | Ok(()) | |
113 | } | |
dc9dc135 XL |
114 | let desc = TestDescAndFn { |
115 | desc: TestDesc { | |
116 | name: StaticTestName("whatever"), | |
117 | ignore: true, | |
5e7ed085 | 118 | ignore_message: None, |
353b0b11 | 119 | source_file: "", |
353b0b11 | 120 | start_line: 0, |
353b0b11 | 121 | start_col: 0, |
353b0b11 | 122 | end_line: 0, |
353b0b11 | 123 | end_col: 0, |
dc9dc135 | 124 | should_panic: ShouldPanic::No, |
17df50a5 | 125 | compile_fail: false, |
17df50a5 | 126 | no_run: false, |
e74abb32 | 127 | test_type: TestType::Unknown, |
dc9dc135 XL |
128 | }, |
129 | testfn: DynTestFn(Box::new(f)), | |
130 | }; | |
131 | let (tx, rx) = channel(); | |
487cf647 | 132 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 | 133 | let result = rx.recv().unwrap().result; |
60c5eb7d | 134 | assert_eq!(result, TrIgnored); |
dc9dc135 XL |
135 | } |
136 | ||
e74abb32 | 137 | // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) |
dc9dc135 | 138 | #[test] |
e74abb32 | 139 | #[cfg(not(target_os = "emscripten"))] |
fe692bf9 | 140 | #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] |
dc9dc135 | 141 | fn test_should_panic() { |
2b03887a | 142 | fn f() -> Result<(), String> { |
dc9dc135 XL |
143 | panic!(); |
144 | } | |
145 | let desc = TestDescAndFn { | |
146 | desc: TestDesc { | |
147 | name: StaticTestName("whatever"), | |
148 | ignore: false, | |
5e7ed085 | 149 | ignore_message: None, |
353b0b11 | 150 | source_file: "", |
353b0b11 | 151 | start_line: 0, |
353b0b11 | 152 | start_col: 0, |
353b0b11 | 153 | end_line: 0, |
353b0b11 | 154 | end_col: 0, |
dc9dc135 | 155 | should_panic: ShouldPanic::Yes, |
17df50a5 | 156 | compile_fail: false, |
17df50a5 | 157 | no_run: false, |
e74abb32 | 158 | test_type: TestType::Unknown, |
dc9dc135 XL |
159 | }, |
160 | testfn: DynTestFn(Box::new(f)), | |
161 | }; | |
162 | let (tx, rx) = channel(); | |
487cf647 | 163 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 | 164 | let result = rx.recv().unwrap().result; |
60c5eb7d | 165 | assert_eq!(result, TrOk); |
dc9dc135 XL |
166 | } |
167 | ||
e74abb32 | 168 | // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) |
dc9dc135 | 169 | #[test] |
e74abb32 | 170 | #[cfg(not(target_os = "emscripten"))] |
fe692bf9 | 171 | #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] |
dc9dc135 | 172 | fn test_should_panic_good_message() { |
2b03887a | 173 | fn f() -> Result<(), String> { |
dc9dc135 XL |
174 | panic!("an error message"); |
175 | } | |
176 | let desc = TestDescAndFn { | |
177 | desc: TestDesc { | |
178 | name: StaticTestName("whatever"), | |
179 | ignore: false, | |
5e7ed085 | 180 | ignore_message: None, |
353b0b11 | 181 | source_file: "", |
353b0b11 | 182 | start_line: 0, |
353b0b11 | 183 | start_col: 0, |
353b0b11 | 184 | end_line: 0, |
353b0b11 | 185 | end_col: 0, |
dc9dc135 | 186 | should_panic: ShouldPanic::YesWithMessage("error message"), |
17df50a5 | 187 | compile_fail: false, |
17df50a5 | 188 | no_run: false, |
e74abb32 | 189 | test_type: TestType::Unknown, |
dc9dc135 XL |
190 | }, |
191 | testfn: DynTestFn(Box::new(f)), | |
192 | }; | |
193 | let (tx, rx) = channel(); | |
487cf647 | 194 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 | 195 | let result = rx.recv().unwrap().result; |
60c5eb7d | 196 | assert_eq!(result, TrOk); |
dc9dc135 XL |
197 | } |
198 | ||
e74abb32 | 199 | // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) |
dc9dc135 | 200 | #[test] |
e74abb32 | 201 | #[cfg(not(target_os = "emscripten"))] |
fe692bf9 | 202 | #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] |
dc9dc135 | 203 | fn test_should_panic_bad_message() { |
e74abb32 | 204 | use crate::tests::TrFailedMsg; |
2b03887a | 205 | fn f() -> Result<(), String> { |
dc9dc135 XL |
206 | panic!("an error message"); |
207 | } | |
208 | let expected = "foobar"; | |
60c5eb7d XL |
209 | let failed_msg = r#"panic did not contain expected string |
210 | panic message: `"an error message"`, | |
211 | expected substring: `"foobar"`"#; | |
dc9dc135 XL |
212 | let desc = TestDescAndFn { |
213 | desc: TestDesc { | |
214 | name: StaticTestName("whatever"), | |
215 | ignore: false, | |
5e7ed085 | 216 | ignore_message: None, |
353b0b11 | 217 | source_file: "", |
353b0b11 | 218 | start_line: 0, |
353b0b11 | 219 | start_col: 0, |
353b0b11 | 220 | end_line: 0, |
353b0b11 | 221 | end_col: 0, |
dc9dc135 | 222 | should_panic: ShouldPanic::YesWithMessage(expected), |
17df50a5 | 223 | compile_fail: false, |
17df50a5 | 224 | no_run: false, |
e74abb32 | 225 | test_type: TestType::Unknown, |
dc9dc135 XL |
226 | }, |
227 | testfn: DynTestFn(Box::new(f)), | |
228 | }; | |
229 | let (tx, rx) = channel(); | |
487cf647 | 230 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 | 231 | let result = rx.recv().unwrap().result; |
60c5eb7d XL |
232 | assert_eq!(result, TrFailedMsg(failed_msg.to_string())); |
233 | } | |
234 | ||
235 | // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) | |
236 | #[test] | |
237 | #[cfg(not(target_os = "emscripten"))] | |
fe692bf9 | 238 | #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] |
60c5eb7d XL |
239 | fn test_should_panic_non_string_message_type() { |
240 | use crate::tests::TrFailedMsg; | |
6a06907d | 241 | use std::any::TypeId; |
2b03887a | 242 | fn f() -> Result<(), String> { |
5869c6ff | 243 | std::panic::panic_any(1i32); |
60c5eb7d XL |
244 | } |
245 | let expected = "foobar"; | |
dfeec247 XL |
246 | let failed_msg = format!( |
247 | r#"expected panic with string value, | |
60c5eb7d | 248 | found non-string value: `{:?}` |
dfeec247 XL |
249 | expected substring: `"foobar"`"#, |
250 | TypeId::of::<i32>() | |
251 | ); | |
60c5eb7d XL |
252 | let desc = TestDescAndFn { |
253 | desc: TestDesc { | |
254 | name: StaticTestName("whatever"), | |
255 | ignore: false, | |
5e7ed085 | 256 | ignore_message: None, |
353b0b11 | 257 | source_file: "", |
353b0b11 | 258 | start_line: 0, |
353b0b11 | 259 | start_col: 0, |
353b0b11 | 260 | end_line: 0, |
353b0b11 | 261 | end_col: 0, |
60c5eb7d | 262 | should_panic: ShouldPanic::YesWithMessage(expected), |
17df50a5 | 263 | compile_fail: false, |
17df50a5 | 264 | no_run: false, |
60c5eb7d XL |
265 | test_type: TestType::Unknown, |
266 | }, | |
267 | testfn: DynTestFn(Box::new(f)), | |
268 | }; | |
269 | let (tx, rx) = channel(); | |
487cf647 | 270 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
60c5eb7d XL |
271 | let result = rx.recv().unwrap().result; |
272 | assert_eq!(result, TrFailedMsg(failed_msg)); | |
dc9dc135 XL |
273 | } |
274 | ||
e74abb32 | 275 | // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251) |
dc9dc135 | 276 | #[test] |
e74abb32 | 277 | #[cfg(not(target_os = "emscripten"))] |
fe692bf9 | 278 | #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] |
dc9dc135 | 279 | fn test_should_panic_but_succeeds() { |
5869c6ff XL |
280 | let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")]; |
281 | ||
282 | for &should_panic in should_panic_variants.iter() { | |
2b03887a FG |
283 | fn f() -> Result<(), String> { |
284 | Ok(()) | |
285 | } | |
5869c6ff XL |
286 | let desc = TestDescAndFn { |
287 | desc: TestDesc { | |
288 | name: StaticTestName("whatever"), | |
289 | ignore: false, | |
5e7ed085 | 290 | ignore_message: None, |
353b0b11 | 291 | source_file: "", |
353b0b11 | 292 | start_line: 0, |
353b0b11 | 293 | start_col: 0, |
353b0b11 | 294 | end_line: 0, |
353b0b11 | 295 | end_col: 0, |
5869c6ff | 296 | should_panic, |
17df50a5 | 297 | compile_fail: false, |
17df50a5 | 298 | no_run: false, |
5869c6ff XL |
299 | test_type: TestType::Unknown, |
300 | }, | |
301 | testfn: DynTestFn(Box::new(f)), | |
302 | }; | |
303 | let (tx, rx) = channel(); | |
487cf647 | 304 | run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx); |
5869c6ff XL |
305 | let result = rx.recv().unwrap().result; |
306 | assert_eq!( | |
307 | result, | |
308 | TrFailedMsg("test did not panic as expected".to_string()), | |
309 | "should_panic == {:?}", | |
310 | should_panic | |
311 | ); | |
312 | } | |
e74abb32 XL |
313 | } |
314 | ||
315 | fn report_time_test_template(report_time: bool) -> Option<TestExecTime> { | |
2b03887a FG |
316 | fn f() -> Result<(), String> { |
317 | Ok(()) | |
318 | } | |
e74abb32 XL |
319 | let desc = TestDescAndFn { |
320 | desc: TestDesc { | |
321 | name: StaticTestName("whatever"), | |
322 | ignore: false, | |
5e7ed085 | 323 | ignore_message: None, |
353b0b11 | 324 | source_file: "", |
353b0b11 | 325 | start_line: 0, |
353b0b11 | 326 | start_col: 0, |
353b0b11 | 327 | end_line: 0, |
353b0b11 | 328 | end_col: 0, |
e74abb32 | 329 | should_panic: ShouldPanic::No, |
17df50a5 | 330 | compile_fail: false, |
17df50a5 | 331 | no_run: false, |
e74abb32 XL |
332 | test_type: TestType::Unknown, |
333 | }, | |
334 | testfn: DynTestFn(Box::new(f)), | |
335 | }; | |
dfeec247 | 336 | let time_options = if report_time { Some(TestTimeOptions::default()) } else { None }; |
e74abb32 | 337 | |
dfeec247 | 338 | let test_opts = TestOpts { time_options, ..TestOpts::new() }; |
e74abb32 | 339 | let (tx, rx) = channel(); |
487cf647 | 340 | run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 XL |
341 | let exec_time = rx.recv().unwrap().exec_time; |
342 | exec_time | |
343 | } | |
344 | ||
345 | #[test] | |
346 | fn test_should_not_report_time() { | |
347 | let exec_time = report_time_test_template(false); | |
348 | assert!(exec_time.is_none()); | |
349 | } | |
350 | ||
351 | #[test] | |
352 | fn test_should_report_time() { | |
353 | let exec_time = report_time_test_template(true); | |
354 | assert!(exec_time.is_some()); | |
355 | } | |
356 | ||
357 | fn time_test_failure_template(test_type: TestType) -> TestResult { | |
2b03887a FG |
358 | fn f() -> Result<(), String> { |
359 | Ok(()) | |
360 | } | |
e74abb32 XL |
361 | let desc = TestDescAndFn { |
362 | desc: TestDesc { | |
363 | name: StaticTestName("whatever"), | |
364 | ignore: false, | |
5e7ed085 | 365 | ignore_message: None, |
353b0b11 | 366 | source_file: "", |
353b0b11 | 367 | start_line: 0, |
353b0b11 | 368 | start_col: 0, |
353b0b11 | 369 | end_line: 0, |
353b0b11 | 370 | end_col: 0, |
e74abb32 | 371 | should_panic: ShouldPanic::No, |
17df50a5 | 372 | compile_fail: false, |
17df50a5 | 373 | no_run: false, |
dfeec247 | 374 | test_type, |
e74abb32 XL |
375 | }, |
376 | testfn: DynTestFn(Box::new(f)), | |
377 | }; | |
378 | // `Default` will initialize all the thresholds to 0 milliseconds. | |
379 | let mut time_options = TestTimeOptions::default(); | |
380 | time_options.error_on_excess = true; | |
381 | ||
dfeec247 | 382 | let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() }; |
e74abb32 | 383 | let (tx, rx) = channel(); |
487cf647 | 384 | run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx); |
e74abb32 XL |
385 | let result = rx.recv().unwrap().result; |
386 | ||
387 | result | |
388 | } | |
389 | ||
390 | #[test] | |
391 | fn test_error_on_exceed() { | |
392 | let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest]; | |
393 | ||
60c5eb7d | 394 | for test_type in types.iter() { |
e74abb32 XL |
395 | let result = time_test_failure_template(*test_type); |
396 | ||
397 | assert_eq!(result, TestResult::TrTimedFail); | |
398 | } | |
399 | ||
400 | // Check that for unknown tests thresholds aren't applied. | |
401 | let result = time_test_failure_template(TestType::Unknown); | |
402 | assert_eq!(result, TestResult::TrOk); | |
403 | } | |
404 | ||
405 | fn typed_test_desc(test_type: TestType) -> TestDesc { | |
406 | TestDesc { | |
407 | name: StaticTestName("whatever"), | |
408 | ignore: false, | |
5e7ed085 | 409 | ignore_message: None, |
353b0b11 | 410 | source_file: "", |
353b0b11 | 411 | start_line: 0, |
353b0b11 | 412 | start_col: 0, |
353b0b11 | 413 | end_line: 0, |
353b0b11 | 414 | end_col: 0, |
e74abb32 | 415 | should_panic: ShouldPanic::No, |
17df50a5 | 416 | compile_fail: false, |
17df50a5 | 417 | no_run: false, |
dfeec247 | 418 | test_type, |
e74abb32 XL |
419 | } |
420 | } | |
421 | ||
422 | fn test_exec_time(millis: u64) -> TestExecTime { | |
423 | TestExecTime(Duration::from_millis(millis)) | |
424 | } | |
425 | ||
426 | #[test] | |
427 | fn test_time_options_threshold() { | |
428 | let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100)); | |
429 | let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000)); | |
430 | let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000)); | |
431 | ||
432 | let options = TestTimeOptions { | |
433 | error_on_excess: false, | |
e74abb32 XL |
434 | unit_threshold: unit.clone(), |
435 | integration_threshold: integration.clone(), | |
436 | doctest_threshold: doc.clone(), | |
437 | }; | |
438 | ||
439 | let test_vector = [ | |
440 | (TestType::UnitTest, unit.warn.as_millis() - 1, false, false), | |
441 | (TestType::UnitTest, unit.warn.as_millis(), true, false), | |
442 | (TestType::UnitTest, unit.critical.as_millis(), true, true), | |
443 | (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false), | |
444 | (TestType::IntegrationTest, integration.warn.as_millis(), true, false), | |
445 | (TestType::IntegrationTest, integration.critical.as_millis(), true, true), | |
446 | (TestType::DocTest, doc.warn.as_millis() - 1, false, false), | |
447 | (TestType::DocTest, doc.warn.as_millis(), true, false), | |
448 | (TestType::DocTest, doc.critical.as_millis(), true, true), | |
449 | ]; | |
450 | ||
60c5eb7d | 451 | for (test_type, time, expected_warn, expected_critical) in test_vector.iter() { |
e74abb32 XL |
452 | let test_desc = typed_test_desc(*test_type); |
453 | let exec_time = test_exec_time(*time as u64); | |
454 | ||
455 | assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn); | |
456 | assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical); | |
457 | } | |
dc9dc135 XL |
458 | } |
459 | ||
460 | #[test] | |
461 | fn parse_ignored_flag() { | |
dfeec247 | 462 | let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()]; |
dc9dc135 XL |
463 | let opts = parse_opts(&args).unwrap().unwrap(); |
464 | assert_eq!(opts.run_ignored, RunIgnored::Only); | |
465 | } | |
466 | ||
e1599b0c XL |
467 | #[test] |
468 | fn parse_show_output_flag() { | |
dfeec247 | 469 | let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()]; |
e1599b0c XL |
470 | let opts = parse_opts(&args).unwrap().unwrap(); |
471 | assert!(opts.options.display_output); | |
472 | } | |
473 | ||
dc9dc135 XL |
474 | #[test] |
475 | fn parse_include_ignored_flag() { | |
5869c6ff | 476 | let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()]; |
dc9dc135 XL |
477 | let opts = parse_opts(&args).unwrap().unwrap(); |
478 | assert_eq!(opts.run_ignored, RunIgnored::Yes); | |
479 | } | |
480 | ||
481 | #[test] | |
482 | pub fn filter_for_ignored_option() { | |
483 | // When we run ignored tests the test filter should filter out all the | |
484 | // unignored tests and flip the ignore flag on the rest to false | |
485 | ||
486 | let mut opts = TestOpts::new(); | |
487 | opts.run_tests = true; | |
488 | opts.run_ignored = RunIgnored::Only; | |
489 | ||
490 | let tests = one_ignored_one_unignored_test(); | |
491 | let filtered = filter_tests(&opts, tests); | |
492 | ||
493 | assert_eq!(filtered.len(), 1); | |
494 | assert_eq!(filtered[0].desc.name.to_string(), "1"); | |
495 | assert!(!filtered[0].desc.ignore); | |
496 | } | |
497 | ||
498 | #[test] | |
499 | pub fn run_include_ignored_option() { | |
500 | // When we "--include-ignored" tests, the ignore flag should be set to false on | |
501 | // all tests and no test filtered out | |
502 | ||
503 | let mut opts = TestOpts::new(); | |
504 | opts.run_tests = true; | |
505 | opts.run_ignored = RunIgnored::Yes; | |
506 | ||
507 | let tests = one_ignored_one_unignored_test(); | |
508 | let filtered = filter_tests(&opts, tests); | |
509 | ||
510 | assert_eq!(filtered.len(), 2); | |
511 | assert!(!filtered[0].desc.ignore); | |
512 | assert!(!filtered[1].desc.ignore); | |
513 | } | |
514 | ||
515 | #[test] | |
516 | pub fn exclude_should_panic_option() { | |
517 | let mut opts = TestOpts::new(); | |
518 | opts.run_tests = true; | |
519 | opts.exclude_should_panic = true; | |
520 | ||
521 | let mut tests = one_ignored_one_unignored_test(); | |
522 | tests.push(TestDescAndFn { | |
523 | desc: TestDesc { | |
524 | name: StaticTestName("3"), | |
525 | ignore: false, | |
5e7ed085 | 526 | ignore_message: None, |
353b0b11 | 527 | source_file: "", |
353b0b11 | 528 | start_line: 0, |
353b0b11 | 529 | start_col: 0, |
353b0b11 | 530 | end_line: 0, |
353b0b11 | 531 | end_col: 0, |
dc9dc135 | 532 | should_panic: ShouldPanic::Yes, |
17df50a5 | 533 | compile_fail: false, |
17df50a5 | 534 | no_run: false, |
e74abb32 | 535 | test_type: TestType::Unknown, |
dc9dc135 | 536 | }, |
2b03887a | 537 | testfn: DynTestFn(Box::new(move || Ok(()))), |
dc9dc135 XL |
538 | }); |
539 | ||
540 | let filtered = filter_tests(&opts, tests); | |
541 | ||
542 | assert_eq!(filtered.len(), 2); | |
543 | assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No)); | |
544 | } | |
545 | ||
546 | #[test] | |
547 | pub fn exact_filter_match() { | |
548 | fn tests() -> Vec<TestDescAndFn> { | |
5099ac24 | 549 | ["base", "base::test", "base::test1", "base::test2"] |
dc9dc135 XL |
550 | .into_iter() |
551 | .map(|name| TestDescAndFn { | |
552 | desc: TestDesc { | |
553 | name: StaticTestName(name), | |
554 | ignore: false, | |
5e7ed085 | 555 | ignore_message: None, |
353b0b11 | 556 | source_file: "", |
353b0b11 | 557 | start_line: 0, |
353b0b11 | 558 | start_col: 0, |
353b0b11 | 559 | end_line: 0, |
353b0b11 | 560 | end_col: 0, |
dc9dc135 | 561 | should_panic: ShouldPanic::No, |
17df50a5 | 562 | compile_fail: false, |
17df50a5 | 563 | no_run: false, |
e74abb32 | 564 | test_type: TestType::Unknown, |
dc9dc135 | 565 | }, |
2b03887a | 566 | testfn: DynTestFn(Box::new(move || Ok(()))), |
dc9dc135 XL |
567 | }) |
568 | .collect() | |
569 | } | |
570 | ||
dfeec247 | 571 | let substr = |
6a06907d | 572 | filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests()); |
dc9dc135 XL |
573 | assert_eq!(substr.len(), 4); |
574 | ||
6a06907d XL |
575 | let substr = |
576 | filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests()); | |
dc9dc135 XL |
577 | assert_eq!(substr.len(), 4); |
578 | ||
dfeec247 | 579 | let substr = |
6a06907d | 580 | filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests()); |
dc9dc135 XL |
581 | assert_eq!(substr.len(), 3); |
582 | ||
dfeec247 | 583 | let substr = |
6a06907d | 584 | filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests()); |
dc9dc135 XL |
585 | assert_eq!(substr.len(), 3); |
586 | ||
6a06907d XL |
587 | let substr = filter_tests( |
588 | &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() }, | |
589 | tests(), | |
590 | ); | |
591 | assert_eq!(substr.len(), 2); | |
592 | ||
dc9dc135 | 593 | let exact = filter_tests( |
6a06907d | 594 | &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() }, |
dc9dc135 XL |
595 | tests(), |
596 | ); | |
597 | assert_eq!(exact.len(), 1); | |
598 | ||
599 | let exact = filter_tests( | |
6a06907d | 600 | &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() }, |
dc9dc135 XL |
601 | tests(), |
602 | ); | |
603 | assert_eq!(exact.len(), 0); | |
604 | ||
605 | let exact = filter_tests( | |
6a06907d | 606 | &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() }, |
dc9dc135 XL |
607 | tests(), |
608 | ); | |
609 | assert_eq!(exact.len(), 0); | |
610 | ||
611 | let exact = filter_tests( | |
6a06907d | 612 | &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() }, |
dc9dc135 XL |
613 | tests(), |
614 | ); | |
615 | assert_eq!(exact.len(), 1); | |
6a06907d XL |
616 | |
617 | let exact = filter_tests( | |
618 | &TestOpts { | |
619 | filters: vec!["base".into(), "base::test".into()], | |
620 | filter_exact: true, | |
621 | ..TestOpts::new() | |
622 | }, | |
623 | tests(), | |
624 | ); | |
625 | assert_eq!(exact.len(), 2); | |
dc9dc135 XL |
626 | } |
627 | ||
c295e0f8 | 628 | fn sample_tests() -> Vec<TestDescAndFn> { |
dc9dc135 XL |
629 | let names = vec![ |
630 | "sha1::test".to_string(), | |
631 | "isize::test_to_str".to_string(), | |
632 | "isize::test_pow".to_string(), | |
633 | "test::do_not_run_ignored_tests".to_string(), | |
634 | "test::ignored_tests_result_in_ignored".to_string(), | |
635 | "test::first_free_arg_should_be_a_filter".to_string(), | |
636 | "test::parse_ignored_flag".to_string(), | |
637 | "test::parse_include_ignored_flag".to_string(), | |
638 | "test::filter_for_ignored_option".to_string(), | |
639 | "test::run_include_ignored_option".to_string(), | |
640 | "test::sort_tests".to_string(), | |
641 | ]; | |
2b03887a FG |
642 | fn testfn() -> Result<(), String> { |
643 | Ok(()) | |
644 | } | |
c295e0f8 XL |
645 | let mut tests = Vec::new(); |
646 | for name in &names { | |
647 | let test = TestDescAndFn { | |
648 | desc: TestDesc { | |
649 | name: DynTestName((*name).clone()), | |
650 | ignore: false, | |
5e7ed085 | 651 | ignore_message: None, |
353b0b11 | 652 | source_file: "", |
353b0b11 | 653 | start_line: 0, |
353b0b11 | 654 | start_col: 0, |
353b0b11 | 655 | end_line: 0, |
353b0b11 | 656 | end_col: 0, |
c295e0f8 | 657 | should_panic: ShouldPanic::No, |
c295e0f8 XL |
658 | compile_fail: false, |
659 | no_run: false, | |
660 | test_type: TestType::Unknown, | |
661 | }, | |
662 | testfn: DynTestFn(Box::new(testfn)), | |
663 | }; | |
664 | tests.push(test); | |
665 | } | |
666 | tests | |
667 | } | |
668 | ||
c295e0f8 XL |
669 | #[test] |
670 | pub fn shuffle_tests() { | |
671 | let mut opts = TestOpts::new(); | |
672 | opts.shuffle = true; | |
673 | ||
674 | let shuffle_seed = get_shuffle_seed(&opts).unwrap(); | |
675 | ||
676 | let left = | |
677 | sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>(); | |
678 | let mut right = | |
679 | sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>(); | |
680 | ||
681 | assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name)); | |
682 | ||
683 | helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); | |
684 | ||
685 | assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name)); | |
686 | } | |
687 | ||
688 | #[test] | |
689 | pub fn shuffle_tests_with_seed() { | |
690 | let mut opts = TestOpts::new(); | |
691 | opts.shuffle = true; | |
692 | ||
693 | let shuffle_seed = get_shuffle_seed(&opts).unwrap(); | |
694 | ||
695 | let mut left = | |
696 | sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>(); | |
697 | let mut right = | |
698 | sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>(); | |
699 | ||
700 | helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice()); | |
701 | helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); | |
702 | ||
703 | assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name)); | |
704 | } | |
705 | ||
706 | #[test] | |
707 | pub fn order_depends_on_more_than_seed() { | |
708 | let mut opts = TestOpts::new(); | |
709 | opts.shuffle = true; | |
710 | ||
711 | let shuffle_seed = get_shuffle_seed(&opts).unwrap(); | |
712 | ||
713 | let mut left_tests = sample_tests(); | |
714 | let mut right_tests = sample_tests(); | |
715 | ||
716 | left_tests.pop(); | |
717 | right_tests.remove(0); | |
718 | ||
719 | let mut left = | |
720 | left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>(); | |
721 | let mut right = | |
722 | right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>(); | |
723 | ||
724 | assert_eq!(left.len(), right.len()); | |
725 | ||
726 | assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0)); | |
727 | ||
728 | helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice()); | |
729 | helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice()); | |
730 | ||
731 | assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0)); | |
732 | } | |
733 | ||
dc9dc135 XL |
734 | #[test] |
735 | pub fn test_metricmap_compare() { | |
736 | let mut m1 = MetricMap::new(); | |
737 | let mut m2 = MetricMap::new(); | |
738 | m1.insert_metric("in-both-noise", 1000.0, 200.0); | |
739 | m2.insert_metric("in-both-noise", 1100.0, 200.0); | |
740 | ||
741 | m1.insert_metric("in-first-noise", 1000.0, 2.0); | |
742 | m2.insert_metric("in-second-noise", 1000.0, 2.0); | |
743 | ||
744 | m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0); | |
745 | m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0); | |
746 | ||
747 | m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0); | |
748 | m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0); | |
749 | ||
750 | m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0); | |
751 | m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0); | |
752 | ||
753 | m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0); | |
754 | m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0); | |
755 | } | |
756 | ||
757 | #[test] | |
758 | pub fn test_bench_once_no_iter() { | |
2b03887a FG |
759 | fn f(_: &mut Bencher) -> Result<(), String> { |
760 | Ok(()) | |
761 | } | |
762 | bench::run_once(f).unwrap(); | |
dc9dc135 XL |
763 | } |
764 | ||
765 | #[test] | |
766 | pub fn test_bench_once_iter() { | |
2b03887a FG |
767 | fn f(b: &mut Bencher) -> Result<(), String> { |
768 | b.iter(|| {}); | |
769 | Ok(()) | |
dc9dc135 | 770 | } |
2b03887a | 771 | bench::run_once(f).unwrap(); |
dc9dc135 XL |
772 | } |
773 | ||
774 | #[test] | |
775 | pub fn test_bench_no_iter() { | |
2b03887a FG |
776 | fn f(_: &mut Bencher) -> Result<(), String> { |
777 | Ok(()) | |
778 | } | |
dc9dc135 XL |
779 | |
780 | let (tx, rx) = channel(); | |
781 | ||
782 | let desc = TestDesc { | |
783 | name: StaticTestName("f"), | |
784 | ignore: false, | |
5e7ed085 | 785 | ignore_message: None, |
353b0b11 | 786 | source_file: "", |
353b0b11 | 787 | start_line: 0, |
353b0b11 | 788 | start_col: 0, |
353b0b11 | 789 | end_line: 0, |
353b0b11 | 790 | end_col: 0, |
dc9dc135 | 791 | should_panic: ShouldPanic::No, |
17df50a5 | 792 | compile_fail: false, |
17df50a5 | 793 | no_run: false, |
e74abb32 | 794 | test_type: TestType::Unknown, |
dc9dc135 XL |
795 | }; |
796 | ||
cdc7bbd5 | 797 | crate::bench::benchmark(TestId(0), desc, tx, true, f); |
dc9dc135 XL |
798 | rx.recv().unwrap(); |
799 | } | |
800 | ||
801 | #[test] | |
802 | pub fn test_bench_iter() { | |
2b03887a FG |
803 | fn f(b: &mut Bencher) -> Result<(), String> { |
804 | b.iter(|| {}); | |
805 | Ok(()) | |
dc9dc135 XL |
806 | } |
807 | ||
808 | let (tx, rx) = channel(); | |
809 | ||
810 | let desc = TestDesc { | |
811 | name: StaticTestName("f"), | |
812 | ignore: false, | |
5e7ed085 | 813 | ignore_message: None, |
353b0b11 | 814 | source_file: "", |
353b0b11 | 815 | start_line: 0, |
353b0b11 | 816 | start_col: 0, |
353b0b11 | 817 | end_line: 0, |
353b0b11 | 818 | end_col: 0, |
dc9dc135 | 819 | should_panic: ShouldPanic::No, |
17df50a5 | 820 | compile_fail: false, |
17df50a5 | 821 | no_run: false, |
e74abb32 | 822 | test_type: TestType::Unknown, |
dc9dc135 XL |
823 | }; |
824 | ||
cdc7bbd5 | 825 | crate::bench::benchmark(TestId(0), desc, tx, true, f); |
dc9dc135 XL |
826 | rx.recv().unwrap(); |
827 | } | |
416331ca XL |
828 | |
829 | #[test] | |
830 | fn should_sort_failures_before_printing_them() { | |
831 | let test_a = TestDesc { | |
832 | name: StaticTestName("a"), | |
833 | ignore: false, | |
5e7ed085 | 834 | ignore_message: None, |
353b0b11 | 835 | source_file: "", |
353b0b11 | 836 | start_line: 0, |
353b0b11 | 837 | start_col: 0, |
353b0b11 | 838 | end_line: 0, |
353b0b11 | 839 | end_col: 0, |
416331ca | 840 | should_panic: ShouldPanic::No, |
17df50a5 | 841 | compile_fail: false, |
17df50a5 | 842 | no_run: false, |
e74abb32 | 843 | test_type: TestType::Unknown, |
416331ca XL |
844 | }; |
845 | ||
846 | let test_b = TestDesc { | |
847 | name: StaticTestName("b"), | |
848 | ignore: false, | |
5e7ed085 | 849 | ignore_message: None, |
353b0b11 | 850 | source_file: "", |
353b0b11 | 851 | start_line: 0, |
353b0b11 | 852 | start_col: 0, |
353b0b11 | 853 | end_line: 0, |
353b0b11 | 854 | end_col: 0, |
416331ca | 855 | should_panic: ShouldPanic::No, |
17df50a5 | 856 | compile_fail: false, |
17df50a5 | 857 | no_run: false, |
e74abb32 | 858 | test_type: TestType::Unknown, |
416331ca XL |
859 | }; |
860 | ||
e74abb32 | 861 | let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None); |
416331ca | 862 | |
e74abb32 | 863 | let st = console::ConsoleTestState { |
416331ca XL |
864 | log_out: None, |
865 | total: 0, | |
866 | passed: 0, | |
867 | failed: 0, | |
868 | ignored: 0, | |
416331ca XL |
869 | filtered_out: 0, |
870 | measured: 0, | |
fc512014 | 871 | exec_time: None, |
416331ca XL |
872 | metrics: MetricMap::new(), |
873 | failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], | |
874 | options: Options::new(), | |
875 | not_failures: Vec::new(), | |
9ffffee4 | 876 | ignores: Vec::new(), |
e74abb32 | 877 | time_failures: Vec::new(), |
416331ca XL |
878 | }; |
879 | ||
880 | out.write_failures(&st).unwrap(); | |
881 | let s = match out.output_location() { | |
e74abb32 XL |
882 | &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]), |
883 | &OutputLocation::Pretty(_) => unreachable!(), | |
416331ca XL |
884 | }; |
885 | ||
886 | let apos = s.find("a").unwrap(); | |
887 | let bpos = s.find("b").unwrap(); | |
888 | assert!(apos < bpos); | |
889 | } | |
2b03887a FG |
890 | |
891 | #[test] | |
892 | #[cfg(not(target_os = "emscripten"))] | |
893 | fn test_dyn_bench_returning_err_fails_when_run_as_test() { | |
894 | fn f(_: &mut Bencher) -> Result<(), String> { | |
895 | Result::Err("An error".into()) | |
896 | } | |
897 | let desc = TestDescAndFn { | |
898 | desc: TestDesc { | |
899 | name: StaticTestName("whatever"), | |
900 | ignore: false, | |
901 | ignore_message: None, | |
353b0b11 | 902 | source_file: "", |
353b0b11 | 903 | start_line: 0, |
353b0b11 | 904 | start_col: 0, |
353b0b11 | 905 | end_line: 0, |
353b0b11 | 906 | end_col: 0, |
2b03887a FG |
907 | should_panic: ShouldPanic::No, |
908 | compile_fail: false, | |
909 | no_run: false, | |
910 | test_type: TestType::Unknown, | |
911 | }, | |
912 | testfn: DynBenchFn(Box::new(f)), | |
913 | }; | |
914 | let (tx, rx) = channel(); | |
915 | let notify = move |event: TestEvent| { | |
916 | if let TestEvent::TeResult(result) = event { | |
917 | tx.send(result).unwrap(); | |
918 | } | |
919 | Ok(()) | |
920 | }; | |
921 | run_tests(&TestOpts { run_tests: true, ..TestOpts::new() }, vec![desc], notify).unwrap(); | |
922 | let result = rx.recv().unwrap().result; | |
923 | assert_eq!(result, TrFailed); | |
924 | } |