]> git.proxmox.com Git - rustc.git/blob - library/test/src/cli.rs
Update upstream source from tag 'upstream/1.67.1+dfsg1'
[rustc.git] / library / test / src / cli.rs
1 //! Module converting command-line arguments into test configuration.
2
3 use std::env;
4 use std::path::PathBuf;
5
6 use super::options::{ColorConfig, Options, OutputFormat, RunIgnored};
7 use super::time::TestTimeOptions;
8 use std::io::{self, IsTerminal};
9
10 #[derive(Debug)]
11 pub struct TestOpts {
12 pub list: bool,
13 pub filters: Vec<String>,
14 pub filter_exact: bool,
15 pub force_run_in_process: bool,
16 pub exclude_should_panic: bool,
17 pub run_ignored: RunIgnored,
18 pub run_tests: bool,
19 pub bench_benchmarks: bool,
20 pub logfile: Option<PathBuf>,
21 pub nocapture: bool,
22 pub color: ColorConfig,
23 pub format: OutputFormat,
24 pub shuffle: bool,
25 pub shuffle_seed: Option<u64>,
26 pub test_threads: Option<usize>,
27 pub skip: Vec<String>,
28 pub time_options: Option<TestTimeOptions>,
29 /// Stop at first failing test.
30 /// May run a few more tests due to threading, but will
31 /// abort as soon as possible.
32 pub fail_fast: bool,
33 pub options: Options,
34 }
35
36 impl TestOpts {
37 pub fn use_color(&self) -> bool {
38 match self.color {
39 ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(),
40 ColorConfig::AlwaysColor => true,
41 ColorConfig::NeverColor => false,
42 }
43 }
44 }
45
46 /// Result of parsing the options.
47 pub type OptRes = Result<TestOpts, String>;
48 /// Result of parsing the option part.
49 type OptPartRes<T> = Result<T, String>;
50
51 fn optgroups() -> getopts::Options {
52 let mut opts = getopts::Options::new();
53 opts.optflag("", "include-ignored", "Run ignored and not ignored tests")
54 .optflag("", "ignored", "Run only ignored tests")
55 .optflag("", "force-run-in-process", "Forces tests to run in-process when panic=abort")
56 .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic")
57 .optflag("", "test", "Run tests and not benchmarks")
58 .optflag("", "bench", "Run benchmarks instead of tests")
59 .optflag("", "list", "List all tests and benchmarks")
60 .optflag("h", "help", "Display this message")
61 .optopt("", "logfile", "Write logs to the specified file", "PATH")
62 .optflag(
63 "",
64 "nocapture",
65 "don't capture stdout/stderr of each \
66 task, allow printing directly",
67 )
68 .optopt(
69 "",
70 "test-threads",
71 "Number of threads used for running tests \
72 in parallel",
73 "n_threads",
74 )
75 .optmulti(
76 "",
77 "skip",
78 "Skip tests whose names contain FILTER (this flag can \
79 be used multiple times)",
80 "FILTER",
81 )
82 .optflag(
83 "q",
84 "quiet",
85 "Display one character per test instead of one line. \
86 Alias to --format=terse",
87 )
88 .optflag("", "exact", "Exactly match filters rather than by substring")
89 .optopt(
90 "",
91 "color",
92 "Configure coloring of output:
93 auto = colorize if stdout is a tty and tests are run on serially (default);
94 always = always colorize output;
95 never = never colorize output;",
96 "auto|always|never",
97 )
98 .optopt(
99 "",
100 "format",
101 "Configure formatting of output:
102 pretty = Print verbose output;
103 terse = Display one character per test;
104 json = Output a json document;
105 junit = Output a JUnit document",
106 "pretty|terse|json|junit",
107 )
108 .optflag("", "show-output", "Show captured stdout of successful tests")
109 .optopt(
110 "Z",
111 "",
112 "Enable nightly-only flags:
113 unstable-options = Allow use of experimental features",
114 "unstable-options",
115 )
116 .optflag(
117 "",
118 "report-time",
119 "Show execution time of each test.
120
121 Threshold values for colorized output can be configured via
122 `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
123 `RUST_TEST_TIME_DOCTEST` environment variables.
124
125 Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
126 Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time
127 is 0.5 seconds, and the critical time is 2 seconds.
128
129 Not available for --format=terse",
130 )
131 .optflag(
132 "",
133 "ensure-time",
134 "Treat excess of the test execution time limit as error.
135
136 Threshold values for this option can be configured via
137 `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
138 `RUST_TEST_TIME_DOCTEST` environment variables.
139
140 Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
141
142 `CRITICAL_TIME` here means the limit that should not be exceeded by test.
143 ",
144 )
145 .optflag("", "shuffle", "Run tests in random order")
146 .optopt(
147 "",
148 "shuffle-seed",
149 "Run tests in random order; seed the random number generator with SEED",
150 "SEED",
151 );
152 opts
153 }
154
155 fn usage(binary: &str, options: &getopts::Options) {
156 let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]");
157 println!(
158 r#"{usage}
159
160 The FILTER string is tested against the name of all tests, and only those
161 tests whose names contain the filter are run. Multiple filter strings may
162 be passed, which will run all tests matching any of the filters.
163
164 By default, all tests are run in parallel. This can be altered with the
165 --test-threads flag or the RUST_TEST_THREADS environment variable when running
166 tests (set it to 1).
167
168 By default, the tests are run in alphabetical order. Use --shuffle or set
169 RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated
170 "shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the
171 tests in the same order again. Note that --shuffle and --shuffle-seed do not
172 affect whether the tests are run in parallel.
173
174 All tests have their standard output and standard error captured by default.
175 This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE
176 environment variable to a value other than "0". Logging is not captured by default.
177
178 Test Attributes:
179
180 `#[test]` - Indicates a function is a test to be run. This function
181 takes no arguments.
182 `#[bench]` - Indicates a function is a benchmark to be run. This
183 function takes one argument (test::Bencher).
184 `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if
185 the code causes a panic (an assertion failure or panic!)
186 A message may be provided, which the failure string must
187 contain: #[should_panic(expected = "foo")].
188 `#[ignore]` - When applied to a function which is already attributed as a
189 test, then the test runner will ignore these tests during
190 normal test runs. Running with --ignored or --include-ignored will run
191 these tests."#,
192 usage = options.usage(&message)
193 );
194 }
195
196 /// Parses command line arguments into test options.
197 /// Returns `None` if help was requested (since we only show help message and don't run tests),
198 /// returns `Some(Err(..))` if provided arguments are incorrect,
199 /// otherwise creates a `TestOpts` object and returns it.
200 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
201 // Parse matches.
202 let opts = optgroups();
203 let binary = args.get(0).map(|c| &**c).unwrap_or("...");
204 let args = args.get(1..).unwrap_or(args);
205 let matches = match opts.parse(args) {
206 Ok(m) => m,
207 Err(f) => return Some(Err(f.to_string())),
208 };
209
210 // Check if help was requested.
211 if matches.opt_present("h") {
212 // Show help and do nothing more.
213 usage(binary, &opts);
214 return None;
215 }
216
217 // Actually parse the opts.
218 let opts_result = parse_opts_impl(matches);
219
220 Some(opts_result)
221 }
222
223 // Gets the option value and checks if unstable features are enabled.
224 macro_rules! unstable_optflag {
225 ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
226 let opt = $matches.opt_present($option_name);
227 if !$allow_unstable && opt {
228 return Err(format!(
229 "The \"{}\" flag is only accepted on the nightly compiler with -Z unstable-options",
230 $option_name
231 ));
232 }
233
234 opt
235 }};
236 }
237
238 // Gets the option value and checks if unstable features are enabled.
239 macro_rules! unstable_optopt {
240 ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
241 let opt = $matches.opt_str($option_name);
242 if !$allow_unstable && opt.is_some() {
243 return Err(format!(
244 "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options",
245 $option_name
246 ));
247 }
248
249 opt
250 }};
251 }
252
253 // Implementation of `parse_opts` that doesn't care about help message
254 // and returns a `Result`.
255 fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
256 let allow_unstable = get_allow_unstable(&matches)?;
257
258 // Unstable flags
259 let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process");
260 let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic");
261 let time_options = get_time_options(&matches, allow_unstable)?;
262 let shuffle = get_shuffle(&matches, allow_unstable)?;
263 let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?;
264
265 let include_ignored = matches.opt_present("include-ignored");
266 let quiet = matches.opt_present("quiet");
267 let exact = matches.opt_present("exact");
268 let list = matches.opt_present("list");
269 let skip = matches.opt_strs("skip");
270
271 let bench_benchmarks = matches.opt_present("bench");
272 let run_tests = !bench_benchmarks || matches.opt_present("test");
273
274 let logfile = get_log_file(&matches)?;
275 let run_ignored = get_run_ignored(&matches, include_ignored)?;
276 let filters = matches.free.clone();
277 let nocapture = get_nocapture(&matches)?;
278 let test_threads = get_test_threads(&matches)?;
279 let color = get_color_config(&matches)?;
280 let format = get_format(&matches, quiet, allow_unstable)?;
281
282 let options = Options::new().display_output(matches.opt_present("show-output"));
283
284 let test_opts = TestOpts {
285 list,
286 filters,
287 filter_exact: exact,
288 force_run_in_process,
289 exclude_should_panic,
290 run_ignored,
291 run_tests,
292 bench_benchmarks,
293 logfile,
294 nocapture,
295 color,
296 format,
297 shuffle,
298 shuffle_seed,
299 test_threads,
300 skip,
301 time_options,
302 options,
303 fail_fast: false,
304 };
305
306 Ok(test_opts)
307 }
308
309 // FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566
310 fn is_nightly() -> bool {
311 // Whether this is a feature-staged build, i.e., on the beta or stable channel
312 let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some();
313 // Whether we should enable unstable features for bootstrapping
314 let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
315
316 bootstrap || !disable_unstable_features
317 }
318
319 // Gets the CLI options associated with `report-time` feature.
320 fn get_time_options(
321 matches: &getopts::Matches,
322 allow_unstable: bool,
323 ) -> OptPartRes<Option<TestTimeOptions>> {
324 let report_time = unstable_optflag!(matches, allow_unstable, "report-time");
325 let ensure_test_time = unstable_optflag!(matches, allow_unstable, "ensure-time");
326
327 // If `ensure-test-time` option is provided, time output is enforced,
328 // so user won't be confused if any of tests will silently fail.
329 let options = if report_time || ensure_test_time {
330 Some(TestTimeOptions::new_from_env(ensure_test_time))
331 } else {
332 None
333 };
334
335 Ok(options)
336 }
337
338 fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<bool> {
339 let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle");
340 if !shuffle && allow_unstable {
341 shuffle = match env::var("RUST_TEST_SHUFFLE") {
342 Ok(val) => &val != "0",
343 Err(_) => false,
344 };
345 }
346
347 Ok(shuffle)
348 }
349
350 fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<Option<u64>> {
351 let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") {
352 Some(n_str) => match n_str.parse::<u64>() {
353 Ok(n) => Some(n),
354 Err(e) => {
355 return Err(format!(
356 "argument for --shuffle-seed must be a number \
357 (error: {})",
358 e
359 ));
360 }
361 },
362 None => None,
363 };
364
365 if shuffle_seed.is_none() && allow_unstable {
366 shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") {
367 Ok(val) => match val.parse::<u64>() {
368 Ok(n) => Some(n),
369 Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number."),
370 },
371 Err(_) => None,
372 };
373 }
374
375 Ok(shuffle_seed)
376 }
377
378 fn get_test_threads(matches: &getopts::Matches) -> OptPartRes<Option<usize>> {
379 let test_threads = match matches.opt_str("test-threads") {
380 Some(n_str) => match n_str.parse::<usize>() {
381 Ok(0) => return Err("argument for --test-threads must not be 0".to_string()),
382 Ok(n) => Some(n),
383 Err(e) => {
384 return Err(format!(
385 "argument for --test-threads must be a number > 0 \
386 (error: {})",
387 e
388 ));
389 }
390 },
391 None => None,
392 };
393
394 Ok(test_threads)
395 }
396
397 fn get_format(
398 matches: &getopts::Matches,
399 quiet: bool,
400 allow_unstable: bool,
401 ) -> OptPartRes<OutputFormat> {
402 let format = match matches.opt_str("format").as_deref() {
403 None if quiet => OutputFormat::Terse,
404 Some("pretty") | None => OutputFormat::Pretty,
405 Some("terse") => OutputFormat::Terse,
406 Some("json") => {
407 if !allow_unstable {
408 return Err("The \"json\" format is only accepted on the nightly compiler".into());
409 }
410 OutputFormat::Json
411 }
412 Some("junit") => {
413 if !allow_unstable {
414 return Err("The \"junit\" format is only accepted on the nightly compiler".into());
415 }
416 OutputFormat::Junit
417 }
418 Some(v) => {
419 return Err(format!(
420 "argument for --format must be pretty, terse, json or junit (was \
421 {})",
422 v
423 ));
424 }
425 };
426
427 Ok(format)
428 }
429
430 fn get_color_config(matches: &getopts::Matches) -> OptPartRes<ColorConfig> {
431 let color = match matches.opt_str("color").as_deref() {
432 Some("auto") | None => ColorConfig::AutoColor,
433 Some("always") => ColorConfig::AlwaysColor,
434 Some("never") => ColorConfig::NeverColor,
435
436 Some(v) => {
437 return Err(format!(
438 "argument for --color must be auto, always, or never (was \
439 {})",
440 v
441 ));
442 }
443 };
444
445 Ok(color)
446 }
447
448 fn get_nocapture(matches: &getopts::Matches) -> OptPartRes<bool> {
449 let mut nocapture = matches.opt_present("nocapture");
450 if !nocapture {
451 nocapture = match env::var("RUST_TEST_NOCAPTURE") {
452 Ok(val) => &val != "0",
453 Err(_) => false,
454 };
455 }
456
457 Ok(nocapture)
458 }
459
460 fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes<RunIgnored> {
461 let run_ignored = match (include_ignored, matches.opt_present("ignored")) {
462 (true, true) => {
463 return Err("the options --include-ignored and --ignored are mutually exclusive".into());
464 }
465 (true, false) => RunIgnored::Yes,
466 (false, true) => RunIgnored::Only,
467 (false, false) => RunIgnored::No,
468 };
469
470 Ok(run_ignored)
471 }
472
473 fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes<bool> {
474 let mut allow_unstable = false;
475
476 if let Some(opt) = matches.opt_str("Z") {
477 if !is_nightly() {
478 return Err("the option `Z` is only accepted on the nightly compiler".into());
479 }
480
481 match &*opt {
482 "unstable-options" => {
483 allow_unstable = true;
484 }
485 _ => {
486 return Err("Unrecognized option to `Z`".into());
487 }
488 }
489 };
490
491 Ok(allow_unstable)
492 }
493
494 fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
495 let logfile = matches.opt_str("logfile").map(|s| PathBuf::from(&s));
496
497 Ok(logfile)
498 }