library/test/src/cli.rs

   1 //! Module converting command-line arguments into test configuration.
   2
   3 use std::env;
   4 use std::path::PathBuf;
   5
   6 use super::options::{ColorConfig, Options, OutputFormat, RunIgnored};
   7 use super::time::TestTimeOptions;
   8 use std::io::{self, IsTerminal};
   9
  10 #[derive(Debug)]
  11 pub struct TestOpts {
  12     pub list: bool,
  13     pub filters: Vec<String>,
  14     pub filter_exact: bool,
  15     pub force_run_in_process: bool,
  16     pub exclude_should_panic: bool,
  17     pub run_ignored: RunIgnored,
  18     pub run_tests: bool,
  19     pub bench_benchmarks: bool,
  20     pub logfile: Option<PathBuf>,
  21     pub nocapture: bool,
  22     pub color: ColorConfig,
  23     pub format: OutputFormat,
  24     pub shuffle: bool,
  25     pub shuffle_seed: Option<u64>,
  26     pub test_threads: Option<usize>,
  27     pub skip: Vec<String>,
  28     pub time_options: Option<TestTimeOptions>,
  29     /// Stop at first failing test.
  30     /// May run a few more tests due to threading, but will
  31     /// abort as soon as possible.
  32     pub fail_fast: bool,
  33     pub options: Options,
  34 }
  35
  36 impl TestOpts {
  37     pub fn use_color(&self) -> bool {
  38         match self.color {
  39             ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(),
  40             ColorConfig::AlwaysColor => true,
  41             ColorConfig::NeverColor => false,
  42         }
  43     }
  44 }
  45
  46 /// Result of parsing the options.
  47 pub type OptRes = Result<TestOpts, String>;
  48 /// Result of parsing the option part.
  49 type OptPartRes<T> = Result<T, String>;
  50
  51 fn optgroups() -> getopts::Options {
  52     let mut opts = getopts::Options::new();
  53     opts.optflag("", "include-ignored", "Run ignored and not ignored tests")
  54         .optflag("", "ignored", "Run only ignored tests")
  55         .optflag("", "force-run-in-process", "Forces tests to run in-process when panic=abort")
  56         .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic")
  57         .optflag("", "test", "Run tests and not benchmarks")
  58         .optflag("", "bench", "Run benchmarks instead of tests")
  59         .optflag("", "list", "List all tests and benchmarks")
  60         .optflag("h", "help", "Display this message")
  61         .optopt("", "logfile", "Write logs to the specified file", "PATH")
  62         .optflag(
  63             "",
  64             "nocapture",
  65             "don't capture stdout/stderr of each \
  66              task, allow printing directly",
  67         )
  68         .optopt(
  69             "",
  70             "test-threads",
  71             "Number of threads used for running tests \
  72              in parallel",
  73             "n_threads",
  74         )
  75         .optmulti(
  76             "",
  77             "skip",
  78             "Skip tests whose names contain FILTER (this flag can \
  79              be used multiple times)",
  80             "FILTER",
  81         )
  82         .optflag(
  83             "q",
  84             "quiet",
  85             "Display one character per test instead of one line. \
  86              Alias to --format=terse",
  87         )
  88         .optflag("", "exact", "Exactly match filters rather than by substring")
  89         .optopt(
  90             "",
  91             "color",
  92             "Configure coloring of output:
  93             auto   = colorize if stdout is a tty and tests are run on serially (default);
  94             always = always colorize output;
  95             never  = never colorize output;",
  96             "auto|always|never",
  97         )
  98         .optopt(
  99             "",
 100             "format",
 101             "Configure formatting of output:
 102             pretty = Print verbose output;
 103             terse  = Display one character per test;
 104             json   = Output a json document;
 105             junit  = Output a JUnit document",
 106             "pretty|terse|json|junit",
 107         )
 108         .optflag("", "show-output", "Show captured stdout of successful tests")
 109         .optopt(
 110             "Z",
 111             "",
 112             "Enable nightly-only flags:
 113             unstable-options = Allow use of experimental features",
 114             "unstable-options",
 115         )
 116         .optflag(
 117             "",
 118             "report-time",
 119             "Show execution time of each test.
 120
 121             Threshold values for colorized output can be configured via
 122             `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
 123             `RUST_TEST_TIME_DOCTEST` environment variables.
 124
 125             Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
 126             Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time
 127             is 0.5 seconds, and the critical time is 2 seconds.
 128
 129             Not available for --format=terse",
 130         )
 131         .optflag(
 132             "",
 133             "ensure-time",
 134             "Treat excess of the test execution time limit as error.
 135
 136             Threshold values for this option can be configured via
 137             `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
 138             `RUST_TEST_TIME_DOCTEST` environment variables.
 139
 140             Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
 141
 142             `CRITICAL_TIME` here means the limit that should not be exceeded by test.
 143             ",
 144         )
 145         .optflag("", "shuffle", "Run tests in random order")
 146         .optopt(
 147             "",
 148             "shuffle-seed",
 149             "Run tests in random order; seed the random number generator with SEED",
 150             "SEED",
 151         );
 152     opts
 153 }
 154
 155 fn usage(binary: &str, options: &getopts::Options) {
 156     let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]");
 157     println!(
 158         r#"{usage}
 159
 160 The FILTER string is tested against the name of all tests, and only those
 161 tests whose names contain the filter are run. Multiple filter strings may
 162 be passed, which will run all tests matching any of the filters.
 163
 164 By default, all tests are run in parallel. This can be altered with the
 165 --test-threads flag or the RUST_TEST_THREADS environment variable when running
 166 tests (set it to 1).
 167
 168 By default, the tests are run in alphabetical order. Use --shuffle or set
 169 RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated
 170 "shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the
 171 tests in the same order again. Note that --shuffle and --shuffle-seed do not
 172 affect whether the tests are run in parallel.
 173
 174 All tests have their standard output and standard error captured by default.
 175 This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE
 176 environment variable to a value other than "0". Logging is not captured by default.
 177
 178 Test Attributes:
 179
 180     `#[test]`        - Indicates a function is a test to be run. This function
 181                        takes no arguments.
 182     `#[bench]`       - Indicates a function is a benchmark to be run. This
 183                        function takes one argument (test::Bencher).
 184     `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if
 185                         the code causes a panic (an assertion failure or panic!)
 186                         A message may be provided, which the failure string must
 187                         contain: #[should_panic(expected = "foo")].
 188     `#[ignore]`       - When applied to a function which is already attributed as a
 189                         test, then the test runner will ignore these tests during
 190                         normal test runs. Running with --ignored or --include-ignored will run
 191                         these tests."#,
 192         usage = options.usage(&message)
 193     );
 194 }
 195
 196 /// Parses command line arguments into test options.
 197 /// Returns `None` if help was requested (since we only show help message and don't run tests),
 198 /// returns `Some(Err(..))` if provided arguments are incorrect,
 199 /// otherwise creates a `TestOpts` object and returns it.
 200 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
 201     // Parse matches.
 202     let opts = optgroups();
 203     let binary = args.get(0).map(|c| &**c).unwrap_or("...");
 204     let args = args.get(1..).unwrap_or(args);
 205     let matches = match opts.parse(args) {
 206         Ok(m) => m,
 207         Err(f) => return Some(Err(f.to_string())),
 208     };
 209
 210     // Check if help was requested.
 211     if matches.opt_present("h") {
 212         // Show help and do nothing more.
 213         usage(binary, &opts);
 214         return None;
 215     }
 216
 217     // Actually parse the opts.
 218     let opts_result = parse_opts_impl(matches);
 219
 220     Some(opts_result)
 221 }
 222
 223 // Gets the option value and checks if unstable features are enabled.
 224 macro_rules! unstable_optflag {
 225     ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
 226         let opt = $matches.opt_present($option_name);
 227         if !$allow_unstable && opt {
 228             return Err(format!(
 229                 "The \"{}\" flag is only accepted on the nightly compiler with -Z unstable-options",
 230                 $option_name
 231             ));
 232         }
 233
 234         opt
 235     }};
 236 }
 237
 238 // Gets the option value and checks if unstable features are enabled.
 239 macro_rules! unstable_optopt {
 240     ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
 241         let opt = $matches.opt_str($option_name);
 242         if !$allow_unstable && opt.is_some() {
 243             return Err(format!(
 244                 "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options",
 245                 $option_name
 246             ));
 247         }
 248
 249         opt
 250     }};
 251 }
 252
 253 // Implementation of `parse_opts` that doesn't care about help message
 254 // and returns a `Result`.
 255 fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
 256     let allow_unstable = get_allow_unstable(&matches)?;
 257
 258     // Unstable flags
 259     let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process");
 260     let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic");
 261     let time_options = get_time_options(&matches, allow_unstable)?;
 262     let shuffle = get_shuffle(&matches, allow_unstable)?;
 263     let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?;
 264
 265     let include_ignored = matches.opt_present("include-ignored");
 266     let quiet = matches.opt_present("quiet");
 267     let exact = matches.opt_present("exact");
 268     let list = matches.opt_present("list");
 269     let skip = matches.opt_strs("skip");
 270
 271     let bench_benchmarks = matches.opt_present("bench");
 272     let run_tests = !bench_benchmarks || matches.opt_present("test");
 273
 274     let logfile = get_log_file(&matches)?;
 275     let run_ignored = get_run_ignored(&matches, include_ignored)?;
 276     let filters = matches.free.clone();
 277     let nocapture = get_nocapture(&matches)?;
 278     let test_threads = get_test_threads(&matches)?;
 279     let color = get_color_config(&matches)?;
 280     let format = get_format(&matches, quiet, allow_unstable)?;
 281
 282     let options = Options::new().display_output(matches.opt_present("show-output"));
 283
 284     let test_opts = TestOpts {
 285         list,
 286         filters,
 287         filter_exact: exact,
 288         force_run_in_process,
 289         exclude_should_panic,
 290         run_ignored,
 291         run_tests,
 292         bench_benchmarks,
 293         logfile,
 294         nocapture,
 295         color,
 296         format,
 297         shuffle,
 298         shuffle_seed,
 299         test_threads,
 300         skip,
 301         time_options,
 302         options,
 303         fail_fast: false,
 304     };
 305
 306     Ok(test_opts)
 307 }
 308
 309 // FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566
 310 fn is_nightly() -> bool {
 311     // Whether this is a feature-staged build, i.e., on the beta or stable channel
 312     let disable_unstable_features =
 313         option_env!("CFG_DISABLE_UNSTABLE_FEATURES").map(|s| s != "0").unwrap_or(false);
 314     // Whether we should enable unstable features for bootstrapping
 315     let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
 316
 317     bootstrap || !disable_unstable_features
 318 }
 319
 320 // Gets the CLI options associated with `report-time` feature.
 321 fn get_time_options(
 322     matches: &getopts::Matches,
 323     allow_unstable: bool,
 324 ) -> OptPartRes<Option<TestTimeOptions>> {
 325     let report_time = unstable_optflag!(matches, allow_unstable, "report-time");
 326     let ensure_test_time = unstable_optflag!(matches, allow_unstable, "ensure-time");
 327
 328     // If `ensure-test-time` option is provided, time output is enforced,
 329     // so user won't be confused if any of tests will silently fail.
 330     let options = if report_time || ensure_test_time {
 331         Some(TestTimeOptions::new_from_env(ensure_test_time))
 332     } else {
 333         None
 334     };
 335
 336     Ok(options)
 337 }
 338
 339 fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<bool> {
 340     let mut shuffle = unstable_optflag!(matches, allow_unstable, "shuffle");
 341     if !shuffle && allow_unstable {
 342         shuffle = match env::var("RUST_TEST_SHUFFLE") {
 343             Ok(val) => &val != "0",
 344             Err(_) => false,
 345         };
 346     }
 347
 348     Ok(shuffle)
 349 }
 350
 351 fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<Option<u64>> {
 352     let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") {
 353         Some(n_str) => match n_str.parse::<u64>() {
 354             Ok(n) => Some(n),
 355             Err(e) => {
 356                 return Err(format!(
 357                     "argument for --shuffle-seed must be a number \
 358                      (error: {e})"
 359                 ));
 360             }
 361         },
 362         None => None,
 363     };
 364
 365     if shuffle_seed.is_none() && allow_unstable {
 366         shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") {
 367             Ok(val) => match val.parse::<u64>() {
 368                 Ok(n) => Some(n),
 369                 Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number."),
 370             },
 371             Err(_) => None,
 372         };
 373     }
 374
 375     Ok(shuffle_seed)
 376 }
 377
 378 fn get_test_threads(matches: &getopts::Matches) -> OptPartRes<Option<usize>> {
 379     let test_threads = match matches.opt_str("test-threads") {
 380         Some(n_str) => match n_str.parse::<usize>() {
 381             Ok(0) => return Err("argument for --test-threads must not be 0".to_string()),
 382             Ok(n) => Some(n),
 383             Err(e) => {
 384                 return Err(format!(
 385                     "argument for --test-threads must be a number > 0 \
 386                      (error: {e})"
 387                 ));
 388             }
 389         },
 390         None => None,
 391     };
 392
 393     Ok(test_threads)
 394 }
 395
 396 fn get_format(
 397     matches: &getopts::Matches,
 398     quiet: bool,
 399     allow_unstable: bool,
 400 ) -> OptPartRes<OutputFormat> {
 401     let format = match matches.opt_str("format").as_deref() {
 402         None if quiet => OutputFormat::Terse,
 403         Some("pretty") | None => OutputFormat::Pretty,
 404         Some("terse") => OutputFormat::Terse,
 405         Some("json") => {
 406             if !allow_unstable {
 407                 return Err("The \"json\" format is only accepted on the nightly compiler with -Z unstable-options".into());
 408             }
 409             OutputFormat::Json
 410         }
 411         Some("junit") => {
 412             if !allow_unstable {
 413                 return Err("The \"junit\" format is only accepted on the nightly compiler with -Z unstable-options".into());
 414             }
 415             OutputFormat::Junit
 416         }
 417         Some(v) => {
 418             return Err(format!(
 419                 "argument for --format must be pretty, terse, json or junit (was \
 420                  {v})"
 421             ));
 422         }
 423     };
 424
 425     Ok(format)
 426 }
 427
 428 fn get_color_config(matches: &getopts::Matches) -> OptPartRes<ColorConfig> {
 429     let color = match matches.opt_str("color").as_deref() {
 430         Some("auto") | None => ColorConfig::AutoColor,
 431         Some("always") => ColorConfig::AlwaysColor,
 432         Some("never") => ColorConfig::NeverColor,
 433
 434         Some(v) => {
 435             return Err(format!(
 436                 "argument for --color must be auto, always, or never (was \
 437                  {v})"
 438             ));
 439         }
 440     };
 441
 442     Ok(color)
 443 }
 444
 445 fn get_nocapture(matches: &getopts::Matches) -> OptPartRes<bool> {
 446     let mut nocapture = matches.opt_present("nocapture");
 447     if !nocapture {
 448         nocapture = match env::var("RUST_TEST_NOCAPTURE") {
 449             Ok(val) => &val != "0",
 450             Err(_) => false,
 451         };
 452     }
 453
 454     Ok(nocapture)
 455 }
 456
 457 fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes<RunIgnored> {
 458     let run_ignored = match (include_ignored, matches.opt_present("ignored")) {
 459         (true, true) => {
 460             return Err("the options --include-ignored and --ignored are mutually exclusive".into());
 461         }
 462         (true, false) => RunIgnored::Yes,
 463         (false, true) => RunIgnored::Only,
 464         (false, false) => RunIgnored::No,
 465     };
 466
 467     Ok(run_ignored)
 468 }
 469
 470 fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes<bool> {
 471     let mut allow_unstable = false;
 472
 473     if let Some(opt) = matches.opt_str("Z") {
 474         if !is_nightly() {
 475             return Err("the option `Z` is only accepted on the nightly compiler".into());
 476         }
 477
 478         match &*opt {
 479             "unstable-options" => {
 480                 allow_unstable = true;
 481             }
 482             _ => {
 483                 return Err("Unrecognized option to `Z`".into());
 484             }
 485         }
 486     };
 487
 488     Ok(allow_unstable)
 489 }
 490
 491 fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
 492     let logfile = matches.opt_str("logfile").map(|s| PathBuf::from(&s));
 493
 494     Ok(logfile)
 495 }