src/tools/clippy/lintcheck/src/main.rs

   1 // Run clippy on a fixed set of crates and collect the warnings.
   2 // This helps observing the impact clippy changes have on a set of real-world code (and not just our
   3 // testsuite).
   4 //
   5 // When a new lint is introduced, we can search the results for new warnings and check for false
   6 // positives.
   7
   8 #![allow(clippy::collapsible_else_if)]
   9
  10 use std::ffi::OsStr;
  11 use std::process::Command;
  12 use std::sync::atomic::{AtomicUsize, Ordering};
  13 use std::{collections::HashMap, io::ErrorKind};
  14 use std::{
  15     env, fmt,
  16     fs::write,
  17     path::{Path, PathBuf},
  18 };
  19
  20 use clap::{App, Arg, ArgMatches};
  21 use rayon::prelude::*;
  22 use serde::{Deserialize, Serialize};
  23 use serde_json::Value;
  24
  25 #[cfg(not(windows))]
  26 const CLIPPY_DRIVER_PATH: &str = "target/debug/clippy-driver";
  27 #[cfg(not(windows))]
  28 const CARGO_CLIPPY_PATH: &str = "target/debug/cargo-clippy";
  29
  30 #[cfg(windows)]
  31 const CLIPPY_DRIVER_PATH: &str = "target/debug/clippy-driver.exe";
  32 #[cfg(windows)]
  33 const CARGO_CLIPPY_PATH: &str = "target/debug/cargo-clippy.exe";
  34
  35 const LINTCHECK_DOWNLOADS: &str = "target/lintcheck/downloads";
  36 const LINTCHECK_SOURCES: &str = "target/lintcheck/sources";
  37
  38 /// List of sources to check, loaded from a .toml file
  39 #[derive(Debug, Serialize, Deserialize)]
  40 struct SourceList {
  41     crates: HashMap<String, TomlCrate>,
  42 }
  43
  44 /// A crate source stored inside the .toml
  45 /// will be translated into on one of the `CrateSource` variants
  46 #[derive(Debug, Serialize, Deserialize)]
  47 struct TomlCrate {
  48     name: String,
  49     versions: Option<Vec<String>>,
  50     git_url: Option<String>,
  51     git_hash: Option<String>,
  52     path: Option<String>,
  53     options: Option<Vec<String>>,
  54 }
  55
  56 /// Represents an archive we download from crates.io, or a git repo, or a local repo/folder
  57 /// Once processed (downloaded/extracted/cloned/copied...), this will be translated into a `Crate`
  58 #[derive(Debug, Serialize, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)]
  59 enum CrateSource {
  60     CratesIo {
  61         name: String,
  62         version: String,
  63         options: Option<Vec<String>>,
  64     },
  65     Git {
  66         name: String,
  67         url: String,
  68         commit: String,
  69         options: Option<Vec<String>>,
  70     },
  71     Path {
  72         name: String,
  73         path: PathBuf,
  74         options: Option<Vec<String>>,
  75     },
  76 }
  77
  78 /// Represents the actual source code of a crate that we ran "cargo clippy" on
  79 #[derive(Debug)]
  80 struct Crate {
  81     version: String,
  82     name: String,
  83     // path to the extracted sources that clippy can check
  84     path: PathBuf,
  85     options: Option<Vec<String>>,
  86 }
  87
  88 /// A single warning that clippy issued while checking a `Crate`
  89 #[derive(Debug)]
  90 struct ClippyWarning {
  91     crate_name: String,
  92     crate_version: String,
  93     file: String,
  94     line: String,
  95     column: String,
  96     linttype: String,
  97     message: String,
  98     is_ice: bool,
  99 }
 100
 101 impl std::fmt::Display for ClippyWarning {
 102     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 103         writeln!(
 104             f,
 105             r#"target/lintcheck/sources/{}-{}/{}:{}:{} {} "{}""#,
 106             &self.crate_name, &self.crate_version, &self.file, &self.line, &self.column, &self.linttype, &self.message
 107         )
 108     }
 109 }
 110
 111 impl CrateSource {
 112     /// Makes the sources available on the disk for clippy to check.
 113     /// Clones a git repo and checks out the specified commit or downloads a crate from crates.io or
 114     /// copies a local folder
 115     fn download_and_extract(&self) -> Crate {
 116         match self {
 117             CrateSource::CratesIo { name, version, options } => {
 118                 let extract_dir = PathBuf::from(LINTCHECK_SOURCES);
 119                 let krate_download_dir = PathBuf::from(LINTCHECK_DOWNLOADS);
 120
 121                 // url to download the crate from crates.io
 122                 let url = format!("https://crates.io/api/v1/crates/{}/{}/download", name, version);
 123                 println!("Downloading and extracting {} {} from {}", name, version, url);
 124                 create_dirs(&krate_download_dir, &extract_dir);
 125
 126                 let krate_file_path = krate_download_dir.join(format!("{}-{}.crate.tar.gz", name, version));
 127                 // don't download/extract if we already have done so
 128                 if !krate_file_path.is_file() {
 129                     // create a file path to download and write the crate data into
 130                     let mut krate_dest = std::fs::File::create(&krate_file_path).unwrap();
 131                     let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
 132                     // copy the crate into the file
 133                     std::io::copy(&mut krate_req, &mut krate_dest).unwrap();
 134
 135                     // unzip the tarball
 136                     let ungz_tar = flate2::read::GzDecoder::new(std::fs::File::open(&krate_file_path).unwrap());
 137                     // extract the tar archive
 138                     let mut archive = tar::Archive::new(ungz_tar);
 139                     archive.unpack(&extract_dir).expect("Failed to extract!");
 140                 }
 141                 // crate is extracted, return a new Krate object which contains the path to the extracted
 142                 // sources that clippy can check
 143                 Crate {
 144                     version: version.clone(),
 145                     name: name.clone(),
 146                     path: extract_dir.join(format!("{}-{}/", name, version)),
 147                     options: options.clone(),
 148                 }
 149             },
 150             CrateSource::Git {
 151                 name,
 152                 url,
 153                 commit,
 154                 options,
 155             } => {
 156                 let repo_path = {
 157                     let mut repo_path = PathBuf::from(LINTCHECK_SOURCES);
 158                     // add a -git suffix in case we have the same crate from crates.io and a git repo
 159                     repo_path.push(format!("{}-git", name));
 160                     repo_path
 161                 };
 162                 // clone the repo if we have not done so
 163                 if !repo_path.is_dir() {
 164                     println!("Cloning {} and checking out {}", url, commit);
 165                     if !Command::new("git")
 166                         .arg("clone")
 167                         .arg(url)
 168                         .arg(&repo_path)
 169                         .status()
 170                         .expect("Failed to clone git repo!")
 171                         .success()
 172                     {
 173                         eprintln!("Failed to clone {} into {}", url, repo_path.display())
 174                     }
 175                 }
 176                 // check out the commit/branch/whatever
 177                 if !Command::new("git")
 178                     .arg("checkout")
 179                     .arg(commit)
 180                     .current_dir(&repo_path)
 181                     .status()
 182                     .expect("Failed to check out commit")
 183                     .success()
 184                 {
 185                     eprintln!("Failed to checkout {} of repo at {}", commit, repo_path.display())
 186                 }
 187
 188                 Crate {
 189                     version: commit.clone(),
 190                     name: name.clone(),
 191                     path: repo_path,
 192                     options: options.clone(),
 193                 }
 194             },
 195             CrateSource::Path { name, path, options } => {
 196                 use fs_extra::dir;
 197
 198                 // simply copy the entire directory into our target dir
 199                 let copy_dest = PathBuf::from(format!("{}/", LINTCHECK_SOURCES));
 200
 201                 // the source path of the crate we copied,  ${copy_dest}/crate_name
 202                 let crate_root = copy_dest.join(name); // .../crates/local_crate
 203
 204                 if crate_root.exists() {
 205                     println!(
 206                         "Not copying {} to {}, destination already exists",
 207                         path.display(),
 208                         crate_root.display()
 209                     );
 210                 } else {
 211                     println!("Copying {} to {}", path.display(), copy_dest.display());
 212
 213                     dir::copy(path, &copy_dest, &dir::CopyOptions::new()).unwrap_or_else(|_| {
 214                         panic!("Failed to copy from {}, to  {}", path.display(), crate_root.display())
 215                     });
 216                 }
 217
 218                 Crate {
 219                     version: String::from("local"),
 220                     name: name.clone(),
 221                     path: crate_root,
 222                     options: options.clone(),
 223                 }
 224             },
 225         }
 226     }
 227 }
 228
 229 impl Crate {
 230     /// Run `cargo clippy` on the `Crate` and collect and return all the lint warnings that clippy
 231     /// issued
 232     fn run_clippy_lints(
 233         &self,
 234         cargo_clippy_path: &Path,
 235         target_dir_index: &AtomicUsize,
 236         thread_limit: usize,
 237         total_crates_to_lint: usize,
 238         fix: bool,
 239     ) -> Vec<ClippyWarning> {
 240         // advance the atomic index by one
 241         let index = target_dir_index.fetch_add(1, Ordering::SeqCst);
 242         // "loop" the index within 0..thread_limit
 243         let thread_index = index % thread_limit;
 244         let perc = (index * 100) / total_crates_to_lint;
 245
 246         if thread_limit == 1 {
 247             println!(
 248                 "{}/{} {}% Linting {} {}",
 249                 index, total_crates_to_lint, perc, &self.name, &self.version
 250             );
 251         } else {
 252             println!(
 253                 "{}/{} {}% Linting {} {} in target dir {:?}",
 254                 index, total_crates_to_lint, perc, &self.name, &self.version, thread_index
 255             );
 256         }
 257
 258         let cargo_clippy_path = std::fs::canonicalize(cargo_clippy_path).unwrap();
 259
 260         let shared_target_dir = clippy_project_root().join("target/lintcheck/shared_target_dir");
 261
 262         let mut args = if fix {
 263             vec![
 264                 "-Zunstable-options",
 265                 "--fix",
 266                 "-Zunstable-options",
 267                 "--allow-no-vcs",
 268                 "--",
 269                 "--cap-lints=warn",
 270             ]
 271         } else {
 272             vec!["--", "--message-format=json", "--", "--cap-lints=warn"]
 273         };
 274
 275         if let Some(options) = &self.options {
 276             for opt in options {
 277                 args.push(opt);
 278             }
 279         } else {
 280             args.extend(&["-Wclippy::pedantic", "-Wclippy::cargo"])
 281         }
 282
 283         let all_output = std::process::Command::new(&cargo_clippy_path)
 284             // use the looping index to create individual target dirs
 285             .env(
 286                 "CARGO_TARGET_DIR",
 287                 shared_target_dir.join(format!("_{:?}", thread_index)),
 288             )
 289             // lint warnings will look like this:
 290             // src/cargo/ops/cargo_compile.rs:127:35: warning: usage of `FromIterator::from_iter`
 291             .args(&args)
 292             .current_dir(&self.path)
 293             .output()
 294             .unwrap_or_else(|error| {
 295                 panic!(
 296                     "Encountered error:\n{:?}\ncargo_clippy_path: {}\ncrate path:{}\n",
 297                     error,
 298                     &cargo_clippy_path.display(),
 299                     &self.path.display()
 300                 );
 301             });
 302         let stdout = String::from_utf8_lossy(&all_output.stdout);
 303         let stderr = String::from_utf8_lossy(&all_output.stderr);
 304         let status = &all_output.status;
 305
 306         if !status.success() {
 307             eprintln!(
 308                 "\nWARNING: bad exit status after checking {} {} \n",
 309                 self.name, self.version
 310             );
 311         }
 312
 313         if fix {
 314             if let Some(stderr) = stderr
 315                 .lines()
 316                 .find(|line| line.contains("failed to automatically apply fixes suggested by rustc to crate"))
 317             {
 318                 let subcrate = &stderr[63..];
 319                 println!(
 320                     "ERROR: failed to apply some suggetion to {} / to (sub)crate {}",
 321                     self.name, subcrate
 322                 );
 323             }
 324             // fast path, we don't need the warnings anyway
 325             return Vec::new();
 326         }
 327
 328         let output_lines = stdout.lines();
 329         let warnings: Vec<ClippyWarning> = output_lines
 330             .into_iter()
 331             // get all clippy warnings and ICEs
 332             .filter(|line| filter_clippy_warnings(&line))
 333             .map(|json_msg| parse_json_message(json_msg, &self))
 334             .collect();
 335
 336         warnings
 337     }
 338 }
 339
 340 #[derive(Debug)]
 341 struct LintcheckConfig {
 342     // max number of jobs to spawn (default 1)
 343     max_jobs: usize,
 344     // we read the sources to check from here
 345     sources_toml_path: PathBuf,
 346     // we save the clippy lint results here
 347     lintcheck_results_path: PathBuf,
 348     // whether to just run --fix and not collect all the warnings
 349     fix: bool,
 350 }
 351
 352 impl LintcheckConfig {
 353     fn from_clap(clap_config: &ArgMatches) -> Self {
 354         // first, check if we got anything passed via the LINTCHECK_TOML env var,
 355         // if not, ask clap if we got any value for --crates-toml  <foo>
 356         // if not, use the default "lintcheck/lintcheck_crates.toml"
 357         let sources_toml = env::var("LINTCHECK_TOML").unwrap_or_else(|_| {
 358             clap_config
 359                 .value_of("crates-toml")
 360                 .clone()
 361                 .unwrap_or("lintcheck/lintcheck_crates.toml")
 362                 .to_string()
 363         });
 364
 365         let sources_toml_path = PathBuf::from(sources_toml);
 366
 367         // for the path where we save the lint results, get the filename without extension (so for
 368         // wasd.toml, use "wasd"...)
 369         let filename: PathBuf = sources_toml_path.file_stem().unwrap().into();
 370         let lintcheck_results_path = PathBuf::from(format!("lintcheck-logs/{}_logs.txt", filename.display()));
 371
 372         // look at the --threads arg, if 0 is passed, ask rayon rayon how many threads it would spawn and
 373         // use half of that for the physical core count
 374         // by default use a single thread
 375         let max_jobs = match clap_config.value_of("threads") {
 376             Some(threads) => {
 377                 let threads: usize = threads
 378                     .parse()
 379                     .unwrap_or_else(|_| panic!("Failed to parse '{}' to a digit", threads));
 380                 if threads == 0 {
 381                     // automatic choice
 382                     // Rayon seems to return thread count so half that for core count
 383                     (rayon::current_num_threads() / 2) as usize
 384                 } else {
 385                     threads
 386                 }
 387             },
 388             // no -j passed, use a single thread
 389             None => 1,
 390         };
 391         let fix: bool = clap_config.is_present("fix");
 392
 393         LintcheckConfig {
 394             max_jobs,
 395             sources_toml_path,
 396             lintcheck_results_path,
 397             fix,
 398         }
 399     }
 400 }
 401
 402 /// takes a single json-formatted clippy warnings and returns true (we are interested in that line)
 403 /// or false (we aren't)
 404 fn filter_clippy_warnings(line: &str) -> bool {
 405     // we want to collect ICEs because clippy might have crashed.
 406     // these are summarized later
 407     if line.contains("internal compiler error: ") {
 408         return true;
 409     }
 410     // in general, we want all clippy warnings
 411     // however due to some kind of bug, sometimes there are absolute paths
 412     // to libcore files inside the message
 413     // or we end up with cargo-metadata output (https://github.com/rust-lang/rust-clippy/issues/6508)
 414
 415     // filter out these message to avoid unnecessary noise in the logs
 416     if line.contains("clippy::")
 417         && !(line.contains("could not read cargo metadata")
 418             || (line.contains(".rustup") && line.contains("toolchains")))
 419     {
 420         return true;
 421     }
 422     false
 423 }
 424
 425 /// Builds clippy inside the repo to make sure we have a clippy executable we can use.
 426 fn build_clippy() {
 427     let status = Command::new("cargo")
 428         .arg("build")
 429         .status()
 430         .expect("Failed to build clippy!");
 431     if !status.success() {
 432         eprintln!("Error: Failed to compile Clippy!");
 433         std::process::exit(1);
 434     }
 435 }
 436
 437 /// Read a `toml` file and return a list of `CrateSources` that we want to check with clippy
 438 fn read_crates(toml_path: &Path) -> Vec<CrateSource> {
 439     let toml_content: String =
 440         std::fs::read_to_string(&toml_path).unwrap_or_else(|_| panic!("Failed to read {}", toml_path.display()));
 441     let crate_list: SourceList =
 442         toml::from_str(&toml_content).unwrap_or_else(|e| panic!("Failed to parse {}: \n{}", toml_path.display(), e));
 443     // parse the hashmap of the toml file into a list of crates
 444     let tomlcrates: Vec<TomlCrate> = crate_list
 445         .crates
 446         .into_iter()
 447         .map(|(_cratename, tomlcrate)| tomlcrate)
 448         .collect();
 449
 450     // flatten TomlCrates into CrateSources (one TomlCrates may represent several versions of a crate =>
 451     // multiple Cratesources)
 452     let mut crate_sources = Vec::new();
 453     tomlcrates.into_iter().for_each(|tk| {
 454         if let Some(ref path) = tk.path {
 455             crate_sources.push(CrateSource::Path {
 456                 name: tk.name.clone(),
 457                 path: PathBuf::from(path),
 458                 options: tk.options.clone(),
 459             });
 460         }
 461
 462         // if we have multiple versions, save each one
 463         if let Some(ref versions) = tk.versions {
 464             versions.iter().for_each(|ver| {
 465                 crate_sources.push(CrateSource::CratesIo {
 466                     name: tk.name.clone(),
 467                     version: ver.to_string(),
 468                     options: tk.options.clone(),
 469                 });
 470             })
 471         }
 472         // otherwise, we should have a git source
 473         if tk.git_url.is_some() && tk.git_hash.is_some() {
 474             crate_sources.push(CrateSource::Git {
 475                 name: tk.name.clone(),
 476                 url: tk.git_url.clone().unwrap(),
 477                 commit: tk.git_hash.clone().unwrap(),
 478                 options: tk.options.clone(),
 479             });
 480         }
 481         // if we have a version as well as a git data OR only one git data, something is funky
 482         if tk.versions.is_some() && (tk.git_url.is_some() || tk.git_hash.is_some())
 483             || tk.git_hash.is_some() != tk.git_url.is_some()
 484         {
 485             eprintln!("tomlkrate: {:?}", tk);
 486             if tk.git_hash.is_some() != tk.git_url.is_some() {
 487                 panic!("Error: Encountered TomlCrate with only one of git_hash and git_url!");
 488             }
 489             if tk.path.is_some() && (tk.git_hash.is_some() || tk.versions.is_some()) {
 490                 panic!("Error: TomlCrate can only have one of 'git_.*', 'version' or 'path' fields");
 491             }
 492             unreachable!("Failed to translate TomlCrate into CrateSource!");
 493         }
 494     });
 495     // sort the crates
 496     crate_sources.sort();
 497
 498     crate_sources
 499 }
 500
 501 /// Parse the json output of clippy and return a `ClippyWarning`
 502 fn parse_json_message(json_message: &str, krate: &Crate) -> ClippyWarning {
 503     let jmsg: Value = serde_json::from_str(&json_message).unwrap_or_else(|e| panic!("Failed to parse json:\n{:?}", e));
 504
 505     let file: String = jmsg["message"]["spans"][0]["file_name"]
 506         .to_string()
 507         .trim_matches('"')
 508         .into();
 509
 510     let file = if file.contains(".cargo") {
 511         // if we deal with macros, a filename may show the origin of a macro which can be inside a dep from
 512         // the registry.
 513         // don't show the full path in that case.
 514
 515         // /home/matthias/.cargo/registry/src/github.com-1ecc6299db9ec823/syn-1.0.63/src/custom_keyword.rs
 516         let path = PathBuf::from(file);
 517         let mut piter = path.iter();
 518         // consume all elements until we find ".cargo", so that "/home/matthias" is skipped
 519         let _: Option<&OsStr> = piter.find(|x| x == &std::ffi::OsString::from(".cargo"));
 520         // collect the remaining segments
 521         let file = piter.collect::<PathBuf>();
 522         format!("{}", file.display())
 523     } else {
 524         file
 525     };
 526
 527     ClippyWarning {
 528         crate_name: krate.name.to_string(),
 529         crate_version: krate.version.to_string(),
 530         file,
 531         line: jmsg["message"]["spans"][0]["line_start"]
 532             .to_string()
 533             .trim_matches('"')
 534             .into(),
 535         column: jmsg["message"]["spans"][0]["text"][0]["highlight_start"]
 536             .to_string()
 537             .trim_matches('"')
 538             .into(),
 539         linttype: jmsg["message"]["code"]["code"].to_string().trim_matches('"').into(),
 540         message: jmsg["message"]["message"].to_string().trim_matches('"').into(),
 541         is_ice: json_message.contains("internal compiler error: "),
 542     }
 543 }
 544
 545 /// Generate a short list of occuring lints-types and their count
 546 fn gather_stats(clippy_warnings: &[ClippyWarning]) -> (String, HashMap<&String, usize>) {
 547     // count lint type occurrences
 548     let mut counter: HashMap<&String, usize> = HashMap::new();
 549     clippy_warnings
 550         .iter()
 551         .for_each(|wrn| *counter.entry(&wrn.linttype).or_insert(0) += 1);
 552
 553     // collect into a tupled list for sorting
 554     let mut stats: Vec<(&&String, &usize)> = counter.iter().map(|(lint, count)| (lint, count)).collect();
 555     // sort by "000{count} {clippy::lintname}"
 556     // to not have a lint with 200 and 2 warnings take the same spot
 557     stats.sort_by_key(|(lint, count)| format!("{:0>4}, {}", count, lint));
 558
 559     let stats_string = stats
 560         .iter()
 561         .map(|(lint, count)| format!("{} {}\n", lint, count))
 562         .collect::<String>();
 563
 564     (stats_string, counter)
 565 }
 566
 567 /// check if the latest modification of the logfile is older than the modification date of the
 568 /// clippy binary, if this is true, we should clean the lintchec shared target directory and recheck
 569 fn lintcheck_needs_rerun(lintcheck_logs_path: &Path) -> bool {
 570     if !lintcheck_logs_path.exists() {
 571         return true;
 572     }
 573
 574     let clippy_modified: std::time::SystemTime = {
 575         let mut times = [CLIPPY_DRIVER_PATH, CARGO_CLIPPY_PATH].iter().map(|p| {
 576             std::fs::metadata(p)
 577                 .expect("failed to get metadata of file")
 578                 .modified()
 579                 .expect("failed to get modification date")
 580         });
 581         // the oldest modification of either of the binaries
 582         std::cmp::max(times.next().unwrap(), times.next().unwrap())
 583     };
 584
 585     let logs_modified: std::time::SystemTime = std::fs::metadata(lintcheck_logs_path)
 586         .expect("failed to get metadata of file")
 587         .modified()
 588         .expect("failed to get modification date");
 589
 590     // time is represented in seconds since X
 591     // logs_modified 2 and clippy_modified 5 means clippy binary is older and we need to recheck
 592     logs_modified < clippy_modified
 593 }
 594
 595 fn is_in_clippy_root() -> bool {
 596     if let Ok(pb) = std::env::current_dir() {
 597         if let Some(file) = pb.file_name() {
 598             return file == PathBuf::from("rust-clippy");
 599         }
 600     }
 601
 602     false
 603 }
 604
 605 /// lintchecks `main()` function
 606 ///
 607 /// # Panics
 608 ///
 609 /// This function panics if the clippy binaries don't exist
 610 /// or if lintcheck is executed from the wrong directory (aka none-repo-root)
 611 pub fn main() {
 612     // assert that we launch lintcheck from the repo root (via cargo lintcheck)
 613     if !is_in_clippy_root() {
 614         eprintln!("lintcheck needs to be run from clippys repo root!\nUse `cargo lintcheck` alternatively.");
 615         std::process::exit(3);
 616     }
 617
 618     let clap_config = &get_clap_config();
 619
 620     let config = LintcheckConfig::from_clap(clap_config);
 621
 622     println!("Compiling clippy...");
 623     build_clippy();
 624     println!("Done compiling");
 625
 626     // if the clippy bin is newer than our logs, throw away target dirs to force clippy to
 627     // refresh the logs
 628     if lintcheck_needs_rerun(&config.lintcheck_results_path) {
 629         let shared_target_dir = "target/lintcheck/shared_target_dir";
 630         // if we get an Err here, the shared target dir probably does simply not exist
 631         if let Ok(metadata) = std::fs::metadata(&shared_target_dir) {
 632             if metadata.is_dir() {
 633                 println!("Clippy is newer than lint check logs, clearing lintcheck shared target dir...");
 634                 std::fs::remove_dir_all(&shared_target_dir)
 635                     .expect("failed to remove target/lintcheck/shared_target_dir");
 636             }
 637         }
 638     }
 639
 640     let cargo_clippy_path: PathBuf = PathBuf::from(CARGO_CLIPPY_PATH)
 641         .canonicalize()
 642         .expect("failed to canonicalize path to clippy binary");
 643
 644     // assert that clippy is found
 645     assert!(
 646         cargo_clippy_path.is_file(),
 647         "target/debug/cargo-clippy binary not found! {}",
 648         cargo_clippy_path.display()
 649     );
 650
 651     let clippy_ver = std::process::Command::new(CARGO_CLIPPY_PATH)
 652         .arg("--version")
 653         .output()
 654         .map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
 655         .expect("could not get clippy version!");
 656
 657     // download and extract the crates, then run clippy on them and collect clippys warnings
 658     // flatten into one big list of warnings
 659
 660     let crates = read_crates(&config.sources_toml_path);
 661     let old_stats = read_stats_from_file(&config.lintcheck_results_path);
 662
 663     let counter = AtomicUsize::new(1);
 664
 665     let clippy_warnings: Vec<ClippyWarning> = if let Some(only_one_crate) = clap_config.value_of("only") {
 666         // if we don't have the specified crate in the .toml, throw an error
 667         if !crates.iter().any(|krate| {
 668             let name = match krate {
 669                 CrateSource::CratesIo { name, .. } | CrateSource::Git { name, .. } | CrateSource::Path { name, .. } => {
 670                     name
 671                 },
 672             };
 673             name == only_one_crate
 674         }) {
 675             eprintln!(
 676                 "ERROR: could not find crate '{}' in lintcheck/lintcheck_crates.toml",
 677                 only_one_crate
 678             );
 679             std::process::exit(1);
 680         }
 681
 682         // only check a single crate that was passed via cmdline
 683         crates
 684             .into_iter()
 685             .map(|krate| krate.download_and_extract())
 686             .filter(|krate| krate.name == only_one_crate)
 687             .flat_map(|krate| krate.run_clippy_lints(&cargo_clippy_path, &AtomicUsize::new(0), 1, 1, config.fix))
 688             .collect()
 689     } else {
 690         if config.max_jobs > 1 {
 691             // run parallel with rayon
 692
 693             // Ask rayon for thread count. Assume that half of that is the number of physical cores
 694             // Use one target dir for each core so that we can run N clippys in parallel.
 695             // We need to use different target dirs because cargo would lock them for a single build otherwise,
 696             // killing the parallelism. However this also means that deps will only be reused half/a
 697             // quarter of the time which might result in a longer wall clock runtime
 698
 699             // This helps when we check many small crates with dep-trees that don't have a lot of branches in
 700             // order to achive some kind of parallelism
 701
 702             // by default, use a single thread
 703             let num_cpus = config.max_jobs;
 704             let num_crates = crates.len();
 705
 706             // check all crates (default)
 707             crates
 708                 .into_par_iter()
 709                 .map(|krate| krate.download_and_extract())
 710                 .flat_map(|krate| {
 711                     krate.run_clippy_lints(&cargo_clippy_path, &counter, num_cpus, num_crates, config.fix)
 712                 })
 713                 .collect()
 714         } else {
 715             // run sequential
 716             let num_crates = crates.len();
 717             crates
 718                 .into_iter()
 719                 .map(|krate| krate.download_and_extract())
 720                 .flat_map(|krate| krate.run_clippy_lints(&cargo_clippy_path, &counter, 1, num_crates, config.fix))
 721                 .collect()
 722         }
 723     };
 724
 725     // if we are in --fix mode, don't change the log files, terminate here
 726     if config.fix {
 727         return;
 728     }
 729
 730     // generate some stats
 731     let (stats_formatted, new_stats) = gather_stats(&clippy_warnings);
 732
 733     // grab crashes/ICEs, save the crate name and the ice message
 734     let ices: Vec<(&String, &String)> = clippy_warnings
 735         .iter()
 736         .filter(|warning| warning.is_ice)
 737         .map(|w| (&w.crate_name, &w.message))
 738         .collect();
 739
 740     let mut all_msgs: Vec<String> = clippy_warnings.iter().map(ToString::to_string).collect();
 741     all_msgs.sort();
 742     all_msgs.push("\n\n\n\nStats:\n".into());
 743     all_msgs.push(stats_formatted);
 744
 745     // save the text into lintcheck-logs/logs.txt
 746     let mut text = clippy_ver; // clippy version number on top
 747     text.push_str(&format!("\n{}", all_msgs.join("")));
 748     text.push_str("ICEs:\n");
 749     ices.iter()
 750         .for_each(|(cratename, msg)| text.push_str(&format!("{}: '{}'", cratename, msg)));
 751
 752     println!("Writing logs to {}", config.lintcheck_results_path.display());
 753     write(&config.lintcheck_results_path, text).unwrap();
 754
 755     print_stats(old_stats, new_stats);
 756 }
 757
 758 /// read the previous stats from the lintcheck-log file
 759 fn read_stats_from_file(file_path: &Path) -> HashMap<String, usize> {
 760     let file_content: String = match std::fs::read_to_string(file_path).ok() {
 761         Some(content) => content,
 762         None => {
 763             return HashMap::new();
 764         },
 765     };
 766
 767     let lines: Vec<String> = file_content.lines().map(ToString::to_string).collect();
 768
 769     // search for the beginning "Stats:" and the end "ICEs:" of the section we want
 770     let start = lines.iter().position(|line| line == "Stats:").unwrap();
 771     let end = lines.iter().position(|line| line == "ICEs:").unwrap();
 772
 773     let stats_lines = &lines[start + 1..end];
 774
 775     stats_lines
 776         .iter()
 777         .map(|line| {
 778             let mut spl = line.split(' ');
 779             (
 780                 spl.next().unwrap().to_string(),
 781                 spl.next().unwrap().parse::<usize>().unwrap(),
 782             )
 783         })
 784         .collect::<HashMap<String, usize>>()
 785 }
 786
 787 /// print how lint counts changed between runs
 788 fn print_stats(old_stats: HashMap<String, usize>, new_stats: HashMap<&String, usize>) {
 789     let same_in_both_hashmaps = old_stats
 790         .iter()
 791         .filter(|(old_key, old_val)| new_stats.get::<&String>(&old_key) == Some(old_val))
 792         .map(|(k, v)| (k.to_string(), *v))
 793         .collect::<Vec<(String, usize)>>();
 794
 795     let mut old_stats_deduped = old_stats;
 796     let mut new_stats_deduped = new_stats;
 797
 798     // remove duplicates from both hashmaps
 799     same_in_both_hashmaps.iter().for_each(|(k, v)| {
 800         assert!(old_stats_deduped.remove(k) == Some(*v));
 801         assert!(new_stats_deduped.remove(k) == Some(*v));
 802     });
 803
 804     println!("\nStats:");
 805
 806     // list all new counts  (key is in new stats but not in old stats)
 807     new_stats_deduped
 808         .iter()
 809         .filter(|(new_key, _)| old_stats_deduped.get::<str>(&new_key).is_none())
 810         .for_each(|(new_key, new_value)| {
 811             println!("{} 0 => {}", new_key, new_value);
 812         });
 813
 814     // list all changed counts (key is in both maps but value differs)
 815     new_stats_deduped
 816         .iter()
 817         .filter(|(new_key, _new_val)| old_stats_deduped.get::<str>(&new_key).is_some())
 818         .for_each(|(new_key, new_val)| {
 819             let old_val = old_stats_deduped.get::<str>(&new_key).unwrap();
 820             println!("{} {} => {}", new_key, old_val, new_val);
 821         });
 822
 823     // list all gone counts (key is in old status but not in new stats)
 824     old_stats_deduped
 825         .iter()
 826         .filter(|(old_key, _)| new_stats_deduped.get::<&String>(&old_key).is_none())
 827         .for_each(|(old_key, old_value)| {
 828             println!("{} {} => 0", old_key, old_value);
 829         });
 830 }
 831
 832 /// Create necessary directories to run the lintcheck tool.
 833 ///
 834 /// # Panics
 835 ///
 836 /// This function panics if creating one of the dirs fails.
 837 fn create_dirs(krate_download_dir: &Path, extract_dir: &Path) {
 838     std::fs::create_dir("target/lintcheck/").unwrap_or_else(|err| {
 839         if err.kind() != ErrorKind::AlreadyExists {
 840             panic!("cannot create lintcheck target dir");
 841         }
 842     });
 843     std::fs::create_dir(&krate_download_dir).unwrap_or_else(|err| {
 844         if err.kind() != ErrorKind::AlreadyExists {
 845             panic!("cannot create crate download dir");
 846         }
 847     });
 848     std::fs::create_dir(&extract_dir).unwrap_or_else(|err| {
 849         if err.kind() != ErrorKind::AlreadyExists {
 850             panic!("cannot create crate extraction dir");
 851         }
 852     });
 853 }
 854
 855 fn get_clap_config<'a>() -> ArgMatches<'a> {
 856     App::new("lintcheck")
 857         .about("run clippy on a set of crates and check output")
 858         .arg(
 859             Arg::with_name("only")
 860                 .takes_value(true)
 861                 .value_name("CRATE")
 862                 .long("only")
 863                 .help("only process a single crate of the list"),
 864         )
 865         .arg(
 866             Arg::with_name("crates-toml")
 867                 .takes_value(true)
 868                 .value_name("CRATES-SOURCES-TOML-PATH")
 869                 .long("crates-toml")
 870                 .help("set the path for a crates.toml where lintcheck should read the sources from"),
 871         )
 872         .arg(
 873             Arg::with_name("threads")
 874                 .takes_value(true)
 875                 .value_name("N")
 876                 .short("j")
 877                 .long("jobs")
 878                 .help("number of threads to use, 0 automatic choice"),
 879         )
 880         .arg(
 881             Arg::with_name("fix")
 882                 .long("--fix")
 883                 .help("runs cargo clippy --fix and checks if all suggestions apply"),
 884         )
 885         .get_matches()
 886 }
 887
 888 /// Returns the path to the Clippy project directory
 889 ///
 890 /// # Panics
 891 ///
 892 /// Panics if the current directory could not be retrieved, there was an error reading any of the
 893 /// Cargo.toml files or ancestor directory is the clippy root directory
 894 #[must_use]
 895 pub fn clippy_project_root() -> PathBuf {
 896     let current_dir = std::env::current_dir().unwrap();
 897     for path in current_dir.ancestors() {
 898         let result = std::fs::read_to_string(path.join("Cargo.toml"));
 899         if let Err(err) = &result {
 900             if err.kind() == std::io::ErrorKind::NotFound {
 901                 continue;
 902             }
 903         }
 904
 905         let content = result.unwrap();
 906         if content.contains("[package]\nname = \"clippy\"") {
 907             return path.to_path_buf();
 908         }
 909     }
 910     panic!("error: Can't determine root of project. Please run inside a Clippy working dir.");
 911 }
 912
 913 #[test]
 914 fn lintcheck_test() {
 915     let args = [
 916         "run",
 917         "--target-dir",
 918         "lintcheck/target",
 919         "--manifest-path",
 920         "./lintcheck/Cargo.toml",
 921         "--",
 922         "--crates-toml",
 923         "lintcheck/test_sources.toml",
 924     ];
 925     let status = std::process::Command::new("cargo")
 926         .args(&args)
 927         .current_dir("..") // repo root
 928         .status();
 929     //.output();
 930
 931     assert!(status.unwrap().success());
 932 }