src/tools/lint-docs/src/lib.rs

   1 use std::error::Error;
   2 use std::fmt::Write;
   3 use std::fs;
   4 use std::path::{Path, PathBuf};
   5 use std::process::Command;
   6 use walkdir::WalkDir;
   7
   8 mod groups;
   9
  10 struct Lint {
  11     name: String,
  12     doc: Vec<String>,
  13     level: Level,
  14     path: PathBuf,
  15     lineno: usize,
  16 }
  17
  18 impl Lint {
  19     fn doc_contains(&self, text: &str) -> bool {
  20         self.doc.iter().any(|line| line.contains(text))
  21     }
  22
  23     fn is_ignored(&self) -> bool {
  24         self.doc
  25             .iter()
  26             .filter(|line| line.starts_with("```rust"))
  27             .all(|line| line.contains(",ignore"))
  28     }
  29 }
  30
  31 #[derive(Clone, Copy, PartialEq)]
  32 enum Level {
  33     Allow,
  34     Warn,
  35     Deny,
  36 }
  37
  38 impl Level {
  39     fn doc_filename(&self) -> &str {
  40         match self {
  41             Level::Allow => "allowed-by-default.md",
  42             Level::Warn => "warn-by-default.md",
  43             Level::Deny => "deny-by-default.md",
  44         }
  45     }
  46 }
  47
  48 #[derive(Copy, Clone)]
  49 pub struct Rustc<'a> {
  50     pub path: &'a Path,
  51     pub target: &'a str,
  52 }
  53
  54 /// Collects all lints, and writes the markdown documentation at the given directory.
  55 pub fn extract_lint_docs(
  56     src_path: &Path,
  57     out_path: &Path,
  58     rustc: Rustc<'_>,
  59     verbose: bool,
  60 ) -> Result<(), Box<dyn Error>> {
  61     let mut lints = gather_lints(src_path)?;
  62     for lint in &mut lints {
  63         generate_output_example(lint, rustc, verbose).map_err(|e| {
  64             format!(
  65                 "failed to test example in lint docs for `{}` in {}:{}: {}",
  66                 lint.name,
  67                 lint.path.display(),
  68                 lint.lineno,
  69                 e
  70             )
  71         })?;
  72     }
  73     save_lints_markdown(&lints, &out_path.join("listing"))?;
  74     groups::generate_group_docs(&lints, rustc, out_path)?;
  75     Ok(())
  76 }
  77
  78 /// Collects all lints from all files in the given directory.
  79 fn gather_lints(src_path: &Path) -> Result<Vec<Lint>, Box<dyn Error>> {
  80     let mut lints = Vec::new();
  81     for entry in WalkDir::new(src_path).into_iter().filter_map(|e| e.ok()) {
  82         if !entry.path().extension().map_or(false, |ext| ext == "rs") {
  83             continue;
  84         }
  85         lints.extend(lints_from_file(entry.path())?);
  86     }
  87     if lints.is_empty() {
  88         return Err("no lints were found!".into());
  89     }
  90     Ok(lints)
  91 }
  92
  93 /// Collects all lints from the given file.
  94 fn lints_from_file(path: &Path) -> Result<Vec<Lint>, Box<dyn Error>> {
  95     let mut lints = Vec::new();
  96     let contents = fs::read_to_string(path)
  97         .map_err(|e| format!("could not read {}: {}", path.display(), e))?;
  98     let mut lines = contents.lines().enumerate();
  99     loop {
 100         // Find a lint declaration.
 101         let lint_start = loop {
 102             match lines.next() {
 103                 Some((lineno, line)) => {
 104                     if line.trim().starts_with("declare_lint!") {
 105                         break lineno + 1;
 106                     }
 107                 }
 108                 None => return Ok(lints),
 109             }
 110         };
 111         // Read the lint.
 112         let mut doc_lines = Vec::new();
 113         let (doc, name) = loop {
 114             match lines.next() {
 115                 Some((lineno, line)) => {
 116                     let line = line.trim();
 117                     if line.starts_with("/// ") {
 118                         doc_lines.push(line.trim()[4..].to_string());
 119                     } else if line.starts_with("///") {
 120                         doc_lines.push("".to_string());
 121                     } else if line.starts_with("// ") {
 122                         // Ignore comments.
 123                         continue;
 124                     } else {
 125                         let name = lint_name(line).map_err(|e| {
 126                             format!(
 127                                 "could not determine lint name in {}:{}: {}, line was `{}`",
 128                                 path.display(),
 129                                 lineno,
 130                                 e,
 131                                 line
 132                             )
 133                         })?;
 134                         if doc_lines.is_empty() {
 135                             return Err(format!(
 136                                 "did not find doc lines for lint `{}` in {}",
 137                                 name,
 138                                 path.display()
 139                             )
 140                             .into());
 141                         }
 142                         break (doc_lines, name);
 143                     }
 144                 }
 145                 None => {
 146                     return Err(format!(
 147                         "unexpected EOF for lint definition at {}:{}",
 148                         path.display(),
 149                         lint_start
 150                     )
 151                     .into());
 152                 }
 153             }
 154         };
 155         // These lints are specifically undocumented. This should be reserved
 156         // for internal rustc-lints only.
 157         if name == "deprecated_in_future" {
 158             continue;
 159         }
 160         // Read the level.
 161         let level = loop {
 162             match lines.next() {
 163                 // Ignore comments.
 164                 Some((_, line)) if line.trim().starts_with("// ") => {}
 165                 Some((lineno, line)) => match line.trim() {
 166                     "Allow," => break Level::Allow,
 167                     "Warn," => break Level::Warn,
 168                     "Deny," => break Level::Deny,
 169                     _ => {
 170                         return Err(format!(
 171                             "unexpected lint level `{}` in {}:{}",
 172                             line,
 173                             path.display(),
 174                             lineno
 175                         )
 176                         .into());
 177                     }
 178                 },
 179                 None => {
 180                     return Err(format!(
 181                         "expected lint level in {}:{}, got EOF",
 182                         path.display(),
 183                         lint_start
 184                     )
 185                     .into());
 186                 }
 187             }
 188         };
 189         // The rest of the lint definition is ignored.
 190         assert!(!doc.is_empty());
 191         lints.push(Lint { name, doc, level, path: PathBuf::from(path), lineno: lint_start });
 192     }
 193 }
 194
 195 /// Extracts the lint name (removing the visibility modifier, and checking validity).
 196 fn lint_name(line: &str) -> Result<String, &'static str> {
 197     // Skip over any potential `pub` visibility.
 198     match line.trim().split(' ').next_back() {
 199         Some(name) => {
 200             if !name.ends_with(',') {
 201                 return Err("lint name should end with comma");
 202             }
 203             let name = &name[..name.len() - 1];
 204             if !name.chars().all(|ch| ch.is_uppercase() || ch == '_') || name.is_empty() {
 205                 return Err("lint name did not have expected format");
 206             }
 207             Ok(name.to_lowercase().to_string())
 208         }
 209         None => Err("could not find lint name"),
 210     }
 211 }
 212
 213 /// Mutates the lint definition to replace the `{{produces}}` marker with the
 214 /// actual output from the compiler.
 215 fn generate_output_example(
 216     lint: &mut Lint,
 217     rustc: Rustc<'_>,
 218     verbose: bool,
 219 ) -> Result<(), Box<dyn Error>> {
 220     // Explicit list of lints that are allowed to not have an example. Please
 221     // try to avoid adding to this list.
 222     if matches!(
 223         lint.name.as_str(),
 224         "unused_features" // broken lint
 225         | "unstable_features" // deprecated
 226     ) {
 227         return Ok(());
 228     }
 229     if lint.doc_contains("[rustdoc book]") && !lint.doc_contains("{{produces}}") {
 230         // Rustdoc lints are documented in the rustdoc book, don't check these.
 231         return Ok(());
 232     }
 233     check_style(lint)?;
 234     // Unfortunately some lints have extra requirements that this simple test
 235     // setup can't handle (like extern crates). An alternative is to use a
 236     // separate test suite, and use an include mechanism such as mdbook's
 237     // `{{#rustdoc_include}}`.
 238     if !lint.is_ignored() {
 239         replace_produces(lint, rustc, verbose)?;
 240     }
 241     Ok(())
 242 }
 243
 244 /// Checks the doc style of the lint.
 245 fn check_style(lint: &Lint) -> Result<(), Box<dyn Error>> {
 246     for &expected in &["### Example", "### Explanation", "{{produces}}"] {
 247         if expected == "{{produces}}" && lint.is_ignored() {
 248             continue;
 249         }
 250         if !lint.doc_contains(expected) {
 251             return Err(format!("lint docs should contain the line `{}`", expected).into());
 252         }
 253     }
 254     if let Some(first) = lint.doc.first() {
 255         if !first.starts_with(&format!("The `{}` lint", lint.name)) {
 256             return Err(format!(
 257                 "lint docs should start with the text \"The `{}` lint\" to introduce the lint",
 258                 lint.name
 259             )
 260             .into());
 261         }
 262     }
 263     Ok(())
 264 }
 265
 266 /// Mutates the lint docs to replace the `{{produces}}` marker with the actual
 267 /// output from the compiler.
 268 fn replace_produces(
 269     lint: &mut Lint,
 270     rustc: Rustc<'_>,
 271     verbose: bool,
 272 ) -> Result<(), Box<dyn Error>> {
 273     let mut lines = lint.doc.iter_mut();
 274     loop {
 275         // Find start of example.
 276         let options = loop {
 277             match lines.next() {
 278                 Some(line) if line.starts_with("```rust") => {
 279                     break line[7..].split(',').collect::<Vec<_>>();
 280                 }
 281                 Some(line) if line.contains("{{produces}}") => {
 282                     return Err("lint marker {{{{produces}}}} found, \
 283                         but expected to immediately follow a rust code block"
 284                         .into());
 285                 }
 286                 Some(_) => {}
 287                 None => return Ok(()),
 288             }
 289         };
 290         // Find the end of example.
 291         let mut example = Vec::new();
 292         loop {
 293             match lines.next() {
 294                 Some(line) if line == "```" => break,
 295                 Some(line) => example.push(line),
 296                 None => {
 297                     return Err(format!(
 298                         "did not find end of example triple ticks ```, docs were:\n{:?}",
 299                         lint.doc
 300                     )
 301                     .into());
 302                 }
 303             }
 304         }
 305         // Find the {{produces}} line.
 306         loop {
 307             match lines.next() {
 308                 Some(line) if line.is_empty() => {}
 309                 Some(line) if line == "{{produces}}" => {
 310                     let output =
 311                         generate_lint_output(&lint.name, &example, &options, rustc, verbose)?;
 312                     line.replace_range(
 313                         ..,
 314                         &format!(
 315                             "This will produce:\n\
 316                         \n\
 317                         ```text\n\
 318                         {}\
 319                         ```",
 320                             output
 321                         ),
 322                     );
 323                     break;
 324                 }
 325                 // No {{produces}} after example, find next example.
 326                 Some(_line) => break,
 327                 None => return Ok(()),
 328             }
 329         }
 330     }
 331 }
 332
 333 /// Runs the compiler against the example, and extracts the output.
 334 fn generate_lint_output(
 335     name: &str,
 336     example: &[&mut String],
 337     options: &[&str],
 338     rustc: Rustc<'_>,
 339     verbose: bool,
 340 ) -> Result<String, Box<dyn Error>> {
 341     if verbose {
 342         eprintln!("compiling lint {}", name);
 343     }
 344     let tempdir = tempfile::TempDir::new()?;
 345     let tempfile = tempdir.path().join("lint_example.rs");
 346     let mut source = String::new();
 347     let needs_main = !example.iter().any(|line| line.contains("fn main"));
 348     // Remove `# ` prefix for hidden lines.
 349     let unhidden =
 350         example.iter().map(|line| if line.starts_with("# ") { &line[2..] } else { line });
 351     let mut lines = unhidden.peekable();
 352     while let Some(line) = lines.peek() {
 353         if line.starts_with("#!") {
 354             source.push_str(line);
 355             source.push('\n');
 356             lines.next();
 357         } else {
 358             break;
 359         }
 360     }
 361     if needs_main {
 362         source.push_str("fn main() {\n");
 363     }
 364     for line in lines {
 365         source.push_str(line);
 366         source.push('\n')
 367     }
 368     if needs_main {
 369         source.push_str("}\n");
 370     }
 371     fs::write(&tempfile, source)
 372         .map_err(|e| format!("failed to write {}: {}", tempfile.display(), e))?;
 373     let mut cmd = Command::new(rustc.path);
 374     if options.contains(&"edition2015") {
 375         cmd.arg("--edition=2015");
 376     } else {
 377         cmd.arg("--edition=2018");
 378     }
 379     cmd.arg("--error-format=json");
 380     cmd.arg("--target").arg(rustc.target);
 381     if options.contains(&"test") {
 382         cmd.arg("--test");
 383     }
 384     cmd.arg("lint_example.rs");
 385     cmd.current_dir(tempdir.path());
 386     let output = cmd.output().map_err(|e| format!("failed to run command {:?}\n{}", cmd, e))?;
 387     let stderr = std::str::from_utf8(&output.stderr).unwrap();
 388     let msgs = stderr
 389         .lines()
 390         .filter(|line| line.starts_with('{'))
 391         .map(serde_json::from_str)
 392         .collect::<Result<Vec<serde_json::Value>, _>>()?;
 393     match msgs
 394         .iter()
 395         .find(|msg| matches!(&msg["code"]["code"], serde_json::Value::String(s) if s==name))
 396     {
 397         Some(msg) => {
 398             let rendered = msg["rendered"].as_str().expect("rendered field should exist");
 399             Ok(rendered.to_string())
 400         }
 401         None => {
 402             match msgs.iter().find(
 403                 |msg| matches!(&msg["rendered"], serde_json::Value::String(s) if s.contains(name)),
 404             ) {
 405                 Some(msg) => {
 406                     let rendered = msg["rendered"].as_str().expect("rendered field should exist");
 407                     Ok(rendered.to_string())
 408                 }
 409                 None => {
 410                     let rendered: Vec<&str> =
 411                         msgs.iter().filter_map(|msg| msg["rendered"].as_str()).collect();
 412                     let non_json: Vec<&str> =
 413                         stderr.lines().filter(|line| !line.starts_with('{')).collect();
 414                     Err(format!(
 415                         "did not find lint `{}` in output of example, got:\n{}\n{}",
 416                         name,
 417                         non_json.join("\n"),
 418                         rendered.join("\n")
 419                     )
 420                     .into())
 421                 }
 422             }
 423         }
 424     }
 425 }
 426
 427 static ALLOWED_MD: &str = r#"# Allowed-by-default lints
 428
 429 These lints are all set to the 'allow' level by default. As such, they won't show up
 430 unless you set them to a higher lint level with a flag or attribute.
 431
 432 "#;
 433
 434 static WARN_MD: &str = r#"# Warn-by-default lints
 435
 436 These lints are all set to the 'warn' level by default.
 437
 438 "#;
 439
 440 static DENY_MD: &str = r#"# Deny-by-default lints
 441
 442 These lints are all set to the 'deny' level by default.
 443
 444 "#;
 445
 446 /// Saves the mdbook lint chapters at the given path.
 447 fn save_lints_markdown(lints: &[Lint], out_dir: &Path) -> Result<(), Box<dyn Error>> {
 448     save_level(lints, Level::Allow, out_dir, ALLOWED_MD)?;
 449     save_level(lints, Level::Warn, out_dir, WARN_MD)?;
 450     save_level(lints, Level::Deny, out_dir, DENY_MD)?;
 451     Ok(())
 452 }
 453
 454 fn save_level(
 455     lints: &[Lint],
 456     level: Level,
 457     out_dir: &Path,
 458     header: &str,
 459 ) -> Result<(), Box<dyn Error>> {
 460     let mut result = String::new();
 461     result.push_str(header);
 462     let mut these_lints: Vec<_> = lints.iter().filter(|lint| lint.level == level).collect();
 463     these_lints.sort_unstable_by_key(|lint| &lint.name);
 464     for lint in &these_lints {
 465         write!(result, "* [`{}`](#{})\n", lint.name, lint.name.replace("_", "-")).unwrap();
 466     }
 467     result.push('\n');
 468     for lint in &these_lints {
 469         write!(result, "## {}\n\n", lint.name.replace("_", "-")).unwrap();
 470         for line in &lint.doc {
 471             result.push_str(line);
 472             result.push('\n');
 473         }
 474         result.push('\n');
 475     }
 476     let out_path = out_dir.join(level.doc_filename());
 477     // Delete the output because rustbuild uses hard links in its copies.
 478     let _ = fs::remove_file(&out_path);
 479     fs::write(&out_path, result)
 480         .map_err(|e| format!("could not write to {}: {}", out_path.display(), e))?;
 481     Ok(())
 482 }