src/tools/linkchecker/main.rs

   1 //! Script to check the validity of `href` links in our HTML documentation.
   2 //!
   3 //! In the past we've been quite error prone to writing in broken links as most
   4 //! of them are manually rather than automatically added. As files move over
   5 //! time or apis change old links become stale or broken. The purpose of this
   6 //! script is to check all relative links in our documentation to make sure they
   7 //! actually point to a valid place.
   8 //!
   9 //! Currently this doesn't actually do any HTML parsing or anything fancy like
  10 //! that, it just has a simple "regex" to search for `href` and `id` tags.
  11 //! These values are then translated to file URLs if possible and then the
  12 //! destination is asserted to exist.
  13 //!
  14 //! A few exceptions are allowed as there's known bugs in rustdoc, but this
  15 //! should catch the majority of "broken link" cases.
  16
  17 #![feature(str_split_once)]
  18
  19 use std::collections::hash_map::Entry;
  20 use std::collections::{HashMap, HashSet};
  21 use std::env;
  22 use std::fs;
  23 use std::path::{Component, Path, PathBuf};
  24 use std::rc::Rc;
  25
  26 use once_cell::sync::Lazy;
  27 use regex::Regex;
  28
  29 use crate::Redirect::*;
  30
  31 // Add linkcheck exceptions here
  32 // If at all possible you should use intra-doc links to avoid linkcheck issues. These
  33 // are cases where that does not work
  34 // [(generated_documentation_page, &[broken_links])]
  35 const LINKCHECK_EXCEPTIONS: &[(&str, &[&str])] = &[
  36     // These are methods on slice, and `Self` does not work on primitive impls
  37     // in intra-doc links (primitive impls are weird)
  38     // https://github.com/rust-lang/rust/issues/62834 is necessary to be
  39     // able to link to slices
  40     ("std/io/struct.IoSlice.html", &["#method.as_mut_ptr", "#method.sort_by_key"]),
  41     // These try to link to std::collections, but are defined in alloc
  42     // https://github.com/rust-lang/rust/issues/74481
  43     ("std/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
  44     ("std/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
  45     ("alloc/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
  46     ("alloc/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
  47 ];
  48
  49 #[rustfmt::skip]
  50 const INTRA_DOC_LINK_EXCEPTIONS: &[(&str, &[&str])] = &[
  51     // This will never have links that are not in other pages.
  52     // To avoid repeating the exceptions twice, an empty list means all broken links are allowed.
  53     ("reference/print.html", &[]),
  54     // All the reference 'links' are actually ENBF highlighted as code
  55     ("reference/comments.html", &[
  56          "/</code> <code>!",
  57          "*</code> <code>!",
  58     ]),
  59     ("reference/identifiers.html", &[
  60          "a</code>-<code>z</code> <code>A</code>-<code>Z",
  61          "a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
  62          "a</code>-<code>z</code> <code>A</code>-<code>Z</code>] [<code>a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
  63     ]),
  64     ("reference/tokens.html", &[
  65          "0</code>-<code>1",
  66          "0</code>-<code>7",
  67          "0</code>-<code>9",
  68          "0</code>-<code>9",
  69          "0</code>-<code>9</code> <code>a</code>-<code>f</code> <code>A</code>-<code>F",
  70     ]),
  71     ("reference/notation.html", &[
  72          "b</code> <code>B",
  73          "a</code>-<code>z",
  74     ]),
  75     // This is being used in the sense of 'inclusive range', not a markdown link
  76     ("core/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
  77     ("std/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
  78     ("core/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
  79     ("alloc/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
  80     ("std/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
  81
  82 ];
  83
  84 static BROKEN_INTRA_DOC_LINK: Lazy<Regex> =
  85     Lazy::new(|| Regex::new(r#"\[<code>(.*)</code>\]"#).unwrap());
  86
  87 macro_rules! t {
  88     ($e:expr) => {
  89         match $e {
  90             Ok(e) => e,
  91             Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
  92         }
  93     };
  94 }
  95
  96 fn main() {
  97     let docs = env::args_os().nth(1).unwrap();
  98     let docs = env::current_dir().unwrap().join(docs);
  99     let mut errors = false;
 100     walk(&mut HashMap::new(), &docs, &docs, &mut errors);
 101     if errors {
 102         panic!("found some broken links");
 103     }
 104 }
 105
 106 #[derive(Debug)]
 107 pub enum LoadError {
 108     IOError(std::io::Error),
 109     BrokenRedirect(PathBuf, std::io::Error),
 110     IsRedirect,
 111 }
 112
 113 enum Redirect {
 114     SkipRedirect,
 115     FromRedirect(bool),
 116 }
 117
 118 struct FileEntry {
 119     source: Rc<String>,
 120     ids: HashSet<String>,
 121 }
 122
 123 type Cache = HashMap<PathBuf, FileEntry>;
 124
 125 fn small_url_encode(s: &str) -> String {
 126     s.replace("<", "%3C")
 127         .replace(">", "%3E")
 128         .replace(" ", "%20")
 129         .replace("?", "%3F")
 130         .replace("'", "%27")
 131         .replace("&", "%26")
 132         .replace(",", "%2C")
 133         .replace(":", "%3A")
 134         .replace(";", "%3B")
 135         .replace("[", "%5B")
 136         .replace("]", "%5D")
 137         .replace("\"", "%22")
 138 }
 139
 140 impl FileEntry {
 141     fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
 142         if self.ids.is_empty() {
 143             with_attrs_in_source(contents, " id", |fragment, i, _| {
 144                 let frag = fragment.trim_start_matches("#").to_owned();
 145                 let encoded = small_url_encode(&frag);
 146                 if !self.ids.insert(frag) {
 147                     *errors = true;
 148                     println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
 149                 }
 150                 // Just in case, we also add the encoded id.
 151                 self.ids.insert(encoded);
 152             });
 153         }
 154     }
 155 }
 156
 157 fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
 158     for entry in t!(dir.read_dir()).map(|e| t!(e)) {
 159         let path = entry.path();
 160         let kind = t!(entry.file_type());
 161         if kind.is_dir() {
 162             walk(cache, root, &path, errors);
 163         } else {
 164             let pretty_path = check(cache, root, &path, errors);
 165             if let Some(pretty_path) = pretty_path {
 166                 let entry = cache.get_mut(&pretty_path).unwrap();
 167                 // we don't need the source anymore,
 168                 // so drop to reduce memory-usage
 169                 entry.source = Rc::new(String::new());
 170             }
 171         }
 172     }
 173 }
 174
 175 fn is_intra_doc_exception(file: &Path, link: &str) -> bool {
 176     if let Some(entry) = INTRA_DOC_LINK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
 177         entry.1.is_empty() || entry.1.contains(&link)
 178     } else {
 179         false
 180     }
 181 }
 182
 183 fn is_exception(file: &Path, link: &str) -> bool {
 184     if let Some(entry) = LINKCHECK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
 185         entry.1.contains(&link)
 186     } else {
 187         // FIXME(#63351): Concat trait in alloc/slice reexported in primitive page
 188         //
 189         // NOTE: This cannot be added to `LINKCHECK_EXCEPTIONS` because the resolved path
 190         // calculated in `check` function is outside `build/<triple>/doc` dir.
 191         // So the `strip_prefix` method just returns the old absolute broken path.
 192         if file.ends_with("std/primitive.slice.html") {
 193             if link.ends_with("primitive.slice.html") {
 194                 return true;
 195             }
 196         }
 197         false
 198     }
 199 }
 200
 201 fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
 202     // Ignore non-HTML files.
 203     if file.extension().and_then(|s| s.to_str()) != Some("html") {
 204         return None;
 205     }
 206
 207     let res = load_file(cache, root, file, SkipRedirect);
 208     let (pretty_file, contents) = match res {
 209         Ok(res) => res,
 210         Err(_) => return None,
 211     };
 212     {
 213         cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
 214     }
 215
 216     // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
 217     with_attrs_in_source(&contents, " href", |url, i, base| {
 218         // Ignore external URLs
 219         if url.starts_with("http:")
 220             || url.starts_with("https:")
 221             || url.starts_with("javascript:")
 222             || url.starts_with("ftp:")
 223             || url.starts_with("irc:")
 224             || url.starts_with("data:")
 225         {
 226             return;
 227         }
 228         let (url, fragment) = match url.split_once('#') {
 229             None => (url, None),
 230             Some((url, fragment)) => (url, Some(fragment)),
 231         };
 232         // NB: the `splitn` always succeeds, even if the delimiter is not present.
 233         let url = url.splitn(2, '?').next().unwrap();
 234
 235         // Once we've plucked out the URL, parse it using our base url and
 236         // then try to extract a file path.
 237         let mut path = file.to_path_buf();
 238         if !base.is_empty() || !url.is_empty() {
 239             path.pop();
 240             for part in Path::new(base).join(url).components() {
 241                 match part {
 242                     Component::Prefix(_) | Component::RootDir => {
 243                         // Avoid absolute paths as they make the docs not
 244                         // relocatable by making assumptions on where the docs
 245                         // are hosted relative to the site root.
 246                         *errors = true;
 247                         println!(
 248                             "{}:{}: absolute path - {}",
 249                             pretty_file.display(),
 250                             i + 1,
 251                             Path::new(base).join(url).display()
 252                         );
 253                         return;
 254                     }
 255                     Component::CurDir => {}
 256                     Component::ParentDir => {
 257                         path.pop();
 258                     }
 259                     Component::Normal(s) => {
 260                         path.push(s);
 261                     }
 262                 }
 263             }
 264         }
 265
 266         // Alright, if we've found a file name then this file had better
 267         // exist! If it doesn't then we register and print an error.
 268         if path.exists() {
 269             if path.is_dir() {
 270                 // Links to directories show as directory listings when viewing
 271                 // the docs offline so it's best to avoid them.
 272                 *errors = true;
 273                 let pretty_path = path.strip_prefix(root).unwrap_or(&path);
 274                 println!(
 275                     "{}:{}: directory link - {}",
 276                     pretty_file.display(),
 277                     i + 1,
 278                     pretty_path.display()
 279                 );
 280                 return;
 281             }
 282             if let Some(extension) = path.extension() {
 283                 // Ignore none HTML files.
 284                 if extension != "html" {
 285                     return;
 286                 }
 287             }
 288             let res = load_file(cache, root, &path, FromRedirect(false));
 289             let (pretty_path, contents) = match res {
 290                 Ok(res) => res,
 291                 Err(LoadError::IOError(err)) => {
 292                     panic!("error loading {}: {}", path.display(), err);
 293                 }
 294                 Err(LoadError::BrokenRedirect(target, _)) => {
 295                     *errors = true;
 296                     println!(
 297                         "{}:{}: broken redirect to {}",
 298                         pretty_file.display(),
 299                         i + 1,
 300                         target.display()
 301                     );
 302                     return;
 303                 }
 304                 Err(LoadError::IsRedirect) => unreachable!(),
 305             };
 306
 307             if let Some(ref fragment) = fragment {
 308                 // Fragments like `#1-6` are most likely line numbers to be
 309                 // interpreted by javascript, so we're ignoring these
 310                 if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
 311                     return;
 312                 }
 313
 314                 // These appear to be broken in mdbook right now?
 315                 if fragment.starts_with('-') {
 316                     return;
 317                 }
 318
 319                 let entry = &mut cache.get_mut(&pretty_path).unwrap();
 320                 entry.parse_ids(&pretty_path, &contents, errors);
 321
 322                 if !entry.ids.contains(*fragment) && !is_exception(file, &format!("#{}", fragment))
 323                 {
 324                     *errors = true;
 325                     print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
 326                     println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
 327                 };
 328             }
 329         } else {
 330             let pretty_path = path.strip_prefix(root).unwrap_or(&path);
 331             if !is_exception(file, pretty_path.to_str().unwrap()) {
 332                 *errors = true;
 333                 print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
 334                 println!("{}", pretty_path.display());
 335             }
 336         }
 337     });
 338
 339     // Search for intra-doc links that rustdoc didn't warn about
 340     // FIXME(#77199, 77200) Rustdoc should just warn about these directly.
 341     // NOTE: only looks at one line at a time; in practice this should find most links
 342     for (i, line) in contents.lines().enumerate() {
 343         for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
 344             if !is_intra_doc_exception(file, &broken_link[1]) {
 345                 *errors = true;
 346                 print!("{}:{}: broken intra-doc link - ", pretty_file.display(), i + 1);
 347                 println!("{}", &broken_link[0]);
 348             }
 349         }
 350     }
 351     Some(pretty_file)
 352 }
 353
 354 fn load_file(
 355     cache: &mut Cache,
 356     root: &Path,
 357     file: &Path,
 358     redirect: Redirect,
 359 ) -> Result<(PathBuf, Rc<String>), LoadError> {
 360     let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));
 361
 362     let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
 363         Entry::Occupied(entry) => (None, entry.get().source.clone()),
 364         Entry::Vacant(entry) => {
 365             let contents = match fs::read_to_string(file) {
 366                 Ok(s) => Rc::new(s),
 367                 Err(err) => {
 368                     return Err(if let FromRedirect(true) = redirect {
 369                         LoadError::BrokenRedirect(file.to_path_buf(), err)
 370                     } else {
 371                         LoadError::IOError(err)
 372                     });
 373                 }
 374             };
 375
 376             let maybe = maybe_redirect(&contents);
 377             if maybe.is_some() {
 378                 if let SkipRedirect = redirect {
 379                     return Err(LoadError::IsRedirect);
 380                 }
 381             } else {
 382                 entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
 383             }
 384             (maybe, contents)
 385         }
 386     };
 387     match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
 388         Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
 389         None => Ok((pretty_file, contents)),
 390     }
 391 }
 392
 393 fn maybe_redirect(source: &str) -> Option<String> {
 394     const REDIRECT: &str = "<p>Redirecting to <a href=";
 395
 396     let mut lines = source.lines();
 397     let redirect_line = lines.nth(6)?;
 398
 399     redirect_line.find(REDIRECT).map(|i| {
 400         let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
 401         let pos_quote = rest.find('"').unwrap();
 402         rest[..pos_quote].to_owned()
 403     })
 404 }
 405
 406 fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
 407     let mut base = "";
 408     for (i, mut line) in contents.lines().enumerate() {
 409         while let Some(j) = line.find(attr) {
 410             let rest = &line[j + attr.len()..];
 411             // The base tag should always be the first link in the document so
 412             // we can get away with using one pass.
 413             let is_base = line[..j].ends_with("<base");
 414             line = rest;
 415             let pos_equals = match rest.find('=') {
 416                 Some(i) => i,
 417                 None => continue,
 418             };
 419             if rest[..pos_equals].trim_start_matches(' ') != "" {
 420                 continue;
 421             }
 422
 423             let rest = &rest[pos_equals + 1..];
 424
 425             let pos_quote = match rest.find(&['"', '\''][..]) {
 426                 Some(i) => i,
 427                 None => continue,
 428             };
 429             let quote_delim = rest.as_bytes()[pos_quote] as char;
 430
 431             if rest[..pos_quote].trim_start_matches(' ') != "" {
 432                 continue;
 433             }
 434             let rest = &rest[pos_quote + 1..];
 435             let url = match rest.find(quote_delim) {
 436                 Some(i) => &rest[..i],
 437                 None => continue,
 438             };
 439             if is_base {
 440                 base = url;
 441                 continue;
 442             }
 443             f(url, i, base)
 444         }
 445     }
 446 }