src/tools/linkchecker/main.rs

   1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Script to check the validity of `href` links in our HTML documentation.
  12 //!
  13 //! In the past we've been quite error prone to writing in broken links as most
  14 //! of them are manually rather than automatically added. As files move over
  15 //! time or apis change old links become stale or broken. The purpose of this
  16 //! script is to check all relative links in our documentation to make sure they
  17 //! actually point to a valid place.
  18 //!
  19 //! Currently this doesn't actually do any HTML parsing or anything fancy like
  20 //! that, it just has a simple "regex" to search for `href` and `id` tags.
  21 //! These values are then translated to file URLs if possible and then the
  22 //! destination is asserted to exist.
  23 //!
  24 //! A few whitelisted exceptions are allowed as there's known bugs in rustdoc,
  25 //! but this should catch the majority of "broken link" cases.
  26
  27 use std::env;
  28 use std::fs::File;
  29 use std::io::prelude::*;
  30 use std::path::{Path, PathBuf, Component};
  31 use std::collections::{HashMap, HashSet};
  32 use std::collections::hash_map::Entry;
  33
  34 use Redirect::*;
  35
  36 macro_rules! t {
  37     ($e:expr) => (match $e {
  38         Ok(e) => e,
  39         Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
  40     })
  41 }
  42
  43 fn main() {
  44     let docs = env::args().nth(1).unwrap();
  45     let docs = env::current_dir().unwrap().join(docs);
  46     let mut errors = false;
  47     walk(&mut HashMap::new(), &docs, &docs, &mut errors);
  48     if errors {
  49         panic!("found some broken links");
  50     }
  51 }
  52
  53 #[derive(Debug)]
  54 pub enum LoadError {
  55     IOError(std::io::Error),
  56     BrokenRedirect(PathBuf, std::io::Error),
  57     IsRedirect,
  58 }
  59
  60 enum Redirect {
  61     SkipRedirect,
  62     FromRedirect(bool),
  63 }
  64
  65 struct FileEntry {
  66     source: String,
  67     ids: HashSet<String>,
  68     names: HashSet<String>,
  69 }
  70
  71 type Cache = HashMap<PathBuf, FileEntry>;
  72
  73 impl FileEntry {
  74     fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
  75         if self.ids.is_empty() {
  76             with_attrs_in_source(contents, " id", |fragment, i| {
  77                 let frag = fragment.trim_left_matches("#").to_owned();
  78                 if !self.ids.insert(frag) {
  79                     *errors = true;
  80                     println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
  81                 }
  82             });
  83         }
  84     }
  85
  86     fn parse_names(&mut self, contents: &str) {
  87         if self.names.is_empty() {
  88             with_attrs_in_source(contents, " name", |fragment, _| {
  89                 let frag = fragment.trim_left_matches("#").to_owned();
  90                 self.names.insert(frag);
  91             });
  92         }
  93     }
  94 }
  95
  96 fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
  97     for entry in t!(dir.read_dir()).map(|e| t!(e)) {
  98         let path = entry.path();
  99         let kind = t!(entry.file_type());
 100         if kind.is_dir() {
 101             walk(cache, root, &path, errors);
 102         } else {
 103             let pretty_path = check(cache, root, &path, errors);
 104             if let Some(pretty_path) = pretty_path {
 105                 let entry = cache.get_mut(&pretty_path).unwrap();
 106                 // we don't need the source anymore,
 107                 // so drop to reduce memory-usage
 108                 entry.source = String::new();
 109             }
 110         }
 111     }
 112 }
 113
 114 fn check(cache: &mut Cache,
 115          root: &Path,
 116          file: &Path,
 117          errors: &mut bool)
 118          -> Option<PathBuf> {
 119     // ignore js files as they are not prone to errors as the rest of the
 120     // documentation is and they otherwise bring up false positives.
 121     if file.extension().and_then(|s| s.to_str()) == Some("js") {
 122         return None;
 123     }
 124
 125     // Unfortunately we're not 100% full of valid links today to we need a few
 126     // whitelists to get this past `make check` today.
 127     // FIXME(#32129)
 128     if file.ends_with("std/string/struct.String.html") {
 129         return None;
 130     }
 131     // FIXME(#32553)
 132     if file.ends_with("collections/string/struct.String.html") {
 133         return None;
 134     }
 135     // FIXME(#32130)
 136     if file.ends_with("btree_set/struct.BTreeSet.html") ||
 137        file.ends_with("collections/struct.BTreeSet.html") ||
 138        file.ends_with("collections/btree_map/struct.BTreeMap.html") ||
 139        file.ends_with("collections/hash_map/struct.HashMap.html") {
 140         return None;
 141     }
 142
 143     let res = load_file(cache, root, PathBuf::from(file), SkipRedirect);
 144     let (pretty_file, contents) = match res {
 145         Ok(res) => res,
 146         Err(_) => return None,
 147     };
 148     {
 149         cache.get_mut(&pretty_file)
 150              .unwrap()
 151              .parse_ids(&pretty_file, &contents, errors);
 152         cache.get_mut(&pretty_file)
 153              .unwrap()
 154              .parse_names(&contents);
 155     }
 156
 157     // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
 158     with_attrs_in_source(&contents, " href", |url, i| {
 159         // Ignore external URLs
 160         if url.starts_with("http:") || url.starts_with("https:") ||
 161            url.starts_with("javascript:") || url.starts_with("ftp:") ||
 162            url.starts_with("irc:") || url.starts_with("data:") {
 163             return;
 164         }
 165         let mut parts = url.splitn(2, "#");
 166         let url = parts.next().unwrap();
 167         let fragment = parts.next();
 168         let mut parts = url.splitn(2, "?");
 169         let url = parts.next().unwrap();
 170
 171         // Once we've plucked out the URL, parse it using our base url and
 172         // then try to extract a file path.
 173         let mut path = file.to_path_buf();
 174         if !url.is_empty() {
 175             path.pop();
 176             for part in Path::new(url).components() {
 177                 match part {
 178                     Component::Prefix(_) |
 179                     Component::RootDir => panic!(),
 180                     Component::CurDir => {}
 181                     Component::ParentDir => { path.pop(); }
 182                     Component::Normal(s) => { path.push(s); }
 183                 }
 184             }
 185         }
 186
 187         if let Some(extension) = path.extension() {
 188             // don't check these files
 189             if extension == "png" {
 190                 return;
 191             }
 192         }
 193
 194         // Alright, if we've found a file name then this file had better
 195         // exist! If it doesn't then we register and print an error.
 196         if path.exists() {
 197             if path.is_dir() {
 198                 // Links to directories show as directory listings when viewing
 199                 // the docs offline so it's best to avoid them.
 200                 *errors = true;
 201                 let pretty_path = path.strip_prefix(root).unwrap_or(&path);
 202                 println!("{}:{}: directory link - {}",
 203                          pretty_file.display(),
 204                          i + 1,
 205                          pretty_path.display());
 206                 return;
 207             }
 208             let res = load_file(cache, root, path.clone(), FromRedirect(false));
 209             let (pretty_path, contents) = match res {
 210                 Ok(res) => res,
 211                 Err(LoadError::IOError(err)) => {
 212                     panic!(format!("error loading {}: {}", path.display(), err));
 213                 }
 214                 Err(LoadError::BrokenRedirect(target, _)) => {
 215                     *errors = true;
 216                     println!("{}:{}: broken redirect to {}",
 217                              pretty_file.display(),
 218                              i + 1,
 219                              target.display());
 220                     return;
 221                 }
 222                 Err(LoadError::IsRedirect) => unreachable!(),
 223             };
 224
 225             if let Some(ref fragment) = fragment {
 226                 // Fragments like `#1-6` are most likely line numbers to be
 227                 // interpreted by javascript, so we're ignoring these
 228                 if fragment.splitn(2, '-')
 229                            .all(|f| f.chars().all(|c| c.is_numeric())) {
 230                     return;
 231                 }
 232
 233                 let entry = &mut cache.get_mut(&pretty_path).unwrap();
 234                 entry.parse_ids(&pretty_path, &contents, errors);
 235                 entry.parse_names(&contents);
 236
 237                 if !(entry.ids.contains(*fragment) || entry.names.contains(*fragment)) {
 238                     *errors = true;
 239                     print!("{}:{}: broken link fragment  ",
 240                            pretty_file.display(),
 241                            i + 1);
 242                     println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
 243                 };
 244             }
 245         } else {
 246             *errors = true;
 247             print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
 248             let pretty_path = path.strip_prefix(root).unwrap_or(&path);
 249             println!("{}", pretty_path.display());
 250         }
 251     });
 252     Some(pretty_file)
 253 }
 254
 255 fn load_file(cache: &mut Cache,
 256              root: &Path,
 257              mut file: PathBuf,
 258              redirect: Redirect)
 259              -> Result<(PathBuf, String), LoadError> {
 260     let mut contents = String::new();
 261     let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));
 262
 263     let maybe_redirect = match cache.entry(pretty_file.clone()) {
 264         Entry::Occupied(entry) => {
 265             contents = entry.get().source.clone();
 266             None
 267         }
 268         Entry::Vacant(entry) => {
 269             let mut fp = File::open(file.clone()).map_err(|err| {
 270                 if let FromRedirect(true) = redirect {
 271                     LoadError::BrokenRedirect(file.clone(), err)
 272                 } else {
 273                     LoadError::IOError(err)
 274                 }
 275             })?;
 276             fp.read_to_string(&mut contents).map_err(|err| LoadError::IOError(err))?;
 277
 278             let maybe = maybe_redirect(&contents);
 279             if maybe.is_some() {
 280                 if let SkipRedirect = redirect {
 281                     return Err(LoadError::IsRedirect);
 282                 }
 283             } else {
 284                 entry.insert(FileEntry {
 285                     source: contents.clone(),
 286                     ids: HashSet::new(),
 287                     names: HashSet::new(),
 288                 });
 289             }
 290             maybe
 291         }
 292     };
 293     file.pop();
 294     match maybe_redirect.map(|url| file.join(url)) {
 295         Some(redirect_file) => {
 296             let path = PathBuf::from(redirect_file);
 297             load_file(cache, root, path, FromRedirect(true))
 298         }
 299         None => Ok((pretty_file, contents)),
 300     }
 301 }
 302
 303 fn maybe_redirect(source: &str) -> Option<String> {
 304     const REDIRECT: &'static str = "<p>Redirecting to <a href=";
 305
 306     let mut lines = source.lines();
 307     let redirect_line = match lines.nth(6) {
 308         Some(l) => l,
 309         None => return None,
 310     };
 311
 312     redirect_line.find(REDIRECT).map(|i| {
 313         let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
 314         let pos_quote = rest.find('"').unwrap();
 315         rest[..pos_quote].to_owned()
 316     })
 317 }
 318
 319 fn with_attrs_in_source<F: FnMut(&str, usize)>(contents: &str, attr: &str, mut f: F) {
 320     for (i, mut line) in contents.lines().enumerate() {
 321         while let Some(j) = line.find(attr) {
 322             let rest = &line[j + attr.len()..];
 323             line = rest;
 324             let pos_equals = match rest.find("=") {
 325                 Some(i) => i,
 326                 None => continue,
 327             };
 328             if rest[..pos_equals].trim_left_matches(" ") != "" {
 329                 continue;
 330             }
 331
 332             let rest = &rest[pos_equals + 1..];
 333
 334             let pos_quote = match rest.find(&['"', '\''][..]) {
 335                 Some(i) => i,
 336                 None => continue,
 337             };
 338             let quote_delim = rest.as_bytes()[pos_quote] as char;
 339
 340             if rest[..pos_quote].trim_left_matches(" ") != "" {
 341                 continue;
 342             }
 343             let rest = &rest[pos_quote + 1..];
 344             let url = match rest.find(quote_delim) {
 345                 Some(i) => &rest[..i],
 346                 None => continue,
 347             };
 348             f(url, i)
 349         }
 350     }
 351 }