]>
Commit | Line | Data |
---|---|---|
54a0048b SL |
1 | // Copyright 2016 The Rust Project Developers. See the COPYRIGHT |
2 | // file at the top-level directory of this distribution and at | |
3 | // http://rust-lang.org/COPYRIGHT. | |
4 | // | |
5 | // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | // option. This file may not be copied, modified, or distributed | |
9 | // except according to those terms. | |
10 | ||
11 | //! Script to check the validity of `href` links in our HTML documentation. | |
12 | //! | |
13 | //! In the past we've been quite error prone to writing in broken links as most | |
14 | //! of them are manually rather than automatically added. As files move over | |
15 | //! time or apis change old links become stale or broken. The purpose of this | |
16 | //! script is to check all relative links in our documentation to make sure they | |
17 | //! actually point to a valid place. | |
18 | //! | |
19 | //! Currently this doesn't actually do any HTML parsing or anything fancy like | |
20 | //! that, it just has a simple "regex" to search for `href` and `id` tags. | |
21 | //! These values are then translated to file URLs if possible and then the | |
22 | //! destination is asserted to exist. | |
23 | //! | |
24 | //! A few whitelisted exceptions are allowed as there's known bugs in rustdoc, | |
25 | //! but this should catch the majority of "broken link" cases. | |
26 | ||
54a0048b SL |
27 | use std::env; |
28 | use std::fs::File; | |
29 | use std::io::prelude::*; | |
476ff2be | 30 | use std::path::{Path, PathBuf, Component}; |
54a0048b SL |
31 | use std::collections::{HashMap, HashSet}; |
32 | use std::collections::hash_map::Entry; | |
33 | ||
54a0048b SL |
34 | use Redirect::*; |
35 | ||
36 | macro_rules! t { | |
37 | ($e:expr) => (match $e { | |
38 | Ok(e) => e, | |
39 | Err(e) => panic!("{} failed with {:?}", stringify!($e), e), | |
40 | }) | |
41 | } | |
42 | ||
43 | fn main() { | |
7cac9316 | 44 | let docs = env::args_os().nth(1).unwrap(); |
54a0048b | 45 | let docs = env::current_dir().unwrap().join(docs); |
54a0048b | 46 | let mut errors = false; |
476ff2be | 47 | walk(&mut HashMap::new(), &docs, &docs, &mut errors); |
54a0048b SL |
48 | if errors { |
49 | panic!("found some broken links"); | |
50 | } | |
51 | } | |
52 | ||
53 | #[derive(Debug)] | |
54 | pub enum LoadError { | |
55 | IOError(std::io::Error), | |
56 | BrokenRedirect(PathBuf, std::io::Error), | |
57 | IsRedirect, | |
58 | } | |
59 | ||
60 | enum Redirect { | |
61 | SkipRedirect, | |
62 | FromRedirect(bool), | |
63 | } | |
64 | ||
65 | struct FileEntry { | |
66 | source: String, | |
67 | ids: HashSet<String>, | |
68 | } | |
69 | ||
70 | type Cache = HashMap<PathBuf, FileEntry>; | |
71 | ||
abe05a73 XL |
72 | fn small_url_encode(s: &str) -> String { |
73 | s.replace("<", "%3C") | |
74 | .replace(">", "%3E") | |
75 | .replace(" ", "%20") | |
76 | .replace("?", "%3F") | |
77 | .replace("'", "%27") | |
78 | .replace("&", "%26") | |
79 | .replace(",", "%2C") | |
80 | .replace(":", "%3A") | |
81 | .replace(";", "%3B") | |
82 | .replace("[", "%5B") | |
83 | .replace("]", "%5D") | |
84 | } | |
85 | ||
54a0048b | 86 | impl FileEntry { |
3157f602 | 87 | fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) { |
54a0048b | 88 | if self.ids.is_empty() { |
7cac9316 | 89 | with_attrs_in_source(contents, " id", |fragment, i, _| { |
54a0048b | 90 | let frag = fragment.trim_left_matches("#").to_owned(); |
abe05a73 | 91 | let encoded = small_url_encode(&frag); |
54a0048b SL |
92 | if !self.ids.insert(frag) { |
93 | *errors = true; | |
3157f602 | 94 | println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment); |
54a0048b | 95 | } |
abe05a73 XL |
96 | // Just in case, we also add the encoded id. |
97 | self.ids.insert(encoded); | |
54a0048b SL |
98 | }); |
99 | } | |
100 | } | |
101 | } | |
102 | ||
476ff2be | 103 | fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) { |
54a0048b SL |
104 | for entry in t!(dir.read_dir()).map(|e| t!(e)) { |
105 | let path = entry.path(); | |
106 | let kind = t!(entry.file_type()); | |
54a0048b | 107 | if kind.is_dir() { |
476ff2be | 108 | walk(cache, root, &path, errors); |
54a0048b | 109 | } else { |
476ff2be | 110 | let pretty_path = check(cache, root, &path, errors); |
54a0048b SL |
111 | if let Some(pretty_path) = pretty_path { |
112 | let entry = cache.get_mut(&pretty_path).unwrap(); | |
113 | // we don't need the source anymore, | |
a7813a04 | 114 | // so drop to reduce memory-usage |
54a0048b SL |
115 | entry.source = String::new(); |
116 | } | |
117 | } | |
54a0048b SL |
118 | } |
119 | } | |
120 | ||
121 | fn check(cache: &mut Cache, | |
122 | root: &Path, | |
123 | file: &Path, | |
3157f602 XL |
124 | errors: &mut bool) |
125 | -> Option<PathBuf> { | |
7cac9316 XL |
126 | // Ignore none HTML files. |
127 | if file.extension().and_then(|s| s.to_str()) != Some("html") { | |
cc61c64b XL |
128 | return None; |
129 | } | |
130 | ||
54a0048b SL |
131 | // Unfortunately we're not 100% full of valid links today to we need a few |
132 | // whitelists to get this past `make check` today. | |
133 | // FIXME(#32129) | |
134 | if file.ends_with("std/string/struct.String.html") { | |
135 | return None; | |
136 | } | |
137 | // FIXME(#32553) | |
041b39d2 | 138 | if file.ends_with("string/struct.String.html") { |
54a0048b SL |
139 | return None; |
140 | } | |
141 | // FIXME(#32130) | |
142 | if file.ends_with("btree_set/struct.BTreeSet.html") || | |
041b39d2 XL |
143 | file.ends_with("struct.BTreeSet.html") || |
144 | file.ends_with("btree_map/struct.BTreeMap.html") || | |
145 | file.ends_with("hash_map/struct.HashMap.html") || | |
146 | file.ends_with("hash_set/struct.HashSet.html") { | |
cc61c64b XL |
147 | return None; |
148 | } | |
149 | ||
7cac9316 | 150 | let res = load_file(cache, root, file, SkipRedirect); |
54a0048b SL |
151 | let (pretty_file, contents) = match res { |
152 | Ok(res) => res, | |
153 | Err(_) => return None, | |
154 | }; | |
155 | { | |
3157f602 XL |
156 | cache.get_mut(&pretty_file) |
157 | .unwrap() | |
158 | .parse_ids(&pretty_file, &contents, errors); | |
54a0048b SL |
159 | } |
160 | ||
161 | // Search for anything that's the regex 'href[ ]*=[ ]*".*?"' | |
7cac9316 | 162 | with_attrs_in_source(&contents, " href", |url, i, base| { |
3157f602 XL |
163 | // Ignore external URLs |
164 | if url.starts_with("http:") || url.starts_with("https:") || | |
165 | url.starts_with("javascript:") || url.starts_with("ftp:") || | |
166 | url.starts_with("irc:") || url.starts_with("data:") { | |
167 | return; | |
168 | } | |
476ff2be SL |
169 | let mut parts = url.splitn(2, "#"); |
170 | let url = parts.next().unwrap(); | |
476ff2be SL |
171 | let fragment = parts.next(); |
172 | let mut parts = url.splitn(2, "?"); | |
173 | let url = parts.next().unwrap(); | |
174 | ||
54a0048b | 175 | // Once we've plucked out the URL, parse it using our base url and |
3157f602 | 176 | // then try to extract a file path. |
476ff2be | 177 | let mut path = file.to_path_buf(); |
7cac9316 | 178 | if !base.is_empty() || !url.is_empty() { |
32a655c1 | 179 | path.pop(); |
7cac9316 | 180 | for part in Path::new(base).join(url).components() { |
32a655c1 SL |
181 | match part { |
182 | Component::Prefix(_) | | |
183 | Component::RootDir => panic!(), | |
184 | Component::CurDir => {} | |
185 | Component::ParentDir => { path.pop(); } | |
186 | Component::Normal(s) => { path.push(s); } | |
187 | } | |
3157f602 | 188 | } |
476ff2be | 189 | } |
54a0048b SL |
190 | |
191 | // Alright, if we've found a file name then this file had better | |
192 | // exist! If it doesn't then we register and print an error. | |
193 | if path.exists() { | |
194 | if path.is_dir() { | |
3157f602 XL |
195 | // Links to directories show as directory listings when viewing |
196 | // the docs offline so it's best to avoid them. | |
197 | *errors = true; | |
198 | let pretty_path = path.strip_prefix(root).unwrap_or(&path); | |
199 | println!("{}:{}: directory link - {}", | |
200 | pretty_file.display(), | |
201 | i + 1, | |
202 | pretty_path.display()); | |
54a0048b SL |
203 | return; |
204 | } | |
7cac9316 XL |
205 | if let Some(extension) = path.extension() { |
206 | // Ignore none HTML files. | |
207 | if extension != "html" { | |
208 | return; | |
209 | } | |
210 | } | |
211 | let res = load_file(cache, root, &path, FromRedirect(false)); | |
54a0048b SL |
212 | let (pretty_path, contents) = match res { |
213 | Ok(res) => res, | |
8bb4bdeb | 214 | Err(LoadError::IOError(err)) => { |
7cac9316 | 215 | panic!("error loading {}: {}", path.display(), err); |
8bb4bdeb | 216 | } |
54a0048b | 217 | Err(LoadError::BrokenRedirect(target, _)) => { |
3157f602 XL |
218 | *errors = true; |
219 | println!("{}:{}: broken redirect to {}", | |
220 | pretty_file.display(), | |
221 | i + 1, | |
222 | target.display()); | |
54a0048b SL |
223 | return; |
224 | } | |
225 | Err(LoadError::IsRedirect) => unreachable!(), | |
226 | }; | |
227 | ||
476ff2be | 228 | if let Some(ref fragment) = fragment { |
54a0048b SL |
229 | // Fragments like `#1-6` are most likely line numbers to be |
230 | // interpreted by javascript, so we're ignoring these | |
231 | if fragment.splitn(2, '-') | |
232 | .all(|f| f.chars().all(|c| c.is_numeric())) { | |
233 | return; | |
234 | } | |
235 | ||
236 | let entry = &mut cache.get_mut(&pretty_path).unwrap(); | |
237 | entry.parse_ids(&pretty_path, &contents, errors); | |
238 | ||
7cac9316 | 239 | if !entry.ids.contains(*fragment) { |
54a0048b | 240 | *errors = true; |
7cac9316 | 241 | print!("{}:{}: broken link fragment ", |
3157f602 XL |
242 | pretty_file.display(), |
243 | i + 1); | |
244 | println!("`#{}` pointing to `{}`", fragment, pretty_path.display()); | |
54a0048b SL |
245 | }; |
246 | } | |
247 | } else { | |
248 | *errors = true; | |
249 | print!("{}:{}: broken link - ", pretty_file.display(), i + 1); | |
250 | let pretty_path = path.strip_prefix(root).unwrap_or(&path); | |
251 | println!("{}", pretty_path.display()); | |
252 | } | |
253 | }); | |
254 | Some(pretty_file) | |
255 | } | |
256 | ||
257 | fn load_file(cache: &mut Cache, | |
258 | root: &Path, | |
7cac9316 | 259 | file: &Path, |
3157f602 XL |
260 | redirect: Redirect) |
261 | -> Result<(PathBuf, String), LoadError> { | |
54a0048b SL |
262 | let mut contents = String::new(); |
263 | let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file)); | |
264 | ||
265 | let maybe_redirect = match cache.entry(pretty_file.clone()) { | |
266 | Entry::Occupied(entry) => { | |
267 | contents = entry.get().source.clone(); | |
268 | None | |
3157f602 | 269 | } |
54a0048b | 270 | Entry::Vacant(entry) => { |
7cac9316 | 271 | let mut fp = File::open(file).map_err(|err| { |
54a0048b | 272 | if let FromRedirect(true) = redirect { |
7cac9316 | 273 | LoadError::BrokenRedirect(file.to_path_buf(), err) |
54a0048b SL |
274 | } else { |
275 | LoadError::IOError(err) | |
276 | } | |
9e0c209e SL |
277 | })?; |
278 | fp.read_to_string(&mut contents).map_err(|err| LoadError::IOError(err))?; | |
54a0048b SL |
279 | |
280 | let maybe = maybe_redirect(&contents); | |
281 | if maybe.is_some() { | |
282 | if let SkipRedirect = redirect { | |
283 | return Err(LoadError::IsRedirect); | |
284 | } | |
285 | } else { | |
286 | entry.insert(FileEntry { | |
287 | source: contents.clone(), | |
288 | ids: HashSet::new(), | |
289 | }); | |
290 | } | |
291 | maybe | |
3157f602 | 292 | } |
54a0048b | 293 | }; |
7cac9316 | 294 | match maybe_redirect.map(|url| file.parent().unwrap().join(url)) { |
476ff2be | 295 | Some(redirect_file) => { |
7cac9316 | 296 | load_file(cache, root, &redirect_file, FromRedirect(true)) |
54a0048b | 297 | } |
3157f602 | 298 | None => Ok((pretty_file, contents)), |
54a0048b SL |
299 | } |
300 | } | |
301 | ||
302 | fn maybe_redirect(source: &str) -> Option<String> { | |
303 | const REDIRECT: &'static str = "<p>Redirecting to <a href="; | |
304 | ||
305 | let mut lines = source.lines(); | |
306 | let redirect_line = match lines.nth(6) { | |
307 | Some(l) => l, | |
308 | None => return None, | |
309 | }; | |
310 | ||
311 | redirect_line.find(REDIRECT).map(|i| { | |
312 | let rest = &redirect_line[(i + REDIRECT.len() + 1)..]; | |
313 | let pos_quote = rest.find('"').unwrap(); | |
314 | rest[..pos_quote].to_owned() | |
315 | }) | |
316 | } | |
317 | ||
7cac9316 XL |
318 | fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) { |
319 | let mut base = ""; | |
54a0048b SL |
320 | for (i, mut line) in contents.lines().enumerate() { |
321 | while let Some(j) = line.find(attr) { | |
3157f602 | 322 | let rest = &line[j + attr.len()..]; |
7cac9316 XL |
323 | // The base tag should always be the first link in the document so |
324 | // we can get away with using one pass. | |
325 | let is_base = line[..j].ends_with("<base"); | |
54a0048b SL |
326 | line = rest; |
327 | let pos_equals = match rest.find("=") { | |
328 | Some(i) => i, | |
329 | None => continue, | |
330 | }; | |
331 | if rest[..pos_equals].trim_left_matches(" ") != "" { | |
3157f602 | 332 | continue; |
54a0048b SL |
333 | } |
334 | ||
335 | let rest = &rest[pos_equals + 1..]; | |
336 | ||
337 | let pos_quote = match rest.find(&['"', '\''][..]) { | |
338 | Some(i) => i, | |
339 | None => continue, | |
340 | }; | |
341 | let quote_delim = rest.as_bytes()[pos_quote] as char; | |
342 | ||
343 | if rest[..pos_quote].trim_left_matches(" ") != "" { | |
3157f602 | 344 | continue; |
54a0048b SL |
345 | } |
346 | let rest = &rest[pos_quote + 1..]; | |
347 | let url = match rest.find(quote_delim) { | |
348 | Some(i) => &rest[..i], | |
349 | None => continue, | |
350 | }; | |
7cac9316 XL |
351 | if is_base { |
352 | base = url; | |
353 | continue; | |
354 | } | |
355 | f(url, i, base) | |
54a0048b SL |
356 | } |
357 | } | |
358 | } |