]> git.proxmox.com Git - rustc.git/blob - src/librustdoc/html/markdown.rs
Imported Upstream version 1.2.0+dfsg1
[rustc.git] / src / librustdoc / html / markdown.rs
1 // Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 //! Markdown formatting for rustdoc
12 //!
13 //! This module implements markdown formatting through the hoedown C-library
14 //! (bundled into the rust runtime). This module self-contains the C bindings
15 //! and necessary legwork to render markdown, and exposes all of the
16 //! functionality through a unit-struct, `Markdown`, which has an implementation
17 //! of `fmt::String`. Example usage:
18 //!
19 //! ```rust,ignore
20 //! use rustdoc::html::markdown::Markdown;
21 //!
22 //! let s = "My *markdown* _text_";
23 //! let html = format!("{}", Markdown(s));
24 //! // ... something using html
25 //! ```
26
27 #![allow(dead_code)]
28 #![allow(non_camel_case_types)]
29
30 use libc;
31 use std::ascii::AsciiExt;
32 use std::cell::RefCell;
33 use std::collections::HashMap;
34 use std::default::Default;
35 use std::ffi::CString;
36 use std::fmt;
37 use std::slice;
38 use std::str;
39
40 use html::toc::TocBuilder;
41 use html::highlight;
42 use html::escape::Escape;
43 use test;
44
45 /// A unit struct which has the `fmt::String` trait implemented. When
46 /// formatted, this struct will emit the HTML corresponding to the rendered
47 /// version of the contained markdown string.
48 pub struct Markdown<'a>(pub &'a str);
49 /// A unit struct like `Markdown`, that renders the markdown with a
50 /// table of contents.
51 pub struct MarkdownWithToc<'a>(pub &'a str);
52
53 const DEF_OUNIT: libc::size_t = 64;
54 const HOEDOWN_EXT_NO_INTRA_EMPHASIS: libc::c_uint = 1 << 10;
55 const HOEDOWN_EXT_TABLES: libc::c_uint = 1 << 0;
56 const HOEDOWN_EXT_FENCED_CODE: libc::c_uint = 1 << 1;
57 const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3;
58 const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4;
59 const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8;
60 const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2;
61
62 const HOEDOWN_EXTENSIONS: libc::c_uint =
63 HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES |
64 HOEDOWN_EXT_FENCED_CODE | HOEDOWN_EXT_AUTOLINK |
65 HOEDOWN_EXT_STRIKETHROUGH | HOEDOWN_EXT_SUPERSCRIPT |
66 HOEDOWN_EXT_FOOTNOTES;
67
68 type hoedown_document = libc::c_void; // this is opaque to us
69
70 type blockcodefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
71 *const hoedown_buffer, *mut libc::c_void);
72
73 type headerfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
74 libc::c_int, *mut libc::c_void);
75
76 type codespanfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
77 *mut libc::c_void);
78
79 type linkfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
80 *const hoedown_buffer, *const hoedown_buffer,
81 *mut libc::c_void) -> libc::c_int;
82
83 type normaltextfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
84 *mut libc::c_void);
85
86 #[repr(C)]
87 struct hoedown_renderer {
88 opaque: *mut libc::c_void,
89
90 blockcode: Option<blockcodefn>,
91 blockquote: Option<extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
92 *mut libc::c_void)>,
93 blockhtml: Option<extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
94 *mut libc::c_void)>,
95 header: Option<headerfn>,
96 other_block_level_callbacks: [libc::size_t; 9],
97
98 /* span level callbacks - NULL or return 0 prints the span verbatim */
99 autolink: libc::size_t, // unused
100 codespan: Option<codespanfn>,
101 other_span_level_callbacks_1: [libc::size_t; 7],
102 link: Option<linkfn>,
103 other_span_level_callbacks_2: [libc::size_t; 5],
104 // hoedown will add `math` callback here, but we use an old version of it.
105
106 /* low level callbacks - NULL copies input directly into the output */
107 entity: Option<extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
108 *mut libc::c_void)>,
109 normal_text: Option<normaltextfn>,
110
111 /* header and footer */
112 doc_header: Option<extern "C" fn(*mut hoedown_buffer, *mut libc::c_void)>,
113 doc_footer: Option<extern "C" fn(*mut hoedown_buffer, *mut libc::c_void)>,
114 }
115
116 #[repr(C)]
117 struct hoedown_html_renderer_state {
118 opaque: *mut libc::c_void,
119 toc_data: html_toc_data,
120 flags: libc::c_uint,
121 link_attributes: Option<extern "C" fn(*mut hoedown_buffer,
122 *const hoedown_buffer,
123 *mut libc::c_void)>,
124 }
125
126 #[repr(C)]
127 struct html_toc_data {
128 header_count: libc::c_int,
129 current_level: libc::c_int,
130 level_offset: libc::c_int,
131 nesting_level: libc::c_int,
132 }
133
134 struct MyOpaque {
135 dfltblk: extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
136 *const hoedown_buffer, *mut libc::c_void),
137 toc_builder: Option<TocBuilder>,
138 }
139
140 #[repr(C)]
141 struct hoedown_buffer {
142 data: *const u8,
143 size: libc::size_t,
144 asize: libc::size_t,
145 unit: libc::size_t,
146 }
147
148 // hoedown FFI
149 #[link(name = "hoedown", kind = "static")]
150 extern {
151 fn hoedown_html_renderer_new(render_flags: libc::c_uint,
152 nesting_level: libc::c_int)
153 -> *mut hoedown_renderer;
154 fn hoedown_html_renderer_free(renderer: *mut hoedown_renderer);
155
156 fn hoedown_document_new(rndr: *mut hoedown_renderer,
157 extensions: libc::c_uint,
158 max_nesting: libc::size_t) -> *mut hoedown_document;
159 fn hoedown_document_render(doc: *mut hoedown_document,
160 ob: *mut hoedown_buffer,
161 document: *const u8,
162 doc_size: libc::size_t);
163 fn hoedown_document_free(md: *mut hoedown_document);
164
165 fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer;
166 fn hoedown_buffer_put(b: *mut hoedown_buffer, c: *const libc::c_char,
167 n: libc::size_t);
168 fn hoedown_buffer_puts(b: *mut hoedown_buffer, c: *const libc::c_char);
169 fn hoedown_buffer_free(b: *mut hoedown_buffer);
170
171 }
172
173 // hoedown_buffer helpers
174 impl hoedown_buffer {
175 fn as_bytes(&self) -> &[u8] {
176 unsafe { slice::from_raw_parts(self.data, self.size as usize) }
177 }
178 }
179
180 /// Returns Some(code) if `s` is a line that should be stripped from
181 /// documentation but used in example code. `code` is the portion of
182 /// `s` that should be used in tests. (None for lines that should be
183 /// left as-is.)
184 fn stripped_filtered_line<'a>(s: &'a str) -> Option<&'a str> {
185 let trimmed = s.trim();
186 if trimmed == "#" {
187 Some("")
188 } else if trimmed.starts_with("# ") {
189 Some(&trimmed[2..])
190 } else {
191 None
192 }
193 }
194
195 /// Returns a new string with all consecutive whitespace collapsed into
196 /// single spaces.
197 ///
198 /// Any leading or trailing whitespace will be trimmed.
199 fn collapse_whitespace(s: &str) -> String {
200 s.split(|c: char| c.is_whitespace()).filter(|s| {
201 !s.is_empty()
202 }).collect::<Vec<_>>().connect(" ")
203 }
204
205 thread_local!(static USED_HEADER_MAP: RefCell<HashMap<String, usize>> = {
206 RefCell::new(HashMap::new())
207 });
208
209 thread_local!(pub static PLAYGROUND_KRATE: RefCell<Option<Option<String>>> = {
210 RefCell::new(None)
211 });
212
213 pub fn render(w: &mut fmt::Formatter, s: &str, print_toc: bool) -> fmt::Result {
214 extern fn block(ob: *mut hoedown_buffer, orig_text: *const hoedown_buffer,
215 lang: *const hoedown_buffer, opaque: *mut libc::c_void) {
216 unsafe {
217 if orig_text.is_null() { return }
218
219 let opaque = opaque as *mut hoedown_html_renderer_state;
220 let my_opaque: &MyOpaque = &*((*opaque).opaque as *const MyOpaque);
221 let text = (*orig_text).as_bytes();
222 let origtext = str::from_utf8(text).unwrap();
223 debug!("docblock: ==============\n{:?}\n=======", text);
224 let rendered = if lang.is_null() {
225 false
226 } else {
227 let rlang = (*lang).as_bytes();
228 let rlang = str::from_utf8(rlang).unwrap();
229 if !LangString::parse(rlang).rust {
230 (my_opaque.dfltblk)(ob, orig_text, lang,
231 opaque as *mut libc::c_void);
232 true
233 } else {
234 false
235 }
236 };
237
238 let lines = origtext.lines().filter(|l| {
239 stripped_filtered_line(*l).is_none()
240 });
241 let text = lines.collect::<Vec<&str>>().connect("\n");
242 if rendered { return }
243 PLAYGROUND_KRATE.with(|krate| {
244 let mut s = String::new();
245 krate.borrow().as_ref().map(|krate| {
246 let test = origtext.lines().map(|l| {
247 stripped_filtered_line(l).unwrap_or(l)
248 }).collect::<Vec<&str>>().connect("\n");
249 let krate = krate.as_ref().map(|s| &**s);
250 let test = test::maketest(&test, krate, false,
251 &Default::default());
252 s.push_str(&format!("<span class='rusttest'>{}</span>", Escape(&test)));
253 });
254 s.push_str(&highlight::highlight(&text,
255 Some("rust-example-rendered"),
256 None));
257 let output = CString::new(s).unwrap();
258 hoedown_buffer_puts(ob, output.as_ptr());
259 })
260 }
261 }
262
263 extern fn header(ob: *mut hoedown_buffer, text: *const hoedown_buffer,
264 level: libc::c_int, opaque: *mut libc::c_void) {
265 // hoedown does this, we may as well too
266 unsafe { hoedown_buffer_puts(ob, "\n\0".as_ptr() as *const _); }
267
268 // Extract the text provided
269 let s = if text.is_null() {
270 "".to_string()
271 } else {
272 let s = unsafe { (*text).as_bytes() };
273 str::from_utf8(s).unwrap().to_string()
274 };
275
276 // Transform the contents of the header into a hyphenated string
277 let id = s.split_whitespace().map(|s| s.to_ascii_lowercase())
278 .collect::<Vec<String>>().connect("-");
279
280 // This is a terrible hack working around how hoedown gives us rendered
281 // html for text rather than the raw text.
282
283 let opaque = opaque as *mut hoedown_html_renderer_state;
284 let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) };
285
286 // Make sure our hyphenated ID is unique for this page
287 let id = USED_HEADER_MAP.with(|map| {
288 let id = id.replace("<code>", "").replace("</code>", "").to_string();
289 let id = match map.borrow_mut().get_mut(&id) {
290 None => id,
291 Some(a) => { *a += 1; format!("{}-{}", id, *a - 1) }
292 };
293 map.borrow_mut().insert(id.clone(), 1);
294 id
295 });
296
297 let sec = match opaque.toc_builder {
298 Some(ref mut builder) => {
299 builder.push(level as u32, s.clone(), id.clone())
300 }
301 None => {""}
302 };
303
304 // Render the HTML
305 let text = format!(r##"<h{lvl} id="{id}" class='section-header'><a
306 href="#{id}">{sec}{}</a></h{lvl}>"##,
307 s, lvl = level, id = id,
308 sec = if sec.is_empty() {
309 sec.to_string()
310 } else {
311 format!("{} ", sec)
312 });
313
314 let text = CString::new(text).unwrap();
315 unsafe { hoedown_buffer_puts(ob, text.as_ptr()) }
316 }
317
318 reset_headers();
319
320 extern fn codespan(ob: *mut hoedown_buffer, text: *const hoedown_buffer, _: *mut libc::c_void) {
321 let content = if text.is_null() {
322 "".to_string()
323 } else {
324 let bytes = unsafe { (*text).as_bytes() };
325 let s = str::from_utf8(bytes).unwrap();
326 collapse_whitespace(s)
327 };
328
329 let content = format!("<code>{}</code>", Escape(&content));
330 let element = CString::new(content).unwrap();
331 unsafe { hoedown_buffer_puts(ob, element.as_ptr()); }
332 }
333
334 unsafe {
335 let ob = hoedown_buffer_new(DEF_OUNIT);
336 let renderer = hoedown_html_renderer_new(0, 0);
337 let mut opaque = MyOpaque {
338 dfltblk: (*renderer).blockcode.unwrap(),
339 toc_builder: if print_toc {Some(TocBuilder::new())} else {None}
340 };
341 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
342 = &mut opaque as *mut _ as *mut libc::c_void;
343 (*renderer).blockcode = Some(block);
344 (*renderer).header = Some(header);
345 (*renderer).codespan = Some(codespan);
346
347 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
348 hoedown_document_render(document, ob, s.as_ptr(),
349 s.len() as libc::size_t);
350 hoedown_document_free(document);
351
352 hoedown_html_renderer_free(renderer);
353
354 let mut ret = match opaque.toc_builder {
355 Some(b) => write!(w, "<nav id=\"TOC\">{}</nav>", b.into_toc()),
356 None => Ok(())
357 };
358
359 if ret.is_ok() {
360 let buf = (*ob).as_bytes();
361 ret = w.write_str(str::from_utf8(buf).unwrap());
362 }
363 hoedown_buffer_free(ob);
364 ret
365 }
366 }
367
368 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
369 extern fn block(_ob: *mut hoedown_buffer,
370 text: *const hoedown_buffer,
371 lang: *const hoedown_buffer,
372 opaque: *mut libc::c_void) {
373 unsafe {
374 if text.is_null() { return }
375 let block_info = if lang.is_null() {
376 LangString::all_false()
377 } else {
378 let lang = (*lang).as_bytes();
379 let s = str::from_utf8(lang).unwrap();
380 LangString::parse(s)
381 };
382 if !block_info.rust { return }
383 let text = (*text).as_bytes();
384 let opaque = opaque as *mut hoedown_html_renderer_state;
385 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
386 let text = str::from_utf8(text).unwrap();
387 let lines = text.lines().map(|l| {
388 stripped_filtered_line(l).unwrap_or(l)
389 });
390 let text = lines.collect::<Vec<&str>>().connect("\n");
391 tests.add_test(text.to_string(),
392 block_info.should_panic, block_info.no_run,
393 block_info.ignore, block_info.test_harness);
394 }
395 }
396
397 extern fn header(_ob: *mut hoedown_buffer,
398 text: *const hoedown_buffer,
399 level: libc::c_int, opaque: *mut libc::c_void) {
400 unsafe {
401 let opaque = opaque as *mut hoedown_html_renderer_state;
402 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
403 if text.is_null() {
404 tests.register_header("", level as u32);
405 } else {
406 let text = (*text).as_bytes();
407 let text = str::from_utf8(text).unwrap();
408 tests.register_header(text, level as u32);
409 }
410 }
411 }
412
413 unsafe {
414 let ob = hoedown_buffer_new(DEF_OUNIT);
415 let renderer = hoedown_html_renderer_new(0, 0);
416 (*renderer).blockcode = Some(block);
417 (*renderer).header = Some(header);
418 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
419 = tests as *mut _ as *mut libc::c_void;
420
421 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
422 hoedown_document_render(document, ob, doc.as_ptr(),
423 doc.len() as libc::size_t);
424 hoedown_document_free(document);
425
426 hoedown_html_renderer_free(renderer);
427 hoedown_buffer_free(ob);
428 }
429 }
430
431 #[derive(Eq, PartialEq, Clone, Debug)]
432 struct LangString {
433 should_panic: bool,
434 no_run: bool,
435 ignore: bool,
436 rust: bool,
437 test_harness: bool,
438 }
439
440 impl LangString {
441 fn all_false() -> LangString {
442 LangString {
443 should_panic: false,
444 no_run: false,
445 ignore: false,
446 rust: true, // NB This used to be `notrust = false`
447 test_harness: false,
448 }
449 }
450
451 fn parse(string: &str) -> LangString {
452 let mut seen_rust_tags = false;
453 let mut seen_other_tags = false;
454 let mut data = LangString::all_false();
455
456 let tokens = string.split(|c: char|
457 !(c == '_' || c == '-' || c.is_alphanumeric())
458 );
459
460 for token in tokens {
461 match token {
462 "" => {},
463 "should_panic" => { data.should_panic = true; seen_rust_tags = true; },
464 "no_run" => { data.no_run = true; seen_rust_tags = true; },
465 "ignore" => { data.ignore = true; seen_rust_tags = true; },
466 "rust" => { data.rust = true; seen_rust_tags = true; },
467 "test_harness" => { data.test_harness = true; seen_rust_tags = true; }
468 _ => { seen_other_tags = true }
469 }
470 }
471
472 data.rust &= !seen_other_tags || seen_rust_tags;
473
474 data
475 }
476 }
477
478 /// By default this markdown renderer generates anchors for each header in the
479 /// rendered document. The anchor name is the contents of the header separated
480 /// by hyphens, and a thread-local map is used to disambiguate among duplicate
481 /// headers (numbers are appended).
482 ///
483 /// This method will reset the local table for these headers. This is typically
484 /// used at the beginning of rendering an entire HTML page to reset from the
485 /// previous state (if any).
486 pub fn reset_headers() {
487 USED_HEADER_MAP.with(|s| s.borrow_mut().clear());
488 }
489
490 impl<'a> fmt::Display for Markdown<'a> {
491 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
492 let Markdown(md) = *self;
493 // This is actually common enough to special-case
494 if md.is_empty() { return Ok(()) }
495 render(fmt, md, false)
496 }
497 }
498
499 impl<'a> fmt::Display for MarkdownWithToc<'a> {
500 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
501 let MarkdownWithToc(md) = *self;
502 render(fmt, md, true)
503 }
504 }
505
506 pub fn plain_summary_line(md: &str) -> String {
507 extern fn link(_ob: *mut hoedown_buffer,
508 _link: *const hoedown_buffer,
509 _title: *const hoedown_buffer,
510 content: *const hoedown_buffer,
511 opaque: *mut libc::c_void) -> libc::c_int
512 {
513 unsafe {
514 if !content.is_null() && (*content).size > 0 {
515 let ob = opaque as *mut hoedown_buffer;
516 hoedown_buffer_put(ob, (*content).data as *const libc::c_char,
517 (*content).size);
518 }
519 }
520 1
521 }
522
523 extern fn normal_text(_ob: *mut hoedown_buffer,
524 text: *const hoedown_buffer,
525 opaque: *mut libc::c_void)
526 {
527 unsafe {
528 let ob = opaque as *mut hoedown_buffer;
529 hoedown_buffer_put(ob, (*text).data as *const libc::c_char,
530 (*text).size);
531 }
532 }
533
534 unsafe {
535 let ob = hoedown_buffer_new(DEF_OUNIT);
536 let mut plain_renderer: hoedown_renderer = ::std::mem::zeroed();
537 let renderer: *mut hoedown_renderer = &mut plain_renderer;
538 (*renderer).opaque = ob as *mut libc::c_void;
539 (*renderer).link = Some(link);
540 (*renderer).normal_text = Some(normal_text);
541
542 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
543 hoedown_document_render(document, ob, md.as_ptr(),
544 md.len() as libc::size_t);
545 hoedown_document_free(document);
546 let plain_slice = (*ob).as_bytes();
547 let plain = match str::from_utf8(plain_slice) {
548 Ok(s) => s.to_string(),
549 Err(_) => "".to_string(),
550 };
551 hoedown_buffer_free(ob);
552 plain
553 }
554 }
555
556 #[cfg(test)]
557 mod tests {
558 use super::{LangString, Markdown};
559 use super::{collapse_whitespace, plain_summary_line};
560
561 #[test]
562 fn test_lang_string_parse() {
563 fn t(s: &str,
564 should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool) {
565 assert_eq!(LangString::parse(s), LangString {
566 should_panic: should_panic,
567 no_run: no_run,
568 ignore: ignore,
569 rust: rust,
570 test_harness: test_harness,
571 })
572 }
573
574 // marker | should_panic| no_run | ignore | rust | test_harness
575 t("", false, false, false, true, false);
576 t("rust", false, false, false, true, false);
577 t("sh", false, false, false, false, false);
578 t("ignore", false, false, true, true, false);
579 t("should_panic", true, false, false, true, false);
580 t("no_run", false, true, false, true, false);
581 t("test_harness", false, false, false, true, true);
582 t("{.no_run .example}", false, true, false, true, false);
583 t("{.sh .should_panic}", true, false, false, true, false);
584 t("{.example .rust}", false, false, false, true, false);
585 t("{.test_harness .rust}", false, false, false, true, true);
586 }
587
588 #[test]
589 fn issue_17736() {
590 let markdown = "# title";
591 format!("{}", Markdown(markdown));
592 }
593
594 #[test]
595 fn test_plain_summary_line() {
596 fn t(input: &str, expect: &str) {
597 let output = plain_summary_line(input);
598 assert_eq!(output, expect);
599 }
600
601 t("hello [Rust](http://rust-lang.org) :)", "hello Rust :)");
602 t("code `let x = i32;` ...", "code `let x = i32;` ...");
603 t("type `Type<'static>` ...", "type `Type<'static>` ...");
604 t("# top header", "top header");
605 t("## header", "header");
606 }
607
608 #[test]
609 fn test_collapse_whitespace() {
610 fn t(input: &str, expected: &str) {
611 let actual = collapse_whitespace(input);
612 assert_eq!(actual, expected);
613 }
614
615 t("foo", "foo");
616 t("foo bar baz", "foo bar baz");
617 t(" foo bar", "foo bar");
618 t("\tfoo bar\nbaz", "foo bar baz");
619 t("foo bar \n baz\t\tqux\n", "foo bar baz qux");
620 }
621 }