src/librustdoc/html/markdown.rs

   1 // Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Markdown formatting for rustdoc
  12 //!
  13 //! This module implements markdown formatting through the hoedown C-library
  14 //! (bundled into the rust runtime). This module self-contains the C bindings
  15 //! and necessary legwork to render markdown, and exposes all of the
  16 //! functionality through a unit-struct, `Markdown`, which has an implementation
  17 //! of `fmt::String`. Example usage:
  18 //!
  19 //! ```rust,ignore
  20 //! use rustdoc::html::markdown::Markdown;
  21 //!
  22 //! let s = "My *markdown* _text_";
  23 //! let html = format!("{}", Markdown(s));
  24 //! // ... something using html
  25 //! ```
  26
  27 #![allow(dead_code)]
  28 #![allow(non_camel_case_types)]
  29
  30 use libc;
  31 use std::ascii::AsciiExt;
  32 use std::ffi::CString;
  33 use std::cell::RefCell;
  34 use std::collections::HashMap;
  35 use std::fmt;
  36 use std::slice;
  37 use std::str;
  38
  39 use html::toc::TocBuilder;
  40 use html::highlight;
  41 use html::escape::Escape;
  42 use test;
  43
  44 /// A unit struct which has the `fmt::String` trait implemented. When
  45 /// formatted, this struct will emit the HTML corresponding to the rendered
  46 /// version of the contained markdown string.
  47 pub struct Markdown<'a>(pub &'a str);
  48 /// A unit struct like `Markdown`, that renders the markdown with a
  49 /// table of contents.
  50 pub struct MarkdownWithToc<'a>(pub &'a str);
  51
  52 const DEF_OUNIT: libc::size_t = 64;
  53 const HOEDOWN_EXT_NO_INTRA_EMPHASIS: libc::c_uint = 1 << 10;
  54 const HOEDOWN_EXT_TABLES: libc::c_uint = 1 << 0;
  55 const HOEDOWN_EXT_FENCED_CODE: libc::c_uint = 1 << 1;
  56 const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3;
  57 const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4;
  58 const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8;
  59 const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2;
  60
  61 const HOEDOWN_EXTENSIONS: libc::c_uint =
  62     HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES |
  63     HOEDOWN_EXT_FENCED_CODE | HOEDOWN_EXT_AUTOLINK |
  64     HOEDOWN_EXT_STRIKETHROUGH | HOEDOWN_EXT_SUPERSCRIPT |
  65     HOEDOWN_EXT_FOOTNOTES;
  66
  67 type hoedown_document = libc::c_void;  // this is opaque to us
  68
  69 type blockcodefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
  70                                  *const hoedown_buffer, *mut libc::c_void);
  71
  72 type headerfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
  73                               libc::c_int, *mut libc::c_void);
  74
  75 type linkfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
  76                              *const hoedown_buffer, *const hoedown_buffer,
  77                              *mut libc::c_void) -> libc::c_int;
  78
  79 type normaltextfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
  80                                   *mut libc::c_void);
  81
  82 #[repr(C)]
  83 struct hoedown_renderer {
  84     opaque: *mut libc::c_void,
  85
  86     blockcode: Option<blockcodefn>,
  87     blockquote: Option<extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
  88                                      *mut libc::c_void)>,
  89     blockhtml: Option<extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
  90                                     *mut libc::c_void)>,
  91     header: Option<headerfn>,
  92
  93     other_block_level_callbacks: [libc::size_t; 9],
  94
  95     /* span level callbacks - NULL or return 0 prints the span verbatim */
  96     other_span_level_callbacks_1: [libc::size_t; 9],
  97     link: Option<linkfn>,
  98     other_span_level_callbacks_2: [libc::size_t; 5],
  99     // hoedown will add `math` callback here, but we use an old version of it.
 100
 101     /* low level callbacks - NULL copies input directly into the output */
 102     entity: Option<extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
 103                                  *mut libc::c_void)>,
 104     normal_text: Option<normaltextfn>,
 105
 106     /* header and footer */
 107     doc_header: Option<extern "C" fn(*mut hoedown_buffer, *mut libc::c_void)>,
 108     doc_footer: Option<extern "C" fn(*mut hoedown_buffer, *mut libc::c_void)>,
 109 }
 110
 111 #[repr(C)]
 112 struct hoedown_html_renderer_state {
 113     opaque: *mut libc::c_void,
 114     toc_data: html_toc_data,
 115     flags: libc::c_uint,
 116     link_attributes: Option<extern "C" fn(*mut hoedown_buffer,
 117                                           *const hoedown_buffer,
 118                                           *mut libc::c_void)>,
 119 }
 120
 121 #[repr(C)]
 122 struct html_toc_data {
 123     header_count: libc::c_int,
 124     current_level: libc::c_int,
 125     level_offset: libc::c_int,
 126     nesting_level: libc::c_int,
 127 }
 128
 129 struct MyOpaque {
 130     dfltblk: extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
 131                            *const hoedown_buffer, *mut libc::c_void),
 132     toc_builder: Option<TocBuilder>,
 133 }
 134
 135 #[repr(C)]
 136 struct hoedown_buffer {
 137     data: *const u8,
 138     size: libc::size_t,
 139     asize: libc::size_t,
 140     unit: libc::size_t,
 141 }
 142
 143 // hoedown FFI
 144 #[link(name = "hoedown", kind = "static")]
 145 extern {
 146     fn hoedown_html_renderer_new(render_flags: libc::c_uint,
 147                                  nesting_level: libc::c_int)
 148         -> *mut hoedown_renderer;
 149     fn hoedown_html_renderer_free(renderer: *mut hoedown_renderer);
 150
 151     fn hoedown_document_new(rndr: *mut hoedown_renderer,
 152                             extensions: libc::c_uint,
 153                             max_nesting: libc::size_t) -> *mut hoedown_document;
 154     fn hoedown_document_render(doc: *mut hoedown_document,
 155                                ob: *mut hoedown_buffer,
 156                                document: *const u8,
 157                                doc_size: libc::size_t);
 158     fn hoedown_document_free(md: *mut hoedown_document);
 159
 160     fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer;
 161     fn hoedown_buffer_put(b: *mut hoedown_buffer, c: *const libc::c_char,
 162                           n: libc::size_t);
 163     fn hoedown_buffer_puts(b: *mut hoedown_buffer, c: *const libc::c_char);
 164     fn hoedown_buffer_free(b: *mut hoedown_buffer);
 165
 166 }
 167
 168 // hoedown_buffer helpers
 169 impl hoedown_buffer {
 170     fn as_bytes(&self) -> &[u8] {
 171         unsafe { slice::from_raw_parts(self.data, self.size as usize) }
 172     }
 173 }
 174
 175 /// Returns Some(code) if `s` is a line that should be stripped from
 176 /// documentation but used in example code. `code` is the portion of
 177 /// `s` that should be used in tests. (None for lines that should be
 178 /// left as-is.)
 179 fn stripped_filtered_line<'a>(s: &'a str) -> Option<&'a str> {
 180     let trimmed = s.trim();
 181     if trimmed.starts_with("# ") {
 182         Some(&trimmed[2..])
 183     } else {
 184         None
 185     }
 186 }
 187
 188 thread_local!(static USED_HEADER_MAP: RefCell<HashMap<String, uint>> = {
 189     RefCell::new(HashMap::new())
 190 });
 191
 192 thread_local!(pub static PLAYGROUND_KRATE: RefCell<Option<Option<String>>> = {
 193     RefCell::new(None)
 194 });
 195
 196 pub fn render(w: &mut fmt::Formatter, s: &str, print_toc: bool) -> fmt::Result {
 197     extern fn block(ob: *mut hoedown_buffer, orig_text: *const hoedown_buffer,
 198                     lang: *const hoedown_buffer, opaque: *mut libc::c_void) {
 199         unsafe {
 200             if orig_text.is_null() { return }
 201
 202             let opaque = opaque as *mut hoedown_html_renderer_state;
 203             let my_opaque: &MyOpaque = &*((*opaque).opaque as *const MyOpaque);
 204             let text = (*orig_text).as_bytes();
 205             let origtext = str::from_utf8(text).unwrap();
 206             debug!("docblock: ==============\n{:?}\n=======", text);
 207             let rendered = if lang.is_null() {
 208                 false
 209             } else {
 210                 let rlang = (*lang).as_bytes();
 211                 let rlang = str::from_utf8(rlang).unwrap();
 212                 if !LangString::parse(rlang).rust {
 213                     (my_opaque.dfltblk)(ob, orig_text, lang,
 214                                         opaque as *mut libc::c_void);
 215                     true
 216                 } else {
 217                     false
 218                 }
 219             };
 220
 221             let lines = origtext.lines().filter(|l| {
 222                 stripped_filtered_line(*l).is_none()
 223             });
 224             let text = lines.collect::<Vec<&str>>().connect("\n");
 225             if rendered { return }
 226             PLAYGROUND_KRATE.with(|krate| {
 227                 let mut s = String::new();
 228                 krate.borrow().as_ref().map(|krate| {
 229                     let test = origtext.lines().map(|l| {
 230                         stripped_filtered_line(l).unwrap_or(l)
 231                     }).collect::<Vec<&str>>().connect("\n");
 232                     let krate = krate.as_ref().map(|s| &**s);
 233                     let test = test::maketest(&test, krate, false, false);
 234                     s.push_str(&format!("<span class='rusttest'>{}</span>", Escape(&test)));
 235                 });
 236                 s.push_str(&highlight::highlight(&text,
 237                                                  None,
 238                                                  Some("rust-example-rendered")));
 239                 let output = CString::new(s).unwrap();
 240                 hoedown_buffer_puts(ob, output.as_ptr());
 241             })
 242         }
 243     }
 244
 245     extern fn header(ob: *mut hoedown_buffer, text: *const hoedown_buffer,
 246                      level: libc::c_int, opaque: *mut libc::c_void) {
 247         // hoedown does this, we may as well too
 248         unsafe { hoedown_buffer_puts(ob, "\n\0".as_ptr() as *const _); }
 249
 250         // Extract the text provided
 251         let s = if text.is_null() {
 252             "".to_string()
 253         } else {
 254             let s = unsafe { (*text).as_bytes() };
 255             str::from_utf8(s).unwrap().to_string()
 256         };
 257
 258         // Transform the contents of the header into a hyphenated string
 259         let id = s.words().map(|s| s.to_ascii_lowercase())
 260             .collect::<Vec<String>>().connect("-");
 261
 262         // This is a terrible hack working around how hoedown gives us rendered
 263         // html for text rather than the raw text.
 264
 265         let opaque = opaque as *mut hoedown_html_renderer_state;
 266         let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) };
 267
 268         // Make sure our hyphenated ID is unique for this page
 269         let id = USED_HEADER_MAP.with(|map| {
 270             let id = id.replace("<code>", "").replace("</code>", "").to_string();
 271             let id = match map.borrow_mut().get_mut(&id) {
 272                 None => id,
 273                 Some(a) => { *a += 1; format!("{}-{}", id, *a - 1) }
 274             };
 275             map.borrow_mut().insert(id.clone(), 1);
 276             id
 277         });
 278
 279         let sec = match opaque.toc_builder {
 280             Some(ref mut builder) => {
 281                 builder.push(level as u32, s.clone(), id.clone())
 282             }
 283             None => {""}
 284         };
 285
 286         // Render the HTML
 287         let text = format!(r##"<h{lvl} id="{id}" class='section-header'><a
 288                            href="#{id}">{sec}{}</a></h{lvl}>"##,
 289                            s, lvl = level, id = id,
 290                            sec = if sec.len() == 0 {
 291                                sec.to_string()
 292                            } else {
 293                                format!("{} ", sec)
 294                            });
 295
 296         let text = CString::new(text).unwrap();
 297         unsafe { hoedown_buffer_puts(ob, text.as_ptr()) }
 298     }
 299
 300     reset_headers();
 301
 302     unsafe {
 303         let ob = hoedown_buffer_new(DEF_OUNIT);
 304         let renderer = hoedown_html_renderer_new(0, 0);
 305         let mut opaque = MyOpaque {
 306             dfltblk: (*renderer).blockcode.unwrap(),
 307             toc_builder: if print_toc {Some(TocBuilder::new())} else {None}
 308         };
 309         (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
 310                 = &mut opaque as *mut _ as *mut libc::c_void;
 311         (*renderer).blockcode = Some(block as blockcodefn);
 312         (*renderer).header = Some(header as headerfn);
 313
 314         let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
 315         hoedown_document_render(document, ob, s.as_ptr(),
 316                                 s.len() as libc::size_t);
 317         hoedown_document_free(document);
 318
 319         hoedown_html_renderer_free(renderer);
 320
 321         let mut ret = match opaque.toc_builder {
 322             Some(b) => write!(w, "<nav id=\"TOC\">{}</nav>", b.into_toc()),
 323             None => Ok(())
 324         };
 325
 326         if ret.is_ok() {
 327             let buf = (*ob).as_bytes();
 328             ret = w.write_str(str::from_utf8(buf).unwrap());
 329         }
 330         hoedown_buffer_free(ob);
 331         ret
 332     }
 333 }
 334
 335 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
 336     extern fn block(_ob: *mut hoedown_buffer,
 337                     text: *const hoedown_buffer,
 338                     lang: *const hoedown_buffer,
 339                     opaque: *mut libc::c_void) {
 340         unsafe {
 341             if text.is_null() { return }
 342             let block_info = if lang.is_null() {
 343                 LangString::all_false()
 344             } else {
 345                 let lang = (*lang).as_bytes();
 346                 let s = str::from_utf8(lang).unwrap();
 347                 LangString::parse(s)
 348             };
 349             if !block_info.rust { return }
 350             let text = (*text).as_bytes();
 351             let opaque = opaque as *mut hoedown_html_renderer_state;
 352             let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
 353             let text = str::from_utf8(text).unwrap();
 354             let lines = text.lines().map(|l| {
 355                 stripped_filtered_line(l).unwrap_or(l)
 356             });
 357             let text = lines.collect::<Vec<&str>>().connect("\n");
 358             tests.add_test(text.to_string(),
 359                            block_info.should_fail, block_info.no_run,
 360                            block_info.ignore, block_info.test_harness);
 361         }
 362     }
 363
 364     extern fn header(_ob: *mut hoedown_buffer,
 365                      text: *const hoedown_buffer,
 366                      level: libc::c_int, opaque: *mut libc::c_void) {
 367         unsafe {
 368             let opaque = opaque as *mut hoedown_html_renderer_state;
 369             let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
 370             if text.is_null() {
 371                 tests.register_header("", level as u32);
 372             } else {
 373                 let text = (*text).as_bytes();
 374                 let text = str::from_utf8(text).unwrap();
 375                 tests.register_header(text, level as u32);
 376             }
 377         }
 378     }
 379
 380     unsafe {
 381         let ob = hoedown_buffer_new(DEF_OUNIT);
 382         let renderer = hoedown_html_renderer_new(0, 0);
 383         (*renderer).blockcode = Some(block as blockcodefn);
 384         (*renderer).header = Some(header as headerfn);
 385         (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
 386                 = tests as *mut _ as *mut libc::c_void;
 387
 388         let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
 389         hoedown_document_render(document, ob, doc.as_ptr(),
 390                                 doc.len() as libc::size_t);
 391         hoedown_document_free(document);
 392
 393         hoedown_html_renderer_free(renderer);
 394         hoedown_buffer_free(ob);
 395     }
 396 }
 397
 398 #[derive(Eq, PartialEq, Clone, Debug)]
 399 struct LangString {
 400     should_fail: bool,
 401     no_run: bool,
 402     ignore: bool,
 403     rust: bool,
 404     test_harness: bool,
 405 }
 406
 407 impl LangString {
 408     fn all_false() -> LangString {
 409         LangString {
 410             should_fail: false,
 411             no_run: false,
 412             ignore: false,
 413             rust: true,  // NB This used to be `notrust = false`
 414             test_harness: false,
 415         }
 416     }
 417
 418     fn parse(string: &str) -> LangString {
 419         let mut seen_rust_tags = false;
 420         let mut seen_other_tags = false;
 421         let mut data = LangString::all_false();
 422
 423         let tokens = string.split(|c: char|
 424             !(c == '_' || c == '-' || c.is_alphanumeric())
 425         );
 426
 427         for token in tokens {
 428             match token {
 429                 "" => {},
 430                 "should_fail" => { data.should_fail = true; seen_rust_tags = true; },
 431                 "no_run" => { data.no_run = true; seen_rust_tags = true; },
 432                 "ignore" => { data.ignore = true; seen_rust_tags = true; },
 433                 "rust" => { data.rust = true; seen_rust_tags = true; },
 434                 "test_harness" => { data.test_harness = true; seen_rust_tags = true; }
 435                 _ => { seen_other_tags = true }
 436             }
 437         }
 438
 439         data.rust &= !seen_other_tags || seen_rust_tags;
 440
 441         data
 442     }
 443 }
 444
 445 /// By default this markdown renderer generates anchors for each header in the
 446 /// rendered document. The anchor name is the contents of the header separated
 447 /// by hyphens, and a task-local map is used to disambiguate among duplicate
 448 /// headers (numbers are appended).
 449 ///
 450 /// This method will reset the local table for these headers. This is typically
 451 /// used at the beginning of rendering an entire HTML page to reset from the
 452 /// previous state (if any).
 453 pub fn reset_headers() {
 454     USED_HEADER_MAP.with(|s| s.borrow_mut().clear());
 455 }
 456
 457 impl<'a> fmt::Display for Markdown<'a> {
 458     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
 459         let Markdown(md) = *self;
 460         // This is actually common enough to special-case
 461         if md.len() == 0 { return Ok(()) }
 462         render(fmt, md, false)
 463     }
 464 }
 465
 466 impl<'a> fmt::Display for MarkdownWithToc<'a> {
 467     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
 468         let MarkdownWithToc(md) = *self;
 469         render(fmt, md, true)
 470     }
 471 }
 472
 473 pub fn plain_summary_line(md: &str) -> String {
 474     extern fn link(_ob: *mut hoedown_buffer,
 475                        _link: *const hoedown_buffer,
 476                        _title: *const hoedown_buffer,
 477                        content: *const hoedown_buffer,
 478                        opaque: *mut libc::c_void) -> libc::c_int
 479     {
 480         unsafe {
 481             if !content.is_null() && (*content).size > 0 {
 482                 let ob = opaque as *mut hoedown_buffer;
 483                 hoedown_buffer_put(ob, (*content).data as *const libc::c_char,
 484                                    (*content).size);
 485             }
 486         }
 487         1
 488     }
 489
 490     extern fn normal_text(_ob: *mut hoedown_buffer,
 491                               text: *const hoedown_buffer,
 492                               opaque: *mut libc::c_void)
 493     {
 494         unsafe {
 495             let ob = opaque as *mut hoedown_buffer;
 496             hoedown_buffer_put(ob, (*text).data as *const libc::c_char,
 497                                (*text).size);
 498         }
 499     }
 500
 501     unsafe {
 502         let ob = hoedown_buffer_new(DEF_OUNIT);
 503         let mut plain_renderer: hoedown_renderer = ::std::mem::zeroed();
 504         let renderer = &mut plain_renderer as *mut hoedown_renderer;
 505         (*renderer).opaque = ob as *mut libc::c_void;
 506         (*renderer).link = Some(link as linkfn);
 507         (*renderer).normal_text = Some(normal_text as normaltextfn);
 508
 509         let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
 510         hoedown_document_render(document, ob, md.as_ptr(),
 511                                 md.len() as libc::size_t);
 512         hoedown_document_free(document);
 513         let plain_slice = (*ob).as_bytes();
 514         let plain = match str::from_utf8(plain_slice) {
 515             Ok(s) => s.to_string(),
 516             Err(_) => "".to_string(),
 517         };
 518         hoedown_buffer_free(ob);
 519         plain
 520     }
 521 }
 522
 523 #[cfg(test)]
 524 mod tests {
 525     use super::{LangString, Markdown};
 526     use super::plain_summary_line;
 527
 528     #[test]
 529     fn test_lang_string_parse() {
 530         fn t(s: &str,
 531             should_fail: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool) {
 532             assert_eq!(LangString::parse(s), LangString {
 533                 should_fail: should_fail,
 534                 no_run: no_run,
 535                 ignore: ignore,
 536                 rust: rust,
 537                 test_harness: test_harness,
 538             })
 539         }
 540
 541         // marker                | should_fail | no_run | ignore | rust | test_harness
 542         t("",                      false,        false,   false,   true,  false);
 543         t("rust",                  false,        false,   false,   true,  false);
 544         t("sh",                    false,        false,   false,   false, false);
 545         t("ignore",                false,        false,   true,    true,  false);
 546         t("should_fail",           true,         false,   false,   true,  false);
 547         t("no_run",                false,        true,    false,   true,  false);
 548         t("test_harness",          false,        false,   false,   true,  true);
 549         t("{.no_run .example}",    false,        true,    false,   true,  false);
 550         t("{.sh .should_fail}",    true,         false,   false,   true,  false);
 551         t("{.example .rust}",      false,        false,   false,   true,  false);
 552         t("{.test_harness .rust}", false,        false,   false,   true,  true);
 553     }
 554
 555     #[test]
 556     fn issue_17736() {
 557         let markdown = "# title";
 558         format!("{}", Markdown(markdown));
 559     }
 560
 561     #[test]
 562     fn test_plain_summary_line() {
 563         fn t(input: &str, expect: &str) {
 564             let output = plain_summary_line(input);
 565             assert_eq!(output, expect);
 566         }
 567
 568         t("hello [Rust](http://rust-lang.org) :)", "hello Rust :)");
 569         t("code `let x = i32;` ...", "code `let x = i32;` ...");
 570         t("type `Type<'static>` ...", "type `Type<'static>` ...");
 571         t("# top header", "top header");
 572         t("## header", "header");
 573     }
 574 }