1 // Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Markdown formatting for rustdoc
13 //! This module implements markdown formatting through the hoedown C-library
14 //! (bundled into the rust runtime). This module self-contains the C bindings
15 //! and necessary legwork to render markdown, and exposes all of the
16 //! functionality through a unit-struct, `Markdown`, which has an implementation
17 //! of `fmt::String`. Example usage:
20 //! use rustdoc::html::markdown::Markdown;
22 //! let s = "My *markdown* _text_";
23 //! let html = format!("{}", Markdown(s));
24 //! // ... something using html
28 #![allow(non_camel_case_types)]
31 use std
::ascii
::AsciiExt
;
32 use std
::cell
::RefCell
;
33 use std
::collections
::HashMap
;
34 use std
::default::Default
;
35 use std
::ffi
::CString
;
40 use html
::toc
::TocBuilder
;
42 use html
::escape
::Escape
;
45 /// A unit struct which has the `fmt::String` trait implemented. When
46 /// formatted, this struct will emit the HTML corresponding to the rendered
47 /// version of the contained markdown string.
48 pub struct Markdown
<'a
>(pub &'a
str);
49 /// A unit struct like `Markdown`, that renders the markdown with a
50 /// table of contents.
51 pub struct MarkdownWithToc
<'a
>(pub &'a
str);
53 const DEF_OUNIT
: libc
::size_t
= 64;
54 const HOEDOWN_EXT_NO_INTRA_EMPHASIS
: libc
::c_uint
= 1 << 10;
55 const HOEDOWN_EXT_TABLES
: libc
::c_uint
= 1 << 0;
56 const HOEDOWN_EXT_FENCED_CODE
: libc
::c_uint
= 1 << 1;
57 const HOEDOWN_EXT_AUTOLINK
: libc
::c_uint
= 1 << 3;
58 const HOEDOWN_EXT_STRIKETHROUGH
: libc
::c_uint
= 1 << 4;
59 const HOEDOWN_EXT_SUPERSCRIPT
: libc
::c_uint
= 1 << 8;
60 const HOEDOWN_EXT_FOOTNOTES
: libc
::c_uint
= 1 << 2;
62 const HOEDOWN_EXTENSIONS
: libc
::c_uint
=
63 HOEDOWN_EXT_NO_INTRA_EMPHASIS
| HOEDOWN_EXT_TABLES
|
64 HOEDOWN_EXT_FENCED_CODE
| HOEDOWN_EXT_AUTOLINK
|
65 HOEDOWN_EXT_STRIKETHROUGH
| HOEDOWN_EXT_SUPERSCRIPT
|
66 HOEDOWN_EXT_FOOTNOTES
;
68 type hoedown_document
= libc
::c_void
; // this is opaque to us
70 type blockcodefn
= extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
71 *const hoedown_buffer
, *mut libc
::c_void
);
73 type headerfn
= extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
74 libc
::c_int
, *mut libc
::c_void
);
76 type codespanfn
= extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
79 type linkfn
= extern "C" fn (*mut hoedown_buffer
, *const hoedown_buffer
,
80 *const hoedown_buffer
, *const hoedown_buffer
,
81 *mut libc
::c_void
) -> libc
::c_int
;
83 type normaltextfn
= extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
87 struct hoedown_renderer
{
88 opaque
: *mut libc
::c_void
,
90 blockcode
: Option
<blockcodefn
>,
91 blockquote
: Option
<extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
93 blockhtml
: Option
<extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
95 header
: Option
<headerfn
>,
96 other_block_level_callbacks
: [libc
::size_t
; 9],
98 /* span level callbacks - NULL or return 0 prints the span verbatim */
99 autolink
: libc
::size_t
, // unused
100 codespan
: Option
<codespanfn
>,
101 other_span_level_callbacks_1
: [libc
::size_t
; 7],
102 link
: Option
<linkfn
>,
103 other_span_level_callbacks_2
: [libc
::size_t
; 5],
104 // hoedown will add `math` callback here, but we use an old version of it.
106 /* low level callbacks - NULL copies input directly into the output */
107 entity
: Option
<extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
109 normal_text
: Option
<normaltextfn
>,
111 /* header and footer */
112 doc_header
: Option
<extern "C" fn(*mut hoedown_buffer
, *mut libc
::c_void
)>,
113 doc_footer
: Option
<extern "C" fn(*mut hoedown_buffer
, *mut libc
::c_void
)>,
117 struct hoedown_html_renderer_state
{
118 opaque
: *mut libc
::c_void
,
119 toc_data
: html_toc_data
,
121 link_attributes
: Option
<extern "C" fn(*mut hoedown_buffer
,
122 *const hoedown_buffer
,
127 struct html_toc_data
{
128 header_count
: libc
::c_int
,
129 current_level
: libc
::c_int
,
130 level_offset
: libc
::c_int
,
131 nesting_level
: libc
::c_int
,
135 dfltblk
: extern "C" fn(*mut hoedown_buffer
, *const hoedown_buffer
,
136 *const hoedown_buffer
, *mut libc
::c_void
),
137 toc_builder
: Option
<TocBuilder
>,
141 struct hoedown_buffer
{
149 #[link(name = "hoedown", kind = "static")]
151 fn hoedown_html_renderer_new(render_flags
: libc
::c_uint
,
152 nesting_level
: libc
::c_int
)
153 -> *mut hoedown_renderer
;
154 fn hoedown_html_renderer_free(renderer
: *mut hoedown_renderer
);
156 fn hoedown_document_new(rndr
: *mut hoedown_renderer
,
157 extensions
: libc
::c_uint
,
158 max_nesting
: libc
::size_t
) -> *mut hoedown_document
;
159 fn hoedown_document_render(doc
: *mut hoedown_document
,
160 ob
: *mut hoedown_buffer
,
162 doc_size
: libc
::size_t
);
163 fn hoedown_document_free(md
: *mut hoedown_document
);
165 fn hoedown_buffer_new(unit
: libc
::size_t
) -> *mut hoedown_buffer
;
166 fn hoedown_buffer_put(b
: *mut hoedown_buffer
, c
: *const libc
::c_char
,
168 fn hoedown_buffer_puts(b
: *mut hoedown_buffer
, c
: *const libc
::c_char
);
169 fn hoedown_buffer_free(b
: *mut hoedown_buffer
);
173 // hoedown_buffer helpers
174 impl hoedown_buffer
{
175 fn as_bytes(&self) -> &[u8] {
176 unsafe { slice::from_raw_parts(self.data, self.size as usize) }
180 /// Returns Some(code) if `s` is a line that should be stripped from
181 /// documentation but used in example code. `code` is the portion of
182 /// `s` that should be used in tests. (None for lines that should be
184 fn stripped_filtered_line
<'a
>(s
: &'a
str) -> Option
<&'a
str> {
185 let trimmed
= s
.trim();
188 } else if trimmed
.starts_with("# ") {
195 /// Returns a new string with all consecutive whitespace collapsed into
198 /// Any leading or trailing whitespace will be trimmed.
199 fn collapse_whitespace(s
: &str) -> String
{
200 s
.split(|c
: char| c
.is_whitespace()).filter(|s
| {
202 }).collect
::<Vec
<_
>>().connect(" ")
205 thread_local
!(static USED_HEADER_MAP
: RefCell
<HashMap
<String
, usize>> = {
206 RefCell
::new(HashMap
::new())
209 thread_local
!(pub static PLAYGROUND_KRATE
: RefCell
<Option
<Option
<String
>>> = {
213 pub fn render(w
: &mut fmt
::Formatter
, s
: &str, print_toc
: bool
) -> fmt
::Result
{
214 extern fn block(ob
: *mut hoedown_buffer
, orig_text
: *const hoedown_buffer
,
215 lang
: *const hoedown_buffer
, opaque
: *mut libc
::c_void
) {
217 if orig_text
.is_null() { return }
219 let opaque
= opaque
as *mut hoedown_html_renderer_state
;
220 let my_opaque
: &MyOpaque
= &*((*opaque
).opaque
as *const MyOpaque
);
221 let text
= (*orig_text
).as_bytes();
222 let origtext
= str::from_utf8(text
).unwrap();
223 debug
!("docblock: ==============\n{:?}\n=======", text
);
224 let rendered
= if lang
.is_null() {
227 let rlang
= (*lang
).as_bytes();
228 let rlang
= str::from_utf8(rlang
).unwrap();
229 if !LangString
::parse(rlang
).rust
{
230 (my_opaque
.dfltblk
)(ob
, orig_text
, lang
,
231 opaque
as *mut libc
::c_void
);
238 let lines
= origtext
.lines().filter(|l
| {
239 stripped_filtered_line(*l
).is_none()
241 let text
= lines
.collect
::<Vec
<&str>>().connect("\n");
242 if rendered { return }
243 PLAYGROUND_KRATE
.with(|krate
| {
244 let mut s
= String
::new();
245 krate
.borrow().as_ref().map(|krate
| {
246 let test
= origtext
.lines().map(|l
| {
247 stripped_filtered_line(l
).unwrap_or(l
)
248 }).collect
::<Vec
<&str>>().connect("\n");
249 let krate
= krate
.as_ref().map(|s
| &**s
);
250 let test
= test
::maketest(&test
, krate
, false,
251 &Default
::default());
252 s
.push_str(&format
!("<span class='rusttest'>{}</span>", Escape(&test
)));
254 s
.push_str(&highlight
::highlight(&text
,
255 Some("rust-example-rendered"),
257 let output
= CString
::new(s
).unwrap();
258 hoedown_buffer_puts(ob
, output
.as_ptr());
263 extern fn header(ob
: *mut hoedown_buffer
, text
: *const hoedown_buffer
,
264 level
: libc
::c_int
, opaque
: *mut libc
::c_void
) {
265 // hoedown does this, we may as well too
266 unsafe { hoedown_buffer_puts(ob, "\n\0".as_ptr() as *const _); }
268 // Extract the text provided
269 let s
= if text
.is_null() {
272 let s
= unsafe { (*text).as_bytes() }
;
273 str::from_utf8(s
).unwrap().to_string()
276 // Transform the contents of the header into a hyphenated string
277 let id
= s
.split_whitespace().map(|s
| s
.to_ascii_lowercase())
278 .collect
::<Vec
<String
>>().connect("-");
280 // This is a terrible hack working around how hoedown gives us rendered
281 // html for text rather than the raw text.
283 let opaque
= opaque
as *mut hoedown_html_renderer_state
;
284 let opaque
= unsafe { &mut *((*opaque).opaque as *mut MyOpaque) }
;
286 // Make sure our hyphenated ID is unique for this page
287 let id
= USED_HEADER_MAP
.with(|map
| {
288 let id
= id
.replace("<code>", "").replace("</code>", "").to_string();
289 let id
= match map
.borrow_mut().get_mut(&id
) {
291 Some(a
) => { *a += 1; format!("{}
-{}
", id, *a - 1) }
293 map.borrow_mut().insert(id.clone(), 1);
297 let sec = match opaque.toc_builder {
298 Some(ref mut builder) => {
299 builder.push(level as u32, s.clone(), id.clone())
305 let text = format!(r##"<h{lvl} id
="{id}" class
='section
-header'
><a
306 href
="#{id}">{sec}{}
</a
></h{lvl}
>"##,
307 s, lvl = level, id = id,
308 sec = if sec.is_empty() {
314 let text = CString::new(text).unwrap();
315 unsafe { hoedown_buffer_puts(ob, text.as_ptr()) }
320 extern fn codespan(ob: *mut hoedown_buffer, text: *const hoedown_buffer, _: *mut libc::c_void) {
321 let content = if text.is_null() {
324 let bytes = unsafe { (*text).as_bytes() };
325 let s = str::from_utf8(bytes).unwrap();
326 collapse_whitespace(s)
329 let content = format!("<code
>{}
</code
>", Escape(&content));
330 let element = CString::new(content).unwrap();
331 unsafe { hoedown_buffer_puts(ob, element.as_ptr()); }
335 let ob = hoedown_buffer_new(DEF_OUNIT);
336 let renderer = hoedown_html_renderer_new(0, 0);
337 let mut opaque = MyOpaque {
338 dfltblk: (*renderer).blockcode.unwrap(),
339 toc_builder: if print_toc {Some(TocBuilder::new())} else {None}
341 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
342 = &mut opaque as *mut _ as *mut libc::c_void;
343 (*renderer).blockcode = Some(block);
344 (*renderer).header = Some(header);
345 (*renderer).codespan = Some(codespan);
347 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
348 hoedown_document_render(document, ob, s.as_ptr(),
349 s.len() as libc::size_t);
350 hoedown_document_free(document);
352 hoedown_html_renderer_free(renderer);
354 let mut ret = match opaque.toc_builder {
355 Some(b) => write!(w, "<nav id
=\"TOC
\">{}
</nav
>", b.into_toc()),
360 let buf = (*ob).as_bytes();
361 ret = w.write_str(str::from_utf8(buf).unwrap());
363 hoedown_buffer_free(ob);
368 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
369 extern fn block(_ob: *mut hoedown_buffer,
370 text: *const hoedown_buffer,
371 lang: *const hoedown_buffer,
372 opaque: *mut libc::c_void) {
374 if text.is_null() { return }
375 let block_info = if lang.is_null() {
376 LangString::all_false()
378 let lang = (*lang).as_bytes();
379 let s = str::from_utf8(lang).unwrap();
382 if !block_info.rust { return }
383 let text = (*text).as_bytes();
384 let opaque = opaque as *mut hoedown_html_renderer_state;
385 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
386 let text = str::from_utf8(text).unwrap();
387 let lines = text.lines().map(|l| {
388 stripped_filtered_line(l).unwrap_or(l)
390 let text = lines.collect::<Vec<&str>>().connect("\n");
391 tests.add_test(text.to_string(),
392 block_info.should_panic, block_info.no_run,
393 block_info.ignore, block_info.test_harness);
397 extern fn header(_ob: *mut hoedown_buffer,
398 text: *const hoedown_buffer,
399 level: libc::c_int, opaque: *mut libc::c_void) {
401 let opaque = opaque as *mut hoedown_html_renderer_state;
402 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
404 tests.register_header("", level as u32);
406 let text = (*text).as_bytes();
407 let text = str::from_utf8(text).unwrap();
408 tests.register_header(text, level as u32);
414 let ob = hoedown_buffer_new(DEF_OUNIT);
415 let renderer = hoedown_html_renderer_new(0, 0);
416 (*renderer).blockcode = Some(block);
417 (*renderer).header = Some(header);
418 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
419 = tests as *mut _ as *mut libc::c_void;
421 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
422 hoedown_document_render(document, ob, doc.as_ptr(),
423 doc.len() as libc::size_t);
424 hoedown_document_free(document);
426 hoedown_html_renderer_free(renderer);
427 hoedown_buffer_free(ob);
431 #[derive(Eq, PartialEq, Clone, Debug)]
441 fn all_false() -> LangString {
446 rust: true, // NB This used to be `notrust = false`
451 fn parse(string: &str) -> LangString {
452 let mut seen_rust_tags = false;
453 let mut seen_other_tags = false;
454 let mut data = LangString::all_false();
456 let tokens = string.split(|c: char|
457 !(c == '_' || c == '-' || c.is_alphanumeric())
460 for token in tokens {
463 "should_panic
" => { data.should_panic = true; seen_rust_tags = true; },
464 "no_run
" => { data.no_run = true; seen_rust_tags = true; },
465 "ignore
" => { data.ignore = true; seen_rust_tags = true; },
466 "rust
" => { data.rust = true; seen_rust_tags = true; },
467 "test_harness
" => { data.test_harness = true; seen_rust_tags = true; }
468 _ => { seen_other_tags = true }
472 data.rust &= !seen_other_tags || seen_rust_tags;
478 /// By default this markdown renderer generates anchors for each header in the
479 /// rendered document. The anchor name is the contents of the header separated
480 /// by hyphens, and a thread-local map is used to disambiguate among duplicate
481 /// headers (numbers are appended).
483 /// This method will reset the local table for these headers. This is typically
484 /// used at the beginning of rendering an entire HTML page to reset from the
485 /// previous state (if any).
486 pub fn reset_headers() {
487 USED_HEADER_MAP.with(|s| s.borrow_mut().clear());
490 impl<'a> fmt::Display for Markdown<'a> {
491 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
492 let Markdown(md) = *self;
493 // This is actually common enough to special-case
494 if md.is_empty() { return Ok(()) }
495 render(fmt, md, false)
499 impl<'a> fmt::Display for MarkdownWithToc<'a> {
500 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
501 let MarkdownWithToc(md) = *self;
502 render(fmt, md, true)
506 pub fn plain_summary_line(md: &str) -> String {
507 extern fn link(_ob: *mut hoedown_buffer,
508 _link: *const hoedown_buffer,
509 _title: *const hoedown_buffer,
510 content: *const hoedown_buffer,
511 opaque: *mut libc::c_void) -> libc::c_int
514 if !content.is_null() && (*content).size > 0 {
515 let ob = opaque as *mut hoedown_buffer;
516 hoedown_buffer_put(ob, (*content).data as *const libc::c_char,
523 extern fn normal_text(_ob: *mut hoedown_buffer,
524 text: *const hoedown_buffer,
525 opaque: *mut libc::c_void)
528 let ob = opaque as *mut hoedown_buffer;
529 hoedown_buffer_put(ob, (*text).data as *const libc::c_char,
535 let ob = hoedown_buffer_new(DEF_OUNIT);
536 let mut plain_renderer: hoedown_renderer = ::std::mem::zeroed();
537 let renderer: *mut hoedown_renderer = &mut plain_renderer;
538 (*renderer).opaque = ob as *mut libc::c_void;
539 (*renderer).link = Some(link);
540 (*renderer).normal_text = Some(normal_text);
542 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
543 hoedown_document_render(document, ob, md.as_ptr(),
544 md.len() as libc::size_t);
545 hoedown_document_free(document);
546 let plain_slice = (*ob).as_bytes();
547 let plain = match str::from_utf8(plain_slice) {
548 Ok(s) => s.to_string(),
549 Err(_) => "".to_string(),
551 hoedown_buffer_free(ob);
558 use super::{LangString, Markdown};
559 use super::{collapse_whitespace, plain_summary_line};
562 fn test_lang_string_parse() {
564 should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool) {
565 assert_eq!(LangString::parse(s), LangString {
566 should_panic: should_panic,
570 test_harness: test_harness,
574 // marker | should_panic| no_run | ignore | rust | test_harness
575 t("", false, false, false, true, false);
576 t("rust
", false, false, false, true, false);
577 t("sh
", false, false, false, false, false);
578 t("ignore
", false, false, true, true, false);
579 t("should_panic
", true, false, false, true, false);
580 t("no_run
", false, true, false, true, false);
581 t("test_harness
", false, false, false, true, true);
582 t("{.no_run .example}
", false, true, false, true, false);
583 t("{.sh .should_panic}
", true, false, false, true, false);
584 t("{.example .rust}
", false, false, false, true, false);
585 t("{.test_harness .rust}
", false, false, false, true, true);
590 let markdown = "# title";
591 format!("{}", Markdown(markdown));
595 fn test_plain_summary_line() {
596 fn t(input: &str, expect: &str) {
597 let output = plain_summary_line(input);
598 assert_eq!(output, expect);
601 t("hello [Rust](http://rust-lang.org) :)", "hello Rust :)");
602 t("code `let x = i32;` ...", "code `let x = i32;` ...");
603 t("type `Type<'static>` ...", "type `Type<'static>` ...");
604 t("# top header", "top header");
605 t("## header", "header");
609 fn test_collapse_whitespace() {
610 fn t(input
: &str, expected
: &str) {
611 let actual
= collapse_whitespace(input
);
612 assert_eq
!(actual
, expected
);
616 t("foo bar baz", "foo bar baz");
617 t(" foo bar", "foo bar");
618 t("\tfoo bar\nbaz", "foo bar baz");
619 t("foo bar \n baz\t\tqux\n", "foo bar baz qux");