1 // Copyright 2014-2016 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Basic syntax highlighting functionality.
13 //! This module uses libsyntax's lexer to provide token-based highlighting for
14 //! the HTML documentation generated by rustdoc.
16 //! If you just want to syntax highlighting for a Rust program, then you can use
17 //! the `render_inner_with_highlighting` or `render_with_highlighting`
18 //! functions. For more advanced use cases (if you want to supply your own css
19 //! classes or control how the HTML is generated, or even generate something
20 //! other then HTML), then you should implement the `Writer` trait and use a
23 use html
::escape
::Escape
;
25 use std
::fmt
::Display
;
27 use std
::io
::prelude
::*;
29 use syntax
::codemap
::{CodeMap, FilePathMapping}
;
30 use syntax
::parse
::lexer
::{self, TokenAndSpan}
;
31 use syntax
::parse
::token
;
35 /// Highlights `src`, returning the HTML output.
36 pub fn render_with_highlighting(src
: &str, class
: Option
<&str>, id
: Option
<&str>,
37 extension
: Option
<&str>,
38 tooltip
: Option
<(&str, &str)>) -> String
{
39 debug
!("highlighting: ================\n{}\n==============", src
);
40 let sess
= parse
::ParseSess
::new(FilePathMapping
::empty());
41 let fm
= sess
.codemap().new_filemap("<stdin>".to_string(), src
.to_string());
43 let mut out
= Vec
::new();
44 if let Some((tooltip
, class
)) = tooltip
{
45 write
!(out
, "<div class='information'><div class='tooltip {}'>ⓘ<span \
46 class='tooltiptext'>{}</span></div></div>",
47 class
, tooltip
).unwrap();
49 write_header(class
, id
, &mut out
).unwrap();
51 let mut classifier
= Classifier
::new(lexer
::StringReader
::new(&sess
, fm
), sess
.codemap());
52 if let Err(_
) = classifier
.write_source(&mut out
) {
53 return format
!("<pre>{}</pre>", src
);
56 if let Some(extension
) = extension
{
57 write
!(out
, "{}", extension
).unwrap();
59 write_footer(&mut out
).unwrap();
60 String
::from_utf8_lossy(&out
[..]).into_owned()
63 /// Highlights `src`, returning the HTML output. Returns only the inner html to
64 /// be inserted into an element. C.f., `render_with_highlighting` which includes
65 /// an enclosing `<pre>` block.
66 pub fn render_inner_with_highlighting(src
: &str) -> io
::Result
<String
> {
67 let sess
= parse
::ParseSess
::new(FilePathMapping
::empty());
68 let fm
= sess
.codemap().new_filemap("<stdin>".to_string(), src
.to_string());
70 let mut out
= Vec
::new();
71 let mut classifier
= Classifier
::new(lexer
::StringReader
::new(&sess
, fm
), sess
.codemap());
72 classifier
.write_source(&mut out
)?
;
74 Ok(String
::from_utf8_lossy(&out
).into_owned())
77 /// Processes a program (nested in the internal `lexer`), classifying strings of
78 /// text by highlighting category (`Class`). Calls out to a `Writer` to write
79 /// each span of text in sequence.
80 pub struct Classifier
<'a
> {
81 lexer
: lexer
::StringReader
<'a
>,
84 // State of the classifier.
87 in_macro_nonterminal
: bool
,
90 /// How a span of text is classified. Mostly corresponds to token kinds.
91 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
98 // Keywords that do pointer/reference stuff.
114 /// Trait that controls writing the output of syntax highlighting. Users should
115 /// implement this trait to customize writing output.
117 /// The classifier will call into the `Writer` implementation as it finds spans
118 /// of text to highlight. Exactly how that text should be highlighted is up to
119 /// the implementation.
121 /// Called when we start processing a span of text that should be highlighted.
122 /// The `Class` argument specifies how it should be highlighted.
123 fn enter_span(&mut self, _
: Class
) -> io
::Result
<()>;
125 /// Called at the end of a span of highlighted text.
126 fn exit_span(&mut self) -> io
::Result
<()>;
128 /// Called for a span of text, usually, but not always, a single token. If
129 /// the string of text (`T`) does correspond to a token, then the token will
130 /// also be passed. If the text should be highlighted differently from the
131 /// surrounding text, then the `Class` argument will be a value other than
133 /// The following sequences of callbacks are equivalent:
135 /// enter_span(Foo), string("text", None), exit_span()
136 /// string("text", Foo)
138 /// The latter can be thought of as a shorthand for the former, which is
140 fn string
<T
: Display
>(&mut self,
143 tok
: Option
<&TokenAndSpan
>)
147 // Implement `Writer` for anthing that can be written to, this just implements
148 // the default rustdoc behaviour.
149 impl<U
: Write
> Writer
for U
{
150 fn string
<T
: Display
>(&mut self,
153 _tas
: Option
<&TokenAndSpan
>)
156 Class
::None
=> write
!(self, "{}", text
),
157 klass
=> write
!(self, "<span class=\"{}\">{}</span>", klass
.rustdoc_class(), text
),
161 fn enter_span(&mut self, klass
: Class
) -> io
::Result
<()> {
162 write
!(self, "<span class=\"{}\">", klass
.rustdoc_class())
165 fn exit_span(&mut self) -> io
::Result
<()> {
166 write
!(self, "</span>")
170 impl<'a
> Classifier
<'a
> {
171 pub fn new(lexer
: lexer
::StringReader
<'a
>, codemap
: &'a CodeMap
) -> Classifier
<'a
> {
177 in_macro_nonterminal
: false,
181 /// Gets the next token out of the lexer, emitting fatal errors if lexing fails.
182 fn try_next_token(&mut self) -> io
::Result
<TokenAndSpan
> {
183 match self.lexer
.try_next_token() {
186 self.lexer
.emit_fatal_errors();
187 self.lexer
.sess
.span_diagnostic
188 .struct_warn("Backing out of syntax highlighting")
189 .note("You probably did not intend to render this as a rust code-block")
191 Err(io
::Error
::new(io
::ErrorKind
::Other
, ""))
196 /// Exhausts the `lexer` writing the output into `out`.
198 /// The general structure for this method is to iterate over each token,
199 /// possibly giving it an HTML span with a class specifying what flavor of token
200 /// is used. All source code emission is done as slices from the source map,
201 /// not from the tokens themselves, in order to stay true to the original
203 pub fn write_source
<W
: Writer
>(&mut self,
207 let next
= self.try_next_token()?
;
208 if next
.tok
== token
::Eof
{
212 self.write_token(out
, next
)?
;
218 // Handles an individual token from the lexer.
219 fn write_token
<W
: Writer
>(&mut self,
223 let klass
= match tas
.tok
{
224 token
::Shebang(s
) => {
225 out
.string(Escape(&s
.as_str()), Class
::None
, Some(&tas
))?
;
229 token
::Whitespace
=> Class
::None
,
230 token
::Comment
=> Class
::Comment
,
231 token
::DocComment(..) => Class
::DocComment
,
233 // If this '&' or '*' token is followed by a non-whitespace token, assume that it's the
234 // reference or dereference operator or a reference or pointer type, instead of the
235 // bit-and or multiplication operator.
236 token
::BinOp(token
::And
) | token
::BinOp(token
::Star
)
237 if self.lexer
.peek().tok
!= token
::Whitespace
=> Class
::RefKeyWord
,
239 // Consider this as part of a macro invocation if there was a
240 // leading identifier.
241 token
::Not
if self.in_macro
=> {
242 self.in_macro
= false;
247 token
::Eq
| token
::Lt
| token
::Le
| token
::EqEq
| token
::Ne
| token
::Ge
| token
::Gt
|
248 token
::AndAnd
| token
::OrOr
| token
::Not
| token
::BinOp(..) | token
::RArrow
|
249 token
::BinOpEq(..) | token
::FatArrow
=> Class
::Op
,
251 // Miscellaneous, no highlighting.
252 token
::Dot
| token
::DotDot
| token
::DotDotDot
| token
::DotDotEq
| token
::Comma
|
253 token
::Semi
| token
::Colon
| token
::ModSep
| token
::LArrow
| token
::OpenDelim(_
) |
254 token
::CloseDelim(token
::Brace
) | token
::CloseDelim(token
::Paren
) |
255 token
::CloseDelim(token
::NoDelim
) => Class
::None
,
257 token
::Question
=> Class
::QuestionMark
,
260 if self.lexer
.peek().tok
.is_ident() {
261 self.in_macro_nonterminal
= true;
262 Class
::MacroNonTerminal
268 // This might be the start of an attribute. We're going to want to
269 // continue highlighting it as an attribute until the ending ']' is
270 // seen, so skip out early. Down below we terminate the attribute
271 // span when we see the ']'.
273 // We can't be sure that our # begins an attribute (it could
274 // just be appearing in a macro) until we read either `#![` or
275 // `#[` from the input stream.
277 // We don't want to start highlighting as an attribute until
278 // we're confident there is going to be a ] coming up, as
279 // otherwise # tokens in macros highlight the rest of the input
282 // Case 1: #![inner_attribute]
283 if self.lexer
.peek().tok
== token
::Not
{
284 self.try_next_token()?
; // NOTE: consumes `!` token!
285 if self.lexer
.peek().tok
== token
::OpenDelim(token
::Bracket
) {
286 self.in_attribute
= true;
287 out
.enter_span(Class
::Attribute
)?
;
289 out
.string("#", Class
::None
, None
)?
;
290 out
.string("!", Class
::None
, None
)?
;
294 // Case 2: #[outer_attribute]
295 if self.lexer
.peek().tok
== token
::OpenDelim(token
::Bracket
) {
296 self.in_attribute
= true;
297 out
.enter_span(Class
::Attribute
)?
;
299 out
.string("#", Class
::None
, None
)?
;
302 token
::CloseDelim(token
::Bracket
) => {
303 if self.in_attribute
{
304 self.in_attribute
= false;
305 out
.string("]", Class
::None
, None
)?
;
313 token
::Literal(lit
, _suf
) => {
316 token
::Byte(..) | token
::Char(..) |
317 token
::ByteStr(..) | token
::ByteStrRaw(..) |
318 token
::Str_(..) | token
::StrRaw(..) => Class
::String
,
321 token
::Integer(..) | token
::Float(..) => Class
::Number
,
325 // Keywords are also included in the identifier set.
326 token
::Ident(ident
) => {
327 match &*ident
.name
.as_str() {
328 "ref" | "mut" => Class
::RefKeyWord
,
330 "self" |"Self" => Class
::Self_
,
331 "false" | "true" => Class
::Bool
,
333 "Option" | "Result" => Class
::PreludeTy
,
334 "Some" | "None" | "Ok" | "Err" => Class
::PreludeVal
,
336 "$crate" => Class
::KeyWord
,
337 _
if tas
.tok
.is_reserved_ident() => Class
::KeyWord
,
340 if self.in_macro_nonterminal
{
341 self.in_macro_nonterminal
= false;
342 Class
::MacroNonTerminal
343 } else if self.lexer
.peek().tok
== token
::Not
{
344 self.in_macro
= true;
353 token
::Lifetime(..) => Class
::Lifetime
,
355 token
::Underscore
| token
::Eof
| token
::Interpolated(..) |
356 token
::Tilde
| token
::At
| token
::DotEq
=> Class
::None
,
359 // Anything that didn't return above is the simple case where we the
360 // class just spans a single token, so we can use the `string` method.
361 out
.string(Escape(&self.snip(tas
.sp
)), klass
, Some(&tas
))
364 // Helper function to get a snippet from the codemap.
365 fn snip(&self, sp
: Span
) -> String
{
366 self.codemap
.span_to_snippet(sp
).unwrap()
371 /// Returns the css class expected by rustdoc for each `Class`.
372 pub fn rustdoc_class(self) -> &'
static str {
375 Class
::Comment
=> "comment",
376 Class
::DocComment
=> "doccomment",
377 Class
::Attribute
=> "attribute",
378 Class
::KeyWord
=> "kw",
379 Class
::RefKeyWord
=> "kw-2",
380 Class
::Self_
=> "self",
382 Class
::Macro
=> "macro",
383 Class
::MacroNonTerminal
=> "macro-nonterminal",
384 Class
::String
=> "string",
385 Class
::Number
=> "number",
386 Class
::Bool
=> "bool-val",
387 Class
::Ident
=> "ident",
388 Class
::Lifetime
=> "lifetime",
389 Class
::PreludeTy
=> "prelude-ty",
390 Class
::PreludeVal
=> "prelude-val",
391 Class
::QuestionMark
=> "question-mark"
396 fn write_header(class
: Option
<&str>,
400 write
!(out
, "<pre ")?
;
401 if let Some(id
) = id
{
402 write
!(out
, "id='{}' ", id
)?
;
404 write
!(out
, "class=\"rust {}\">\n", class
.unwrap_or(""))
407 fn write_footer(out
: &mut Write
) -> io
::Result
<()> {
408 write
!(out
, "</pre>\n")