1 //! Basic syntax highlighting functionality.
3 //! This module uses librustc_ast's lexer to provide token-based highlighting for
4 //! the HTML documentation generated by rustdoc.
6 //! Use the `render_with_highlighting` to highlight some rust code.
8 use crate::clean
::PrimitiveType
;
9 use crate::html
::escape
::Escape
;
10 use crate::html
::render
::{Context, LinkFromSrc}
;
12 use std
::collections
::VecDeque
;
13 use std
::fmt
::{Display, Write}
;
15 use rustc_data_structures
::fx
::FxHashMap
;
16 use rustc_lexer
::{Cursor, LiteralKind, TokenKind}
;
17 use rustc_span
::edition
::Edition
;
18 use rustc_span
::symbol
::Symbol
;
19 use rustc_span
::{BytePos, Span, DUMMY_SP}
;
21 use super::format
::{self, Buffer}
;
23 /// This type is needed in case we want to render links on items to allow to go to their definition.
24 pub(crate) struct HrefContext
<'a
, 'b
, 'c
> {
25 pub(crate) context
: &'a Context
<'b
>,
26 /// This span contains the current file we're going through.
27 pub(crate) file_span
: Span
,
28 /// This field is used to know "how far" from the top of the directory we are to link to either
29 /// documentation pages or other source pages.
30 pub(crate) root_path
: &'c
str,
31 /// This field is used to calculate precise local URLs.
32 pub(crate) current_href
: &'c
str,
35 /// Decorations are represented as a map from CSS class to vector of character ranges.
36 /// Each range will be wrapped in a span with that class.
38 pub(crate) struct DecorationInfo(pub(crate) FxHashMap
<&'
static str, Vec
<(u32, u32)>>);
40 #[derive(Eq, PartialEq, Clone, Copy)]
41 pub(crate) enum Tooltip
{
49 /// Highlights `src` as an inline example, returning the HTML output.
50 pub(crate) fn render_example_with_highlighting(
54 playground_button
: Option
<&str>,
56 write_header(out
, "rust-example-rendered", None
, tooltip
);
57 write_code(out
, src
, None
, None
);
58 write_footer(out
, playground_button
);
61 /// Highlights `src` as a macro, returning the HTML output.
62 pub(crate) fn render_macro_with_highlighting(src
: &str, out
: &mut Buffer
) {
63 write_header(out
, "macro", None
, Tooltip
::None
);
64 write_code(out
, src
, None
, None
);
65 write_footer(out
, None
);
68 /// Highlights `src` as a source code page, returning the HTML output.
69 pub(crate) fn render_source_with_highlighting(
73 href_context
: HrefContext
<'_
, '_
, '_
>,
74 decoration_info
: DecorationInfo
,
76 write_header(out
, "", Some(line_numbers
), Tooltip
::None
);
77 write_code(out
, src
, Some(href_context
), Some(decoration_info
));
78 write_footer(out
, None
);
81 fn write_header(out
: &mut Buffer
, class
: &str, extra_content
: Option
<Buffer
>, tooltip
: Tooltip
) {
84 "<div class=\"example-wrap{}\">",
86 Tooltip
::Ignore
=> " ignore",
87 Tooltip
::CompileFail
=> " compile_fail",
88 Tooltip
::ShouldPanic
=> " should_panic",
89 Tooltip
::Edition(_
) => " edition",
94 if tooltip
!= Tooltip
::None
{
97 "<div class='tooltip'{}>ⓘ</div>",
98 if let Tooltip
::Edition(edition_info
) = tooltip
{
99 format
!(" data-edition=\"{}\"", edition_info
)
106 if let Some(extra
) = extra_content
{
107 out
.push_buffer(extra
);
109 if class
.is_empty() {
110 write
!(out
, "<pre class=\"rust\">");
112 write
!(out
, "<pre class=\"rust {class}\">");
114 write
!(out
, "<code>");
117 /// Check if two `Class` can be merged together. In the following rules, "unclassified" means `None`
118 /// basically (since it's `Option<Class>`). The following rules apply:
120 /// * If two `Class` have the same variant, then they can be merged.
121 /// * If the other `Class` is unclassified and only contains white characters (backline,
122 /// whitespace, etc), it can be merged.
123 /// * `Class::Ident` is considered the same as unclassified (because it doesn't have an associated
125 fn can_merge(class1
: Option
<Class
>, class2
: Option
<Class
>, text
: &str) -> bool
{
126 match (class1
, class2
) {
127 (Some(c1
), Some(c2
)) => c1
.is_equal_to(c2
),
128 (Some(Class
::Ident(_
)), None
) | (None
, Some(Class
::Ident(_
))) => true,
129 (Some(_
), None
) | (None
, Some(_
)) => text
.trim().is_empty(),
130 (None
, None
) => true,
134 /// This type is used as a conveniency to prevent having to pass all its fields as arguments into
135 /// the various functions (which became its methods).
136 struct TokenHandler
<'a
, 'b
, 'c
, 'd
, 'e
> {
138 /// It contains the closing tag and the associated `Class`.
139 closing_tags
: Vec
<(&'
static str, Class
)>,
140 /// This is used because we don't automatically generate the closing tag on `ExitSpan` in
141 /// case an `EnterSpan` event with the same class follows.
142 pending_exit_span
: Option
<Class
>,
143 /// `current_class` and `pending_elems` are used to group HTML elements with same `class`
144 /// attributes to reduce the DOM size.
145 current_class
: Option
<Class
>,
146 /// We need to keep the `Class` for each element because it could contain a `Span` which is
147 /// used to generate links.
148 pending_elems
: Vec
<(&'b
str, Option
<Class
>)>,
149 href_context
: Option
<HrefContext
<'c
, 'd
, 'e
>>,
152 impl<'a
, 'b
, 'c
, 'd
, 'e
> TokenHandler
<'a
, 'b
, 'c
, 'd
, 'e
> {
153 fn handle_exit_span(&mut self) {
154 // We can't get the last `closing_tags` element using `pop()` because `closing_tags` is
155 // being used in `write_pending_elems`.
156 let class
= self.closing_tags
.last().expect("ExitSpan without EnterSpan").1;
157 // We flush everything just in case...
158 self.write_pending_elems(Some(class
));
160 exit_span(self.out
, self.closing_tags
.pop().expect("ExitSpan without EnterSpan").0);
161 self.pending_exit_span
= None
;
164 /// Write all the pending elements sharing a same (or at mergeable) `Class`.
166 /// If there is a "parent" (if a `EnterSpan` event was encountered) and the parent can be merged
167 /// with the elements' class, then we simply write the elements since the `ExitSpan` event will
170 /// Otherwise, if there is only one pending element, we let the `string` function handle both
171 /// opening and closing the tag, otherwise we do it into this function.
173 /// It returns `true` if `current_class` must be set to `None` afterwards.
174 fn write_pending_elems(&mut self, current_class
: Option
<Class
>) -> bool
{
175 if self.pending_elems
.is_empty() {
178 if let Some((_
, parent_class
)) = self.closing_tags
.last() &&
179 can_merge(current_class
, Some(*parent_class
), "")
181 for (text
, class
) in self.pending_elems
.iter() {
182 string(self.out
, Escape(text
), *class
, &self.href_context
, false);
185 // We only want to "open" the tag ourselves if we have more than one pending and if the
186 // current parent tag is not the same as our pending content.
187 let close_tag
= if self.pending_elems
.len() > 1 && current_class
.is_some() {
188 Some(enter_span(self.out
, current_class
.unwrap(), &self.href_context
))
192 for (text
, class
) in self.pending_elems
.iter() {
193 string(self.out
, Escape(text
), *class
, &self.href_context
, close_tag
.is_none());
195 if let Some(close_tag
) = close_tag
{
196 exit_span(self.out
, close_tag
);
199 self.pending_elems
.clear();
204 impl<'a
, 'b
, 'c
, 'd
, 'e
> Drop
for TokenHandler
<'a
, 'b
, 'c
, 'd
, 'e
> {
205 /// When leaving, we need to flush all pending data to not have missing content.
207 if self.pending_exit_span
.is_some() {
208 self.handle_exit_span();
210 self.write_pending_elems(self.current_class
);
215 /// Convert the given `src` source code into HTML by adding classes for highlighting.
217 /// This code is used to render code blocks (in the documentation) as well as the source code pages.
219 /// Some explanations on the last arguments:
221 /// In case we are rendering a code block and not a source code file, `href_context` will be `None`.
222 /// To put it more simply: if `href_context` is `None`, the code won't try to generate links to an
225 /// More explanations about spans and how we use them here are provided in the
229 href_context
: Option
<HrefContext
<'_
, '_
, '_
>>,
230 decoration_info
: Option
<DecorationInfo
>,
232 // This replace allows to fix how the code source with DOS backline characters is displayed.
233 let src
= src
.replace("\r\n", "\n");
234 let mut token_handler
= TokenHandler
{
236 closing_tags
: Vec
::new(),
237 pending_exit_span
: None
,
239 pending_elems
: Vec
::new(),
245 token_handler
.href_context
.as_ref().map(|c
| c
.file_span
).unwrap_or(DUMMY_SP
),
248 .highlight(&mut |highlight
| {
250 Highlight
::Token { text, class }
=> {
251 // If we received a `ExitSpan` event and then have a non-compatible `Class`, we
252 // need to close the `<span>`.
253 let need_current_class_update
= if let Some(pending
) = token_handler
.pending_exit_span
&&
254 !can_merge(Some(pending
), class
, text
) {
255 token_handler
.handle_exit_span();
257 // If the two `Class` are different, time to flush the current content and start
259 } else if !can_merge(token_handler
.current_class
, class
, text
) {
260 token_handler
.write_pending_elems(token_handler
.current_class
);
263 token_handler
.current_class
.is_none()
266 if need_current_class_update
{
267 token_handler
.current_class
= class
.map(Class
::dummy
);
269 token_handler
.pending_elems
.push((text
, class
));
271 Highlight
::EnterSpan { class }
=> {
272 let mut should_add
= true;
273 if let Some(pending_exit_span
) = token_handler
.pending_exit_span
{
274 if class
.is_equal_to(pending_exit_span
) {
277 token_handler
.handle_exit_span();
280 // We flush everything just in case...
281 if token_handler
.write_pending_elems(token_handler
.current_class
) {
282 token_handler
.current_class
= None
;
286 let closing_tag
= enter_span(token_handler
.out
, class
, &token_handler
.href_context
);
287 token_handler
.closing_tags
.push((closing_tag
, class
));
290 token_handler
.current_class
= None
;
291 token_handler
.pending_exit_span
= None
;
293 Highlight
::ExitSpan
=> {
294 token_handler
.current_class
= None
;
295 token_handler
.pending_exit_span
=
296 Some(token_handler
.closing_tags
.last().as_ref().expect("ExitSpan without EnterSpan").1);
302 fn write_footer(out
: &mut Buffer
, playground_button
: Option
<&str>) {
303 writeln
!(out
, "</code></pre>{}</div>", playground_button
.unwrap_or_default());
306 /// How a span of text is classified. Mostly corresponds to token kinds.
307 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
313 /// Keywords that do pointer/reference stuff.
321 /// `Ident` isn't rendered in the HTML but we still need it for the `Span` it contains.
327 Decoration(&'
static str),
331 /// It is only looking at the variant, not the variant content.
333 /// It is used mostly to group multiple similar HTML elements into one `<span>` instead of
335 fn is_equal_to(self, other
: Self) -> bool
{
336 match (self, other
) {
337 (Self::Self_(_
), Self::Self_(_
))
338 | (Self::Macro(_
), Self::Macro(_
))
339 | (Self::Ident(_
), Self::Ident(_
)) => true,
340 (Self::Decoration(c1
), Self::Decoration(c2
)) => c1
== c2
,
345 /// If `self` contains a `Span`, it'll be replaced with `DUMMY_SP` to prevent creating links
346 /// on "empty content" (because of the attributes merge).
347 fn dummy(self) -> Self {
349 Self::Self_(_
) => Self::Self_(DUMMY_SP
),
350 Self::Macro(_
) => Self::Macro(DUMMY_SP
),
351 Self::Ident(_
) => Self::Ident(DUMMY_SP
),
356 /// Returns the css class expected by rustdoc for each `Class`.
357 fn as_html(self) -> &'
static str {
359 Class
::Comment
=> "comment",
360 Class
::DocComment
=> "doccomment",
361 Class
::Attribute
=> "attribute",
362 Class
::KeyWord
=> "kw",
363 Class
::RefKeyWord
=> "kw-2",
364 Class
::Self_(_
) => "self",
365 Class
::Macro(_
) => "macro",
366 Class
::MacroNonTerminal
=> "macro-nonterminal",
367 Class
::String
=> "string",
368 Class
::Number
=> "number",
369 Class
::Bool
=> "bool-val",
370 Class
::Ident(_
) => "",
371 Class
::Lifetime
=> "lifetime",
372 Class
::PreludeTy
=> "prelude-ty",
373 Class
::PreludeVal
=> "prelude-val",
374 Class
::QuestionMark
=> "question-mark",
375 Class
::Decoration(kind
) => kind
,
379 /// In case this is an item which can be converted into a link to a definition, it'll contain
380 /// a "span" (a tuple representing `(lo, hi)` equivalent of `Span`).
381 fn get_span(self) -> Option
<Span
> {
383 Self::Ident(sp
) | Self::Self_(sp
) | Self::Macro(sp
) => Some(sp
),
389 | Self::MacroNonTerminal
397 | Self::Decoration(_
) => None
,
403 Token { text: &'a str, class: Option<Class> }
,
404 EnterSpan { class: Class }
,
408 struct TokenIter
<'a
> {
413 impl<'a
> Iterator
for TokenIter
<'a
> {
414 type Item
= (TokenKind
, &'a
str);
415 fn next(&mut self) -> Option
<(TokenKind
, &'a
str)> {
416 let token
= self.cursor
.advance_token();
417 if token
.kind
== TokenKind
::Eof
{
420 let (text
, rest
) = self.src
.split_at(token
.len
as usize);
422 Some((token
.kind
, text
))
426 /// Classifies into identifier class; returns `None` if this is a non-keyword identifier.
427 fn get_real_ident_class(text
: &str, allow_path_keywords
: bool
) -> Option
<Class
> {
428 let ignore
: &[&str] =
429 if allow_path_keywords { &["self", "Self", "super", "crate"] }
else { &["self", "Self"] }
;
430 if ignore
.iter().any(|k
| *k
== text
) {
434 "ref" | "mut" => Class
::RefKeyWord
,
435 "false" | "true" => Class
::Bool
,
436 _
if Symbol
::intern(text
).is_reserved(|| Edition
::Edition2021
) => Class
::KeyWord
,
441 /// This iterator comes from the same idea than "Peekable" except that it allows to "peek" more than
442 /// just the next item by using `peek_next`. The `peek` method always returns the next item after
443 /// the current one whereas `peek_next` will return the next item after the last one peeked.
445 /// You can use both `peek` and `peek_next` at the same time without problem.
446 struct PeekIter
<'a
> {
447 stored
: VecDeque
<(TokenKind
, &'a
str)>,
448 /// This position is reinitialized when using `next`. It is used in `peek_next`.
453 impl<'a
> PeekIter
<'a
> {
454 fn new(iter
: TokenIter
<'a
>) -> Self {
455 Self { stored: VecDeque::new(), peek_pos: 0, iter }
457 /// Returns the next item after the current one. It doesn't interfere with `peek_next` output.
458 fn peek(&mut self) -> Option
<&(TokenKind
, &'a
str)> {
459 if self.stored
.is_empty() {
460 if let Some(next
) = self.iter
.next() {
461 self.stored
.push_back(next
);
466 /// Returns the next item after the last one peeked. It doesn't interfere with `peek` output.
467 fn peek_next(&mut self) -> Option
<&(TokenKind
, &'a
str)> {
469 if self.peek_pos
- 1 < self.stored
.len() {
470 self.stored
.get(self.peek_pos
- 1)
471 } else if let Some(next
) = self.iter
.next() {
472 self.stored
.push_back(next
);
480 impl<'a
> Iterator
for PeekIter
<'a
> {
481 type Item
= (TokenKind
, &'a
str);
482 fn next(&mut self) -> Option
<Self::Item
> {
484 if let Some(first
) = self.stored
.pop_front() { Some(first) }
else { self.iter.next() }
488 /// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
490 starts
: Vec
<(u32, &'
static str)>,
495 fn new(info
: DecorationInfo
) -> Self {
496 // Extract tuples (start, end, kind) into separate sequences of (start, kind) and (end).
497 let (mut starts
, mut ends
): (Vec
<_
>, Vec
<_
>) = info
500 .flat_map(|(kind
, ranges
)| ranges
.into_iter().map(move |(lo
, hi
)| ((lo
, kind
), hi
)))
503 // Sort the sequences in document order.
504 starts
.sort_by_key(|(lo
, _
)| *lo
);
507 Decorations { starts, ends }
511 /// Processes program tokens, classifying strings of text by highlighting
512 /// category (`Class`).
513 struct Classifier
<'a
> {
514 tokens
: PeekIter
<'a
>,
517 in_macro_nonterminal
: bool
,
521 decorations
: Option
<Decorations
>,
524 impl<'a
> Classifier
<'a
> {
525 /// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
526 /// file span which will be used later on by the `span_correspondance_map`.
527 fn new(src
: &str, file_span
: Span
, decoration_info
: Option
<DecorationInfo
>) -> Classifier
<'_
> {
528 let tokens
= PeekIter
::new(TokenIter { src, cursor: Cursor::new(src) }
);
529 let decorations
= decoration_info
.map(Decorations
::new
);
534 in_macro_nonterminal
: false,
542 /// Convenient wrapper to create a [`Span`] from a position in the file.
543 fn new_span(&self, lo
: u32, text
: &str) -> Span
{
544 let hi
= lo
+ text
.len() as u32;
545 let file_lo
= self.file_span
.lo();
546 self.file_span
.with_lo(file_lo
+ BytePos(lo
)).with_hi(file_lo
+ BytePos(hi
))
549 /// Concatenate colons and idents as one when possible.
550 fn get_full_ident_path(&mut self) -> Vec
<(TokenKind
, usize, usize)> {
551 let start
= self.byte_pos
as usize;
553 let mut has_ident
= false;
557 while let Some((TokenKind
::Colon
, _
)) = self.tokens
.peek() {
561 // Ident path can start with "::" but if we already have content in the ident path,
562 // the "::" is mandatory.
563 if has_ident
&& nb
== 0 {
564 return vec
![(TokenKind
::Ident
, start
, pos
)];
565 } else if nb
!= 0 && nb
!= 2 {
567 return vec
![(TokenKind
::Ident
, start
, pos
), (TokenKind
::Colon
, pos
, pos
+ nb
)];
569 return vec
![(TokenKind
::Colon
, start
, pos
+ nb
)];
573 if let Some((None
, text
)) = self.tokens
.peek().map(|(token
, text
)| {
574 if *token
== TokenKind
::Ident
{
575 let class
= get_real_ident_class(text
, true);
578 // Doesn't matter which Class we put in here...
579 (Some(Class
::Comment
), text
)
582 // We only "add" the colon if there is an ident behind.
583 pos
+= text
.len() + nb
;
586 } else if nb
> 0 && has_ident
{
587 return vec
![(TokenKind
::Ident
, start
, pos
), (TokenKind
::Colon
, pos
, pos
+ nb
)];
589 return vec
![(TokenKind
::Colon
, start
, start
+ nb
)];
590 } else if has_ident
{
591 return vec
![(TokenKind
::Ident
, start
, pos
)];
598 /// Wraps the tokens iteration to ensure that the `byte_pos` is always correct.
600 /// It returns the token's kind, the token as a string and its byte position in the source
602 fn next(&mut self) -> Option
<(TokenKind
, &'a
str, u32)> {
603 if let Some((kind
, text
)) = self.tokens
.next() {
604 let before
= self.byte_pos
;
605 self.byte_pos
+= text
.len() as u32;
606 Some((kind
, text
, before
))
612 /// Exhausts the `Classifier` writing the output into `sink`.
614 /// The general structure for this method is to iterate over each token,
615 /// possibly giving it an HTML span with a class specifying what flavor of
617 fn highlight(mut self, sink
: &mut dyn FnMut(Highlight
<'a
>)) {
619 if let Some(decs
) = self.decorations
.as_mut() {
620 let byte_pos
= self.byte_pos
;
621 let n_starts
= decs
.starts
.iter().filter(|(i
, _
)| byte_pos
>= *i
).count();
622 for (_
, kind
) in decs
.starts
.drain(0..n_starts
) {
623 sink(Highlight
::EnterSpan { class: Class::Decoration(kind) }
);
626 let n_ends
= decs
.ends
.iter().filter(|i
| byte_pos
>= **i
).count();
627 for _
in decs
.ends
.drain(0..n_ends
) {
628 sink(Highlight
::ExitSpan
);
635 .map(|t
| matches
!(t
.0, TokenKind
::Colon
| TokenKind
::Ident
))
638 let tokens
= self.get_full_ident_path();
639 for (token
, start
, end
) in &tokens
{
640 let text
= &self.src
[*start
..*end
];
641 self.advance(*token
, text
, sink
, *start
as u32);
642 self.byte_pos
+= text
.len() as u32;
644 if !tokens
.is_empty() {
648 if let Some((token
, text
, before
)) = self.next() {
649 self.advance(token
, text
, sink
, before
);
656 /// Single step of highlighting. This will classify `token`, but maybe also a couple of
657 /// following ones as well.
659 /// `before` is the position of the given token in the `source` string and is used as "lo" byte
660 /// in case we want to try to generate a link for this token using the
661 /// `span_correspondance_map`.
666 sink
: &mut dyn FnMut(Highlight
<'a
>),
669 let lookahead
= self.peek();
670 let no_highlight
= |sink
: &mut dyn FnMut(_
)| sink(Highlight
::Token { text, class: None }
);
671 let class
= match token
{
672 TokenKind
::Whitespace
=> return no_highlight(sink
),
673 TokenKind
::LineComment { doc_style }
| TokenKind
::BlockComment { doc_style, .. }
=> {
674 if doc_style
.is_some() {
680 // Consider this as part of a macro invocation if there was a
681 // leading identifier.
682 TokenKind
::Bang
if self.in_macro
=> {
683 self.in_macro
= false;
684 sink(Highlight
::Token { text, class: None }
);
685 sink(Highlight
::ExitSpan
);
689 // Assume that '&' or '*' is the reference or dereference operator
690 // or a reference or pointer type. Unless, of course, it looks like
691 // a logical and or a multiplication operator: `&&` or `* `.
692 TokenKind
::Star
=> match self.tokens
.peek() {
693 Some((TokenKind
::Whitespace
, _
)) => return no_highlight(sink
),
694 Some((TokenKind
::Ident
, "mut")) => {
696 sink(Highlight
::Token { text: "*mut", class: Some(Class::RefKeyWord) }
);
699 Some((TokenKind
::Ident
, "const")) => {
701 sink(Highlight
::Token { text: "*const", class: Some(Class::RefKeyWord) }
);
704 _
=> Class
::RefKeyWord
,
706 TokenKind
::And
=> match self.tokens
.peek() {
707 Some((TokenKind
::And
, _
)) => {
709 sink(Highlight
::Token { text: "&&", class: None }
);
712 Some((TokenKind
::Eq
, _
)) => {
714 sink(Highlight
::Token { text: "&=", class: None }
);
717 Some((TokenKind
::Whitespace
, _
)) => return no_highlight(sink
),
718 Some((TokenKind
::Ident
, "mut")) => {
720 sink(Highlight
::Token { text: "&mut", class: Some(Class::RefKeyWord) }
);
723 _
=> Class
::RefKeyWord
,
726 // These can either be operators, or arrows.
727 TokenKind
::Eq
=> match lookahead
{
728 Some(TokenKind
::Eq
) => {
730 sink(Highlight
::Token { text: "==", class: None }
);
733 Some(TokenKind
::Gt
) => {
735 sink(Highlight
::Token { text: "=>", class: None }
);
738 _
=> return no_highlight(sink
),
740 TokenKind
::Minus
if lookahead
== Some(TokenKind
::Gt
) => {
742 sink(Highlight
::Token { text: "->", class: None }
);
755 | TokenKind
::Gt
=> return no_highlight(sink
),
757 // Miscellaneous, no highlighting.
761 | TokenKind
::OpenParen
762 | TokenKind
::CloseParen
763 | TokenKind
::OpenBrace
764 | TokenKind
::CloseBrace
765 | TokenKind
::OpenBracket
769 | TokenKind
::Unknown
=> return no_highlight(sink
),
771 TokenKind
::Question
=> Class
::QuestionMark
,
773 TokenKind
::Dollar
=> match lookahead
{
774 Some(TokenKind
::Ident
) => {
775 self.in_macro_nonterminal
= true;
776 Class
::MacroNonTerminal
778 _
=> return no_highlight(sink
),
781 // This might be the start of an attribute. We're going to want to
782 // continue highlighting it as an attribute until the ending ']' is
783 // seen, so skip out early. Down below we terminate the attribute
784 // span when we see the ']'.
785 TokenKind
::Pound
=> {
787 // Case 1: #![inner_attribute]
788 Some(TokenKind
::Bang
) => {
790 if let Some(TokenKind
::OpenBracket
) = self.peek() {
791 self.in_attribute
= true;
792 sink(Highlight
::EnterSpan { class: Class::Attribute }
);
794 sink(Highlight
::Token { text: "#", class: None }
);
795 sink(Highlight
::Token { text: "!", class: None }
);
798 // Case 2: #[outer_attribute]
799 Some(TokenKind
::OpenBracket
) => {
800 self.in_attribute
= true;
801 sink(Highlight
::EnterSpan { class: Class::Attribute }
);
805 return no_highlight(sink
);
807 TokenKind
::CloseBracket
=> {
808 if self.in_attribute
{
809 self.in_attribute
= false;
810 sink(Highlight
::Token { text: "]", class: None }
);
811 sink(Highlight
::ExitSpan
);
814 return no_highlight(sink
);
816 TokenKind
::Literal { kind, .. }
=> match kind
{
818 LiteralKind
::Byte { .. }
819 | LiteralKind
::Char { .. }
820 | LiteralKind
::Str { .. }
821 | LiteralKind
::ByteStr { .. }
822 | LiteralKind
::RawStr { .. }
823 | LiteralKind
::RawByteStr { .. }
=> Class
::String
,
825 LiteralKind
::Float { .. }
| LiteralKind
::Int { .. }
=> Class
::Number
,
827 TokenKind
::Ident
| TokenKind
::RawIdent
if lookahead
== Some(TokenKind
::Bang
) => {
828 self.in_macro
= true;
829 sink(Highlight
::EnterSpan { class: Class::Macro(self.new_span(before, text)) }
);
830 sink(Highlight
::Token { text, class: None }
);
833 TokenKind
::Ident
=> match get_real_ident_class(text
, false) {
835 "Option" | "Result" => Class
::PreludeTy
,
836 "Some" | "None" | "Ok" | "Err" => Class
::PreludeVal
,
837 // "union" is a weak keyword and is only considered as a keyword when declaring
839 "union" if self.check_if_is_union_keyword() => Class
::KeyWord
,
840 _
if self.in_macro_nonterminal
=> {
841 self.in_macro_nonterminal
= false;
842 Class
::MacroNonTerminal
844 "self" | "Self" => Class
::Self_(self.new_span(before
, text
)),
845 _
=> Class
::Ident(self.new_span(before
, text
)),
849 TokenKind
::RawIdent
| TokenKind
::UnknownPrefix
| TokenKind
::InvalidIdent
=> {
850 Class
::Ident(self.new_span(before
, text
))
852 TokenKind
::Lifetime { .. }
=> Class
::Lifetime
,
853 TokenKind
::Eof
=> panic
!("Eof in advance"),
855 // Anything that didn't return above is the simple case where we the
856 // class just spans a single token, so we can use the `string` method.
857 sink(Highlight
::Token { text, class: Some(class) }
);
860 fn peek(&mut self) -> Option
<TokenKind
> {
861 self.tokens
.peek().map(|(token_kind
, _text
)| *token_kind
)
864 fn check_if_is_union_keyword(&mut self) -> bool
{
865 while let Some(kind
) = self.tokens
.peek_next().map(|(token_kind
, _text
)| token_kind
) {
866 if *kind
== TokenKind
::Whitespace
{
869 return *kind
== TokenKind
::Ident
;
875 /// Called when we start processing a span of text that should be highlighted.
876 /// The `Class` argument specifies how it should be highlighted.
880 href_context
: &Option
<HrefContext
<'_
, '_
, '_
>>,
882 string_without_closing_tag(out
, "", Some(klass
), href_context
, true).expect(
883 "internal error: enter_span was called with Some(klass) but did not return a \
888 /// Called at the end of a span of highlighted text.
889 fn exit_span(out
: &mut Buffer
, closing_tag
: &str) {
890 out
.write_str(closing_tag
);
893 /// Called for a span of text. If the text should be highlighted differently
894 /// from the surrounding text, then the `Class` argument will be a value other
897 /// The following sequences of callbacks are equivalent:
899 /// enter_span(Foo), string("text", None), exit_span()
900 /// string("text", Foo)
903 /// The latter can be thought of as a shorthand for the former, which is more
906 /// Note that if `context` is not `None` and that the given `klass` contains a `Span`, the function
907 /// will then try to find this `span` in the `span_correspondance_map`. If found, it'll then
908 /// generate a link for this element (which corresponds to where its definition is located).
909 fn string
<T
: Display
>(
912 klass
: Option
<Class
>,
913 href_context
: &Option
<HrefContext
<'_
, '_
, '_
>>,
916 if let Some(closing_tag
) = string_without_closing_tag(out
, text
, klass
, href_context
, open_tag
)
918 out
.write_str(closing_tag
);
922 /// This function writes `text` into `out` with some modifications depending on `klass`:
924 /// * If `klass` is `None`, `text` is written into `out` with no modification.
925 /// * If `klass` is `Some` but `klass.get_span()` is `None`, it writes the text wrapped in a
926 /// `<span>` with the provided `klass`.
927 /// * If `klass` is `Some` and has a [`rustc_span::Span`], it then tries to generate a link (`<a>`
928 /// element) by retrieving the link information from the `span_correspondance_map` that was filled
929 /// in `span_map.rs::collect_spans_and_sources`. If it cannot retrieve the information, then it's
930 /// the same as the second point (`klass` is `Some` but doesn't have a [`rustc_span::Span`]).
931 fn string_without_closing_tag
<T
: Display
>(
934 klass
: Option
<Class
>,
935 href_context
: &Option
<HrefContext
<'_
, '_
, '_
>>,
937 ) -> Option
<&'
static str> {
938 let Some(klass
) = klass
940 write
!(out
, "{}", text
);
943 let Some(def_span
) = klass
.get_span()
946 write
!(out
, "{}", text
);
949 write
!(out
, "<span class=\"{}\">{}", klass
.as_html(), text
);
950 return Some("</span>");
953 let mut text_s
= text
.to_string();
954 if text_s
.contains("::") {
955 text_s
= text_s
.split("::").intersperse("::").fold(String
::new(), |mut path
, t
| {
957 "self" | "Self" => write
!(
959 "<span class=\"{}\">{}</span>",
960 Class
::Self_(DUMMY_SP
).as_html(),
963 "crate" | "super" => {
964 write
!(&mut path
, "<span class=\"{}\">{}</span>", Class
::KeyWord
.as_html(), t
)
966 t
=> write
!(&mut path
, "{}", t
),
968 .expect("Failed to build source HTML path");
973 if let Some(href_context
) = href_context
{
975 href_context
.context
.shared
.span_correspondance_map
.get(&def_span
).and_then(|href
| {
976 let context
= href_context
.context
;
977 // FIXME: later on, it'd be nice to provide two links (if possible) for all items:
978 // one to the documentation page and one to the source definition.
979 // FIXME: currently, external items only generate a link to their documentation,
980 // a link to their definition can be generated using this:
981 // https://github.com/rust-lang/rust/blob/60f1a2fc4b535ead9c85ce085fdce49b1b097531/src/librustdoc/html/render/context.rs#L315-L338
983 LinkFromSrc
::Local(span
) => {
984 context
.href_from_span_relative(*span
, href_context
.current_href
)
986 LinkFromSrc
::External(def_id
) => {
987 format
::href_with_root_path(*def_id
, context
, Some(href_context
.root_path
))
989 .map(|(url
, _
, _
)| url
)
991 LinkFromSrc
::Primitive(prim
) => format
::href_with_root_path(
992 PrimitiveType
::primitive_locations(context
.tcx())[prim
],
994 Some(href_context
.root_path
),
997 .map(|(url
, _
, _
)| url
),
1002 // We're already inside an element which has the same klass, no need to give it
1004 write
!(out
, "<a href=\"{}\">{}", href
, text_s
);
1006 let klass_s
= klass
.as_html();
1007 if klass_s
.is_empty() {
1008 write
!(out
, "<a href=\"{}\">{}", href
, text_s
);
1010 write
!(out
, "<a class=\"{}\" href=\"{}\">{}", klass_s
, href
, text_s
);
1013 return Some("</a>");
1017 write
!(out
, "{}", text_s
);
1020 let klass_s
= klass
.as_html();
1021 if klass_s
.is_empty() {
1022 write
!(out
, "{}", text_s
);
1025 write
!(out
, "<span class=\"{}\">{}", klass_s
, text_s
);