1 //! Basic syntax highlighting functionality.
3 //! This module uses librustc_ast's lexer to provide token-based highlighting for
4 //! the HTML documentation generated by rustdoc.
6 //! Use the `render_with_highlighting` to highlight some rust code.
8 use crate::clean
::PrimitiveType
;
9 use crate::html
::escape
::Escape
;
10 use crate::html
::render
::{Context, LinkFromSrc}
;
12 use std
::collections
::VecDeque
;
13 use std
::fmt
::{Display, Write}
;
15 use rustc_data_structures
::fx
::FxHashMap
;
16 use rustc_lexer
::{Cursor, LiteralKind, TokenKind}
;
17 use rustc_span
::edition
::Edition
;
18 use rustc_span
::symbol
::Symbol
;
19 use rustc_span
::{BytePos, Span, DUMMY_SP}
;
21 use super::format
::{self, Buffer}
;
23 /// This type is needed in case we want to render links on items to allow to go to their definition.
24 pub(crate) struct HrefContext
<'a
, 'tcx
> {
25 pub(crate) context
: &'a Context
<'tcx
>,
26 /// This span contains the current file we're going through.
27 pub(crate) file_span
: Span
,
28 /// This field is used to know "how far" from the top of the directory we are to link to either
29 /// documentation pages or other source pages.
30 pub(crate) root_path
: &'a
str,
31 /// This field is used to calculate precise local URLs.
32 pub(crate) current_href
: String
,
35 /// Decorations are represented as a map from CSS class to vector of character ranges.
36 /// Each range will be wrapped in a span with that class.
38 pub(crate) struct DecorationInfo(pub(crate) FxHashMap
<&'
static str, Vec
<(u32, u32)>>);
40 #[derive(Eq, PartialEq, Clone, Copy)]
41 pub(crate) enum Tooltip
{
49 /// Highlights `src` as an inline example, returning the HTML output.
50 pub(crate) fn render_example_with_highlighting(
54 playground_button
: Option
<&str>,
55 extra_classes
: &[String
],
57 write_header(out
, "rust-example-rendered", None
, tooltip
, extra_classes
);
58 write_code(out
, src
, None
, None
);
59 write_footer(out
, playground_button
);
62 /// Highlights `src` as an item-decl, returning the HTML output.
63 pub(crate) fn render_item_decl_with_highlighting(src
: &str, out
: &mut Buffer
) {
64 write
!(out
, "<pre class=\"rust item-decl\">");
65 write_code(out
, src
, None
, None
);
66 write
!(out
, "</pre>");
72 extra_content
: Option
<Buffer
>,
74 extra_classes
: &[String
],
78 "<div class=\"example-wrap{}\">",
80 Tooltip
::Ignore
=> " ignore",
81 Tooltip
::CompileFail
=> " compile_fail",
82 Tooltip
::ShouldPanic
=> " should_panic",
83 Tooltip
::Edition(_
) => " edition",
88 if tooltip
!= Tooltip
::None
{
92 "<a href=\"#\" class=\"tooltip\" title=\"{}\">ⓘ</a>",
94 Tooltip
::Ignore
=> "This example is not tested",
95 Tooltip
::CompileFail
=> "This example deliberately fails to compile",
96 Tooltip
::ShouldPanic
=> "This example panics",
97 Tooltip
::Edition(edition
) => {
98 edition_code
= format
!("This example runs with edition {edition}");
101 Tooltip
::None
=> unreachable
!(),
106 if let Some(extra
) = extra_content
{
107 out
.push_buffer(extra
);
109 if class
.is_empty() {
112 "<pre class=\"rust{}{}\">",
113 if extra_classes
.is_empty() { "" }
else { " " }
,
114 extra_classes
.join(" "),
119 "<pre class=\"rust {class}{}{}\">",
120 if extra_classes
.is_empty() { "" }
else { " " }
,
121 extra_classes
.join(" "),
124 write
!(out
, "<code>");
127 /// Check if two `Class` can be merged together. In the following rules, "unclassified" means `None`
128 /// basically (since it's `Option<Class>`). The following rules apply:
130 /// * If two `Class` have the same variant, then they can be merged.
131 /// * If the other `Class` is unclassified and only contains white characters (backline,
132 /// whitespace, etc), it can be merged.
133 /// * `Class::Ident` is considered the same as unclassified (because it doesn't have an associated
135 fn can_merge(class1
: Option
<Class
>, class2
: Option
<Class
>, text
: &str) -> bool
{
136 match (class1
, class2
) {
137 (Some(c1
), Some(c2
)) => c1
.is_equal_to(c2
),
138 (Some(Class
::Ident(_
)), None
) | (None
, Some(Class
::Ident(_
))) => true,
139 (Some(_
), None
) | (None
, Some(_
)) => text
.trim().is_empty(),
140 (None
, None
) => true,
144 /// This type is used as a conveniency to prevent having to pass all its fields as arguments into
145 /// the various functions (which became its methods).
146 struct TokenHandler
<'a
, 'tcx
, F
: Write
> {
148 /// It contains the closing tag and the associated `Class`.
149 closing_tags
: Vec
<(&'
static str, Class
)>,
150 /// This is used because we don't automatically generate the closing tag on `ExitSpan` in
151 /// case an `EnterSpan` event with the same class follows.
152 pending_exit_span
: Option
<Class
>,
153 /// `current_class` and `pending_elems` are used to group HTML elements with same `class`
154 /// attributes to reduce the DOM size.
155 current_class
: Option
<Class
>,
156 /// We need to keep the `Class` for each element because it could contain a `Span` which is
157 /// used to generate links.
158 pending_elems
: Vec
<(&'a
str, Option
<Class
>)>,
159 href_context
: Option
<HrefContext
<'a
, 'tcx
>>,
162 impl<'a
, 'tcx
, F
: Write
> TokenHandler
<'a
, 'tcx
, F
> {
163 fn handle_exit_span(&mut self) {
164 // We can't get the last `closing_tags` element using `pop()` because `closing_tags` is
165 // being used in `write_pending_elems`.
166 let class
= self.closing_tags
.last().expect("ExitSpan without EnterSpan").1;
167 // We flush everything just in case...
168 self.write_pending_elems(Some(class
));
170 exit_span(self.out
, self.closing_tags
.pop().expect("ExitSpan without EnterSpan").0);
171 self.pending_exit_span
= None
;
174 /// Write all the pending elements sharing a same (or at mergeable) `Class`.
176 /// If there is a "parent" (if a `EnterSpan` event was encountered) and the parent can be merged
177 /// with the elements' class, then we simply write the elements since the `ExitSpan` event will
180 /// Otherwise, if there is only one pending element, we let the `string` function handle both
181 /// opening and closing the tag, otherwise we do it into this function.
183 /// It returns `true` if `current_class` must be set to `None` afterwards.
184 fn write_pending_elems(&mut self, current_class
: Option
<Class
>) -> bool
{
185 if self.pending_elems
.is_empty() {
188 if let Some((_
, parent_class
)) = self.closing_tags
.last() &&
189 can_merge(current_class
, Some(*parent_class
), "")
191 for (text
, class
) in self.pending_elems
.iter() {
192 string(self.out
, Escape(text
), *class
, &self.href_context
, false);
195 // We only want to "open" the tag ourselves if we have more than one pending and if the
196 // current parent tag is not the same as our pending content.
197 let close_tag
= if self.pending_elems
.len() > 1 && let Some(current_class
) = current_class
{
198 Some(enter_span(self.out
, current_class
, &self.href_context
))
202 for (text
, class
) in self.pending_elems
.iter() {
203 string(self.out
, Escape(text
), *class
, &self.href_context
, close_tag
.is_none());
205 if let Some(close_tag
) = close_tag
{
206 exit_span(self.out
, close_tag
);
209 self.pending_elems
.clear();
214 impl<'a
, 'tcx
, F
: Write
> Drop
for TokenHandler
<'a
, 'tcx
, F
> {
215 /// When leaving, we need to flush all pending data to not have missing content.
217 if self.pending_exit_span
.is_some() {
218 self.handle_exit_span();
220 self.write_pending_elems(self.current_class
);
225 /// Convert the given `src` source code into HTML by adding classes for highlighting.
227 /// This code is used to render code blocks (in the documentation) as well as the source code pages.
229 /// Some explanations on the last arguments:
231 /// In case we are rendering a code block and not a source code file, `href_context` will be `None`.
232 /// To put it more simply: if `href_context` is `None`, the code won't try to generate links to an
235 /// More explanations about spans and how we use them here are provided in the
236 pub(super) fn write_code(
237 out
: &mut impl Write
,
239 href_context
: Option
<HrefContext
<'_
, '_
>>,
240 decoration_info
: Option
<DecorationInfo
>,
242 // This replace allows to fix how the code source with DOS backline characters is displayed.
243 let src
= src
.replace("\r\n", "\n");
244 let mut token_handler
= TokenHandler
{
246 closing_tags
: Vec
::new(),
247 pending_exit_span
: None
,
249 pending_elems
: Vec
::new(),
255 token_handler
.href_context
.as_ref().map(|c
| c
.file_span
).unwrap_or(DUMMY_SP
),
258 .highlight(&mut |highlight
| {
260 Highlight
::Token { text, class }
=> {
261 // If we received a `ExitSpan` event and then have a non-compatible `Class`, we
262 // need to close the `<span>`.
263 let need_current_class_update
= if let Some(pending
) = token_handler
.pending_exit_span
&&
264 !can_merge(Some(pending
), class
, text
) {
265 token_handler
.handle_exit_span();
267 // If the two `Class` are different, time to flush the current content and start
269 } else if !can_merge(token_handler
.current_class
, class
, text
) {
270 token_handler
.write_pending_elems(token_handler
.current_class
);
273 token_handler
.current_class
.is_none()
276 if need_current_class_update
{
277 token_handler
.current_class
= class
.map(Class
::dummy
);
279 token_handler
.pending_elems
.push((text
, class
));
281 Highlight
::EnterSpan { class }
=> {
282 let mut should_add
= true;
283 if let Some(pending_exit_span
) = token_handler
.pending_exit_span
{
284 if class
.is_equal_to(pending_exit_span
) {
287 token_handler
.handle_exit_span();
290 // We flush everything just in case...
291 if token_handler
.write_pending_elems(token_handler
.current_class
) {
292 token_handler
.current_class
= None
;
296 let closing_tag
= enter_span(token_handler
.out
, class
, &token_handler
.href_context
);
297 token_handler
.closing_tags
.push((closing_tag
, class
));
300 token_handler
.current_class
= None
;
301 token_handler
.pending_exit_span
= None
;
303 Highlight
::ExitSpan
=> {
304 token_handler
.current_class
= None
;
305 token_handler
.pending_exit_span
=
306 Some(token_handler
.closing_tags
.last().as_ref().expect("ExitSpan without EnterSpan").1);
312 fn write_footer(out
: &mut Buffer
, playground_button
: Option
<&str>) {
313 writeln
!(out
, "</code></pre>{}</div>", playground_button
.unwrap_or_default());
316 /// How a span of text is classified. Mostly corresponds to token kinds.
317 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
323 /// Keywords that do pointer/reference stuff.
331 /// `Ident` isn't rendered in the HTML but we still need it for the `Span` it contains.
337 Decoration(&'
static str),
341 /// It is only looking at the variant, not the variant content.
343 /// It is used mostly to group multiple similar HTML elements into one `<span>` instead of
345 fn is_equal_to(self, other
: Self) -> bool
{
346 match (self, other
) {
347 (Self::Self_(_
), Self::Self_(_
))
348 | (Self::Macro(_
), Self::Macro(_
))
349 | (Self::Ident(_
), Self::Ident(_
)) => true,
350 (Self::Decoration(c1
), Self::Decoration(c2
)) => c1
== c2
,
355 /// If `self` contains a `Span`, it'll be replaced with `DUMMY_SP` to prevent creating links
356 /// on "empty content" (because of the attributes merge).
357 fn dummy(self) -> Self {
359 Self::Self_(_
) => Self::Self_(DUMMY_SP
),
360 Self::Macro(_
) => Self::Macro(DUMMY_SP
),
361 Self::Ident(_
) => Self::Ident(DUMMY_SP
),
366 /// Returns the css class expected by rustdoc for each `Class`.
367 fn as_html(self) -> &'
static str {
369 Class
::Comment
=> "comment",
370 Class
::DocComment
=> "doccomment",
371 Class
::Attribute
=> "attr",
372 Class
::KeyWord
=> "kw",
373 Class
::RefKeyWord
=> "kw-2",
374 Class
::Self_(_
) => "self",
375 Class
::Macro(_
) => "macro",
376 Class
::MacroNonTerminal
=> "macro-nonterminal",
377 Class
::String
=> "string",
378 Class
::Number
=> "number",
379 Class
::Bool
=> "bool-val",
380 Class
::Ident(_
) => "",
381 Class
::Lifetime
=> "lifetime",
382 Class
::PreludeTy
=> "prelude-ty",
383 Class
::PreludeVal
=> "prelude-val",
384 Class
::QuestionMark
=> "question-mark",
385 Class
::Decoration(kind
) => kind
,
389 /// In case this is an item which can be converted into a link to a definition, it'll contain
390 /// a "span" (a tuple representing `(lo, hi)` equivalent of `Span`).
391 fn get_span(self) -> Option
<Span
> {
393 Self::Ident(sp
) | Self::Self_(sp
) | Self::Macro(sp
) => Some(sp
),
399 | Self::MacroNonTerminal
407 | Self::Decoration(_
) => None
,
413 Token { text: &'a str, class: Option<Class> }
,
414 EnterSpan { class: Class }
,
418 struct TokenIter
<'a
> {
423 impl<'a
> Iterator
for TokenIter
<'a
> {
424 type Item
= (TokenKind
, &'a
str);
425 fn next(&mut self) -> Option
<(TokenKind
, &'a
str)> {
426 let token
= self.cursor
.advance_token();
427 if token
.kind
== TokenKind
::Eof
{
430 let (text
, rest
) = self.src
.split_at(token
.len
as usize);
432 Some((token
.kind
, text
))
436 /// Classifies into identifier class; returns `None` if this is a non-keyword identifier.
437 fn get_real_ident_class(text
: &str, allow_path_keywords
: bool
) -> Option
<Class
> {
438 let ignore
: &[&str] =
439 if allow_path_keywords { &["self", "Self", "super", "crate"] }
else { &["self", "Self"] }
;
440 if ignore
.iter().any(|k
| *k
== text
) {
444 "ref" | "mut" => Class
::RefKeyWord
,
445 "false" | "true" => Class
::Bool
,
446 _
if Symbol
::intern(text
).is_reserved(|| Edition
::Edition2021
) => Class
::KeyWord
,
451 /// This iterator comes from the same idea than "Peekable" except that it allows to "peek" more than
452 /// just the next item by using `peek_next`. The `peek` method always returns the next item after
453 /// the current one whereas `peek_next` will return the next item after the last one peeked.
455 /// You can use both `peek` and `peek_next` at the same time without problem.
456 struct PeekIter
<'a
> {
457 stored
: VecDeque
<(TokenKind
, &'a
str)>,
458 /// This position is reinitialized when using `next`. It is used in `peek_next`.
463 impl<'a
> PeekIter
<'a
> {
464 fn new(iter
: TokenIter
<'a
>) -> Self {
465 Self { stored: VecDeque::new(), peek_pos: 0, iter }
467 /// Returns the next item after the current one. It doesn't interfere with `peek_next` output.
468 fn peek(&mut self) -> Option
<&(TokenKind
, &'a
str)> {
469 if self.stored
.is_empty() && let Some(next
) = self.iter
.next() {
470 self.stored
.push_back(next
);
474 /// Returns the next item after the last one peeked. It doesn't interfere with `peek` output.
475 fn peek_next(&mut self) -> Option
<&(TokenKind
, &'a
str)> {
477 if self.peek_pos
- 1 < self.stored
.len() {
478 self.stored
.get(self.peek_pos
- 1)
479 } else if let Some(next
) = self.iter
.next() {
480 self.stored
.push_back(next
);
488 impl<'a
> Iterator
for PeekIter
<'a
> {
489 type Item
= (TokenKind
, &'a
str);
490 fn next(&mut self) -> Option
<Self::Item
> {
492 if let Some(first
) = self.stored
.pop_front() { Some(first) }
else { self.iter.next() }
496 /// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
498 starts
: Vec
<(u32, &'
static str)>,
503 fn new(info
: DecorationInfo
) -> Self {
504 // Extract tuples (start, end, kind) into separate sequences of (start, kind) and (end).
505 let (mut starts
, mut ends
): (Vec
<_
>, Vec
<_
>) = info
508 .flat_map(|(kind
, ranges
)| ranges
.into_iter().map(move |(lo
, hi
)| ((lo
, kind
), hi
)))
511 // Sort the sequences in document order.
512 starts
.sort_by_key(|(lo
, _
)| *lo
);
515 Decorations { starts, ends }
519 /// Processes program tokens, classifying strings of text by highlighting
520 /// category (`Class`).
521 struct Classifier
<'src
> {
522 tokens
: PeekIter
<'src
>,
525 in_macro_nonterminal
: bool
,
529 decorations
: Option
<Decorations
>,
532 impl<'src
> Classifier
<'src
> {
533 /// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
534 /// file span which will be used later on by the `span_correspondence_map`.
535 fn new(src
: &str, file_span
: Span
, decoration_info
: Option
<DecorationInfo
>) -> Classifier
<'_
> {
536 let tokens
= PeekIter
::new(TokenIter { src, cursor: Cursor::new(src) }
);
537 let decorations
= decoration_info
.map(Decorations
::new
);
542 in_macro_nonterminal
: false,
550 /// Convenient wrapper to create a [`Span`] from a position in the file.
551 fn new_span(&self, lo
: u32, text
: &str) -> Span
{
552 let hi
= lo
+ text
.len() as u32;
553 let file_lo
= self.file_span
.lo();
554 self.file_span
.with_lo(file_lo
+ BytePos(lo
)).with_hi(file_lo
+ BytePos(hi
))
557 /// Concatenate colons and idents as one when possible.
558 fn get_full_ident_path(&mut self) -> Vec
<(TokenKind
, usize, usize)> {
559 let start
= self.byte_pos
as usize;
561 let mut has_ident
= false;
565 while let Some((TokenKind
::Colon
, _
)) = self.tokens
.peek() {
569 // Ident path can start with "::" but if we already have content in the ident path,
570 // the "::" is mandatory.
571 if has_ident
&& nb
== 0 {
572 return vec
![(TokenKind
::Ident
, start
, pos
)];
573 } else if nb
!= 0 && nb
!= 2 {
575 return vec
![(TokenKind
::Ident
, start
, pos
), (TokenKind
::Colon
, pos
, pos
+ nb
)];
577 return vec
![(TokenKind
::Colon
, start
, pos
+ nb
)];
581 if let Some((None
, text
)) = self.tokens
.peek().map(|(token
, text
)| {
582 if *token
== TokenKind
::Ident
{
583 let class
= get_real_ident_class(text
, true);
586 // Doesn't matter which Class we put in here...
587 (Some(Class
::Comment
), text
)
590 // We only "add" the colon if there is an ident behind.
591 pos
+= text
.len() + nb
;
594 } else if nb
> 0 && has_ident
{
595 return vec
![(TokenKind
::Ident
, start
, pos
), (TokenKind
::Colon
, pos
, pos
+ nb
)];
597 return vec
![(TokenKind
::Colon
, start
, start
+ nb
)];
598 } else if has_ident
{
599 return vec
![(TokenKind
::Ident
, start
, pos
)];
606 /// Wraps the tokens iteration to ensure that the `byte_pos` is always correct.
608 /// It returns the token's kind, the token as a string and its byte position in the source
610 fn next(&mut self) -> Option
<(TokenKind
, &'src
str, u32)> {
611 if let Some((kind
, text
)) = self.tokens
.next() {
612 let before
= self.byte_pos
;
613 self.byte_pos
+= text
.len() as u32;
614 Some((kind
, text
, before
))
620 /// Exhausts the `Classifier` writing the output into `sink`.
622 /// The general structure for this method is to iterate over each token,
623 /// possibly giving it an HTML span with a class specifying what flavor of
625 fn highlight(mut self, sink
: &mut dyn FnMut(Highlight
<'src
>)) {
627 if let Some(decs
) = self.decorations
.as_mut() {
628 let byte_pos
= self.byte_pos
;
629 let n_starts
= decs
.starts
.iter().filter(|(i
, _
)| byte_pos
>= *i
).count();
630 for (_
, kind
) in decs
.starts
.drain(0..n_starts
) {
631 sink(Highlight
::EnterSpan { class: Class::Decoration(kind) }
);
634 let n_ends
= decs
.ends
.iter().filter(|i
| byte_pos
>= **i
).count();
635 for _
in decs
.ends
.drain(0..n_ends
) {
636 sink(Highlight
::ExitSpan
);
643 .map(|t
| matches
!(t
.0, TokenKind
::Colon
| TokenKind
::Ident
))
646 let tokens
= self.get_full_ident_path();
647 for (token
, start
, end
) in &tokens
{
648 let text
= &self.src
[*start
..*end
];
649 self.advance(*token
, text
, sink
, *start
as u32);
650 self.byte_pos
+= text
.len() as u32;
652 if !tokens
.is_empty() {
656 if let Some((token
, text
, before
)) = self.next() {
657 self.advance(token
, text
, sink
, before
);
664 /// Single step of highlighting. This will classify `token`, but maybe also a couple of
665 /// following ones as well.
667 /// `before` is the position of the given token in the `source` string and is used as "lo" byte
668 /// in case we want to try to generate a link for this token using the
669 /// `span_correspondence_map`.
674 sink
: &mut dyn FnMut(Highlight
<'src
>),
677 let lookahead
= self.peek();
678 let no_highlight
= |sink
: &mut dyn FnMut(_
)| sink(Highlight
::Token { text, class: None }
);
679 let class
= match token
{
680 TokenKind
::Whitespace
=> return no_highlight(sink
),
681 TokenKind
::LineComment { doc_style }
| TokenKind
::BlockComment { doc_style, .. }
=> {
682 if doc_style
.is_some() {
688 // Consider this as part of a macro invocation if there was a
689 // leading identifier.
690 TokenKind
::Bang
if self.in_macro
=> {
691 self.in_macro
= false;
692 sink(Highlight
::Token { text, class: None }
);
693 sink(Highlight
::ExitSpan
);
697 // Assume that '&' or '*' is the reference or dereference operator
698 // or a reference or pointer type. Unless, of course, it looks like
699 // a logical and or a multiplication operator: `&&` or `* `.
700 TokenKind
::Star
=> match self.tokens
.peek() {
701 Some((TokenKind
::Whitespace
, _
)) => return no_highlight(sink
),
702 Some((TokenKind
::Ident
, "mut")) => {
704 sink(Highlight
::Token { text: "*mut", class: Some(Class::RefKeyWord) }
);
707 Some((TokenKind
::Ident
, "const")) => {
709 sink(Highlight
::Token { text: "*const", class: Some(Class::RefKeyWord) }
);
712 _
=> Class
::RefKeyWord
,
714 TokenKind
::And
=> match self.tokens
.peek() {
715 Some((TokenKind
::And
, _
)) => {
717 sink(Highlight
::Token { text: "&&", class: None }
);
720 Some((TokenKind
::Eq
, _
)) => {
722 sink(Highlight
::Token { text: "&=", class: None }
);
725 Some((TokenKind
::Whitespace
, _
)) => return no_highlight(sink
),
726 Some((TokenKind
::Ident
, "mut")) => {
728 sink(Highlight
::Token { text: "&mut", class: Some(Class::RefKeyWord) }
);
731 _
=> Class
::RefKeyWord
,
734 // These can either be operators, or arrows.
735 TokenKind
::Eq
=> match lookahead
{
736 Some(TokenKind
::Eq
) => {
738 sink(Highlight
::Token { text: "==", class: None }
);
741 Some(TokenKind
::Gt
) => {
743 sink(Highlight
::Token { text: "=>", class: None }
);
746 _
=> return no_highlight(sink
),
748 TokenKind
::Minus
if lookahead
== Some(TokenKind
::Gt
) => {
750 sink(Highlight
::Token { text: "->", class: None }
);
763 | TokenKind
::Gt
=> return no_highlight(sink
),
765 // Miscellaneous, no highlighting.
769 | TokenKind
::OpenParen
770 | TokenKind
::CloseParen
771 | TokenKind
::OpenBrace
772 | TokenKind
::CloseBrace
773 | TokenKind
::OpenBracket
777 | TokenKind
::Unknown
=> return no_highlight(sink
),
779 TokenKind
::Question
=> Class
::QuestionMark
,
781 TokenKind
::Dollar
=> match lookahead
{
782 Some(TokenKind
::Ident
) => {
783 self.in_macro_nonterminal
= true;
784 Class
::MacroNonTerminal
786 _
=> return no_highlight(sink
),
789 // This might be the start of an attribute. We're going to want to
790 // continue highlighting it as an attribute until the ending ']' is
791 // seen, so skip out early. Down below we terminate the attribute
792 // span when we see the ']'.
793 TokenKind
::Pound
=> {
795 // Case 1: #![inner_attribute]
796 Some(TokenKind
::Bang
) => {
798 if let Some(TokenKind
::OpenBracket
) = self.peek() {
799 self.in_attribute
= true;
800 sink(Highlight
::EnterSpan { class: Class::Attribute }
);
802 sink(Highlight
::Token { text: "#", class: None }
);
803 sink(Highlight
::Token { text: "!", class: None }
);
806 // Case 2: #[outer_attribute]
807 Some(TokenKind
::OpenBracket
) => {
808 self.in_attribute
= true;
809 sink(Highlight
::EnterSpan { class: Class::Attribute }
);
813 return no_highlight(sink
);
815 TokenKind
::CloseBracket
=> {
816 if self.in_attribute
{
817 self.in_attribute
= false;
818 sink(Highlight
::Token { text: "]", class: None }
);
819 sink(Highlight
::ExitSpan
);
822 return no_highlight(sink
);
824 TokenKind
::Literal { kind, .. }
=> match kind
{
826 LiteralKind
::Byte { .. }
827 | LiteralKind
::Char { .. }
828 | LiteralKind
::Str { .. }
829 | LiteralKind
::ByteStr { .. }
830 | LiteralKind
::RawStr { .. }
831 | LiteralKind
::RawByteStr { .. }
832 | LiteralKind
::CStr { .. }
833 | LiteralKind
::RawCStr { .. }
=> Class
::String
,
835 LiteralKind
::Float { .. }
| LiteralKind
::Int { .. }
=> Class
::Number
,
837 TokenKind
::Ident
| TokenKind
::RawIdent
if lookahead
== Some(TokenKind
::Bang
) => {
838 self.in_macro
= true;
839 sink(Highlight
::EnterSpan { class: Class::Macro(self.new_span(before, text)) }
);
840 sink(Highlight
::Token { text, class: None }
);
843 TokenKind
::Ident
=> match get_real_ident_class(text
, false) {
845 "Option" | "Result" => Class
::PreludeTy
,
846 "Some" | "None" | "Ok" | "Err" => Class
::PreludeVal
,
847 // "union" is a weak keyword and is only considered as a keyword when declaring
849 "union" if self.check_if_is_union_keyword() => Class
::KeyWord
,
850 _
if self.in_macro_nonterminal
=> {
851 self.in_macro_nonterminal
= false;
852 Class
::MacroNonTerminal
854 "self" | "Self" => Class
::Self_(self.new_span(before
, text
)),
855 _
=> Class
::Ident(self.new_span(before
, text
)),
859 TokenKind
::RawIdent
| TokenKind
::UnknownPrefix
| TokenKind
::InvalidIdent
=> {
860 Class
::Ident(self.new_span(before
, text
))
862 TokenKind
::Lifetime { .. }
=> Class
::Lifetime
,
863 TokenKind
::Eof
=> panic
!("Eof in advance"),
865 // Anything that didn't return above is the simple case where we the
866 // class just spans a single token, so we can use the `string` method.
867 sink(Highlight
::Token { text, class: Some(class) }
);
870 fn peek(&mut self) -> Option
<TokenKind
> {
871 self.tokens
.peek().map(|(token_kind
, _text
)| *token_kind
)
874 fn check_if_is_union_keyword(&mut self) -> bool
{
875 while let Some(kind
) = self.tokens
.peek_next().map(|(token_kind
, _text
)| token_kind
) {
876 if *kind
== TokenKind
::Whitespace
{
879 return *kind
== TokenKind
::Ident
;
885 /// Called when we start processing a span of text that should be highlighted.
886 /// The `Class` argument specifies how it should be highlighted.
888 out
: &mut impl Write
,
890 href_context
: &Option
<HrefContext
<'_
, '_
>>,
892 string_without_closing_tag(out
, "", Some(klass
), href_context
, true).expect(
893 "internal error: enter_span was called with Some(klass) but did not return a \
898 /// Called at the end of a span of highlighted text.
899 fn exit_span(out
: &mut impl Write
, closing_tag
: &str) {
900 out
.write_str(closing_tag
).unwrap();
903 /// Called for a span of text. If the text should be highlighted differently
904 /// from the surrounding text, then the `Class` argument will be a value other
907 /// The following sequences of callbacks are equivalent:
909 /// enter_span(Foo), string("text", None), exit_span()
910 /// string("text", Foo)
913 /// The latter can be thought of as a shorthand for the former, which is more
916 /// Note that if `context` is not `None` and that the given `klass` contains a `Span`, the function
917 /// will then try to find this `span` in the `span_correspondence_map`. If found, it'll then
918 /// generate a link for this element (which corresponds to where its definition is located).
919 fn string
<T
: Display
>(
920 out
: &mut impl Write
,
922 klass
: Option
<Class
>,
923 href_context
: &Option
<HrefContext
<'_
, '_
>>,
926 if let Some(closing_tag
) = string_without_closing_tag(out
, text
, klass
, href_context
, open_tag
)
928 out
.write_str(closing_tag
).unwrap();
932 /// This function writes `text` into `out` with some modifications depending on `klass`:
934 /// * If `klass` is `None`, `text` is written into `out` with no modification.
935 /// * If `klass` is `Some` but `klass.get_span()` is `None`, it writes the text wrapped in a
936 /// `<span>` with the provided `klass`.
937 /// * If `klass` is `Some` and has a [`rustc_span::Span`], it then tries to generate a link (`<a>`
938 /// element) by retrieving the link information from the `span_correspondence_map` that was filled
939 /// in `span_map.rs::collect_spans_and_sources`. If it cannot retrieve the information, then it's
940 /// the same as the second point (`klass` is `Some` but doesn't have a [`rustc_span::Span`]).
941 fn string_without_closing_tag
<T
: Display
>(
942 out
: &mut impl Write
,
944 klass
: Option
<Class
>,
945 href_context
: &Option
<HrefContext
<'_
, '_
>>,
947 ) -> Option
<&'
static str> {
948 let Some(klass
) = klass
else {
949 write
!(out
, "{text}").unwrap();
952 let Some(def_span
) = klass
.get_span() else {
954 write
!(out
, "{text}").unwrap();
957 write
!(out
, "<span class=\"{klass}\">{text}", klass
= klass
.as_html()).unwrap();
958 return Some("</span>");
961 let mut text_s
= text
.to_string();
962 if text_s
.contains("::") {
963 text_s
= text_s
.split("::").intersperse("::").fold(String
::new(), |mut path
, t
| {
965 "self" | "Self" => write
!(
967 "<span class=\"{klass}\">{t}</span>",
968 klass
= Class
::Self_(DUMMY_SP
).as_html(),
970 "crate" | "super" => {
973 "<span class=\"{klass}\">{t}</span>",
974 klass
= Class
::KeyWord
.as_html(),
977 t
=> write
!(&mut path
, "{t}"),
979 .expect("Failed to build source HTML path");
984 if let Some(href_context
) = href_context
{
986 href_context
.context
.shared
.span_correspondence_map
.get(&def_span
).and_then(|href
| {
987 let context
= href_context
.context
;
988 // FIXME: later on, it'd be nice to provide two links (if possible) for all items:
989 // one to the documentation page and one to the source definition.
990 // FIXME: currently, external items only generate a link to their documentation,
991 // a link to their definition can be generated using this:
992 // https://github.com/rust-lang/rust/blob/60f1a2fc4b535ead9c85ce085fdce49b1b097531/src/librustdoc/html/render/context.rs#L315-L338
994 LinkFromSrc
::Local(span
) => {
995 context
.href_from_span_relative(*span
, &href_context
.current_href
)
997 LinkFromSrc
::External(def_id
) => {
998 format
::href_with_root_path(*def_id
, context
, Some(href_context
.root_path
))
1000 .map(|(url
, _
, _
)| url
)
1002 LinkFromSrc
::Primitive(prim
) => format
::href_with_root_path(
1003 PrimitiveType
::primitive_locations(context
.tcx())[prim
],
1005 Some(href_context
.root_path
),
1008 .map(|(url
, _
, _
)| url
),
1009 LinkFromSrc
::Doc(def_id
) => {
1010 format
::href_with_root_path(*def_id
, context
, Some(&href_context
.root_path
))
1012 .map(|(doc_link
, _
, _
)| doc_link
)
1018 // We're already inside an element which has the same klass, no need to give it
1020 write
!(out
, "<a href=\"{href}\">{text_s}").unwrap();
1022 let klass_s
= klass
.as_html();
1023 if klass_s
.is_empty() {
1024 write
!(out
, "<a href=\"{href}\">{text_s}").unwrap();
1026 write
!(out
, "<a class=\"{klass_s}\" href=\"{href}\">{text_s}").unwrap();
1029 return Some("</a>");
1033 write
!(out
, "{}", text_s
).unwrap();
1036 let klass_s
= klass
.as_html();
1037 if klass_s
.is_empty() {
1038 out
.write_str(&text_s
).unwrap();
1041 write
!(out
, "<span class=\"{klass_s}\">{text_s}").unwrap();