1 //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].
3 use rustc_hash
::FxHashMap
;
4 use stdx
::{always, non_empty_vec::NonEmptyVec}
;
6 ast
::{self, make::tokens::doc_comment}
,
7 AstToken
, Parse
, PreorderWithTokens
, SmolStr
, SyntaxElement
, SyntaxKind
,
9 SyntaxNode
, SyntaxToken
, SyntaxTreeBuilder
, TextRange
, TextSize
, WalkEvent
, T
,
13 to_parser_input
::to_parser_input
,
16 buffer
::{Cursor, TokenBuffer}
,
25 /// Convert the syntax node to a `TokenTree` (what macro
27 pub fn syntax_node_to_token_tree(node
: &SyntaxNode
) -> (tt
::Subtree
, TokenMap
) {
28 let (subtree
, token_map
, _
) = syntax_node_to_token_tree_with_modifications(
38 /// Convert the syntax node to a `TokenTree` (what macro will consume)
39 /// with the censored range excluded.
40 pub fn syntax_node_to_token_tree_with_modifications(
42 existing_token_map
: TokenMap
,
44 replace
: FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
45 append
: FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
46 ) -> (tt
::Subtree
, TokenMap
, u32) {
47 let global_offset
= node
.text_range().start();
48 let mut c
= Converter
::new(node
, global_offset
, existing_token_map
, next_id
, replace
, append
);
49 let subtree
= convert_tokens(&mut c
);
50 c
.id_alloc
.map
.shrink_to_fit();
51 always
!(c
.replace
.is_empty(), "replace: {:?}", c
.replace
);
52 always
!(c
.append
.is_empty(), "append: {:?}", c
.append
);
53 (subtree
, c
.id_alloc
.map
, c
.id_alloc
.next_id
)
56 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
57 pub struct SyntheticTokenId(pub u32);
59 #[derive(Debug, Clone)]
60 pub struct SyntheticToken
{
64 pub id
: SyntheticTokenId
,
67 // The following items are what `rustc` macro can be parsed into :
68 // link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141
69 // * Expr(P<ast::Expr>) -> token_tree_to_expr
70 // * Pat(P<ast::Pat>) -> token_tree_to_pat
71 // * Ty(P<ast::Ty>) -> token_tree_to_ty
72 // * Stmts(SmallVec<[ast::Stmt; 1]>) -> token_tree_to_stmts
73 // * Items(SmallVec<[P<ast::Item>; 1]>) -> token_tree_to_items
75 // * TraitItems(SmallVec<[ast::TraitItem; 1]>)
76 // * AssocItems(SmallVec<[ast::AssocItem; 1]>)
77 // * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
79 pub fn token_tree_to_syntax_node(
81 entry_point
: parser
::TopEntryPoint
,
82 ) -> (Parse
<SyntaxNode
>, TokenMap
) {
83 let buffer
= match tt
{
85 delimiter
: tt
::Delimiter { kind: tt::DelimiterKind::Invisible, .. }
,
87 } => TokenBuffer
::from_tokens(token_trees
.as_slice()),
88 _
=> TokenBuffer
::from_subtree(tt
),
90 let parser_input
= to_parser_input(&buffer
);
91 let parser_output
= entry_point
.parse(&parser_input
);
92 let mut tree_sink
= TtTreeSink
::new(buffer
.begin());
93 for event
in parser_output
.iter() {
95 parser
::Step
::Token { kind, n_input_tokens: n_raw_tokens }
=> {
96 tree_sink
.token(kind
, n_raw_tokens
)
98 parser
::Step
::FloatSplit { ends_in_dot: has_pseudo_dot }
=> {
99 tree_sink
.float_split(has_pseudo_dot
)
101 parser
::Step
::Enter { kind }
=> tree_sink
.start_node(kind
),
102 parser
::Step
::Exit
=> tree_sink
.finish_node(),
103 parser
::Step
::Error { msg }
=> tree_sink
.error(msg
.to_string()),
109 /// Convert a string to a `TokenTree`
110 pub fn parse_to_token_tree(text
: &str) -> Option
<(tt
::Subtree
, TokenMap
)> {
111 let lexed
= parser
::LexedStr
::new(text
);
112 if lexed
.errors().next().is_some() {
116 let mut conv
= RawConverter
{
119 id_alloc
: TokenIdAlloc
{
120 map
: Default
::default(),
121 global_offset
: TextSize
::default(),
126 let subtree
= convert_tokens(&mut conv
);
127 Some((subtree
, conv
.id_alloc
.map
))
130 /// Split token tree with separate expr: $($e:expr)SEP*
131 pub fn parse_exprs_with_sep(tt
: &tt
::Subtree
, sep
: char) -> Vec
<tt
::Subtree
> {
132 if tt
.token_trees
.is_empty() {
136 let mut iter
= TtIter
::new(tt
);
137 let mut res
= Vec
::new();
139 while iter
.peek_n(0).is_some() {
140 let expanded
= iter
.expect_fragment(parser
::PrefixEntryPoint
::Expr
);
142 res
.push(match expanded
.value
{
144 Some(tt @ tt
::TokenTree
::Leaf(_
)) => {
145 tt
::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![tt] }
147 Some(tt
::TokenTree
::Subtree(tt
)) => tt
,
150 let mut fork
= iter
.clone();
151 if fork
.expect_char(sep
).is_err() {
157 if iter
.peek_n(0).is_some() {
158 res
.push(tt
::Subtree
{
159 delimiter
: tt
::Delimiter
::unspecified(),
160 token_trees
: iter
.cloned().collect(),
167 fn convert_tokens
<C
: TokenConverter
>(conv
: &mut C
) -> tt
::Subtree
{
169 subtree
: tt
::Subtree
,
171 open_range
: TextRange
,
174 let entry
= StackEntry
{
175 subtree
: tt
::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] }
,
176 // never used (delimiter is `None`)
178 open_range
: TextRange
::empty(TextSize
::of('
.'
)),
180 let mut stack
= NonEmptyVec
::new(entry
);
183 let StackEntry { subtree, .. }
= stack
.last_mut();
184 let result
= &mut subtree
.token_trees
;
185 let (token
, range
) = match conv
.bump() {
189 let synth_id
= token
.synthetic_id(conv
);
191 let kind
= token
.kind(conv
);
193 if let Some(tokens
) = conv
.convert_doc_comment(&token
) {
194 // FIXME: There has to be a better way to do this
195 // Add the comments token id to the converted doc string
196 let id
= conv
.id_alloc().alloc(range
, synth_id
);
197 result
.extend(tokens
.into_iter().map(|mut tt
| {
198 if let tt
::TokenTree
::Subtree(sub
) = &mut tt
{
199 if let Some(tt
::TokenTree
::Leaf(tt
::Leaf
::Literal(lit
))) =
200 sub
.token_trees
.get_mut(2)
210 let tt
= if kind
.is_punct() && kind
!= UNDERSCORE
{
211 if synth_id
.is_none() {
212 assert_eq
!(range
.len(), TextSize
::of('
.'
));
215 let expected
= match subtree
.delimiter
.kind
{
216 tt
::DelimiterKind
::Parenthesis
=> Some(T
!['
)'
]),
217 tt
::DelimiterKind
::Brace
=> Some(T
!['
}'
]),
218 tt
::DelimiterKind
::Bracket
=> Some(T
!['
]'
]),
219 tt
::DelimiterKind
::Invisible
=> None
,
222 if let Some(expected
) = expected
{
223 if kind
== expected
{
224 if let Some(entry
) = stack
.pop() {
225 conv
.id_alloc().close_delim(entry
.idx
, Some(range
));
226 stack
.last_mut().subtree
.token_trees
.push(entry
.subtree
.into());
232 let delim
= match kind
{
233 T
!['
('
] => Some(tt
::DelimiterKind
::Parenthesis
),
234 T
!['
{'
] => Some(tt
::DelimiterKind
::Brace
),
235 T
!['
['
] => Some(tt
::DelimiterKind
::Bracket
),
239 if let Some(kind
) = delim
{
240 let (id
, idx
) = conv
.id_alloc().open_delim(range
, synth_id
);
241 let subtree
= tt
::Subtree
{
242 delimiter
: tt
::Delimiter { open: id, close: tt::TokenId::UNSPECIFIED, kind }
,
245 stack
.push(StackEntry { subtree, idx, open_range: range }
);
249 let spacing
= match conv
.peek().map(|next
| next
.kind(conv
)) {
250 Some(kind
) if is_single_token_op(kind
) => tt
::Spacing
::Joint
,
251 _
=> tt
::Spacing
::Alone
,
253 let char = match token
.to_char(conv
) {
256 panic
!("Token from lexer must be single char: token = {token:#?}");
259 tt
::Leaf
::from(tt
::Punct
{
262 span
: conv
.id_alloc().alloc(range
, synth_id
),
266 macro_rules
! make_leaf
{
269 span
: conv
.id_alloc().alloc(range
, synth_id
),
270 text
: token
.to_text(conv
),
275 let leaf
: tt
::Leaf
= match kind
{
276 T
![true] | T
![false] => make_leaf
!(Ident
),
277 IDENT
=> make_leaf
!(Ident
),
278 UNDERSCORE
=> make_leaf
!(Ident
),
279 k
if k
.is_keyword() => make_leaf
!(Ident
),
280 k
if k
.is_literal() => make_leaf
!(Literal
),
282 let char_unit
= TextSize
::of('
\''
);
283 let r
= TextRange
::at(range
.start(), char_unit
);
284 let apostrophe
= tt
::Leaf
::from(tt
::Punct
{
286 spacing
: tt
::Spacing
::Joint
,
287 span
: conv
.id_alloc().alloc(r
, synth_id
),
289 result
.push(apostrophe
.into());
291 let r
= TextRange
::at(range
.start() + char_unit
, range
.len() - char_unit
);
292 let ident
= tt
::Leaf
::from(tt
::Ident
{
293 text
: SmolStr
::new(&token
.to_text(conv
)[1..]),
294 span
: conv
.id_alloc().alloc(r
, synth_id
),
296 result
.push(ident
.into());
307 // If we get here, we've consumed all input tokens.
308 // We might have more than one subtree in the stack, if the delimiters are improperly balanced.
309 // Merge them so we're left with one.
310 while let Some(entry
) = stack
.pop() {
311 let parent
= stack
.last_mut();
313 conv
.id_alloc().close_delim(entry
.idx
, None
);
314 let leaf
: tt
::Leaf
= tt
::Punct
{
315 span
: conv
.id_alloc().alloc(entry
.open_range
, None
),
316 char: match entry
.subtree
.delimiter
.kind
{
317 tt
::DelimiterKind
::Parenthesis
=> '
('
,
318 tt
::DelimiterKind
::Brace
=> '
{'
,
319 tt
::DelimiterKind
::Bracket
=> '
['
,
320 tt
::DelimiterKind
::Invisible
=> '$'
,
322 spacing
: tt
::Spacing
::Alone
,
325 parent
.subtree
.token_trees
.push(leaf
.into());
326 parent
.subtree
.token_trees
.extend(entry
.subtree
.token_trees
);
329 let subtree
= stack
.into_last().subtree
;
330 if let [tt
::TokenTree
::Subtree(first
)] = &*subtree
.token_trees
{
337 fn is_single_token_op(kind
: SyntaxKind
) -> bool
{
360 // LIFETIME_IDENT will be split into a sequence of `'` (a single quote) and an
366 /// Returns the textual content of a doc comment block as a quoted string
367 /// That is, strips leading `///` (or `/**`, etc)
368 /// and strips the ending `*/`
369 /// And then quote the string, which is needed to convert to `tt::Literal`
370 fn doc_comment_text(comment
: &ast
::Comment
) -> SmolStr
{
371 let prefix_len
= comment
.prefix().len();
372 let mut text
= &comment
.text()[prefix_len
..];
374 // Remove ending "*/"
375 if comment
.kind().shape
== ast
::CommentShape
::Block
{
376 text
= &text
[0..text
.len() - 2];
380 // Note that `tt::Literal` expect an escaped string
381 let text
= format
!("\"{}\"", text
.escape_debug());
385 fn convert_doc_comment(token
: &syntax
::SyntaxToken
) -> Option
<Vec
<tt
::TokenTree
>> {
386 cov_mark
::hit
!(test_meta_doc_comments
);
387 let comment
= ast
::Comment
::cast(token
.clone())?
;
388 let doc
= comment
.kind().doc?
;
390 // Make `doc="\" Comments\""
391 let meta_tkns
= vec
![mk_ident("doc"), mk_punct('
='
), mk_doc_literal(&comment
)];
394 let mut token_trees
= Vec
::with_capacity(3);
395 token_trees
.push(mk_punct('
#'));
396 if let ast
::CommentPlacement
::Inner
= doc
{
397 token_trees
.push(mk_punct('
!'
));
399 token_trees
.push(tt
::TokenTree
::from(tt
::Subtree
{
400 delimiter
: tt
::Delimiter
{
401 open
: tt
::TokenId
::UNSPECIFIED
,
402 close
: tt
::TokenId
::UNSPECIFIED
,
403 kind
: tt
::DelimiterKind
::Bracket
,
405 token_trees
: meta_tkns
,
408 return Some(token_trees
);
411 fn mk_ident(s
: &str) -> tt
::TokenTree
{
412 tt
::TokenTree
::from(tt
::Leaf
::from(tt
::Ident
{
414 span
: tt
::TokenId
::unspecified(),
418 fn mk_punct(c
: char) -> tt
::TokenTree
{
419 tt
::TokenTree
::from(tt
::Leaf
::from(tt
::Punct
{
421 spacing
: tt
::Spacing
::Alone
,
422 span
: tt
::TokenId
::unspecified(),
426 fn mk_doc_literal(comment
: &ast
::Comment
) -> tt
::TokenTree
{
427 let lit
= tt
::Literal { text: doc_comment_text(comment), span: tt::TokenId::unspecified() }
;
429 tt
::TokenTree
::from(tt
::Leaf
::from(lit
))
433 struct TokenIdAlloc
{
435 global_offset
: TextSize
,
442 absolute_range
: TextRange
,
443 synthetic_id
: Option
<SyntheticTokenId
>,
445 let relative_range
= absolute_range
- self.global_offset
;
446 let token_id
= tt
::TokenId(self.next_id
);
448 self.map
.insert(token_id
, relative_range
);
449 if let Some(id
) = synthetic_id
{
450 self.map
.insert_synthetic(token_id
, id
);
457 open_abs_range
: TextRange
,
458 synthetic_id
: Option
<SyntheticTokenId
>,
459 ) -> (tt
::TokenId
, usize) {
460 let token_id
= tt
::TokenId(self.next_id
);
462 let idx
= self.map
.insert_delim(
464 open_abs_range
- self.global_offset
,
465 open_abs_range
- self.global_offset
,
467 if let Some(id
) = synthetic_id
{
468 self.map
.insert_synthetic(token_id
, id
);
473 fn close_delim(&mut self, idx
: usize, close_abs_range
: Option
<TextRange
>) {
474 match close_abs_range
{
476 self.map
.remove_delim(idx
);
479 self.map
.update_close_delim(idx
, close
- self.global_offset
);
485 /// A raw token (straight from lexer) converter
486 struct RawConverter
<'a
> {
487 lexed
: parser
::LexedStr
<'a
>,
489 id_alloc
: TokenIdAlloc
,
492 trait SrcToken
<Ctx
>: std
::fmt
::Debug
{
493 fn kind(&self, ctx
: &Ctx
) -> SyntaxKind
;
495 fn to_char(&self, ctx
: &Ctx
) -> Option
<char>;
497 fn to_text(&self, ctx
: &Ctx
) -> SmolStr
;
499 fn synthetic_id(&self, ctx
: &Ctx
) -> Option
<SyntheticTokenId
>;
502 trait TokenConverter
: Sized
{
503 type Token
: SrcToken
<Self>;
505 fn convert_doc_comment(&self, token
: &Self::Token
) -> Option
<Vec
<tt
::TokenTree
>>;
507 fn bump(&mut self) -> Option
<(Self::Token
, TextRange
)>;
509 fn peek(&self) -> Option
<Self::Token
>;
511 fn id_alloc(&mut self) -> &mut TokenIdAlloc
;
514 impl<'a
> SrcToken
<RawConverter
<'a
>> for usize {
515 fn kind(&self, ctx
: &RawConverter
<'a
>) -> SyntaxKind
{
516 ctx
.lexed
.kind(*self)
519 fn to_char(&self, ctx
: &RawConverter
<'a
>) -> Option
<char> {
520 ctx
.lexed
.text(*self).chars().next()
523 fn to_text(&self, ctx
: &RawConverter
<'_
>) -> SmolStr
{
524 ctx
.lexed
.text(*self).into()
527 fn synthetic_id(&self, _ctx
: &RawConverter
<'a
>) -> Option
<SyntheticTokenId
> {
532 impl<'a
> TokenConverter
for RawConverter
<'a
> {
535 fn convert_doc_comment(&self, &token
: &usize) -> Option
<Vec
<tt
::TokenTree
>> {
536 let text
= self.lexed
.text(token
);
537 convert_doc_comment(&doc_comment(text
))
540 fn bump(&mut self) -> Option
<(Self::Token
, TextRange
)> {
541 if self.pos
== self.lexed
.len() {
544 let token
= self.pos
;
546 let range
= self.lexed
.text_range(token
);
547 let range
= TextRange
::new(range
.start
.try_into().unwrap(), range
.end
.try_into().unwrap());
552 fn peek(&self) -> Option
<Self::Token
> {
553 if self.pos
== self.lexed
.len() {
559 fn id_alloc(&mut self) -> &mut TokenIdAlloc
{
565 id_alloc
: TokenIdAlloc
,
566 current
: Option
<SyntaxToken
>,
567 current_synthetic
: Vec
<SyntheticToken
>,
568 preorder
: PreorderWithTokens
,
569 replace
: FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
570 append
: FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
572 punct_offset
: Option
<(SyntaxToken
, TextSize
)>,
578 global_offset
: TextSize
,
579 existing_token_map
: TokenMap
,
581 mut replace
: FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
582 mut append
: FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
584 let range
= node
.text_range();
585 let mut preorder
= node
.preorder_with_tokens();
586 let (first
, synthetic
) = Self::next_token(&mut preorder
, &mut replace
, &mut append
);
588 id_alloc
: { TokenIdAlloc { map: existing_token_map, global_offset, next_id }
},
590 current_synthetic
: synthetic
,
600 preorder
: &mut PreorderWithTokens
,
601 replace
: &mut FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
602 append
: &mut FxHashMap
<SyntaxElement
, Vec
<SyntheticToken
>>,
603 ) -> (Option
<SyntaxToken
>, Vec
<SyntheticToken
>) {
604 while let Some(ev
) = preorder
.next() {
606 WalkEvent
::Enter(ele
) => ele
,
607 WalkEvent
::Leave(ele
) => {
608 if let Some(mut v
) = append
.remove(&ele
) {
617 if let Some(mut v
) = replace
.remove(&ele
) {
618 preorder
.skip_subtree();
625 SyntaxElement
::Token(t
) => return (Some(t
), Vec
::new()),
635 Ordinary(SyntaxToken
),
636 // FIXME is this supposed to be `Punct`?
637 Punch(SyntaxToken
, TextSize
),
638 Synthetic(SyntheticToken
),
642 fn token(&self) -> Option
<&SyntaxToken
> {
644 SynToken
::Ordinary(it
) | SynToken
::Punch(it
, _
) => Some(it
),
645 SynToken
::Synthetic(_
) => None
,
650 impl SrcToken
<Converter
> for SynToken
{
651 fn kind(&self, ctx
: &Converter
) -> SyntaxKind
{
653 SynToken
::Ordinary(token
) => token
.kind(),
654 SynToken
::Punch(..) => SyntaxKind
::from_char(self.to_char(ctx
).unwrap()).unwrap(),
655 SynToken
::Synthetic(token
) => token
.kind
,
658 fn to_char(&self, _ctx
: &Converter
) -> Option
<char> {
660 SynToken
::Ordinary(_
) => None
,
661 SynToken
::Punch(it
, i
) => it
.text().chars().nth((*i
).into()),
662 SynToken
::Synthetic(token
) if token
.text
.len() == 1 => token
.text
.chars().next(),
663 SynToken
::Synthetic(_
) => None
,
666 fn to_text(&self, _ctx
: &Converter
) -> SmolStr
{
668 SynToken
::Ordinary(token
) => token
.text().into(),
669 SynToken
::Punch(token
, _
) => token
.text().into(),
670 SynToken
::Synthetic(token
) => token
.text
.clone(),
674 fn synthetic_id(&self, _ctx
: &Converter
) -> Option
<SyntheticTokenId
> {
676 SynToken
::Synthetic(token
) => Some(token
.id
),
682 impl TokenConverter
for Converter
{
683 type Token
= SynToken
;
684 fn convert_doc_comment(&self, token
: &Self::Token
) -> Option
<Vec
<tt
::TokenTree
>> {
685 convert_doc_comment(token
.token()?
)
688 fn bump(&mut self) -> Option
<(Self::Token
, TextRange
)> {
689 if let Some((punct
, offset
)) = self.punct_offset
.clone() {
690 if usize::from(offset
) + 1 < punct
.text().len() {
691 let offset
= offset
+ TextSize
::of('
.'
);
692 let range
= punct
.text_range();
693 self.punct_offset
= Some((punct
.clone(), offset
));
694 let range
= TextRange
::at(range
.start() + offset
, TextSize
::of('
.'
));
695 return Some((SynToken
::Punch(punct
, offset
), range
));
699 if let Some(synth_token
) = self.current_synthetic
.pop() {
700 if self.current_synthetic
.is_empty() {
701 let (new_current
, new_synth
) =
702 Self::next_token(&mut self.preorder
, &mut self.replace
, &mut self.append
);
703 self.current
= new_current
;
704 self.current_synthetic
= new_synth
;
706 let range
= synth_token
.range
;
707 return Some((SynToken
::Synthetic(synth_token
), range
));
710 let curr
= self.current
.clone()?
;
711 if !self.range
.contains_range(curr
.text_range()) {
714 let (new_current
, new_synth
) =
715 Self::next_token(&mut self.preorder
, &mut self.replace
, &mut self.append
);
716 self.current
= new_current
;
717 self.current_synthetic
= new_synth
;
718 let token
= if curr
.kind().is_punct() {
719 self.punct_offset
= Some((curr
.clone(), 0.into
()));
720 let range
= curr
.text_range();
721 let range
= TextRange
::at(range
.start(), TextSize
::of('
.'
));
722 (SynToken
::Punch(curr
, 0.into
()), range
)
724 self.punct_offset
= None
;
725 let range
= curr
.text_range();
726 (SynToken
::Ordinary(curr
), range
)
732 fn peek(&self) -> Option
<Self::Token
> {
733 if let Some((punct
, mut offset
)) = self.punct_offset
.clone() {
734 offset
+= TextSize
::of('
.'
);
735 if usize::from(offset
) < punct
.text().len() {
736 return Some(SynToken
::Punch(punct
, offset
));
740 if let Some(synth_token
) = self.current_synthetic
.last() {
741 return Some(SynToken
::Synthetic(synth_token
.clone()));
744 let curr
= self.current
.clone()?
;
745 if !self.range
.contains_range(curr
.text_range()) {
749 let token
= if curr
.kind().is_punct() {
750 SynToken
::Punch(curr
, 0.into
())
752 SynToken
::Ordinary(curr
)
757 fn id_alloc(&mut self) -> &mut TokenIdAlloc
{
762 struct TtTreeSink
<'a
> {
765 open_delims
: FxHashMap
<tt
::TokenId
, TextSize
>,
767 inner
: SyntaxTreeBuilder
,
771 impl<'a
> TtTreeSink
<'a
> {
772 fn new(cursor
: Cursor
<'a
>) -> Self {
776 open_delims
: FxHashMap
::default(),
778 inner
: SyntaxTreeBuilder
::default(),
779 token_map
: TokenMap
::default(),
783 fn finish(mut self) -> (Parse
<SyntaxNode
>, TokenMap
) {
784 self.token_map
.shrink_to_fit();
785 (self.inner
.finish(), self.token_map
)
789 fn delim_to_str(d
: tt
::DelimiterKind
, closing
: bool
) -> Option
<&'
static str> {
790 let texts
= match d
{
791 tt
::DelimiterKind
::Parenthesis
=> "()",
792 tt
::DelimiterKind
::Brace
=> "{}",
793 tt
::DelimiterKind
::Bracket
=> "[]",
794 tt
::DelimiterKind
::Invisible
=> return None
,
797 let idx
= closing
as usize;
798 Some(&texts
[idx
..texts
.len() - (1 - idx
)])
801 impl<'a
> TtTreeSink
<'a
> {
802 /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween.
803 /// This occurs when a float literal is used as a field access.
804 fn float_split(&mut self, has_pseudo_dot
: bool
) {
805 let (text
, _span
) = match self.cursor
.token_tree() {
806 Some(tt
::buffer
::TokenTreeRef
::Leaf(tt
::Leaf
::Literal(lit
), _
)) => {
807 (lit
.text
.as_str(), lit
.span
)
811 match text
.split_once('
.'
) {
812 Some((left
, right
)) => {
813 assert
!(!left
.is_empty());
814 self.inner
.start_node(SyntaxKind
::NAME_REF
);
815 self.inner
.token(SyntaxKind
::INT_NUMBER
, left
);
816 self.inner
.finish_node();
818 // here we move the exit up, the original exit has been deleted in process
819 self.inner
.finish_node();
821 self.inner
.token(SyntaxKind
::DOT
, ".");
824 assert
!(right
.is_empty(), "{left}.{right}");
826 self.inner
.start_node(SyntaxKind
::NAME_REF
);
827 self.inner
.token(SyntaxKind
::INT_NUMBER
, right
);
828 self.inner
.finish_node();
830 // the parser creates an unbalanced start node, we are required to close it here
831 self.inner
.finish_node();
834 None
=> unreachable
!(),
836 self.cursor
= self.cursor
.bump();
839 fn token(&mut self, kind
: SyntaxKind
, mut n_tokens
: u8) {
840 if kind
== LIFETIME_IDENT
{
844 let mut last
= self.cursor
;
845 for _
in 0..n_tokens
{
847 if self.cursor
.eof() {
851 let text
: &str = loop {
852 break match self.cursor
.token_tree() {
853 Some(tt
::buffer
::TokenTreeRef
::Leaf(leaf
, _
)) => {
854 // Mark the range if needed
855 let (text
, id
) = match leaf
{
856 tt
::Leaf
::Ident(ident
) => (ident
.text
.as_str(), ident
.span
),
857 tt
::Leaf
::Punct(punct
) => {
858 assert
!(punct
.char.is_ascii());
859 tmp
= punct
.char as u8;
861 std
::str::from_utf8(std
::slice
::from_ref(&tmp
)).unwrap(),
865 tt
::Leaf
::Literal(lit
) => (lit
.text
.as_str(), lit
.span
),
867 let range
= TextRange
::at(self.text_pos
, TextSize
::of(text
));
868 self.token_map
.insert(id
, range
);
869 self.cursor
= self.cursor
.bump();
872 Some(tt
::buffer
::TokenTreeRef
::Subtree(subtree
, _
)) => {
873 self.cursor
= self.cursor
.subtree().unwrap();
874 match delim_to_str(subtree
.delimiter
.kind
, false) {
876 self.open_delims
.insert(subtree
.delimiter
.open
, self.text_pos
);
883 let parent
= self.cursor
.end().unwrap();
884 self.cursor
= self.cursor
.bump();
885 match delim_to_str(parent
.delimiter
.kind
, true) {
887 if let Some(open_delim
) =
888 self.open_delims
.get(&parent
.delimiter
.open
)
890 let open_range
= TextRange
::at(*open_delim
, TextSize
::of('
('
));
892 TextRange
::at(self.text_pos
, TextSize
::of('
('
));
893 self.token_map
.insert_delim(
894 parent
.delimiter
.open
,
907 self.text_pos
+= TextSize
::of(text
);
910 self.inner
.token(kind
, self.buf
.as_str());
912 // Add whitespace between adjoint puncts
913 let next
= last
.bump();
915 Some(tt
::buffer
::TokenTreeRef
::Leaf(tt
::Leaf
::Punct(curr
), _
)),
916 Some(tt
::buffer
::TokenTreeRef
::Leaf(tt
::Leaf
::Punct(next
), _
)),
917 ) = (last
.token_tree(), next
.token_tree())
919 // Note: We always assume the semi-colon would be the last token in
920 // other parts of RA such that we don't add whitespace here.
922 // When `next` is a `Punct` of `'`, that's a part of a lifetime identifier so we don't
923 // need to add whitespace either.
924 if curr
.spacing
== tt
::Spacing
::Alone
&& curr
.char != '
;'
&& next
.char != '
\''
{
925 self.inner
.token(WHITESPACE
, " ");
926 self.text_pos
+= TextSize
::of(' '
);
931 fn start_node(&mut self, kind
: SyntaxKind
) {
932 self.inner
.start_node(kind
);
935 fn finish_node(&mut self) {
936 self.inner
.finish_node();
939 fn error(&mut self, error
: String
) {
940 self.inner
.error(error
, self.text_pos
)