src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs

   1 //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].
   2
   3 use rustc_hash::FxHashMap;
   4 use stdx::{always, non_empty_vec::NonEmptyVec};
   5 use syntax::{
   6     ast::{self, make::tokens::doc_comment},
   7     AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind,
   8     SyntaxKind::*,
   9     SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
  10 };
  11 use tt::buffer::{Cursor, TokenBuffer};
  12
  13 use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
  14
  15 #[cfg(test)]
  16 mod tests;
  17
  18 /// Convert the syntax node to a `TokenTree` (what macro
  19 /// will consume).
  20 pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) {
  21     let (subtree, token_map, _) = syntax_node_to_token_tree_with_modifications(
  22         node,
  23         Default::default(),
  24         0,
  25         Default::default(),
  26         Default::default(),
  27     );
  28     (subtree, token_map)
  29 }
  30
  31 /// Convert the syntax node to a `TokenTree` (what macro will consume)
  32 /// with the censored range excluded.
  33 pub fn syntax_node_to_token_tree_with_modifications(
  34     node: &SyntaxNode,
  35     existing_token_map: TokenMap,
  36     next_id: u32,
  37     replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
  38     append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
  39 ) -> (tt::Subtree, TokenMap, u32) {
  40     let global_offset = node.text_range().start();
  41     let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append);
  42     let subtree = convert_tokens(&mut c);
  43     c.id_alloc.map.shrink_to_fit();
  44     always!(c.replace.is_empty(), "replace: {:?}", c.replace);
  45     always!(c.append.is_empty(), "append: {:?}", c.append);
  46     (subtree, c.id_alloc.map, c.id_alloc.next_id)
  47 }
  48
  49 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
  50 pub struct SyntheticTokenId(pub u32);
  51
  52 #[derive(Debug, Clone)]
  53 pub struct SyntheticToken {
  54     pub kind: SyntaxKind,
  55     pub text: SmolStr,
  56     pub range: TextRange,
  57     pub id: SyntheticTokenId,
  58 }
  59
  60 // The following items are what `rustc` macro can be parsed into :
  61 // link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141
  62 // * Expr(P<ast::Expr>)                     -> token_tree_to_expr
  63 // * Pat(P<ast::Pat>)                       -> token_tree_to_pat
  64 // * Ty(P<ast::Ty>)                         -> token_tree_to_ty
  65 // * Stmts(SmallVec<[ast::Stmt; 1]>)        -> token_tree_to_stmts
  66 // * Items(SmallVec<[P<ast::Item>; 1]>)     -> token_tree_to_items
  67 //
  68 // * TraitItems(SmallVec<[ast::TraitItem; 1]>)
  69 // * AssocItems(SmallVec<[ast::AssocItem; 1]>)
  70 // * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
  71
  72 pub fn token_tree_to_syntax_node(
  73     tt: &tt::Subtree,
  74     entry_point: parser::TopEntryPoint,
  75 ) -> (Parse<SyntaxNode>, TokenMap) {
  76     let buffer = match tt {
  77         tt::Subtree { delimiter: None, token_trees } => {
  78             TokenBuffer::from_tokens(token_trees.as_slice())
  79         }
  80         _ => TokenBuffer::from_subtree(tt),
  81     };
  82     let parser_input = to_parser_input(&buffer);
  83     let parser_output = entry_point.parse(&parser_input);
  84     let mut tree_sink = TtTreeSink::new(buffer.begin());
  85     for event in parser_output.iter() {
  86         match event {
  87             parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
  88                 tree_sink.token(kind, n_raw_tokens)
  89             }
  90             parser::Step::Enter { kind } => tree_sink.start_node(kind),
  91             parser::Step::Exit => tree_sink.finish_node(),
  92             parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
  93         }
  94     }
  95     let (parse, range_map) = tree_sink.finish();
  96     (parse, range_map)
  97 }
  98
  99 /// Convert a string to a `TokenTree`
 100 pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
 101     let lexed = parser::LexedStr::new(text);
 102     if lexed.errors().next().is_some() {
 103         return None;
 104     }
 105
 106     let mut conv = RawConverter {
 107         lexed,
 108         pos: 0,
 109         id_alloc: TokenIdAlloc {
 110             map: Default::default(),
 111             global_offset: TextSize::default(),
 112             next_id: 0,
 113         },
 114     };
 115
 116     let subtree = convert_tokens(&mut conv);
 117     Some((subtree, conv.id_alloc.map))
 118 }
 119
 120 /// Split token tree with separate expr: $($e:expr)SEP*
 121 pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
 122     if tt.token_trees.is_empty() {
 123         return Vec::new();
 124     }
 125
 126     let mut iter = TtIter::new(tt);
 127     let mut res = Vec::new();
 128
 129     while iter.peek_n(0).is_some() {
 130         let expanded = iter.expect_fragment(parser::PrefixEntryPoint::Expr);
 131
 132         res.push(match expanded.value {
 133             None => break,
 134             Some(tt @ tt::TokenTree::Leaf(_)) => {
 135                 tt::Subtree { delimiter: None, token_trees: vec![tt] }
 136             }
 137             Some(tt::TokenTree::Subtree(tt)) => tt,
 138         });
 139
 140         let mut fork = iter.clone();
 141         if fork.expect_char(sep).is_err() {
 142             break;
 143         }
 144         iter = fork;
 145     }
 146
 147     if iter.peek_n(0).is_some() {
 148         res.push(tt::Subtree { delimiter: None, token_trees: iter.into_iter().cloned().collect() });
 149     }
 150
 151     res
 152 }
 153
 154 fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree {
 155     struct StackEntry {
 156         subtree: tt::Subtree,
 157         idx: usize,
 158         open_range: TextRange,
 159     }
 160
 161     let entry = StackEntry {
 162         subtree: tt::Subtree { delimiter: None, ..Default::default() },
 163         // never used (delimiter is `None`)
 164         idx: !0,
 165         open_range: TextRange::empty(TextSize::of('.')),
 166     };
 167     let mut stack = NonEmptyVec::new(entry);
 168
 169     loop {
 170         let StackEntry { subtree, .. } = stack.last_mut();
 171         let result = &mut subtree.token_trees;
 172         let (token, range) = match conv.bump() {
 173             Some(it) => it,
 174             None => break,
 175         };
 176         let synth_id = token.synthetic_id(conv);
 177
 178         let kind = token.kind(conv);
 179         if kind == COMMENT {
 180             if let Some(tokens) = conv.convert_doc_comment(&token) {
 181                 // FIXME: There has to be a better way to do this
 182                 // Add the comments token id to the converted doc string
 183                 let id = conv.id_alloc().alloc(range, synth_id);
 184                 result.extend(tokens.into_iter().map(|mut tt| {
 185                     if let tt::TokenTree::Subtree(sub) = &mut tt {
 186                         if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) =
 187                             sub.token_trees.get_mut(2)
 188                         {
 189                             lit.id = id
 190                         }
 191                     }
 192                     tt
 193                 }));
 194             }
 195             continue;
 196         }
 197         let tt = if kind.is_punct() && kind != UNDERSCORE {
 198             if synth_id.is_none() {
 199                 assert_eq!(range.len(), TextSize::of('.'));
 200             }
 201
 202             if let Some(delim) = subtree.delimiter {
 203                 let expected = match delim.kind {
 204                     tt::DelimiterKind::Parenthesis => T![')'],
 205                     tt::DelimiterKind::Brace => T!['}'],
 206                     tt::DelimiterKind::Bracket => T![']'],
 207                 };
 208
 209                 if kind == expected {
 210                     if let Some(entry) = stack.pop() {
 211                         conv.id_alloc().close_delim(entry.idx, Some(range));
 212                         stack.last_mut().subtree.token_trees.push(entry.subtree.into());
 213                     }
 214                     continue;
 215                 }
 216             }
 217
 218             let delim = match kind {
 219                 T!['('] => Some(tt::DelimiterKind::Parenthesis),
 220                 T!['{'] => Some(tt::DelimiterKind::Brace),
 221                 T!['['] => Some(tt::DelimiterKind::Bracket),
 222                 _ => None,
 223             };
 224
 225             if let Some(kind) = delim {
 226                 let mut subtree = tt::Subtree::default();
 227                 let (id, idx) = conv.id_alloc().open_delim(range, synth_id);
 228                 subtree.delimiter = Some(tt::Delimiter { id, kind });
 229                 stack.push(StackEntry { subtree, idx, open_range: range });
 230                 continue;
 231             }
 232
 233             let spacing = match conv.peek().map(|next| next.kind(conv)) {
 234                 Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint,
 235                 _ => tt::Spacing::Alone,
 236             };
 237             let char = match token.to_char(conv) {
 238                 Some(c) => c,
 239                 None => {
 240                     panic!("Token from lexer must be single char: token = {:#?}", token);
 241                 }
 242             };
 243             tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) })
 244                 .into()
 245         } else {
 246             macro_rules! make_leaf {
 247                 ($i:ident) => {
 248                     tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) }
 249                         .into()
 250                 };
 251             }
 252             let leaf: tt::Leaf = match kind {
 253                 T![true] | T![false] => make_leaf!(Ident),
 254                 IDENT => make_leaf!(Ident),
 255                 UNDERSCORE => make_leaf!(Ident),
 256                 k if k.is_keyword() => make_leaf!(Ident),
 257                 k if k.is_literal() => make_leaf!(Literal),
 258                 LIFETIME_IDENT => {
 259                     let char_unit = TextSize::of('\'');
 260                     let r = TextRange::at(range.start(), char_unit);
 261                     let apostrophe = tt::Leaf::from(tt::Punct {
 262                         char: '\'',
 263                         spacing: tt::Spacing::Joint,
 264                         id: conv.id_alloc().alloc(r, synth_id),
 265                     });
 266                     result.push(apostrophe.into());
 267
 268                     let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
 269                     let ident = tt::Leaf::from(tt::Ident {
 270                         text: SmolStr::new(&token.to_text(conv)[1..]),
 271                         id: conv.id_alloc().alloc(r, synth_id),
 272                     });
 273                     result.push(ident.into());
 274                     continue;
 275                 }
 276                 _ => continue,
 277             };
 278
 279             leaf.into()
 280         };
 281         result.push(tt);
 282     }
 283
 284     // If we get here, we've consumed all input tokens.
 285     // We might have more than one subtree in the stack, if the delimiters are improperly balanced.
 286     // Merge them so we're left with one.
 287     while let Some(entry) = stack.pop() {
 288         let parent = stack.last_mut();
 289
 290         conv.id_alloc().close_delim(entry.idx, None);
 291         let leaf: tt::Leaf = tt::Punct {
 292             id: conv.id_alloc().alloc(entry.open_range, None),
 293             char: match entry.subtree.delimiter.unwrap().kind {
 294                 tt::DelimiterKind::Parenthesis => '(',
 295                 tt::DelimiterKind::Brace => '{',
 296                 tt::DelimiterKind::Bracket => '[',
 297             },
 298             spacing: tt::Spacing::Alone,
 299         }
 300         .into();
 301         parent.subtree.token_trees.push(leaf.into());
 302         parent.subtree.token_trees.extend(entry.subtree.token_trees);
 303     }
 304
 305     let subtree = stack.into_last().subtree;
 306     if let [tt::TokenTree::Subtree(first)] = &*subtree.token_trees {
 307         first.clone()
 308     } else {
 309         subtree
 310     }
 311 }
 312
 313 fn is_single_token_op(kind: SyntaxKind) -> bool {
 314     matches!(
 315         kind,
 316         EQ | L_ANGLE
 317             | R_ANGLE
 318             | BANG
 319             | AMP
 320             | PIPE
 321             | TILDE
 322             | AT
 323             | DOT
 324             | COMMA
 325             | SEMICOLON
 326             | COLON
 327             | POUND
 328             | DOLLAR
 329             | QUESTION
 330             | PLUS
 331             | MINUS
 332             | STAR
 333             | SLASH
 334             | PERCENT
 335             | CARET
 336             // LIFETIME_IDENT will be split into a sequence of `'` (a single quote) and an
 337             // identifier.
 338             | LIFETIME_IDENT
 339     )
 340 }
 341
 342 /// Returns the textual content of a doc comment block as a quoted string
 343 /// That is, strips leading `///` (or `/**`, etc)
 344 /// and strips the ending `*/`
 345 /// And then quote the string, which is needed to convert to `tt::Literal`
 346 fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
 347     let prefix_len = comment.prefix().len();
 348     let mut text = &comment.text()[prefix_len..];
 349
 350     // Remove ending "*/"
 351     if comment.kind().shape == ast::CommentShape::Block {
 352         text = &text[0..text.len() - 2];
 353     }
 354
 355     // Quote the string
 356     // Note that `tt::Literal` expect an escaped string
 357     let text = format!("\"{}\"", text.escape_debug());
 358     text.into()
 359 }
 360
 361 fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>> {
 362     cov_mark::hit!(test_meta_doc_comments);
 363     let comment = ast::Comment::cast(token.clone())?;
 364     let doc = comment.kind().doc?;
 365
 366     // Make `doc="\" Comments\""
 367     let meta_tkns = vec![mk_ident("doc"), mk_punct('='), mk_doc_literal(&comment)];
 368
 369     // Make `#![]`
 370     let mut token_trees = Vec::with_capacity(3);
 371     token_trees.push(mk_punct('#'));
 372     if let ast::CommentPlacement::Inner = doc {
 373         token_trees.push(mk_punct('!'));
 374     }
 375     token_trees.push(tt::TokenTree::from(tt::Subtree {
 376         delimiter: Some(tt::Delimiter {
 377             kind: tt::DelimiterKind::Bracket,
 378             id: tt::TokenId::unspecified(),
 379         }),
 380         token_trees: meta_tkns,
 381     }));
 382
 383     return Some(token_trees);
 384
 385     // Helper functions
 386     fn mk_ident(s: &str) -> tt::TokenTree {
 387         tt::TokenTree::from(tt::Leaf::from(tt::Ident {
 388             text: s.into(),
 389             id: tt::TokenId::unspecified(),
 390         }))
 391     }
 392
 393     fn mk_punct(c: char) -> tt::TokenTree {
 394         tt::TokenTree::from(tt::Leaf::from(tt::Punct {
 395             char: c,
 396             spacing: tt::Spacing::Alone,
 397             id: tt::TokenId::unspecified(),
 398         }))
 399     }
 400
 401     fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
 402         let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
 403
 404         tt::TokenTree::from(tt::Leaf::from(lit))
 405     }
 406 }
 407
 408 struct TokenIdAlloc {
 409     map: TokenMap,
 410     global_offset: TextSize,
 411     next_id: u32,
 412 }
 413
 414 impl TokenIdAlloc {
 415     fn alloc(
 416         &mut self,
 417         absolute_range: TextRange,
 418         synthetic_id: Option<SyntheticTokenId>,
 419     ) -> tt::TokenId {
 420         let relative_range = absolute_range - self.global_offset;
 421         let token_id = tt::TokenId(self.next_id);
 422         self.next_id += 1;
 423         self.map.insert(token_id, relative_range);
 424         if let Some(id) = synthetic_id {
 425             self.map.insert_synthetic(token_id, id);
 426         }
 427         token_id
 428     }
 429
 430     fn open_delim(
 431         &mut self,
 432         open_abs_range: TextRange,
 433         synthetic_id: Option<SyntheticTokenId>,
 434     ) -> (tt::TokenId, usize) {
 435         let token_id = tt::TokenId(self.next_id);
 436         self.next_id += 1;
 437         let idx = self.map.insert_delim(
 438             token_id,
 439             open_abs_range - self.global_offset,
 440             open_abs_range - self.global_offset,
 441         );
 442         if let Some(id) = synthetic_id {
 443             self.map.insert_synthetic(token_id, id);
 444         }
 445         (token_id, idx)
 446     }
 447
 448     fn close_delim(&mut self, idx: usize, close_abs_range: Option<TextRange>) {
 449         match close_abs_range {
 450             None => {
 451                 self.map.remove_delim(idx);
 452             }
 453             Some(close) => {
 454                 self.map.update_close_delim(idx, close - self.global_offset);
 455             }
 456         }
 457     }
 458 }
 459
 460 /// A raw token (straight from lexer) converter
 461 struct RawConverter<'a> {
 462     lexed: parser::LexedStr<'a>,
 463     pos: usize,
 464     id_alloc: TokenIdAlloc,
 465 }
 466
 467 trait SrcToken<Ctx>: std::fmt::Debug {
 468     fn kind(&self, ctx: &Ctx) -> SyntaxKind;
 469
 470     fn to_char(&self, ctx: &Ctx) -> Option<char>;
 471
 472     fn to_text(&self, ctx: &Ctx) -> SmolStr;
 473
 474     fn synthetic_id(&self, ctx: &Ctx) -> Option<SyntheticTokenId>;
 475 }
 476
 477 trait TokenConverter: Sized {
 478     type Token: SrcToken<Self>;
 479
 480     fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>;
 481
 482     fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
 483
 484     fn peek(&self) -> Option<Self::Token>;
 485
 486     fn id_alloc(&mut self) -> &mut TokenIdAlloc;
 487 }
 488
 489 impl<'a> SrcToken<RawConverter<'a>> for usize {
 490     fn kind(&self, ctx: &RawConverter<'a>) -> SyntaxKind {
 491         ctx.lexed.kind(*self)
 492     }
 493
 494     fn to_char(&self, ctx: &RawConverter<'a>) -> Option<char> {
 495         ctx.lexed.text(*self).chars().next()
 496     }
 497
 498     fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr {
 499         ctx.lexed.text(*self).into()
 500     }
 501
 502     fn synthetic_id(&self, _ctx: &RawConverter<'a>) -> Option<SyntheticTokenId> {
 503         None
 504     }
 505 }
 506
 507 impl<'a> TokenConverter for RawConverter<'a> {
 508     type Token = usize;
 509
 510     fn convert_doc_comment(&self, &token: &usize) -> Option<Vec<tt::TokenTree>> {
 511         let text = self.lexed.text(token);
 512         convert_doc_comment(&doc_comment(text))
 513     }
 514
 515     fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
 516         if self.pos == self.lexed.len() {
 517             return None;
 518         }
 519         let token = self.pos;
 520         self.pos += 1;
 521         let range = self.lexed.text_range(token);
 522         let range = TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap());
 523
 524         Some((token, range))
 525     }
 526
 527     fn peek(&self) -> Option<Self::Token> {
 528         if self.pos == self.lexed.len() {
 529             return None;
 530         }
 531         Some(self.pos)
 532     }
 533
 534     fn id_alloc(&mut self) -> &mut TokenIdAlloc {
 535         &mut self.id_alloc
 536     }
 537 }
 538
 539 struct Converter {
 540     id_alloc: TokenIdAlloc,
 541     current: Option<SyntaxToken>,
 542     current_synthetic: Vec<SyntheticToken>,
 543     preorder: PreorderWithTokens,
 544     replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
 545     append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
 546     range: TextRange,
 547     punct_offset: Option<(SyntaxToken, TextSize)>,
 548 }
 549
 550 impl Converter {
 551     fn new(
 552         node: &SyntaxNode,
 553         global_offset: TextSize,
 554         existing_token_map: TokenMap,
 555         next_id: u32,
 556         mut replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
 557         mut append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
 558     ) -> Converter {
 559         let range = node.text_range();
 560         let mut preorder = node.preorder_with_tokens();
 561         let (first, synthetic) = Self::next_token(&mut preorder, &mut replace, &mut append);
 562         Converter {
 563             id_alloc: { TokenIdAlloc { map: existing_token_map, global_offset, next_id } },
 564             current: first,
 565             current_synthetic: synthetic,
 566             preorder,
 567             range,
 568             replace,
 569             append,
 570             punct_offset: None,
 571         }
 572     }
 573
 574     fn next_token(
 575         preorder: &mut PreorderWithTokens,
 576         replace: &mut FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
 577         append: &mut FxHashMap<SyntaxElement, Vec<SyntheticToken>>,
 578     ) -> (Option<SyntaxToken>, Vec<SyntheticToken>) {
 579         while let Some(ev) = preorder.next() {
 580             let ele = match ev {
 581                 WalkEvent::Enter(ele) => ele,
 582                 WalkEvent::Leave(ele) => {
 583                     if let Some(mut v) = append.remove(&ele) {
 584                         if !v.is_empty() {
 585                             v.reverse();
 586                             return (None, v);
 587                         }
 588                     }
 589                     continue;
 590                 }
 591             };
 592             if let Some(mut v) = replace.remove(&ele) {
 593                 preorder.skip_subtree();
 594                 if !v.is_empty() {
 595                     v.reverse();
 596                     return (None, v);
 597                 }
 598             }
 599             match ele {
 600                 SyntaxElement::Token(t) => return (Some(t), Vec::new()),
 601                 _ => {}
 602             }
 603         }
 604         (None, Vec::new())
 605     }
 606 }
 607
 608 #[derive(Debug)]
 609 enum SynToken {
 610     Ordinary(SyntaxToken),
 611     // FIXME is this supposed to be `Punct`?
 612     Punch(SyntaxToken, TextSize),
 613     Synthetic(SyntheticToken),
 614 }
 615
 616 impl SynToken {
 617     fn token(&self) -> Option<&SyntaxToken> {
 618         match self {
 619             SynToken::Ordinary(it) | SynToken::Punch(it, _) => Some(it),
 620             SynToken::Synthetic(_) => None,
 621         }
 622     }
 623 }
 624
 625 impl SrcToken<Converter> for SynToken {
 626     fn kind(&self, ctx: &Converter) -> SyntaxKind {
 627         match self {
 628             SynToken::Ordinary(token) => token.kind(),
 629             SynToken::Punch(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(),
 630             SynToken::Synthetic(token) => token.kind,
 631         }
 632     }
 633     fn to_char(&self, _ctx: &Converter) -> Option<char> {
 634         match self {
 635             SynToken::Ordinary(_) => None,
 636             SynToken::Punch(it, i) => it.text().chars().nth((*i).into()),
 637             SynToken::Synthetic(token) if token.text.len() == 1 => token.text.chars().next(),
 638             SynToken::Synthetic(_) => None,
 639         }
 640     }
 641     fn to_text(&self, _ctx: &Converter) -> SmolStr {
 642         match self {
 643             SynToken::Ordinary(token) => token.text().into(),
 644             SynToken::Punch(token, _) => token.text().into(),
 645             SynToken::Synthetic(token) => token.text.clone(),
 646         }
 647     }
 648
 649     fn synthetic_id(&self, _ctx: &Converter) -> Option<SyntheticTokenId> {
 650         match self {
 651             SynToken::Synthetic(token) => Some(token.id),
 652             _ => None,
 653         }
 654     }
 655 }
 656
 657 impl TokenConverter for Converter {
 658     type Token = SynToken;
 659     fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
 660         convert_doc_comment(token.token()?)
 661     }
 662
 663     fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
 664         if let Some((punct, offset)) = self.punct_offset.clone() {
 665             if usize::from(offset) + 1 < punct.text().len() {
 666                 let offset = offset + TextSize::of('.');
 667                 let range = punct.text_range();
 668                 self.punct_offset = Some((punct.clone(), offset));
 669                 let range = TextRange::at(range.start() + offset, TextSize::of('.'));
 670                 return Some((SynToken::Punch(punct, offset), range));
 671             }
 672         }
 673
 674         if let Some(synth_token) = self.current_synthetic.pop() {
 675             if self.current_synthetic.is_empty() {
 676                 let (new_current, new_synth) =
 677                     Self::next_token(&mut self.preorder, &mut self.replace, &mut self.append);
 678                 self.current = new_current;
 679                 self.current_synthetic = new_synth;
 680             }
 681             let range = synth_token.range;
 682             return Some((SynToken::Synthetic(synth_token), range));
 683         }
 684
 685         let curr = self.current.clone()?;
 686         if !self.range.contains_range(curr.text_range()) {
 687             return None;
 688         }
 689         let (new_current, new_synth) =
 690             Self::next_token(&mut self.preorder, &mut self.replace, &mut self.append);
 691         self.current = new_current;
 692         self.current_synthetic = new_synth;
 693         let token = if curr.kind().is_punct() {
 694             self.punct_offset = Some((curr.clone(), 0.into()));
 695             let range = curr.text_range();
 696             let range = TextRange::at(range.start(), TextSize::of('.'));
 697             (SynToken::Punch(curr, 0.into()), range)
 698         } else {
 699             self.punct_offset = None;
 700             let range = curr.text_range();
 701             (SynToken::Ordinary(curr), range)
 702         };
 703
 704         Some(token)
 705     }
 706
 707     fn peek(&self) -> Option<Self::Token> {
 708         if let Some((punct, mut offset)) = self.punct_offset.clone() {
 709             offset += TextSize::of('.');
 710             if usize::from(offset) < punct.text().len() {
 711                 return Some(SynToken::Punch(punct, offset));
 712             }
 713         }
 714
 715         if let Some(synth_token) = self.current_synthetic.last() {
 716             return Some(SynToken::Synthetic(synth_token.clone()));
 717         }
 718
 719         let curr = self.current.clone()?;
 720         if !self.range.contains_range(curr.text_range()) {
 721             return None;
 722         }
 723
 724         let token = if curr.kind().is_punct() {
 725             SynToken::Punch(curr, 0.into())
 726         } else {
 727             SynToken::Ordinary(curr)
 728         };
 729         Some(token)
 730     }
 731
 732     fn id_alloc(&mut self) -> &mut TokenIdAlloc {
 733         &mut self.id_alloc
 734     }
 735 }
 736
 737 struct TtTreeSink<'a> {
 738     buf: String,
 739     cursor: Cursor<'a>,
 740     open_delims: FxHashMap<tt::TokenId, TextSize>,
 741     text_pos: TextSize,
 742     inner: SyntaxTreeBuilder,
 743     token_map: TokenMap,
 744 }
 745
 746 impl<'a> TtTreeSink<'a> {
 747     fn new(cursor: Cursor<'a>) -> Self {
 748         TtTreeSink {
 749             buf: String::new(),
 750             cursor,
 751             open_delims: FxHashMap::default(),
 752             text_pos: 0.into(),
 753             inner: SyntaxTreeBuilder::default(),
 754             token_map: TokenMap::default(),
 755         }
 756     }
 757
 758     fn finish(mut self) -> (Parse<SyntaxNode>, TokenMap) {
 759         self.token_map.shrink_to_fit();
 760         (self.inner.finish(), self.token_map)
 761     }
 762 }
 763
 764 fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
 765     let texts = match d {
 766         tt::DelimiterKind::Parenthesis => "()",
 767         tt::DelimiterKind::Brace => "{}",
 768         tt::DelimiterKind::Bracket => "[]",
 769     };
 770
 771     let idx = closing as usize;
 772     &texts[idx..texts.len() - (1 - idx)]
 773 }
 774
 775 impl<'a> TtTreeSink<'a> {
 776     fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
 777         if kind == LIFETIME_IDENT {
 778             n_tokens = 2;
 779         }
 780
 781         let mut last = self.cursor;
 782         for _ in 0..n_tokens {
 783             let tmp: u8;
 784             if self.cursor.eof() {
 785                 break;
 786             }
 787             last = self.cursor;
 788             let text: &str = loop {
 789                 break match self.cursor.token_tree() {
 790                     Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
 791                         // Mark the range if needed
 792                         let (text, id) = match leaf {
 793                             tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id),
 794                             tt::Leaf::Punct(punct) => {
 795                                 assert!(punct.char.is_ascii());
 796                                 tmp = punct.char as u8;
 797                                 (std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id)
 798                             }
 799                             tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id),
 800                         };
 801                         let range = TextRange::at(self.text_pos, TextSize::of(text));
 802                         self.token_map.insert(id, range);
 803                         self.cursor = self.cursor.bump();
 804                         text
 805                     }
 806                     Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
 807                         self.cursor = self.cursor.subtree().unwrap();
 808                         match subtree.delimiter {
 809                             Some(d) => {
 810                                 self.open_delims.insert(d.id, self.text_pos);
 811                                 delim_to_str(d.kind, false)
 812                             }
 813                             None => continue,
 814                         }
 815                     }
 816                     None => {
 817                         let parent = self.cursor.end().unwrap();
 818                         self.cursor = self.cursor.bump();
 819                         match parent.delimiter {
 820                             Some(d) => {
 821                                 if let Some(open_delim) = self.open_delims.get(&d.id) {
 822                                     let open_range = TextRange::at(*open_delim, TextSize::of('('));
 823                                     let close_range =
 824                                         TextRange::at(self.text_pos, TextSize::of('('));
 825                                     self.token_map.insert_delim(d.id, open_range, close_range);
 826                                 }
 827                                 delim_to_str(d.kind, true)
 828                             }
 829                             None => continue,
 830                         }
 831                     }
 832                 };
 833             };
 834             self.buf += text;
 835             self.text_pos += TextSize::of(text);
 836         }
 837
 838         self.inner.token(kind, self.buf.as_str());
 839         self.buf.clear();
 840         // Add whitespace between adjoint puncts
 841         let next = last.bump();
 842         if let (
 843             Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(curr), _)),
 844             Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(next), _)),
 845         ) = (last.token_tree(), next.token_tree())
 846         {
 847             // Note: We always assume the semi-colon would be the last token in
 848             // other parts of RA such that we don't add whitespace here.
 849             //
 850             // When `next` is a `Punct` of `'`, that's a part of a lifetime identifier so we don't
 851             // need to add whitespace either.
 852             if curr.spacing == tt::Spacing::Alone && curr.char != ';' && next.char != '\'' {
 853                 self.inner.token(WHITESPACE, " ");
 854                 self.text_pos += TextSize::of(' ');
 855             }
 856         }
 857     }
 858
 859     fn start_node(&mut self, kind: SyntaxKind) {
 860         self.inner.start_node(kind);
 861     }
 862
 863     fn finish_node(&mut self) {
 864         self.inner.finish_node();
 865     }
 866
 867     fn error(&mut self, error: String) {
 868         self.inner.error(error, self.text_pos)
 869     }
 870 }