src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs

   1 //! To make attribute macros work reliably when typing, we need to take care to
   2 //! fix up syntax errors in the code we're passing to them.
   3
   4 use base_db::{
   5     span::{ErasedFileAstId, SpanAnchor, SpanData},
   6     FileId,
   7 };
   8 use la_arena::RawIdx;
   9 use rustc_hash::{FxHashMap, FxHashSet};
  10 use smallvec::SmallVec;
  11 use stdx::never;
  12 use syntax::{
  13     ast::{self, AstNode, HasLoopBody},
  14     match_ast, SyntaxElement, SyntaxKind, SyntaxNode, TextRange, TextSize,
  15 };
  16 use triomphe::Arc;
  17 use tt::{Spacing, Span};
  18
  19 use crate::{
  20     span::SpanMapRef,
  21     tt::{Ident, Leaf, Punct, Subtree},
  22 };
  23
  24 /// The result of calculating fixes for a syntax node -- a bunch of changes
  25 /// (appending to and replacing nodes), the information that is needed to
  26 /// reverse those changes afterwards, and a token map.
  27 #[derive(Debug, Default)]
  28 pub(crate) struct SyntaxFixups {
  29     pub(crate) append: FxHashMap<SyntaxElement, Vec<Leaf>>,
  30     pub(crate) remove: FxHashSet<SyntaxNode>,
  31     pub(crate) undo_info: SyntaxFixupUndoInfo,
  32 }
  33
  34 /// This is the information needed to reverse the fixups.
  35 #[derive(Clone, Debug, Default, PartialEq, Eq)]
  36 pub struct SyntaxFixupUndoInfo {
  37     // FIXME: ThinArc<[Subtree]>
  38     original: Option<Arc<Box<[Subtree]>>>,
  39 }
  40
  41 impl SyntaxFixupUndoInfo {
  42     pub(crate) const NONE: Self = SyntaxFixupUndoInfo { original: None };
  43 }
  44
  45 // censoring -> just don't convert the node
  46 // replacement -> censor + append
  47 // append -> insert a fake node, here we need to assemble some dummy span that we can figure out how
  48 // to remove later
  49 const FIXUP_DUMMY_FILE: FileId = FileId::from_raw(FileId::MAX_FILE_ID);
  50 const FIXUP_DUMMY_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(RawIdx::from_u32(!0));
  51 const FIXUP_DUMMY_RANGE: TextRange = TextRange::empty(TextSize::new(0));
  52 const FIXUP_DUMMY_RANGE_END: TextSize = TextSize::new(!0);
  53
  54 pub(crate) fn fixup_syntax(span_map: SpanMapRef<'_>, node: &SyntaxNode) -> SyntaxFixups {
  55     let mut append = FxHashMap::<SyntaxElement, _>::default();
  56     let mut remove = FxHashSet::<SyntaxNode>::default();
  57     let mut preorder = node.preorder();
  58     let mut original = Vec::new();
  59     let dummy_range = FIXUP_DUMMY_RANGE;
  60     // we use a file id of `FileId(!0)` to signal a fake node, and the text range's start offset as
  61     // the index into the replacement vec but only if the end points to !0
  62     let dummy_anchor = SpanAnchor { file_id: FIXUP_DUMMY_FILE, ast_id: FIXUP_DUMMY_AST_ID };
  63     let fake_span = |range| SpanData {
  64         range: dummy_range,
  65         anchor: dummy_anchor,
  66         ctx: span_map.span_for_range(range).ctx,
  67     };
  68     while let Some(event) = preorder.next() {
  69         let syntax::WalkEvent::Enter(node) = event else { continue };
  70
  71         let node_range = node.text_range();
  72         if can_handle_error(&node) && has_error_to_handle(&node) {
  73             remove.insert(node.clone().into());
  74             // the node contains an error node, we have to completely replace it by something valid
  75             let original_tree = mbe::syntax_node_to_token_tree(&node, span_map);
  76             let idx = original.len() as u32;
  77             original.push(original_tree);
  78             let replacement = Leaf::Ident(Ident {
  79                 text: "__ra_fixup".into(),
  80                 span: SpanData {
  81                     range: TextRange::new(TextSize::new(idx), FIXUP_DUMMY_RANGE_END),
  82                     anchor: dummy_anchor,
  83                     ctx: span_map.span_for_range(node_range).ctx,
  84                 },
  85             });
  86             append.insert(node.clone().into(), vec![replacement]);
  87             preorder.skip_subtree();
  88             continue;
  89         }
  90
  91         // In some other situations, we can fix things by just appending some tokens.
  92         match_ast! {
  93             match node {
  94                 ast::FieldExpr(it) => {
  95                     if it.name_ref().is_none() {
  96                         // incomplete field access: some_expr.|
  97                         append.insert(node.clone().into(), vec![
  98                             Leaf::Ident(Ident {
  99                                 text: "__ra_fixup".into(),
 100                                 span: fake_span(node_range),
 101                             }),
 102                         ]);
 103                     }
 104                 },
 105                 ast::ExprStmt(it) => {
 106                     if it.semicolon_token().is_none() {
 107                         append.insert(node.clone().into(), vec![
 108                             Leaf::Punct(Punct {
 109                                 char: ';',
 110                                 spacing: Spacing::Alone,
 111                                 span: fake_span(node_range),
 112                             }),
 113                         ]);
 114                     }
 115                 },
 116                 ast::LetStmt(it) => {
 117                     if it.semicolon_token().is_none() {
 118                         append.insert(node.clone().into(), vec![
 119                             Leaf::Punct(Punct {
 120                                 char: ';',
 121                                 spacing: Spacing::Alone,
 122                                 span: fake_span(node_range)
 123                             }),
 124                         ]);
 125                     }
 126                 },
 127                 ast::IfExpr(it) => {
 128                     if it.condition().is_none() {
 129                         // insert placeholder token after the if token
 130                         let if_token = match it.if_token() {
 131                             Some(t) => t,
 132                             None => continue,
 133                         };
 134                         append.insert(if_token.into(), vec![
 135                             Leaf::Ident(Ident {
 136                                 text: "__ra_fixup".into(),
 137                                 span: fake_span(node_range)
 138                             }),
 139                         ]);
 140                     }
 141                     if it.then_branch().is_none() {
 142                         append.insert(node.clone().into(), vec![
 143                             // FIXME: THis should be a subtree no?
 144                             Leaf::Punct(Punct {
 145                                 char: '{',
 146                                 spacing: Spacing::Alone,
 147                                 span: fake_span(node_range)
 148                             }),
 149                             Leaf::Punct(Punct {
 150                                 char: '}',
 151                                 spacing: Spacing::Alone,
 152                                 span: fake_span(node_range)
 153                             }),
 154                         ]);
 155                     }
 156                 },
 157                 ast::WhileExpr(it) => {
 158                     if it.condition().is_none() {
 159                         // insert placeholder token after the while token
 160                         let while_token = match it.while_token() {
 161                             Some(t) => t,
 162                             None => continue,
 163                         };
 164                         append.insert(while_token.into(), vec![
 165                             Leaf::Ident(Ident {
 166                                 text: "__ra_fixup".into(),
 167                                 span: fake_span(node_range)
 168                             }),
 169                         ]);
 170                     }
 171                     if it.loop_body().is_none() {
 172                         append.insert(node.clone().into(), vec![
 173                             // FIXME: THis should be a subtree no?
 174                             Leaf::Punct(Punct {
 175                                 char: '{',
 176                                 spacing: Spacing::Alone,
 177                                 span: fake_span(node_range)
 178                             }),
 179                             Leaf::Punct(Punct {
 180                                 char: '}',
 181                                 spacing: Spacing::Alone,
 182                                 span: fake_span(node_range)
 183                             }),
 184                         ]);
 185                     }
 186                 },
 187                 ast::LoopExpr(it) => {
 188                     if it.loop_body().is_none() {
 189                         append.insert(node.clone().into(), vec![
 190                             // FIXME: THis should be a subtree no?
 191                             Leaf::Punct(Punct {
 192                                 char: '{',
 193                                 spacing: Spacing::Alone,
 194                                 span: fake_span(node_range)
 195                             }),
 196                             Leaf::Punct(Punct {
 197                                 char: '}',
 198                                 spacing: Spacing::Alone,
 199                                 span: fake_span(node_range)
 200                             }),
 201                         ]);
 202                     }
 203                 },
 204                 // FIXME: foo::
 205                 ast::MatchExpr(it) => {
 206                     if it.expr().is_none() {
 207                         let match_token = match it.match_token() {
 208                             Some(t) => t,
 209                             None => continue
 210                         };
 211                         append.insert(match_token.into(), vec![
 212                             Leaf::Ident(Ident {
 213                                 text: "__ra_fixup".into(),
 214                                 span: fake_span(node_range)
 215                             }),
 216                         ]);
 217                     }
 218                     if it.match_arm_list().is_none() {
 219                         // No match arms
 220                         append.insert(node.clone().into(), vec![
 221                             // FIXME: THis should be a subtree no?
 222                             Leaf::Punct(Punct {
 223                                 char: '{',
 224                                 spacing: Spacing::Alone,
 225                                 span: fake_span(node_range)
 226                             }),
 227                             Leaf::Punct(Punct {
 228                                 char: '}',
 229                                 spacing: Spacing::Alone,
 230                                 span: fake_span(node_range)
 231                             }),
 232                         ]);
 233                     }
 234                 },
 235                 ast::ForExpr(it) => {
 236                     let for_token = match it.for_token() {
 237                         Some(token) => token,
 238                         None => continue
 239                     };
 240
 241                     let [pat, in_token, iter] = [
 242                          "_",
 243                          "in",
 244                          "__ra_fixup"
 245                     ].map(|text|
 246                         Leaf::Ident(Ident {
 247                             text: text.into(),
 248                             span: fake_span(node_range)
 249                         }),
 250                     );
 251
 252                     if it.pat().is_none() && it.in_token().is_none() && it.iterable().is_none() {
 253                         append.insert(for_token.into(), vec![pat, in_token, iter]);
 254                     // does something funky -- see test case for_no_pat
 255                     } else if it.pat().is_none() {
 256                         append.insert(for_token.into(), vec![pat]);
 257                     }
 258
 259                     if it.loop_body().is_none() {
 260                         append.insert(node.clone().into(), vec![
 261                             // FIXME: THis should be a subtree no?
 262                             Leaf::Punct(Punct {
 263                                 char: '{',
 264                                 spacing: Spacing::Alone,
 265                                 span: fake_span(node_range)
 266                             }),
 267                             Leaf::Punct(Punct {
 268                                 char: '}',
 269                                 spacing: Spacing::Alone,
 270                                 span: fake_span(node_range)
 271                             }),
 272                         ]);
 273                     }
 274                 },
 275                 _ => (),
 276             }
 277         }
 278     }
 279     let needs_fixups = !append.is_empty() || !original.is_empty();
 280     SyntaxFixups {
 281         append,
 282         remove,
 283         undo_info: SyntaxFixupUndoInfo {
 284             original: needs_fixups.then(|| Arc::new(original.into_boxed_slice())),
 285         },
 286     }
 287 }
 288
 289 fn has_error(node: &SyntaxNode) -> bool {
 290     node.children().any(|c| c.kind() == SyntaxKind::ERROR)
 291 }
 292
 293 fn can_handle_error(node: &SyntaxNode) -> bool {
 294     ast::Expr::can_cast(node.kind())
 295 }
 296
 297 fn has_error_to_handle(node: &SyntaxNode) -> bool {
 298     has_error(node) || node.children().any(|c| !can_handle_error(&c) && has_error_to_handle(&c))
 299 }
 300
 301 pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo) {
 302     let Some(undo_info) = undo_info.original.as_deref() else { return };
 303     let undo_info = &**undo_info;
 304     if never!(
 305         tt.delimiter.close.anchor.file_id == FIXUP_DUMMY_FILE
 306             || tt.delimiter.open.anchor.file_id == FIXUP_DUMMY_FILE
 307     ) {
 308         tt.delimiter.close = SpanData::DUMMY;
 309         tt.delimiter.open = SpanData::DUMMY;
 310     }
 311     reverse_fixups_(tt, undo_info);
 312 }
 313
 314 fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
 315     let tts = std::mem::take(&mut tt.token_trees);
 316     tt.token_trees = tts
 317         .into_iter()
 318         // delete all fake nodes
 319         .filter(|tt| match tt {
 320             tt::TokenTree::Leaf(leaf) => {
 321                 let span = leaf.span();
 322                 let is_real_leaf = span.anchor.file_id != FIXUP_DUMMY_FILE;
 323                 let is_replaced_node = span.range.end() == FIXUP_DUMMY_RANGE_END;
 324                 is_real_leaf || is_replaced_node
 325             }
 326             tt::TokenTree::Subtree(_) => true,
 327         })
 328         .flat_map(|tt| match tt {
 329             tt::TokenTree::Subtree(mut tt) => {
 330                 if tt.delimiter.close.anchor.file_id == FIXUP_DUMMY_FILE
 331                     || tt.delimiter.open.anchor.file_id == FIXUP_DUMMY_FILE
 332                 {
 333                     // Even though fixup never creates subtrees with fixup spans, the old proc-macro server
 334                     // might copy them if the proc-macro asks for it, so we need to filter those out
 335                     // here as well.
 336                     return SmallVec::new_const();
 337                 }
 338                 reverse_fixups_(&mut tt, undo_info);
 339                 SmallVec::from_const([tt.into()])
 340             }
 341             tt::TokenTree::Leaf(leaf) => {
 342                 if leaf.span().anchor.file_id == FIXUP_DUMMY_FILE {
 343                     // we have a fake node here, we need to replace it again with the original
 344                     let original = undo_info[u32::from(leaf.span().range.start()) as usize].clone();
 345                     if original.delimiter.kind == tt::DelimiterKind::Invisible {
 346                         original.token_trees.into()
 347                     } else {
 348                         SmallVec::from_const([original.into()])
 349                     }
 350                 } else {
 351                     // just a normal leaf
 352                     SmallVec::from_const([leaf.into()])
 353                 }
 354             }
 355         })
 356         .collect();
 357 }
 358
 359 #[cfg(test)]
 360 mod tests {
 361     use base_db::FileId;
 362     use expect_test::{expect, Expect};
 363     use triomphe::Arc;
 364
 365     use crate::{
 366         fixup::reverse_fixups,
 367         span::{RealSpanMap, SpanMap},
 368         tt,
 369     };
 370
 371     // The following three functions are only meant to check partial structural equivalence of
 372     // `TokenTree`s, see the last assertion in `check()`.
 373     fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool {
 374         match (a, b) {
 375             (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text == b.text,
 376             (tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char,
 377             (tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.text == b.text,
 378             _ => false,
 379         }
 380     }
 381
 382     fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
 383         a.delimiter.kind == b.delimiter.kind
 384             && a.token_trees.len() == b.token_trees.len()
 385             && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
 386     }
 387
 388     fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool {
 389         match (a, b) {
 390             (tt::TokenTree::Leaf(a), tt::TokenTree::Leaf(b)) => check_leaf_eq(a, b),
 391             (tt::TokenTree::Subtree(a), tt::TokenTree::Subtree(b)) => check_subtree_eq(a, b),
 392             _ => false,
 393         }
 394     }
 395
 396     #[track_caller]
 397     fn check(ra_fixture: &str, mut expect: Expect) {
 398         let parsed = syntax::SourceFile::parse(ra_fixture);
 399         let span_map = SpanMap::RealSpanMap(Arc::new(RealSpanMap::absolute(FileId::from_raw(0))));
 400         let fixups = super::fixup_syntax(span_map.as_ref(), &parsed.syntax_node());
 401         let mut tt = mbe::syntax_node_to_token_tree_modified(
 402             &parsed.syntax_node(),
 403             span_map.as_ref(),
 404             fixups.append,
 405             fixups.remove,
 406         );
 407
 408         let actual = format!("{tt}\n");
 409
 410         expect.indent(false);
 411         expect.assert_eq(&actual);
 412
 413         // the fixed-up tree should be syntactically valid
 414         let (parse, _) = mbe::token_tree_to_syntax_node(&tt, ::mbe::TopEntryPoint::MacroItems);
 415         assert!(
 416             parse.errors().is_empty(),
 417             "parse has syntax errors. parse tree:\n{:#?}",
 418             parse.syntax_node()
 419         );
 420
 421         reverse_fixups(&mut tt, &fixups.undo_info);
 422
 423         // the fixed-up + reversed version should be equivalent to the original input
 424         // modulo token IDs and `Punct`s' spacing.
 425         let original_as_tt =
 426             mbe::syntax_node_to_token_tree(&parsed.syntax_node(), span_map.as_ref());
 427         assert!(
 428             check_subtree_eq(&tt, &original_as_tt),
 429             "different token tree:\n{tt:?}\n\n{original_as_tt:?}"
 430         );
 431     }
 432
 433     #[test]
 434     fn just_for_token() {
 435         check(
 436             r#"
 437 fn foo() {
 438     for
 439 }
 440 "#,
 441             expect![[r#"
 442 fn foo () {for _ in __ra_fixup { }}
 443 "#]],
 444         )
 445     }
 446
 447     #[test]
 448     fn for_no_iter_pattern() {
 449         check(
 450             r#"
 451 fn foo() {
 452     for {}
 453 }
 454 "#,
 455             expect![[r#"
 456 fn foo () {for _ in __ra_fixup {}}
 457 "#]],
 458         )
 459     }
 460
 461     #[test]
 462     fn for_no_body() {
 463         check(
 464             r#"
 465 fn foo() {
 466     for bar in qux
 467 }
 468 "#,
 469             expect![[r#"
 470 fn foo () {for bar in qux { }}
 471 "#]],
 472         )
 473     }
 474
 475     // FIXME: https://github.com/rust-lang/rust-analyzer/pull/12937#discussion_r937633695
 476     #[test]
 477     fn for_no_pat() {
 478         check(
 479             r#"
 480 fn foo() {
 481     for in qux {
 482
 483     }
 484 }
 485 "#,
 486             expect![[r#"
 487 fn foo () {__ra_fixup}
 488 "#]],
 489         )
 490     }
 491
 492     #[test]
 493     fn match_no_expr_no_arms() {
 494         check(
 495             r#"
 496 fn foo() {
 497     match
 498 }
 499 "#,
 500             expect![[r#"
 501 fn foo () {match __ra_fixup { }}
 502 "#]],
 503         )
 504     }
 505
 506     #[test]
 507     fn match_expr_no_arms() {
 508         check(
 509             r#"
 510 fn foo() {
 511     match it {
 512
 513     }
 514 }
 515 "#,
 516             expect![[r#"
 517 fn foo () {match it {}}
 518 "#]],
 519         )
 520     }
 521
 522     #[test]
 523     fn match_no_expr() {
 524         check(
 525             r#"
 526 fn foo() {
 527     match {
 528         _ => {}
 529     }
 530 }
 531 "#,
 532             expect![[r#"
 533 fn foo () {match __ra_fixup { }}
 534 "#]],
 535         )
 536     }
 537
 538     #[test]
 539     fn incomplete_field_expr_1() {
 540         check(
 541             r#"
 542 fn foo() {
 543     a.
 544 }
 545 "#,
 546             expect![[r#"
 547 fn foo () {a . __ra_fixup}
 548 "#]],
 549         )
 550     }
 551
 552     #[test]
 553     fn incomplete_field_expr_2() {
 554         check(
 555             r#"
 556 fn foo() {
 557     a.;
 558 }
 559 "#,
 560             expect![[r#"
 561 fn foo () {a . __ra_fixup ;}
 562 "#]],
 563         )
 564     }
 565
 566     #[test]
 567     fn incomplete_field_expr_3() {
 568         check(
 569             r#"
 570 fn foo() {
 571     a.;
 572     bar();
 573 }
 574 "#,
 575             expect![[r#"
 576 fn foo () {a . __ra_fixup ; bar () ;}
 577 "#]],
 578         )
 579     }
 580
 581     #[test]
 582     fn incomplete_let() {
 583         check(
 584             r#"
 585 fn foo() {
 586     let it = a
 587 }
 588 "#,
 589             expect![[r#"
 590 fn foo () {let it = a ;}
 591 "#]],
 592         )
 593     }
 594
 595     #[test]
 596     fn incomplete_field_expr_in_let() {
 597         check(
 598             r#"
 599 fn foo() {
 600     let it = a.
 601 }
 602 "#,
 603             expect![[r#"
 604 fn foo () {let it = a . __ra_fixup ;}
 605 "#]],
 606         )
 607     }
 608
 609     #[test]
 610     fn field_expr_before_call() {
 611         // another case that easily happens while typing
 612         check(
 613             r#"
 614 fn foo() {
 615     a.b
 616     bar();
 617 }
 618 "#,
 619             expect![[r#"
 620 fn foo () {a . b ; bar () ;}
 621 "#]],
 622         )
 623     }
 624
 625     #[test]
 626     fn extraneous_comma() {
 627         check(
 628             r#"
 629 fn foo() {
 630     bar(,);
 631 }
 632 "#,
 633             expect![[r#"
 634 fn foo () {__ra_fixup ;}
 635 "#]],
 636         )
 637     }
 638
 639     #[test]
 640     fn fixup_if_1() {
 641         check(
 642             r#"
 643 fn foo() {
 644     if a
 645 }
 646 "#,
 647             expect![[r#"
 648 fn foo () {if a { }}
 649 "#]],
 650         )
 651     }
 652
 653     #[test]
 654     fn fixup_if_2() {
 655         check(
 656             r#"
 657 fn foo() {
 658     if
 659 }
 660 "#,
 661             expect![[r#"
 662 fn foo () {if __ra_fixup { }}
 663 "#]],
 664         )
 665     }
 666
 667     #[test]
 668     fn fixup_if_3() {
 669         check(
 670             r#"
 671 fn foo() {
 672     if {}
 673 }
 674 "#,
 675             expect![[r#"
 676 fn foo () {if __ra_fixup {} { }}
 677 "#]],
 678         )
 679     }
 680
 681     #[test]
 682     fn fixup_while_1() {
 683         check(
 684             r#"
 685 fn foo() {
 686     while
 687 }
 688 "#,
 689             expect![[r#"
 690 fn foo () {while __ra_fixup { }}
 691 "#]],
 692         )
 693     }
 694
 695     #[test]
 696     fn fixup_while_2() {
 697         check(
 698             r#"
 699 fn foo() {
 700     while foo
 701 }
 702 "#,
 703             expect![[r#"
 704 fn foo () {while foo { }}
 705 "#]],
 706         )
 707     }
 708     #[test]
 709     fn fixup_while_3() {
 710         check(
 711             r#"
 712 fn foo() {
 713     while {}
 714 }
 715 "#,
 716             expect![[r#"
 717 fn foo () {while __ra_fixup {}}
 718 "#]],
 719         )
 720     }
 721
 722     #[test]
 723     fn fixup_loop() {
 724         check(
 725             r#"
 726 fn foo() {
 727     loop
 728 }
 729 "#,
 730             expect![[r#"
 731 fn foo () {loop { }}
 732 "#]],
 733         )
 734     }
 735 }