1 //! Defines database & queries for macro expansion.
5 use base_db
::{salsa, SourceDatabase}
;
8 use mbe
::syntax_node_to_token_tree
;
9 use rustc_hash
::FxHashSet
;
11 ast
::{self, HasAttrs, HasDocComments}
,
12 AstNode
, GreenNode
, Parse
, SyntaxNode
, SyntaxToken
, T
,
16 ast_id_map
::AstIdMap
, builtin_attr_macro
::pseudo_derive_attr_expansion
, fixup
,
17 hygiene
::HygieneFrame
, BuiltinAttrExpander
, BuiltinDeriveExpander
, BuiltinFnLikeExpander
,
18 ExpandError
, ExpandResult
, ExpandTo
, HirFileId
, HirFileIdRepr
, MacroCallId
, MacroCallKind
,
19 MacroCallLoc
, MacroDefId
, MacroDefKind
, MacroFile
, ProcMacroExpander
,
22 /// Total limit on the number of tokens produced by any macro invocation.
24 /// If an invocation produces more tokens than this limit, it will not be stored in the database and
25 /// an error will be emitted.
27 /// Actual max for `analysis-stats .` at some point: 30672.
28 static TOKEN_LIMIT
: Limit
= Limit
::new(524_288);
30 #[derive(Debug, Clone, Eq, PartialEq)]
31 pub enum TokenExpander
{
32 /// Old-style `macro_rules` or the new macros 2.0
33 DeclarativeMacro { mac: mbe::DeclarativeMacro, def_site_token_map: mbe::TokenMap }
,
34 /// Stuff like `line!` and `file!`.
35 Builtin(BuiltinFnLikeExpander
),
36 /// `global_allocator` and such.
37 BuiltinAttr(BuiltinAttrExpander
),
38 /// `derive(Copy)` and such.
39 BuiltinDerive(BuiltinDeriveExpander
),
40 /// The thing we love the most here in rust-analyzer -- procedural macros.
41 ProcMacro(ProcMacroExpander
),
50 ) -> ExpandResult
<tt
::Subtree
> {
52 TokenExpander
::DeclarativeMacro { mac, .. }
=> mac
.expand(tt
).map_err(Into
::into
),
53 TokenExpander
::Builtin(it
) => it
.expand(db
, id
, tt
).map_err(Into
::into
),
54 TokenExpander
::BuiltinAttr(it
) => it
.expand(db
, id
, tt
),
55 TokenExpander
::BuiltinDerive(it
) => it
.expand(db
, id
, tt
),
56 TokenExpander
::ProcMacro(_
) => {
57 // We store the result in salsa db to prevent non-deterministic behavior in
58 // some proc-macro implementation
59 // See #4315 for details
60 db
.expand_proc_macro(id
)
65 pub(crate) fn map_id_down(&self, id
: tt
::TokenId
) -> tt
::TokenId
{
67 TokenExpander
::DeclarativeMacro { mac, .. }
=> mac
.map_id_down(id
),
68 TokenExpander
::Builtin(..)
69 | TokenExpander
::BuiltinAttr(..)
70 | TokenExpander
::BuiltinDerive(..)
71 | TokenExpander
::ProcMacro(..) => id
,
75 pub(crate) fn map_id_up(&self, id
: tt
::TokenId
) -> (tt
::TokenId
, mbe
::Origin
) {
77 TokenExpander
::DeclarativeMacro { mac, .. }
=> mac
.map_id_up(id
),
78 TokenExpander
::Builtin(..)
79 | TokenExpander
::BuiltinAttr(..)
80 | TokenExpander
::BuiltinDerive(..)
81 | TokenExpander
::ProcMacro(..) => (id
, mbe
::Origin
::Call
),
86 // FIXME: rename to ExpandDatabase
87 #[salsa::query_group(AstDatabaseStorage)]
88 pub trait AstDatabase
: SourceDatabase
{
89 fn ast_id_map(&self, file_id
: HirFileId
) -> Arc
<AstIdMap
>;
91 /// Main public API -- parses a hir file, not caring whether it's a real
92 /// file or a macro expansion.
94 fn parse_or_expand(&self, file_id
: HirFileId
) -> Option
<SyntaxNode
>;
95 /// Implementation for the macro case.
96 fn parse_macro_expansion(
98 macro_file
: MacroFile
,
99 ) -> ExpandResult
<Option
<(Parse
<SyntaxNode
>, Arc
<mbe
::TokenMap
>)>>;
101 /// Macro ids. That's probably the tricksiest bit in rust-analyzer, and the
102 /// reason why we use salsa at all.
104 /// We encode macro definitions into ids of macro calls, this what allows us
105 /// to be incremental.
107 fn intern_macro_call(&self, macro_call
: MacroCallLoc
) -> MacroCallId
;
109 /// Lowers syntactic macro call to a token tree representation.
110 #[salsa::transparent]
114 ) -> Option
<Arc
<(tt
::Subtree
, mbe
::TokenMap
, fixup
::SyntaxFixupUndoInfo
)>>;
115 /// Extracts syntax node, corresponding to a macro call. That's a firewall
116 /// query, only typing in the macro call itself changes the returned
118 fn macro_arg_text(&self, id
: MacroCallId
) -> Option
<GreenNode
>;
119 /// Gets the expander for this macro. This compiles declarative macros, and
120 /// just fetches procedural ones.
121 fn macro_def(&self, id
: MacroDefId
) -> Result
<Arc
<TokenExpander
>, mbe
::ParseError
>;
123 /// Expand macro call to a token tree. This query is LRUed (we keep 128 or so results in memory)
124 fn macro_expand(&self, macro_call
: MacroCallId
) -> ExpandResult
<Option
<Arc
<tt
::Subtree
>>>;
125 /// Special case of the previous query for procedural macros. We can't LRU
126 /// proc macros, since they are not deterministic in general, and
127 /// non-determinism breaks salsa in a very, very, very bad way. @edwin0cheng
128 /// heroically debugged this once!
129 fn expand_proc_macro(&self, call
: MacroCallId
) -> ExpandResult
<tt
::Subtree
>;
130 /// Firewall query that returns the error from the `macro_expand` query.
131 fn macro_expand_error(&self, macro_call
: MacroCallId
) -> Option
<ExpandError
>;
133 fn hygiene_frame(&self, file_id
: HirFileId
) -> Arc
<HygieneFrame
>;
136 /// This expands the given macro call, but with different arguments. This is
137 /// used for completion, where we want to see what 'would happen' if we insert a
138 /// token. The `token_to_map` mapped down into the expansion, with the mapped
140 pub fn expand_speculative(
141 db
: &dyn AstDatabase
,
142 actual_macro_call
: MacroCallId
,
143 speculative_args
: &SyntaxNode
,
144 token_to_map
: SyntaxToken
,
145 ) -> Option
<(SyntaxNode
, SyntaxToken
)> {
146 let loc
= db
.lookup_intern_macro_call(actual_macro_call
);
147 let macro_def
= db
.macro_def(loc
.def
).ok()?
;
148 let token_range
= token_to_map
.text_range();
150 // Build the subtree and token mapping for the speculative args
151 let censor
= censor_for_macro_input(&loc
, speculative_args
);
152 let mut fixups
= fixup
::fixup_syntax(speculative_args
);
153 fixups
.replace
.extend(censor
.into_iter().map(|node
| (node
.into(), Vec
::new())));
154 let (mut tt
, spec_args_tmap
, _
) = mbe
::syntax_node_to_token_tree_with_modifications(
162 let (attr_arg
, token_id
) = match loc
.kind
{
163 MacroCallKind
::Attr { invoc_attr_index, is_derive, .. }
=> {
164 let attr
= if is_derive
{
165 // for pseudo-derive expansion we actually pass the attribute itself only
166 ast
::Attr
::cast(speculative_args
.clone())
168 // Attributes may have an input token tree, build the subtree and map for this as well
169 // then try finding a token id for our token if it is inside this input subtree.
170 let item
= ast
::Item
::cast(speculative_args
.clone())?
;
171 item
.doc_comments_and_attrs().nth(invoc_attr_index
as usize).and_then(Either
::left
)
173 match attr
.token_tree() {
174 Some(token_tree
) => {
175 let (mut tree
, map
) = syntax_node_to_token_tree(attr
.token_tree()?
.syntax());
176 tree
.delimiter
= None
;
178 let shift
= mbe
::Shift
::new(&tt
);
179 shift
.shift_all(&mut tree
);
181 let token_id
= if token_tree
.syntax().text_range().contains_range(token_range
) {
182 let attr_input_start
=
183 token_tree
.left_delimiter_token()?
.text_range().start();
184 let range
= token_range
.checked_sub(attr_input_start
)?
;
185 let token_id
= shift
.shift(map
.token_by_range(range
)?
);
190 (Some(tree
), token_id
)
197 let token_id
= match token_id
{
198 Some(token_id
) => token_id
,
199 // token wasn't inside an attribute input so it has to be in the general macro input
201 let range
= token_range
.checked_sub(speculative_args
.text_range().start())?
;
202 let token_id
= spec_args_tmap
.token_by_range(range
)?
;
203 macro_def
.map_id_down(token_id
)
207 // Do the actual expansion, we need to directly expand the proc macro due to the attribute args
208 // Otherwise the expand query will fetch the non speculative attribute args and pass those instead.
209 let mut speculative_expansion
= match loc
.def
.kind
{
210 MacroDefKind
::ProcMacro(expander
, ..) => {
212 expander
.expand(db
, loc
.krate
, &tt
, attr_arg
.as_ref())
214 MacroDefKind
::BuiltInAttr(BuiltinAttrExpander
::Derive
, _
) => {
215 pseudo_derive_attr_expansion(&tt
, attr_arg
.as_ref()?
)
217 _
=> macro_def
.expand(db
, actual_macro_call
, &tt
),
220 let expand_to
= macro_expand_to(db
, actual_macro_call
);
221 fixup
::reverse_fixups(&mut speculative_expansion
.value
, &spec_args_tmap
, &fixups
.undo_info
);
222 let (node
, rev_tmap
) = token_tree_to_syntax_node(&speculative_expansion
.value
, expand_to
);
224 let range
= rev_tmap
.first_range_by_token(token_id
, token_to_map
.kind())?
;
225 let token
= node
.syntax_node().covering_element(range
).into_token()?
;
226 Some((node
.syntax_node(), token
))
229 fn ast_id_map(db
: &dyn AstDatabase
, file_id
: HirFileId
) -> Arc
<AstIdMap
> {
230 let map
= db
.parse_or_expand(file_id
).map(|it
| AstIdMap
::from_source(&it
)).unwrap_or_default();
234 fn parse_or_expand(db
: &dyn AstDatabase
, file_id
: HirFileId
) -> Option
<SyntaxNode
> {
236 HirFileIdRepr
::FileId(file_id
) => Some(db
.parse(file_id
).tree().syntax().clone()),
237 HirFileIdRepr
::MacroFile(macro_file
) => {
238 // FIXME: Note how we convert from `Parse` to `SyntaxNode` here,
239 // forgetting about parse errors.
240 db
.parse_macro_expansion(macro_file
).value
.map(|(it
, _
)| it
.syntax_node())
245 fn parse_macro_expansion(
246 db
: &dyn AstDatabase
,
247 macro_file
: MacroFile
,
248 ) -> ExpandResult
<Option
<(Parse
<SyntaxNode
>, Arc
<mbe
::TokenMap
>)>> {
249 let _p
= profile
::span("parse_macro_expansion");
250 let result
= db
.macro_expand(macro_file
.macro_call_id
);
252 if let Some(err
) = &result
.err
{
254 // The final goal we would like to make all parse_macro success,
255 // such that the following log will not call anyway.
256 let loc
: MacroCallLoc
= db
.lookup_intern_macro_call(macro_file
.macro_call_id
);
257 let node
= loc
.kind
.to_node(db
);
259 // collect parent information for warning log
261 std
::iter
::successors(loc
.kind
.file_id().call_node(db
), |it
| it
.file_id
.call_node(db
))
262 .map(|n
| format
!("{:#}", n
.value
))
267 "fail on macro_parse: (reason: {:?} macro_call: {:#}) parents: {}",
273 let tt
= match result
.value
{
275 None
=> return ExpandResult { value: None, err: result.err }
,
278 let expand_to
= macro_expand_to(db
, macro_file
.macro_call_id
);
280 tracing
::debug
!("expanded = {}", tt
.as_debug_string());
281 tracing
::debug
!("kind = {:?}", expand_to
);
283 let (parse
, rev_token_map
) = token_tree_to_syntax_node(&tt
, expand_to
);
285 ExpandResult { value: Some((parse, Arc::new(rev_token_map))), err: result.err }
289 db
: &dyn AstDatabase
,
291 ) -> Option
<Arc
<(tt
::Subtree
, mbe
::TokenMap
, fixup
::SyntaxFixupUndoInfo
)>> {
292 let arg
= db
.macro_arg_text(id
)?
;
293 let loc
= db
.lookup_intern_macro_call(id
);
295 let node
= SyntaxNode
::new_root(arg
);
296 let censor
= censor_for_macro_input(&loc
, &node
);
297 let mut fixups
= fixup
::fixup_syntax(&node
);
298 fixups
.replace
.extend(censor
.into_iter().map(|node
| (node
.into(), Vec
::new())));
299 let (mut tt
, tmap
, _
) = mbe
::syntax_node_to_token_tree_with_modifications(
307 if loc
.def
.is_proc_macro() {
308 // proc macros expect their inputs without parentheses, MBEs expect it with them included
312 Some(Arc
::new((tt
, tmap
, fixups
.undo_info
)))
315 fn censor_for_macro_input(loc
: &MacroCallLoc
, node
: &SyntaxNode
) -> FxHashSet
<SyntaxNode
> {
317 let censor
= match loc
.kind
{
318 MacroCallKind
::FnLike { .. }
=> return None
,
319 MacroCallKind
::Derive { derive_attr_index, .. }
=> {
320 cov_mark
::hit
!(derive_censoring
);
321 ast
::Item
::cast(node
.clone())?
323 .take(derive_attr_index
as usize + 1)
324 // FIXME, this resolution should not be done syntactically
325 // derive is a proper macro now, no longer builtin
326 // But we do not have resolution at this stage, this means
327 // we need to know about all macro calls for the given ast item here
328 // so we require some kind of mapping...
329 .filter(|attr
| attr
.simple_name().as_deref() == Some("derive"))
330 .map(|it
| it
.syntax().clone())
333 MacroCallKind
::Attr { is_derive: true, .. }
=> return None
,
334 MacroCallKind
::Attr { invoc_attr_index, .. }
=> {
335 cov_mark
::hit
!(attribute_macro_attr_censoring
);
336 ast
::Item
::cast(node
.clone())?
337 .doc_comments_and_attrs()
338 .nth(invoc_attr_index
as usize)
339 .and_then(Either
::left
)
340 .map(|attr
| attr
.syntax().clone())
350 fn macro_arg_text(db
: &dyn AstDatabase
, id
: MacroCallId
) -> Option
<GreenNode
> {
351 let loc
= db
.lookup_intern_macro_call(id
);
352 let arg
= loc
.kind
.arg(db
)?
;
353 if matches
!(loc
.kind
, MacroCallKind
::FnLike { .. }
) {
354 let first
= arg
.first_child_or_token().map_or(T
![.], |it
| it
.kind());
355 let last
= arg
.last_child_or_token().map_or(T
![.], |it
| it
.kind());
357 matches
!((first
, last
), (T
!['
('
], T
!['
)'
]) | (T
!['
['
], T
!['
]'
]) | (T
!['{'], T!['}'
]));
359 // Don't expand malformed (unbalanced) macro invocations. This is
360 // less than ideal, but trying to expand unbalanced macro calls
361 // sometimes produces pathological, deeply nested code which breaks
362 // all kinds of things.
364 // Some day, we'll have explicit recursion counters for all
365 // recursive things, at which point this code might be removed.
366 cov_mark
::hit
!(issue9358_bad_macro_stack_overflow
);
370 Some(arg
.green().into())
373 fn macro_def(db
: &dyn AstDatabase
, id
: MacroDefId
) -> Result
<Arc
<TokenExpander
>, mbe
::ParseError
> {
375 MacroDefKind
::Declarative(ast_id
) => {
376 let (mac
, def_site_token_map
) = match ast_id
.to_node(db
) {
377 ast
::Macro
::MacroRules(macro_rules
) => {
378 let arg
= macro_rules
380 .ok_or_else(|| mbe
::ParseError
::Expected("expected a token tree".into()))?
;
381 let (tt
, def_site_token_map
) = mbe
::syntax_node_to_token_tree(arg
.syntax());
382 let mac
= mbe
::DeclarativeMacro
::parse_macro_rules(&tt
)?
;
383 (mac
, def_site_token_map
)
385 ast
::Macro
::MacroDef(macro_def
) => {
388 .ok_or_else(|| mbe
::ParseError
::Expected("expected a token tree".into()))?
;
389 let (tt
, def_site_token_map
) = mbe
::syntax_node_to_token_tree(arg
.syntax());
390 let mac
= mbe
::DeclarativeMacro
::parse_macro2(&tt
)?
;
391 (mac
, def_site_token_map
)
394 Ok(Arc
::new(TokenExpander
::DeclarativeMacro { mac, def_site_token_map }
))
396 MacroDefKind
::BuiltIn(expander
, _
) => Ok(Arc
::new(TokenExpander
::Builtin(expander
))),
397 MacroDefKind
::BuiltInAttr(expander
, _
) => {
398 Ok(Arc
::new(TokenExpander
::BuiltinAttr(expander
)))
400 MacroDefKind
::BuiltInDerive(expander
, _
) => {
401 Ok(Arc
::new(TokenExpander
::BuiltinDerive(expander
)))
403 MacroDefKind
::BuiltInEager(..) => {
404 // FIXME: Return a random error here just to make the types align.
405 // This obviously should do something real instead.
406 Err(mbe
::ParseError
::UnexpectedToken("unexpected eager macro".into()))
408 MacroDefKind
::ProcMacro(expander
, ..) => Ok(Arc
::new(TokenExpander
::ProcMacro(expander
))),
412 fn macro_expand(db
: &dyn AstDatabase
, id
: MacroCallId
) -> ExpandResult
<Option
<Arc
<tt
::Subtree
>>> {
413 let _p
= profile
::span("macro_expand");
414 let loc
: MacroCallLoc
= db
.lookup_intern_macro_call(id
);
415 if let Some(eager
) = &loc
.eager
{
416 return ExpandResult
{
417 value
: Some(eager
.arg_or_expansion
.clone()),
418 // FIXME: There could be errors here!
423 let macro_arg
= match db
.macro_arg(id
) {
426 return ExpandResult
::only_err(ExpandError
::Other(
427 "Failed to lower macro args to token tree".into(),
432 let expander
= match db
.macro_def(loc
.def
) {
434 // FIXME: This is weird -- we effectively report macro *definition*
435 // errors lazily, when we try to expand the macro. Instead, they should
436 // be reported at the definition site (when we construct a def map).
438 return ExpandResult
::only_err(ExpandError
::Other(
439 format
!("invalid macro definition: {}", err
).into(),
443 let ExpandResult { value: mut tt, err }
= expander
.expand(db
, id
, ¯o_arg
.0);
444 // Set a hard limit for the expanded tt
445 let count
= tt
.count();
446 if TOKEN_LIMIT
.check(count
).is_err() {
447 return ExpandResult
::only_err(ExpandError
::Other(
449 "macro invocation exceeds token limit: produced {} tokens, limit is {}",
457 fixup
::reverse_fixups(&mut tt
, ¯o_arg
.1, ¯o_arg
.2);
459 ExpandResult { value: Some(Arc::new(tt)), err }
462 fn macro_expand_error(db
: &dyn AstDatabase
, macro_call
: MacroCallId
) -> Option
<ExpandError
> {
463 db
.macro_expand(macro_call
).err
466 fn expand_proc_macro(db
: &dyn AstDatabase
, id
: MacroCallId
) -> ExpandResult
<tt
::Subtree
> {
467 let loc
: MacroCallLoc
= db
.lookup_intern_macro_call(id
);
468 let macro_arg
= match db
.macro_arg(id
) {
471 return ExpandResult
::only_err(ExpandError
::Other("No arguments for proc-macro".into()))
475 let expander
= match loc
.def
.kind
{
476 MacroDefKind
::ProcMacro(expander
, ..) => expander
,
480 let attr_arg
= match &loc
.kind
{
481 MacroCallKind
::Attr { attr_args, .. }
=> {
482 let mut attr_args
= attr_args
.0.clone();
483 mbe
::Shift
::new(¯o_arg
.0).shift_all(&mut attr_args
);
489 expander
.expand(db
, loc
.krate
, ¯o_arg
.0, attr_arg
.as_ref())
492 fn hygiene_frame(db
: &dyn AstDatabase
, file_id
: HirFileId
) -> Arc
<HygieneFrame
> {
493 Arc
::new(HygieneFrame
::new(db
, file_id
))
496 fn macro_expand_to(db
: &dyn AstDatabase
, id
: MacroCallId
) -> ExpandTo
{
497 let loc
: MacroCallLoc
= db
.lookup_intern_macro_call(id
);
501 fn token_tree_to_syntax_node(
504 ) -> (Parse
<SyntaxNode
>, mbe
::TokenMap
) {
505 let entry_point
= match expand_to
{
506 ExpandTo
::Statements
=> mbe
::TopEntryPoint
::MacroStmts
,
507 ExpandTo
::Items
=> mbe
::TopEntryPoint
::MacroItems
,
508 ExpandTo
::Pattern
=> mbe
::TopEntryPoint
::Pattern
,
509 ExpandTo
::Type
=> mbe
::TopEntryPoint
::Type
,
510 ExpandTo
::Expr
=> mbe
::TopEntryPoint
::Expr
,
512 mbe
::token_tree_to_syntax_node(tt
, entry_point
)