]>
Commit | Line | Data |
---|---|---|
5bcae85e SL |
1 | //! # Token Streams |
2 | //! | |
7cac9316 | 3 | //! `TokenStream`s represent syntactic objects before they are converted into ASTs. |
5869c6ff XL |
4 | //! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s, |
5 | //! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens. | |
5bcae85e SL |
6 | //! |
7 | //! ## Ownership | |
9fa01778 | 8 | //! |
e1599b0c | 9 | //! `TokenStream`s are persistent data structures constructed as ropes with reference |
7cac9316 XL |
10 | //! counted-children. In general, this means that calling an operation on a `TokenStream` |
11 | //! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to | |
5869c6ff | 12 | //! the original. This essentially coerces `TokenStream`s into "views" of their subparts, |
7cac9316 | 13 | //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking |
5bcae85e | 14 | //! ownership of the original. |
3157f602 | 15 | |
923072b8 FG |
16 | use crate::ast::StmtKind; |
17 | use crate::ast_traits::{HasAttrs, HasSpan, HasTokens}; | |
18 | use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind}; | |
cdc7bbd5 | 19 | use crate::AttrVec; |
9fa01778 | 20 | |
60c5eb7d | 21 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
29967ef6 | 22 | use rustc_data_structures::sync::{self, Lrc}; |
dfeec247 | 23 | use rustc_macros::HashStable_Generic; |
29967ef6 | 24 | use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; |
dfeec247 XL |
25 | use rustc_span::{Span, DUMMY_SP}; |
26 | use smallvec::{smallvec, SmallVec}; | |
5bcae85e | 27 | |
923072b8 | 28 | use std::{fmt, iter}; |
3157f602 | 29 | |
5869c6ff XL |
30 | /// When the main Rust parser encounters a syntax-extension invocation, it |
31 | /// parses the arguments to the invocation as a token tree. This is a very | |
32 | /// loose structure, such that all sorts of different AST fragments can | |
3157f602 XL |
33 | /// be passed to syntax extensions using a uniform type. |
34 | /// | |
35 | /// If the syntax extension is an MBE macro, it will attempt to match its | |
36 | /// LHS token tree against the provided token tree, and if it finds a | |
37 | /// match, will transcribe the RHS token tree, splicing in any captured | |
7cac9316 | 38 | /// `macro_parser::matched_nonterminals` into the `SubstNt`s it finds. |
3157f602 XL |
39 | /// |
40 | /// The RHS of an MBE macro is the only place `SubstNt`s are substituted. | |
41 | /// Nothing special happens to misnamed or misplaced `SubstNt`s. | |
3dfed10e | 42 | #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] |
3157f602 | 43 | pub enum TokenTree { |
5869c6ff | 44 | /// A single token. |
064997fb | 45 | Token(Token, Spacing), |
5869c6ff | 46 | /// A delimited sequence of token trees. |
04454e1e | 47 | Delimited(DelimSpan, Delimiter, TokenStream), |
3157f602 XL |
48 | } |
49 | ||
dc9dc135 XL |
50 | // Ensure all fields of `TokenTree` is `Send` and `Sync`. |
51 | #[cfg(parallel_compiler)] | |
52 | fn _dummy() | |
53 | where | |
54 | Token: Send + Sync, | |
55 | DelimSpan: Send + Sync, | |
04454e1e | 56 | Delimiter: Send + Sync, |
dc9dc135 | 57 | TokenStream: Send + Sync, |
dfeec247 XL |
58 | { |
59 | } | |
dc9dc135 | 60 | |
3157f602 | 61 | impl TokenTree { |
5869c6ff | 62 | /// Checks if this `TokenTree` is equal to the other, regardless of span information. |
5bcae85e SL |
63 | pub fn eq_unspanned(&self, other: &TokenTree) -> bool { |
64 | match (self, other) { | |
064997fb | 65 | (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind, |
dc9dc135 | 66 | (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { |
487cf647 | 67 | delim == delim2 && tts.eq_unspanned(tts2) |
83c7162d | 68 | } |
dc9dc135 | 69 | _ => false, |
83c7162d XL |
70 | } |
71 | } | |
72 | ||
5869c6ff | 73 | /// Retrieves the `TokenTree`'s span. |
5bcae85e | 74 | pub fn span(&self) -> Span { |
dc9dc135 | 75 | match self { |
064997fb | 76 | TokenTree::Token(token, _) => token.span, |
0731742a | 77 | TokenTree::Delimited(sp, ..) => sp.entire(), |
5bcae85e SL |
78 | } |
79 | } | |
80 | ||
a1dfa0c6 | 81 | /// Modify the `TokenTree`'s span in-place. |
3b2f2976 | 82 | pub fn set_span(&mut self, span: Span) { |
dc9dc135 | 83 | match self { |
064997fb | 84 | TokenTree::Token(token, _) => token.span = span, |
dc9dc135 | 85 | TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span), |
5bcae85e SL |
86 | } |
87 | } | |
041b39d2 | 88 | |
487cf647 | 89 | /// Create a `TokenTree::Token` with alone spacing. |
064997fb FG |
90 | pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree { |
91 | TokenTree::Token(Token::new(kind, span), Spacing::Alone) | |
92 | } | |
93 | ||
487cf647 | 94 | /// Create a `TokenTree::Token` with joint spacing. |
064997fb FG |
95 | pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree { |
96 | TokenTree::Token(Token::new(kind, span), Spacing::Joint) | |
dc9dc135 XL |
97 | } |
98 | ||
74b04a01 XL |
99 | pub fn uninterpolate(self) -> TokenTree { |
100 | match self { | |
064997fb FG |
101 | TokenTree::Token(token, spacing) => { |
102 | TokenTree::Token(token.uninterpolate().into_owned(), spacing) | |
103 | } | |
74b04a01 XL |
104 | tt => tt, |
105 | } | |
106 | } | |
60c5eb7d XL |
107 | } |
108 | ||
109 | impl<CTX> HashStable<CTX> for TokenStream | |
dfeec247 XL |
110 | where |
111 | CTX: crate::HashStableContext, | |
60c5eb7d XL |
112 | { |
113 | fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { | |
114 | for sub_tt in self.trees() { | |
115 | sub_tt.hash_stable(hcx, hasher); | |
116 | } | |
041b39d2 | 117 | } |
3157f602 XL |
118 | } |
119 | ||
f2b60f7d FG |
120 | pub trait ToAttrTokenStream: sync::Send + sync::Sync { |
121 | fn to_attr_token_stream(&self) -> AttrTokenStream; | |
29967ef6 XL |
122 | } |
123 | ||
f2b60f7d FG |
124 | impl ToAttrTokenStream for AttrTokenStream { |
125 | fn to_attr_token_stream(&self) -> AttrTokenStream { | |
29967ef6 XL |
126 | self.clone() |
127 | } | |
128 | } | |
129 | ||
5869c6ff | 130 | /// A lazy version of [`TokenStream`], which defers creation |
29967ef6 XL |
131 | /// of an actual `TokenStream` until it is needed. |
132 | /// `Box` is here only to reduce the structure size. | |
133 | #[derive(Clone)] | |
f2b60f7d | 134 | pub struct LazyAttrTokenStream(Lrc<Box<dyn ToAttrTokenStream>>); |
29967ef6 | 135 | |
f2b60f7d FG |
136 | impl LazyAttrTokenStream { |
137 | pub fn new(inner: impl ToAttrTokenStream + 'static) -> LazyAttrTokenStream { | |
138 | LazyAttrTokenStream(Lrc::new(Box::new(inner))) | |
29967ef6 XL |
139 | } |
140 | ||
f2b60f7d FG |
141 | pub fn to_attr_token_stream(&self) -> AttrTokenStream { |
142 | self.0.to_attr_token_stream() | |
29967ef6 XL |
143 | } |
144 | } | |
145 | ||
f2b60f7d | 146 | impl fmt::Debug for LazyAttrTokenStream { |
29967ef6 | 147 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
f2b60f7d | 148 | write!(f, "LazyAttrTokenStream({:?})", self.to_attr_token_stream()) |
29967ef6 XL |
149 | } |
150 | } | |
151 | ||
f2b60f7d | 152 | impl<S: Encoder> Encodable<S> for LazyAttrTokenStream { |
923072b8 | 153 | fn encode(&self, s: &mut S) { |
29967ef6 | 154 | // Used by AST json printing. |
f2b60f7d | 155 | Encodable::encode(&self.to_attr_token_stream(), s); |
29967ef6 XL |
156 | } |
157 | } | |
158 | ||
f2b60f7d | 159 | impl<D: Decoder> Decodable<D> for LazyAttrTokenStream { |
5099ac24 | 160 | fn decode(_d: &mut D) -> Self { |
f2b60f7d | 161 | panic!("Attempted to decode LazyAttrTokenStream"); |
29967ef6 XL |
162 | } |
163 | } | |
164 | ||
f2b60f7d | 165 | impl<CTX> HashStable<CTX> for LazyAttrTokenStream { |
29967ef6 | 166 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { |
f2b60f7d | 167 | panic!("Attempted to compute stable hash for LazyAttrTokenStream"); |
29967ef6 XL |
168 | } |
169 | } | |
170 | ||
f2b60f7d | 171 | /// An `AttrTokenStream` is similar to a `TokenStream`, but with extra |
cdc7bbd5 XL |
172 | /// information about the tokens for attribute targets. This is used |
173 | /// during expansion to perform early cfg-expansion, and to process attributes | |
174 | /// during proc-macro invocations. | |
175 | #[derive(Clone, Debug, Default, Encodable, Decodable)] | |
f2b60f7d | 176 | pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>); |
cdc7bbd5 | 177 | |
f2b60f7d | 178 | /// Like `TokenTree`, but for `AttrTokenStream`. |
cdc7bbd5 | 179 | #[derive(Clone, Debug, Encodable, Decodable)] |
f2b60f7d FG |
180 | pub enum AttrTokenTree { |
181 | Token(Token, Spacing), | |
182 | Delimited(DelimSpan, Delimiter, AttrTokenStream), | |
cdc7bbd5 XL |
183 | /// Stores the attributes for an attribute target, |
184 | /// along with the tokens for that attribute target. | |
185 | /// See `AttributesData` for more information | |
186 | Attributes(AttributesData), | |
187 | } | |
188 | ||
f2b60f7d FG |
189 | impl AttrTokenStream { |
190 | pub fn new(tokens: Vec<AttrTokenTree>) -> AttrTokenStream { | |
191 | AttrTokenStream(Lrc::new(tokens)) | |
cdc7bbd5 XL |
192 | } |
193 | ||
f2b60f7d FG |
194 | /// Converts this `AttrTokenStream` to a plain `TokenStream`. |
195 | /// During conversion, `AttrTokenTree::Attributes` get 'flattened' | |
cdc7bbd5 XL |
196 | /// back to a `TokenStream` of the form `outer_attr attr_target`. |
197 | /// If there are inner attributes, they are inserted into the proper | |
198 | /// place in the attribute target tokens. | |
199 | pub fn to_tokenstream(&self) -> TokenStream { | |
200 | let trees: Vec<_> = self | |
201 | .0 | |
202 | .iter() | |
f2b60f7d FG |
203 | .flat_map(|tree| match &tree { |
204 | AttrTokenTree::Token(inner, spacing) => { | |
205 | smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter() | |
064997fb | 206 | } |
f2b60f7d | 207 | AttrTokenTree::Delimited(span, delim, stream) => { |
064997fb FG |
208 | smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),] |
209 | .into_iter() | |
cdc7bbd5 | 210 | } |
f2b60f7d | 211 | AttrTokenTree::Attributes(data) => { |
cdc7bbd5 XL |
212 | let mut outer_attrs = Vec::new(); |
213 | let mut inner_attrs = Vec::new(); | |
17df50a5 | 214 | for attr in &data.attrs { |
cdc7bbd5 | 215 | match attr.style { |
f2b60f7d FG |
216 | crate::AttrStyle::Outer => outer_attrs.push(attr), |
217 | crate::AttrStyle::Inner => inner_attrs.push(attr), | |
cdc7bbd5 XL |
218 | } |
219 | } | |
220 | ||
221 | let mut target_tokens: Vec<_> = data | |
222 | .tokens | |
f2b60f7d | 223 | .to_attr_token_stream() |
cdc7bbd5 XL |
224 | .to_tokenstream() |
225 | .0 | |
226 | .iter() | |
227 | .cloned() | |
228 | .collect(); | |
229 | if !inner_attrs.is_empty() { | |
230 | let mut found = false; | |
231 | // Check the last two trees (to account for a trailing semi) | |
064997fb | 232 | for tree in target_tokens.iter_mut().rev().take(2) { |
cdc7bbd5 | 233 | if let TokenTree::Delimited(span, delim, delim_tokens) = tree { |
f2b60f7d FG |
234 | // Inner attributes are only supported on extern blocks, functions, |
235 | // impls, and modules. All of these have their inner attributes | |
236 | // placed at the beginning of the rightmost outermost braced group: | |
cdc7bbd5 XL |
237 | // e.g. fn foo() { #![my_attr} } |
238 | // | |
239 | // Therefore, we can insert them back into the right location | |
240 | // without needing to do any extra position tracking. | |
241 | // | |
242 | // Note: Outline modules are an exception - they can | |
243 | // have attributes like `#![my_attr]` at the start of a file. | |
244 | // Support for custom attributes in this position is not | |
245 | // properly implemented - we always synthesize fake tokens, | |
246 | // so we never reach this code. | |
247 | ||
2b03887a | 248 | let mut stream = TokenStream::default(); |
17df50a5 | 249 | for inner_attr in inner_attrs { |
2b03887a | 250 | stream.push_stream(inner_attr.tokens()); |
cdc7bbd5 | 251 | } |
2b03887a FG |
252 | stream.push_stream(delim_tokens.clone()); |
253 | *tree = TokenTree::Delimited(*span, *delim, stream); | |
cdc7bbd5 XL |
254 | found = true; |
255 | break; | |
256 | } | |
257 | } | |
258 | ||
259 | assert!( | |
260 | found, | |
261 | "Failed to find trailing delimited group in: {:?}", | |
262 | target_tokens | |
263 | ); | |
264 | } | |
265 | let mut flat: SmallVec<[_; 1]> = SmallVec::new(); | |
266 | for attr in outer_attrs { | |
267 | // FIXME: Make this more efficient | |
f2b60f7d | 268 | flat.extend(attr.tokens().0.clone().iter().cloned()); |
cdc7bbd5 XL |
269 | } |
270 | flat.extend(target_tokens); | |
271 | flat.into_iter() | |
272 | } | |
273 | }) | |
274 | .collect(); | |
275 | TokenStream::new(trees) | |
276 | } | |
277 | } | |
278 | ||
279 | /// Stores the tokens for an attribute target, along | |
280 | /// with its attributes. | |
281 | /// | |
282 | /// This is constructed during parsing when we need to capture | |
283 | /// tokens. | |
284 | /// | |
285 | /// For example, `#[cfg(FALSE)] struct Foo {}` would | |
286 | /// have an `attrs` field containing the `#[cfg(FALSE)]` attr, | |
5e7ed085 | 287 | /// and a `tokens` field storing the (unparsed) tokens `struct Foo {}` |
cdc7bbd5 XL |
288 | #[derive(Clone, Debug, Encodable, Decodable)] |
289 | pub struct AttributesData { | |
290 | /// Attributes, both outer and inner. | |
291 | /// These are stored in the original order that they were parsed in. | |
292 | pub attrs: AttrVec, | |
293 | /// The underlying tokens for the attribute target that `attrs` | |
294 | /// are applied to | |
f2b60f7d | 295 | pub tokens: LazyAttrTokenStream, |
cdc7bbd5 XL |
296 | } |
297 | ||
5869c6ff | 298 | /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. |
e1599b0c | 299 | /// |
32a655c1 SL |
300 | /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s |
301 | /// instead of a representation of the abstract syntax tree. | |
5869c6ff | 302 | /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for |
cdc7bbd5 | 303 | /// backwards compatibility. |
3dfed10e | 304 | #[derive(Clone, Debug, Default, Encodable, Decodable)] |
064997fb | 305 | pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>); |
0731742a | 306 | |
2b03887a FG |
307 | /// Similar to `proc_macro::Spacing`, but for tokens. |
308 | /// | |
309 | /// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when | |
310 | /// we convert to `proc_macro::TokenTree` for proc macros only `Punct` | |
311 | /// `TokenTree`s have a `proc_macro::Spacing`. | |
064997fb | 312 | #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] |
1b1a35ee | 313 | pub enum Spacing { |
2b03887a FG |
314 | /// The token is not immediately followed by an operator token (as |
315 | /// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`, | |
316 | /// `+/*foo*/=`, `+ident`, and `+()`. | |
1b1a35ee | 317 | Alone, |
2b03887a FG |
318 | |
319 | /// The token is immediately followed by an operator token. E.g. a `+` | |
320 | /// token is `Joint` in `+=` and `++`. | |
0731742a | 321 | Joint, |
0731742a XL |
322 | } |
323 | ||
b7449926 XL |
324 | impl TokenStream { |
325 | /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` | |
326 | /// separating the two arguments with a comma for diagnostic suggestions. | |
e74abb32 | 327 | pub fn add_comma(&self) -> Option<(TokenStream, Span)> { |
b7449926 | 328 | // Used to suggest if a user writes `foo!(a b);` |
e74abb32 XL |
329 | let mut suggestion = None; |
330 | let mut iter = self.0.iter().enumerate().peekable(); | |
331 | while let Some((pos, ts)) = iter.next() { | |
332 | if let Some((_, next)) = iter.peek() { | |
333 | let sp = match (&ts, &next) { | |
064997fb | 334 | (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, |
dfeec247 | 335 | ( |
064997fb FG |
336 | TokenTree::Token(token_left, Spacing::Alone), |
337 | TokenTree::Token(token_right, _), | |
dfeec247 XL |
338 | ) if ((token_left.is_ident() && !token_left.is_reserved_ident()) |
339 | || token_left.is_lit()) | |
340 | && ((token_right.is_ident() && !token_right.is_reserved_ident()) | |
341 | || token_right.is_lit()) => | |
342 | { | |
343 | token_left.span | |
344 | } | |
064997fb | 345 | (TokenTree::Delimited(sp, ..), _) => sp.entire(), |
e74abb32 XL |
346 | _ => continue, |
347 | }; | |
348 | let sp = sp.shrink_to_hi(); | |
064997fb | 349 | let comma = TokenTree::token_alone(token::Comma, sp); |
e74abb32 | 350 | suggestion = Some((pos, comma, sp)); |
b7449926 XL |
351 | } |
352 | } | |
e74abb32 | 353 | if let Some((pos, comma, sp)) = suggestion { |
fc512014 | 354 | let mut new_stream = Vec::with_capacity(self.0.len() + 1); |
e74abb32 XL |
355 | let parts = self.0.split_at(pos + 1); |
356 | new_stream.extend_from_slice(parts.0); | |
357 | new_stream.push(comma); | |
358 | new_stream.extend_from_slice(parts.1); | |
359 | return Some((TokenStream::new(new_stream), sp)); | |
360 | } | |
b7449926 XL |
361 | None |
362 | } | |
363 | } | |
364 | ||
e74abb32 XL |
365 | impl iter::FromIterator<TokenTree> for TokenStream { |
366 | fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self { | |
064997fb | 367 | TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>()) |
b7449926 XL |
368 | } |
369 | } | |
370 | ||
32a655c1 SL |
371 | impl Eq for TokenStream {} |
372 | ||
5bcae85e SL |
373 | impl PartialEq<TokenStream> for TokenStream { |
374 | fn eq(&self, other: &TokenStream) -> bool { | |
32a655c1 | 375 | self.trees().eq(other.trees()) |
5bcae85e SL |
376 | } |
377 | } | |
378 | ||
5bcae85e | 379 | impl TokenStream { |
064997fb | 380 | pub fn new(streams: Vec<TokenTree>) -> TokenStream { |
e74abb32 | 381 | TokenStream(Lrc::new(streams)) |
3b2f2976 XL |
382 | } |
383 | ||
e74abb32 XL |
384 | pub fn is_empty(&self) -> bool { |
385 | self.0.is_empty() | |
9e0c209e SL |
386 | } |
387 | ||
e74abb32 XL |
388 | pub fn len(&self) -> usize { |
389 | self.0.len() | |
5bcae85e SL |
390 | } |
391 | ||
923072b8 FG |
392 | pub fn trees(&self) -> CursorRef<'_> { |
393 | CursorRef::new(self) | |
5bcae85e SL |
394 | } |
395 | ||
8bb4bdeb | 396 | pub fn into_trees(self) -> Cursor { |
32a655c1 | 397 | Cursor::new(self) |
5bcae85e SL |
398 | } |
399 | ||
e1599b0c | 400 | /// Compares two `TokenStream`s, checking equality without regarding span information. |
5bcae85e | 401 | pub fn eq_unspanned(&self, other: &TokenStream) -> bool { |
83c7162d XL |
402 | let mut t1 = self.trees(); |
403 | let mut t2 = other.trees(); | |
cdc7bbd5 | 404 | for (t1, t2) in iter::zip(&mut t1, &mut t2) { |
487cf647 | 405 | if !t1.eq_unspanned(t2) { |
5bcae85e SL |
406 | return false; |
407 | } | |
408 | } | |
83c7162d XL |
409 | t1.next().is_none() && t2.next().is_none() |
410 | } | |
411 | ||
29967ef6 | 412 | pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { |
064997fb | 413 | TokenStream(Lrc::new(self.0.iter().enumerate().map(|(i, tree)| f(i, tree)).collect())) |
3b2f2976 | 414 | } |
923072b8 | 415 | |
487cf647 | 416 | /// Create a token stream containing a single token with alone spacing. |
064997fb FG |
417 | pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream { |
418 | TokenStream::new(vec![TokenTree::token_alone(kind, span)]) | |
419 | } | |
420 | ||
487cf647 | 421 | /// Create a token stream containing a single token with joint spacing. |
064997fb FG |
422 | pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream { |
423 | TokenStream::new(vec![TokenTree::token_joint(kind, span)]) | |
424 | } | |
425 | ||
487cf647 | 426 | /// Create a token stream containing a single `Delimited`. |
064997fb FG |
427 | pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream { |
428 | TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)]) | |
429 | } | |
430 | ||
923072b8 | 431 | pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream { |
f2b60f7d FG |
432 | let Some(tokens) = node.tokens() else { |
433 | panic!("missing tokens for node at {:?}: {:?}", node.span(), node); | |
434 | }; | |
435 | let attrs = node.attrs(); | |
436 | let attr_stream = if attrs.is_empty() { | |
437 | tokens.to_attr_token_stream() | |
438 | } else { | |
439 | let attr_data = | |
440 | AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; | |
441 | AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)]) | |
442 | }; | |
443 | attr_stream.to_tokenstream() | |
923072b8 FG |
444 | } |
445 | ||
446 | pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { | |
447 | match nt { | |
448 | Nonterminal::NtIdent(ident, is_raw) => { | |
064997fb | 449 | TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span) |
923072b8 FG |
450 | } |
451 | Nonterminal::NtLifetime(ident) => { | |
064997fb | 452 | TokenStream::token_alone(token::Lifetime(ident.name), ident.span) |
923072b8 FG |
453 | } |
454 | Nonterminal::NtItem(item) => TokenStream::from_ast(item), | |
455 | Nonterminal::NtBlock(block) => TokenStream::from_ast(block), | |
456 | Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { | |
457 | // FIXME: Properly collect tokens for empty statements. | |
064997fb | 458 | TokenStream::token_alone(token::Semi, stmt.span) |
923072b8 FG |
459 | } |
460 | Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt), | |
461 | Nonterminal::NtPat(pat) => TokenStream::from_ast(pat), | |
462 | Nonterminal::NtTy(ty) => TokenStream::from_ast(ty), | |
463 | Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr), | |
464 | Nonterminal::NtPath(path) => TokenStream::from_ast(path), | |
465 | Nonterminal::NtVis(vis) => TokenStream::from_ast(vis), | |
466 | Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr), | |
467 | } | |
468 | } | |
469 | ||
064997fb | 470 | fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { |
923072b8 FG |
471 | match &token.kind { |
472 | token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => { | |
064997fb | 473 | TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) |
923072b8 FG |
474 | } |
475 | token::Interpolated(nt) => TokenTree::Delimited( | |
476 | DelimSpan::from_single(token.span), | |
477 | Delimiter::Invisible, | |
487cf647 | 478 | TokenStream::from_nonterminal_ast(nt).flattened(), |
923072b8 | 479 | ), |
064997fb | 480 | _ => TokenTree::Token(token.clone(), spacing), |
923072b8 FG |
481 | } |
482 | } | |
483 | ||
484 | fn flatten_token_tree(tree: &TokenTree) -> TokenTree { | |
485 | match tree { | |
064997fb | 486 | TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing), |
923072b8 FG |
487 | TokenTree::Delimited(span, delim, tts) => { |
488 | TokenTree::Delimited(*span, *delim, tts.flattened()) | |
489 | } | |
490 | } | |
491 | } | |
492 | ||
493 | #[must_use] | |
494 | pub fn flattened(&self) -> TokenStream { | |
495 | fn can_skip(stream: &TokenStream) -> bool { | |
496 | stream.trees().all(|tree| match tree { | |
064997fb | 497 | TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), |
923072b8 FG |
498 | TokenTree::Delimited(_, _, inner) => can_skip(inner), |
499 | }) | |
500 | } | |
501 | ||
502 | if can_skip(self) { | |
503 | return self.clone(); | |
504 | } | |
505 | ||
506 | self.trees().map(|tree| TokenStream::flatten_token_tree(tree)).collect() | |
507 | } | |
041b39d2 | 508 | |
2b03887a FG |
509 | // If `vec` is not empty, try to glue `tt` onto its last token. The return |
510 | // value indicates if gluing took place. | |
511 | fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool { | |
512 | if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last() | |
513 | && let TokenTree::Token(tok, spacing) = tt | |
487cf647 | 514 | && let Some(glued_tok) = last_tok.glue(tok) |
2b03887a FG |
515 | { |
516 | // ...then overwrite the last token tree in `vec` with the | |
517 | // glued token, and skip the first token tree from `stream`. | |
518 | *vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing); | |
519 | true | |
520 | } else { | |
521 | false | |
522 | } | |
523 | } | |
923072b8 | 524 | |
487cf647 FG |
525 | /// Push `tt` onto the end of the stream, possibly gluing it to the last |
526 | /// token. Uses `make_mut` to maximize efficiency. | |
2b03887a FG |
527 | pub fn push_tree(&mut self, tt: TokenTree) { |
528 | let vec_mut = Lrc::make_mut(&mut self.0); | |
529 | ||
530 | if Self::try_glue_to_last(vec_mut, &tt) { | |
531 | // nothing else to do | |
532 | } else { | |
533 | vec_mut.push(tt); | |
534 | } | |
535 | } | |
536 | ||
487cf647 FG |
537 | /// Push `stream` onto the end of the stream, possibly gluing the first |
538 | /// token tree to the last token. (No other token trees will be glued.) | |
539 | /// Uses `make_mut` to maximize efficiency. | |
2b03887a FG |
540 | pub fn push_stream(&mut self, stream: TokenStream) { |
541 | let vec_mut = Lrc::make_mut(&mut self.0); | |
542 | ||
543 | let stream_iter = stream.0.iter().cloned(); | |
544 | ||
545 | if let Some(first) = stream.0.first() && Self::try_glue_to_last(vec_mut, first) { | |
546 | // Now skip the first token tree from `stream`. | |
547 | vec_mut.extend(stream_iter.skip(1)); | |
548 | } else { | |
549 | // Append all of `stream`. | |
550 | vec_mut.extend(stream_iter); | |
923072b8 | 551 | } |
041b39d2 | 552 | } |
5bcae85e SL |
553 | } |
554 | ||
5869c6ff | 555 | /// By-reference iterator over a [`TokenStream`]. |
29967ef6 XL |
556 | #[derive(Clone)] |
557 | pub struct CursorRef<'t> { | |
558 | stream: &'t TokenStream, | |
559 | index: usize, | |
560 | } | |
561 | ||
562 | impl<'t> CursorRef<'t> { | |
923072b8 FG |
563 | fn new(stream: &'t TokenStream) -> Self { |
564 | CursorRef { stream, index: 0 } | |
565 | } | |
566 | ||
923072b8 | 567 | pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { |
064997fb | 568 | self.stream.0.get(self.index + n) |
923072b8 | 569 | } |
29967ef6 XL |
570 | } |
571 | ||
572 | impl<'t> Iterator for CursorRef<'t> { | |
573 | type Item = &'t TokenTree; | |
574 | ||
575 | fn next(&mut self) -> Option<&'t TokenTree> { | |
064997fb FG |
576 | self.stream.0.get(self.index).map(|tree| { |
577 | self.index += 1; | |
578 | tree | |
579 | }) | |
29967ef6 XL |
580 | } |
581 | } | |
582 | ||
5869c6ff XL |
583 | /// Owning by-value iterator over a [`TokenStream`]. |
584 | // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones. | |
041b39d2 | 585 | #[derive(Clone)] |
0731742a XL |
586 | pub struct Cursor { |
587 | pub stream: TokenStream, | |
8bb4bdeb | 588 | index: usize, |
041b39d2 XL |
589 | } |
590 | ||
591 | impl Iterator for Cursor { | |
592 | type Item = TokenTree; | |
593 | ||
594 | fn next(&mut self) -> Option<TokenTree> { | |
064997fb FG |
595 | self.stream.0.get(self.index).map(|tree| { |
596 | self.index += 1; | |
597 | tree.clone() | |
598 | }) | |
041b39d2 | 599 | } |
32a655c1 | 600 | } |
5bcae85e | 601 | |
8bb4bdeb XL |
602 | impl Cursor { |
603 | fn new(stream: TokenStream) -> Self { | |
0731742a XL |
604 | Cursor { stream, index: 0 } |
605 | } | |
606 | ||
04454e1e | 607 | #[inline] |
064997fb | 608 | pub fn next_ref(&mut self) -> Option<&TokenTree> { |
04454e1e FG |
609 | self.stream.0.get(self.index).map(|tree| { |
610 | self.index += 1; | |
611 | tree | |
612 | }) | |
041b39d2 XL |
613 | } |
614 | ||
29967ef6 | 615 | pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { |
064997fb | 616 | self.stream.0.get(self.index + n) |
5bcae85e SL |
617 | } |
618 | } | |
619 | ||
3dfed10e | 620 | #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] |
b7449926 XL |
621 | pub struct DelimSpan { |
622 | pub open: Span, | |
623 | pub close: Span, | |
624 | } | |
625 | ||
626 | impl DelimSpan { | |
627 | pub fn from_single(sp: Span) -> Self { | |
dfeec247 | 628 | DelimSpan { open: sp, close: sp } |
b7449926 XL |
629 | } |
630 | ||
631 | pub fn from_pair(open: Span, close: Span) -> Self { | |
632 | DelimSpan { open, close } | |
633 | } | |
634 | ||
635 | pub fn dummy() -> Self { | |
636 | Self::from_single(DUMMY_SP) | |
637 | } | |
638 | ||
639 | pub fn entire(self) -> Span { | |
640 | self.open.with_hi(self.close.hi()) | |
641 | } | |
b7449926 | 642 | } |
2b03887a FG |
643 | |
644 | // Some types are used a lot. Make sure they don't unintentionally get bigger. | |
645 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] | |
646 | mod size_asserts { | |
647 | use super::*; | |
648 | use rustc_data_structures::static_assert_size; | |
649 | // tidy-alphabetical-start | |
650 | static_assert_size!(AttrTokenStream, 8); | |
651 | static_assert_size!(AttrTokenTree, 32); | |
652 | static_assert_size!(LazyAttrTokenStream, 8); | |
653 | static_assert_size!(TokenStream, 8); | |
654 | static_assert_size!(TokenTree, 32); | |
655 | // tidy-alphabetical-end | |
656 | } |