]>
Commit | Line | Data |
---|---|---|
5bcae85e SL |
1 | //! # Token Streams |
2 | //! | |
7cac9316 | 3 | //! `TokenStream`s represent syntactic objects before they are converted into ASTs. |
5869c6ff XL |
4 | //! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s, |
5 | //! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens. | |
5bcae85e SL |
6 | //! |
7 | //! ## Ownership | |
9fa01778 | 8 | //! |
e1599b0c | 9 | //! `TokenStream`s are persistent data structures constructed as ropes with reference |
7cac9316 XL |
10 | //! counted-children. In general, this means that calling an operation on a `TokenStream` |
11 | //! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to | |
5869c6ff | 12 | //! the original. This essentially coerces `TokenStream`s into "views" of their subparts, |
7cac9316 | 13 | //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking |
5bcae85e | 14 | //! ownership of the original. |
3157f602 | 15 | |
60c5eb7d | 16 | use crate::token::{self, DelimToken, Token, TokenKind}; |
cdc7bbd5 | 17 | use crate::AttrVec; |
9fa01778 | 18 | |
60c5eb7d | 19 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
29967ef6 | 20 | use rustc_data_structures::sync::{self, Lrc}; |
dfeec247 | 21 | use rustc_macros::HashStable_Generic; |
29967ef6 | 22 | use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; |
dfeec247 XL |
23 | use rustc_span::{Span, DUMMY_SP}; |
24 | use smallvec::{smallvec, SmallVec}; | |
5bcae85e | 25 | |
29967ef6 | 26 | use std::{fmt, iter, mem}; |
3157f602 | 27 | |
5869c6ff XL |
28 | /// When the main Rust parser encounters a syntax-extension invocation, it |
29 | /// parses the arguments to the invocation as a token tree. This is a very | |
30 | /// loose structure, such that all sorts of different AST fragments can | |
3157f602 XL |
31 | /// be passed to syntax extensions using a uniform type. |
32 | /// | |
33 | /// If the syntax extension is an MBE macro, it will attempt to match its | |
34 | /// LHS token tree against the provided token tree, and if it finds a | |
35 | /// match, will transcribe the RHS token tree, splicing in any captured | |
7cac9316 | 36 | /// `macro_parser::matched_nonterminals` into the `SubstNt`s it finds. |
3157f602 XL |
37 | /// |
38 | /// The RHS of an MBE macro is the only place `SubstNt`s are substituted. | |
39 | /// Nothing special happens to misnamed or misplaced `SubstNt`s. | |
3dfed10e | 40 | #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] |
3157f602 | 41 | pub enum TokenTree { |
5869c6ff | 42 | /// A single token. |
dc9dc135 | 43 | Token(Token), |
5869c6ff | 44 | /// A delimited sequence of token trees. |
9fa01778 | 45 | Delimited(DelimSpan, DelimToken, TokenStream), |
3157f602 XL |
46 | } |
47 | ||
5869c6ff XL |
48 | #[derive(Copy, Clone)] |
49 | pub enum CanSynthesizeMissingTokens { | |
50 | Yes, | |
51 | No, | |
52 | } | |
53 | ||
dc9dc135 XL |
54 | // Ensure all fields of `TokenTree` is `Send` and `Sync`. |
55 | #[cfg(parallel_compiler)] | |
56 | fn _dummy() | |
57 | where | |
58 | Token: Send + Sync, | |
59 | DelimSpan: Send + Sync, | |
60 | DelimToken: Send + Sync, | |
61 | TokenStream: Send + Sync, | |
dfeec247 XL |
62 | { |
63 | } | |
dc9dc135 | 64 | |
3157f602 | 65 | impl TokenTree { |
5869c6ff | 66 | /// Checks if this `TokenTree` is equal to the other, regardless of span information. |
5bcae85e SL |
67 | pub fn eq_unspanned(&self, other: &TokenTree) -> bool { |
68 | match (self, other) { | |
dc9dc135 XL |
69 | (TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind, |
70 | (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { | |
9fa01778 | 71 | delim == delim2 && tts.eq_unspanned(&tts2) |
83c7162d | 72 | } |
dc9dc135 | 73 | _ => false, |
83c7162d XL |
74 | } |
75 | } | |
76 | ||
5869c6ff | 77 | /// Retrieves the `TokenTree`'s span. |
5bcae85e | 78 | pub fn span(&self) -> Span { |
dc9dc135 XL |
79 | match self { |
80 | TokenTree::Token(token) => token.span, | |
0731742a | 81 | TokenTree::Delimited(sp, ..) => sp.entire(), |
5bcae85e SL |
82 | } |
83 | } | |
84 | ||
a1dfa0c6 | 85 | /// Modify the `TokenTree`'s span in-place. |
3b2f2976 | 86 | pub fn set_span(&mut self, span: Span) { |
dc9dc135 XL |
87 | match self { |
88 | TokenTree::Token(token) => token.span = span, | |
89 | TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span), | |
5bcae85e SL |
90 | } |
91 | } | |
041b39d2 | 92 | |
dc9dc135 XL |
93 | pub fn token(kind: TokenKind, span: Span) -> TokenTree { |
94 | TokenTree::Token(Token::new(kind, span)) | |
95 | } | |
96 | ||
0731742a | 97 | /// Returns the opening delimiter as a token tree. |
60c5eb7d XL |
98 | pub fn open_tt(span: DelimSpan, delim: DelimToken) -> TokenTree { |
99 | TokenTree::token(token::OpenDelim(delim), span.open) | |
0731742a XL |
100 | } |
101 | ||
102 | /// Returns the closing delimiter as a token tree. | |
60c5eb7d XL |
103 | pub fn close_tt(span: DelimSpan, delim: DelimToken) -> TokenTree { |
104 | TokenTree::token(token::CloseDelim(delim), span.close) | |
105 | } | |
74b04a01 XL |
106 | |
107 | pub fn uninterpolate(self) -> TokenTree { | |
108 | match self { | |
109 | TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()), | |
110 | tt => tt, | |
111 | } | |
112 | } | |
60c5eb7d XL |
113 | } |
114 | ||
115 | impl<CTX> HashStable<CTX> for TokenStream | |
dfeec247 XL |
116 | where |
117 | CTX: crate::HashStableContext, | |
60c5eb7d XL |
118 | { |
119 | fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { | |
120 | for sub_tt in self.trees() { | |
121 | sub_tt.hash_stable(hcx, hasher); | |
122 | } | |
041b39d2 | 123 | } |
3157f602 XL |
124 | } |
125 | ||
29967ef6 | 126 | pub trait CreateTokenStream: sync::Send + sync::Sync { |
cdc7bbd5 | 127 | fn create_token_stream(&self) -> AttrAnnotatedTokenStream; |
29967ef6 XL |
128 | } |
129 | ||
cdc7bbd5 XL |
130 | impl CreateTokenStream for AttrAnnotatedTokenStream { |
131 | fn create_token_stream(&self) -> AttrAnnotatedTokenStream { | |
29967ef6 XL |
132 | self.clone() |
133 | } | |
134 | } | |
135 | ||
5869c6ff | 136 | /// A lazy version of [`TokenStream`], which defers creation |
29967ef6 XL |
137 | /// of an actual `TokenStream` until it is needed. |
138 | /// `Box` is here only to reduce the structure size. | |
139 | #[derive(Clone)] | |
140 | pub struct LazyTokenStream(Lrc<Box<dyn CreateTokenStream>>); | |
141 | ||
142 | impl LazyTokenStream { | |
143 | pub fn new(inner: impl CreateTokenStream + 'static) -> LazyTokenStream { | |
144 | LazyTokenStream(Lrc::new(Box::new(inner))) | |
145 | } | |
146 | ||
cdc7bbd5 | 147 | pub fn create_token_stream(&self) -> AttrAnnotatedTokenStream { |
29967ef6 XL |
148 | self.0.create_token_stream() |
149 | } | |
150 | } | |
151 | ||
152 | impl fmt::Debug for LazyTokenStream { | |
153 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
cdc7bbd5 | 154 | write!(f, "LazyTokenStream({:?})", self.create_token_stream()) |
29967ef6 XL |
155 | } |
156 | } | |
157 | ||
158 | impl<S: Encoder> Encodable<S> for LazyTokenStream { | |
159 | fn encode(&self, s: &mut S) -> Result<(), S::Error> { | |
160 | // Used by AST json printing. | |
161 | Encodable::encode(&self.create_token_stream(), s) | |
162 | } | |
163 | } | |
164 | ||
165 | impl<D: Decoder> Decodable<D> for LazyTokenStream { | |
5099ac24 | 166 | fn decode(_d: &mut D) -> Self { |
29967ef6 XL |
167 | panic!("Attempted to decode LazyTokenStream"); |
168 | } | |
169 | } | |
170 | ||
171 | impl<CTX> HashStable<CTX> for LazyTokenStream { | |
172 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
173 | panic!("Attempted to compute stable hash for LazyTokenStream"); | |
174 | } | |
175 | } | |
176 | ||
cdc7bbd5 XL |
177 | /// A `AttrAnnotatedTokenStream` is similar to a `TokenStream`, but with extra |
178 | /// information about the tokens for attribute targets. This is used | |
179 | /// during expansion to perform early cfg-expansion, and to process attributes | |
180 | /// during proc-macro invocations. | |
181 | #[derive(Clone, Debug, Default, Encodable, Decodable)] | |
182 | pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>); | |
183 | ||
184 | /// Like `TokenTree`, but for `AttrAnnotatedTokenStream` | |
185 | #[derive(Clone, Debug, Encodable, Decodable)] | |
186 | pub enum AttrAnnotatedTokenTree { | |
187 | Token(Token), | |
188 | Delimited(DelimSpan, DelimToken, AttrAnnotatedTokenStream), | |
189 | /// Stores the attributes for an attribute target, | |
190 | /// along with the tokens for that attribute target. | |
191 | /// See `AttributesData` for more information | |
192 | Attributes(AttributesData), | |
193 | } | |
194 | ||
195 | impl AttrAnnotatedTokenStream { | |
196 | pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream { | |
197 | AttrAnnotatedTokenStream(Lrc::new(tokens)) | |
198 | } | |
199 | ||
200 | /// Converts this `AttrAnnotatedTokenStream` to a plain `TokenStream | |
201 | /// During conversion, `AttrAnnotatedTokenTree::Attributes` get 'flattened' | |
202 | /// back to a `TokenStream` of the form `outer_attr attr_target`. | |
203 | /// If there are inner attributes, they are inserted into the proper | |
204 | /// place in the attribute target tokens. | |
205 | pub fn to_tokenstream(&self) -> TokenStream { | |
206 | let trees: Vec<_> = self | |
207 | .0 | |
208 | .iter() | |
209 | .flat_map(|tree| match &tree.0 { | |
210 | AttrAnnotatedTokenTree::Token(inner) => { | |
211 | smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter() | |
212 | } | |
213 | AttrAnnotatedTokenTree::Delimited(span, delim, stream) => smallvec![( | |
214 | TokenTree::Delimited(*span, *delim, stream.to_tokenstream()), | |
215 | tree.1, | |
216 | )] | |
217 | .into_iter(), | |
218 | AttrAnnotatedTokenTree::Attributes(data) => { | |
219 | let mut outer_attrs = Vec::new(); | |
220 | let mut inner_attrs = Vec::new(); | |
17df50a5 | 221 | for attr in &data.attrs { |
cdc7bbd5 XL |
222 | match attr.style { |
223 | crate::AttrStyle::Outer => { | |
cdc7bbd5 XL |
224 | outer_attrs.push(attr); |
225 | } | |
226 | crate::AttrStyle::Inner => { | |
227 | inner_attrs.push(attr); | |
228 | } | |
229 | } | |
230 | } | |
231 | ||
232 | let mut target_tokens: Vec<_> = data | |
233 | .tokens | |
234 | .create_token_stream() | |
235 | .to_tokenstream() | |
236 | .0 | |
237 | .iter() | |
238 | .cloned() | |
239 | .collect(); | |
240 | if !inner_attrs.is_empty() { | |
241 | let mut found = false; | |
242 | // Check the last two trees (to account for a trailing semi) | |
243 | for (tree, _) in target_tokens.iter_mut().rev().take(2) { | |
244 | if let TokenTree::Delimited(span, delim, delim_tokens) = tree { | |
245 | // Inner attributes are only supported on extern blocks, functions, impls, | |
246 | // and modules. All of these have their inner attributes placed at | |
247 | // the beginning of the rightmost outermost braced group: | |
248 | // e.g. fn foo() { #![my_attr} } | |
249 | // | |
250 | // Therefore, we can insert them back into the right location | |
251 | // without needing to do any extra position tracking. | |
252 | // | |
253 | // Note: Outline modules are an exception - they can | |
254 | // have attributes like `#![my_attr]` at the start of a file. | |
255 | // Support for custom attributes in this position is not | |
256 | // properly implemented - we always synthesize fake tokens, | |
257 | // so we never reach this code. | |
258 | ||
259 | let mut builder = TokenStreamBuilder::new(); | |
17df50a5 | 260 | for inner_attr in inner_attrs { |
cdc7bbd5 XL |
261 | builder.push(inner_attr.tokens().to_tokenstream()); |
262 | } | |
263 | builder.push(delim_tokens.clone()); | |
264 | *tree = TokenTree::Delimited(*span, *delim, builder.build()); | |
265 | found = true; | |
266 | break; | |
267 | } | |
268 | } | |
269 | ||
270 | assert!( | |
271 | found, | |
272 | "Failed to find trailing delimited group in: {:?}", | |
273 | target_tokens | |
274 | ); | |
275 | } | |
276 | let mut flat: SmallVec<[_; 1]> = SmallVec::new(); | |
277 | for attr in outer_attrs { | |
278 | // FIXME: Make this more efficient | |
279 | flat.extend(attr.tokens().to_tokenstream().0.clone().iter().cloned()); | |
280 | } | |
281 | flat.extend(target_tokens); | |
282 | flat.into_iter() | |
283 | } | |
284 | }) | |
285 | .collect(); | |
286 | TokenStream::new(trees) | |
287 | } | |
288 | } | |
289 | ||
290 | /// Stores the tokens for an attribute target, along | |
291 | /// with its attributes. | |
292 | /// | |
293 | /// This is constructed during parsing when we need to capture | |
294 | /// tokens. | |
295 | /// | |
296 | /// For example, `#[cfg(FALSE)] struct Foo {}` would | |
297 | /// have an `attrs` field containing the `#[cfg(FALSE)]` attr, | |
5e7ed085 | 298 | /// and a `tokens` field storing the (unparsed) tokens `struct Foo {}` |
cdc7bbd5 XL |
299 | #[derive(Clone, Debug, Encodable, Decodable)] |
300 | pub struct AttributesData { | |
301 | /// Attributes, both outer and inner. | |
302 | /// These are stored in the original order that they were parsed in. | |
303 | pub attrs: AttrVec, | |
304 | /// The underlying tokens for the attribute target that `attrs` | |
305 | /// are applied to | |
306 | pub tokens: LazyTokenStream, | |
307 | } | |
308 | ||
5869c6ff | 309 | /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. |
e1599b0c | 310 | /// |
32a655c1 SL |
311 | /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s |
312 | /// instead of a representation of the abstract syntax tree. | |
5869c6ff | 313 | /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for |
cdc7bbd5 | 314 | /// backwards compatibility. |
3dfed10e | 315 | #[derive(Clone, Debug, Default, Encodable, Decodable)] |
29967ef6 | 316 | pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>); |
5bcae85e | 317 | |
1b1a35ee | 318 | pub type TreeAndSpacing = (TokenTree, Spacing); |
0731742a XL |
319 | |
320 | // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. | |
6a06907d | 321 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] |
60c5eb7d | 322 | rustc_data_structures::static_assert_size!(TokenStream, 8); |
0731742a | 323 | |
3dfed10e | 324 | #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)] |
1b1a35ee XL |
325 | pub enum Spacing { |
326 | Alone, | |
0731742a | 327 | Joint, |
0731742a XL |
328 | } |
329 | ||
b7449926 XL |
330 | impl TokenStream { |
331 | /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` | |
332 | /// separating the two arguments with a comma for diagnostic suggestions. | |
e74abb32 | 333 | pub fn add_comma(&self) -> Option<(TokenStream, Span)> { |
b7449926 | 334 | // Used to suggest if a user writes `foo!(a b);` |
e74abb32 XL |
335 | let mut suggestion = None; |
336 | let mut iter = self.0.iter().enumerate().peekable(); | |
337 | while let Some((pos, ts)) = iter.next() { | |
338 | if let Some((_, next)) = iter.peek() { | |
339 | let sp = match (&ts, &next) { | |
340 | (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, | |
dfeec247 | 341 | ( |
1b1a35ee | 342 | (TokenTree::Token(token_left), Spacing::Alone), |
dfeec247 XL |
343 | (TokenTree::Token(token_right), _), |
344 | ) if ((token_left.is_ident() && !token_left.is_reserved_ident()) | |
345 | || token_left.is_lit()) | |
346 | && ((token_right.is_ident() && !token_right.is_reserved_ident()) | |
347 | || token_right.is_lit()) => | |
348 | { | |
349 | token_left.span | |
350 | } | |
1b1a35ee | 351 | ((TokenTree::Delimited(sp, ..), Spacing::Alone), _) => sp.entire(), |
e74abb32 XL |
352 | _ => continue, |
353 | }; | |
354 | let sp = sp.shrink_to_hi(); | |
1b1a35ee | 355 | let comma = (TokenTree::token(token::Comma, sp), Spacing::Alone); |
e74abb32 | 356 | suggestion = Some((pos, comma, sp)); |
b7449926 XL |
357 | } |
358 | } | |
e74abb32 | 359 | if let Some((pos, comma, sp)) = suggestion { |
fc512014 | 360 | let mut new_stream = Vec::with_capacity(self.0.len() + 1); |
e74abb32 XL |
361 | let parts = self.0.split_at(pos + 1); |
362 | new_stream.extend_from_slice(parts.0); | |
363 | new_stream.push(comma); | |
364 | new_stream.extend_from_slice(parts.1); | |
365 | return Some((TokenStream::new(new_stream), sp)); | |
366 | } | |
b7449926 XL |
367 | None |
368 | } | |
369 | } | |
370 | ||
cdc7bbd5 XL |
371 | impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream { |
372 | fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream { | |
373 | AttrAnnotatedTokenStream::new(vec![(tree, spacing)]) | |
374 | } | |
375 | } | |
376 | ||
0731742a XL |
377 | impl From<TokenTree> for TokenStream { |
378 | fn from(tree: TokenTree) -> TokenStream { | |
1b1a35ee | 379 | TokenStream::new(vec![(tree, Spacing::Alone)]) |
0731742a | 380 | } |
5bcae85e SL |
381 | } |
382 | ||
1b1a35ee XL |
383 | impl From<TokenTree> for TreeAndSpacing { |
384 | fn from(tree: TokenTree) -> TreeAndSpacing { | |
385 | (tree, Spacing::Alone) | |
5bcae85e SL |
386 | } |
387 | } | |
388 | ||
e74abb32 XL |
389 | impl iter::FromIterator<TokenTree> for TokenStream { |
390 | fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self { | |
1b1a35ee | 391 | TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndSpacing>>()) |
b7449926 XL |
392 | } |
393 | } | |
394 | ||
32a655c1 SL |
395 | impl Eq for TokenStream {} |
396 | ||
5bcae85e SL |
397 | impl PartialEq<TokenStream> for TokenStream { |
398 | fn eq(&self, other: &TokenStream) -> bool { | |
32a655c1 | 399 | self.trees().eq(other.trees()) |
5bcae85e SL |
400 | } |
401 | } | |
402 | ||
5bcae85e | 403 | impl TokenStream { |
1b1a35ee | 404 | pub fn new(streams: Vec<TreeAndSpacing>) -> TokenStream { |
e74abb32 | 405 | TokenStream(Lrc::new(streams)) |
3b2f2976 XL |
406 | } |
407 | ||
e74abb32 XL |
408 | pub fn is_empty(&self) -> bool { |
409 | self.0.is_empty() | |
9e0c209e SL |
410 | } |
411 | ||
e74abb32 XL |
412 | pub fn len(&self) -> usize { |
413 | self.0.len() | |
5bcae85e SL |
414 | } |
415 | ||
60c5eb7d | 416 | pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { |
8bb4bdeb | 417 | match streams.len() { |
e74abb32 | 418 | 0 => TokenStream::default(), |
041b39d2 | 419 | 1 => streams.pop().unwrap(), |
0731742a | 420 | _ => { |
e74abb32 XL |
421 | // We are going to extend the first stream in `streams` with |
422 | // the elements from the subsequent streams. This requires | |
423 | // using `make_mut()` on the first stream, and in practice this | |
424 | // doesn't cause cloning 99.9% of the time. | |
425 | // | |
426 | // One very common use case is when `streams` has two elements, | |
427 | // where the first stream has any number of elements within | |
428 | // (often 1, but sometimes many more) and the second stream has | |
429 | // a single element within. | |
430 | ||
431 | // Determine how much the first stream will be extended. | |
432 | // Needed to avoid quadratic blow up from on-the-fly | |
433 | // reallocations (#57735). | |
dfeec247 | 434 | let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum(); |
9fa01778 | 435 | |
e74abb32 XL |
436 | // Get the first stream. If it's `None`, create an empty |
437 | // stream. | |
60c5eb7d | 438 | let mut iter = streams.drain(..); |
e74abb32 XL |
439 | let mut first_stream_lrc = iter.next().unwrap().0; |
440 | ||
441 | // Append the elements to the first stream, after reserving | |
442 | // space for them. | |
443 | let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); | |
444 | first_vec_mut.reserve(num_appends); | |
445 | for stream in iter { | |
446 | first_vec_mut.extend(stream.0.iter().cloned()); | |
0731742a | 447 | } |
8bb4bdeb | 448 | |
e74abb32 XL |
449 | // Create the final `TokenStream`. |
450 | TokenStream(first_stream_lrc) | |
451 | } | |
0731742a | 452 | } |
8bb4bdeb XL |
453 | } |
454 | ||
455 | pub fn trees(&self) -> Cursor { | |
456 | self.clone().into_trees() | |
5bcae85e SL |
457 | } |
458 | ||
8bb4bdeb | 459 | pub fn into_trees(self) -> Cursor { |
32a655c1 | 460 | Cursor::new(self) |
5bcae85e SL |
461 | } |
462 | ||
e1599b0c | 463 | /// Compares two `TokenStream`s, checking equality without regarding span information. |
5bcae85e | 464 | pub fn eq_unspanned(&self, other: &TokenStream) -> bool { |
83c7162d XL |
465 | let mut t1 = self.trees(); |
466 | let mut t2 = other.trees(); | |
cdc7bbd5 | 467 | for (t1, t2) in iter::zip(&mut t1, &mut t2) { |
8bb4bdeb | 468 | if !t1.eq_unspanned(&t2) { |
5bcae85e SL |
469 | return false; |
470 | } | |
471 | } | |
83c7162d XL |
472 | t1.next().is_none() && t2.next().is_none() |
473 | } | |
474 | ||
29967ef6 | 475 | pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { |
e74abb32 XL |
476 | TokenStream(Lrc::new( |
477 | self.0 | |
478 | .iter() | |
479 | .enumerate() | |
29967ef6 | 480 | .map(|(i, (tree, is_joint))| (f(i, tree), *is_joint)) |
dfeec247 | 481 | .collect(), |
e74abb32 | 482 | )) |
3b2f2976 | 483 | } |
041b39d2 XL |
484 | } |
485 | ||
532ac7d7 | 486 | // 99.5%+ of the time we have 1 or 2 elements in this vector. |
8faf50e0 | 487 | #[derive(Clone)] |
532ac7d7 | 488 | pub struct TokenStreamBuilder(SmallVec<[TokenStream; 2]>); |
041b39d2 XL |
489 | |
490 | impl TokenStreamBuilder { | |
491 | pub fn new() -> TokenStreamBuilder { | |
532ac7d7 | 492 | TokenStreamBuilder(SmallVec::new()) |
041b39d2 XL |
493 | } |
494 | ||
495 | pub fn push<T: Into<TokenStream>>(&mut self, stream: T) { | |
e74abb32 XL |
496 | let mut stream = stream.into(); |
497 | ||
498 | // If `self` is not empty and the last tree within the last stream is a | |
499 | // token tree marked with `Joint`... | |
5e7ed085 FG |
500 | if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() |
501 | && let Some((TokenTree::Token(last_token), Spacing::Joint)) = last_stream_lrc.last() | |
502 | // ...and `stream` is not empty and the first tree within it is | |
503 | // a token tree... | |
504 | && let TokenStream(ref mut stream_lrc) = stream | |
505 | && let Some((TokenTree::Token(token), spacing)) = stream_lrc.first() | |
506 | // ...and the two tokens can be glued together... | |
507 | && let Some(glued_tok) = last_token.glue(&token) | |
508 | { | |
509 | // ...then do so, by overwriting the last token | |
510 | // tree in `self` and removing the first token tree | |
511 | // from `stream`. This requires using `make_mut()` | |
512 | // on the last stream in `self` and on `stream`, | |
513 | // and in practice this doesn't cause cloning 99.9% | |
514 | // of the time. | |
515 | ||
516 | // Overwrite the last token tree with the merged | |
517 | // token. | |
518 | let last_vec_mut = Lrc::make_mut(last_stream_lrc); | |
519 | *last_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); | |
520 | ||
521 | // Remove the first token tree from `stream`. (This | |
522 | // is almost always the only tree in `stream`.) | |
523 | let stream_vec_mut = Lrc::make_mut(stream_lrc); | |
524 | stream_vec_mut.remove(0); | |
525 | ||
526 | // Don't push `stream` if it's empty -- that could | |
527 | // block subsequent token gluing, by getting | |
528 | // between two token trees that should be glued | |
529 | // together. | |
530 | if !stream.is_empty() { | |
531 | self.0.push(stream); | |
041b39d2 | 532 | } |
5e7ed085 | 533 | return; |
041b39d2 XL |
534 | } |
535 | self.0.push(stream); | |
536 | } | |
537 | ||
041b39d2 | 538 | pub fn build(self) -> TokenStream { |
0731742a | 539 | TokenStream::from_streams(self.0) |
041b39d2 | 540 | } |
5bcae85e SL |
541 | } |
542 | ||
5869c6ff | 543 | /// By-reference iterator over a [`TokenStream`]. |
29967ef6 XL |
544 | #[derive(Clone)] |
545 | pub struct CursorRef<'t> { | |
546 | stream: &'t TokenStream, | |
547 | index: usize, | |
548 | } | |
549 | ||
550 | impl<'t> CursorRef<'t> { | |
29967ef6 XL |
551 | fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> { |
552 | self.stream.0.get(self.index).map(|tree| { | |
553 | self.index += 1; | |
554 | tree | |
555 | }) | |
556 | } | |
557 | } | |
558 | ||
559 | impl<'t> Iterator for CursorRef<'t> { | |
560 | type Item = &'t TokenTree; | |
561 | ||
562 | fn next(&mut self) -> Option<&'t TokenTree> { | |
563 | self.next_with_spacing().map(|(tree, _)| tree) | |
564 | } | |
565 | } | |
566 | ||
5869c6ff XL |
567 | /// Owning by-value iterator over a [`TokenStream`]. |
568 | // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones. | |
041b39d2 | 569 | #[derive(Clone)] |
0731742a XL |
570 | pub struct Cursor { |
571 | pub stream: TokenStream, | |
8bb4bdeb | 572 | index: usize, |
041b39d2 XL |
573 | } |
574 | ||
575 | impl Iterator for Cursor { | |
576 | type Item = TokenTree; | |
577 | ||
578 | fn next(&mut self) -> Option<TokenTree> { | |
1b1a35ee | 579 | self.next_with_spacing().map(|(tree, _)| tree) |
041b39d2 | 580 | } |
32a655c1 | 581 | } |
5bcae85e | 582 | |
8bb4bdeb XL |
583 | impl Cursor { |
584 | fn new(stream: TokenStream) -> Self { | |
0731742a XL |
585 | Cursor { stream, index: 0 } |
586 | } | |
587 | ||
1b1a35ee | 588 | pub fn next_with_spacing(&mut self) -> Option<TreeAndSpacing> { |
e74abb32 XL |
589 | if self.index < self.stream.len() { |
590 | self.index += 1; | |
591 | Some(self.stream.0[self.index - 1].clone()) | |
592 | } else { | |
593 | None | |
041b39d2 XL |
594 | } |
595 | } | |
596 | ||
cdc7bbd5 XL |
597 | pub fn index(&self) -> usize { |
598 | self.index | |
599 | } | |
600 | ||
0731742a XL |
601 | pub fn append(&mut self, new_stream: TokenStream) { |
602 | if new_stream.is_empty() { | |
603 | return; | |
5bcae85e | 604 | } |
0731742a | 605 | let index = self.index; |
e74abb32 | 606 | let stream = mem::take(&mut self.stream); |
532ac7d7 | 607 | *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees(); |
0731742a | 608 | self.index = index; |
8bb4bdeb XL |
609 | } |
610 | ||
29967ef6 XL |
611 | pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { |
612 | self.stream.0[self.index..].get(n).map(|(tree, _)| tree) | |
5bcae85e SL |
613 | } |
614 | } | |
615 | ||
3dfed10e | 616 | #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] |
b7449926 XL |
617 | pub struct DelimSpan { |
618 | pub open: Span, | |
619 | pub close: Span, | |
620 | } | |
621 | ||
622 | impl DelimSpan { | |
623 | pub fn from_single(sp: Span) -> Self { | |
dfeec247 | 624 | DelimSpan { open: sp, close: sp } |
b7449926 XL |
625 | } |
626 | ||
627 | pub fn from_pair(open: Span, close: Span) -> Self { | |
628 | DelimSpan { open, close } | |
629 | } | |
630 | ||
631 | pub fn dummy() -> Self { | |
632 | Self::from_single(DUMMY_SP) | |
633 | } | |
634 | ||
635 | pub fn entire(self) -> Span { | |
636 | self.open.with_hi(self.close.hi()) | |
637 | } | |
b7449926 | 638 | } |