1 // Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
13 //! `TokenStream`s represent syntactic objects before they are converted into ASTs.
14 //! A `TokenStream` is, roughly speaking, a sequence (eg stream) of `TokenTree`s,
15 //! which are themselves a single `Token` or a `Delimited` subsequence of tokens.
18 //! `TokenStreams` are persistent data structures constructed as ropes with reference
19 //! counted-children. In general, this means that calling an operation on a `TokenStream`
20 //! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to
21 //! the original. This essentially coerces `TokenStream`s into 'views' of their subparts,
22 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
23 //! ownership of the original.
25 use syntax_pos
::{BytePos, Span, DUMMY_SP}
;
27 use ext
::tt
::{macro_parser, quoted}
;
29 use parse
::token
::{self, Token}
;
31 use serialize
::{Decoder, Decodable, Encoder, Encodable}
;
35 use std
::{fmt, iter, mem}
;
36 use std
::hash
::{self, Hash}
;
38 /// A delimited sequence of token trees
39 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
40 pub struct Delimited
{
41 /// The type of delimiter
42 pub delim
: token
::DelimToken
,
43 /// The delimited sequence of token trees
44 pub tts
: ThinTokenStream
,
48 /// Returns the opening delimiter as a token.
49 pub fn open_token(&self) -> token
::Token
{
50 token
::OpenDelim(self.delim
)
53 /// Returns the closing delimiter as a token.
54 pub fn close_token(&self) -> token
::Token
{
55 token
::CloseDelim(self.delim
)
58 /// Returns the opening delimiter as a token tree.
59 pub fn open_tt(&self, span
: Span
) -> TokenTree
{
60 let open_span
= if span
== DUMMY_SP
{
63 span
.with_hi(span
.lo() + BytePos(self.delim
.len() as u32))
65 TokenTree
::Token(open_span
, self.open_token())
68 /// Returns the closing delimiter as a token tree.
69 pub fn close_tt(&self, span
: Span
) -> TokenTree
{
70 let close_span
= if span
== DUMMY_SP
{
73 span
.with_lo(span
.hi() - BytePos(self.delim
.len() as u32))
75 TokenTree
::Token(close_span
, self.close_token())
78 /// Returns the token trees inside the delimiters.
79 pub fn stream(&self) -> TokenStream
{
80 self.tts
.clone().into()
84 /// When the main rust parser encounters a syntax-extension invocation, it
85 /// parses the arguments to the invocation as a token-tree. This is a very
86 /// loose structure, such that all sorts of different AST-fragments can
87 /// be passed to syntax extensions using a uniform type.
89 /// If the syntax extension is an MBE macro, it will attempt to match its
90 /// LHS token tree against the provided token tree, and if it finds a
91 /// match, will transcribe the RHS token tree, splicing in any captured
92 /// `macro_parser::matched_nonterminals` into the `SubstNt`s it finds.
94 /// The RHS of an MBE macro is the only place `SubstNt`s are substituted.
95 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
96 #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)]
99 Token(Span
, token
::Token
),
100 /// A delimited sequence of token trees
101 Delimited(Span
, Delimited
),
105 /// Use this token tree as a matcher to parse given tts.
106 pub fn parse(cx
: &base
::ExtCtxt
, mtch
: &[quoted
::TokenTree
], tts
: TokenStream
)
107 -> macro_parser
::NamedParseResult
{
108 // `None` is because we're not interpolating
109 let directory
= Directory
{
110 path
: Cow
::from(cx
.current_expansion
.module
.directory
.as_path()),
111 ownership
: cx
.current_expansion
.directory_ownership
,
113 macro_parser
::parse(cx
.parse_sess(), tts
, mtch
, Some(directory
), true)
116 /// Check if this TokenTree is equal to the other, regardless of span information.
117 pub fn eq_unspanned(&self, other
: &TokenTree
) -> bool
{
118 match (self, other
) {
119 (&TokenTree
::Token(_
, ref tk
), &TokenTree
::Token(_
, ref tk2
)) => tk
== tk2
,
120 (&TokenTree
::Delimited(_
, ref dl
), &TokenTree
::Delimited(_
, ref dl2
)) => {
121 dl
.delim
== dl2
.delim
&&
122 dl
.stream().eq_unspanned(&dl2
.stream())
128 // See comments in `interpolated_to_tokenstream` for why we care about
129 // *probably* equal here rather than actual equality
131 // This is otherwise the same as `eq_unspanned`, only recursing with a
133 pub fn probably_equal_for_proc_macro(&self, other
: &TokenTree
) -> bool
{
134 match (self, other
) {
135 (&TokenTree
::Token(_
, ref tk
), &TokenTree
::Token(_
, ref tk2
)) => {
136 tk
.probably_equal_for_proc_macro(tk2
)
138 (&TokenTree
::Delimited(_
, ref dl
), &TokenTree
::Delimited(_
, ref dl2
)) => {
139 dl
.delim
== dl2
.delim
&&
140 dl
.stream().probably_equal_for_proc_macro(&dl2
.stream())
146 /// Retrieve the TokenTree's span.
147 pub fn span(&self) -> Span
{
149 TokenTree
::Token(sp
, _
) | TokenTree
::Delimited(sp
, _
) => sp
,
153 /// Modify the `TokenTree`'s span inplace.
154 pub fn set_span(&mut self, span
: Span
) {
156 TokenTree
::Token(ref mut sp
, _
) | TokenTree
::Delimited(ref mut sp
, _
) => {
162 /// Indicates if the stream is a token that is equal to the provided token.
163 pub fn eq_token(&self, t
: Token
) -> bool
{
165 TokenTree
::Token(_
, ref tk
) => *tk
== t
,
170 pub fn joint(self) -> TokenStream
{
171 TokenStream { kind: TokenStreamKind::JointTree(self) }
177 /// A `TokenStream` is an abstract sequence of tokens, organized into `TokenTree`s.
178 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
179 /// instead of a representation of the abstract syntax tree.
180 /// Today's `TokenTree`s can still contain AST via `Token::Interpolated` for back-compat.
181 #[derive(Clone, Debug)]
182 pub struct TokenStream
{
183 kind
: TokenStreamKind
,
186 #[derive(Clone, Debug)]
187 enum TokenStreamKind
{
190 JointTree(TokenTree
),
191 Stream(RcSlice
<TokenStream
>),
194 impl From
<TokenTree
> for TokenStream
{
195 fn from(tt
: TokenTree
) -> TokenStream
{
196 TokenStream { kind: TokenStreamKind::Tree(tt) }
200 impl From
<Token
> for TokenStream
{
201 fn from(token
: Token
) -> TokenStream
{
202 TokenTree
::Token(DUMMY_SP
, token
).into()
206 impl<T
: Into
<TokenStream
>> iter
::FromIterator
<T
> for TokenStream
{
207 fn from_iter
<I
: IntoIterator
<Item
= T
>>(iter
: I
) -> Self {
208 TokenStream
::concat(iter
.into_iter().map(Into
::into
).collect
::<Vec
<_
>>())
212 impl Eq
for TokenStream {}
214 impl PartialEq
<TokenStream
> for TokenStream
{
215 fn eq(&self, other
: &TokenStream
) -> bool
{
216 self.trees().eq(other
.trees())
221 pub fn len(&self) -> usize {
222 if let TokenStreamKind
::Stream(ref slice
) = self.kind
{
229 pub fn empty() -> TokenStream
{
230 TokenStream { kind: TokenStreamKind::Empty }
233 pub fn is_empty(&self) -> bool
{
235 TokenStreamKind
::Empty
=> true,
240 pub fn concat(mut streams
: Vec
<TokenStream
>) -> TokenStream
{
241 match streams
.len() {
242 0 => TokenStream
::empty(),
243 1 => streams
.pop().unwrap(),
244 _
=> TokenStream
::concat_rc_slice(RcSlice
::new(streams
)),
248 fn concat_rc_slice(streams
: RcSlice
<TokenStream
>) -> TokenStream
{
249 TokenStream { kind: TokenStreamKind::Stream(streams) }
252 pub fn trees(&self) -> Cursor
{
253 self.clone().into_trees()
256 pub fn into_trees(self) -> Cursor
{
260 /// Compares two TokenStreams, checking equality without regarding span information.
261 pub fn eq_unspanned(&self, other
: &TokenStream
) -> bool
{
262 let mut t1
= self.trees();
263 let mut t2
= other
.trees();
264 for (t1
, t2
) in t1
.by_ref().zip(t2
.by_ref()) {
265 if !t1
.eq_unspanned(&t2
) {
269 t1
.next().is_none() && t2
.next().is_none()
272 // See comments in `interpolated_to_tokenstream` for why we care about
273 // *probably* equal here rather than actual equality
275 // This is otherwise the same as `eq_unspanned`, only recursing with a
277 pub fn probably_equal_for_proc_macro(&self, other
: &TokenStream
) -> bool
{
278 let mut t1
= self.trees();
279 let mut t2
= other
.trees();
280 for (t1
, t2
) in t1
.by_ref().zip(t2
.by_ref()) {
281 if !t1
.probably_equal_for_proc_macro(&t2
) {
285 t1
.next().is_none() && t2
.next().is_none()
288 /// Precondition: `self` consists of a single token tree.
289 /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
290 pub fn as_tree(self) -> (TokenTree
, bool
/* joint? */) {
292 TokenStreamKind
::Tree(tree
) => (tree
, false),
293 TokenStreamKind
::JointTree(tree
) => (tree
, true),
298 pub fn map_enumerated
<F
: FnMut(usize, TokenTree
) -> TokenTree
>(self, mut f
: F
) -> TokenStream
{
299 let mut trees
= self.into_trees();
300 let mut result
= Vec
::new();
302 while let Some(stream
) = trees
.next_as_stream() {
303 result
.push(match stream
.kind
{
304 TokenStreamKind
::Tree(tree
) => f(i
, tree
).into(),
305 TokenStreamKind
::JointTree(tree
) => f(i
, tree
).joint(),
310 TokenStream
::concat(result
)
313 pub fn map
<F
: FnMut(TokenTree
) -> TokenTree
>(self, mut f
: F
) -> TokenStream
{
314 let mut trees
= self.into_trees();
315 let mut result
= Vec
::new();
316 while let Some(stream
) = trees
.next_as_stream() {
317 result
.push(match stream
.kind
{
318 TokenStreamKind
::Tree(tree
) => f(tree
).into(),
319 TokenStreamKind
::JointTree(tree
) => f(tree
).joint(),
323 TokenStream
::concat(result
)
326 fn first_tree_and_joint(&self) -> Option
<(TokenTree
, bool
)> {
328 TokenStreamKind
::Empty
=> None
,
329 TokenStreamKind
::Tree(ref tree
) => Some((tree
.clone(), false)),
330 TokenStreamKind
::JointTree(ref tree
) => Some((tree
.clone(), true)),
331 TokenStreamKind
::Stream(ref stream
) => stream
.first().unwrap().first_tree_and_joint(),
335 fn last_tree_if_joint(&self) -> Option
<TokenTree
> {
337 TokenStreamKind
::Empty
| TokenStreamKind
::Tree(..) => None
,
338 TokenStreamKind
::JointTree(ref tree
) => Some(tree
.clone()),
339 TokenStreamKind
::Stream(ref stream
) => stream
.last().unwrap().last_tree_if_joint(),
344 pub struct TokenStreamBuilder(Vec
<TokenStream
>);
346 impl TokenStreamBuilder
{
347 pub fn new() -> TokenStreamBuilder
{
348 TokenStreamBuilder(Vec
::new())
351 pub fn push
<T
: Into
<TokenStream
>>(&mut self, stream
: T
) {
352 let stream
= stream
.into();
353 let last_tree_if_joint
= self.0.last().and_then(TokenStream
::last_tree_if_joint
);
354 if let Some(TokenTree
::Token(last_span
, last_tok
)) = last_tree_if_joint
{
355 if let Some((TokenTree
::Token(span
, tok
), is_joint
)) = stream
.first_tree_and_joint() {
356 if let Some(glued_tok
) = last_tok
.glue(tok
) {
357 let last_stream
= self.0.pop().unwrap();
358 self.push_all_but_last_tree(&last_stream
);
359 let glued_span
= last_span
.to(span
);
360 let glued_tt
= TokenTree
::Token(glued_span
, glued_tok
);
361 let glued_tokenstream
= if is_joint
{
366 self.0.push(glued_tokenstream
);
367 self.push_all_but_first_tree(&stream
);
375 pub fn add
<T
: Into
<TokenStream
>>(mut self, stream
: T
) -> Self {
380 pub fn build(self) -> TokenStream
{
381 TokenStream
::concat(self.0)
384 fn push_all_but_last_tree(&mut self, stream
: &TokenStream
) {
385 if let TokenStreamKind
::Stream(ref streams
) = stream
.kind
{
386 let len
= streams
.len();
389 2 => self.0.push(streams
[0].clone().into()),
390 _
=> self.0.push(TokenStream
::concat_rc_slice(streams
.sub_slice(0 .. len
- 1))),
392 self.push_all_but_last_tree(&streams
[len
- 1])
396 fn push_all_but_first_tree(&mut self, stream
: &TokenStream
) {
397 if let TokenStreamKind
::Stream(ref streams
) = stream
.kind
{
398 let len
= streams
.len();
401 2 => self.0.push(streams
[1].clone().into()),
402 _
=> self.0.push(TokenStream
::concat_rc_slice(streams
.sub_slice(1 .. len
))),
404 self.push_all_but_first_tree(&streams
[0])
410 pub struct Cursor(CursorKind
);
415 Tree(TokenTree
, bool
/* consumed? */),
416 JointTree(TokenTree
, bool
/* consumed? */),
417 Stream(StreamCursor
),
421 struct StreamCursor
{
422 stream
: RcSlice
<TokenStream
>,
424 stack
: Vec
<(RcSlice
<TokenStream
>, usize)>,
428 fn new(stream
: RcSlice
<TokenStream
>) -> Self {
429 StreamCursor { stream: stream, index: 0, stack: Vec::new() }
432 fn next_as_stream(&mut self) -> Option
<TokenStream
> {
434 if self.index
< self.stream
.len() {
436 let next
= self.stream
[self.index
- 1].clone();
438 TokenStreamKind
::Tree(..) | TokenStreamKind
::JointTree(..) => return Some(next
),
439 TokenStreamKind
::Stream(stream
) => self.insert(stream
),
440 TokenStreamKind
::Empty
=> {}
442 } else if let Some((stream
, index
)) = self.stack
.pop() {
443 self.stream
= stream
;
451 fn insert(&mut self, stream
: RcSlice
<TokenStream
>) {
452 self.stack
.push((mem
::replace(&mut self.stream
, stream
),
453 mem
::replace(&mut self.index
, 0)));
457 impl Iterator
for Cursor
{
458 type Item
= TokenTree
;
460 fn next(&mut self) -> Option
<TokenTree
> {
461 self.next_as_stream().map(|stream
| match stream
.kind
{
462 TokenStreamKind
::Tree(tree
) | TokenStreamKind
::JointTree(tree
) => tree
,
469 fn new(stream
: TokenStream
) -> Self {
470 Cursor(match stream
.kind
{
471 TokenStreamKind
::Empty
=> CursorKind
::Empty
,
472 TokenStreamKind
::Tree(tree
) => CursorKind
::Tree(tree
, false),
473 TokenStreamKind
::JointTree(tree
) => CursorKind
::JointTree(tree
, false),
474 TokenStreamKind
::Stream(stream
) => CursorKind
::Stream(StreamCursor
::new(stream
)),
478 pub fn next_as_stream(&mut self) -> Option
<TokenStream
> {
479 let (stream
, consumed
) = match self.0 {
480 CursorKind
::Tree(ref tree
, ref mut consumed @
false) =>
481 (tree
.clone().into(), consumed
),
482 CursorKind
::JointTree(ref tree
, ref mut consumed @
false) =>
483 (tree
.clone().joint(), consumed
),
484 CursorKind
::Stream(ref mut cursor
) => return cursor
.next_as_stream(),
492 pub fn insert(&mut self, stream
: TokenStream
) {
494 _
if stream
.is_empty() => return,
495 CursorKind
::Empty
=> *self = stream
.trees(),
496 CursorKind
::Tree(_
, consumed
) | CursorKind
::JointTree(_
, consumed
) => {
497 *self = TokenStream
::concat(vec
![self.original_stream(), stream
]).trees();
502 CursorKind
::Stream(ref mut cursor
) => {
503 cursor
.insert(ThinTokenStream
::from(stream
).0.unwrap
());
508 pub fn original_stream(&self) -> TokenStream
{
510 CursorKind
::Empty
=> TokenStream
::empty(),
511 CursorKind
::Tree(ref tree
, _
) => tree
.clone().into(),
512 CursorKind
::JointTree(ref tree
, _
) => tree
.clone().joint(),
513 CursorKind
::Stream(ref cursor
) => TokenStream
::concat_rc_slice({
514 cursor
.stack
.get(0).cloned().map(|(stream
, _
)| stream
)
515 .unwrap_or(cursor
.stream
.clone())
520 pub fn look_ahead(&self, n
: usize) -> Option
<TokenTree
> {
521 fn look_ahead(streams
: &[TokenStream
], mut n
: usize) -> Result
<TokenTree
, usize> {
522 for stream
in streams
{
523 n
= match stream
.kind
{
524 TokenStreamKind
::Tree(ref tree
) | TokenStreamKind
::JointTree(ref tree
)
525 if n
== 0 => return Ok(tree
.clone()),
526 TokenStreamKind
::Tree(..) | TokenStreamKind
::JointTree(..) => n
- 1,
527 TokenStreamKind
::Stream(ref stream
) => match look_ahead(stream
, n
) {
528 Ok(tree
) => return Ok(tree
),
539 CursorKind
::Tree(_
, true) |
540 CursorKind
::JointTree(_
, true) => Err(n
),
541 CursorKind
::Tree(ref tree
, false) |
542 CursorKind
::JointTree(ref tree
, false) => look_ahead(&[tree
.clone().into()], n
),
543 CursorKind
::Stream(ref cursor
) => {
544 look_ahead(&cursor
.stream
[cursor
.index
..], n
).or_else(|mut n
| {
545 for &(ref stream
, index
) in cursor
.stack
.iter().rev() {
546 n
= match look_ahead(&stream
[index
..], n
) {
547 Ok(tree
) => return Ok(tree
),
559 /// The `TokenStream` type is large enough to represent a single `TokenTree` without allocation.
560 /// `ThinTokenStream` is smaller, but needs to allocate to represent a single `TokenTree`.
561 /// We must use `ThinTokenStream` in `TokenTree::Delimited` to avoid infinite size due to recursion.
562 #[derive(Debug, Clone)]
563 pub struct ThinTokenStream(Option
<RcSlice
<TokenStream
>>);
565 impl From
<TokenStream
> for ThinTokenStream
{
566 fn from(stream
: TokenStream
) -> ThinTokenStream
{
567 ThinTokenStream(match stream
.kind
{
568 TokenStreamKind
::Empty
=> None
,
569 TokenStreamKind
::Tree(tree
) => Some(RcSlice
::new(vec
![tree
.into()])),
570 TokenStreamKind
::JointTree(tree
) => Some(RcSlice
::new(vec
![tree
.joint()])),
571 TokenStreamKind
::Stream(stream
) => Some(stream
),
576 impl From
<ThinTokenStream
> for TokenStream
{
577 fn from(stream
: ThinTokenStream
) -> TokenStream
{
578 stream
.0.map(TokenStream
::concat_rc_slice
).unwrap_or_else(TokenStream
::empty
)
582 impl Eq
for ThinTokenStream {}
584 impl PartialEq
<ThinTokenStream
> for ThinTokenStream
{
585 fn eq(&self, other
: &ThinTokenStream
) -> bool
{
586 TokenStream
::from(self.clone()) == TokenStream
::from(other
.clone())
590 impl fmt
::Display
for TokenStream
{
591 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
592 f
.write_str(&pprust
::tokens_to_string(self.clone()))
596 impl Encodable
for TokenStream
{
597 fn encode
<E
: Encoder
>(&self, encoder
: &mut E
) -> Result
<(), E
::Error
> {
598 self.trees().collect
::<Vec
<_
>>().encode(encoder
)
602 impl Decodable
for TokenStream
{
603 fn decode
<D
: Decoder
>(decoder
: &mut D
) -> Result
<TokenStream
, D
::Error
> {
604 Vec
::<TokenTree
>::decode(decoder
).map(|vec
| vec
.into_iter().collect())
608 impl Hash
for TokenStream
{
609 fn hash
<H
: hash
::Hasher
>(&self, state
: &mut H
) {
610 for tree
in self.trees() {
616 impl Encodable
for ThinTokenStream
{
617 fn encode
<E
: Encoder
>(&self, encoder
: &mut E
) -> Result
<(), E
::Error
> {
618 TokenStream
::from(self.clone()).encode(encoder
)
622 impl Decodable
for ThinTokenStream
{
623 fn decode
<D
: Decoder
>(decoder
: &mut D
) -> Result
<ThinTokenStream
, D
::Error
> {
624 TokenStream
::decode(decoder
).map(Into
::into
)
628 impl Hash
for ThinTokenStream
{
629 fn hash
<H
: hash
::Hasher
>(&self, state
: &mut H
) {
630 TokenStream
::from(self.clone()).hash(state
);
638 use syntax
::ast
::Ident
;
640 use syntax_pos
::{Span, BytePos, NO_EXPANSION}
;
641 use parse
::token
::Token
;
642 use util
::parser_testing
::string_to_stream
;
644 fn string_to_ts(string
: &str) -> TokenStream
{
645 string_to_stream(string
.to_owned())
648 fn sp(a
: u32, b
: u32) -> Span
{
649 Span
::new(BytePos(a
), BytePos(b
), NO_EXPANSION
)
655 let test_res
= string_to_ts("foo::bar::baz");
656 let test_fst
= string_to_ts("foo::bar");
657 let test_snd
= string_to_ts("::baz");
658 let eq_res
= TokenStream
::concat(vec
![test_fst
, test_snd
]);
659 assert_eq
!(test_res
.trees().count(), 5);
660 assert_eq
!(eq_res
.trees().count(), 5);
661 assert_eq
!(test_res
.eq_unspanned(&eq_res
), true);
666 fn test_to_from_bijection() {
668 let test_start
= string_to_ts("foo::bar(baz)");
669 let test_end
= test_start
.trees().collect();
670 assert_eq
!(test_start
, test_end
)
677 let test_res
= string_to_ts("foo");
678 let test_eqs
= string_to_ts("foo");
679 assert_eq
!(test_res
, test_eqs
)
686 let test_res
= string_to_ts("::bar::baz");
687 let test_eqs
= string_to_ts("::bar::baz");
688 assert_eq
!(test_res
, test_eqs
)
695 let test_res
= string_to_ts("");
696 let test_eqs
= string_to_ts("");
697 assert_eq
!(test_res
, test_eqs
)
704 let test_res
= string_to_ts("::bar::baz");
705 let test_eqs
= string_to_ts("bar::baz");
706 assert_eq
!(test_res
== test_eqs
, false)
713 let test_res
= string_to_ts("(bar,baz)");
714 let test_eqs
= string_to_ts("bar,baz");
715 assert_eq
!(test_res
== test_eqs
, false)
722 let test0
: TokenStream
= Vec
::<TokenTree
>::new().into_iter().collect();
723 let test1
: TokenStream
=
724 TokenTree
::Token(sp(0, 1), Token
::Ident(Ident
::from_str("a"), false)).into();
725 let test2
= string_to_ts("foo(bar::baz)");
727 assert_eq
!(test0
.is_empty(), true);
728 assert_eq
!(test1
.is_empty(), false);
729 assert_eq
!(test2
.is_empty(), false);
734 fn test_dotdotdot() {
735 let mut builder
= TokenStreamBuilder
::new();
736 builder
.push(TokenTree
::Token(sp(0, 1), Token
::Dot
).joint());
737 builder
.push(TokenTree
::Token(sp(1, 2), Token
::Dot
).joint());
738 builder
.push(TokenTree
::Token(sp(2, 3), Token
::Dot
));
739 let stream
= builder
.build();
740 assert
!(stream
.eq_unspanned(&string_to_ts("...")));
741 assert_eq
!(stream
.trees().count(), 1);