New upstream version 1.51.0+dfsg1

[rustc.git] / compiler / rustc_ast / src / tokenstream.rs

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index b2207f22816205f0c11dc16fa5ae7c21da3a65aa..9ac05f316f0346e84a12e0882dc761c5d469441d 100644 (file)
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -1,15 +1,15 @@
 //! # Token Streams
 //!
 //! `TokenStream`s represent syntactic objects before they are converted into ASTs.
-//! A `TokenStream` is, roughly speaking, a sequence (eg stream) of `TokenTree`s,
-//! which are themselves a single `Token` or a `Delimited` subsequence of tokens.
+//! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s,
+//! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens.
 //!
 //! ## Ownership
 //!
 //! `TokenStream`s are persistent data structures constructed as ropes with reference
 //! counted-children. In general, this means that calling an operation on a `TokenStream`
 //! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to
-//! the original. This essentially coerces `TokenStream`s into 'views' of their subparts,
+//! the original. This essentially coerces `TokenStream`s into "views" of their subparts,
 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
 //! ownership of the original.
 
@@ -24,9 +24,9 @@ use smallvec::{smallvec, SmallVec};
 
 use std::{fmt, iter, mem};
 
-/// When the main rust parser encounters a syntax-extension invocation, it
-/// parses the arguments to the invocation as a token-tree. This is a very
-/// loose structure, such that all sorts of different AST-fragments can
+/// When the main Rust parser encounters a syntax-extension invocation, it
+/// parses the arguments to the invocation as a token tree. This is a very
+/// loose structure, such that all sorts of different AST fragments can
 /// be passed to syntax extensions using a uniform type.
 ///
 /// If the syntax extension is an MBE macro, it will attempt to match its
@@ -38,12 +38,18 @@ use std::{fmt, iter, mem};
 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
-    /// A single token
+    /// A single token.
     Token(Token),
-    /// A delimited sequence of token trees
+    /// A delimited sequence of token trees.
     Delimited(DelimSpan, DelimToken, TokenStream),
 }
 
+#[derive(Copy, Clone)]
+pub enum CanSynthesizeMissingTokens {
+    Yes,
+    No,
+}
+
 // Ensure all fields of `TokenTree` is `Send` and `Sync`.
 #[cfg(parallel_compiler)]
 fn _dummy()
@@ -56,7 +62,7 @@ where
 }
 
 impl TokenTree {
-    /// Checks if this TokenTree is equal to the other, regardless of span information.
+    /// Checks if this `TokenTree` is equal to the other, regardless of span information.
     pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
         match (self, other) {
             (TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind,
@@ -67,7 +73,7 @@ impl TokenTree {
         }
     }
 
-    /// Retrieves the TokenTree's span.
+    /// Retrieves the `TokenTree`'s span.
     pub fn span(&self) -> Span {
         match self {
             TokenTree::Token(token) => token.span,
@@ -121,20 +127,16 @@ where
 }
 
 pub trait CreateTokenStream: sync::Send + sync::Sync {
-    fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream>;
     fn create_token_stream(&self) -> TokenStream;
 }
 
 impl CreateTokenStream for TokenStream {
-    fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream> {
-        panic!("Cannot call `add_trailing_semi` on a `TokenStream`!");
-    }
     fn create_token_stream(&self) -> TokenStream {
         self.clone()
     }
 }
 
-/// A lazy version of `TokenStream`, which defers creation
+/// A lazy version of [`TokenStream`], which defers creation
 /// of an actual `TokenStream` until it is needed.
 /// `Box` is here only to reduce the structure size.
 #[derive(Clone)]
@@ -145,13 +147,6 @@ impl LazyTokenStream {
         LazyTokenStream(Lrc::new(Box::new(inner)))
     }
 
-    /// Extends the captured stream by one token,
-    /// which must be a trailing semicolon. This
-    /// affects the `TokenStream` created by `make_tokenstream`.
-    pub fn add_trailing_semi(&self) -> LazyTokenStream {
-        LazyTokenStream(Lrc::new(self.0.add_trailing_semi()))
-    }
-
     pub fn create_token_stream(&self) -> TokenStream {
         self.0.create_token_stream()
     }
@@ -182,11 +177,12 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
     }
 }
 
-/// A `TokenStream` is an abstract sequence of tokens, organized into `TokenTree`s.
+/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
 ///
 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
 /// instead of a representation of the abstract syntax tree.
-/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
+/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
+/// backwards compatability.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>);
 
@@ -423,7 +419,7 @@ impl TokenStreamBuilder {
     }
 }
 
-/// By-reference iterator over a `TokenStream`.
+/// By-reference iterator over a [`TokenStream`].
 #[derive(Clone)]
 pub struct CursorRef<'t> {
     stream: &'t TokenStream,
@@ -451,8 +447,8 @@ impl<'t> Iterator for CursorRef<'t> {
     }
 }
 
-/// Owning by-value iterator over a `TokenStream`.
-/// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
+/// Owning by-value iterator over a [`TokenStream`].
+// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
 #[derive(Clone)]
 pub struct Cursor {
     pub stream: TokenStream,