1 use crate::parse
::{self, Cursor}
;
2 use crate::{Delimiter, Spacing, TokenTree}
;
4 use std
::cell
::RefCell
;
7 use std
::fmt
::{self, Debug, Display}
;
8 use std
::iter
::FromIterator
;
10 use std
::ops
::RangeBounds
;
11 #[cfg(procmacro2_semver_exempt)]
13 use std
::path
::PathBuf
;
14 use std
::str::FromStr
;
16 use unicode_xid
::UnicodeXID
;
18 /// Force use of proc-macro2's fallback implementation of the API for now, even
19 /// if the compiler's implementation is available.
21 #[cfg(wrap_proc_macro)]
22 crate::detection
::force_fallback();
25 /// Resume using the compiler's implementation of the proc macro API if it is
28 #[cfg(wrap_proc_macro)]
29 crate::detection
::unforce_fallback();
33 pub(crate) struct TokenStream
{
34 pub(crate) inner
: Vec
<TokenTree
>,
38 pub(crate) struct LexError
{
39 pub(crate) span
: Span
,
43 pub(crate) fn span(&self) -> Span
{
49 pub fn new() -> TokenStream
{
50 TokenStream { inner: Vec::new() }
53 pub fn is_empty(&self) -> bool
{
57 fn take_inner(&mut self) -> Vec
<TokenTree
> {
58 mem
::replace(&mut self.inner
, Vec
::new())
61 fn push_token(&mut self, token
: TokenTree
) {
62 // https://github.com/alexcrichton/proc-macro2/issues/235
64 #[cfg(not(no_bind_by_move_pattern_guard))]
65 TokenTree
::Literal(crate::Literal
{
66 #[cfg(wrap_proc_macro)]
67 inner
: crate::imp
::Literal
::Fallback(literal
),
68 #[cfg(not(wrap_proc_macro))]
71 }) if literal
.text
.starts_with('
-'
) => {
72 push_negative_literal(self, literal
);
74 #[cfg(no_bind_by_move_pattern_guard)]
75 TokenTree
::Literal(crate::Literal
{
76 #[cfg(wrap_proc_macro)]
77 inner
: crate::imp
::Literal
::Fallback(literal
),
78 #[cfg(not(wrap_proc_macro))]
82 if literal
.text
.starts_with('
-'
) {
83 push_negative_literal(self, literal
);
86 .push(TokenTree
::Literal(crate::Literal
::_new_stable(literal
)));
89 _
=> self.inner
.push(token
),
93 fn push_negative_literal(stream
: &mut TokenStream
, mut literal
: Literal
) {
94 literal
.text
.remove(0);
95 let mut punct
= crate::Punct
::new('
-'
, Spacing
::Alone
);
96 punct
.set_span(crate::Span
::_new_stable(literal
.span
));
97 stream
.inner
.push(TokenTree
::Punct(punct
));
100 .push(TokenTree
::Literal(crate::Literal
::_new_stable(literal
)));
105 // Nonrecursive to prevent stack overflow.
106 impl Drop
for TokenStream
{
108 while let Some(token
) = self.inner
.pop() {
109 let group
= match token
{
110 TokenTree
::Group(group
) => group
.inner
,
113 #[cfg(wrap_proc_macro)]
114 let group
= match group
{
115 crate::imp
::Group
::Fallback(group
) => group
,
118 let mut group
= group
;
119 self.inner
.extend(group
.stream
.take_inner());
124 #[cfg(span_locations)]
125 fn get_cursor(src
: &str) -> Cursor
{
126 // Create a dummy file & add it to the source map
127 SOURCE_MAP
.with(|cm
| {
128 let mut cm
= cm
.borrow_mut();
129 let name
= format
!("<parsed string {}>", cm
.files
.len());
130 let span
= cm
.add_file(&name
, src
);
138 #[cfg(not(span_locations))]
139 fn get_cursor(src
: &str) -> Cursor
{
143 impl FromStr
for TokenStream
{
146 fn from_str(src
: &str) -> Result
<TokenStream
, LexError
> {
147 // Create a dummy file & add it to the source map
148 let cursor
= get_cursor(src
);
150 parse
::token_stream(cursor
)
154 impl Display
for LexError
{
155 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
156 f
.write_str("cannot parse string into token stream")
160 impl Display
for TokenStream
{
161 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
162 let mut joint
= false;
163 for (i
, tt
) in self.inner
.iter().enumerate() {
164 if i
!= 0 && !joint
{
169 TokenTree
::Group(tt
) => Display
::fmt(tt
, f
),
170 TokenTree
::Ident(tt
) => Display
::fmt(tt
, f
),
171 TokenTree
::Punct(tt
) => {
172 joint
= tt
.spacing() == Spacing
::Joint
;
175 TokenTree
::Literal(tt
) => Display
::fmt(tt
, f
),
183 impl Debug
for TokenStream
{
184 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
185 f
.write_str("TokenStream ")?
;
186 f
.debug_list().entries(self.clone()).finish()
190 #[cfg(use_proc_macro)]
191 impl From
<proc_macro
::TokenStream
> for TokenStream
{
192 fn from(inner
: proc_macro
::TokenStream
) -> TokenStream
{
196 .expect("compiler token stream parse failed")
200 #[cfg(use_proc_macro)]
201 impl From
<TokenStream
> for proc_macro
::TokenStream
{
202 fn from(inner
: TokenStream
) -> proc_macro
::TokenStream
{
206 .expect("failed to parse to compiler tokens")
210 impl From
<TokenTree
> for TokenStream
{
211 fn from(tree
: TokenTree
) -> TokenStream
{
212 let mut stream
= TokenStream
::new();
213 stream
.push_token(tree
);
218 impl FromIterator
<TokenTree
> for TokenStream
{
219 fn from_iter
<I
: IntoIterator
<Item
= TokenTree
>>(tokens
: I
) -> Self {
220 let mut stream
= TokenStream
::new();
221 stream
.extend(tokens
);
226 impl FromIterator
<TokenStream
> for TokenStream
{
227 fn from_iter
<I
: IntoIterator
<Item
= TokenStream
>>(streams
: I
) -> Self {
228 let mut v
= Vec
::new();
230 for mut stream
in streams
{
231 v
.extend(stream
.take_inner());
234 TokenStream { inner: v }
238 impl Extend
<TokenTree
> for TokenStream
{
239 fn extend
<I
: IntoIterator
<Item
= TokenTree
>>(&mut self, tokens
: I
) {
240 tokens
.into_iter().for_each(|token
| self.push_token(token
));
244 impl Extend
<TokenStream
> for TokenStream
{
245 fn extend
<I
: IntoIterator
<Item
= TokenStream
>>(&mut self, streams
: I
) {
246 self.inner
.extend(streams
.into_iter().flatten());
250 pub(crate) type TokenTreeIter
= vec
::IntoIter
<TokenTree
>;
252 impl IntoIterator
for TokenStream
{
253 type Item
= TokenTree
;
254 type IntoIter
= TokenTreeIter
;
256 fn into_iter(mut self) -> TokenTreeIter
{
257 self.take_inner().into_iter()
261 #[derive(Clone, PartialEq, Eq)]
262 pub(crate) struct SourceFile
{
267 /// Get the path to this source file as a string.
268 pub fn path(&self) -> PathBuf
{
272 pub fn is_real(&self) -> bool
{
273 // XXX(nika): Support real files in the future?
278 impl Debug
for SourceFile
{
279 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
280 f
.debug_struct("SourceFile")
281 .field("path", &self.path())
282 .field("is_real", &self.is_real())
287 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
288 pub(crate) struct LineColumn
{
293 #[cfg(span_locations)]
295 static SOURCE_MAP
: RefCell
<SourceMap
> = RefCell
::new(SourceMap
{
296 // NOTE: We start with a single dummy file which all call_site() and
297 // def_site() spans reference.
298 files
: vec
![FileInfo
{
299 #[cfg(procmacro2_semver_exempt)]
300 name
: "<unspecified>".to_owned(),
301 span
: Span { lo: 0, hi: 0 }
,
307 #[cfg(span_locations)]
309 #[cfg(procmacro2_semver_exempt)]
315 #[cfg(span_locations)]
317 fn offset_line_column(&self, offset
: usize) -> LineColumn
{
318 assert
!(self.span_within(Span
{
322 let offset
= offset
- self.span
.lo
as usize;
323 match self.lines
.binary_search(&offset
) {
324 Ok(found
) => LineColumn
{
328 Err(idx
) => LineColumn
{
330 column
: offset
- self.lines
[idx
- 1],
335 fn span_within(&self, span
: Span
) -> bool
{
336 span
.lo
>= self.span
.lo
&& span
.hi
<= self.span
.hi
340 /// Computes the offsets of each line in the given source string
341 /// and the total number of characters
342 #[cfg(span_locations)]
343 fn lines_offsets(s
: &str) -> (usize, Vec
<usize>) {
344 let mut lines
= vec
![0];
347 for ch
in s
.chars() {
357 #[cfg(span_locations)]
359 files
: Vec
<FileInfo
>,
362 #[cfg(span_locations)]
364 fn next_start_pos(&self) -> u32 {
365 // Add 1 so there's always space between files.
367 // We'll always have at least 1 file, as we initialize our files list
368 // with a dummy file.
369 self.files
.last().unwrap().span
.hi
+ 1
372 fn add_file(&mut self, name
: &str, src
: &str) -> Span
{
373 let (len
, lines
) = lines_offsets(src
);
374 let lo
= self.next_start_pos();
375 // XXX(nika): Shouild we bother doing a checked cast or checked add here?
378 hi
: lo
+ (len
as u32),
381 self.files
.push(FileInfo
{
382 #[cfg(procmacro2_semver_exempt)]
383 name
: name
.to_owned(),
388 #[cfg(not(procmacro2_semver_exempt))]
394 fn fileinfo(&self, span
: Span
) -> &FileInfo
{
395 for file
in &self.files
{
396 if file
.span_within(span
) {
400 panic
!("Invalid span with no related FileInfo!");
404 #[derive(Clone, Copy, PartialEq, Eq)]
405 pub(crate) struct Span
{
406 #[cfg(span_locations)]
408 #[cfg(span_locations)]
413 #[cfg(not(span_locations))]
414 pub fn call_site() -> Span
{
418 #[cfg(span_locations)]
419 pub fn call_site() -> Span
{
420 Span { lo: 0, hi: 0 }
424 pub fn mixed_site() -> Span
{
428 #[cfg(procmacro2_semver_exempt)]
429 pub fn def_site() -> Span
{
433 pub fn resolved_at(&self, _other
: Span
) -> Span
{
434 // Stable spans consist only of line/column information, so
435 // `resolved_at` and `located_at` only select which span the
436 // caller wants line/column information from.
440 pub fn located_at(&self, other
: Span
) -> Span
{
444 #[cfg(procmacro2_semver_exempt)]
445 pub fn source_file(&self) -> SourceFile
{
446 SOURCE_MAP
.with(|cm
| {
447 let cm
= cm
.borrow();
448 let fi
= cm
.fileinfo(*self);
450 path
: Path
::new(&fi
.name
).to_owned(),
455 #[cfg(span_locations)]
456 pub fn start(&self) -> LineColumn
{
457 SOURCE_MAP
.with(|cm
| {
458 let cm
= cm
.borrow();
459 let fi
= cm
.fileinfo(*self);
460 fi
.offset_line_column(self.lo
as usize)
464 #[cfg(span_locations)]
465 pub fn end(&self) -> LineColumn
{
466 SOURCE_MAP
.with(|cm
| {
467 let cm
= cm
.borrow();
468 let fi
= cm
.fileinfo(*self);
469 fi
.offset_line_column(self.hi
as usize)
473 #[cfg(not(span_locations))]
474 pub fn join(&self, _other
: Span
) -> Option
<Span
> {
478 #[cfg(span_locations)]
479 pub fn join(&self, other
: Span
) -> Option
<Span
> {
480 SOURCE_MAP
.with(|cm
| {
481 let cm
= cm
.borrow();
482 // If `other` is not within the same FileInfo as us, return None.
483 if !cm
.fileinfo(*self).span_within(other
) {
487 lo
: cmp
::min(self.lo
, other
.lo
),
488 hi
: cmp
::max(self.hi
, other
.hi
),
493 #[cfg(not(span_locations))]
494 fn first_byte(self) -> Self {
498 #[cfg(span_locations)]
499 fn first_byte(self) -> Self {
502 hi
: cmp
::min(self.lo
.saturating_add(1), self.hi
),
506 #[cfg(not(span_locations))]
507 fn last_byte(self) -> Self {
511 #[cfg(span_locations)]
512 fn last_byte(self) -> Self {
514 lo
: cmp
::max(self.hi
.saturating_sub(1), self.lo
),
520 impl Debug
for Span
{
521 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
522 #[cfg(span_locations)]
523 return write
!(f
, "bytes({}..{})", self.lo
, self.hi
);
525 #[cfg(not(span_locations))]
530 pub(crate) fn debug_span_field_if_nontrivial(debug
: &mut fmt
::DebugStruct
, span
: Span
) {
531 #[cfg(span_locations)]
533 if span
.lo
== 0 && span
.hi
== 0 {
538 if cfg
!(span_locations
) {
539 debug
.field("span", &span
);
544 pub(crate) struct Group
{
545 delimiter
: Delimiter
,
551 pub fn new(delimiter
: Delimiter
, stream
: TokenStream
) -> Group
{
555 span
: Span
::call_site(),
559 pub fn delimiter(&self) -> Delimiter
{
563 pub fn stream(&self) -> TokenStream
{
567 pub fn span(&self) -> Span
{
571 pub fn span_open(&self) -> Span
{
572 self.span
.first_byte()
575 pub fn span_close(&self) -> Span
{
576 self.span
.last_byte()
579 pub fn set_span(&mut self, span
: Span
) {
584 impl Display
for Group
{
585 // We attempt to match libproc_macro's formatting.
587 // Nonempty parens: (...)
588 // Empty brackets: []
589 // Nonempty brackets: [...]
591 // Nonempty braces: { ... }
592 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
593 let (open
, close
) = match self.delimiter
{
594 Delimiter
::Parenthesis
=> ("(", ")"),
595 Delimiter
::Brace
=> ("{ ", "}"),
596 Delimiter
::Bracket
=> ("[", "]"),
597 Delimiter
::None
=> ("", ""),
601 Display
::fmt(&self.stream
, f
)?
;
602 if self.delimiter
== Delimiter
::Brace
&& !self.stream
.inner
.is_empty() {
611 impl Debug
for Group
{
612 fn fmt(&self, fmt
: &mut fmt
::Formatter
) -> fmt
::Result
{
613 let mut debug
= fmt
.debug_struct("Group");
614 debug
.field("delimiter", &self.delimiter
);
615 debug
.field("stream", &self.stream
);
616 debug_span_field_if_nontrivial(&mut debug
, self.span
);
622 pub(crate) struct Ident
{
629 fn _new(string
: &str, raw
: bool
, span
: Span
) -> Ident
{
630 validate_ident(string
);
633 sym
: string
.to_owned(),
639 pub fn new(string
: &str, span
: Span
) -> Ident
{
640 Ident
::_new(string
, false, span
)
643 pub fn new_raw(string
: &str, span
: Span
) -> Ident
{
644 Ident
::_new(string
, true, span
)
647 pub fn span(&self) -> Span
{
651 pub fn set_span(&mut self, span
: Span
) {
656 pub(crate) fn is_ident_start(c
: char) -> bool
{
657 ('a'
<= c
&& c
<= 'z'
)
658 || ('A'
<= c
&& c
<= 'Z'
)
660 || (c
> '
\x7f'
&& UnicodeXID
::is_xid_start(c
))
663 pub(crate) fn is_ident_continue(c
: char) -> bool
{
664 ('a'
<= c
&& c
<= 'z'
)
665 || ('A'
<= c
&& c
<= 'Z'
)
667 || ('
0'
<= c
&& c
<= '
9'
)
668 || (c
> '
\x7f'
&& UnicodeXID
::is_xid_continue(c
))
671 fn validate_ident(string
: &str) {
672 let validate
= string
;
673 if validate
.is_empty() {
674 panic
!("Ident is not allowed to be empty; use Option<Ident>");
677 if validate
.bytes().all(|digit
| digit
>= b'
0'
&& digit
<= b'
9'
) {
678 panic
!("Ident cannot be a number; use Literal instead");
681 fn ident_ok(string
: &str) -> bool
{
682 let mut chars
= string
.chars();
683 let first
= chars
.next().unwrap();
684 if !is_ident_start(first
) {
688 if !is_ident_continue(ch
) {
695 if !ident_ok(validate
) {
696 panic
!("{:?} is not a valid Ident", string
);
700 impl PartialEq
for Ident
{
701 fn eq(&self, other
: &Ident
) -> bool
{
702 self.sym
== other
.sym
&& self.raw
== other
.raw
706 impl<T
> PartialEq
<T
> for Ident
708 T
: ?Sized
+ AsRef
<str>,
710 fn eq(&self, other
: &T
) -> bool
{
711 let other
= other
.as_ref();
713 other
.starts_with("r#") && self.sym
== other
[2..]
720 impl Display
for Ident
{
721 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
725 Display
::fmt(&self.sym
, f
)
729 impl Debug
for Ident
{
730 // Ident(proc_macro), Ident(r#union)
731 #[cfg(not(span_locations))]
732 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
733 let mut debug
= f
.debug_tuple("Ident");
734 debug
.field(&format_args
!("{}", self));
740 // span: bytes(128..138)
742 #[cfg(span_locations)]
743 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
744 let mut debug
= f
.debug_struct("Ident");
745 debug
.field("sym", &format_args
!("{}", self));
746 debug_span_field_if_nontrivial(&mut debug
, self.span
);
752 pub(crate) struct Literal
{
757 macro_rules
! suffixed_numbers
{
758 ($
($name
:ident
=> $kind
:ident
,)*) => ($
(
759 pub fn $
name(n
: $kind
) -> Literal
{
760 Literal
::_new(format
!(concat
!("{}", stringify
!($kind
)), n
))
765 macro_rules
! unsuffixed_numbers
{
766 ($
($name
:ident
=> $kind
:ident
,)*) => ($
(
767 pub fn $
name(n
: $kind
) -> Literal
{
768 Literal
::_new(n
.to_string())
774 pub(crate) fn _new(text
: String
) -> Literal
{
777 span
: Span
::call_site(),
786 u128_suffixed
=> u128
,
787 usize_suffixed
=> usize,
792 i128_suffixed
=> i128
,
793 isize_suffixed
=> isize,
799 unsuffixed_numbers
! {
801 u16_unsuffixed
=> u16,
802 u32_unsuffixed
=> u32,
803 u64_unsuffixed
=> u64,
804 u128_unsuffixed
=> u128
,
805 usize_unsuffixed
=> usize,
807 i16_unsuffixed
=> i16,
808 i32_unsuffixed
=> i32,
809 i64_unsuffixed
=> i64,
810 i128_unsuffixed
=> i128
,
811 isize_unsuffixed
=> isize,
814 pub fn f32_unsuffixed(f
: f32) -> Literal
{
815 let mut s
= f
.to_string();
816 if !s
.contains('
.'
) {
822 pub fn f64_unsuffixed(f
: f64) -> Literal
{
823 let mut s
= f
.to_string();
824 if !s
.contains('
.'
) {
830 pub fn string(t
: &str) -> Literal
{
831 let mut text
= String
::with_capacity(t
.len() + 2);
835 // escape_debug turns this into "\'" which is unnecessary.
838 text.extend(c.escape_debug());
845 pub fn character(t
: char) -> Literal
{
846 let mut text
= String
::new();
849 // escape_debug turns this into '\"' which is unnecessary.
852 text.extend(t.escape_debug());
858 pub fn byte_string(bytes: &[u8]) -> Literal {
859 let mut escaped = "b
\"".to_string();
861 #[allow(clippy::match_overlapping_arm)]
863 b'\0' => escaped.push_str(r"\0"),
864 b'\t' => escaped.push_str(r"\t"),
865 b'\n' => escaped.push_str(r"\n"),
866 b'\r' => escaped.push_str(r"\r"),
867 b'"'
=> escaped
.push_str("\\\""),
868 b'
\\'
=> escaped
.push_str("\\\\"),
869 b'
\x20'
..=b'
\x7E'
=> escaped
.push(*b
as char),
870 _
=> escaped
.push_str(&format
!("\\x{:02X}", b
)),
874 Literal::_new(escaped)
877 pub fn span(&self) -> Span {
881 pub fn set_span(&mut self, span: Span) {
885 pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
890 impl Display for Literal {
891 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
892 Display::fmt(&self.text, f)
896 impl Debug for Literal {
897 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
898 let mut debug = fmt.debug_struct("Literal
");
899 debug.field("lit
", &format_args!("{}
", self.text));
900 debug_span_field_if_nontrivial(&mut debug, self.span);