1 use crate::parse
::{token_stream, Cursor}
;
2 use crate::{Delimiter, Spacing, TokenTree}
;
4 use std
::cell
::RefCell
;
7 use std
::fmt
::{self, Debug, Display}
;
8 use std
::iter
::FromIterator
;
10 use std
::ops
::RangeBounds
;
11 #[cfg(procmacro2_semver_exempt)]
13 use std
::path
::PathBuf
;
14 use std
::str::FromStr
;
16 use unicode_xid
::UnicodeXID
;
18 /// Force use of proc-macro2's fallback implementation of the API for now, even
19 /// if the compiler's implementation is available.
21 #[cfg(wrap_proc_macro)]
22 crate::detection
::force_fallback();
25 /// Resume using the compiler's implementation of the proc macro API if it is
28 #[cfg(wrap_proc_macro)]
29 crate::detection
::unforce_fallback();
33 pub(crate) struct TokenStream
{
34 pub(crate) inner
: Vec
<TokenTree
>,
38 pub(crate) struct LexError
;
41 pub fn new() -> TokenStream
{
42 TokenStream { inner: Vec::new() }
45 pub fn is_empty(&self) -> bool
{
49 fn take_inner(&mut self) -> Vec
<TokenTree
> {
50 mem
::replace(&mut self.inner
, Vec
::new())
53 fn push_token(&mut self, token
: TokenTree
) {
54 // https://github.com/alexcrichton/proc-macro2/issues/235
56 #[cfg(not(no_bind_by_move_pattern_guard))]
57 TokenTree
::Literal(crate::Literal
{
58 #[cfg(wrap_proc_macro)]
59 inner
: crate::imp
::Literal
::Fallback(literal
),
60 #[cfg(not(wrap_proc_macro))]
63 }) if literal
.text
.starts_with('
-'
) => {
64 push_negative_literal(self, literal
);
66 #[cfg(no_bind_by_move_pattern_guard)]
67 TokenTree
::Literal(crate::Literal
{
68 #[cfg(wrap_proc_macro)]
69 inner
: crate::imp
::Literal
::Fallback(literal
),
70 #[cfg(not(wrap_proc_macro))]
74 if literal
.text
.starts_with('
-'
) {
75 push_negative_literal(self, literal
);
78 .push(TokenTree
::Literal(crate::Literal
::_new_stable(literal
)));
81 _
=> self.inner
.push(token
),
85 fn push_negative_literal(stream
: &mut TokenStream
, mut literal
: Literal
) {
86 literal
.text
.remove(0);
87 let mut punct
= crate::Punct
::new('
-'
, Spacing
::Alone
);
88 punct
.set_span(crate::Span
::_new_stable(literal
.span
));
89 stream
.inner
.push(TokenTree
::Punct(punct
));
92 .push(TokenTree
::Literal(crate::Literal
::_new_stable(literal
)));
97 // Nonrecursive to prevent stack overflow.
98 impl Drop
for TokenStream
{
100 while let Some(token
) = self.inner
.pop() {
101 let group
= match token
{
102 TokenTree
::Group(group
) => group
.inner
,
105 #[cfg(wrap_proc_macro)]
106 let group
= match group
{
107 crate::imp
::Group
::Fallback(group
) => group
,
110 let mut group
= group
;
111 self.inner
.extend(group
.stream
.take_inner());
116 #[cfg(span_locations)]
117 fn get_cursor(src
: &str) -> Cursor
{
118 // Create a dummy file & add it to the source map
119 SOURCE_MAP
.with(|cm
| {
120 let mut cm
= cm
.borrow_mut();
121 let name
= format
!("<parsed string {}>", cm
.files
.len());
122 let span
= cm
.add_file(&name
, src
);
130 #[cfg(not(span_locations))]
131 fn get_cursor(src
: &str) -> Cursor
{
135 impl FromStr
for TokenStream
{
138 fn from_str(src
: &str) -> Result
<TokenStream
, LexError
> {
139 // Create a dummy file & add it to the source map
140 let cursor
= get_cursor(src
);
142 let (rest
, tokens
) = token_stream(cursor
)?
;
151 impl Display
for TokenStream
{
152 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
153 let mut joint
= false;
154 for (i
, tt
) in self.inner
.iter().enumerate() {
155 if i
!= 0 && !joint
{
160 TokenTree
::Group(tt
) => Display
::fmt(tt
, f
),
161 TokenTree
::Ident(tt
) => Display
::fmt(tt
, f
),
162 TokenTree
::Punct(tt
) => {
163 joint
= tt
.spacing() == Spacing
::Joint
;
166 TokenTree
::Literal(tt
) => Display
::fmt(tt
, f
),
174 impl Debug
for TokenStream
{
175 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
176 f
.write_str("TokenStream ")?
;
177 f
.debug_list().entries(self.clone()).finish()
181 #[cfg(use_proc_macro)]
182 impl From
<proc_macro
::TokenStream
> for TokenStream
{
183 fn from(inner
: proc_macro
::TokenStream
) -> TokenStream
{
187 .expect("compiler token stream parse failed")
191 #[cfg(use_proc_macro)]
192 impl From
<TokenStream
> for proc_macro
::TokenStream
{
193 fn from(inner
: TokenStream
) -> proc_macro
::TokenStream
{
197 .expect("failed to parse to compiler tokens")
201 impl From
<TokenTree
> for TokenStream
{
202 fn from(tree
: TokenTree
) -> TokenStream
{
203 let mut stream
= TokenStream
::new();
204 stream
.push_token(tree
);
209 impl FromIterator
<TokenTree
> for TokenStream
{
210 fn from_iter
<I
: IntoIterator
<Item
= TokenTree
>>(tokens
: I
) -> Self {
211 let mut stream
= TokenStream
::new();
212 stream
.extend(tokens
);
217 impl FromIterator
<TokenStream
> for TokenStream
{
218 fn from_iter
<I
: IntoIterator
<Item
= TokenStream
>>(streams
: I
) -> Self {
219 let mut v
= Vec
::new();
221 for mut stream
in streams
{
222 v
.extend(stream
.take_inner());
225 TokenStream { inner: v }
229 impl Extend
<TokenTree
> for TokenStream
{
230 fn extend
<I
: IntoIterator
<Item
= TokenTree
>>(&mut self, tokens
: I
) {
231 tokens
.into_iter().for_each(|token
| self.push_token(token
));
235 impl Extend
<TokenStream
> for TokenStream
{
236 fn extend
<I
: IntoIterator
<Item
= TokenStream
>>(&mut self, streams
: I
) {
237 self.inner
.extend(streams
.into_iter().flatten());
241 pub(crate) type TokenTreeIter
= vec
::IntoIter
<TokenTree
>;
243 impl IntoIterator
for TokenStream
{
244 type Item
= TokenTree
;
245 type IntoIter
= TokenTreeIter
;
247 fn into_iter(mut self) -> TokenTreeIter
{
248 self.take_inner().into_iter()
252 #[derive(Clone, PartialEq, Eq)]
253 pub(crate) struct SourceFile
{
258 /// Get the path to this source file as a string.
259 pub fn path(&self) -> PathBuf
{
263 pub fn is_real(&self) -> bool
{
264 // XXX(nika): Support real files in the future?
269 impl Debug
for SourceFile
{
270 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
271 f
.debug_struct("SourceFile")
272 .field("path", &self.path())
273 .field("is_real", &self.is_real())
278 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
279 pub(crate) struct LineColumn
{
284 #[cfg(span_locations)]
286 static SOURCE_MAP
: RefCell
<SourceMap
> = RefCell
::new(SourceMap
{
287 // NOTE: We start with a single dummy file which all call_site() and
288 // def_site() spans reference.
289 files
: vec
![FileInfo
{
290 #[cfg(procmacro2_semver_exempt)]
291 name
: "<unspecified>".to_owned(),
292 span
: Span { lo: 0, hi: 0 }
,
298 #[cfg(span_locations)]
300 #[cfg(procmacro2_semver_exempt)]
306 #[cfg(span_locations)]
308 fn offset_line_column(&self, offset
: usize) -> LineColumn
{
309 assert
!(self.span_within(Span
{
313 let offset
= offset
- self.span
.lo
as usize;
314 match self.lines
.binary_search(&offset
) {
315 Ok(found
) => LineColumn
{
319 Err(idx
) => LineColumn
{
321 column
: offset
- self.lines
[idx
- 1],
326 fn span_within(&self, span
: Span
) -> bool
{
327 span
.lo
>= self.span
.lo
&& span
.hi
<= self.span
.hi
331 /// Computes the offsets of each line in the given source string
332 /// and the total number of characters
333 #[cfg(span_locations)]
334 fn lines_offsets(s
: &str) -> (usize, Vec
<usize>) {
335 let mut lines
= vec
![0];
338 for ch
in s
.chars() {
348 #[cfg(span_locations)]
350 files
: Vec
<FileInfo
>,
353 #[cfg(span_locations)]
355 fn next_start_pos(&self) -> u32 {
356 // Add 1 so there's always space between files.
358 // We'll always have at least 1 file, as we initialize our files list
359 // with a dummy file.
360 self.files
.last().unwrap().span
.hi
+ 1
363 fn add_file(&mut self, name
: &str, src
: &str) -> Span
{
364 let (len
, lines
) = lines_offsets(src
);
365 let lo
= self.next_start_pos();
366 // XXX(nika): Shouild we bother doing a checked cast or checked add here?
369 hi
: lo
+ (len
as u32),
372 self.files
.push(FileInfo
{
373 #[cfg(procmacro2_semver_exempt)]
374 name
: name
.to_owned(),
379 #[cfg(not(procmacro2_semver_exempt))]
385 fn fileinfo(&self, span
: Span
) -> &FileInfo
{
386 for file
in &self.files
{
387 if file
.span_within(span
) {
391 panic
!("Invalid span with no related FileInfo!");
395 #[derive(Clone, Copy, PartialEq, Eq)]
396 pub(crate) struct Span
{
397 #[cfg(span_locations)]
399 #[cfg(span_locations)]
404 #[cfg(not(span_locations))]
405 pub fn call_site() -> Span
{
409 #[cfg(span_locations)]
410 pub fn call_site() -> Span
{
411 Span { lo: 0, hi: 0 }
415 pub fn mixed_site() -> Span
{
419 #[cfg(procmacro2_semver_exempt)]
420 pub fn def_site() -> Span
{
424 pub fn resolved_at(&self, _other
: Span
) -> Span
{
425 // Stable spans consist only of line/column information, so
426 // `resolved_at` and `located_at` only select which span the
427 // caller wants line/column information from.
431 pub fn located_at(&self, other
: Span
) -> Span
{
435 #[cfg(procmacro2_semver_exempt)]
436 pub fn source_file(&self) -> SourceFile
{
437 SOURCE_MAP
.with(|cm
| {
438 let cm
= cm
.borrow();
439 let fi
= cm
.fileinfo(*self);
441 path
: Path
::new(&fi
.name
).to_owned(),
446 #[cfg(span_locations)]
447 pub fn start(&self) -> LineColumn
{
448 SOURCE_MAP
.with(|cm
| {
449 let cm
= cm
.borrow();
450 let fi
= cm
.fileinfo(*self);
451 fi
.offset_line_column(self.lo
as usize)
455 #[cfg(span_locations)]
456 pub fn end(&self) -> LineColumn
{
457 SOURCE_MAP
.with(|cm
| {
458 let cm
= cm
.borrow();
459 let fi
= cm
.fileinfo(*self);
460 fi
.offset_line_column(self.hi
as usize)
464 #[cfg(not(span_locations))]
465 pub fn join(&self, _other
: Span
) -> Option
<Span
> {
469 #[cfg(span_locations)]
470 pub fn join(&self, other
: Span
) -> Option
<Span
> {
471 SOURCE_MAP
.with(|cm
| {
472 let cm
= cm
.borrow();
473 // If `other` is not within the same FileInfo as us, return None.
474 if !cm
.fileinfo(*self).span_within(other
) {
478 lo
: cmp
::min(self.lo
, other
.lo
),
479 hi
: cmp
::max(self.hi
, other
.hi
),
484 #[cfg(not(span_locations))]
485 fn first_byte(self) -> Self {
489 #[cfg(span_locations)]
490 fn first_byte(self) -> Self {
493 hi
: cmp
::min(self.lo
.saturating_add(1), self.hi
),
497 #[cfg(not(span_locations))]
498 fn last_byte(self) -> Self {
502 #[cfg(span_locations)]
503 fn last_byte(self) -> Self {
505 lo
: cmp
::max(self.hi
.saturating_sub(1), self.lo
),
511 impl Debug
for Span
{
512 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
513 #[cfg(span_locations)]
514 return write
!(f
, "bytes({}..{})", self.lo
, self.hi
);
516 #[cfg(not(span_locations))]
521 pub(crate) fn debug_span_field_if_nontrivial(debug
: &mut fmt
::DebugStruct
, span
: Span
) {
522 #[cfg(span_locations)]
524 if span
.lo
== 0 && span
.hi
== 0 {
529 if cfg
!(span_locations
) {
530 debug
.field("span", &span
);
535 pub(crate) struct Group
{
536 delimiter
: Delimiter
,
542 pub fn new(delimiter
: Delimiter
, stream
: TokenStream
) -> Group
{
546 span
: Span
::call_site(),
550 pub fn delimiter(&self) -> Delimiter
{
554 pub fn stream(&self) -> TokenStream
{
558 pub fn span(&self) -> Span
{
562 pub fn span_open(&self) -> Span
{
563 self.span
.first_byte()
566 pub fn span_close(&self) -> Span
{
567 self.span
.last_byte()
570 pub fn set_span(&mut self, span
: Span
) {
575 impl Display
for Group
{
576 // We attempt to match libproc_macro's formatting.
578 // Nonempty parens: (...)
579 // Empty brackets: []
580 // Nonempty brackets: [...]
582 // Nonempty braces: { ... }
583 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
584 let (open
, close
) = match self.delimiter
{
585 Delimiter
::Parenthesis
=> ("(", ")"),
586 Delimiter
::Brace
=> ("{ ", "}"),
587 Delimiter
::Bracket
=> ("[", "]"),
588 Delimiter
::None
=> ("", ""),
592 Display
::fmt(&self.stream
, f
)?
;
593 if self.delimiter
== Delimiter
::Brace
&& !self.stream
.inner
.is_empty() {
602 impl Debug
for Group
{
603 fn fmt(&self, fmt
: &mut fmt
::Formatter
) -> fmt
::Result
{
604 let mut debug
= fmt
.debug_struct("Group");
605 debug
.field("delimiter", &self.delimiter
);
606 debug
.field("stream", &self.stream
);
607 debug_span_field_if_nontrivial(&mut debug
, self.span
);
613 pub(crate) struct Ident
{
620 fn _new(string
: &str, raw
: bool
, span
: Span
) -> Ident
{
621 validate_ident(string
);
624 sym
: string
.to_owned(),
630 pub fn new(string
: &str, span
: Span
) -> Ident
{
631 Ident
::_new(string
, false, span
)
634 pub fn new_raw(string
: &str, span
: Span
) -> Ident
{
635 Ident
::_new(string
, true, span
)
638 pub fn span(&self) -> Span
{
642 pub fn set_span(&mut self, span
: Span
) {
647 pub(crate) fn is_ident_start(c
: char) -> bool
{
648 ('a'
<= c
&& c
<= 'z'
)
649 || ('A'
<= c
&& c
<= 'Z'
)
651 || (c
> '
\x7f'
&& UnicodeXID
::is_xid_start(c
))
654 pub(crate) fn is_ident_continue(c
: char) -> bool
{
655 ('a'
<= c
&& c
<= 'z'
)
656 || ('A'
<= c
&& c
<= 'Z'
)
658 || ('
0'
<= c
&& c
<= '
9'
)
659 || (c
> '
\x7f'
&& UnicodeXID
::is_xid_continue(c
))
662 fn validate_ident(string
: &str) {
663 let validate
= string
;
664 if validate
.is_empty() {
665 panic
!("Ident is not allowed to be empty; use Option<Ident>");
668 if validate
.bytes().all(|digit
| digit
>= b'
0'
&& digit
<= b'
9'
) {
669 panic
!("Ident cannot be a number; use Literal instead");
672 fn ident_ok(string
: &str) -> bool
{
673 let mut chars
= string
.chars();
674 let first
= chars
.next().unwrap();
675 if !is_ident_start(first
) {
679 if !is_ident_continue(ch
) {
686 if !ident_ok(validate
) {
687 panic
!("{:?} is not a valid Ident", string
);
691 impl PartialEq
for Ident
{
692 fn eq(&self, other
: &Ident
) -> bool
{
693 self.sym
== other
.sym
&& self.raw
== other
.raw
697 impl<T
> PartialEq
<T
> for Ident
699 T
: ?Sized
+ AsRef
<str>,
701 fn eq(&self, other
: &T
) -> bool
{
702 let other
= other
.as_ref();
704 other
.starts_with("r#") && self.sym
== other
[2..]
711 impl Display
for Ident
{
712 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
716 Display
::fmt(&self.sym
, f
)
720 impl Debug
for Ident
{
721 // Ident(proc_macro), Ident(r#union)
722 #[cfg(not(span_locations))]
723 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
724 let mut debug
= f
.debug_tuple("Ident");
725 debug
.field(&format_args
!("{}", self));
731 // span: bytes(128..138)
733 #[cfg(span_locations)]
734 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
735 let mut debug
= f
.debug_struct("Ident");
736 debug
.field("sym", &format_args
!("{}", self));
737 debug_span_field_if_nontrivial(&mut debug
, self.span
);
743 pub(crate) struct Literal
{
748 macro_rules
! suffixed_numbers
{
749 ($
($name
:ident
=> $kind
:ident
,)*) => ($
(
750 pub fn $
name(n
: $kind
) -> Literal
{
751 Literal
::_new(format
!(concat
!("{}", stringify
!($kind
)), n
))
756 macro_rules
! unsuffixed_numbers
{
757 ($
($name
:ident
=> $kind
:ident
,)*) => ($
(
758 pub fn $
name(n
: $kind
) -> Literal
{
759 Literal
::_new(n
.to_string())
765 pub(crate) fn _new(text
: String
) -> Literal
{
768 span
: Span
::call_site(),
777 u128_suffixed
=> u128
,
778 usize_suffixed
=> usize,
783 i128_suffixed
=> i128
,
784 isize_suffixed
=> isize,
790 unsuffixed_numbers
! {
792 u16_unsuffixed
=> u16,
793 u32_unsuffixed
=> u32,
794 u64_unsuffixed
=> u64,
795 u128_unsuffixed
=> u128
,
796 usize_unsuffixed
=> usize,
798 i16_unsuffixed
=> i16,
799 i32_unsuffixed
=> i32,
800 i64_unsuffixed
=> i64,
801 i128_unsuffixed
=> i128
,
802 isize_unsuffixed
=> isize,
805 pub fn f32_unsuffixed(f
: f32) -> Literal
{
806 let mut s
= f
.to_string();
807 if !s
.contains('
.'
) {
813 pub fn f64_unsuffixed(f
: f64) -> Literal
{
814 let mut s
= f
.to_string();
815 if !s
.contains('
.'
) {
821 pub fn string(t
: &str) -> Literal
{
822 let mut text
= String
::with_capacity(t
.len() + 2);
826 // escape_debug turns this into "\'" which is unnecessary.
829 text.extend(c.escape_debug());
836 pub fn character(t
: char) -> Literal
{
837 let mut text
= String
::new();
840 // escape_debug turns this into '\"' which is unnecessary.
843 text.extend(t.escape_debug());
849 pub fn byte_string(bytes: &[u8]) -> Literal {
850 let mut escaped = "b
\"".to_string();
852 #[allow(clippy::match_overlapping_arm)]
854 b'\0' => escaped.push_str(r"\0"),
855 b'\t' => escaped.push_str(r"\t"),
856 b'\n' => escaped.push_str(r"\n"),
857 b'\r' => escaped.push_str(r"\r"),
858 b'"'
=> escaped
.push_str("\\\""),
859 b'
\\'
=> escaped
.push_str("\\\\"),
860 b'
\x20'
..=b'
\x7E'
=> escaped
.push(*b
as char),
861 _
=> escaped
.push_str(&format
!("\\x{:02X}", b
)),
865 Literal::_new(escaped)
868 pub fn span(&self) -> Span {
872 pub fn set_span(&mut self, span: Span) {
876 pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
881 impl Display for Literal {
882 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
883 Display::fmt(&self.text, f)
887 impl Debug for Literal {
888 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
889 let mut debug = fmt.debug_struct("Literal
");
890 debug.field("lit
", &format_args!("{}
", self.text));
891 debug_span_field_if_nontrivial(&mut debug, self.span);