2 clippy
::assertions_on_result_states
,
3 clippy
::items_after_statements
,
4 clippy
::non_ascii_literal
,
8 use proc_macro2
::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}
;
10 use std
::str::{self, FromStr}
;
15 Ident
::new("String", Span
::call_site()).to_string(),
18 assert_eq
!(Ident
::new("fn", Span
::call_site()).to_string(), "fn");
19 assert_eq
!(Ident
::new("_", Span
::call_site()).to_string(), "_");
25 Ident
::new_raw("String", Span
::call_site()).to_string(),
28 assert_eq
!(Ident
::new_raw("fn", Span
::call_site()).to_string(), "r#fn");
32 #[should_panic(expected = "`r#_` cannot be a raw identifier")]
33 fn ident_raw_underscore() {
34 Ident
::new_raw("_", Span
::call_site());
38 #[should_panic(expected = "`r#super` cannot be a raw identifier")]
39 fn ident_raw_reserved() {
40 Ident
::new_raw("super", Span
::call_site());
44 #[should_panic(expected = "Ident is not allowed to be empty; use Option<Ident>")]
46 Ident
::new("", Span
::call_site());
50 #[should_panic(expected = "Ident cannot be a number; use Literal instead")]
52 Ident
::new("255", Span
::call_site());
56 #[should_panic(expected = "\"a#\" is not a valid Ident")]
58 Ident
::new("a#", Span
::call_site());
62 #[should_panic(expected = "not a valid Ident")]
63 fn raw_ident_empty() {
64 Ident
::new("r#", Span
::call_site());
68 #[should_panic(expected = "not a valid Ident")]
69 fn raw_ident_number() {
70 Ident
::new("r#255", Span
::call_site());
74 #[should_panic(expected = "\"r#a#\" is not a valid Ident")]
75 fn raw_ident_invalid() {
76 Ident
::new("r#a#", Span
::call_site());
80 #[should_panic(expected = "not a valid Ident")]
82 Ident
::new("'", Span
::call_site());
86 #[should_panic(expected = "not a valid Ident")]
87 fn lifetime_number() {
88 Ident
::new("'255", Span
::call_site());
92 #[should_panic(expected = r#""'a#" is not a valid Ident"#)]
93 fn lifetime_invalid() {
94 Ident
::new("'a#", Span
::call_site());
99 assert_eq
!(Literal
::string("foo").to_string(), "\"foo\"");
100 assert_eq
!(Literal
::string("\"").to_string(), "\"\\\"\"");
101 assert_eq
!(Literal
::string("didn't").to_string(), "\"didn't\"");
103 Literal
::string("a\00b\07c\08d\0e\0").to_string(),
104 "\"a\\x000b\\x007c\\08d\\0e\\0\"",
107 "\"\\\r\n x\"".parse
::<TokenStream
>().unwrap();
108 "\"\\\r\n \rx\"".parse
::<TokenStream
>().unwrap_err();
112 fn literal_raw_string() {
113 "r\"\r\n\"".parse
::<TokenStream
>().unwrap();
115 fn raw_string_literal_with_hashes(n
: usize) -> String
{
116 let mut literal
= String
::new();
118 literal
.extend(iter
::repeat('
#').take(n));
121 literal
.extend(iter
::repeat('
#').take(n));
125 raw_string_literal_with_hashes(255)
126 .parse
::<TokenStream
>()
129 // https://github.com/rust-lang/rust/pull/95251
130 raw_string_literal_with_hashes(256)
131 .parse
::<TokenStream
>()
136 fn literal_byte_string() {
137 assert_eq
!(Literal
::byte_string(b
"").to_string(), "b\"\"");
139 Literal
::byte_string(b
"\0\t\n\r\"\\2\x10").to_string(),
140 "b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"",
143 Literal
::byte_string(b
"a\00b\07c\08d\0e\0").to_string(),
144 "b\"a\\x000b\\x007c\\08d\\0e\\0\"",
147 "b\"\\\r\n x\"".parse
::<TokenStream
>().unwrap();
148 "b\"\\\r\n \rx\"".parse
::<TokenStream
>().unwrap_err();
149 "b\"\\\r\n \u{a0}x\"".parse
::<TokenStream
>().unwrap_err();
150 "br\"\u{a0}\"".parse
::<TokenStream
>().unwrap_err();
154 fn literal_c_string() {
156 c"hello\x80我叫\u{1F980}" // from the RFC
158 cr
##"Hello "world"!"##
162 let mut tokens = strings.parse::<TokenStream>().unwrap().into_iter();
165 r#"c
"hello\x80我叫\u{1F980}""#,
167 r###"cr
##"Hello "world"!"##"###,
170 match tokens.next().unwrap() {
171 TokenTree::Literal(literal) => {
172 assert_eq!(literal.to_string(), *expected);
174 unexpected => panic!("unexpected token: {:?}", unexpected),
178 if let Some(unexpected) = tokens.next() {
179 panic!("unexpected token: {:?}", unexpected);
182 for invalid in &[r#"c"\0""#, r#"c"\x00""#, r#"c"\u{0}""#, "c\"\0\""] {
183 if let Ok(unexpected) = invalid.parse::<TokenStream>() {
184 panic!("unexpected token: {:?}", unexpected);
190 fn literal_character() {
191 assert_eq!(Literal::character('x').to_string(), "'x'");
192 assert_eq!(Literal::character('\'').to_string(), "'\\''");
193 assert_eq!(Literal::character('"').to_string(), "'\"'");
197 fn literal_integer() {
198 assert_eq
!(Literal
::u8_suffixed(10).to_string(), "10u8");
199 assert_eq
!(Literal
::u16_suffixed(10).to_string(), "10u16");
200 assert_eq
!(Literal
::u32_suffixed(10).to_string(), "10u32");
201 assert_eq
!(Literal
::u64_suffixed(10).to_string(), "10u64");
202 assert_eq
!(Literal
::u128_suffixed(10).to_string(), "10u128");
203 assert_eq
!(Literal
::usize_suffixed(10).to_string(), "10usize");
205 assert_eq
!(Literal
::i8_suffixed(10).to_string(), "10i8");
206 assert_eq
!(Literal
::i16_suffixed(10).to_string(), "10i16");
207 assert_eq
!(Literal
::i32_suffixed(10).to_string(), "10i32");
208 assert_eq
!(Literal
::i64_suffixed(10).to_string(), "10i64");
209 assert_eq
!(Literal
::i128_suffixed(10).to_string(), "10i128");
210 assert_eq
!(Literal
::isize_suffixed(10).to_string(), "10isize");
212 assert_eq
!(Literal
::u8_unsuffixed(10).to_string(), "10");
213 assert_eq
!(Literal
::u16_unsuffixed(10).to_string(), "10");
214 assert_eq
!(Literal
::u32_unsuffixed(10).to_string(), "10");
215 assert_eq
!(Literal
::u64_unsuffixed(10).to_string(), "10");
216 assert_eq
!(Literal
::u128_unsuffixed(10).to_string(), "10");
217 assert_eq
!(Literal
::usize_unsuffixed(10).to_string(), "10");
219 assert_eq
!(Literal
::i8_unsuffixed(10).to_string(), "10");
220 assert_eq
!(Literal
::i16_unsuffixed(10).to_string(), "10");
221 assert_eq
!(Literal
::i32_unsuffixed(10).to_string(), "10");
222 assert_eq
!(Literal
::i64_unsuffixed(10).to_string(), "10");
223 assert_eq
!(Literal
::i128_unsuffixed(10).to_string(), "10");
224 assert_eq
!(Literal
::isize_unsuffixed(10).to_string(), "10");
229 assert_eq
!(Literal
::f32_suffixed(10.0).to_string(), "10f32");
230 assert_eq
!(Literal
::f64_suffixed(10.0).to_string(), "10f64");
232 assert_eq
!(Literal
::f32_unsuffixed(10.0).to_string(), "10.0");
233 assert_eq
!(Literal
::f64_unsuffixed(10.0).to_string(), "10.0");
237 fn literal_suffix() {
238 fn token_count(p
: &str) -> usize {
239 p
.parse
::<TokenStream
>().unwrap().into_iter().count()
242 assert_eq
!(token_count("999u256"), 1);
243 assert_eq
!(token_count("999r#u256"), 3);
244 assert_eq
!(token_count("1."), 1);
245 assert_eq
!(token_count("1.f32"), 3);
246 assert_eq
!(token_count("1.0_0"), 1);
247 assert_eq
!(token_count("1._0"), 3);
248 assert_eq
!(token_count("1._m"), 3);
249 assert_eq
!(token_count("\"\"s"), 1);
250 assert_eq
!(token_count("r\"\"r"), 1);
251 assert_eq
!(token_count("b\"\"b"), 1);
252 assert_eq
!(token_count("br\"\"br"), 1);
253 assert_eq
!(token_count("r#\"\"#r"), 1);
254 assert_eq
!(token_count("'c'c"), 1);
255 assert_eq
!(token_count("b'b'b"), 1);
256 assert_eq
!(token_count("0E"), 1);
257 assert_eq
!(token_count("0o0A"), 1);
258 assert_eq
!(token_count("0E--0"), 4);
259 assert_eq
!(token_count("0.0ECMA"), 1);
263 fn literal_iter_negative() {
264 let negative_literal
= Literal
::i32_suffixed(-3);
265 let tokens
= TokenStream
::from(TokenTree
::Literal(negative_literal
));
266 let mut iter
= tokens
.into_iter();
267 match iter
.next().unwrap() {
268 TokenTree
::Punct(punct
) => {
269 assert_eq
!(punct
.as_char(), '
-'
);
270 assert_eq
!(punct
.spacing(), Spacing
::Alone
);
272 unexpected
=> panic
!("unexpected token {:?}", unexpected
),
274 match iter
.next().unwrap() {
275 TokenTree
::Literal(literal
) => {
276 assert_eq
!(literal
.to_string(), "3i32");
278 unexpected
=> panic
!("unexpected token {:?}", unexpected
),
280 assert
!(iter
.next().is_none());
285 assert
!("1".parse
::<Literal
>().is_ok());
286 assert
!("-1".parse
::<Literal
>().is_ok());
287 assert
!("-1u12".parse
::<Literal
>().is_ok());
288 assert
!("1.0".parse
::<Literal
>().is_ok());
289 assert
!("-1.0".parse
::<Literal
>().is_ok());
290 assert
!("-1.0f12".parse
::<Literal
>().is_ok());
291 assert
!("'a'".parse
::<Literal
>().is_ok());
292 assert
!("\"\n\"".parse
::<Literal
>().is_ok());
293 assert
!("0 1".parse
::<Literal
>().is_err());
294 assert
!(" 0".parse
::<Literal
>().is_err());
295 assert
!("0 ".parse
::<Literal
>().is_err());
296 assert
!("/* comment */0".parse
::<Literal
>().is_err());
297 assert
!("0/* comment */".parse
::<Literal
>().is_err());
298 assert
!("0// comment".parse
::<Literal
>().is_err());
299 assert
!("- 1".parse
::<Literal
>().is_err());
300 assert
!("- 1.0".parse
::<Literal
>().is_err());
301 assert
!("-\"\"".parse
::<Literal
>().is_err());
306 let positive
= "0.1".parse
::<Literal
>().unwrap();
307 let negative
= "-0.1".parse
::<Literal
>().unwrap();
308 let subspan
= positive
.subspan(1..2);
310 #[cfg(not(span_locations))]
313 assert
!(subspan
.is_none());
316 #[cfg(span_locations)]
318 assert_eq
!(positive
.span().start().column
, 0);
319 assert_eq
!(positive
.span().end().column
, 3);
320 assert_eq
!(negative
.span().start().column
, 0);
321 assert_eq
!(negative
.span().end().column
, 4);
322 assert_eq
!(subspan
.unwrap().source_text().unwrap(), ".");
325 assert
!(positive
.subspan(1..4).is_none());
328 #[cfg(span_locations)]
331 let input
= " 𓀕 a z ";
332 let mut tokens
= input
333 .parse
::<proc_macro2
::TokenStream
>()
337 let first
= tokens
.next().unwrap();
338 assert_eq
!("𓀕", first
.span().source_text().unwrap());
340 let second
= tokens
.next().unwrap();
341 let third
= tokens
.next().unwrap();
342 assert_eq
!("z", third
.span().source_text().unwrap());
343 assert_eq
!("a", second
.span().source_text().unwrap());
348 fn roundtrip(p
: &str) {
349 println
!("parse: {}", p
);
350 let s
= p
.parse
::<TokenStream
>().unwrap().to_string();
351 println
!("first: {}", s
);
352 let s2
= s
.parse
::<TokenStream
>().unwrap().to_string();
372 0xffffffffffffffffffffffffffffffff
380 roundtrip("'static");
381 roundtrip("'\\u{10__FFFF}'");
382 roundtrip("\"\\u{10_F0FF__}foo\\u{1_0_0_0__}\"");
388 if let Ok(s
) = p
.parse
::<TokenStream
>() {
389 panic
!("should have failed to parse: {}\n{:#?}", p
, s
);
395 fail("\"\\u{0000000}\""); // overlong unicode escape (rust allows at most 6 hex digits)
396 fail("\"\\u{999999}\""); // outside of valid range of char
397 fail("\"\\u{_0}\""); // leading underscore
398 fail("\"\\u{}\""); // empty
399 fail("b\"\r\""); // bare carriage return in byte string
400 fail("r\"\r\""); // bare carriage return in raw string
401 fail("\"\\\r \""); // backslash carriage return
404 fail("\"\\\n\u{85}\r\"");
407 #[cfg(span_locations)]
412 /// This is a document comment
419 (1, 0, 1, 30), // [ ... ]
420 (1, 0, 1, 30), // doc
422 (1, 0, 1, 30), // "This is..."
423 (2, 0, 2, 7), // testing
424 (2, 8, 2, 11), // 123
425 (3, 0, 5, 1), // { ... }
426 (4, 2, 4, 9), // testing
427 (4, 10, 4, 13), // 234
432 #[cfg(procmacro2_semver_exempt)]
436 let start
= Span
::call_site().start();
437 assert_eq
!(start
.line
, 1);
438 assert_eq
!(start
.column
, 0);
439 let end
= Span
::call_site().end();
440 assert_eq
!(end
.line
, 1);
441 assert_eq
!(end
.column
, 0);
442 let source_file
= Span
::call_site().source_file();
443 assert_eq
!(source_file
.path().to_string_lossy(), "<unspecified>");
444 assert
!(!source_file
.is_real());
447 #[cfg(procmacro2_semver_exempt)]
450 let source1
= "aaa\nbbb"
451 .parse
::<TokenStream
>()
454 .collect
::<Vec
<_
>>();
455 let source2
= "ccc\nddd"
456 .parse
::<TokenStream
>()
459 .collect
::<Vec
<_
>>();
461 assert
!(source1
[0].span().source_file() != source2
[0].span().source_file());
463 source1
[0].span().source_file(),
464 source1
[1].span().source_file()
467 let joined1
= source1
[0].span().join(source1
[1].span());
468 let joined2
= source1
[0].span().join(source2
[0].span());
469 assert
!(joined1
.is_some());
470 assert
!(joined2
.is_none());
472 let start
= joined1
.unwrap().start();
473 let end
= joined1
.unwrap().end();
474 assert_eq
!(start
.line
, 1);
475 assert_eq
!(start
.column
, 0);
476 assert_eq
!(end
.line
, 2);
477 assert_eq
!(end
.column
, 3);
480 joined1
.unwrap().source_file(),
481 source1
[0].span().source_file()
487 let s
= str::from_utf8(b
"b\'\xc2\x86 \x00\x00\x00^\"").unwrap();
488 assert
!(s
.parse
::<TokenStream
>().is_err());
492 fn punct_before_comment() {
493 let mut tts
= TokenStream
::from_str("~// comment").unwrap().into_iter();
494 match tts
.next().unwrap() {
495 TokenTree
::Punct(tt
) => {
496 assert_eq
!(tt
.as_char(), '
~'
);
497 assert_eq
!(tt
.spacing(), Spacing
::Alone
);
499 wrong
=> panic
!("wrong token {:?}", wrong
),
504 fn joint_last_token() {
505 // This test verifies that we match the behavior of libproc_macro *not* in
506 // the range nightly-2020-09-06 through nightly-2020-09-10, in which this
507 // behavior was temporarily broken.
508 // See https://github.com/rust-lang/rust/issues/76399
510 let joint_punct
= Punct
::new('
:'
, Spacing
::Joint
);
511 let stream
= TokenStream
::from(TokenTree
::Punct(joint_punct
));
512 let punct
= match stream
.into_iter().next().unwrap() {
513 TokenTree
::Punct(punct
) => punct
,
516 assert_eq
!(punct
.spacing(), Spacing
::Joint
);
520 fn raw_identifier() {
521 let mut tts
= TokenStream
::from_str("r#dyn").unwrap().into_iter();
522 match tts
.next().unwrap() {
523 TokenTree
::Ident(raw
) => assert_eq
!("r#dyn", raw
.to_string()),
524 wrong
=> panic
!("wrong token {:?}", wrong
),
526 assert
!(tts
.next().is_none());
530 fn test_debug_ident() {
531 let ident
= Ident
::new("proc_macro", Span
::call_site());
533 #[cfg(not(span_locations))]
534 let expected
= "Ident(proc_macro)";
536 #[cfg(span_locations)]
537 let expected
= "Ident { sym: proc_macro }";
539 assert_eq
!(expected
, format
!("{:?}", ident
));
543 fn test_debug_tokenstream() {
544 let tts
= TokenStream
::from_str("[a + 1]").unwrap();
546 #[cfg(not(span_locations))]
551 stream: TokenStream [
567 #[cfg(not(span_locations))]
568 let expected_before_trailing_commas
= "\
572 stream: TokenStream [
588 #[cfg(span_locations)]
593 stream: TokenStream [
613 #[cfg(span_locations)]
614 let expected_before_trailing_commas
= "\
618 stream: TokenStream [
638 let actual
= format
!("{:#?}", tts
);
639 if actual
.ends_with(",\n]") {
640 assert_eq
!(expected
, actual
);
642 assert_eq
!(expected_before_trailing_commas
, actual
);
647 fn default_tokenstream_is_empty() {
648 let default_token_stream
= <TokenStream
as Default
>::default();
650 assert
!(default_token_stream
.is_empty());
654 fn tokenstream_size_hint() {
655 let tokens
= "a b (c d) e".parse
::<TokenStream
>().unwrap();
657 assert_eq
!(tokens
.into_iter().size_hint(), (4, Some(4)));
661 fn tuple_indexing() {
662 // This behavior may change depending on https://github.com/rust-lang/rust/pull/71322
663 let mut tokens
= "tuple.0.0".parse
::<TokenStream
>().unwrap().into_iter();
664 assert_eq
!("tuple", tokens
.next().unwrap().to_string());
665 assert_eq
!(".", tokens
.next().unwrap().to_string());
666 assert_eq
!("0.0", tokens
.next().unwrap().to_string());
667 assert
!(tokens
.next().is_none());
670 #[cfg(span_locations)]
672 fn non_ascii_tokens() {
673 check_spans("// abc", &[]);
674 check_spans("// ábc", &[]);
675 check_spans("// abc x", &[]);
676 check_spans("// ábc x", &[]);
677 check_spans("/* abc */ x", &[(1, 10, 1, 11)]);
678 check_spans("/* ábc */ x", &[(1, 10, 1, 11)]);
679 check_spans("/* ab\nc */ x", &[(2, 5, 2, 6)]);
680 check_spans("/* áb\nc */ x", &[(2, 5, 2, 6)]);
681 check_spans("/*** abc */ x", &[(1, 12, 1, 13)]);
682 check_spans("/*** ábc */ x", &[(1, 12, 1, 13)]);
683 check_spans(r
#""abc""#, &[(1, 0, 1, 5)]);
684 check_spans(r
#""ábc""#, &[(1, 0, 1, 5)]);
685 check_spans(r
##"r#"abc"#"##, &[(1, 0, 1, 8)]);
686 check_spans(r
##"r#"ábc"#"##, &[(1, 0, 1, 8)]);
687 check_spans("r#\"a\nc\"#", &[(1, 0, 2, 3)]);
688 check_spans("r#\"á\nc\"#", &[(1, 0, 2, 3)]);
689 check_spans("'a'", &[(1, 0, 1, 3)]);
690 check_spans("'á'", &[(1, 0, 1, 3)]);
691 check_spans("//! abc", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
692 check_spans("//! ábc", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
693 check_spans("//! abc\n", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
694 check_spans("//! ábc\n", &[(1, 0, 1, 7), (1, 0, 1, 7), (1, 0, 1, 7)]);
695 check_spans("/*! abc */", &[(1, 0, 1, 10), (1, 0, 1, 10), (1, 0, 1, 10)]);
696 check_spans("/*! ábc */", &[(1, 0, 1, 10), (1, 0, 1, 10), (1, 0, 1, 10)]);
697 check_spans("/*! a\nc */", &[(1, 0, 2, 4), (1, 0, 2, 4), (1, 0, 2, 4)]);
698 check_spans("/*! á\nc */", &[(1, 0, 2, 4), (1, 0, 2, 4), (1, 0, 2, 4)]);
699 check_spans("abc", &[(1, 0, 1, 3)]);
700 check_spans("ábc", &[(1, 0, 1, 3)]);
701 check_spans("ábć", &[(1, 0, 1, 3)]);
702 check_spans("abc// foo", &[(1, 0, 1, 3)]);
703 check_spans("ábc// foo", &[(1, 0, 1, 3)]);
704 check_spans("ábć// foo", &[(1, 0, 1, 3)]);
705 check_spans("b\"a\\\n c\"", &[(1, 0, 2, 3)]);
708 #[cfg(span_locations)]
709 fn check_spans(p
: &str, mut lines
: &[(usize, usize, usize, usize)]) {
710 let ts
= p
.parse
::<TokenStream
>().unwrap();
711 check_spans_internal(ts
, &mut lines
);
712 assert
!(lines
.is_empty(), "leftover ranges: {:?}", lines
);
715 #[cfg(span_locations)]
716 fn check_spans_internal(ts
: TokenStream
, lines
: &mut &[(usize, usize, usize, usize)]) {
718 if let Some((&(sline
, scol
, eline
, ecol
), rest
)) = lines
.split_first() {
721 let start
= i
.span().start();
722 assert_eq
!(start
.line
, sline
, "sline did not match for {}", i
);
723 assert_eq
!(start
.column
, scol
, "scol did not match for {}", i
);
725 let end
= i
.span().end();
726 assert_eq
!(end
.line
, eline
, "eline did not match for {}", i
);
727 assert_eq
!(end
.column
, ecol
, "ecol did not match for {}", i
);
729 if let TokenTree
::Group(g
) = i
{
730 check_spans_internal(g
.stream().clone(), lines
);
738 // space, horizontal tab, vertical tab, form feed, carriage return, line
739 // feed, non-breaking space, left-to-right mark, right-to-left mark
740 let various_spaces
= " \t\u{b}\u{c}\r\n\u{a0}\u{200e}\u{200f}";
741 let tokens
= various_spaces
.parse
::<TokenStream
>().unwrap();
742 assert_eq
!(tokens
.into_iter().count(), 0);
744 let lone_carriage_returns
= " \r \r\r\n ";
745 lone_carriage_returns
.parse
::<TokenStream
>().unwrap();
749 fn byte_order_mark() {
750 let string
= "\u{feff}foo";
751 let tokens
= string
.parse
::<TokenStream
>().unwrap();
752 match tokens
.into_iter().next().unwrap() {
753 TokenTree
::Ident(ident
) => assert_eq
!(ident
, "foo"),
757 let string
= "foo\u{feff}";
758 string
.parse
::<TokenStream
>().unwrap_err();