1 use rustc_ast
::ast
::AttrStyle
;
2 use rustc_ast
::token
::{self, CommentKind, Token, TokenKind}
;
3 use rustc_data_structures
::sync
::Lrc
;
4 use rustc_errors
::{emitter::EmitterWriter, Handler}
;
5 use rustc_parse
::lexer
::StringReader
;
6 use rustc_session
::parse
::ParseSess
;
7 use rustc_span
::source_map
::{FilePathMapping, SourceMap}
;
8 use rustc_span
::symbol
::Symbol
;
9 use rustc_span
::with_default_session_globals
;
10 use rustc_span
::{BytePos, Span}
;
13 use std
::path
::PathBuf
;
15 fn mk_sess(sm
: Lrc
<SourceMap
>) -> ParseSess
{
16 let emitter
= EmitterWriter
::new(
25 ParseSess
::with_span_handler(Handler
::with_emitter(true, None
, Box
::new(emitter
)), sm
)
28 // Creates a string reader for the given string.
29 fn setup
<'a
>(sm
: &SourceMap
, sess
: &'a ParseSess
, teststr
: String
) -> StringReader
<'a
> {
30 let sf
= sm
.new_source_file(PathBuf
::from(teststr
.clone()).into(), teststr
);
31 StringReader
::new(sess
, sf
, None
)
36 with_default_session_globals(|| {
37 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
38 let sh
= mk_sess(sm
.clone());
39 let mut string_reader
= setup(
42 "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(),
44 assert_eq
!(string_reader
.next_token(), token
::Comment
);
45 assert_eq
!(string_reader
.next_token(), token
::Whitespace
);
46 let tok1
= string_reader
.next_token();
47 let tok2
= Token
::new(mk_ident("fn"), Span
::with_root_ctxt(BytePos(21), BytePos(23)));
48 assert_eq
!(tok1
.kind
, tok2
.kind
);
49 assert_eq
!(tok1
.span
, tok2
.span
);
50 assert_eq
!(string_reader
.next_token(), token
::Whitespace
);
51 // Read another token.
52 let tok3
= string_reader
.next_token();
53 assert_eq
!(string_reader
.pos(), BytePos(28));
54 let tok4
= Token
::new(mk_ident("main"), Span
::with_root_ctxt(BytePos(24), BytePos(28)));
55 assert_eq
!(tok3
.kind
, tok4
.kind
);
56 assert_eq
!(tok3
.span
, tok4
.span
);
58 assert_eq
!(string_reader
.next_token(), token
::OpenDelim(token
::Paren
));
59 assert_eq
!(string_reader
.pos(), BytePos(29))
63 // Checks that the given reader produces the desired stream
64 // of tokens (stop checking after exhausting `expected`).
65 fn check_tokenization(mut string_reader
: StringReader
<'_
>, expected
: Vec
<TokenKind
>) {
66 for expected_tok
in &expected
{
67 assert_eq
!(&string_reader
.next_token(), expected_tok
);
71 // Makes the identifier by looking up the string in the interner.
72 fn mk_ident(id
: &str) -> TokenKind
{
73 token
::Ident(Symbol
::intern(id
), false)
76 fn mk_lit(kind
: token
::LitKind
, symbol
: &str, suffix
: Option
<&str>) -> TokenKind
{
77 TokenKind
::lit(kind
, Symbol
::intern(symbol
), suffix
.map(Symbol
::intern
))
81 fn doublecolon_parsing() {
82 with_default_session_globals(|| {
83 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
84 let sh
= mk_sess(sm
.clone());
86 setup(&sm
, &sh
, "a b".to_string()),
87 vec
![mk_ident("a"), token
::Whitespace
, mk_ident("b")],
93 fn doublecolon_parsing_2() {
94 with_default_session_globals(|| {
95 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
96 let sh
= mk_sess(sm
.clone());
98 setup(&sm
, &sh
, "a::b".to_string()),
99 vec
![mk_ident("a"), token
::Colon
, token
::Colon
, mk_ident("b")],
105 fn doublecolon_parsing_3() {
106 with_default_session_globals(|| {
107 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
108 let sh
= mk_sess(sm
.clone());
110 setup(&sm
, &sh
, "a ::b".to_string()),
111 vec
![mk_ident("a"), token
::Whitespace
, token
::Colon
, token
::Colon
, mk_ident("b")],
117 fn doublecolon_parsing_4() {
118 with_default_session_globals(|| {
119 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
120 let sh
= mk_sess(sm
.clone());
122 setup(&sm
, &sh
, "a:: b".to_string()),
123 vec
![mk_ident("a"), token
::Colon
, token
::Colon
, token
::Whitespace
, mk_ident("b")],
130 with_default_session_globals(|| {
131 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
132 let sh
= mk_sess(sm
.clone());
133 assert_eq
!(setup(&sm
, &sh
, "'a'".to_string()).next_token(), mk_lit(token
::Char
, "a", None
),);
138 fn character_space() {
139 with_default_session_globals(|| {
140 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
141 let sh
= mk_sess(sm
.clone());
142 assert_eq
!(setup(&sm
, &sh
, "' '".to_string()).next_token(), mk_lit(token
::Char
, " ", None
),);
147 fn character_escaped() {
148 with_default_session_globals(|| {
149 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
150 let sh
= mk_sess(sm
.clone());
152 setup(&sm
, &sh
, "'\\n'".to_string()).next_token(),
153 mk_lit(token
::Char
, "\\n", None
),
160 with_default_session_globals(|| {
161 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
162 let sh
= mk_sess(sm
.clone());
164 setup(&sm
, &sh
, "'abc".to_string()).next_token(),
165 token
::Lifetime(Symbol
::intern("'abc")),
172 with_default_session_globals(|| {
173 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
174 let sh
= mk_sess(sm
.clone());
176 setup(&sm
, &sh
, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(),
177 mk_lit(token
::StrRaw(3), "\"#a\\b\x00c\"", None
),
183 fn literal_suffixes() {
184 with_default_session_globals(|| {
185 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
186 let sh
= mk_sess(sm
.clone());
188 ($input
: expr
, $tok_type
: ident
, $tok_contents
: expr
) => {{
190 setup(&sm
, &sh
, format
!("{}suffix", $input
)).next_token(),
191 mk_lit(token
::$tok_type
, $tok_contents
, Some("suffix")),
193 // with a whitespace separator
195 setup(&sm
, &sh
, format
!("{} suffix", $input
)).next_token(),
196 mk_lit(token
::$tok_type
, $tok_contents
, None
),
201 test
!("'a'", Char
, "a");
202 test
!("b'a'", Byte
, "a");
203 test
!("\"a\"", Str
, "a");
204 test
!("b\"a\"", ByteStr
, "a");
205 test
!("1234", Integer
, "1234");
206 test
!("0b101", Integer
, "0b101");
207 test
!("0xABC", Integer
, "0xABC");
208 test
!("1.0", Float
, "1.0");
209 test
!("1.0e10", Float
, "1.0e10");
212 setup(&sm
, &sh
, "2us".to_string()).next_token(),
213 mk_lit(token
::Integer
, "2", Some("us")),
216 setup(&sm
, &sh
, "r###\"raw\"###suffix".to_string()).next_token(),
217 mk_lit(token
::StrRaw(3), "raw", Some("suffix")),
220 setup(&sm
, &sh
, "br###\"raw\"###suffix".to_string()).next_token(),
221 mk_lit(token
::ByteStrRaw(3), "raw", Some("suffix")),
227 fn nested_block_comments() {
228 with_default_session_globals(|| {
229 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
230 let sh
= mk_sess(sm
.clone());
231 let mut lexer
= setup(&sm
, &sh
, "/* /* */ */'a'".to_string());
232 assert_eq
!(lexer
.next_token(), token
::Comment
);
233 assert_eq
!(lexer
.next_token(), mk_lit(token
::Char
, "a", None
));
239 with_default_session_globals(|| {
240 let sm
= Lrc
::new(SourceMap
::new(FilePathMapping
::empty()));
241 let sh
= mk_sess(sm
.clone());
242 let mut lexer
= setup(&sm
, &sh
, "// test\r\n/// test\r\n".to_string());
243 let comment
= lexer
.next_token();
244 assert_eq
!(comment
.kind
, token
::Comment
);
245 assert_eq
!((comment
.span
.lo(), comment
.span
.hi()), (BytePos(0), BytePos(7)));
246 assert_eq
!(lexer
.next_token(), token
::Whitespace
);
249 token
::DocComment(CommentKind
::Line
, AttrStyle
::Outer
, Symbol
::intern(" test"))