[rustc.git] / compiler / rustc_lexer / src / tests.rs

use super::*;

use expect_test::{expect, Expect};

fn check_raw_str(s: &str, expected: Result<u8, RawStrError>) {
    let s = &format!("r{}", s);
    let mut cursor = Cursor::new(s);
    cursor.bump();
    let res = cursor.raw_double_quoted_string(0);
    assert_eq!(res, expected);
}

#[test]
fn test_naked_raw_str() {
    check_raw_str(r#""abc""#, Ok(0));
}

#[test]
fn test_raw_no_start() {
    check_raw_str(r##""abc"#"##, Ok(0));
}

#[test]
fn test_too_many_terminators() {
    // this error is handled in the parser later
    check_raw_str(r###"#"abc"##"###, Ok(1));
}

#[test]
fn test_unterminated() {
    check_raw_str(
        r#"#"abc"#,
        Err(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }),
    );
    check_raw_str(
        r###"##"abc"#"###,
        Err(RawStrError::NoTerminator {
            expected: 2,
            found: 1,
            possible_terminator_offset: Some(7),
        }),
    );
    // We're looking for "# not just any #
    check_raw_str(
        r###"##"abc#"###,
        Err(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }),
    )
}

#[test]
fn test_invalid_start() {
    check_raw_str(r##"#~"abc"#"##, Err(RawStrError::InvalidStarter { bad_char: '~' }));
}

#[test]
fn test_unterminated_no_pound() {
    // https://github.com/rust-lang/rust/issues/70677
    check_raw_str(
        r#"""#,
        Err(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }),
    );
}

#[test]
fn test_too_many_hashes() {
    let max_count = u8::MAX;
    let hashes1 = "#".repeat(max_count as usize);
    let hashes2 = "#".repeat(max_count as usize + 1);
    let middle = "\"abc\"";
    let s1 = [&hashes1, middle, &hashes1].join("");
    let s2 = [&hashes2, middle, &hashes2].join("");

    // Valid number of hashes (255 = 2^8 - 1 = u8::MAX).
    check_raw_str(&s1, Ok(255));

    // One more hash sign (256 = 2^8) becomes too many.
    check_raw_str(&s2, Err(RawStrError::TooManyDelimiters { found: u32::from(max_count) + 1 }));
}

#[test]
fn test_valid_shebang() {
    // https://github.com/rust-lang/rust/issues/70528
    let input = "#!/usr/bin/rustrun\nlet x = 5;";
    assert_eq!(strip_shebang(input), Some(18));
}

#[test]
fn test_invalid_shebang_valid_rust_syntax() {
    // https://github.com/rust-lang/rust/issues/70528
    let input = "#!    [bad_attribute]";
    assert_eq!(strip_shebang(input), None);
}

#[test]
fn test_shebang_second_line() {
    // Because shebangs are interpreted by the kernel, they must be on the first line
    let input = "\n#!/bin/bash";
    assert_eq!(strip_shebang(input), None);
}

#[test]
fn test_shebang_space() {
    let input = "#!    /bin/bash";
    assert_eq!(strip_shebang(input), Some(input.len()));
}

#[test]
fn test_shebang_empty_shebang() {
    let input = "#!    \n[attribute(foo)]";
    assert_eq!(strip_shebang(input), None);
}

#[test]
fn test_invalid_shebang_comment() {
    let input = "#!//bin/ami/a/comment\n[";
    assert_eq!(strip_shebang(input), None)
}

#[test]
fn test_invalid_shebang_another_comment() {
    let input = "#!/*bin/ami/a/comment*/\n[attribute";
    assert_eq!(strip_shebang(input), None)
}

#[test]
fn test_shebang_valid_rust_after() {
    let input = "#!/*bin/ami/a/comment*/\npub fn main() {}";
    assert_eq!(strip_shebang(input), Some(23))
}

#[test]
fn test_shebang_followed_by_attrib() {
    let input = "#!/bin/rust-scripts\n#![allow_unused(true)]";
    assert_eq!(strip_shebang(input), Some(19));
}

fn check_lexing(src: &str, expect: Expect) {
    let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect();
    expect.assert_eq(&actual)
}

#[test]
fn smoke_test() {
    check_lexing(
        "/* my source file */ fn main() { println!(\"zebra\"); }\n",
        expect![[r#"
            Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Ident, len: 2 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Ident, len: 4 }
            Token { kind: OpenParen, len: 1 }
            Token { kind: CloseParen, len: 1 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: OpenBrace, len: 1 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Ident, len: 7 }
            Token { kind: Bang, len: 1 }
            Token { kind: OpenParen, len: 1 }
            Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 }
            Token { kind: CloseParen, len: 1 }
            Token { kind: Semi, len: 1 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: CloseBrace, len: 1 }
            Token { kind: Whitespace, len: 1 }
        "#]],
    )
}

#[test]
fn comment_flavors() {
    check_lexing(
        r"
// line
//// line as well
/// outer doc line
//! inner doc line
/* block */
/**/
/*** also block */
/** outer doc block */
/*! inner doc block */
",
        expect![[r#"
            Token { kind: Whitespace, len: 1 }
            Token { kind: LineComment { doc_style: None }, len: 7 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: LineComment { doc_style: None }, len: 17 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
            Token { kind: Whitespace, len: 1 }
        "#]],
    )
}

#[test]
fn nested_block_comments() {
    check_lexing(
        "/* /* */ */'a'",
        expect![[r#"
            Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
            Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
        "#]],
    )
}

#[test]
fn characters() {
    check_lexing(
        "'a' ' ' '\\n'",
        expect![[r#"
            Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 }
        "#]],
    );
}

#[test]
fn lifetime() {
    check_lexing(
        "'abc",
        expect![[r#"
            Token { kind: Lifetime { starts_with_number: false }, len: 4 }
        "#]],
    );
}

#[test]
fn raw_string() {
    check_lexing(
        "r###\"\"#a\\b\x00c\"\"###",
        expect![[r#"
            Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 }
        "#]],
    )
}

#[test]
fn literal_suffixes() {
    check_lexing(
        r####"
'a'
b'a'
"a"
b"a"
1234
0b101
0xABC
1.0
1.0e10
2us
r###"raw"###suffix
br###"raw"###suffix
"####,
        expect![[r#"
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 12 }, len: 18 }
            Token { kind: Whitespace, len: 1 }
            Token { kind: Literal { kind: RawByteStr { n_hashes: Some(3) }, suffix_start: 13 }, len: 19 }
            Token { kind: Whitespace, len: 1 }
        "#]],
    )
}
Commit	Line	Data
1b1a35ee XL	1	use super::*;
	2
	3	use expect_test::{expect, Expect};
	4
064997fb	5	fn check_raw_str(s: &str, expected: Result<u8, RawStrError>) {
1b1a35ee XL	6	let s = &format!("r{}", s);
	7	let mut cursor = Cursor::new(s);
	8	cursor.bump();
064997fb FG	9	let res = cursor.raw_double_quoted_string(0);
064997fb FG	10	assert_eq!(res, expected);
1b1a35ee XL	11	}
	12
	13	#[test]
	14	fn test_naked_raw_str() {
064997fb	15	check_raw_str(r#""abc""#, Ok(0));
1b1a35ee XL	16	}
	17
	18	#[test]
	19	fn test_raw_no_start() {
064997fb	20	check_raw_str(r##""abc"#"##, Ok(0));
1b1a35ee XL	21	}
	22
	23	#[test]
	24	fn test_too_many_terminators() {
	25	// this error is handled in the parser later
064997fb	26	check_raw_str(r###"#"abc"##"###, Ok(1));
1b1a35ee XL	27	}
	28
	29	#[test]
	30	fn test_unterminated() {
	31	check_raw_str(
	32	r#"#"abc"#,
064997fb	33	Err(RawStrError::NoTerminator { expected: 1, found: 0, possible_terminator_offset: None }),
1b1a35ee XL	34	);
	35	check_raw_str(
	36	r###"##"abc"#"###,
064997fb	37	Err(RawStrError::NoTerminator {
1b1a35ee XL	38	expected: 2,
	39	found: 1,
	40	possible_terminator_offset: Some(7),
	41	}),
	42	);
	43	// We're looking for "# not just any #
	44	check_raw_str(
	45	r###"##"abc#"###,
064997fb	46	Err(RawStrError::NoTerminator { expected: 2, found: 0, possible_terminator_offset: None }),
1b1a35ee XL	47	)
	48	}
	49
	50	#[test]
	51	fn test_invalid_start() {
064997fb	52	check_raw_str(r##"#~"abc"#"##, Err(RawStrError::InvalidStarter { bad_char: '~' }));
1b1a35ee XL	53	}
	54
	55	#[test]
	56	fn test_unterminated_no_pound() {
	57	// https://github.com/rust-lang/rust/issues/70677
	58	check_raw_str(
	59	r#"""#,
064997fb	60	Err(RawStrError::NoTerminator { expected: 0, found: 0, possible_terminator_offset: None }),
1b1a35ee XL	61	);
	62	}
	63
5e7ed085 FG	64	#[test]
	65	fn test_too_many_hashes() {
	66	let max_count = u8::MAX;
064997fb FG	67	let hashes1 = "#".repeat(max_count as usize);
	68	let hashes2 = "#".repeat(max_count as usize + 1);
	69	let middle = "\"abc\"";
	70	let s1 = [&hashes1, middle, &hashes1].join("");
	71	let s2 = [&hashes2, middle, &hashes2].join("");
5e7ed085	72
064997fb FG	73	// Valid number of hashes (255 = 2^8 - 1 = u8::MAX).
064997fb FG	74	check_raw_str(&s1, Ok(255));
5e7ed085 FG	75
5e7ed085 FG	76	// One more hash sign (256 = 2^8) becomes too many.
064997fb	77	check_raw_str(&s2, Err(RawStrError::TooManyDelimiters { found: u32::from(max_count) + 1 }));
5e7ed085 FG	78	}
5e7ed085 FG	79
1b1a35ee XL	80	#[test]
	81	fn test_valid_shebang() {
	82	// https://github.com/rust-lang/rust/issues/70528
	83	let input = "#!/usr/bin/rustrun\nlet x = 5;";
	84	assert_eq!(strip_shebang(input), Some(18));
	85	}
	86
	87	#[test]
	88	fn test_invalid_shebang_valid_rust_syntax() {
	89	// https://github.com/rust-lang/rust/issues/70528
	90	let input = "#! [bad_attribute]";
	91	assert_eq!(strip_shebang(input), None);
	92	}
	93
	94	#[test]
	95	fn test_shebang_second_line() {
	96	// Because shebangs are interpreted by the kernel, they must be on the first line
	97	let input = "\n#!/bin/bash";
	98	assert_eq!(strip_shebang(input), None);
	99	}
	100
	101	#[test]
	102	fn test_shebang_space() {
	103	let input = "#! /bin/bash";
	104	assert_eq!(strip_shebang(input), Some(input.len()));
	105	}
	106
	107	#[test]
	108	fn test_shebang_empty_shebang() {
	109	let input = "#! \n[attribute(foo)]";
	110	assert_eq!(strip_shebang(input), None);
	111	}
	112
	113	#[test]
	114	fn test_invalid_shebang_comment() {
	115	let input = "#!//bin/ami/a/comment\n[";
	116	assert_eq!(strip_shebang(input), None)
	117	}
	118
	119	#[test]
	120	fn test_invalid_shebang_another_comment() {
	121	let input = "#!/bin/ami/a/comment/\n[attribute";
	122	assert_eq!(strip_shebang(input), None)
	123	}
	124
	125	#[test]
	126	fn test_shebang_valid_rust_after() {
	127	let input = "#!/bin/ami/a/comment/\npub fn main() {}";
	128	assert_eq!(strip_shebang(input), Some(23))
	129	}
	130
	131	#[test]
	132	fn test_shebang_followed_by_attrib() {
	133	let input = "#!/bin/rust-scripts\n#![allow_unused(true)]";
	134	assert_eq!(strip_shebang(input), Some(19));
	135	}
	136
	137	fn check_lexing(src: &str, expect: Expect) {
	138	let actual: String = tokenize(src).map(\|token\| format!("{:?}\n", token)).collect();
	139	expect.assert_eq(&actual)
	140	}
	141
	142	#[test]
	143	fn smoke_test() {
144	check_lexing(
145	"/* my source file */ fn main() { println!(\"zebra\"); }\n",
146	expect![[r#"
147	Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 }
148	Token { kind: Whitespace, len: 1 }
149	Token { kind: Ident, len: 2 }
150	Token { kind: Whitespace, len: 1 }
151	Token { kind: Ident, len: 4 }
152	Token { kind: OpenParen, len: 1 }
153	Token { kind: CloseParen, len: 1 }
154	Token { kind: Whitespace, len: 1 }
155	Token { kind: OpenBrace, len: 1 }
156	Token { kind: Whitespace, len: 1 }
157	Token { kind: Ident, len: 7 }
158	Token { kind: Bang, len: 1 }
159	Token { kind: OpenParen, len: 1 }
160	Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 7 }, len: 7 }
161	Token { kind: CloseParen, len: 1 }
162	Token { kind: Semi, len: 1 }
163	Token { kind: Whitespace, len: 1 }
164	Token { kind: CloseBrace, len: 1 }
165	Token { kind: Whitespace, len: 1 }
166	"#]],
167	)
168	}
169
170	#[test]
171	fn comment_flavors() {
172	check_lexing(
173	r"
174	// line
175	//// line as well
176	/// outer doc line
177	//! inner doc line
178	/* block */
179	/**/
180	/*** also block */
181	/** outer doc block */
182	/! inner doc block /
183	",
184	expect![[r#"
185	Token { kind: Whitespace, len: 1 }
186	Token { kind: LineComment { doc_style: None }, len: 7 }
187	Token { kind: Whitespace, len: 1 }
188	Token { kind: LineComment { doc_style: None }, len: 17 }
189	Token { kind: Whitespace, len: 1 }
190	Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
191	Token { kind: Whitespace, len: 1 }
192	Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
193	Token { kind: Whitespace, len: 1 }
194	Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
195	Token { kind: Whitespace, len: 1 }
196	Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
197	Token { kind: Whitespace, len: 1 }
198	Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
199	Token { kind: Whitespace, len: 1 }
200	Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
201	Token { kind: Whitespace, len: 1 }
202	Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
203	Token { kind: Whitespace, len: 1 }
204	"#]],
205	)
206	}
207
208	#[test]
209	fn nested_block_comments() {
210	check_lexing(
211	"/* /* / /'a'",
212	expect![[r#"
213	Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
214	Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
215	"#]],
216	)
217	}
218
219	#[test]
220	fn characters() {
221	check_lexing(
222	"'a' ' ' '\\n'",
223	expect![[r#"
224	Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
225	Token { kind: Whitespace, len: 1 }
226	Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
227	Token { kind: Whitespace, len: 1 }
228	Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 4 }, len: 4 }
229	"#]],
230	);
231	}
232
233	#[test]
234	fn lifetime() {
235	check_lexing(
236	"'abc",
237	expect![[r#"
238	Token { kind: Lifetime { starts_with_number: false }, len: 4 }
239	"#]],
240	);
241	}
242
243	#[test]
244	fn raw_string() {
245	check_lexing(
246	"r###\"\"#a\\b\x00c\"\"###",
247	expect![[r#"
064997fb	248	Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 17 }, len: 17 }
1b1a35ee XL	249	"#]],
	250	)
	251	}
	252
	253	#[test]
	254	fn literal_suffixes() {
	255	check_lexing(
	256	r####"
	257	'a'
	258	b'a'
	259	"a"
	260	b"a"
	261	1234
	262	0b101
	263	0xABC
	264	1.0
	265	1.0e10
	266	2us
	267	r###"raw"###suffix
	268	br###"raw"###suffix
	269	"####,
	270	expect![[r#"
	271	Token { kind: Whitespace, len: 1 }
	272	Token { kind: Literal { kind: Char { terminated: true }, suffix_start: 3 }, len: 3 }
	273	Token { kind: Whitespace, len: 1 }
	274	Token { kind: Literal { kind: Byte { terminated: true }, suffix_start: 4 }, len: 4 }
	275	Token { kind: Whitespace, len: 1 }
	276	Token { kind: Literal { kind: Str { terminated: true }, suffix_start: 3 }, len: 3 }
	277	Token { kind: Whitespace, len: 1 }
	278	Token { kind: Literal { kind: ByteStr { terminated: true }, suffix_start: 4 }, len: 4 }
	279	Token { kind: Whitespace, len: 1 }
	280	Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 4 }, len: 4 }
	281	Token { kind: Whitespace, len: 1 }
	282	Token { kind: Literal { kind: Int { base: Binary, empty_int: false }, suffix_start: 5 }, len: 5 }
	283	Token { kind: Whitespace, len: 1 }
	284	Token { kind: Literal { kind: Int { base: Hexadecimal, empty_int: false }, suffix_start: 5 }, len: 5 }
	285	Token { kind: Whitespace, len: 1 }
	286	Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 3 }, len: 3 }
	287	Token { kind: Whitespace, len: 1 }
	288	Token { kind: Literal { kind: Float { base: Decimal, empty_exponent: false }, suffix_start: 6 }, len: 6 }
	289	Token { kind: Whitespace, len: 1 }
	290	Token { kind: Literal { kind: Int { base: Decimal, empty_int: false }, suffix_start: 1 }, len: 3 }
	291	Token { kind: Whitespace, len: 1 }
064997fb	292	Token { kind: Literal { kind: RawStr { n_hashes: Some(3) }, suffix_start: 12 }, len: 18 }
1b1a35ee	293	Token { kind: Whitespace, len: 1 }
064997fb	294	Token { kind: Literal { kind: RawByteStr { n_hashes: Some(3) }, suffix_start: 13 }, len: 19 }
1b1a35ee XL	295	Token { kind: Whitespace, len: 1 }
	296	"#]],
	297	)
	298	}