[rustc.git] / vendor / toml_edit / src / parser / strings.rs

use std::borrow::Cow;
use std::char;
use std::ops::RangeInclusive;

use winnow::combinator::alt;
use winnow::combinator::cut_err;
use winnow::combinator::delimited;
use winnow::combinator::fail;
use winnow::combinator::opt;
use winnow::combinator::peek;
use winnow::combinator::preceded;
use winnow::combinator::repeat;
use winnow::combinator::success;
use winnow::combinator::terminated;
use winnow::prelude::*;
use winnow::stream::Stream;
use winnow::token::any;
use winnow::token::none_of;
use winnow::token::one_of;
use winnow::token::tag;
use winnow::token::take_while;
use winnow::trace::trace;

use crate::parser::errors::CustomError;
use crate::parser::numbers::HEXDIG;
use crate::parser::prelude::*;
use crate::parser::trivia::{from_utf8_unchecked, newline, ws, ws_newlines, NON_ASCII, WSCHAR};

// ;; String

// string = ml-basic-string / basic-string / ml-literal-string / literal-string
pub(crate) fn string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    trace(
        "string",
        alt((
            ml_basic_string,
            basic_string,
            ml_literal_string,
            literal_string.map(Cow::Borrowed),
        )),
    )
    .parse_next(input)
}

// ;; Basic String

// basic-string = quotation-mark *basic-char quotation-mark
pub(crate) fn basic_string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    trace("basic-string", |input: &mut Input<'i>| {
        let _ = one_of(QUOTATION_MARK).parse_next(input)?;

        let mut c = Cow::Borrowed("");
        if let Some(ci) = opt(basic_chars).parse_next(input)? {
            c = ci;
        }
        while let Some(ci) = opt(basic_chars).parse_next(input)? {
            c.to_mut().push_str(&ci);
        }

        let _ = cut_err(one_of(QUOTATION_MARK))
            .context(StrContext::Label("basic string"))
            .parse_next(input)?;

        Ok(c)
    })
    .parse_next(input)
}

// quotation-mark = %x22            ; "
pub(crate) const QUOTATION_MARK: u8 = b'"';

// basic-char = basic-unescaped / escaped
fn basic_chars<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    alt((
        // Deviate from the official grammar by batching the unescaped chars so we build a string a
        // chunk at a time, rather than a `char` at a time.
        take_while(1.., BASIC_UNESCAPED)
            .try_map(std::str::from_utf8)
            .map(Cow::Borrowed),
        escaped.map(|c| Cow::Owned(String::from(c))),
    ))
    .parse_next(input)
}

// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
pub(crate) const BASIC_UNESCAPED: (
    (u8, u8),
    u8,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
) = (WSCHAR, 0x21, 0x23..=0x5B, 0x5D..=0x7E, NON_ASCII);

// escaped = escape escape-seq-char
fn escaped(input: &mut Input<'_>) -> PResult<char> {
    preceded(ESCAPE, escape_seq_char).parse_next(input)
}

// escape = %x5C                    ; \
pub(crate) const ESCAPE: u8 = b'\\';

// escape-seq-char =  %x22         ; "    quotation mark  U+0022
// escape-seq-char =/ %x5C         ; \    reverse solidus U+005C
// escape-seq-char =/ %x62         ; b    backspace       U+0008
// escape-seq-char =/ %x66         ; f    form feed       U+000C
// escape-seq-char =/ %x6E         ; n    line feed       U+000A
// escape-seq-char =/ %x72         ; r    carriage return U+000D
// escape-seq-char =/ %x74         ; t    tab             U+0009
// escape-seq-char =/ %x75 4HEXDIG ; uXXXX                U+XXXX
// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX            U+XXXXXXXX
fn escape_seq_char(input: &mut Input<'_>) -> PResult<char> {
    dispatch! {any;
        b'b' => success('\u{8}'),
        b'f' => success('\u{c}'),
        b'n' => success('\n'),
        b'r' => success('\r'),
        b't' => success('\t'),
        b'u' => cut_err(hexescape::<4>).context(StrContext::Label("unicode 4-digit hex code")),
        b'U' => cut_err(hexescape::<8>).context(StrContext::Label("unicode 8-digit hex code")),
        b'\\' => success('\\'),
        b'"' => success('"'),
        _ => {
            cut_err(fail::<_, char, _>)
            .context(StrContext::Label("escape sequence"))
            .context(StrContext::Expected(StrContextValue::CharLiteral('b')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('f')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('n')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('r')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('t')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('u')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('U')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('\\')))
            .context(StrContext::Expected(StrContextValue::CharLiteral('"')))
        }
    }
    .parse_next(input)
}

pub(crate) fn hexescape<const N: usize>(input: &mut Input<'_>) -> PResult<char> {
    take_while(0..=N, HEXDIG)
        .verify(|b: &[u8]| b.len() == N)
        .map(|b: &[u8]| unsafe { from_utf8_unchecked(b, "`is_ascii_digit` filters out on-ASCII") })
        .verify_map(|s| u32::from_str_radix(s, 16).ok())
        .try_map(|h| char::from_u32(h).ok_or(CustomError::OutOfRange))
        .parse_next(input)
}

// ;; Multiline Basic String

// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
//                   ml-basic-string-delim
fn ml_basic_string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    trace(
        "ml-basic-string",
        delimited(
            ML_BASIC_STRING_DELIM,
            preceded(opt(newline), cut_err(ml_basic_body)),
            cut_err(ML_BASIC_STRING_DELIM),
        )
        .context(StrContext::Label("multiline basic string")),
    )
    .parse_next(input)
}

// ml-basic-string-delim = 3quotation-mark
pub(crate) const ML_BASIC_STRING_DELIM: &[u8] = b"\"\"\"";

// ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
fn ml_basic_body<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    let mut c = Cow::Borrowed("");
    if let Some(ci) = opt(mlb_content).parse_next(input)? {
        c = ci;
    }
    while let Some(ci) = opt(mlb_content).parse_next(input)? {
        c.to_mut().push_str(&ci);
    }

    while let Some(qi) = opt(mlb_quotes(none_of(b'\"').value(()))).parse_next(input)? {
        if let Some(ci) = opt(mlb_content).parse_next(input)? {
            c.to_mut().push_str(qi);
            c.to_mut().push_str(&ci);
            while let Some(ci) = opt(mlb_content).parse_next(input)? {
                c.to_mut().push_str(&ci);
            }
        } else {
            break;
        }
    }

    if let Some(qi) = opt(mlb_quotes(tag(ML_BASIC_STRING_DELIM).value(()))).parse_next(input)? {
        c.to_mut().push_str(qi);
    }

    Ok(c)
}

// mlb-content = mlb-char / newline / mlb-escaped-nl
// mlb-char = mlb-unescaped / escaped
fn mlb_content<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    alt((
        // Deviate from the official grammar by batching the unescaped chars so we build a string a
        // chunk at a time, rather than a `char` at a time.
        take_while(1.., MLB_UNESCAPED)
            .try_map(std::str::from_utf8)
            .map(Cow::Borrowed),
        // Order changed fromg grammar so `escaped` can more easily `cut_err` on bad escape sequences
        mlb_escaped_nl.map(|_| Cow::Borrowed("")),
        escaped.map(|c| Cow::Owned(String::from(c))),
        newline.map(|_| Cow::Borrowed("\n")),
    ))
    .parse_next(input)
}

// mlb-quotes = 1*2quotation-mark
fn mlb_quotes<'i>(
    mut term: impl winnow::Parser<Input<'i>, (), ContextError>,
) -> impl Parser<Input<'i>, &'i str, ContextError> {
    move |input: &mut Input<'i>| {
        let start = input.checkpoint();
        let res = terminated(b"\"\"", peek(term.by_ref()))
            .map(|b| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
            .parse_next(input);

        match res {
            Err(winnow::error::ErrMode::Backtrack(_)) => {
                input.reset(start);
                terminated(b"\"", peek(term.by_ref()))
                    .map(|b| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
                    .parse_next(input)
            }
            res => res,
        }
    }
}

// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
pub(crate) const MLB_UNESCAPED: (
    (u8, u8),
    u8,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
) = (WSCHAR, 0x21, 0x23..=0x5B, 0x5D..=0x7E, NON_ASCII);

// mlb-escaped-nl = escape ws newline *( wschar / newline
// When the last non-whitespace character on a line is a \,
// it will be trimmed along with all whitespace
// (including newlines) up to the next non-whitespace
// character or closing delimiter.
fn mlb_escaped_nl(input: &mut Input<'_>) -> PResult<()> {
    repeat(1.., (ESCAPE, ws, ws_newlines))
        .map(|()| ())
        .value(())
        .parse_next(input)
}

// ;; Literal String

// literal-string = apostrophe *literal-char apostrophe
pub(crate) fn literal_string<'i>(input: &mut Input<'i>) -> PResult<&'i str> {
    trace(
        "literal-string",
        delimited(
            APOSTROPHE,
            cut_err(take_while(0.., LITERAL_CHAR)),
            cut_err(APOSTROPHE),
        )
        .try_map(std::str::from_utf8)
        .context(StrContext::Label("literal string")),
    )
    .parse_next(input)
}

// apostrophe = %x27 ; ' apostrophe
pub(crate) const APOSTROPHE: u8 = b'\'';

// literal-char = %x09 / %x20-26 / %x28-7E / non-ascii
pub(crate) const LITERAL_CHAR: (
    u8,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
) = (0x9, 0x20..=0x26, 0x28..=0x7E, NON_ASCII);

// ;; Multiline Literal String

// ml-literal-string = ml-literal-string-delim [ newline ] ml-literal-body
//                     ml-literal-string-delim
fn ml_literal_string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
    trace(
        "ml-literal-string",
        delimited(
            (ML_LITERAL_STRING_DELIM, opt(newline)),
            cut_err(ml_literal_body.map(|t| {
                if t.contains("\r\n") {
                    Cow::Owned(t.replace("\r\n", "\n"))
                } else {
                    Cow::Borrowed(t)
                }
            })),
            cut_err(ML_LITERAL_STRING_DELIM),
        )
        .context(StrContext::Label("multiline literal string")),
    )
    .parse_next(input)
}

// ml-literal-string-delim = 3apostrophe
pub(crate) const ML_LITERAL_STRING_DELIM: &[u8] = b"'''";

// ml-literal-body = *mll-content *( mll-quotes 1*mll-content ) [ mll-quotes ]
fn ml_literal_body<'i>(input: &mut Input<'i>) -> PResult<&'i str> {
    (
        repeat(0.., mll_content).map(|()| ()),
        repeat(
            0..,
            (
                mll_quotes(none_of(APOSTROPHE).value(())),
                repeat(1.., mll_content).map(|()| ()),
            ),
        )
        .map(|()| ()),
        opt(mll_quotes(tag(ML_LITERAL_STRING_DELIM).value(()))),
    )
        .recognize()
        .try_map(std::str::from_utf8)
        .parse_next(input)
}

// mll-content = mll-char / newline
fn mll_content(input: &mut Input<'_>) -> PResult<u8> {
    alt((one_of(MLL_CHAR), newline)).parse_next(input)
}

// mll-char = %x09 / %x20-26 / %x28-7E / non-ascii
const MLL_CHAR: (
    u8,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
    RangeInclusive<u8>,
) = (0x9, 0x20..=0x26, 0x28..=0x7E, NON_ASCII);

// mll-quotes = 1*2apostrophe
fn mll_quotes<'i>(
    mut term: impl winnow::Parser<Input<'i>, (), ContextError>,
) -> impl Parser<Input<'i>, &'i str, ContextError> {
    move |input: &mut Input<'i>| {
        let start = input.checkpoint();
        let res = terminated(b"''", peek(term.by_ref()))
            .map(|b| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
            .parse_next(input);

        match res {
            Err(winnow::error::ErrMode::Backtrack(_)) => {
                input.reset(start);
                terminated(b"'", peek(term.by_ref()))
                    .map(|b| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
                    .parse_next(input)
            }
            res => res,
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn basic_string() {
        let input =
            r#""I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF. \U0002070E""#;
        let expected = "I\'m a string. \"You can quote me\". Name\tJosé\nLocation\tSF. \u{2070E}";
        let parsed = string.parse(new_input(input));
        assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
    }

    #[test]
    fn ml_basic_string() {
        let cases = [
            (
                r#""""
Roses are red
Violets are blue""""#,
                r#"Roses are red
Violets are blue"#,
            ),
            (r#"""" \""" """"#, " \"\"\" "),
            (r#"""" \\""""#, " \\"),
        ];

        for &(input, expected) in &cases {
            let parsed = string.parse(new_input(input));
            assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
        }

        let invalid_cases = [r#""""  """#, r#""""  \""""#];

        for input in &invalid_cases {
            let parsed = string.parse(new_input(input));
            assert!(parsed.is_err());
        }
    }

    #[test]
    fn ml_basic_string_escape_ws() {
        let inputs = [
            r#""""
The quick brown \


  fox jumps over \
    the lazy dog.""""#,
            r#""""\
       The quick brown \
       fox jumps over \
       the lazy dog.\
       """"#,
        ];
        for input in &inputs {
            let expected = "The quick brown fox jumps over the lazy dog.";
            let parsed = string.parse(new_input(input));
            assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
        }
        let empties = [
            r#""""\
       """"#,
            r#""""
\
  \
""""#,
        ];
        for input in &empties {
            let expected = "";
            let parsed = string.parse(new_input(input));
            assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
        }
    }

    #[test]
    fn literal_string() {
        let inputs = [
            r"'C:\Users\nodejs\templates'",
            r"'\\ServerX\admin$\system32\'",
            r#"'Tom "Dubs" Preston-Werner'"#,
            r"'<\i\c*\s*>'",
        ];

        for input in &inputs {
            let expected = &input[1..input.len() - 1];
            let parsed = string.parse(new_input(input));
            assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
        }
    }

    #[test]
    fn ml_literal_string() {
        let inputs = [
            r"'''I [dw]on't need \d{2} apples'''",
            r#"''''one_quote''''"#,
        ];
        for input in &inputs {
            let expected = &input[3..input.len() - 3];
            let parsed = string.parse(new_input(input));
            assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
        }

        let input = r#"'''
The first newline is
trimmed in raw strings.
   All other whitespace
   is preserved.
'''"#;
        let expected = &input[4..input.len() - 3];
        let parsed = string.parse(new_input(input));
        assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
    }
}
Commit	Line	Data
0a29b90c FG	1	use std::borrow::Cow;
	2	use std::char;
	3	use std::ops::RangeInclusive;
	4
fe692bf9	5	use winnow::combinator::alt;
49aad941	6	use winnow::combinator::cut_err;
fe692bf9	7	use winnow::combinator::delimited;
49aad941 FG	8	use winnow::combinator::fail;
	9	use winnow::combinator::opt;
	10	use winnow::combinator::peek;
fe692bf9 FG	11	use winnow::combinator::preceded;
fe692bf9 FG	12	use winnow::combinator::repeat;
49aad941	13	use winnow::combinator::success;
fe692bf9	14	use winnow::combinator::terminated;
49aad941	15	use winnow::prelude::*;
add651ee	16	use winnow::stream::Stream;
fe692bf9 FG	17	use winnow::token::any;
	18	use winnow::token::none_of;
	19	use winnow::token::one_of;
	20	use winnow::token::tag;
	21	use winnow::token::take_while;
add651ee	22	use winnow::trace::trace;
0a29b90c FG	23
	24	use crate::parser::errors::CustomError;
	25	use crate::parser::numbers::HEXDIG;
	26	use crate::parser::prelude::*;
	27	use crate::parser::trivia::{from_utf8_unchecked, newline, ws, ws_newlines, NON_ASCII, WSCHAR};
	28
	29	// ;; String
	30
	31	// string = ml-basic-string / basic-string / ml-literal-string / literal-string
add651ee FG	32	pub(crate) fn string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
	33	trace(
	34	"string",
	35	alt((
	36	ml_basic_string,
	37	basic_string,
	38	ml_literal_string,
	39	literal_string.map(Cow::Borrowed),
	40	)),
	41	)
49aad941	42	.parse_next(input)
0a29b90c FG	43	}
	44
	45	// ;; Basic String
	46
	47	// basic-string = quotation-mark *basic-char quotation-mark
add651ee FG	48	pub(crate) fn basic_string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
	49	trace("basic-string", \|input: &mut Input<'i>\| {
	50	let _ = one_of(QUOTATION_MARK).parse_next(input)?;
0a29b90c	51
add651ee FG	52	let mut c = Cow::Borrowed("");
	53	if let Some(ci) = opt(basic_chars).parse_next(input)? {
	54	c = ci;
	55	}
	56	while let Some(ci) = opt(basic_chars).parse_next(input)? {
	57	c.to_mut().push_str(&ci);
	58	}
0a29b90c	59
add651ee FG	60	let _ = cut_err(one_of(QUOTATION_MARK))
	61	.context(StrContext::Label("basic string"))
	62	.parse_next(input)?;
0a29b90c	63
add651ee FG	64	Ok(c)
	65	})
	66	.parse_next(input)
0a29b90c FG	67	}
	68
	69	// quotation-mark = %x22 ; "
	70	pub(crate) const QUOTATION_MARK: u8 = b'"';
	71
	72	// basic-char = basic-unescaped / escaped
add651ee	73	fn basic_chars<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
0a29b90c FG	74	alt((
	75	// Deviate from the official grammar by batching the unescaped chars so we build a string a
	76	// chunk at a time, rather than a `char` at a time.
fe692bf9 FG	77	take_while(1.., BASIC_UNESCAPED)
fe692bf9 FG	78	.try_map(std::str::from_utf8)
0a29b90c FG	79	.map(Cow::Borrowed),
	80	escaped.map(\|c\| Cow::Owned(String::from(c))),
	81	))
49aad941	82	.parse_next(input)
0a29b90c FG	83	}
	84
	85	// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
	86	pub(crate) const BASIC_UNESCAPED: (
	87	(u8, u8),
	88	u8,
	89	RangeInclusive<u8>,
	90	RangeInclusive<u8>,
	91	RangeInclusive<u8>,
	92	) = (WSCHAR, 0x21, 0x23..=0x5B, 0x5D..=0x7E, NON_ASCII);
	93
	94	// escaped = escape escape-seq-char
add651ee	95	fn escaped(input: &mut Input<'_>) -> PResult<char> {
49aad941	96	preceded(ESCAPE, escape_seq_char).parse_next(input)
0a29b90c FG	97	}
	98
	99	// escape = %x5C ; \
	100	pub(crate) const ESCAPE: u8 = b'\\';
	101
	102	// escape-seq-char = %x22 ; " quotation mark U+0022
	103	// escape-seq-char =/ %x5C ; \ reverse solidus U+005C
	104	// escape-seq-char =/ %x62 ; b backspace U+0008
	105	// escape-seq-char =/ %x66 ; f form feed U+000C
	106	// escape-seq-char =/ %x6E ; n line feed U+000A
	107	// escape-seq-char =/ %x72 ; r carriage return U+000D
	108	// escape-seq-char =/ %x74 ; t tab U+0009
	109	// escape-seq-char =/ %x75 4HEXDIG ; uXXXX U+XXXX
	110	// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
add651ee	111	fn escape_seq_char(input: &mut Input<'_>) -> PResult<char> {
0a29b90c FG	112	dispatch! {any;
	113	b'b' => success('\u{8}'),
	114	b'f' => success('\u{c}'),
	115	b'n' => success('\n'),
	116	b'r' => success('\r'),
	117	b't' => success('\t'),
add651ee FG	118	b'u' => cut_err(hexescape::<4>).context(StrContext::Label("unicode 4-digit hex code")),
add651ee FG	119	b'U' => cut_err(hexescape::<8>).context(StrContext::Label("unicode 8-digit hex code")),
0a29b90c FG	120	b'\\' => success('\\'),
	121	b'"' => success('"'),
	122	_ => {
49aad941	123	cut_err(fail::<_, char, _>)
add651ee FG	124	.context(StrContext::Label("escape sequence"))
	125	.context(StrContext::Expected(StrContextValue::CharLiteral('b')))
	126	.context(StrContext::Expected(StrContextValue::CharLiteral('f')))
	127	.context(StrContext::Expected(StrContextValue::CharLiteral('n')))
	128	.context(StrContext::Expected(StrContextValue::CharLiteral('r')))
	129	.context(StrContext::Expected(StrContextValue::CharLiteral('t')))
	130	.context(StrContext::Expected(StrContextValue::CharLiteral('u')))
	131	.context(StrContext::Expected(StrContextValue::CharLiteral('U')))
	132	.context(StrContext::Expected(StrContextValue::CharLiteral('\\')))
	133	.context(StrContext::Expected(StrContextValue::CharLiteral('"')))
0a29b90c FG	134	}
0a29b90c FG	135	}
49aad941	136	.parse_next(input)
0a29b90c FG	137	}
0a29b90c FG	138
add651ee	139	pub(crate) fn hexescape<const N: usize>(input: &mut Input<'_>) -> PResult<char> {
fe692bf9	140	take_while(0..=N, HEXDIG)
0a29b90c FG	141	.verify(\|b: &[u8]\| b.len() == N)
0a29b90c FG	142	.map(\|b: &[u8]\| unsafe { from_utf8_unchecked(b, "`is_ascii_digit` filters out on-ASCII") })
49aad941	143	.verify_map(\|s\| u32::from_str_radix(s, 16).ok())
fe692bf9	144	.try_map(\|h\| char::from_u32(h).ok_or(CustomError::OutOfRange))
49aad941	145	.parse_next(input)
0a29b90c FG	146	}
	147
	148	// ;; Multiline Basic String
	149
	150	// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
	151	// ml-basic-string-delim
add651ee FG	152	fn ml_basic_string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
	153	trace(
	154	"ml-basic-string",
	155	delimited(
	156	ML_BASIC_STRING_DELIM,
	157	preceded(opt(newline), cut_err(ml_basic_body)),
	158	cut_err(ML_BASIC_STRING_DELIM),
	159	)
	160	.context(StrContext::Label("multiline basic string")),
0a29b90c	161	)
49aad941	162	.parse_next(input)
0a29b90c FG	163	}
	164
	165	// ml-basic-string-delim = 3quotation-mark
	166	pub(crate) const ML_BASIC_STRING_DELIM: &[u8] = b"\"\"\"";
	167
	168	// ml-basic-body = mlb-content ( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
add651ee	169	fn ml_basic_body<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
0a29b90c	170	let mut c = Cow::Borrowed("");
add651ee	171	if let Some(ci) = opt(mlb_content).parse_next(input)? {
0a29b90c FG	172	c = ci;
0a29b90c FG	173	}
add651ee	174	while let Some(ci) = opt(mlb_content).parse_next(input)? {
0a29b90c FG	175	c.to_mut().push_str(&ci);
	176	}
	177
add651ee FG	178	while let Some(qi) = opt(mlb_quotes(none_of(b'\"').value(()))).parse_next(input)? {
add651ee FG	179	if let Some(ci) = opt(mlb_content).parse_next(input)? {
0a29b90c FG	180	c.to_mut().push_str(qi);
0a29b90c FG	181	c.to_mut().push_str(&ci);
add651ee	182	while let Some(ci) = opt(mlb_content).parse_next(input)? {
0a29b90c FG	183	c.to_mut().push_str(&ci);
	184	}
	185	} else {
	186	break;
	187	}
	188	}
	189
add651ee	190	if let Some(qi) = opt(mlb_quotes(tag(ML_BASIC_STRING_DELIM).value(()))).parse_next(input)? {
0a29b90c FG	191	c.to_mut().push_str(qi);
	192	}
	193
add651ee	194	Ok(c)
0a29b90c FG	195	}
	196
	197	// mlb-content = mlb-char / newline / mlb-escaped-nl
	198	// mlb-char = mlb-unescaped / escaped
add651ee	199	fn mlb_content<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
0a29b90c FG	200	alt((
	201	// Deviate from the official grammar by batching the unescaped chars so we build a string a
	202	// chunk at a time, rather than a `char` at a time.
fe692bf9 FG	203	take_while(1.., MLB_UNESCAPED)
fe692bf9 FG	204	.try_map(std::str::from_utf8)
0a29b90c	205	.map(Cow::Borrowed),
49aad941	206	// Order changed fromg grammar so `escaped` can more easily `cut_err` on bad escape sequences
0a29b90c FG	207	mlb_escaped_nl.map(\|_\| Cow::Borrowed("")),
	208	escaped.map(\|c\| Cow::Owned(String::from(c))),
	209	newline.map(\|_\| Cow::Borrowed("\n")),
	210	))
49aad941	211	.parse_next(input)
0a29b90c FG	212	}
	213
	214	// mlb-quotes = 1*2quotation-mark
	215	fn mlb_quotes<'i>(
add651ee FG	216	mut term: impl winnow::Parser<Input<'i>, (), ContextError>,
	217	) -> impl Parser<Input<'i>, &'i str, ContextError> {
	218	move \|input: &mut Input<'i>\| {
	219	let start = input.checkpoint();
0a29b90c FG	220	let res = terminated(b"\"\"", peek(term.by_ref()))
0a29b90c FG	221	.map(\|b\| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
49aad941	222	.parse_next(input);
0a29b90c FG	223
0a29b90c FG	224	match res {
add651ee FG	225	Err(winnow::error::ErrMode::Backtrack(_)) => {
	226	input.reset(start);
	227	terminated(b"\"", peek(term.by_ref()))
	228	.map(\|b\| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
	229	.parse_next(input)
	230	}
0a29b90c FG	231	res => res,
	232	}
	233	}
	234	}
	235
	236	// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
	237	pub(crate) const MLB_UNESCAPED: (
	238	(u8, u8),
	239	u8,
	240	RangeInclusive<u8>,
	241	RangeInclusive<u8>,
	242	RangeInclusive<u8>,
	243	) = (WSCHAR, 0x21, 0x23..=0x5B, 0x5D..=0x7E, NON_ASCII);
	244
	245	// mlb-escaped-nl = escape ws newline *( wschar / newline
	246	// When the last non-whitespace character on a line is a \,
	247	// it will be trimmed along with all whitespace
	248	// (including newlines) up to the next non-whitespace
	249	// character or closing delimiter.
add651ee	250	fn mlb_escaped_nl(input: &mut Input<'_>) -> PResult<()> {
fe692bf9	251	repeat(1.., (ESCAPE, ws, ws_newlines))
49aad941	252	.map(\|()\| ())
0a29b90c	253	.value(())
49aad941	254	.parse_next(input)
0a29b90c FG	255	}
	256
	257	// ;; Literal String
	258
	259	// literal-string = apostrophe *literal-char apostrophe
add651ee FG	260	pub(crate) fn literal_string<'i>(input: &mut Input<'i>) -> PResult<&'i str> {
	261	trace(
	262	"literal-string",
	263	delimited(
	264	APOSTROPHE,
	265	cut_err(take_while(0.., LITERAL_CHAR)),
	266	cut_err(APOSTROPHE),
	267	)
	268	.try_map(std::str::from_utf8)
	269	.context(StrContext::Label("literal string")),
49aad941	270	)
49aad941	271	.parse_next(input)
0a29b90c FG	272	}
	273
	274	// apostrophe = %x27 ; ' apostrophe
	275	pub(crate) const APOSTROPHE: u8 = b'\'';
	276
	277	// literal-char = %x09 / %x20-26 / %x28-7E / non-ascii
	278	pub(crate) const LITERAL_CHAR: (
	279	u8,
	280	RangeInclusive<u8>,
	281	RangeInclusive<u8>,
	282	RangeInclusive<u8>,
	283	) = (0x9, 0x20..=0x26, 0x28..=0x7E, NON_ASCII);
	284
	285	// ;; Multiline Literal String
	286
	287	// ml-literal-string = ml-literal-string-delim [ newline ] ml-literal-body
	288	// ml-literal-string-delim
add651ee FG	289	fn ml_literal_string<'i>(input: &mut Input<'i>) -> PResult<Cow<'i, str>> {
	290	trace(
	291	"ml-literal-string",
	292	delimited(
	293	(ML_LITERAL_STRING_DELIM, opt(newline)),
	294	cut_err(ml_literal_body.map(\|t\| {
	295	if t.contains("\r\n") {
	296	Cow::Owned(t.replace("\r\n", "\n"))
	297	} else {
	298	Cow::Borrowed(t)
	299	}
	300	})),
	301	cut_err(ML_LITERAL_STRING_DELIM),
	302	)
	303	.context(StrContext::Label("multiline literal string")),
0a29b90c	304	)
49aad941	305	.parse_next(input)
0a29b90c FG	306	}
	307
	308	// ml-literal-string-delim = 3apostrophe
	309	pub(crate) const ML_LITERAL_STRING_DELIM: &[u8] = b"'''";
	310
	311	// ml-literal-body = mll-content ( mll-quotes 1*mll-content ) [ mll-quotes ]
add651ee	312	fn ml_literal_body<'i>(input: &mut Input<'i>) -> PResult<&'i str> {
0a29b90c	313	(
fe692bf9 FG	314	repeat(0.., mll_content).map(\|()\| ()),
	315	repeat(
	316	0..,
	317	(
	318	mll_quotes(none_of(APOSTROPHE).value(())),
	319	repeat(1.., mll_content).map(\|()\| ()),
	320	),
	321	)
49aad941	322	.map(\|()\| ()),
0a29b90c FG	323	opt(mll_quotes(tag(ML_LITERAL_STRING_DELIM).value(()))),
	324	)
	325	.recognize()
fe692bf9	326	.try_map(std::str::from_utf8)
49aad941	327	.parse_next(input)
0a29b90c FG	328	}
	329
	330	// mll-content = mll-char / newline
add651ee	331	fn mll_content(input: &mut Input<'_>) -> PResult<u8> {
49aad941	332	alt((one_of(MLL_CHAR), newline)).parse_next(input)
0a29b90c FG	333	}
	334
	335	// mll-char = %x09 / %x20-26 / %x28-7E / non-ascii
	336	const MLL_CHAR: (
	337	u8,
	338	RangeInclusive<u8>,
	339	RangeInclusive<u8>,
	340	RangeInclusive<u8>,
	341	) = (0x9, 0x20..=0x26, 0x28..=0x7E, NON_ASCII);
	342
	343	// mll-quotes = 1*2apostrophe
	344	fn mll_quotes<'i>(
add651ee FG	345	mut term: impl winnow::Parser<Input<'i>, (), ContextError>,
	346	) -> impl Parser<Input<'i>, &'i str, ContextError> {
	347	move \|input: &mut Input<'i>\| {
	348	let start = input.checkpoint();
0a29b90c FG	349	let res = terminated(b"''", peek(term.by_ref()))
0a29b90c FG	350	.map(\|b\| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
49aad941	351	.parse_next(input);
0a29b90c FG	352
0a29b90c FG	353	match res {
add651ee FG	354	Err(winnow::error::ErrMode::Backtrack(_)) => {
	355	input.reset(start);
	356	terminated(b"'", peek(term.by_ref()))
	357	.map(\|b\| unsafe { from_utf8_unchecked(b, "`bytes` out non-ASCII") })
	358	.parse_next(input)
	359	}
0a29b90c FG	360	res => res,
	361	}
	362	}
	363	}
	364
	365	#[cfg(test)]
	366	mod test {
	367	use super::*;
	368
	369	#[test]
	370	fn basic_string() {
	371	let input =
	372	r#""I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF. \U0002070E""#;
	373	let expected = "I\'m a string. \"You can quote me\". Name\tJosé\nLocation\tSF. \u{2070E}";
49aad941	374	let parsed = string.parse(new_input(input));
0a29b90c FG	375	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	376	}
	377
	378	#[test]
	379	fn ml_basic_string() {
	380	let cases = [
	381	(
	382	r#""""
	383	Roses are red
	384	Violets are blue""""#,
	385	r#"Roses are red
	386	Violets are blue"#,
	387	),
	388	(r#"""" \""" """"#, " \"\"\" "),
	389	(r#"""" \\""""#, " \\"),
	390	];
	391
	392	for &(input, expected) in &cases {
49aad941	393	let parsed = string.parse(new_input(input));
0a29b90c FG	394	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	395	}
	396
	397	let invalid_cases = [r#"""" """#, r#"""" \""""#];
	398
	399	for input in &invalid_cases {
49aad941	400	let parsed = string.parse(new_input(input));
0a29b90c FG	401	assert!(parsed.is_err());
	402	}
	403	}
	404
	405	#[test]
	406	fn ml_basic_string_escape_ws() {
	407	let inputs = [
	408	r#""""
	409	The quick brown \
	410
	411
	412	fox jumps over \
	413	the lazy dog.""""#,
	414	r#""""\
	415	The quick brown \
	416	fox jumps over \
	417	the lazy dog.\
	418	""""#,
	419	];
	420	for input in &inputs {
	421	let expected = "The quick brown fox jumps over the lazy dog.";
49aad941	422	let parsed = string.parse(new_input(input));
0a29b90c FG	423	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	424	}
	425	let empties = [
	426	r#""""\
	427	""""#,
	428	r#""""
	429	\
	430	\
	431	""""#,
	432	];
	433	for input in &empties {
	434	let expected = "";
49aad941	435	let parsed = string.parse(new_input(input));
0a29b90c FG	436	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	437	}
	438	}
	439
	440	#[test]
	441	fn literal_string() {
	442	let inputs = [
ed00b5ec FG	443	r"'C:\Users\nodejs\templates'",
ed00b5ec FG	444	r"'\\ServerX\admin$\system32\'",
0a29b90c	445	r#"'Tom "Dubs" Preston-Werner'"#,
ed00b5ec	446	r"'<\i\c\s>'",
0a29b90c FG	447	];
	448
	449	for input in &inputs {
	450	let expected = &input[1..input.len() - 1];
49aad941	451	let parsed = string.parse(new_input(input));
0a29b90c FG	452	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	453	}
	454	}
	455
	456	#[test]
	457	fn ml_literal_string() {
	458	let inputs = [
ed00b5ec	459	r"'''I [dw]on't need \d{2} apples'''",
0a29b90c FG	460	r#"''''one_quote''''"#,
	461	];
	462	for input in &inputs {
	463	let expected = &input[3..input.len() - 3];
49aad941	464	let parsed = string.parse(new_input(input));
0a29b90c FG	465	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	466	}
	467
	468	let input = r#"'''
	469	The first newline is
	470	trimmed in raw strings.
	471	All other whitespace
	472	is preserved.
	473	'''"#;
	474	let expected = &input[4..input.len() - 3];
49aad941	475	let parsed = string.parse(new_input(input));
0a29b90c FG	476	assert_eq!(parsed.as_deref(), Ok(expected), "Parsing {input:?}");
	477	}
	478	}