use codemap;
use diagnostic::SpanHandler;
use ext::tt::transcribe::tt_next_token;
-use parse::token;
use parse::token::str_to_ident;
+use parse::token;
+use str::char_at;
-use std::borrow::{IntoCow, Cow};
+use std::borrow::Cow;
use std::char;
-use std::fmt;
use std::mem::replace;
use std::rc::Rc;
pub peek_tok: token::Token,
pub peek_span: Span,
- // FIXME (Issue #16472): This field should go away after ToToken impls
- // are revised to go directly to token-trees.
- /// Is \x00<name>,<ctxt>\x00 is interpreted as encoded ast::Ident?
- read_embedded_ident: bool,
-
// cache a direct reference to the source text, so that we don't have to
// retrieve it via `self.filemap.src.as_ref().unwrap()` all the time.
source_text: Rc<String>
}
}
-// FIXME (Issue #16472): This function should go away after
-// ToToken impls are revised to go directly to token-trees.
-pub fn make_reader_with_embedded_idents<'b>(span_diagnostic: &'b SpanHandler,
- filemap: Rc<codemap::FileMap>)
- -> StringReader<'b> {
- let mut sr = StringReader::new_raw(span_diagnostic, filemap);
- sr.read_embedded_ident = true;
- sr.advance_token();
- sr
-}
-
impl<'a> StringReader<'a> {
/// For comments.rs, which hackily pokes into pos and curr
pub fn new_raw<'b>(span_diagnostic: &'b SpanHandler,
/* dummy values; not read */
peek_tok: token::Eof,
peek_span: codemap::DUMMY_SP,
- read_embedded_ident: false,
source_text: source_text
};
sr.bump();
s: &'b str, errmsg: &'b str) -> Cow<'b, str> {
let mut i = 0;
while i < s.len() {
- let ch = s.char_at(i);
+ let ch = char_at(s, i);
let next = i + ch.len_utf8();
if ch == '\r' {
- if next < s.len() && s.char_at(next) == '\n' {
- return translate_crlf_(self, start, s, errmsg, i).into_cow();
+ if next < s.len() && char_at(s, next) == '\n' {
+ return translate_crlf_(self, start, s, errmsg, i).into();
}
let pos = start + BytePos(i as u32);
let end_pos = start + BytePos(next as u32);
}
i = next;
}
- return s.into_cow();
+ return s.into();
fn translate_crlf_(rdr: &StringReader, start: BytePos,
s: &str, errmsg: &str, mut i: usize) -> String {
let mut buf = String::with_capacity(s.len());
let mut j = 0;
while i < s.len() {
- let ch = s.char_at(i);
+ let ch = char_at(s, i);
let next = i + ch.len_utf8();
if ch == '\r' {
if j < i { buf.push_str(&s[j..i]); }
j = next;
- if next >= s.len() || s.char_at(next) != '\n' {
+ if next >= s.len() || char_at(s, next) != '\n' {
let pos = start + BytePos(i as u32);
let end_pos = start + BytePos(next as u32);
rdr.err_span_(pos, end_pos, errmsg);
if current_byte_offset < self.source_text.len() {
assert!(self.curr.is_some());
let last_char = self.curr.unwrap();
- let ch = self.source_text.char_at(current_byte_offset);
+ let ch = char_at(&self.source_text, current_byte_offset);
let next = current_byte_offset + ch.len_utf8();
let byte_offset_diff = next - current_byte_offset;
self.pos = self.pos + Pos::from_usize(byte_offset_diff);
pub fn nextch(&self) -> Option<char> {
let offset = self.byte_offset(self.pos).to_usize();
if offset < self.source_text.len() {
- Some(self.source_text.char_at(offset))
+ Some(char_at(&self.source_text, offset))
} else {
None
}
let offset = self.byte_offset(self.pos).to_usize();
let s = &self.source_text[..];
if offset >= s.len() { return None }
- let next = offset + s.char_at(offset).len_utf8();
+ let next = offset + char_at(s, offset).len_utf8();
if next < s.len() {
- Some(s.char_at(next))
+ Some(char_at(s, next))
} else {
None
}
let string = if has_cr {
self.translate_crlf(start_bpos, string,
"bare CR not allowed in block doc-comment")
- } else { string.into_cow() };
+ } else { string.into() };
token::DocComment(token::intern(&string[..]))
} else {
token::Comment
})
}
- // FIXME (Issue #16472): The scan_embedded_hygienic_ident function
- // should go away after we revise the syntax::ext::quote::ToToken
- // impls to go directly to token-trees instead of thing -> string
- // -> token-trees. (The function is currently used to resolve
- // Issues #15750 and #15962.)
- //
- // Since this function is only used for certain internal macros,
- // and the functionality it provides is not exposed to end user
- // programs, pnkfelix deliberately chose to write it in a way that
- // favors rustc debugging effectiveness over runtime efficiency.
-
- /// Scan through input of form \x00name_NNNNNN,ctxt_CCCCCCC\x00
- /// whence: `NNNNNN` is a string of characters forming an integer
- /// (the name) and `CCCCCCC` is a string of characters forming an
- /// integer (the ctxt), separate by a comma and delimited by a
- /// `\x00` marker.
- #[inline(never)]
- fn scan_embedded_hygienic_ident(&mut self) -> ast::Ident {
- fn bump_expecting_char<'a,D:fmt::Debug>(r: &mut StringReader<'a>,
- c: char,
- described_c: D,
- whence: &str) {
- match r.curr {
- Some(r_c) if r_c == c => r.bump(),
- Some(r_c) => panic!("expected {:?}, hit {:?}, {}", described_c, r_c, whence),
- None => panic!("expected {:?}, hit EOF, {}", described_c, whence),
- }
- }
-
- let whence = "while scanning embedded hygienic ident";
-
- // skip over the leading `\x00`
- bump_expecting_char(self, '\x00', "nul-byte", whence);
-
- // skip over the "name_"
- for c in "name_".chars() {
- bump_expecting_char(self, c, c, whence);
- }
-
- let start_bpos = self.last_pos;
- let base = 10;
-
- // find the integer representing the name
- self.scan_digits(base, base);
- let encoded_name : u32 = self.with_str_from(start_bpos, |s| {
- u32::from_str_radix(s, 10).unwrap_or_else(|_| {
- panic!("expected digits representing a name, got {:?}, {}, range [{:?},{:?}]",
- s, whence, start_bpos, self.last_pos);
- })
- });
-
- // skip over the `,`
- bump_expecting_char(self, ',', "comma", whence);
-
- // skip over the "ctxt_"
- for c in "ctxt_".chars() {
- bump_expecting_char(self, c, c, whence);
- }
-
- // find the integer representing the ctxt
- let start_bpos = self.last_pos;
- self.scan_digits(base, base);
- let encoded_ctxt : ast::SyntaxContext = self.with_str_from(start_bpos, |s| {
- u32::from_str_radix(s, 10).unwrap_or_else(|_| {
- panic!("expected digits representing a ctxt, got {:?}, {}", s, whence);
- })
- });
-
- // skip over the `\x00`
- bump_expecting_char(self, '\x00', "nul-byte", whence);
-
- ast::Ident { name: ast::Name(encoded_name),
- ctxt: encoded_ctxt, }
- }
-
/// Scan through any digits (base `scan_radix`) or underscores,
/// and return how many digits there were.
///
return token::Literal(num, suffix)
}
- if self.read_embedded_ident {
- match (c.unwrap(), self.nextch(), self.nextnextch()) {
- ('\x00', Some('n'), Some('a')) => {
- let ast_ident = self.scan_embedded_hygienic_ident();
- return if self.curr_is(':') && self.nextch_is(':') {
- token::Ident(ast_ident, token::ModName)
- } else {
- token::Ident(ast_ident, token::Plain)
- };
- }
- _ => {}
- }
- }
-
match c.expect("next_token_inner called at EOF") {
// One-byte tokens.
';' => { self.bump(); return token::Semi; }
}
#[cfg(test)]
-mod test {
+mod tests {
use super::*;
use codemap::{BytePos, CodeMap, Span, NO_EXPANSION};
projects / rustc.git / blobdiff