]>
Commit | Line | Data |
---|---|---|
970d7e83 | 1 | // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT |
223e47cc LB |
2 | // file at the top-level directory of this distribution and at |
3 | // http://rust-lang.org/COPYRIGHT. | |
4 | // | |
5 | // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | // option. This file may not be copied, modified, or distributed | |
9 | // except according to those terms. | |
10 | ||
1a4d82fc JJ |
11 | pub use self::BinOpToken::*; |
12 | pub use self::Nonterminal::*; | |
13 | pub use self::DelimToken::*; | |
14 | pub use self::IdentStyle::*; | |
15 | pub use self::Lit::*; | |
16 | pub use self::Token::*; | |
17 | ||
223e47cc | 18 | use ast; |
1a4d82fc JJ |
19 | use ext::mtwt; |
20 | use ptr::P; | |
21 | use util::interner::{RcStr, StrInterner}; | |
223e47cc LB |
22 | use util::interner; |
23 | ||
1a4d82fc JJ |
24 | use serialize::{Decodable, Decoder, Encodable, Encoder}; |
25 | use std::fmt; | |
26 | use std::mem; | |
27 | use std::ops::Deref; | |
c34b1796 | 28 | #[allow(deprecated)] |
85aaf69f | 29 | use std::old_path::BytesContainer; |
1a4d82fc JJ |
30 | use std::rc::Rc; |
31 | ||
32 | #[allow(non_camel_case_types)] | |
85aaf69f | 33 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] |
1a4d82fc JJ |
34 | pub enum BinOpToken { |
35 | Plus, | |
36 | Minus, | |
37 | Star, | |
38 | Slash, | |
39 | Percent, | |
40 | Caret, | |
41 | And, | |
42 | Or, | |
43 | Shl, | |
44 | Shr, | |
45 | } | |
46 | ||
47 | /// A delimiter token | |
85aaf69f | 48 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] |
1a4d82fc JJ |
49 | pub enum DelimToken { |
50 | /// A round parenthesis: `(` or `)` | |
51 | Paren, | |
52 | /// A square bracket: `[` or `]` | |
53 | Bracket, | |
54 | /// A curly brace: `{` or `}` | |
55 | Brace, | |
56 | } | |
57 | ||
85aaf69f | 58 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] |
1a4d82fc JJ |
59 | pub enum IdentStyle { |
60 | /// `::` follows the identifier with no whitespace in-between. | |
61 | ModName, | |
62 | Plain, | |
63 | } | |
64 | ||
85aaf69f | 65 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] |
1a4d82fc JJ |
66 | pub enum SpecialMacroVar { |
67 | /// `$crate` will be filled in with the name of the crate a macro was | |
68 | /// imported from, if any. | |
69 | CrateMacroVar, | |
70 | } | |
71 | ||
72 | impl SpecialMacroVar { | |
73 | pub fn as_str(self) -> &'static str { | |
74 | match self { | |
75 | SpecialMacroVar::CrateMacroVar => "crate", | |
76 | } | |
77 | } | |
78 | } | |
79 | ||
85aaf69f | 80 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] |
1a4d82fc JJ |
81 | pub enum Lit { |
82 | Byte(ast::Name), | |
83 | Char(ast::Name), | |
84 | Integer(ast::Name), | |
85 | Float(ast::Name), | |
86 | Str_(ast::Name), | |
85aaf69f | 87 | StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */ |
1a4d82fc | 88 | Binary(ast::Name), |
85aaf69f | 89 | BinaryRaw(ast::Name, usize), /* raw binary str delimited by n hash symbols */ |
1a4d82fc JJ |
90 | } |
91 | ||
92 | impl Lit { | |
93 | pub fn short_name(&self) -> &'static str { | |
94 | match *self { | |
95 | Byte(_) => "byte", | |
96 | Char(_) => "char", | |
97 | Integer(_) => "integer", | |
98 | Float(_) => "float", | |
99 | Str_(_) | StrRaw(..) => "str", | |
100 | Binary(_) | BinaryRaw(..) => "binary str" | |
101 | } | |
102 | } | |
103 | } | |
104 | ||
105 | #[allow(non_camel_case_types)] | |
85aaf69f | 106 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)] |
223e47cc LB |
107 | pub enum Token { |
108 | /* Expression-operator symbols. */ | |
1a4d82fc JJ |
109 | Eq, |
110 | Lt, | |
111 | Le, | |
112 | EqEq, | |
113 | Ne, | |
114 | Ge, | |
115 | Gt, | |
116 | AndAnd, | |
117 | OrOr, | |
118 | Not, | |
119 | Tilde, | |
120 | BinOp(BinOpToken), | |
121 | BinOpEq(BinOpToken), | |
223e47cc LB |
122 | |
123 | /* Structural symbols */ | |
1a4d82fc JJ |
124 | At, |
125 | Dot, | |
126 | DotDot, | |
127 | DotDotDot, | |
128 | Comma, | |
129 | Semi, | |
130 | Colon, | |
131 | ModSep, | |
132 | RArrow, | |
133 | LArrow, | |
134 | FatArrow, | |
135 | Pound, | |
136 | Dollar, | |
137 | Question, | |
138 | /// An opening delimiter, eg. `{` | |
139 | OpenDelim(DelimToken), | |
140 | /// A closing delimiter, eg. `}` | |
141 | CloseDelim(DelimToken), | |
223e47cc LB |
142 | |
143 | /* Literals */ | |
1a4d82fc | 144 | Literal(Lit, Option<ast::Name>), |
223e47cc LB |
145 | |
146 | /* Name components */ | |
1a4d82fc JJ |
147 | Ident(ast::Ident, IdentStyle), |
148 | Underscore, | |
149 | Lifetime(ast::Ident), | |
223e47cc LB |
150 | |
151 | /* For interpolation */ | |
1a4d82fc JJ |
152 | Interpolated(Nonterminal), |
153 | // Can be expanded into several tokens. | |
154 | /// Doc comment | |
155 | DocComment(ast::Name), | |
156 | // In left-hand-sides of MBE macros: | |
157 | /// Parse a nonterminal (name to bind, name of NT, styles of their idents) | |
158 | MatchNt(ast::Ident, ast::Ident, IdentStyle, IdentStyle), | |
159 | // In right-hand-sides of MBE macros: | |
160 | /// A syntactic variable that will be filled in by macro expansion. | |
161 | SubstNt(ast::Ident, IdentStyle), | |
162 | /// A macro variable with special meaning. | |
163 | SpecialVarNt(SpecialMacroVar), | |
164 | ||
165 | // Junk. These carry no data because we don't really care about the data | |
166 | // they *would* carry, and don't really want to allocate a new ident for | |
167 | // them. Instead, users could extract that from the associated span. | |
168 | ||
169 | /// Whitespace | |
170 | Whitespace, | |
171 | /// Comment | |
172 | Comment, | |
173 | Shebang(ast::Name), | |
174 | ||
175 | Eof, | |
176 | } | |
177 | ||
178 | impl Token { | |
179 | /// Returns `true` if the token can appear at the start of an expression. | |
180 | pub fn can_begin_expr(&self) -> bool { | |
181 | match *self { | |
182 | OpenDelim(_) => true, | |
183 | Ident(_, _) => true, | |
184 | Underscore => true, | |
185 | Tilde => true, | |
186 | Literal(_, _) => true, | |
1a4d82fc JJ |
187 | Not => true, |
188 | BinOp(Minus) => true, | |
189 | BinOp(Star) => true, | |
190 | BinOp(And) => true, | |
191 | BinOp(Or) => true, // in lambda syntax | |
192 | OrOr => true, // in lambda syntax | |
85aaf69f SL |
193 | AndAnd => true, // double borrow |
194 | DotDot => true, // range notation | |
1a4d82fc JJ |
195 | ModSep => true, |
196 | Interpolated(NtExpr(..)) => true, | |
197 | Interpolated(NtIdent(..)) => true, | |
198 | Interpolated(NtBlock(..)) => true, | |
199 | Interpolated(NtPath(..)) => true, | |
200 | _ => false, | |
201 | } | |
202 | } | |
223e47cc | 203 | |
1a4d82fc JJ |
204 | /// Returns `true` if the token is any literal |
205 | pub fn is_lit(&self) -> bool { | |
206 | match *self { | |
207 | Literal(_, _) => true, | |
208 | _ => false, | |
209 | } | |
210 | } | |
223e47cc | 211 | |
1a4d82fc JJ |
212 | /// Returns `true` if the token is an identifier. |
213 | pub fn is_ident(&self) -> bool { | |
214 | match *self { | |
215 | Ident(_, _) => true, | |
216 | _ => false, | |
223e47cc | 217 | } |
1a4d82fc JJ |
218 | } |
219 | ||
220 | /// Returns `true` if the token is an interpolated path. | |
221 | pub fn is_path(&self) -> bool { | |
222 | match *self { | |
223 | Interpolated(NtPath(..)) => true, | |
224 | _ => false, | |
225 | } | |
226 | } | |
227 | ||
228 | /// Returns `true` if the token is a path that is not followed by a `::` | |
229 | /// token. | |
230 | #[allow(non_upper_case_globals)] | |
231 | pub fn is_plain_ident(&self) -> bool { | |
232 | match *self { | |
233 | Ident(_, Plain) => true, | |
234 | _ => false, | |
235 | } | |
236 | } | |
237 | ||
238 | /// Returns `true` if the token is a lifetime. | |
239 | pub fn is_lifetime(&self) -> bool { | |
240 | match *self { | |
241 | Lifetime(..) => true, | |
242 | _ => false, | |
223e47cc | 243 | } |
1a4d82fc JJ |
244 | } |
245 | ||
246 | /// Returns `true` if the token is either the `mut` or `const` keyword. | |
247 | pub fn is_mutability(&self) -> bool { | |
248 | self.is_keyword(keywords::Mut) || | |
249 | self.is_keyword(keywords::Const) | |
250 | } | |
251 | ||
252 | /// Maps a token to its corresponding binary operator. | |
85aaf69f | 253 | pub fn to_binop(&self) -> Option<ast::BinOp_> { |
1a4d82fc JJ |
254 | match *self { |
255 | BinOp(Star) => Some(ast::BiMul), | |
256 | BinOp(Slash) => Some(ast::BiDiv), | |
257 | BinOp(Percent) => Some(ast::BiRem), | |
258 | BinOp(Plus) => Some(ast::BiAdd), | |
259 | BinOp(Minus) => Some(ast::BiSub), | |
260 | BinOp(Shl) => Some(ast::BiShl), | |
261 | BinOp(Shr) => Some(ast::BiShr), | |
262 | BinOp(And) => Some(ast::BiBitAnd), | |
263 | BinOp(Caret) => Some(ast::BiBitXor), | |
264 | BinOp(Or) => Some(ast::BiBitOr), | |
265 | Lt => Some(ast::BiLt), | |
266 | Le => Some(ast::BiLe), | |
267 | Ge => Some(ast::BiGe), | |
268 | Gt => Some(ast::BiGt), | |
269 | EqEq => Some(ast::BiEq), | |
270 | Ne => Some(ast::BiNe), | |
271 | AndAnd => Some(ast::BiAnd), | |
272 | OrOr => Some(ast::BiOr), | |
273 | _ => None, | |
274 | } | |
275 | } | |
276 | ||
277 | /// Returns `true` if the token is a given keyword, `kw`. | |
278 | #[allow(non_upper_case_globals)] | |
279 | pub fn is_keyword(&self, kw: keywords::Keyword) -> bool { | |
280 | match *self { | |
281 | Ident(sid, Plain) => kw.to_name() == sid.name, | |
282 | _ => false, | |
283 | } | |
284 | } | |
285 | ||
286 | pub fn is_keyword_allow_following_colon(&self, kw: keywords::Keyword) -> bool { | |
287 | match *self { | |
288 | Ident(sid, _) => { kw.to_name() == sid.name } | |
289 | _ => { false } | |
290 | } | |
291 | } | |
292 | ||
293 | /// Returns `true` if the token is either a special identifier, or a strict | |
294 | /// or reserved keyword. | |
295 | #[allow(non_upper_case_globals)] | |
296 | pub fn is_any_keyword(&self) -> bool { | |
297 | match *self { | |
298 | Ident(sid, Plain) => { | |
299 | let n = sid.name; | |
300 | ||
301 | n == SELF_KEYWORD_NAME | |
302 | || n == STATIC_KEYWORD_NAME | |
303 | || n == SUPER_KEYWORD_NAME | |
85aaf69f | 304 | || n == SELF_TYPE_KEYWORD_NAME |
1a4d82fc JJ |
305 | || STRICT_KEYWORD_START <= n |
306 | && n <= RESERVED_KEYWORD_FINAL | |
307 | }, | |
308 | _ => false | |
309 | } | |
310 | } | |
311 | ||
312 | /// Returns `true` if the token may not appear as an identifier. | |
313 | #[allow(non_upper_case_globals)] | |
314 | pub fn is_strict_keyword(&self) -> bool { | |
315 | match *self { | |
316 | Ident(sid, Plain) => { | |
317 | let n = sid.name; | |
318 | ||
319 | n == SELF_KEYWORD_NAME | |
320 | || n == STATIC_KEYWORD_NAME | |
321 | || n == SUPER_KEYWORD_NAME | |
85aaf69f | 322 | || n == SELF_TYPE_KEYWORD_NAME |
1a4d82fc JJ |
323 | || STRICT_KEYWORD_START <= n |
324 | && n <= STRICT_KEYWORD_FINAL | |
325 | }, | |
326 | Ident(sid, ModName) => { | |
327 | let n = sid.name; | |
328 | ||
329 | n != SELF_KEYWORD_NAME | |
330 | && n != SUPER_KEYWORD_NAME | |
331 | && STRICT_KEYWORD_START <= n | |
332 | && n <= STRICT_KEYWORD_FINAL | |
223e47cc | 333 | } |
1a4d82fc JJ |
334 | _ => false, |
335 | } | |
336 | } | |
337 | ||
338 | /// Returns `true` if the token is a keyword that has been reserved for | |
339 | /// possible future use. | |
340 | #[allow(non_upper_case_globals)] | |
341 | pub fn is_reserved_keyword(&self) -> bool { | |
342 | match *self { | |
343 | Ident(sid, Plain) => { | |
344 | let n = sid.name; | |
345 | ||
346 | RESERVED_KEYWORD_START <= n | |
347 | && n <= RESERVED_KEYWORD_FINAL | |
348 | }, | |
349 | _ => false, | |
350 | } | |
351 | } | |
352 | ||
353 | /// Hygienic identifier equality comparison. | |
354 | /// | |
355 | /// See `styntax::ext::mtwt`. | |
356 | pub fn mtwt_eq(&self, other : &Token) -> bool { | |
357 | match (self, other) { | |
358 | (&Ident(id1,_), &Ident(id2,_)) | (&Lifetime(id1), &Lifetime(id2)) => | |
359 | mtwt::resolve(id1) == mtwt::resolve(id2), | |
360 | _ => *self == *other | |
223e47cc | 361 | } |
223e47cc LB |
362 | } |
363 | } | |
364 | ||
1a4d82fc JJ |
365 | #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)] |
366 | /// For interpolation during macro expansion. | |
367 | pub enum Nonterminal { | |
368 | NtItem(P<ast::Item>), | |
369 | NtBlock(P<ast::Block>), | |
370 | NtStmt(P<ast::Stmt>), | |
371 | NtPat(P<ast::Pat>), | |
372 | NtExpr(P<ast::Expr>), | |
373 | NtTy(P<ast::Ty>), | |
374 | NtIdent(Box<ast::Ident>, IdentStyle), | |
375 | /// Stuff inside brackets for attributes | |
376 | NtMeta(P<ast::MetaItem>), | |
377 | NtPath(Box<ast::Path>), | |
378 | NtTT(P<ast::TokenTree>), // needs P'ed to break a circularity | |
379 | } | |
380 | ||
85aaf69f | 381 | impl fmt::Debug for Nonterminal { |
1a4d82fc JJ |
382 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
383 | match *self { | |
384 | NtItem(..) => f.pad("NtItem(..)"), | |
385 | NtBlock(..) => f.pad("NtBlock(..)"), | |
386 | NtStmt(..) => f.pad("NtStmt(..)"), | |
387 | NtPat(..) => f.pad("NtPat(..)"), | |
388 | NtExpr(..) => f.pad("NtExpr(..)"), | |
389 | NtTy(..) => f.pad("NtTy(..)"), | |
390 | NtIdent(..) => f.pad("NtIdent(..)"), | |
391 | NtMeta(..) => f.pad("NtMeta(..)"), | |
392 | NtPath(..) => f.pad("NtPath(..)"), | |
393 | NtTT(..) => f.pad("NtTT(..)"), | |
394 | } | |
970d7e83 LB |
395 | } |
396 | } | |
397 | ||
1a4d82fc JJ |
398 | |
399 | // Get the first "argument" | |
400 | macro_rules! first { | |
401 | ( $first:expr, $( $remainder:expr, )* ) => ( $first ) | |
402 | } | |
403 | ||
404 | // Get the last "argument" (has to be done recursively to avoid phoney local ambiguity error) | |
405 | macro_rules! last { | |
406 | ( $first:expr, $( $remainder:expr, )+ ) => ( last!( $( $remainder, )+ ) ); | |
407 | ( $first:expr, ) => ( $first ) | |
408 | } | |
409 | ||
410 | // In this macro, there is the requirement that the name (the number) must be monotonically | |
411 | // increasing by one in the special identifiers, starting at 0; the same holds for the keywords, | |
412 | // except starting from the next number instead of zero, and with the additional exception that | |
413 | // special identifiers are *also* allowed (they are deduplicated in the important place, the | |
414 | // interner), an exception which is demonstrated by "static" and "self". | |
415 | macro_rules! declare_special_idents_and_keywords {( | |
416 | // So now, in these rules, why is each definition parenthesised? | |
417 | // Answer: otherwise we get a spurious local ambiguity bug on the "}" | |
418 | pub mod special_idents { | |
419 | $( ($si_name:expr, $si_static:ident, $si_str:expr); )* | |
420 | } | |
421 | ||
422 | pub mod keywords { | |
423 | 'strict: | |
424 | $( ($sk_name:expr, $sk_variant:ident, $sk_str:expr); )* | |
425 | 'reserved: | |
426 | $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )* | |
427 | } | |
428 | ) => { | |
c34b1796 AL |
429 | const STRICT_KEYWORD_START: ast::Name = first!($( ast::Name($sk_name), )*); |
430 | const STRICT_KEYWORD_FINAL: ast::Name = last!($( ast::Name($sk_name), )*); | |
431 | const RESERVED_KEYWORD_START: ast::Name = first!($( ast::Name($rk_name), )*); | |
432 | const RESERVED_KEYWORD_FINAL: ast::Name = last!($( ast::Name($rk_name), )*); | |
1a4d82fc JJ |
433 | |
434 | pub mod special_idents { | |
435 | use ast; | |
436 | $( | |
437 | #[allow(non_upper_case_globals)] | |
438 | pub const $si_static: ast::Ident = ast::Ident { | |
439 | name: ast::Name($si_name), | |
440 | ctxt: 0, | |
441 | }; | |
442 | )* | |
443 | } | |
444 | ||
445 | pub mod special_names { | |
446 | use ast; | |
447 | $( | |
448 | #[allow(non_upper_case_globals)] | |
449 | pub const $si_static: ast::Name = ast::Name($si_name); | |
450 | )* | |
451 | } | |
452 | ||
453 | /// All the valid words that have meaning in the Rust language. | |
454 | /// | |
455 | /// Rust keywords are either 'strict' or 'reserved'. Strict keywords may not | |
456 | /// appear as identifiers at all. Reserved keywords are not used anywhere in | |
457 | /// the language and may not appear as identifiers. | |
458 | pub mod keywords { | |
459 | pub use self::Keyword::*; | |
460 | use ast; | |
461 | ||
85aaf69f | 462 | #[derive(Copy, Clone, PartialEq, Eq)] |
1a4d82fc JJ |
463 | pub enum Keyword { |
464 | $( $sk_variant, )* | |
465 | $( $rk_variant, )* | |
466 | } | |
467 | ||
468 | impl Keyword { | |
469 | pub fn to_name(&self) -> ast::Name { | |
470 | match *self { | |
471 | $( $sk_variant => ast::Name($sk_name), )* | |
472 | $( $rk_variant => ast::Name($rk_name), )* | |
473 | } | |
223e47cc LB |
474 | } |
475 | } | |
476 | } | |
1a4d82fc JJ |
477 | |
478 | fn mk_fresh_ident_interner() -> IdentInterner { | |
479 | // The indices here must correspond to the numbers in | |
480 | // special_idents, in Keyword to_name(), and in static | |
481 | // constants below. | |
482 | let mut init_vec = Vec::new(); | |
483 | $(init_vec.push($si_str);)* | |
484 | $(init_vec.push($sk_str);)* | |
485 | $(init_vec.push($rk_str);)* | |
85aaf69f | 486 | interner::StrInterner::prefill(&init_vec[..]) |
1a4d82fc JJ |
487 | } |
488 | }} | |
489 | ||
490 | // If the special idents get renumbered, remember to modify these two as appropriate | |
491 | pub const SELF_KEYWORD_NAME: ast::Name = ast::Name(SELF_KEYWORD_NAME_NUM); | |
492 | const STATIC_KEYWORD_NAME: ast::Name = ast::Name(STATIC_KEYWORD_NAME_NUM); | |
493 | const SUPER_KEYWORD_NAME: ast::Name = ast::Name(SUPER_KEYWORD_NAME_NUM); | |
85aaf69f | 494 | const SELF_TYPE_KEYWORD_NAME: ast::Name = ast::Name(SELF_TYPE_KEYWORD_NAME_NUM); |
1a4d82fc JJ |
495 | |
496 | pub const SELF_KEYWORD_NAME_NUM: u32 = 1; | |
497 | const STATIC_KEYWORD_NAME_NUM: u32 = 2; | |
498 | const SUPER_KEYWORD_NAME_NUM: u32 = 3; | |
85aaf69f | 499 | const SELF_TYPE_KEYWORD_NAME_NUM: u32 = 10; |
1a4d82fc JJ |
500 | |
501 | // NB: leaving holes in the ident table is bad! a different ident will get | |
502 | // interned with the id from the hole, but it will be between the min and max | |
503 | // of the reserved words, and thus tagged as "reserved". | |
504 | ||
505 | declare_special_idents_and_keywords! { | |
506 | pub mod special_idents { | |
507 | // These ones are statics | |
508 | (0, invalid, ""); | |
509 | (super::SELF_KEYWORD_NAME_NUM, self_, "self"); | |
510 | (super::STATIC_KEYWORD_NAME_NUM, statik, "static"); | |
511 | (super::SUPER_KEYWORD_NAME_NUM, super_, "super"); | |
512 | (4, static_lifetime, "'static"); | |
513 | ||
514 | // for matcher NTs | |
515 | (5, tt, "tt"); | |
516 | (6, matchers, "matchers"); | |
517 | ||
518 | // outside of libsyntax | |
519 | (7, clownshoe_abi, "__rust_abi"); | |
520 | (8, opaque, "<opaque>"); | |
521 | (9, unnamed_field, "<unnamed_field>"); | |
85aaf69f | 522 | (super::SELF_TYPE_KEYWORD_NAME_NUM, type_self, "Self"); |
1a4d82fc | 523 | (11, prelude_import, "prelude_import"); |
1a4d82fc JJ |
524 | } |
525 | ||
526 | pub mod keywords { | |
527 | // These ones are variants of the Keyword enum | |
528 | ||
529 | 'strict: | |
85aaf69f SL |
530 | (12, As, "as"); |
531 | (13, Break, "break"); | |
532 | (14, Crate, "crate"); | |
533 | (15, Else, "else"); | |
534 | (16, Enum, "enum"); | |
535 | (17, Extern, "extern"); | |
536 | (18, False, "false"); | |
537 | (19, Fn, "fn"); | |
538 | (20, For, "for"); | |
539 | (21, If, "if"); | |
540 | (22, Impl, "impl"); | |
541 | (23, In, "in"); | |
542 | (24, Let, "let"); | |
543 | (25, Loop, "loop"); | |
544 | (26, Match, "match"); | |
545 | (27, Mod, "mod"); | |
546 | (28, Move, "move"); | |
547 | (29, Mut, "mut"); | |
548 | (30, Pub, "pub"); | |
549 | (31, Ref, "ref"); | |
550 | (32, Return, "return"); | |
1a4d82fc JJ |
551 | // Static and Self are also special idents (prefill de-dupes) |
552 | (super::STATIC_KEYWORD_NAME_NUM, Static, "static"); | |
85aaf69f SL |
553 | (super::SELF_KEYWORD_NAME_NUM, SelfValue, "self"); |
554 | (super::SELF_TYPE_KEYWORD_NAME_NUM, SelfType, "Self"); | |
555 | (33, Struct, "struct"); | |
1a4d82fc | 556 | (super::SUPER_KEYWORD_NAME_NUM, Super, "super"); |
85aaf69f SL |
557 | (34, True, "true"); |
558 | (35, Trait, "trait"); | |
559 | (36, Type, "type"); | |
560 | (37, Unsafe, "unsafe"); | |
561 | (38, Use, "use"); | |
562 | (39, Virtual, "virtual"); | |
563 | (40, While, "while"); | |
564 | (41, Continue, "continue"); | |
c34b1796 AL |
565 | (42, Box, "box"); |
566 | (43, Const, "const"); | |
567 | (44, Where, "where"); | |
1a4d82fc | 568 | 'reserved: |
c34b1796 | 569 | (45, Proc, "proc"); |
85aaf69f SL |
570 | (46, Alignof, "alignof"); |
571 | (47, Become, "become"); | |
572 | (48, Offsetof, "offsetof"); | |
573 | (49, Priv, "priv"); | |
574 | (50, Pure, "pure"); | |
575 | (51, Sizeof, "sizeof"); | |
576 | (52, Typeof, "typeof"); | |
577 | (53, Unsized, "unsized"); | |
578 | (54, Yield, "yield"); | |
579 | (55, Do, "do"); | |
580 | (56, Abstract, "abstract"); | |
581 | (57, Final, "final"); | |
582 | (58, Override, "override"); | |
583 | (59, Macro, "macro"); | |
1a4d82fc | 584 | } |
223e47cc LB |
585 | } |
586 | ||
1a4d82fc JJ |
587 | // looks like we can get rid of this completely... |
588 | pub type IdentInterner = StrInterner; | |
589 | ||
590 | // if an interner exists in TLS, return it. Otherwise, prepare a | |
591 | // fresh one. | |
592 | // FIXME(eddyb) #8726 This should probably use a task-local reference. | |
593 | pub fn get_ident_interner() -> Rc<IdentInterner> { | |
594 | thread_local!(static KEY: Rc<::parse::token::IdentInterner> = { | |
595 | Rc::new(mk_fresh_ident_interner()) | |
596 | }); | |
597 | KEY.with(|k| k.clone()) | |
970d7e83 LB |
598 | } |
599 | ||
1a4d82fc JJ |
600 | /// Reset the ident interner to its initial state. |
601 | pub fn reset_ident_interner() { | |
970d7e83 | 602 | let interner = get_ident_interner(); |
1a4d82fc JJ |
603 | interner.reset(mk_fresh_ident_interner()); |
604 | } | |
605 | ||
606 | /// Represents a string stored in the task-local interner. Because the | |
607 | /// interner lives for the life of the task, this can be safely treated as an | |
608 | /// immortal string, as long as it never crosses between tasks. | |
609 | /// | |
610 | /// FIXME(pcwalton): You must be careful about what you do in the destructors | |
611 | /// of objects stored in TLS, because they may run after the interner is | |
612 | /// destroyed. In particular, they must not access string contents. This can | |
613 | /// be fixed in the future by just leaking all strings until task death | |
614 | /// somehow. | |
615 | #[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)] | |
616 | pub struct InternedString { | |
617 | string: RcStr, | |
618 | } | |
619 | ||
620 | impl InternedString { | |
621 | #[inline] | |
622 | pub fn new(string: &'static str) -> InternedString { | |
623 | InternedString { | |
624 | string: RcStr::new(string), | |
625 | } | |
626 | } | |
627 | ||
628 | #[inline] | |
629 | fn new_from_rc_str(string: RcStr) -> InternedString { | |
630 | InternedString { | |
631 | string: string, | |
632 | } | |
633 | } | |
970d7e83 LB |
634 | } |
635 | ||
1a4d82fc JJ |
636 | impl Deref for InternedString { |
637 | type Target = str; | |
638 | ||
639 | fn deref(&self) -> &str { &*self.string } | |
640 | } | |
641 | ||
c34b1796 | 642 | #[allow(deprecated)] |
1a4d82fc JJ |
643 | impl BytesContainer for InternedString { |
644 | fn container_as_bytes<'a>(&'a self) -> &'a [u8] { | |
645 | // FIXME #12938: This is a workaround for the incorrect signature | |
646 | // of `BytesContainer`, which is itself a workaround for the lack of | |
647 | // DST. | |
648 | unsafe { | |
85aaf69f | 649 | let this = &self[..]; |
1a4d82fc | 650 | mem::transmute::<&[u8],&[u8]>(this.container_as_bytes()) |
970d7e83 | 651 | } |
223e47cc | 652 | } |
223e47cc LB |
653 | } |
654 | ||
85aaf69f | 655 | impl fmt::Debug for InternedString { |
1a4d82fc | 656 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
c34b1796 | 657 | fmt::Debug::fmt(&self.string, f) |
223e47cc | 658 | } |
223e47cc LB |
659 | } |
660 | ||
85aaf69f | 661 | impl fmt::Display for InternedString { |
1a4d82fc | 662 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
c34b1796 | 663 | fmt::Display::fmt(&self.string, f) |
970d7e83 LB |
664 | } |
665 | } | |
666 | ||
1a4d82fc JJ |
667 | impl<'a> PartialEq<&'a str> for InternedString { |
668 | #[inline(always)] | |
669 | fn eq(&self, other: & &'a str) -> bool { | |
c34b1796 | 670 | PartialEq::eq(&self.string[..], *other) |
1a4d82fc JJ |
671 | } |
672 | #[inline(always)] | |
673 | fn ne(&self, other: & &'a str) -> bool { | |
c34b1796 | 674 | PartialEq::ne(&self.string[..], *other) |
970d7e83 LB |
675 | } |
676 | } | |
223e47cc | 677 | |
1a4d82fc JJ |
678 | impl<'a> PartialEq<InternedString > for &'a str { |
679 | #[inline(always)] | |
680 | fn eq(&self, other: &InternedString) -> bool { | |
c34b1796 | 681 | PartialEq::eq(*self, &other.string[..]) |
1a4d82fc JJ |
682 | } |
683 | #[inline(always)] | |
684 | fn ne(&self, other: &InternedString) -> bool { | |
c34b1796 | 685 | PartialEq::ne(*self, &other.string[..]) |
1a4d82fc JJ |
686 | } |
687 | } | |
688 | ||
689 | impl Decodable for InternedString { | |
690 | fn decode<D: Decoder>(d: &mut D) -> Result<InternedString, D::Error> { | |
c34b1796 | 691 | Ok(get_name(get_ident_interner().intern(&try!(d.read_str())[..]))) |
970d7e83 LB |
692 | } |
693 | } | |
694 | ||
1a4d82fc JJ |
695 | impl Encodable for InternedString { |
696 | fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { | |
c34b1796 | 697 | s.emit_str(&self.string) |
1a4d82fc JJ |
698 | } |
699 | } | |
700 | ||
701 | /// Returns the string contents of a name, using the task-local interner. | |
702 | #[inline] | |
703 | pub fn get_name(name: ast::Name) -> InternedString { | |
704 | let interner = get_ident_interner(); | |
705 | InternedString::new_from_rc_str(interner.get(name)) | |
706 | } | |
707 | ||
708 | /// Returns the string contents of an identifier, using the task-local | |
709 | /// interner. | |
710 | #[inline] | |
711 | pub fn get_ident(ident: ast::Ident) -> InternedString { | |
712 | get_name(ident.name) | |
713 | } | |
714 | ||
715 | /// Interns and returns the string contents of an identifier, using the | |
716 | /// task-local interner. | |
717 | #[inline] | |
718 | pub fn intern_and_get_ident(s: &str) -> InternedString { | |
719 | get_name(intern(s)) | |
720 | } | |
721 | ||
722 | /// Maps a string to its interned representation. | |
723 | #[inline] | |
724 | pub fn intern(s: &str) -> ast::Name { | |
725 | get_ident_interner().intern(s) | |
726 | } | |
727 | ||
85aaf69f | 728 | /// gensym's a new usize, using the current interner. |
1a4d82fc JJ |
729 | #[inline] |
730 | pub fn gensym(s: &str) -> ast::Name { | |
731 | get_ident_interner().gensym(s) | |
732 | } | |
733 | ||
734 | /// Maps a string to an identifier with an empty syntax context. | |
735 | #[inline] | |
736 | pub fn str_to_ident(s: &str) -> ast::Ident { | |
737 | ast::Ident::new(intern(s)) | |
738 | } | |
739 | ||
740 | /// Maps a string to a gensym'ed identifier. | |
741 | #[inline] | |
742 | pub fn gensym_ident(s: &str) -> ast::Ident { | |
743 | ast::Ident::new(gensym(s)) | |
744 | } | |
745 | ||
746 | // create a fresh name that maps to the same string as the old one. | |
747 | // note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src))); | |
748 | // that is, that the new name and the old one are connected to ptr_eq strings. | |
749 | pub fn fresh_name(src: &ast::Ident) -> ast::Name { | |
750 | let interner = get_ident_interner(); | |
751 | interner.gensym_copy(src.name) | |
752 | // following: debug version. Could work in final except that it's incompatible with | |
753 | // good error messages and uses of struct names in ambiguous could-be-binding | |
754 | // locations. Also definitely destroys the guarantee given above about ptr_eq. | |
755 | /*let num = rand::thread_rng().gen_uint_range(0,0xffff); | |
756 | gensym(format!("{}_{}",ident_to_string(src),num))*/ | |
757 | } | |
758 | ||
759 | // create a fresh mark. | |
760 | pub fn fresh_mark() -> ast::Mrk { | |
85aaf69f | 761 | gensym("mark").usize() as u32 |
1a4d82fc | 762 | } |
970d7e83 LB |
763 | |
764 | #[cfg(test)] | |
765 | mod test { | |
766 | use super::*; | |
1a4d82fc JJ |
767 | use ast; |
768 | use ext::mtwt; | |
769 | ||
770 | fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident { | |
771 | ast::Ident { name: id.name, ctxt:mtwt::apply_mark(m, id.ctxt) } | |
772 | } | |
773 | ||
774 | #[test] fn mtwt_token_eq_test() { | |
775 | assert!(Gt.mtwt_eq(&Gt)); | |
776 | let a = str_to_ident("bac"); | |
777 | let a1 = mark_ident(a,92); | |
778 | assert!(Ident(a, ModName).mtwt_eq(&Ident(a1, Plain))); | |
970d7e83 LB |
779 | } |
780 | } |