1 // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 pub use self::BinOpToken
::*;
12 pub use self::Nonterminal
::*;
13 pub use self::DelimToken
::*;
14 pub use self::IdentStyle
::*;
16 pub use self::Token
::*;
18 use ast
::{self, BinOpKind}
;
21 use util
::interner
::{RcStr, StrInterner}
;
24 use serialize
::{Decodable, Decoder, Encodable, Encoder}
;
29 #[allow(non_camel_case_types)]
30 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
45 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
47 /// A round parenthesis: `(` or `)`
49 /// A square bracket: `[` or `]`
51 /// A curly brace: `{` or `}`
55 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
57 /// `::` follows the identifier with no whitespace in-between.
62 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
63 pub enum SpecialMacroVar
{
64 /// `$crate` will be filled in with the name of the crate a macro was
65 /// imported from, if any.
69 impl SpecialMacroVar
{
70 pub fn as_str(self) -> &'
static str {
72 SpecialMacroVar
::CrateMacroVar
=> "crate",
77 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
84 StrRaw(ast
::Name
, usize), /* raw str delimited by n hash symbols */
86 ByteStrRaw(ast
::Name
, usize), /* raw byte str delimited by n hash symbols */
90 pub fn short_name(&self) -> &'
static str {
94 Integer(_
) => "integer",
96 Str_(_
) | StrRaw(..) => "string",
97 ByteStr(_
) | ByteStrRaw(..) => "byte string"
102 #[allow(non_camel_case_types)]
103 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
105 /* Expression-operator symbols. */
120 /* Structural symbols */
135 /// An opening delimiter, eg. `{`
136 OpenDelim(DelimToken
),
137 /// A closing delimiter, eg. `}`
138 CloseDelim(DelimToken
),
141 Literal(Lit
, Option
<ast
::Name
>),
143 /* Name components */
144 Ident(ast
::Ident
, IdentStyle
),
146 Lifetime(ast
::Ident
),
148 /* For interpolation */
149 Interpolated(Nonterminal
),
150 // Can be expanded into several tokens.
152 DocComment(ast
::Name
),
153 // In left-hand-sides of MBE macros:
154 /// Parse a nonterminal (name to bind, name of NT, styles of their idents)
155 MatchNt(ast
::Ident
, ast
::Ident
, IdentStyle
, IdentStyle
),
156 // In right-hand-sides of MBE macros:
157 /// A syntactic variable that will be filled in by macro expansion.
158 SubstNt(ast
::Ident
, IdentStyle
),
159 /// A macro variable with special meaning.
160 SpecialVarNt(SpecialMacroVar
),
162 // Junk. These carry no data because we don't really care about the data
163 // they *would* carry, and don't really want to allocate a new ident for
164 // them. Instead, users could extract that from the associated span.
176 /// Returns `true` if the token starts with '>'.
177 pub fn is_like_gt(&self) -> bool
{
179 BinOp(Shr
) | BinOpEq(Shr
) | Gt
| Ge
=> true,
184 /// Returns `true` if the token can appear at the start of an expression.
185 pub fn can_begin_expr(&self) -> bool
{
187 OpenDelim(_
) => true,
191 Literal(_
, _
) => true,
193 BinOp(Minus
) => true,
196 BinOp(Or
) => true, // in lambda syntax
197 OrOr
=> true, // in lambda syntax
198 AndAnd
=> true, // double borrow
199 DotDot
| DotDotDot
=> true, // range notation
201 Interpolated(NtExpr(..)) => true,
202 Interpolated(NtIdent(..)) => true,
203 Interpolated(NtBlock(..)) => true,
204 Interpolated(NtPath(..)) => true,
205 Pound
=> true, // for expression attributes
210 /// Returns `true` if the token is any literal
211 pub fn is_lit(&self) -> bool
{
213 Literal(_
, _
) => true,
218 /// Returns `true` if the token is an identifier.
219 pub fn is_ident(&self) -> bool
{
226 /// Returns `true` if the token is interpolated.
227 pub fn is_interpolated(&self) -> bool
{
229 Interpolated(..) => true,
234 /// Returns `true` if the token is an interpolated path.
235 pub fn is_path(&self) -> bool
{
237 Interpolated(NtPath(..)) => true,
242 /// Returns `true` if the token is a path that is not followed by a `::`
244 #[allow(non_upper_case_globals)]
245 pub fn is_plain_ident(&self) -> bool
{
247 Ident(_
, Plain
) => true,
252 /// Returns `true` if the token is a lifetime.
253 pub fn is_lifetime(&self) -> bool
{
255 Lifetime(..) => true,
260 /// Returns `true` if the token is either the `mut` or `const` keyword.
261 pub fn is_mutability(&self) -> bool
{
262 self.is_keyword(keywords
::Mut
) ||
263 self.is_keyword(keywords
::Const
)
266 /// Maps a token to its corresponding binary operator.
267 pub fn to_binop(&self) -> Option
<BinOpKind
> {
269 BinOp(Star
) => Some(BinOpKind
::Mul
),
270 BinOp(Slash
) => Some(BinOpKind
::Div
),
271 BinOp(Percent
) => Some(BinOpKind
::Rem
),
272 BinOp(Plus
) => Some(BinOpKind
::Add
),
273 BinOp(Minus
) => Some(BinOpKind
::Sub
),
274 BinOp(Shl
) => Some(BinOpKind
::Shl
),
275 BinOp(Shr
) => Some(BinOpKind
::Shr
),
276 BinOp(And
) => Some(BinOpKind
::BitAnd
),
277 BinOp(Caret
) => Some(BinOpKind
::BitXor
),
278 BinOp(Or
) => Some(BinOpKind
::BitOr
),
279 Lt
=> Some(BinOpKind
::Lt
),
280 Le
=> Some(BinOpKind
::Le
),
281 Ge
=> Some(BinOpKind
::Ge
),
282 Gt
=> Some(BinOpKind
::Gt
),
283 EqEq
=> Some(BinOpKind
::Eq
),
284 Ne
=> Some(BinOpKind
::Ne
),
285 AndAnd
=> Some(BinOpKind
::And
),
286 OrOr
=> Some(BinOpKind
::Or
),
291 /// Returns `true` if the token is a given keyword, `kw`.
292 #[allow(non_upper_case_globals)]
293 pub fn is_keyword(&self, kw
: keywords
::Keyword
) -> bool
{
295 Ident(sid
, Plain
) => kw
.to_name() == sid
.name
,
300 pub fn is_keyword_allow_following_colon(&self, kw
: keywords
::Keyword
) -> bool
{
302 Ident(sid
, _
) => { kw.to_name() == sid.name }
307 /// Returns `true` if the token is either a special identifier, or a strict
308 /// or reserved keyword.
309 #[allow(non_upper_case_globals)]
310 pub fn is_any_keyword(&self) -> bool
{
312 Ident(sid
, Plain
) => {
315 n
== SELF_KEYWORD_NAME
316 || n
== STATIC_KEYWORD_NAME
317 || n
== SUPER_KEYWORD_NAME
318 || n
== SELF_TYPE_KEYWORD_NAME
319 || STRICT_KEYWORD_START
<= n
320 && n
<= RESERVED_KEYWORD_FINAL
326 /// Returns `true` if the token may not appear as an identifier.
327 #[allow(non_upper_case_globals)]
328 pub fn is_strict_keyword(&self) -> bool
{
330 Ident(sid
, Plain
) => {
333 n
== SELF_KEYWORD_NAME
334 || n
== STATIC_KEYWORD_NAME
335 || n
== SUPER_KEYWORD_NAME
336 || n
== SELF_TYPE_KEYWORD_NAME
337 || STRICT_KEYWORD_START
<= n
338 && n
<= STRICT_KEYWORD_FINAL
340 Ident(sid
, ModName
) => {
343 n
!= SELF_KEYWORD_NAME
344 && n
!= SUPER_KEYWORD_NAME
345 && STRICT_KEYWORD_START
<= n
346 && n
<= STRICT_KEYWORD_FINAL
352 /// Returns `true` if the token is a keyword that has been reserved for
353 /// possible future use.
354 #[allow(non_upper_case_globals)]
355 pub fn is_reserved_keyword(&self) -> bool
{
357 Ident(sid
, Plain
) => {
360 RESERVED_KEYWORD_START
<= n
361 && n
<= RESERVED_KEYWORD_FINAL
367 /// Hygienic identifier equality comparison.
369 /// See `styntax::ext::mtwt`.
370 pub fn mtwt_eq(&self, other
: &Token
) -> bool
{
371 match (self, other
) {
372 (&Ident(id1
,_
), &Ident(id2
,_
)) | (&Lifetime(id1
), &Lifetime(id2
)) =>
373 mtwt
::resolve(id1
) == mtwt
::resolve(id2
),
379 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
380 /// For interpolation during macro expansion.
381 pub enum Nonterminal
{
382 NtItem(P
<ast
::Item
>),
383 NtBlock(P
<ast
::Block
>),
384 NtStmt(P
<ast
::Stmt
>),
386 NtExpr(P
<ast
::Expr
>),
388 NtIdent(Box
<ast
::SpannedIdent
>, IdentStyle
),
389 /// Stuff inside brackets for attributes
390 NtMeta(P
<ast
::MetaItem
>),
391 NtPath(Box
<ast
::Path
>),
392 NtTT(P
<ast
::TokenTree
>), // needs P'ed to break a circularity
393 // These are not exposed to macros, but are used by quasiquote.
395 NtImplItem(P
<ast
::ImplItem
>),
396 NtTraitItem(P
<ast
::TraitItem
>),
397 NtGenerics(ast
::Generics
),
398 NtWhereClause(ast
::WhereClause
),
402 impl fmt
::Debug
for Nonterminal
{
403 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
405 NtItem(..) => f
.pad("NtItem(..)"),
406 NtBlock(..) => f
.pad("NtBlock(..)"),
407 NtStmt(..) => f
.pad("NtStmt(..)"),
408 NtPat(..) => f
.pad("NtPat(..)"),
409 NtExpr(..) => f
.pad("NtExpr(..)"),
410 NtTy(..) => f
.pad("NtTy(..)"),
411 NtIdent(..) => f
.pad("NtIdent(..)"),
412 NtMeta(..) => f
.pad("NtMeta(..)"),
413 NtPath(..) => f
.pad("NtPath(..)"),
414 NtTT(..) => f
.pad("NtTT(..)"),
415 NtArm(..) => f
.pad("NtArm(..)"),
416 NtImplItem(..) => f
.pad("NtImplItem(..)"),
417 NtTraitItem(..) => f
.pad("NtTraitItem(..)"),
418 NtGenerics(..) => f
.pad("NtGenerics(..)"),
419 NtWhereClause(..) => f
.pad("NtWhereClause(..)"),
420 NtArg(..) => f
.pad("NtArg(..)"),
426 // Get the first "argument"
428 ( $first
:expr
, $
( $remainder
:expr
, )* ) => ( $first
)
431 // Get the last "argument" (has to be done recursively to avoid phoney local ambiguity error)
433 ( $first
:expr
, $
( $remainder
:expr
, )+ ) => ( last
!( $
( $remainder
, )+ ) );
434 ( $first
:expr
, ) => ( $first
)
437 // In this macro, there is the requirement that the name (the number) must be monotonically
438 // increasing by one in the special identifiers, starting at 0; the same holds for the keywords,
439 // except starting from the next number instead of zero, and with the additional exception that
440 // special identifiers are *also* allowed (they are deduplicated in the important place, the
441 // interner), an exception which is demonstrated by "static" and "self".
442 macro_rules
! declare_special_idents_and_keywords
{(
443 // So now, in these rules, why is each definition parenthesised?
444 // Answer: otherwise we get a spurious local ambiguity bug on the "}"
445 pub mod special_idents
{
446 $
( ($si_name
:expr
, $si_static
:ident
, $si_str
:expr
); )*
451 $
( ($sk_name
:expr
, $sk_variant
:ident
, $sk_str
:expr
); )*
453 $
( ($rk_name
:expr
, $rk_variant
:ident
, $rk_str
:expr
); )*
456 const STRICT_KEYWORD_START
: ast
::Name
= first
!($
( ast
::Name($sk_name
), )*);
457 const STRICT_KEYWORD_FINAL
: ast
::Name
= last
!($
( ast
::Name($sk_name
), )*);
458 const RESERVED_KEYWORD_START
: ast
::Name
= first
!($
( ast
::Name($rk_name
), )*);
459 const RESERVED_KEYWORD_FINAL
: ast
::Name
= last
!($
( ast
::Name($rk_name
), )*);
461 pub mod special_idents
{
464 #[allow(non_upper_case_globals)]
465 pub const $si_static
: ast
::Ident
= ast
::Ident
{
466 name
: ast
::Name($si_name
),
467 ctxt
: ast
::EMPTY_CTXT
,
472 pub mod special_names
{
475 #[allow(non_upper_case_globals)]
476 pub const $si_static
: ast
::Name
= ast
::Name($si_name
);
480 /// All the valid words that have meaning in the Rust language.
482 /// Rust keywords are either 'strict' or 'reserved'. Strict keywords may not
483 /// appear as identifiers at all. Reserved keywords are not used anywhere in
484 /// the language and may not appear as identifiers.
486 pub use self::Keyword
::*;
489 #[derive(Copy, Clone, PartialEq, Eq)]
496 pub fn to_name(&self) -> ast
::Name
{
498 $
( $sk_variant
=> ast
::Name($sk_name
), )*
499 $
( $rk_variant
=> ast
::Name($rk_name
), )*
505 fn mk_fresh_ident_interner() -> IdentInterner
{
506 let mut init_vec
= Vec
::new();
507 $
(init_vec
.push($si_str
);)*
508 $
(init_vec
.push($sk_str
);)*
509 $
(init_vec
.push($rk_str
);)*
510 interner
::StrInterner
::prefill(&init_vec
[..])
514 // If the special idents get renumbered, remember to modify these two as appropriate
515 pub const SELF_KEYWORD_NAME
: ast
::Name
= ast
::Name(SELF_KEYWORD_NAME_NUM
);
516 const STATIC_KEYWORD_NAME
: ast
::Name
= ast
::Name(STATIC_KEYWORD_NAME_NUM
);
517 pub const SUPER_KEYWORD_NAME
: ast
::Name
= ast
::Name(SUPER_KEYWORD_NAME_NUM
);
518 const SELF_TYPE_KEYWORD_NAME
: ast
::Name
= ast
::Name(SELF_TYPE_KEYWORD_NAME_NUM
);
520 pub const SELF_KEYWORD_NAME_NUM
: u32 = 1;
521 const STATIC_KEYWORD_NAME_NUM
: u32 = 2;
522 const SUPER_KEYWORD_NAME_NUM
: u32 = 3;
523 const SELF_TYPE_KEYWORD_NAME_NUM
: u32 = 10;
525 // NB: leaving holes in the ident table is bad! a different ident will get
526 // interned with the id from the hole, but it will be between the min and max
527 // of the reserved words, and thus tagged as "reserved".
529 declare_special_idents_and_keywords
! {
530 pub mod special_idents
{
531 // These ones are statics
533 (super::SELF_KEYWORD_NAME_NUM
, self_
, "self");
534 (super::STATIC_KEYWORD_NAME_NUM
, statik
, "static");
535 (super::SUPER_KEYWORD_NAME_NUM
, super_
, "super");
536 (4, static_lifetime
, "'static");
540 (6, matchers
, "matchers");
542 // outside of libsyntax
543 (7, clownshoe_abi
, "__rust_abi");
544 (8, opaque
, "<opaque>");
545 (9, __unused1
, "<__unused1>");
546 (super::SELF_TYPE_KEYWORD_NAME_NUM
, type_self
, "Self");
547 (11, prelude_import
, "prelude_import");
548 (12, DEFAULT
, "default");
552 // These ones are variants of the Keyword enum
556 (14, Break
, "break");
557 (15, Crate
, "crate");
560 (18, Extern
, "extern");
561 (19, False
, "false");
569 (27, Match
, "match");
575 (33, Return
, "return");
576 // Static and Self are also special idents (prefill de-dupes)
577 (super::STATIC_KEYWORD_NAME_NUM
, Static
, "static");
578 (super::SELF_KEYWORD_NAME_NUM
, SelfValue
, "self");
579 (super::SELF_TYPE_KEYWORD_NAME_NUM
, SelfType
, "Self");
580 (34, Struct
, "struct");
581 (super::SUPER_KEYWORD_NAME_NUM
, Super
, "super");
583 (36, Trait
, "trait");
585 (38, Unsafe
, "unsafe");
587 (40, While
, "while");
588 (41, Continue
, "continue");
590 (43, Const
, "const");
591 (44, Where
, "where");
593 (45, Virtual
, "virtual");
595 (47, Alignof
, "alignof");
596 (48, Become
, "become");
597 (49, Offsetof
, "offsetof");
600 (52, Sizeof
, "sizeof");
601 (53, Typeof
, "typeof");
602 (54, Unsized
, "unsized");
603 (55, Yield
, "yield");
605 (57, Abstract
, "abstract");
606 (58, Final
, "final");
607 (59, Override
, "override");
608 (60, Macro
, "macro");
612 // looks like we can get rid of this completely...
613 pub type IdentInterner
= StrInterner
;
615 // if an interner exists in TLS, return it. Otherwise, prepare a
617 // FIXME(eddyb) #8726 This should probably use a thread-local reference.
618 pub fn get_ident_interner() -> Rc
<IdentInterner
> {
619 thread_local
!(static KEY
: Rc
<::parse
::token
::IdentInterner
> = {
620 Rc
::new(mk_fresh_ident_interner())
622 KEY
.with(|k
| k
.clone())
625 /// Reset the ident interner to its initial state.
626 pub fn reset_ident_interner() {
627 let interner
= get_ident_interner();
628 interner
.reset(mk_fresh_ident_interner());
631 /// Represents a string stored in the thread-local interner. Because the
632 /// interner lives for the life of the thread, this can be safely treated as an
633 /// immortal string, as long as it never crosses between threads.
635 /// FIXME(pcwalton): You must be careful about what you do in the destructors
636 /// of objects stored in TLS, because they may run after the interner is
637 /// destroyed. In particular, they must not access string contents. This can
638 /// be fixed in the future by just leaking all strings until thread death
640 #[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)]
641 pub struct InternedString
{
645 impl InternedString
{
647 pub fn new(string
: &'
static str) -> InternedString
{
649 string
: RcStr
::new(string
),
654 fn new_from_rc_str(string
: RcStr
) -> InternedString
{
661 pub fn new_from_name(name
: ast
::Name
) -> InternedString
{
662 let interner
= get_ident_interner();
663 InternedString
::new_from_rc_str(interner
.get(name
))
667 impl Deref
for InternedString
{
670 fn deref(&self) -> &str { &self.string }
673 impl fmt
::Debug
for InternedString
{
674 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
675 fmt
::Debug
::fmt(&self.string
, f
)
679 impl fmt
::Display
for InternedString
{
680 fn fmt(&self, f
: &mut fmt
::Formatter
) -> fmt
::Result
{
681 fmt
::Display
::fmt(&self.string
, f
)
685 impl<'a
> PartialEq
<&'a
str> for InternedString
{
687 fn eq(&self, other
: & &'a
str) -> bool
{
688 PartialEq
::eq(&self.string
[..], *other
)
691 fn ne(&self, other
: & &'a
str) -> bool
{
692 PartialEq
::ne(&self.string
[..], *other
)
696 impl<'a
> PartialEq
<InternedString
> for &'a
str {
698 fn eq(&self, other
: &InternedString
) -> bool
{
699 PartialEq
::eq(*self, &other
.string
[..])
702 fn ne(&self, other
: &InternedString
) -> bool
{
703 PartialEq
::ne(*self, &other
.string
[..])
707 impl Decodable
for InternedString
{
708 fn decode
<D
: Decoder
>(d
: &mut D
) -> Result
<InternedString
, D
::Error
> {
709 Ok(intern(d
.read_str()?
.as_ref()).as_str())
713 impl Encodable
for InternedString
{
714 fn encode
<S
: Encoder
>(&self, s
: &mut S
) -> Result
<(), S
::Error
> {
715 s
.emit_str(&self.string
)
719 /// Interns and returns the string contents of an identifier, using the
720 /// thread-local interner.
722 pub fn intern_and_get_ident(s
: &str) -> InternedString
{
726 /// Maps a string to its interned representation.
728 pub fn intern(s
: &str) -> ast
::Name
{
729 get_ident_interner().intern(s
)
732 /// gensym's a new usize, using the current interner.
734 pub fn gensym(s
: &str) -> ast
::Name
{
735 get_ident_interner().gensym(s
)
738 /// Maps a string to an identifier with an empty syntax context.
740 pub fn str_to_ident(s
: &str) -> ast
::Ident
{
741 ast
::Ident
::with_empty_ctxt(intern(s
))
744 /// Maps a string to a gensym'ed identifier.
746 pub fn gensym_ident(s
: &str) -> ast
::Ident
{
747 ast
::Ident
::with_empty_ctxt(gensym(s
))
750 // create a fresh name that maps to the same string as the old one.
751 // note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src)));
752 // that is, that the new name and the old one are connected to ptr_eq strings.
753 pub fn fresh_name(src
: ast
::Ident
) -> ast
::Name
{
754 let interner
= get_ident_interner();
755 interner
.gensym_copy(src
.name
)
756 // following: debug version. Could work in final except that it's incompatible with
757 // good error messages and uses of struct names in ambiguous could-be-binding
758 // locations. Also definitely destroys the guarantee given above about ptr_eq.
759 /*let num = rand::thread_rng().gen_uint_range(0,0xffff);
760 gensym(format!("{}_{}",ident_to_string(src),num))*/
763 // create a fresh mark.
764 pub fn fresh_mark() -> ast
::Mrk
{
774 fn mark_ident(id
: ast
::Ident
, m
: ast
::Mrk
) -> ast
::Ident
{
775 ast
::Ident
::new(id
.name
, mtwt
::apply_mark(m
, id
.ctxt
))
778 #[test] fn mtwt_token_eq_test() {
779 assert
!(Gt
.mtwt_eq(&Gt
));
780 let a
= str_to_ident("bac");
781 let a1
= mark_ident(a
,92);
782 assert
!(Ident(a
, ModName
).mtwt_eq(&Ident(a1
, Plain
)));