src/libsyntax/parse/token.rs

   1 // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 pub use self::BinOpToken::*;
  12 pub use self::Nonterminal::*;
  13 pub use self::DelimToken::*;
  14 pub use self::Lit::*;
  15 pub use self::Token::*;
  16
  17 use ast::{self};
  18 use ptr::P;
  19 use symbol::keywords;
  20 use tokenstream::TokenTree;
  21
  22 use std::fmt;
  23 use std::rc::Rc;
  24
  25 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
  26 pub enum BinOpToken {
  27     Plus,
  28     Minus,
  29     Star,
  30     Slash,
  31     Percent,
  32     Caret,
  33     And,
  34     Or,
  35     Shl,
  36     Shr,
  37 }
  38
  39 /// A delimiter token
  40 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
  41 pub enum DelimToken {
  42     /// A round parenthesis: `(` or `)`
  43     Paren,
  44     /// A square bracket: `[` or `]`
  45     Bracket,
  46     /// A curly brace: `{` or `}`
  47     Brace,
  48     /// An empty delimiter
  49     NoDelim,
  50 }
  51
  52 impl DelimToken {
  53     pub fn len(self) -> usize {
  54         if self == NoDelim { 0 } else { 1 }
  55     }
  56 }
  57
  58 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
  59 pub enum Lit {
  60     Byte(ast::Name),
  61     Char(ast::Name),
  62     Integer(ast::Name),
  63     Float(ast::Name),
  64     Str_(ast::Name),
  65     StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */
  66     ByteStr(ast::Name),
  67     ByteStrRaw(ast::Name, usize), /* raw byte str delimited by n hash symbols */
  68 }
  69
  70 impl Lit {
  71     pub fn short_name(&self) -> &'static str {
  72         match *self {
  73             Byte(_) => "byte",
  74             Char(_) => "char",
  75             Integer(_) => "integer",
  76             Float(_) => "float",
  77             Str_(_) | StrRaw(..) => "string",
  78             ByteStr(_) | ByteStrRaw(..) => "byte string"
  79         }
  80     }
  81 }
  82
  83 fn ident_can_begin_expr(ident: ast::Ident) -> bool {
  84     let ident_token: Token = Ident(ident);
  85
  86     !ident_token.is_any_keyword() ||
  87     ident_token.is_path_segment_keyword() ||
  88     [
  89         keywords::Do.name(),
  90         keywords::Box.name(),
  91         keywords::Break.name(),
  92         keywords::Continue.name(),
  93         keywords::False.name(),
  94         keywords::For.name(),
  95         keywords::If.name(),
  96         keywords::Loop.name(),
  97         keywords::Match.name(),
  98         keywords::Move.name(),
  99         keywords::Return.name(),
 100         keywords::True.name(),
 101         keywords::Unsafe.name(),
 102         keywords::While.name(),
 103     ].contains(&ident.name)
 104 }
 105
 106 fn ident_can_begin_type(ident: ast::Ident) -> bool {
 107     let ident_token: Token = Ident(ident);
 108
 109     !ident_token.is_any_keyword() ||
 110     ident_token.is_path_segment_keyword() ||
 111     [
 112         keywords::For.name(),
 113         keywords::Impl.name(),
 114         keywords::Fn.name(),
 115         keywords::Unsafe.name(),
 116         keywords::Extern.name(),
 117         keywords::Typeof.name(),
 118     ].contains(&ident.name)
 119 }
 120
 121 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
 122 pub enum Token {
 123     /* Expression-operator symbols. */
 124     Eq,
 125     Lt,
 126     Le,
 127     EqEq,
 128     Ne,
 129     Ge,
 130     Gt,
 131     AndAnd,
 132     OrOr,
 133     Not,
 134     Tilde,
 135     BinOp(BinOpToken),
 136     BinOpEq(BinOpToken),
 137
 138     /* Structural symbols */
 139     At,
 140     Dot,
 141     DotDot,
 142     DotDotDot,
 143     Comma,
 144     Semi,
 145     Colon,
 146     ModSep,
 147     RArrow,
 148     LArrow,
 149     FatArrow,
 150     Pound,
 151     Dollar,
 152     Question,
 153     /// An opening delimiter, eg. `{`
 154     OpenDelim(DelimToken),
 155     /// A closing delimiter, eg. `}`
 156     CloseDelim(DelimToken),
 157
 158     /* Literals */
 159     Literal(Lit, Option<ast::Name>),
 160
 161     /* Name components */
 162     Ident(ast::Ident),
 163     Underscore,
 164     Lifetime(ast::Ident),
 165
 166     /* For interpolation */
 167     Interpolated(Rc<Nonterminal>),
 168     // Can be expanded into several tokens.
 169     /// Doc comment
 170     DocComment(ast::Name),
 171     // In right-hand-sides of MBE macros:
 172     /// A syntactic variable that will be filled in by macro expansion.
 173     SubstNt(ast::Ident),
 174
 175     // Junk. These carry no data because we don't really care about the data
 176     // they *would* carry, and don't really want to allocate a new ident for
 177     // them. Instead, users could extract that from the associated span.
 178
 179     /// Whitespace
 180     Whitespace,
 181     /// Comment
 182     Comment,
 183     Shebang(ast::Name),
 184
 185     Eof,
 186 }
 187
 188 impl Token {
 189     /// Returns `true` if the token starts with '>'.
 190     pub fn is_like_gt(&self) -> bool {
 191         match *self {
 192             BinOp(Shr) | BinOpEq(Shr) | Gt | Ge => true,
 193             _ => false,
 194         }
 195     }
 196
 197     /// Returns `true` if the token can appear at the start of an expression.
 198     pub fn can_begin_expr(&self) -> bool {
 199         match *self {
 200             Ident(ident)                => ident_can_begin_expr(ident), // value name or keyword
 201             OpenDelim(..)               => true, // tuple, array or block
 202             Literal(..)                 => true, // literal
 203             Not                         => true, // operator not
 204             BinOp(Minus)                => true, // unary minus
 205             BinOp(Star)                 => true, // dereference
 206             BinOp(Or) | OrOr            => true, // closure
 207             BinOp(And)                  => true, // reference
 208             AndAnd                      => true, // double reference
 209             DotDot | DotDotDot          => true, // range notation
 210             Lt | BinOp(Shl)             => true, // associated path
 211             ModSep                      => true, // global path
 212             Pound                       => true, // expression attributes
 213             Interpolated(ref nt) => match **nt {
 214                 NtIdent(..) | NtExpr(..) | NtBlock(..) | NtPath(..) => true,
 215                 _ => false,
 216             },
 217             _ => false,
 218         }
 219     }
 220
 221     /// Returns `true` if the token can appear at the start of a type.
 222     pub fn can_begin_type(&self) -> bool {
 223         match *self {
 224             Ident(ident)                => ident_can_begin_type(ident), // type name or keyword
 225             OpenDelim(Paren)            => true, // tuple
 226             OpenDelim(Bracket)          => true, // array
 227             Underscore                  => true, // placeholder
 228             Not                         => true, // never
 229             BinOp(Star)                 => true, // raw pointer
 230             BinOp(And)                  => true, // reference
 231             AndAnd                      => true, // double reference
 232             Question                    => true, // maybe bound in trait object
 233             Lifetime(..)                => true, // lifetime bound in trait object
 234             Lt | BinOp(Shl)             => true, // associated path
 235             ModSep                      => true, // global path
 236             Interpolated(ref nt) => match **nt {
 237                 NtIdent(..) | NtTy(..) | NtPath(..) => true,
 238                 _ => false,
 239             },
 240             _ => false,
 241         }
 242     }
 243
 244     /// Returns `true` if the token is any literal
 245     pub fn is_lit(&self) -> bool {
 246         match *self {
 247             Literal(..) => true,
 248             _           => false,
 249         }
 250     }
 251
 252     pub fn ident(&self) -> Option<ast::Ident> {
 253         match *self {
 254             Ident(ident) => Some(ident),
 255             Interpolated(ref nt) => match **nt {
 256                 NtIdent(ident) => Some(ident.node),
 257                 _ => None,
 258             },
 259             _ => None,
 260         }
 261     }
 262
 263     /// Returns `true` if the token is an identifier.
 264     pub fn is_ident(&self) -> bool {
 265         self.ident().is_some()
 266     }
 267
 268     /// Returns `true` if the token is a documentation comment.
 269     pub fn is_doc_comment(&self) -> bool {
 270         match *self {
 271             DocComment(..)   => true,
 272             _                => false,
 273         }
 274     }
 275
 276     /// Returns `true` if the token is interpolated.
 277     pub fn is_interpolated(&self) -> bool {
 278         match *self {
 279             Interpolated(..) => true,
 280             _                => false,
 281         }
 282     }
 283
 284     /// Returns `true` if the token is an interpolated path.
 285     pub fn is_path(&self) -> bool {
 286         if let Interpolated(ref nt) = *self {
 287             if let NtPath(..) = **nt {
 288                 return true;
 289             }
 290         }
 291         false
 292     }
 293
 294     /// Returns `true` if the token is a lifetime.
 295     pub fn is_lifetime(&self) -> bool {
 296         match *self {
 297             Lifetime(..) => true,
 298             _            => false,
 299         }
 300     }
 301
 302     /// Returns `true` if the token is either the `mut` or `const` keyword.
 303     pub fn is_mutability(&self) -> bool {
 304         self.is_keyword(keywords::Mut) ||
 305         self.is_keyword(keywords::Const)
 306     }
 307
 308     pub fn is_qpath_start(&self) -> bool {
 309         self == &Lt || self == &BinOp(Shl)
 310     }
 311
 312     pub fn is_path_start(&self) -> bool {
 313         self == &ModSep || self.is_qpath_start() || self.is_path() ||
 314         self.is_path_segment_keyword() || self.is_ident() && !self.is_any_keyword()
 315     }
 316
 317     /// Returns `true` if the token is a given keyword, `kw`.
 318     pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
 319         self.ident().map(|ident| ident.name == kw.name()).unwrap_or(false)
 320     }
 321
 322     pub fn is_path_segment_keyword(&self) -> bool {
 323         match self.ident() {
 324             Some(id) => id.name == keywords::Super.name() ||
 325                         id.name == keywords::SelfValue.name() ||
 326                         id.name == keywords::SelfType.name(),
 327             None => false,
 328         }
 329     }
 330
 331     /// Returns `true` if the token is either a strict or reserved keyword.
 332     pub fn is_any_keyword(&self) -> bool {
 333         self.is_strict_keyword() || self.is_reserved_keyword()
 334     }
 335
 336     /// Returns `true` if the token is a strict keyword.
 337     pub fn is_strict_keyword(&self) -> bool {
 338         match self.ident() {
 339             Some(id) => id.name >= keywords::As.name() && id.name <= keywords::While.name(),
 340             _ => false,
 341         }
 342     }
 343
 344     /// Returns `true` if the token is a keyword reserved for possible future use.
 345     pub fn is_reserved_keyword(&self) -> bool {
 346         match self.ident() {
 347             Some(id) => id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name(),
 348             _ => false,
 349         }
 350     }
 351 }
 352
 353 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
 354 /// For interpolation during macro expansion.
 355 pub enum Nonterminal {
 356     NtItem(P<ast::Item>),
 357     NtBlock(P<ast::Block>),
 358     NtStmt(ast::Stmt),
 359     NtPat(P<ast::Pat>),
 360     NtExpr(P<ast::Expr>),
 361     NtTy(P<ast::Ty>),
 362     NtIdent(ast::SpannedIdent),
 363     /// Stuff inside brackets for attributes
 364     NtMeta(ast::MetaItem),
 365     NtPath(ast::Path),
 366     NtVis(ast::Visibility),
 367     NtTT(TokenTree),
 368     // These are not exposed to macros, but are used by quasiquote.
 369     NtArm(ast::Arm),
 370     NtImplItem(ast::ImplItem),
 371     NtTraitItem(ast::TraitItem),
 372     NtGenerics(ast::Generics),
 373     NtWhereClause(ast::WhereClause),
 374     NtArg(ast::Arg),
 375 }
 376
 377 impl fmt::Debug for Nonterminal {
 378     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 379         match *self {
 380             NtItem(..) => f.pad("NtItem(..)"),
 381             NtBlock(..) => f.pad("NtBlock(..)"),
 382             NtStmt(..) => f.pad("NtStmt(..)"),
 383             NtPat(..) => f.pad("NtPat(..)"),
 384             NtExpr(..) => f.pad("NtExpr(..)"),
 385             NtTy(..) => f.pad("NtTy(..)"),
 386             NtIdent(..) => f.pad("NtIdent(..)"),
 387             NtMeta(..) => f.pad("NtMeta(..)"),
 388             NtPath(..) => f.pad("NtPath(..)"),
 389             NtTT(..) => f.pad("NtTT(..)"),
 390             NtArm(..) => f.pad("NtArm(..)"),
 391             NtImplItem(..) => f.pad("NtImplItem(..)"),
 392             NtTraitItem(..) => f.pad("NtTraitItem(..)"),
 393             NtGenerics(..) => f.pad("NtGenerics(..)"),
 394             NtWhereClause(..) => f.pad("NtWhereClause(..)"),
 395             NtArg(..) => f.pad("NtArg(..)"),
 396             NtVis(..) => f.pad("NtVis(..)"),
 397         }
 398     }
 399 }