ceph/src/jaegertracing/opentelemetry-cpp/third_party/googletest/googlemock/scripts/generator/cpp/ast.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Neal Norwitz
   4 # Portions Copyright 2007 Google Inc.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #      http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 """Generate an Abstract Syntax Tree (AST) for C++."""
  19
  20 # FIXME:
  21 #  * Tokens should never be exported, need to convert to Nodes
  22 #    (return types, parameters, etc.)
  23 #  * Handle static class data for templatized classes
  24 #  * Handle casts (both C++ and C-style)
  25 #  * Handle conditions and loops (if/else, switch, for, while/do)
  26 #
  27 # TODO much, much later:
  28 #  * Handle #define
  29 #  * exceptions
  30
  31
  32 try:
  33   # Python 3.x
  34   import builtins
  35 except ImportError:
  36   # Python 2.x
  37   import __builtin__ as builtins
  38
  39 import sys
  40 import traceback
  41
  42 from cpp import keywords
  43 from cpp import tokenize
  44 from cpp import utils
  45
  46
  47 if not hasattr(builtins, 'reversed'):
  48   # Support Python 2.3 and earlier.
  49   def reversed(seq):
  50     for i in range(len(seq)-1, -1, -1):
  51       yield seq[i]
  52
  53 if not hasattr(builtins, 'next'):
  54   # Support Python 2.5 and earlier.
  55   def next(obj):
  56     return obj.next()
  57
  58
  59 VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3)
  60
  61 FUNCTION_NONE = 0x00
  62 FUNCTION_CONST = 0x01
  63 FUNCTION_VIRTUAL = 0x02
  64 FUNCTION_PURE_VIRTUAL = 0x04
  65 FUNCTION_CTOR = 0x08
  66 FUNCTION_DTOR = 0x10
  67 FUNCTION_ATTRIBUTE = 0x20
  68 FUNCTION_UNKNOWN_ANNOTATION = 0x40
  69 FUNCTION_THROW = 0x80
  70 FUNCTION_OVERRIDE = 0x100
  71
  72 """
  73 These are currently unused.  Should really handle these properly at some point.
  74
  75 TYPE_MODIFIER_INLINE   = 0x010000
  76 TYPE_MODIFIER_EXTERN   = 0x020000
  77 TYPE_MODIFIER_STATIC   = 0x040000
  78 TYPE_MODIFIER_CONST    = 0x080000
  79 TYPE_MODIFIER_REGISTER = 0x100000
  80 TYPE_MODIFIER_VOLATILE = 0x200000
  81 TYPE_MODIFIER_MUTABLE  = 0x400000
  82
  83 TYPE_MODIFIER_MAP = {
  84     'inline': TYPE_MODIFIER_INLINE,
  85     'extern': TYPE_MODIFIER_EXTERN,
  86     'static': TYPE_MODIFIER_STATIC,
  87     'const': TYPE_MODIFIER_CONST,
  88     'register': TYPE_MODIFIER_REGISTER,
  89     'volatile': TYPE_MODIFIER_VOLATILE,
  90     'mutable': TYPE_MODIFIER_MUTABLE,
  91     }
  92 """
  93
  94 _INTERNAL_TOKEN = 'internal'
  95 _NAMESPACE_POP = 'ns-pop'
  96
  97
  98 # TODO(nnorwitz): use this as a singleton for templated_types, etc
  99 # where we don't want to create a new empty dict each time.  It is also const.
 100 class _NullDict(object):
 101   __contains__ = lambda self: False
 102   keys = values = items = iterkeys = itervalues = iteritems = lambda self: ()
 103
 104
 105 # TODO(nnorwitz): move AST nodes into a separate module.
 106 class Node(object):
 107   """Base AST node."""
 108
 109   def __init__(self, start, end):
 110     self.start = start
 111     self.end = end
 112
 113   def IsDeclaration(self):
 114     """Returns bool if this node is a declaration."""
 115     return False
 116
 117   def IsDefinition(self):
 118     """Returns bool if this node is a definition."""
 119     return False
 120
 121   def IsExportable(self):
 122     """Returns bool if this node exportable from a header file."""
 123     return False
 124
 125   def Requires(self, node):
 126     """Does this AST node require the definition of the node passed in?"""
 127     return False
 128
 129   def XXX__str__(self):
 130     return self._StringHelper(self.__class__.__name__, '')
 131
 132   def _StringHelper(self, name, suffix):
 133     if not utils.DEBUG:
 134       return '%s(%s)' % (name, suffix)
 135     return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix)
 136
 137   def __repr__(self):
 138     return str(self)
 139
 140
 141 class Define(Node):
 142   def __init__(self, start, end, name, definition):
 143     Node.__init__(self, start, end)
 144     self.name = name
 145     self.definition = definition
 146
 147   def __str__(self):
 148     value = '%s %s' % (self.name, self.definition)
 149     return self._StringHelper(self.__class__.__name__, value)
 150
 151
 152 class Include(Node):
 153   def __init__(self, start, end, filename, system):
 154     Node.__init__(self, start, end)
 155     self.filename = filename
 156     self.system = system
 157
 158   def __str__(self):
 159     fmt = '"%s"'
 160     if self.system:
 161       fmt = '<%s>'
 162     return self._StringHelper(self.__class__.__name__, fmt % self.filename)
 163
 164
 165 class Goto(Node):
 166   def __init__(self, start, end, label):
 167     Node.__init__(self, start, end)
 168     self.label = label
 169
 170   def __str__(self):
 171     return self._StringHelper(self.__class__.__name__, str(self.label))
 172
 173
 174 class Expr(Node):
 175   def __init__(self, start, end, expr):
 176     Node.__init__(self, start, end)
 177     self.expr = expr
 178
 179   def Requires(self, node):
 180     # TODO(nnorwitz): impl.
 181     return False
 182
 183   def __str__(self):
 184     return self._StringHelper(self.__class__.__name__, str(self.expr))
 185
 186
 187 class Return(Expr):
 188   pass
 189
 190
 191 class Delete(Expr):
 192   pass
 193
 194
 195 class Friend(Expr):
 196   def __init__(self, start, end, expr, namespace):
 197     Expr.__init__(self, start, end, expr)
 198     self.namespace = namespace[:]
 199
 200
 201 class Using(Node):
 202   def __init__(self, start, end, names):
 203     Node.__init__(self, start, end)
 204     self.names = names
 205
 206   def __str__(self):
 207     return self._StringHelper(self.__class__.__name__, str(self.names))
 208
 209
 210 class Parameter(Node):
 211   def __init__(self, start, end, name, parameter_type, default):
 212     Node.__init__(self, start, end)
 213     self.name = name
 214     self.type = parameter_type
 215     self.default = default
 216
 217   def Requires(self, node):
 218     # TODO(nnorwitz): handle namespaces, etc.
 219     return self.type.name == node.name
 220
 221   def __str__(self):
 222     name = str(self.type)
 223     suffix = '%s %s' % (name, self.name)
 224     if self.default:
 225       suffix += ' = ' + ''.join([d.name for d in self.default])
 226     return self._StringHelper(self.__class__.__name__, suffix)
 227
 228
 229 class _GenericDeclaration(Node):
 230   def __init__(self, start, end, name, namespace):
 231     Node.__init__(self, start, end)
 232     self.name = name
 233     self.namespace = namespace[:]
 234
 235   def FullName(self):
 236     prefix = ''
 237     if self.namespace and self.namespace[-1]:
 238       prefix = '::'.join(self.namespace) + '::'
 239     return prefix + self.name
 240
 241   def _TypeStringHelper(self, suffix):
 242     if self.namespace:
 243       names = [n or '<anonymous>' for n in self.namespace]
 244       suffix += ' in ' + '::'.join(names)
 245     return self._StringHelper(self.__class__.__name__, suffix)
 246
 247
 248 # TODO(nnorwitz): merge with Parameter in some way?
 249 class VariableDeclaration(_GenericDeclaration):
 250   def __init__(self, start, end, name, var_type, initial_value, namespace):
 251     _GenericDeclaration.__init__(self, start, end, name, namespace)
 252     self.type = var_type
 253     self.initial_value = initial_value
 254
 255   def Requires(self, node):
 256     # TODO(nnorwitz): handle namespaces, etc.
 257     return self.type.name == node.name
 258
 259   def ToString(self):
 260     """Return a string that tries to reconstitute the variable decl."""
 261     suffix = '%s %s' % (self.type, self.name)
 262     if self.initial_value:
 263       suffix += ' = ' + self.initial_value
 264     return suffix
 265
 266   def __str__(self):
 267     return self._StringHelper(self.__class__.__name__, self.ToString())
 268
 269
 270 class Typedef(_GenericDeclaration):
 271   def __init__(self, start, end, name, alias, namespace):
 272     _GenericDeclaration.__init__(self, start, end, name, namespace)
 273     self.alias = alias
 274
 275   def IsDefinition(self):
 276     return True
 277
 278   def IsExportable(self):
 279     return True
 280
 281   def Requires(self, node):
 282     # TODO(nnorwitz): handle namespaces, etc.
 283     name = node.name
 284     for token in self.alias:
 285       if token is not None and name == token.name:
 286         return True
 287     return False
 288
 289   def __str__(self):
 290     suffix = '%s, %s' % (self.name, self.alias)
 291     return self._TypeStringHelper(suffix)
 292
 293
 294 class _NestedType(_GenericDeclaration):
 295   def __init__(self, start, end, name, fields, namespace):
 296     _GenericDeclaration.__init__(self, start, end, name, namespace)
 297     self.fields = fields
 298
 299   def IsDefinition(self):
 300     return True
 301
 302   def IsExportable(self):
 303     return True
 304
 305   def __str__(self):
 306     suffix = '%s, {%s}' % (self.name, self.fields)
 307     return self._TypeStringHelper(suffix)
 308
 309
 310 class Union(_NestedType):
 311   pass
 312
 313
 314 class Enum(_NestedType):
 315   pass
 316
 317
 318 class Class(_GenericDeclaration):
 319   def __init__(self, start, end, name, bases, templated_types, body, namespace):
 320     _GenericDeclaration.__init__(self, start, end, name, namespace)
 321     self.bases = bases
 322     self.body = body
 323     self.templated_types = templated_types
 324
 325   def IsDeclaration(self):
 326     return self.bases is None and self.body is None
 327
 328   def IsDefinition(self):
 329     return not self.IsDeclaration()
 330
 331   def IsExportable(self):
 332     return not self.IsDeclaration()
 333
 334   def Requires(self, node):
 335     # TODO(nnorwitz): handle namespaces, etc.
 336     if self.bases:
 337       for token_list in self.bases:
 338         # TODO(nnorwitz): bases are tokens, do name comparision.
 339         for token in token_list:
 340           if token.name == node.name:
 341             return True
 342     # TODO(nnorwitz): search in body too.
 343     return False
 344
 345   def __str__(self):
 346     name = self.name
 347     if self.templated_types:
 348       name += '<%s>' % self.templated_types
 349     suffix = '%s, %s, %s' % (name, self.bases, self.body)
 350     return self._TypeStringHelper(suffix)
 351
 352
 353 class Struct(Class):
 354   pass
 355
 356
 357 class Function(_GenericDeclaration):
 358   def __init__(self, start, end, name, return_type, parameters,
 359                modifiers, templated_types, body, namespace):
 360     _GenericDeclaration.__init__(self, start, end, name, namespace)
 361     converter = TypeConverter(namespace)
 362     self.return_type = converter.CreateReturnType(return_type)
 363     self.parameters = converter.ToParameters(parameters)
 364     self.modifiers = modifiers
 365     self.body = body
 366     self.templated_types = templated_types
 367
 368   def IsDeclaration(self):
 369     return self.body is None
 370
 371   def IsDefinition(self):
 372     return self.body is not None
 373
 374   def IsExportable(self):
 375     if self.return_type and 'static' in self.return_type.modifiers:
 376       return False
 377     return None not in self.namespace
 378
 379   def Requires(self, node):
 380     if self.parameters:
 381       # TODO(nnorwitz): parameters are tokens, do name comparision.
 382       for p in self.parameters:
 383         if p.name == node.name:
 384           return True
 385     # TODO(nnorwitz): search in body too.
 386     return False
 387
 388   def __str__(self):
 389     # TODO(nnorwitz): add templated_types.
 390     suffix = ('%s %s(%s), 0x%02x, %s' %
 391               (self.return_type, self.name, self.parameters,
 392                self.modifiers, self.body))
 393     return self._TypeStringHelper(suffix)
 394
 395
 396 class Method(Function):
 397   def __init__(self, start, end, name, in_class, return_type, parameters,
 398                modifiers, templated_types, body, namespace):
 399     Function.__init__(self, start, end, name, return_type, parameters,
 400                       modifiers, templated_types, body, namespace)
 401     # TODO(nnorwitz): in_class could also be a namespace which can
 402     # mess up finding functions properly.
 403     self.in_class = in_class
 404
 405
 406 class Type(_GenericDeclaration):
 407   """Type used for any variable (eg class, primitive, struct, etc)."""
 408
 409   def __init__(self, start, end, name, templated_types, modifiers,
 410                reference, pointer, array):
 411     """
 412         Args:
 413           name: str name of main type
 414           templated_types: [Class (Type?)] template type info between <>
 415           modifiers: [str] type modifiers (keywords) eg, const, mutable, etc.
 416           reference, pointer, array: bools
 417         """
 418     _GenericDeclaration.__init__(self, start, end, name, [])
 419     self.templated_types = templated_types
 420     if not name and modifiers:
 421       self.name = modifiers.pop()
 422     self.modifiers = modifiers
 423     self.reference = reference
 424     self.pointer = pointer
 425     self.array = array
 426
 427   def __str__(self):
 428     prefix = ''
 429     if self.modifiers:
 430       prefix = ' '.join(self.modifiers) + ' '
 431     name = str(self.name)
 432     if self.templated_types:
 433       name += '<%s>' % self.templated_types
 434     suffix = prefix + name
 435     if self.reference:
 436       suffix += '&'
 437     if self.pointer:
 438       suffix += '*'
 439     if self.array:
 440       suffix += '[]'
 441     return self._TypeStringHelper(suffix)
 442
 443   # By definition, Is* are always False.  A Type can only exist in
 444   # some sort of variable declaration, parameter, or return value.
 445   def IsDeclaration(self):
 446     return False
 447
 448   def IsDefinition(self):
 449     return False
 450
 451   def IsExportable(self):
 452     return False
 453
 454
 455 class TypeConverter(object):
 456
 457   def __init__(self, namespace_stack):
 458     self.namespace_stack = namespace_stack
 459
 460   def _GetTemplateEnd(self, tokens, start):
 461     count = 1
 462     end = start
 463     while 1:
 464       token = tokens[end]
 465       end += 1
 466       if token.name == '<':
 467         count += 1
 468       elif token.name == '>':
 469         count -= 1
 470         if count == 0:
 471           break
 472     return tokens[start:end-1], end
 473
 474   def ToType(self, tokens):
 475     """Convert [Token,...] to [Class(...), ] useful for base classes.
 476         For example, code like class Foo : public Bar<x, y> { ... };
 477         the "Bar<x, y>" portion gets converted to an AST.
 478
 479         Returns:
 480           [Class(...), ...]
 481         """
 482     result = []
 483     name_tokens = []
 484     reference = pointer = array = False
 485
 486     def AddType(templated_types):
 487       # Partition tokens into name and modifier tokens.
 488       names = []
 489       modifiers = []
 490       for t in name_tokens:
 491         if keywords.IsKeyword(t.name):
 492           modifiers.append(t.name)
 493         else:
 494           names.append(t.name)
 495       name = ''.join(names)
 496       if name_tokens:
 497         result.append(Type(name_tokens[0].start, name_tokens[-1].end,
 498                            name, templated_types, modifiers,
 499                            reference, pointer, array))
 500       del name_tokens[:]
 501
 502     i = 0
 503     end = len(tokens)
 504     while i < end:
 505       token = tokens[i]
 506       if token.name == '<':
 507         new_tokens, new_end = self._GetTemplateEnd(tokens, i+1)
 508         AddType(self.ToType(new_tokens))
 509         # If there is a comma after the template, we need to consume
 510         # that here otherwise it becomes part of the name.
 511         i = new_end
 512         reference = pointer = array = False
 513       elif token.name == ',':
 514         AddType([])
 515         reference = pointer = array = False
 516       elif token.name == '*':
 517         pointer = True
 518       elif token.name == '&':
 519         reference = True
 520       elif token.name == '[':
 521         pointer = True
 522       elif token.name == ']':
 523         pass
 524       else:
 525         name_tokens.append(token)
 526       i += 1
 527
 528     if name_tokens:
 529       # No '<' in the tokens, just a simple name and no template.
 530       AddType([])
 531     return result
 532
 533   def DeclarationToParts(self, parts, needs_name_removed):
 534     name = None
 535     default = []
 536     if needs_name_removed:
 537       # Handle default (initial) values properly.
 538       for i, t in enumerate(parts):
 539         if t.name == '=':
 540           default = parts[i+1:]
 541           name = parts[i-1].name
 542           if name == ']' and parts[i-2].name == '[':
 543             name = parts[i-3].name
 544             i -= 1
 545           parts = parts[:i-1]
 546           break
 547       else:
 548         if parts[-1].token_type == tokenize.NAME:
 549           name = parts.pop().name
 550         else:
 551           # TODO(nnorwitz): this is a hack that happens for code like
 552           # Register(Foo<T>); where it thinks this is a function call
 553           # but it's actually a declaration.
 554           name = '???'
 555     modifiers = []
 556     type_name = []
 557     other_tokens = []
 558     templated_types = []
 559     i = 0
 560     end = len(parts)
 561     while i < end:
 562       p = parts[i]
 563       if keywords.IsKeyword(p.name):
 564         modifiers.append(p.name)
 565       elif p.name == '<':
 566         templated_tokens, new_end = self._GetTemplateEnd(parts, i+1)
 567         templated_types = self.ToType(templated_tokens)
 568         i = new_end - 1
 569         # Don't add a spurious :: to data members being initialized.
 570         next_index = i + 1
 571         if next_index < end and parts[next_index].name == '::':
 572           i += 1
 573       elif p.name in ('[', ']', '='):
 574         # These are handled elsewhere.
 575         other_tokens.append(p)
 576       elif p.name not in ('*', '&', '>'):
 577         # Ensure that names have a space between them.
 578         if (type_name and type_name[-1].token_type == tokenize.NAME and
 579                 p.token_type == tokenize.NAME):
 580           type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
 581         type_name.append(p)
 582       else:
 583         other_tokens.append(p)
 584       i += 1
 585     type_name = ''.join([t.name for t in type_name])
 586     return name, type_name, templated_types, modifiers, default, other_tokens
 587
 588   def ToParameters(self, tokens):
 589     if not tokens:
 590       return []
 591
 592     result = []
 593     name = type_name = ''
 594     type_modifiers = []
 595     pointer = reference = array = False
 596     first_token = None
 597     default = []
 598
 599     def AddParameter(end):
 600       if default:
 601         del default[0]  # Remove flag.
 602       parts = self.DeclarationToParts(type_modifiers, True)
 603       (name, type_name, templated_types, modifiers,
 604        unused_default, unused_other_tokens) = parts
 605       parameter_type = Type(first_token.start, first_token.end,
 606                             type_name, templated_types, modifiers,
 607                             reference, pointer, array)
 608       p = Parameter(first_token.start, end, name,
 609                     parameter_type, default)
 610       result.append(p)
 611
 612     template_count = 0
 613     brace_count = 0
 614     for s in tokens:
 615       if not first_token:
 616         first_token = s
 617
 618       # Check for braces before templates, as we can have unmatched '<>'
 619       # inside default arguments.
 620       if s.name == '{':
 621         brace_count += 1
 622       elif s.name == '}':
 623         brace_count -= 1
 624       if brace_count > 0:
 625         type_modifiers.append(s)
 626         continue
 627
 628       if s.name == '<':
 629         template_count += 1
 630       elif s.name == '>':
 631         template_count -= 1
 632       if template_count > 0:
 633         type_modifiers.append(s)
 634         continue
 635
 636       if s.name == ',':
 637         AddParameter(s.start)
 638         name = type_name = ''
 639         type_modifiers = []
 640         pointer = reference = array = False
 641         first_token = None
 642         default = []
 643       elif s.name == '*':
 644         pointer = True
 645       elif s.name == '&':
 646         reference = True
 647       elif s.name == '[':
 648         array = True
 649       elif s.name == ']':
 650         pass  # Just don't add to type_modifiers.
 651       elif s.name == '=':
 652         # Got a default value.  Add any value (None) as a flag.
 653         default.append(None)
 654       elif default:
 655         default.append(s)
 656       else:
 657         type_modifiers.append(s)
 658     AddParameter(tokens[-1].end)
 659     return result
 660
 661   def CreateReturnType(self, return_type_seq):
 662     if not return_type_seq:
 663       return None
 664     start = return_type_seq[0].start
 665     end = return_type_seq[-1].end
 666     _, name, templated_types, modifiers, default, other_tokens = \
 667         self.DeclarationToParts(return_type_seq, False)
 668     names = [n.name for n in other_tokens]
 669     reference = '&' in names
 670     pointer = '*' in names
 671     array = '[' in names
 672     return Type(start, end, name, templated_types, modifiers,
 673                 reference, pointer, array)
 674
 675   def GetTemplateIndices(self, names):
 676     # names is a list of strings.
 677     start = names.index('<')
 678     end = len(names) - 1
 679     while end > 0:
 680       if names[end] == '>':
 681         break
 682       end -= 1
 683     return start, end+1
 684
 685 class AstBuilder(object):
 686   def __init__(self, token_stream, filename, in_class='', visibility=None,
 687                namespace_stack=[]):
 688     self.tokens = token_stream
 689     self.filename = filename
 690     # TODO(nnorwitz): use a better data structure (deque) for the queue.
 691     # Switching directions of the "queue" improved perf by about 25%.
 692     # Using a deque should be even better since we access from both sides.
 693     self.token_queue = []
 694     self.namespace_stack = namespace_stack[:]
 695     self.in_class = in_class
 696     if in_class is None:
 697       self.in_class_name_only = None
 698     else:
 699       self.in_class_name_only = in_class.split('::')[-1]
 700     self.visibility = visibility
 701     self.in_function = False
 702     self.current_token = None
 703     # Keep the state whether we are currently handling a typedef or not.
 704     self._handling_typedef = False
 705
 706     self.converter = TypeConverter(self.namespace_stack)
 707
 708   def HandleError(self, msg, token):
 709     printable_queue = list(reversed(self.token_queue[-20:]))
 710     sys.stderr.write('Got %s in %s @ %s %s\n' %
 711                      (msg, self.filename, token, printable_queue))
 712
 713   def Generate(self):
 714     while 1:
 715       token = self._GetNextToken()
 716       if not token:
 717         break
 718
 719       # Get the next token.
 720       self.current_token = token
 721
 722       # Dispatch on the next token type.
 723       if token.token_type == _INTERNAL_TOKEN:
 724         if token.name == _NAMESPACE_POP:
 725           self.namespace_stack.pop()
 726         continue
 727
 728       try:
 729         result = self._GenerateOne(token)
 730         if result is not None:
 731           yield result
 732       except:
 733         self.HandleError('exception', token)
 734         raise
 735
 736   def _CreateVariable(self, pos_token, name, type_name, type_modifiers,
 737                       ref_pointer_name_seq, templated_types, value=None):
 738     reference = '&' in ref_pointer_name_seq
 739     pointer = '*' in ref_pointer_name_seq
 740     array = '[' in ref_pointer_name_seq
 741     var_type = Type(pos_token.start, pos_token.end, type_name,
 742                     templated_types, type_modifiers,
 743                     reference, pointer, array)
 744     return VariableDeclaration(pos_token.start, pos_token.end,
 745                                name, var_type, value, self.namespace_stack)
 746
 747   def _GenerateOne(self, token):
 748     if token.token_type == tokenize.NAME:
 749       if (keywords.IsKeyword(token.name) and
 750           not keywords.IsBuiltinType(token.name)):
 751         if token.name == 'enum':
 752           # Pop the next token and only put it back if it's not
 753           # 'class'.  This allows us to support the two-token
 754           # 'enum class' keyword as if it were simply 'enum'.
 755           next = self._GetNextToken()
 756           if next.name != 'class':
 757             self._AddBackToken(next)
 758
 759         method = getattr(self, 'handle_' + token.name)
 760         return method()
 761       elif token.name == self.in_class_name_only:
 762         # The token name is the same as the class, must be a ctor if
 763         # there is a paren.  Otherwise, it's the return type.
 764         # Peek ahead to get the next token to figure out which.
 765         next = self._GetNextToken()
 766         self._AddBackToken(next)
 767         if next.token_type == tokenize.SYNTAX and next.name == '(':
 768           return self._GetMethod([token], FUNCTION_CTOR, None, True)
 769         # Fall through--handle like any other method.
 770
 771       # Handle data or function declaration/definition.
 772       syntax = tokenize.SYNTAX
 773       temp_tokens, last_token = \
 774           self._GetVarTokensUpToIgnoringTemplates(syntax,
 775                                                   '(', ';', '{', '[')
 776       temp_tokens.insert(0, token)
 777       if last_token.name == '(':
 778         # If there is an assignment before the paren,
 779         # this is an expression, not a method.
 780         expr = bool([e for e in temp_tokens if e.name == '='])
 781         if expr:
 782           new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
 783           temp_tokens.append(last_token)
 784           temp_tokens.extend(new_temp)
 785           last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)
 786
 787       if last_token.name == '[':
 788         # Handle array, this isn't a method, unless it's an operator.
 789         # TODO(nnorwitz): keep the size somewhere.
 790         # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']')
 791         temp_tokens.append(last_token)
 792         if temp_tokens[-2].name == 'operator':
 793           temp_tokens.append(self._GetNextToken())
 794         else:
 795           temp_tokens2, last_token = \
 796               self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
 797           temp_tokens.extend(temp_tokens2)
 798
 799       if last_token.name == ';':
 800         # Handle data, this isn't a method.
 801         parts = self.converter.DeclarationToParts(temp_tokens, True)
 802         (name, type_name, templated_types, modifiers, default,
 803          unused_other_tokens) = parts
 804
 805         t0 = temp_tokens[0]
 806         names = [t.name for t in temp_tokens]
 807         if templated_types:
 808           start, end = self.converter.GetTemplateIndices(names)
 809           names = names[:start] + names[end:]
 810         default = ''.join([t.name for t in default])
 811         return self._CreateVariable(t0, name, type_name, modifiers,
 812                                     names, templated_types, default)
 813       if last_token.name == '{':
 814         self._AddBackTokens(temp_tokens[1:])
 815         self._AddBackToken(last_token)
 816         method_name = temp_tokens[0].name
 817         method = getattr(self, 'handle_' + method_name, None)
 818         if not method:
 819           # Must be declaring a variable.
 820           # TODO(nnorwitz): handle the declaration.
 821           return None
 822         return method()
 823       return self._GetMethod(temp_tokens, 0, None, False)
 824     elif token.token_type == tokenize.SYNTAX:
 825       if token.name == '~' and self.in_class:
 826         # Must be a dtor (probably not in method body).
 827         token = self._GetNextToken()
 828         # self.in_class can contain A::Name, but the dtor will only
 829         # be Name.  Make sure to compare against the right value.
 830         if (token.token_type == tokenize.NAME and
 831                 token.name == self.in_class_name_only):
 832           return self._GetMethod([token], FUNCTION_DTOR, None, True)
 833       # TODO(nnorwitz): handle a lot more syntax.
 834     elif token.token_type == tokenize.PREPROCESSOR:
 835       # TODO(nnorwitz): handle more preprocessor directives.
 836       # token starts with a #, so remove it and strip whitespace.
 837       name = token.name[1:].lstrip()
 838       if name.startswith('include'):
 839         # Remove "include".
 840         name = name[7:].strip()
 841         assert name
 842         # Handle #include \<newline> "header-on-second-line.h".
 843         if name.startswith('\\'):
 844           name = name[1:].strip()
 845         assert name[0] in '<"', token
 846         assert name[-1] in '>"', token
 847         system = name[0] == '<'
 848         filename = name[1:-1]
 849         return Include(token.start, token.end, filename, system)
 850       if name.startswith('define'):
 851         # Remove "define".
 852         name = name[6:].strip()
 853         assert name
 854         value = ''
 855         for i, c in enumerate(name):
 856           if c.isspace():
 857             value = name[i:].lstrip()
 858             name = name[:i]
 859             break
 860         return Define(token.start, token.end, name, value)
 861       if name.startswith('if') and name[2:3].isspace():
 862         condition = name[3:].strip()
 863         if condition.startswith('0') or condition.startswith('(0)'):
 864           self._SkipIf0Blocks()
 865     return None
 866
 867   def _GetTokensUpTo(self, expected_token_type, expected_token):
 868     return self._GetVarTokensUpTo(expected_token_type, expected_token)[0]
 869
 870   def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens):
 871     last_token = self._GetNextToken()
 872     tokens = []
 873     while (last_token.token_type != expected_token_type or
 874            last_token.name not in expected_tokens):
 875       tokens.append(last_token)
 876       last_token = self._GetNextToken()
 877     return tokens, last_token
 878
 879   # Same as _GetVarTokensUpTo, but skips over '<...>' which could contain an
 880   # expected token.
 881   def _GetVarTokensUpToIgnoringTemplates(self, expected_token_type,
 882                                          *expected_tokens):
 883     last_token = self._GetNextToken()
 884     tokens = []
 885     nesting = 0
 886     while (nesting > 0 or
 887            last_token.token_type != expected_token_type or
 888            last_token.name not in expected_tokens):
 889       tokens.append(last_token)
 890       last_token = self._GetNextToken()
 891       if last_token.name == '<':
 892         nesting += 1
 893       elif last_token.name == '>':
 894         nesting -= 1
 895     return tokens, last_token
 896
 897   # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necesary.
 898   def _IgnoreUpTo(self, token_type, token):
 899     unused_tokens = self._GetTokensUpTo(token_type, token)
 900
 901   def _SkipIf0Blocks(self):
 902     count = 1
 903     while 1:
 904       token = self._GetNextToken()
 905       if token.token_type != tokenize.PREPROCESSOR:
 906         continue
 907
 908       name = token.name[1:].lstrip()
 909       if name.startswith('endif'):
 910         count -= 1
 911         if count == 0:
 912           break
 913       elif name.startswith('if'):
 914         count += 1
 915
 916   def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None):
 917     if GetNextToken is None:
 918       GetNextToken = self._GetNextToken
 919     # Assumes the current token is open_paren and we will consume
 920     # and return up to the close_paren.
 921     count = 1
 922     token = GetNextToken()
 923     while 1:
 924       if token.token_type == tokenize.SYNTAX:
 925         if token.name == open_paren:
 926           count += 1
 927         elif token.name == close_paren:
 928           count -= 1
 929           if count == 0:
 930             break
 931       yield token
 932       token = GetNextToken()
 933     yield token
 934
 935   def _GetParameters(self):
 936     return self._GetMatchingChar('(', ')')
 937
 938   def GetScope(self):
 939     return self._GetMatchingChar('{', '}')
 940
 941   def _GetNextToken(self):
 942     if self.token_queue:
 943       return self.token_queue.pop()
 944     try:
 945       return next(self.tokens)
 946     except StopIteration:
 947       return
 948
 949   def _AddBackToken(self, token):
 950     if token.whence == tokenize.WHENCE_STREAM:
 951       token.whence = tokenize.WHENCE_QUEUE
 952       self.token_queue.insert(0, token)
 953     else:
 954       assert token.whence == tokenize.WHENCE_QUEUE, token
 955       self.token_queue.append(token)
 956
 957   def _AddBackTokens(self, tokens):
 958     if tokens:
 959       if tokens[-1].whence == tokenize.WHENCE_STREAM:
 960         for token in tokens:
 961           token.whence = tokenize.WHENCE_QUEUE
 962         self.token_queue[:0] = reversed(tokens)
 963       else:
 964         assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
 965         self.token_queue.extend(reversed(tokens))
 966
 967   def GetName(self, seq=None):
 968     """Returns ([tokens], next_token_info)."""
 969     GetNextToken = self._GetNextToken
 970     if seq is not None:
 971       it = iter(seq)
 972       GetNextToken = lambda: next(it)
 973     next_token = GetNextToken()
 974     tokens = []
 975     last_token_was_name = False
 976     while (next_token.token_type == tokenize.NAME or
 977            (next_token.token_type == tokenize.SYNTAX and
 978             next_token.name in ('::', '<'))):
 979       # Two NAMEs in a row means the identifier should terminate.
 980       # It's probably some sort of variable declaration.
 981       if last_token_was_name and next_token.token_type == tokenize.NAME:
 982         break
 983       last_token_was_name = next_token.token_type == tokenize.NAME
 984       tokens.append(next_token)
 985       # Handle templated names.
 986       if next_token.name == '<':
 987         tokens.extend(self._GetMatchingChar('<', '>', GetNextToken))
 988         last_token_was_name = True
 989       next_token = GetNextToken()
 990     return tokens, next_token
 991
 992   def GetMethod(self, modifiers, templated_types):
 993     return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
 994     assert len(return_type_and_name) >= 1
 995     return self._GetMethod(return_type_and_name, modifiers, templated_types,
 996                            False)
 997
 998   def _GetMethod(self, return_type_and_name, modifiers, templated_types,
 999                  get_paren):
1000     template_portion = None
1001     if get_paren:
1002       token = self._GetNextToken()
1003       assert token.token_type == tokenize.SYNTAX, token
1004       if token.name == '<':
1005         # Handle templatized dtors.
1006         template_portion = [token]
1007         template_portion.extend(self._GetMatchingChar('<', '>'))
1008         token = self._GetNextToken()
1009       assert token.token_type == tokenize.SYNTAX, token
1010       assert token.name == '(', token
1011
1012     name = return_type_and_name.pop()
1013     # Handle templatized ctors.
1014     if name.name == '>':
1015       index = 1
1016       while return_type_and_name[index].name != '<':
1017         index += 1
1018       template_portion = return_type_and_name[index:] + [name]
1019       del return_type_and_name[index:]
1020       name = return_type_and_name.pop()
1021     elif name.name == ']':
1022       rt = return_type_and_name
1023       assert rt[-1].name == '[', return_type_and_name
1024       assert rt[-2].name == 'operator', return_type_and_name
1025       name_seq = return_type_and_name[-2:]
1026       del return_type_and_name[-2:]
1027       name = tokenize.Token(tokenize.NAME, 'operator[]',
1028                             name_seq[0].start, name.end)
1029       # Get the open paren so _GetParameters() below works.
1030       unused_open_paren = self._GetNextToken()
1031
1032     # TODO(nnorwitz): store template_portion.
1033     return_type = return_type_and_name
1034     indices = name
1035     if return_type:
1036       indices = return_type[0]
1037
1038     # Force ctor for templatized ctors.
1039     if name.name == self.in_class and not modifiers:
1040       modifiers |= FUNCTION_CTOR
1041     parameters = list(self._GetParameters())
1042     del parameters[-1]              # Remove trailing ')'.
1043
1044     # Handling operator() is especially weird.
1045     if name.name == 'operator' and not parameters:
1046       token = self._GetNextToken()
1047       assert token.name == '(', token
1048       parameters = list(self._GetParameters())
1049       del parameters[-1]          # Remove trailing ')'.
1050
1051     token = self._GetNextToken()
1052     while token.token_type == tokenize.NAME:
1053       modifier_token = token
1054       token = self._GetNextToken()
1055       if modifier_token.name == 'const':
1056         modifiers |= FUNCTION_CONST
1057       elif modifier_token.name == '__attribute__':
1058         # TODO(nnorwitz): handle more __attribute__ details.
1059         modifiers |= FUNCTION_ATTRIBUTE
1060         assert token.name == '(', token
1061         # Consume everything between the (parens).
1062         unused_tokens = list(self._GetMatchingChar('(', ')'))
1063         token = self._GetNextToken()
1064       elif modifier_token.name == 'throw':
1065         modifiers |= FUNCTION_THROW
1066         assert token.name == '(', token
1067         # Consume everything between the (parens).
1068         unused_tokens = list(self._GetMatchingChar('(', ')'))
1069         token = self._GetNextToken()
1070       elif modifier_token.name == 'override':
1071         modifiers |= FUNCTION_OVERRIDE
1072       elif modifier_token.name == modifier_token.name.upper():
1073         # HACK(nnorwitz):  assume that all upper-case names
1074         # are some macro we aren't expanding.
1075         modifiers |= FUNCTION_UNKNOWN_ANNOTATION
1076       else:
1077         self.HandleError('unexpected token', modifier_token)
1078
1079     assert token.token_type == tokenize.SYNTAX, token
1080     # Handle ctor initializers.
1081     if token.name == ':':
1082       # TODO(nnorwitz): anything else to handle for initializer list?
1083       while token.name != ';' and token.name != '{':
1084         token = self._GetNextToken()
1085
1086     # Handle pointer to functions that are really data but look
1087     # like method declarations.
1088     if token.name == '(':
1089       if parameters[0].name == '*':
1090         # name contains the return type.
1091         name = parameters.pop()
1092         # parameters contains the name of the data.
1093         modifiers = [p.name for p in parameters]
1094         # Already at the ( to open the parameter list.
1095         function_parameters = list(self._GetMatchingChar('(', ')'))
1096         del function_parameters[-1]  # Remove trailing ')'.
1097         # TODO(nnorwitz): store the function_parameters.
1098         token = self._GetNextToken()
1099         assert token.token_type == tokenize.SYNTAX, token
1100         assert token.name == ';', token
1101         return self._CreateVariable(indices, name.name, indices.name,
1102                                     modifiers, '', None)
1103       # At this point, we got something like:
1104       #  return_type (type::*name_)(params);
1105       # This is a data member called name_ that is a function pointer.
1106       # With this code: void (sq_type::*field_)(string&);
1107       # We get: name=void return_type=[] parameters=sq_type ... field_
1108       # TODO(nnorwitz): is return_type always empty?
1109       # TODO(nnorwitz): this isn't even close to being correct.
1110       # Just put in something so we don't crash and can move on.
1111       real_name = parameters[-1]
1112       modifiers = [p.name for p in self._GetParameters()]
1113       del modifiers[-1]           # Remove trailing ')'.
1114       return self._CreateVariable(indices, real_name.name, indices.name,
1115                                   modifiers, '', None)
1116
1117     if token.name == '{':
1118       body = list(self.GetScope())
1119       del body[-1]                # Remove trailing '}'.
1120     else:
1121       body = None
1122       if token.name == '=':
1123         token = self._GetNextToken()
1124
1125         if token.name == 'default' or token.name == 'delete':
1126           # Ignore explicitly defaulted and deleted special members
1127           # in C++11.
1128           token = self._GetNextToken()
1129         else:
1130           # Handle pure-virtual declarations.
1131           assert token.token_type == tokenize.CONSTANT, token
1132           assert token.name == '0', token
1133           modifiers |= FUNCTION_PURE_VIRTUAL
1134           token = self._GetNextToken()
1135
1136       if token.name == '[':
1137         # TODO(nnorwitz): store tokens and improve parsing.
1138         # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N];
1139         tokens = list(self._GetMatchingChar('[', ']'))
1140         token = self._GetNextToken()
1141
1142       assert token.name == ';', (token, return_type_and_name, parameters)
1143
1144     # Looks like we got a method, not a function.
1145     if len(return_type) > 2 and return_type[-1].name == '::':
1146       return_type, in_class = \
1147           self._GetReturnTypeAndClassName(return_type)
1148       return Method(indices.start, indices.end, name.name, in_class,
1149                     return_type, parameters, modifiers, templated_types,
1150                     body, self.namespace_stack)
1151     return Function(indices.start, indices.end, name.name, return_type,
1152                     parameters, modifiers, templated_types, body,
1153                     self.namespace_stack)
1154
1155   def _GetReturnTypeAndClassName(self, token_seq):
1156     # Splitting the return type from the class name in a method
1157     # can be tricky.  For example, Return::Type::Is::Hard::To::Find().
1158     # Where is the return type and where is the class name?
1159     # The heuristic used is to pull the last name as the class name.
1160     # This includes all the templated type info.
1161     # TODO(nnorwitz): if there is only One name like in the
1162     # example above, punt and assume the last bit is the class name.
1163
1164     # Ignore a :: prefix, if exists so we can find the first real name.
1165     i = 0
1166     if token_seq[0].name == '::':
1167       i = 1
1168     # Ignore a :: suffix, if exists.
1169     end = len(token_seq) - 1
1170     if token_seq[end-1].name == '::':
1171       end -= 1
1172
1173     # Make a copy of the sequence so we can append a sentinel
1174     # value. This is required for GetName will has to have some
1175     # terminating condition beyond the last name.
1176     seq_copy = token_seq[i:end]
1177     seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
1178     names = []
1179     while i < end:
1180       # Iterate through the sequence parsing out each name.
1181       new_name, next = self.GetName(seq_copy[i:])
1182       assert new_name, 'Got empty new_name, next=%s' % next
1183       # We got a pointer or ref.  Add it to the name.
1184       if next and next.token_type == tokenize.SYNTAX:
1185         new_name.append(next)
1186       names.append(new_name)
1187       i += len(new_name)
1188
1189     # Now that we have the names, it's time to undo what we did.
1190
1191     # Remove the sentinel value.
1192     names[-1].pop()
1193     # Flatten the token sequence for the return type.
1194     return_type = [e for seq in names[:-1] for e in seq]
1195     # The class name is the last name.
1196     class_name = names[-1]
1197     return return_type, class_name
1198
1199   def handle_bool(self):
1200     pass
1201
1202   def handle_char(self):
1203     pass
1204
1205   def handle_int(self):
1206     pass
1207
1208   def handle_long(self):
1209     pass
1210
1211   def handle_short(self):
1212     pass
1213
1214   def handle_double(self):
1215     pass
1216
1217   def handle_float(self):
1218     pass
1219
1220   def handle_void(self):
1221     pass
1222
1223   def handle_wchar_t(self):
1224     pass
1225
1226   def handle_unsigned(self):
1227     pass
1228
1229   def handle_signed(self):
1230     pass
1231
1232   def _GetNestedType(self, ctor):
1233     name = None
1234     name_tokens, token = self.GetName()
1235     if name_tokens:
1236       name = ''.join([t.name for t in name_tokens])
1237
1238     # Handle forward declarations.
1239     if token.token_type == tokenize.SYNTAX and token.name == ';':
1240       return ctor(token.start, token.end, name, None,
1241                   self.namespace_stack)
1242
1243     if token.token_type == tokenize.NAME and self._handling_typedef:
1244       self._AddBackToken(token)
1245       return ctor(token.start, token.end, name, None,
1246                   self.namespace_stack)
1247
1248     # Must be the type declaration.
1249     fields = list(self._GetMatchingChar('{', '}'))
1250     del fields[-1]                  # Remove trailing '}'.
1251     if token.token_type == tokenize.SYNTAX and token.name == '{':
1252       next = self._GetNextToken()
1253       new_type = ctor(token.start, token.end, name, fields,
1254                       self.namespace_stack)
1255       # A name means this is an anonymous type and the name
1256       # is the variable declaration.
1257       if next.token_type != tokenize.NAME:
1258         return new_type
1259       name = new_type
1260       token = next
1261
1262     # Must be variable declaration using the type prefixed with keyword.
1263     assert token.token_type == tokenize.NAME, token
1264     return self._CreateVariable(token, token.name, name, [], '', None)
1265
1266   def handle_struct(self):
1267     # Special case the handling typedef/aliasing of structs here.
1268     # It would be a pain to handle in the class code.
1269     name_tokens, var_token = self.GetName()
1270     if name_tokens:
1271       next_token = self._GetNextToken()
1272       is_syntax = (var_token.token_type == tokenize.SYNTAX and
1273                    var_token.name[0] in '*&')
1274       is_variable = (var_token.token_type == tokenize.NAME and
1275                      next_token.name == ';')
1276       variable = var_token
1277       if is_syntax and not is_variable:
1278         variable = next_token
1279         temp = self._GetNextToken()
1280         if temp.token_type == tokenize.SYNTAX and temp.name == '(':
1281           # Handle methods declared to return a struct.
1282           t0 = name_tokens[0]
1283           struct = tokenize.Token(tokenize.NAME, 'struct',
1284                                   t0.start-7, t0.start-2)
1285           type_and_name = [struct]
1286           type_and_name.extend(name_tokens)
1287           type_and_name.extend((var_token, next_token))
1288           return self._GetMethod(type_and_name, 0, None, False)
1289         assert temp.name == ';', (temp, name_tokens, var_token)
1290       if is_syntax or (is_variable and not self._handling_typedef):
1291         modifiers = ['struct']
1292         type_name = ''.join([t.name for t in name_tokens])
1293         position = name_tokens[0]
1294         return self._CreateVariable(position, variable.name, type_name,
1295                                     modifiers, var_token.name, None)
1296       name_tokens.extend((var_token, next_token))
1297       self._AddBackTokens(name_tokens)
1298     else:
1299       self._AddBackToken(var_token)
1300     return self._GetClass(Struct, VISIBILITY_PUBLIC, None)
1301
1302   def handle_union(self):
1303     return self._GetNestedType(Union)
1304
1305   def handle_enum(self):
1306     return self._GetNestedType(Enum)
1307
1308   def handle_auto(self):
1309     # TODO(nnorwitz): warn about using auto?  Probably not since it
1310     # will be reclaimed and useful for C++0x.
1311     pass
1312
1313   def handle_register(self):
1314     pass
1315
1316   def handle_const(self):
1317     pass
1318
1319   def handle_inline(self):
1320     pass
1321
1322   def handle_extern(self):
1323     pass
1324
1325   def handle_static(self):
1326     pass
1327
1328   def handle_virtual(self):
1329     # What follows must be a method.
1330     token = token2 = self._GetNextToken()
1331     if token.name == 'inline':
1332       # HACK(nnorwitz): handle inline dtors by ignoring 'inline'.
1333       token2 = self._GetNextToken()
1334     if token2.token_type == tokenize.SYNTAX and token2.name == '~':
1335       return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None)
1336     assert token.token_type == tokenize.NAME or token.name == '::', token
1337     return_type_and_name, _ = self._GetVarTokensUpToIgnoringTemplates(
1338         tokenize.SYNTAX, '(')  # )
1339     return_type_and_name.insert(0, token)
1340     if token2 is not token:
1341       return_type_and_name.insert(1, token2)
1342     return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL,
1343                            None, False)
1344
1345   def handle_volatile(self):
1346     pass
1347
1348   def handle_mutable(self):
1349     pass
1350
1351   def handle_public(self):
1352     assert self.in_class
1353     self.visibility = VISIBILITY_PUBLIC
1354
1355   def handle_protected(self):
1356     assert self.in_class
1357     self.visibility = VISIBILITY_PROTECTED
1358
1359   def handle_private(self):
1360     assert self.in_class
1361     self.visibility = VISIBILITY_PRIVATE
1362
1363   def handle_friend(self):
1364     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1365     assert tokens
1366     t0 = tokens[0]
1367     return Friend(t0.start, t0.end, tokens, self.namespace_stack)
1368
1369   def handle_static_cast(self):
1370     pass
1371
1372   def handle_const_cast(self):
1373     pass
1374
1375   def handle_dynamic_cast(self):
1376     pass
1377
1378   def handle_reinterpret_cast(self):
1379     pass
1380
1381   def handle_new(self):
1382     pass
1383
1384   def handle_delete(self):
1385     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1386     assert tokens
1387     return Delete(tokens[0].start, tokens[0].end, tokens)
1388
1389   def handle_typedef(self):
1390     token = self._GetNextToken()
1391     if (token.token_type == tokenize.NAME and
1392             keywords.IsKeyword(token.name)):
1393       # Token must be struct/enum/union/class.
1394       method = getattr(self, 'handle_' + token.name)
1395       self._handling_typedef = True
1396       tokens = [method()]
1397       self._handling_typedef = False
1398     else:
1399       tokens = [token]
1400
1401     # Get the remainder of the typedef up to the semi-colon.
1402     tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))
1403
1404     # TODO(nnorwitz): clean all this up.
1405     assert tokens
1406     name = tokens.pop()
1407     indices = name
1408     if tokens:
1409       indices = tokens[0]
1410     if not indices:
1411       indices = token
1412     if name.name == ')':
1413       # HACK(nnorwitz): Handle pointers to functions "properly".
1414       if (len(tokens) >= 4 and
1415               tokens[1].name == '(' and tokens[2].name == '*'):
1416         tokens.append(name)
1417         name = tokens[3]
1418     elif name.name == ']':
1419       # HACK(nnorwitz): Handle arrays properly.
1420       if len(tokens) >= 2:
1421         tokens.append(name)
1422         name = tokens[1]
1423     new_type = tokens
1424     if tokens and isinstance(tokens[0], tokenize.Token):
1425       new_type = self.converter.ToType(tokens)[0]
1426     return Typedef(indices.start, indices.end, name.name,
1427                    new_type, self.namespace_stack)
1428
1429   def handle_typeid(self):
1430     pass  # Not needed yet.
1431
1432   def handle_typename(self):
1433     pass  # Not needed yet.
1434
1435   def _GetTemplatedTypes(self):
1436     result = {}
1437     tokens = list(self._GetMatchingChar('<', '>'))
1438     len_tokens = len(tokens) - 1    # Ignore trailing '>'.
1439     i = 0
1440     while i < len_tokens:
1441       key = tokens[i].name
1442       i += 1
1443       if keywords.IsKeyword(key) or key == ',':
1444         continue
1445       type_name = default = None
1446       if i < len_tokens:
1447         i += 1
1448         if tokens[i-1].name == '=':
1449           assert i < len_tokens, '%s %s' % (i, tokens)
1450           default, unused_next_token = self.GetName(tokens[i:])
1451           i += len(default)
1452         else:
1453           if tokens[i-1].name != ',':
1454             # We got something like: Type variable.
1455             # Re-adjust the key (variable) and type_name (Type).
1456             key = tokens[i-1].name
1457             type_name = tokens[i-2]
1458
1459       result[key] = (type_name, default)
1460     return result
1461
1462   def handle_template(self):
1463     token = self._GetNextToken()
1464     assert token.token_type == tokenize.SYNTAX, token
1465     assert token.name == '<', token
1466     templated_types = self._GetTemplatedTypes()
1467     # TODO(nnorwitz): for now, just ignore the template params.
1468     token = self._GetNextToken()
1469     if token.token_type == tokenize.NAME:
1470       if token.name == 'class':
1471         return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types)
1472       elif token.name == 'struct':
1473         return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types)
1474       elif token.name == 'friend':
1475         return self.handle_friend()
1476     self._AddBackToken(token)
1477     tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
1478     tokens.append(last)
1479     self._AddBackTokens(tokens)
1480     if last.name == '(':
1481       return self.GetMethod(FUNCTION_NONE, templated_types)
1482     # Must be a variable definition.
1483     return None
1484
1485   def handle_true(self):
1486     pass  # Nothing to do.
1487
1488   def handle_false(self):
1489     pass  # Nothing to do.
1490
1491   def handle_asm(self):
1492     pass  # Not needed yet.
1493
1494   def handle_class(self):
1495     return self._GetClass(Class, VISIBILITY_PRIVATE, None)
1496
1497   def _GetBases(self):
1498     # Get base classes.
1499     bases = []
1500     while 1:
1501       token = self._GetNextToken()
1502       assert token.token_type == tokenize.NAME, token
1503       # TODO(nnorwitz): store kind of inheritance...maybe.
1504       if token.name not in ('public', 'protected', 'private'):
1505         # If inheritance type is not specified, it is private.
1506         # Just put the token back so we can form a name.
1507         # TODO(nnorwitz): it would be good to warn about this.
1508         self._AddBackToken(token)
1509       else:
1510         # Check for virtual inheritance.
1511         token = self._GetNextToken()
1512         if token.name != 'virtual':
1513           self._AddBackToken(token)
1514         else:
1515           # TODO(nnorwitz): store that we got virtual for this base.
1516           pass
1517       base, next_token = self.GetName()
1518       bases_ast = self.converter.ToType(base)
1519       assert len(bases_ast) == 1, bases_ast
1520       bases.append(bases_ast[0])
1521       assert next_token.token_type == tokenize.SYNTAX, next_token
1522       if next_token.name == '{':
1523         token = next_token
1524         break
1525       # Support multiple inheritance.
1526       assert next_token.name == ',', next_token
1527     return bases, token
1528
1529   def _GetClass(self, class_type, visibility, templated_types):
1530     class_name = None
1531     class_token = self._GetNextToken()
1532     if class_token.token_type != tokenize.NAME:
1533       assert class_token.token_type == tokenize.SYNTAX, class_token
1534       token = class_token
1535     else:
1536       # Skip any macro (e.g. storage class specifiers) after the
1537       # 'class' keyword.
1538       next_token = self._GetNextToken()
1539       if next_token.token_type == tokenize.NAME:
1540         self._AddBackToken(next_token)
1541       else:
1542         self._AddBackTokens([class_token, next_token])
1543       name_tokens, token = self.GetName()
1544       class_name = ''.join([t.name for t in name_tokens])
1545     bases = None
1546     if token.token_type == tokenize.SYNTAX:
1547       if token.name == ';':
1548         # Forward declaration.
1549         return class_type(class_token.start, class_token.end,
1550                           class_name, None, templated_types, None,
1551                           self.namespace_stack)
1552       if token.name in '*&':
1553         # Inline forward declaration.  Could be method or data.
1554         name_token = self._GetNextToken()
1555         next_token = self._GetNextToken()
1556         if next_token.name == ';':
1557           # Handle data
1558           modifiers = ['class']
1559           return self._CreateVariable(class_token, name_token.name,
1560                                       class_name,
1561                                       modifiers, token.name, None)
1562         else:
1563           # Assume this is a method.
1564           tokens = (class_token, token, name_token, next_token)
1565           self._AddBackTokens(tokens)
1566           return self.GetMethod(FUNCTION_NONE, None)
1567       if token.name == ':':
1568         bases, token = self._GetBases()
1569
1570     body = None
1571     if token.token_type == tokenize.SYNTAX and token.name == '{':
1572       assert token.token_type == tokenize.SYNTAX, token
1573       assert token.name == '{', token
1574
1575       ast = AstBuilder(self.GetScope(), self.filename, class_name,
1576                        visibility, self.namespace_stack)
1577       body = list(ast.Generate())
1578
1579       if not self._handling_typedef:
1580         token = self._GetNextToken()
1581         if token.token_type != tokenize.NAME:
1582           assert token.token_type == tokenize.SYNTAX, token
1583           assert token.name == ';', token
1584         else:
1585           new_class = class_type(class_token.start, class_token.end,
1586                                  class_name, bases, None,
1587                                  body, self.namespace_stack)
1588
1589           modifiers = []
1590           return self._CreateVariable(class_token,
1591                                       token.name, new_class,
1592                                       modifiers, token.name, None)
1593     else:
1594       if not self._handling_typedef:
1595         self.HandleError('non-typedef token', token)
1596       self._AddBackToken(token)
1597
1598     return class_type(class_token.start, class_token.end, class_name,
1599                       bases, templated_types, body, self.namespace_stack)
1600
1601   def handle_namespace(self):
1602     # Support anonymous namespaces.
1603     name = None
1604     name_tokens, token = self.GetName()
1605     if name_tokens:
1606       name = ''.join([t.name for t in name_tokens])
1607     self.namespace_stack.append(name)
1608     assert token.token_type == tokenize.SYNTAX, token
1609     # Create an internal token that denotes when the namespace is complete.
1610     internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
1611                                     None, None)
1612     internal_token.whence = token.whence
1613     if token.name == '=':
1614       # TODO(nnorwitz): handle aliasing namespaces.
1615       name, next_token = self.GetName()
1616       assert next_token.name == ';', next_token
1617       self._AddBackToken(internal_token)
1618     else:
1619       assert token.name == '{', token
1620       tokens = list(self.GetScope())
1621       # Replace the trailing } with the internal namespace pop token.
1622       tokens[-1] = internal_token
1623       # Handle namespace with nothing in it.
1624       self._AddBackTokens(tokens)
1625     return None
1626
1627   def handle_using(self):
1628     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1629     assert tokens
1630     return Using(tokens[0].start, tokens[0].end, tokens)
1631
1632   def handle_explicit(self):
1633     assert self.in_class
1634     # Nothing much to do.
1635     # TODO(nnorwitz): maybe verify the method name == class name.
1636     # This must be a ctor.
1637     return self.GetMethod(FUNCTION_CTOR, None)
1638
1639   def handle_this(self):
1640     pass  # Nothing to do.
1641
1642   def handle_operator(self):
1643     # Pull off the next token(s?) and make that part of the method name.
1644     pass
1645
1646   def handle_sizeof(self):
1647     pass
1648
1649   def handle_case(self):
1650     pass
1651
1652   def handle_switch(self):
1653     pass
1654
1655   def handle_default(self):
1656     token = self._GetNextToken()
1657     assert token.token_type == tokenize.SYNTAX
1658     assert token.name == ':'
1659
1660   def handle_if(self):
1661     pass
1662
1663   def handle_else(self):
1664     pass
1665
1666   def handle_return(self):
1667     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1668     if not tokens:
1669       return Return(self.current_token.start, self.current_token.end, None)
1670     return Return(tokens[0].start, tokens[0].end, tokens)
1671
1672   def handle_goto(self):
1673     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1674     assert len(tokens) == 1, str(tokens)
1675     return Goto(tokens[0].start, tokens[0].end, tokens[0].name)
1676
1677   def handle_try(self):
1678     pass  # Not needed yet.
1679
1680   def handle_catch(self):
1681     pass  # Not needed yet.
1682
1683   def handle_throw(self):
1684     pass  # Not needed yet.
1685
1686   def handle_while(self):
1687     pass
1688
1689   def handle_do(self):
1690     pass
1691
1692   def handle_for(self):
1693     pass
1694
1695   def handle_break(self):
1696     self._IgnoreUpTo(tokenize.SYNTAX, ';')
1697
1698   def handle_continue(self):
1699     self._IgnoreUpTo(tokenize.SYNTAX, ';')
1700
1701
1702 def BuilderFromSource(source, filename):
1703   """Utility method that returns an AstBuilder from source code.
1704
1705     Args:
1706       source: 'C++ source code'
1707       filename: 'file1'
1708
1709     Returns:
1710       AstBuilder
1711     """
1712   return AstBuilder(tokenize.GetTokens(source), filename)
1713
1714
1715 def PrintIndentifiers(filename, should_print):
1716   """Prints all identifiers for a C++ source file.
1717
1718     Args:
1719       filename: 'file1'
1720       should_print: predicate with signature: bool Function(token)
1721     """
1722   source = utils.ReadFile(filename, False)
1723   if source is None:
1724     sys.stderr.write('Unable to find: %s\n' % filename)
1725     return
1726
1727   #print('Processing %s' % actual_filename)
1728   builder = BuilderFromSource(source, filename)
1729   try:
1730     for node in builder.Generate():
1731       if should_print(node):
1732         print(node.name)
1733   except KeyboardInterrupt:
1734     return
1735   except:
1736     pass
1737
1738
1739 def PrintAllIndentifiers(filenames, should_print):
1740   """Prints all identifiers for each C++ source file in filenames.
1741
1742     Args:
1743       filenames: ['file1', 'file2', ...]
1744       should_print: predicate with signature: bool Function(token)
1745     """
1746   for path in filenames:
1747     PrintIndentifiers(path, should_print)
1748
1749
1750 def main(argv):
1751   for filename in argv[1:]:
1752     source = utils.ReadFile(filename)
1753     if source is None:
1754       continue
1755
1756     print('Processing %s' % filename)
1757     builder = BuilderFromSource(source, filename)
1758     try:
1759       entire_ast = filter(None, builder.Generate())
1760     except KeyboardInterrupt:
1761       return
1762     except:
1763       # Already printed a warning, print the traceback and continue.
1764       traceback.print_exc()
1765     else:
1766       if utils.DEBUG:
1767         for ast in entire_ast:
1768           print(ast)
1769
1770
1771 if __name__ == '__main__':
1772   main(sys.argv)