]>
Commit | Line | Data |
---|---|---|
1e59de90 TL |
1 | #!/usr/bin/env python |
2 | # | |
3 | # Copyright 2007 Neal Norwitz | |
4 | # Portions Copyright 2007 Google Inc. | |
5 | # | |
6 | # Licensed under the Apache License, Version 2.0 (the "License"); | |
7 | # you may not use this file except in compliance with the License. | |
8 | # You may obtain a copy of the License at | |
9 | # | |
10 | # http://www.apache.org/licenses/LICENSE-2.0 | |
11 | # | |
12 | # Unless required by applicable law or agreed to in writing, software | |
13 | # distributed under the License is distributed on an "AS IS" BASIS, | |
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
15 | # See the License for the specific language governing permissions and | |
16 | # limitations under the License. | |
17 | ||
18 | """Generate an Abstract Syntax Tree (AST) for C++.""" | |
19 | ||
20 | # FIXME: | |
21 | # * Tokens should never be exported, need to convert to Nodes | |
22 | # (return types, parameters, etc.) | |
23 | # * Handle static class data for templatized classes | |
24 | # * Handle casts (both C++ and C-style) | |
25 | # * Handle conditions and loops (if/else, switch, for, while/do) | |
26 | # | |
27 | # TODO much, much later: | |
28 | # * Handle #define | |
29 | # * exceptions | |
30 | ||
31 | ||
32 | try: | |
33 | # Python 3.x | |
34 | import builtins | |
35 | except ImportError: | |
36 | # Python 2.x | |
37 | import __builtin__ as builtins | |
38 | ||
39 | import sys | |
40 | import traceback | |
41 | ||
42 | from cpp import keywords | |
43 | from cpp import tokenize | |
44 | from cpp import utils | |
45 | ||
46 | ||
47 | if not hasattr(builtins, 'reversed'): | |
48 | # Support Python 2.3 and earlier. | |
49 | def reversed(seq): | |
50 | for i in range(len(seq)-1, -1, -1): | |
51 | yield seq[i] | |
52 | ||
53 | if not hasattr(builtins, 'next'): | |
54 | # Support Python 2.5 and earlier. | |
55 | def next(obj): | |
56 | return obj.next() | |
57 | ||
58 | ||
59 | VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3) | |
60 | ||
61 | FUNCTION_NONE = 0x00 | |
62 | FUNCTION_CONST = 0x01 | |
63 | FUNCTION_VIRTUAL = 0x02 | |
64 | FUNCTION_PURE_VIRTUAL = 0x04 | |
65 | FUNCTION_CTOR = 0x08 | |
66 | FUNCTION_DTOR = 0x10 | |
67 | FUNCTION_ATTRIBUTE = 0x20 | |
68 | FUNCTION_UNKNOWN_ANNOTATION = 0x40 | |
69 | FUNCTION_THROW = 0x80 | |
70 | FUNCTION_OVERRIDE = 0x100 | |
71 | ||
72 | """ | |
73 | These are currently unused. Should really handle these properly at some point. | |
74 | ||
75 | TYPE_MODIFIER_INLINE = 0x010000 | |
76 | TYPE_MODIFIER_EXTERN = 0x020000 | |
77 | TYPE_MODIFIER_STATIC = 0x040000 | |
78 | TYPE_MODIFIER_CONST = 0x080000 | |
79 | TYPE_MODIFIER_REGISTER = 0x100000 | |
80 | TYPE_MODIFIER_VOLATILE = 0x200000 | |
81 | TYPE_MODIFIER_MUTABLE = 0x400000 | |
82 | ||
83 | TYPE_MODIFIER_MAP = { | |
84 | 'inline': TYPE_MODIFIER_INLINE, | |
85 | 'extern': TYPE_MODIFIER_EXTERN, | |
86 | 'static': TYPE_MODIFIER_STATIC, | |
87 | 'const': TYPE_MODIFIER_CONST, | |
88 | 'register': TYPE_MODIFIER_REGISTER, | |
89 | 'volatile': TYPE_MODIFIER_VOLATILE, | |
90 | 'mutable': TYPE_MODIFIER_MUTABLE, | |
91 | } | |
92 | """ | |
93 | ||
94 | _INTERNAL_TOKEN = 'internal' | |
95 | _NAMESPACE_POP = 'ns-pop' | |
96 | ||
97 | ||
98 | # TODO(nnorwitz): use this as a singleton for templated_types, etc | |
99 | # where we don't want to create a new empty dict each time. It is also const. | |
100 | class _NullDict(object): | |
101 | __contains__ = lambda self: False | |
102 | keys = values = items = iterkeys = itervalues = iteritems = lambda self: () | |
103 | ||
104 | ||
105 | # TODO(nnorwitz): move AST nodes into a separate module. | |
106 | class Node(object): | |
107 | """Base AST node.""" | |
108 | ||
109 | def __init__(self, start, end): | |
110 | self.start = start | |
111 | self.end = end | |
112 | ||
113 | def IsDeclaration(self): | |
114 | """Returns bool if this node is a declaration.""" | |
115 | return False | |
116 | ||
117 | def IsDefinition(self): | |
118 | """Returns bool if this node is a definition.""" | |
119 | return False | |
120 | ||
121 | def IsExportable(self): | |
122 | """Returns bool if this node exportable from a header file.""" | |
123 | return False | |
124 | ||
125 | def Requires(self, node): | |
126 | """Does this AST node require the definition of the node passed in?""" | |
127 | return False | |
128 | ||
129 | def XXX__str__(self): | |
130 | return self._StringHelper(self.__class__.__name__, '') | |
131 | ||
132 | def _StringHelper(self, name, suffix): | |
133 | if not utils.DEBUG: | |
134 | return '%s(%s)' % (name, suffix) | |
135 | return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix) | |
136 | ||
137 | def __repr__(self): | |
138 | return str(self) | |
139 | ||
140 | ||
141 | class Define(Node): | |
142 | def __init__(self, start, end, name, definition): | |
143 | Node.__init__(self, start, end) | |
144 | self.name = name | |
145 | self.definition = definition | |
146 | ||
147 | def __str__(self): | |
148 | value = '%s %s' % (self.name, self.definition) | |
149 | return self._StringHelper(self.__class__.__name__, value) | |
150 | ||
151 | ||
152 | class Include(Node): | |
153 | def __init__(self, start, end, filename, system): | |
154 | Node.__init__(self, start, end) | |
155 | self.filename = filename | |
156 | self.system = system | |
157 | ||
158 | def __str__(self): | |
159 | fmt = '"%s"' | |
160 | if self.system: | |
161 | fmt = '<%s>' | |
162 | return self._StringHelper(self.__class__.__name__, fmt % self.filename) | |
163 | ||
164 | ||
165 | class Goto(Node): | |
166 | def __init__(self, start, end, label): | |
167 | Node.__init__(self, start, end) | |
168 | self.label = label | |
169 | ||
170 | def __str__(self): | |
171 | return self._StringHelper(self.__class__.__name__, str(self.label)) | |
172 | ||
173 | ||
174 | class Expr(Node): | |
175 | def __init__(self, start, end, expr): | |
176 | Node.__init__(self, start, end) | |
177 | self.expr = expr | |
178 | ||
179 | def Requires(self, node): | |
180 | # TODO(nnorwitz): impl. | |
181 | return False | |
182 | ||
183 | def __str__(self): | |
184 | return self._StringHelper(self.__class__.__name__, str(self.expr)) | |
185 | ||
186 | ||
187 | class Return(Expr): | |
188 | pass | |
189 | ||
190 | ||
191 | class Delete(Expr): | |
192 | pass | |
193 | ||
194 | ||
195 | class Friend(Expr): | |
196 | def __init__(self, start, end, expr, namespace): | |
197 | Expr.__init__(self, start, end, expr) | |
198 | self.namespace = namespace[:] | |
199 | ||
200 | ||
201 | class Using(Node): | |
202 | def __init__(self, start, end, names): | |
203 | Node.__init__(self, start, end) | |
204 | self.names = names | |
205 | ||
206 | def __str__(self): | |
207 | return self._StringHelper(self.__class__.__name__, str(self.names)) | |
208 | ||
209 | ||
210 | class Parameter(Node): | |
211 | def __init__(self, start, end, name, parameter_type, default): | |
212 | Node.__init__(self, start, end) | |
213 | self.name = name | |
214 | self.type = parameter_type | |
215 | self.default = default | |
216 | ||
217 | def Requires(self, node): | |
218 | # TODO(nnorwitz): handle namespaces, etc. | |
219 | return self.type.name == node.name | |
220 | ||
221 | def __str__(self): | |
222 | name = str(self.type) | |
223 | suffix = '%s %s' % (name, self.name) | |
224 | if self.default: | |
225 | suffix += ' = ' + ''.join([d.name for d in self.default]) | |
226 | return self._StringHelper(self.__class__.__name__, suffix) | |
227 | ||
228 | ||
229 | class _GenericDeclaration(Node): | |
230 | def __init__(self, start, end, name, namespace): | |
231 | Node.__init__(self, start, end) | |
232 | self.name = name | |
233 | self.namespace = namespace[:] | |
234 | ||
235 | def FullName(self): | |
236 | prefix = '' | |
237 | if self.namespace and self.namespace[-1]: | |
238 | prefix = '::'.join(self.namespace) + '::' | |
239 | return prefix + self.name | |
240 | ||
241 | def _TypeStringHelper(self, suffix): | |
242 | if self.namespace: | |
243 | names = [n or '<anonymous>' for n in self.namespace] | |
244 | suffix += ' in ' + '::'.join(names) | |
245 | return self._StringHelper(self.__class__.__name__, suffix) | |
246 | ||
247 | ||
248 | # TODO(nnorwitz): merge with Parameter in some way? | |
249 | class VariableDeclaration(_GenericDeclaration): | |
250 | def __init__(self, start, end, name, var_type, initial_value, namespace): | |
251 | _GenericDeclaration.__init__(self, start, end, name, namespace) | |
252 | self.type = var_type | |
253 | self.initial_value = initial_value | |
254 | ||
255 | def Requires(self, node): | |
256 | # TODO(nnorwitz): handle namespaces, etc. | |
257 | return self.type.name == node.name | |
258 | ||
259 | def ToString(self): | |
260 | """Return a string that tries to reconstitute the variable decl.""" | |
261 | suffix = '%s %s' % (self.type, self.name) | |
262 | if self.initial_value: | |
263 | suffix += ' = ' + self.initial_value | |
264 | return suffix | |
265 | ||
266 | def __str__(self): | |
267 | return self._StringHelper(self.__class__.__name__, self.ToString()) | |
268 | ||
269 | ||
270 | class Typedef(_GenericDeclaration): | |
271 | def __init__(self, start, end, name, alias, namespace): | |
272 | _GenericDeclaration.__init__(self, start, end, name, namespace) | |
273 | self.alias = alias | |
274 | ||
275 | def IsDefinition(self): | |
276 | return True | |
277 | ||
278 | def IsExportable(self): | |
279 | return True | |
280 | ||
281 | def Requires(self, node): | |
282 | # TODO(nnorwitz): handle namespaces, etc. | |
283 | name = node.name | |
284 | for token in self.alias: | |
285 | if token is not None and name == token.name: | |
286 | return True | |
287 | return False | |
288 | ||
289 | def __str__(self): | |
290 | suffix = '%s, %s' % (self.name, self.alias) | |
291 | return self._TypeStringHelper(suffix) | |
292 | ||
293 | ||
294 | class _NestedType(_GenericDeclaration): | |
295 | def __init__(self, start, end, name, fields, namespace): | |
296 | _GenericDeclaration.__init__(self, start, end, name, namespace) | |
297 | self.fields = fields | |
298 | ||
299 | def IsDefinition(self): | |
300 | return True | |
301 | ||
302 | def IsExportable(self): | |
303 | return True | |
304 | ||
305 | def __str__(self): | |
306 | suffix = '%s, {%s}' % (self.name, self.fields) | |
307 | return self._TypeStringHelper(suffix) | |
308 | ||
309 | ||
310 | class Union(_NestedType): | |
311 | pass | |
312 | ||
313 | ||
314 | class Enum(_NestedType): | |
315 | pass | |
316 | ||
317 | ||
318 | class Class(_GenericDeclaration): | |
319 | def __init__(self, start, end, name, bases, templated_types, body, namespace): | |
320 | _GenericDeclaration.__init__(self, start, end, name, namespace) | |
321 | self.bases = bases | |
322 | self.body = body | |
323 | self.templated_types = templated_types | |
324 | ||
325 | def IsDeclaration(self): | |
326 | return self.bases is None and self.body is None | |
327 | ||
328 | def IsDefinition(self): | |
329 | return not self.IsDeclaration() | |
330 | ||
331 | def IsExportable(self): | |
332 | return not self.IsDeclaration() | |
333 | ||
334 | def Requires(self, node): | |
335 | # TODO(nnorwitz): handle namespaces, etc. | |
336 | if self.bases: | |
337 | for token_list in self.bases: | |
338 | # TODO(nnorwitz): bases are tokens, do name comparision. | |
339 | for token in token_list: | |
340 | if token.name == node.name: | |
341 | return True | |
342 | # TODO(nnorwitz): search in body too. | |
343 | return False | |
344 | ||
345 | def __str__(self): | |
346 | name = self.name | |
347 | if self.templated_types: | |
348 | name += '<%s>' % self.templated_types | |
349 | suffix = '%s, %s, %s' % (name, self.bases, self.body) | |
350 | return self._TypeStringHelper(suffix) | |
351 | ||
352 | ||
353 | class Struct(Class): | |
354 | pass | |
355 | ||
356 | ||
357 | class Function(_GenericDeclaration): | |
358 | def __init__(self, start, end, name, return_type, parameters, | |
359 | modifiers, templated_types, body, namespace): | |
360 | _GenericDeclaration.__init__(self, start, end, name, namespace) | |
361 | converter = TypeConverter(namespace) | |
362 | self.return_type = converter.CreateReturnType(return_type) | |
363 | self.parameters = converter.ToParameters(parameters) | |
364 | self.modifiers = modifiers | |
365 | self.body = body | |
366 | self.templated_types = templated_types | |
367 | ||
368 | def IsDeclaration(self): | |
369 | return self.body is None | |
370 | ||
371 | def IsDefinition(self): | |
372 | return self.body is not None | |
373 | ||
374 | def IsExportable(self): | |
375 | if self.return_type and 'static' in self.return_type.modifiers: | |
376 | return False | |
377 | return None not in self.namespace | |
378 | ||
379 | def Requires(self, node): | |
380 | if self.parameters: | |
381 | # TODO(nnorwitz): parameters are tokens, do name comparision. | |
382 | for p in self.parameters: | |
383 | if p.name == node.name: | |
384 | return True | |
385 | # TODO(nnorwitz): search in body too. | |
386 | return False | |
387 | ||
388 | def __str__(self): | |
389 | # TODO(nnorwitz): add templated_types. | |
390 | suffix = ('%s %s(%s), 0x%02x, %s' % | |
391 | (self.return_type, self.name, self.parameters, | |
392 | self.modifiers, self.body)) | |
393 | return self._TypeStringHelper(suffix) | |
394 | ||
395 | ||
396 | class Method(Function): | |
397 | def __init__(self, start, end, name, in_class, return_type, parameters, | |
398 | modifiers, templated_types, body, namespace): | |
399 | Function.__init__(self, start, end, name, return_type, parameters, | |
400 | modifiers, templated_types, body, namespace) | |
401 | # TODO(nnorwitz): in_class could also be a namespace which can | |
402 | # mess up finding functions properly. | |
403 | self.in_class = in_class | |
404 | ||
405 | ||
406 | class Type(_GenericDeclaration): | |
407 | """Type used for any variable (eg class, primitive, struct, etc).""" | |
408 | ||
409 | def __init__(self, start, end, name, templated_types, modifiers, | |
410 | reference, pointer, array): | |
411 | """ | |
412 | Args: | |
413 | name: str name of main type | |
414 | templated_types: [Class (Type?)] template type info between <> | |
415 | modifiers: [str] type modifiers (keywords) eg, const, mutable, etc. | |
416 | reference, pointer, array: bools | |
417 | """ | |
418 | _GenericDeclaration.__init__(self, start, end, name, []) | |
419 | self.templated_types = templated_types | |
420 | if not name and modifiers: | |
421 | self.name = modifiers.pop() | |
422 | self.modifiers = modifiers | |
423 | self.reference = reference | |
424 | self.pointer = pointer | |
425 | self.array = array | |
426 | ||
427 | def __str__(self): | |
428 | prefix = '' | |
429 | if self.modifiers: | |
430 | prefix = ' '.join(self.modifiers) + ' ' | |
431 | name = str(self.name) | |
432 | if self.templated_types: | |
433 | name += '<%s>' % self.templated_types | |
434 | suffix = prefix + name | |
435 | if self.reference: | |
436 | suffix += '&' | |
437 | if self.pointer: | |
438 | suffix += '*' | |
439 | if self.array: | |
440 | suffix += '[]' | |
441 | return self._TypeStringHelper(suffix) | |
442 | ||
443 | # By definition, Is* are always False. A Type can only exist in | |
444 | # some sort of variable declaration, parameter, or return value. | |
445 | def IsDeclaration(self): | |
446 | return False | |
447 | ||
448 | def IsDefinition(self): | |
449 | return False | |
450 | ||
451 | def IsExportable(self): | |
452 | return False | |
453 | ||
454 | ||
455 | class TypeConverter(object): | |
456 | ||
457 | def __init__(self, namespace_stack): | |
458 | self.namespace_stack = namespace_stack | |
459 | ||
460 | def _GetTemplateEnd(self, tokens, start): | |
461 | count = 1 | |
462 | end = start | |
463 | while 1: | |
464 | token = tokens[end] | |
465 | end += 1 | |
466 | if token.name == '<': | |
467 | count += 1 | |
468 | elif token.name == '>': | |
469 | count -= 1 | |
470 | if count == 0: | |
471 | break | |
472 | return tokens[start:end-1], end | |
473 | ||
474 | def ToType(self, tokens): | |
475 | """Convert [Token,...] to [Class(...), ] useful for base classes. | |
476 | For example, code like class Foo : public Bar<x, y> { ... }; | |
477 | the "Bar<x, y>" portion gets converted to an AST. | |
478 | ||
479 | Returns: | |
480 | [Class(...), ...] | |
481 | """ | |
482 | result = [] | |
483 | name_tokens = [] | |
484 | reference = pointer = array = False | |
485 | ||
486 | def AddType(templated_types): | |
487 | # Partition tokens into name and modifier tokens. | |
488 | names = [] | |
489 | modifiers = [] | |
490 | for t in name_tokens: | |
491 | if keywords.IsKeyword(t.name): | |
492 | modifiers.append(t.name) | |
493 | else: | |
494 | names.append(t.name) | |
495 | name = ''.join(names) | |
496 | if name_tokens: | |
497 | result.append(Type(name_tokens[0].start, name_tokens[-1].end, | |
498 | name, templated_types, modifiers, | |
499 | reference, pointer, array)) | |
500 | del name_tokens[:] | |
501 | ||
502 | i = 0 | |
503 | end = len(tokens) | |
504 | while i < end: | |
505 | token = tokens[i] | |
506 | if token.name == '<': | |
507 | new_tokens, new_end = self._GetTemplateEnd(tokens, i+1) | |
508 | AddType(self.ToType(new_tokens)) | |
509 | # If there is a comma after the template, we need to consume | |
510 | # that here otherwise it becomes part of the name. | |
511 | i = new_end | |
512 | reference = pointer = array = False | |
513 | elif token.name == ',': | |
514 | AddType([]) | |
515 | reference = pointer = array = False | |
516 | elif token.name == '*': | |
517 | pointer = True | |
518 | elif token.name == '&': | |
519 | reference = True | |
520 | elif token.name == '[': | |
521 | pointer = True | |
522 | elif token.name == ']': | |
523 | pass | |
524 | else: | |
525 | name_tokens.append(token) | |
526 | i += 1 | |
527 | ||
528 | if name_tokens: | |
529 | # No '<' in the tokens, just a simple name and no template. | |
530 | AddType([]) | |
531 | return result | |
532 | ||
533 | def DeclarationToParts(self, parts, needs_name_removed): | |
534 | name = None | |
535 | default = [] | |
536 | if needs_name_removed: | |
537 | # Handle default (initial) values properly. | |
538 | for i, t in enumerate(parts): | |
539 | if t.name == '=': | |
540 | default = parts[i+1:] | |
541 | name = parts[i-1].name | |
542 | if name == ']' and parts[i-2].name == '[': | |
543 | name = parts[i-3].name | |
544 | i -= 1 | |
545 | parts = parts[:i-1] | |
546 | break | |
547 | else: | |
548 | if parts[-1].token_type == tokenize.NAME: | |
549 | name = parts.pop().name | |
550 | else: | |
551 | # TODO(nnorwitz): this is a hack that happens for code like | |
552 | # Register(Foo<T>); where it thinks this is a function call | |
553 | # but it's actually a declaration. | |
554 | name = '???' | |
555 | modifiers = [] | |
556 | type_name = [] | |
557 | other_tokens = [] | |
558 | templated_types = [] | |
559 | i = 0 | |
560 | end = len(parts) | |
561 | while i < end: | |
562 | p = parts[i] | |
563 | if keywords.IsKeyword(p.name): | |
564 | modifiers.append(p.name) | |
565 | elif p.name == '<': | |
566 | templated_tokens, new_end = self._GetTemplateEnd(parts, i+1) | |
567 | templated_types = self.ToType(templated_tokens) | |
568 | i = new_end - 1 | |
569 | # Don't add a spurious :: to data members being initialized. | |
570 | next_index = i + 1 | |
571 | if next_index < end and parts[next_index].name == '::': | |
572 | i += 1 | |
573 | elif p.name in ('[', ']', '='): | |
574 | # These are handled elsewhere. | |
575 | other_tokens.append(p) | |
576 | elif p.name not in ('*', '&', '>'): | |
577 | # Ensure that names have a space between them. | |
578 | if (type_name and type_name[-1].token_type == tokenize.NAME and | |
579 | p.token_type == tokenize.NAME): | |
580 | type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0)) | |
581 | type_name.append(p) | |
582 | else: | |
583 | other_tokens.append(p) | |
584 | i += 1 | |
585 | type_name = ''.join([t.name for t in type_name]) | |
586 | return name, type_name, templated_types, modifiers, default, other_tokens | |
587 | ||
588 | def ToParameters(self, tokens): | |
589 | if not tokens: | |
590 | return [] | |
591 | ||
592 | result = [] | |
593 | name = type_name = '' | |
594 | type_modifiers = [] | |
595 | pointer = reference = array = False | |
596 | first_token = None | |
597 | default = [] | |
598 | ||
599 | def AddParameter(end): | |
600 | if default: | |
601 | del default[0] # Remove flag. | |
602 | parts = self.DeclarationToParts(type_modifiers, True) | |
603 | (name, type_name, templated_types, modifiers, | |
604 | unused_default, unused_other_tokens) = parts | |
605 | parameter_type = Type(first_token.start, first_token.end, | |
606 | type_name, templated_types, modifiers, | |
607 | reference, pointer, array) | |
608 | p = Parameter(first_token.start, end, name, | |
609 | parameter_type, default) | |
610 | result.append(p) | |
611 | ||
612 | template_count = 0 | |
613 | brace_count = 0 | |
614 | for s in tokens: | |
615 | if not first_token: | |
616 | first_token = s | |
617 | ||
618 | # Check for braces before templates, as we can have unmatched '<>' | |
619 | # inside default arguments. | |
620 | if s.name == '{': | |
621 | brace_count += 1 | |
622 | elif s.name == '}': | |
623 | brace_count -= 1 | |
624 | if brace_count > 0: | |
625 | type_modifiers.append(s) | |
626 | continue | |
627 | ||
628 | if s.name == '<': | |
629 | template_count += 1 | |
630 | elif s.name == '>': | |
631 | template_count -= 1 | |
632 | if template_count > 0: | |
633 | type_modifiers.append(s) | |
634 | continue | |
635 | ||
636 | if s.name == ',': | |
637 | AddParameter(s.start) | |
638 | name = type_name = '' | |
639 | type_modifiers = [] | |
640 | pointer = reference = array = False | |
641 | first_token = None | |
642 | default = [] | |
643 | elif s.name == '*': | |
644 | pointer = True | |
645 | elif s.name == '&': | |
646 | reference = True | |
647 | elif s.name == '[': | |
648 | array = True | |
649 | elif s.name == ']': | |
650 | pass # Just don't add to type_modifiers. | |
651 | elif s.name == '=': | |
652 | # Got a default value. Add any value (None) as a flag. | |
653 | default.append(None) | |
654 | elif default: | |
655 | default.append(s) | |
656 | else: | |
657 | type_modifiers.append(s) | |
658 | AddParameter(tokens[-1].end) | |
659 | return result | |
660 | ||
661 | def CreateReturnType(self, return_type_seq): | |
662 | if not return_type_seq: | |
663 | return None | |
664 | start = return_type_seq[0].start | |
665 | end = return_type_seq[-1].end | |
666 | _, name, templated_types, modifiers, default, other_tokens = \ | |
667 | self.DeclarationToParts(return_type_seq, False) | |
668 | names = [n.name for n in other_tokens] | |
669 | reference = '&' in names | |
670 | pointer = '*' in names | |
671 | array = '[' in names | |
672 | return Type(start, end, name, templated_types, modifiers, | |
673 | reference, pointer, array) | |
674 | ||
675 | def GetTemplateIndices(self, names): | |
676 | # names is a list of strings. | |
677 | start = names.index('<') | |
678 | end = len(names) - 1 | |
679 | while end > 0: | |
680 | if names[end] == '>': | |
681 | break | |
682 | end -= 1 | |
683 | return start, end+1 | |
684 | ||
685 | class AstBuilder(object): | |
686 | def __init__(self, token_stream, filename, in_class='', visibility=None, | |
687 | namespace_stack=[]): | |
688 | self.tokens = token_stream | |
689 | self.filename = filename | |
690 | # TODO(nnorwitz): use a better data structure (deque) for the queue. | |
691 | # Switching directions of the "queue" improved perf by about 25%. | |
692 | # Using a deque should be even better since we access from both sides. | |
693 | self.token_queue = [] | |
694 | self.namespace_stack = namespace_stack[:] | |
695 | self.in_class = in_class | |
696 | if in_class is None: | |
697 | self.in_class_name_only = None | |
698 | else: | |
699 | self.in_class_name_only = in_class.split('::')[-1] | |
700 | self.visibility = visibility | |
701 | self.in_function = False | |
702 | self.current_token = None | |
703 | # Keep the state whether we are currently handling a typedef or not. | |
704 | self._handling_typedef = False | |
705 | ||
706 | self.converter = TypeConverter(self.namespace_stack) | |
707 | ||
708 | def HandleError(self, msg, token): | |
709 | printable_queue = list(reversed(self.token_queue[-20:])) | |
710 | sys.stderr.write('Got %s in %s @ %s %s\n' % | |
711 | (msg, self.filename, token, printable_queue)) | |
712 | ||
713 | def Generate(self): | |
714 | while 1: | |
715 | token = self._GetNextToken() | |
716 | if not token: | |
717 | break | |
718 | ||
719 | # Get the next token. | |
720 | self.current_token = token | |
721 | ||
722 | # Dispatch on the next token type. | |
723 | if token.token_type == _INTERNAL_TOKEN: | |
724 | if token.name == _NAMESPACE_POP: | |
725 | self.namespace_stack.pop() | |
726 | continue | |
727 | ||
728 | try: | |
729 | result = self._GenerateOne(token) | |
730 | if result is not None: | |
731 | yield result | |
732 | except: | |
733 | self.HandleError('exception', token) | |
734 | raise | |
735 | ||
736 | def _CreateVariable(self, pos_token, name, type_name, type_modifiers, | |
737 | ref_pointer_name_seq, templated_types, value=None): | |
738 | reference = '&' in ref_pointer_name_seq | |
739 | pointer = '*' in ref_pointer_name_seq | |
740 | array = '[' in ref_pointer_name_seq | |
741 | var_type = Type(pos_token.start, pos_token.end, type_name, | |
742 | templated_types, type_modifiers, | |
743 | reference, pointer, array) | |
744 | return VariableDeclaration(pos_token.start, pos_token.end, | |
745 | name, var_type, value, self.namespace_stack) | |
746 | ||
747 | def _GenerateOne(self, token): | |
748 | if token.token_type == tokenize.NAME: | |
749 | if (keywords.IsKeyword(token.name) and | |
750 | not keywords.IsBuiltinType(token.name)): | |
751 | if token.name == 'enum': | |
752 | # Pop the next token and only put it back if it's not | |
753 | # 'class'. This allows us to support the two-token | |
754 | # 'enum class' keyword as if it were simply 'enum'. | |
755 | next = self._GetNextToken() | |
756 | if next.name != 'class': | |
757 | self._AddBackToken(next) | |
758 | ||
759 | method = getattr(self, 'handle_' + token.name) | |
760 | return method() | |
761 | elif token.name == self.in_class_name_only: | |
762 | # The token name is the same as the class, must be a ctor if | |
763 | # there is a paren. Otherwise, it's the return type. | |
764 | # Peek ahead to get the next token to figure out which. | |
765 | next = self._GetNextToken() | |
766 | self._AddBackToken(next) | |
767 | if next.token_type == tokenize.SYNTAX and next.name == '(': | |
768 | return self._GetMethod([token], FUNCTION_CTOR, None, True) | |
769 | # Fall through--handle like any other method. | |
770 | ||
771 | # Handle data or function declaration/definition. | |
772 | syntax = tokenize.SYNTAX | |
773 | temp_tokens, last_token = \ | |
774 | self._GetVarTokensUpToIgnoringTemplates(syntax, | |
775 | '(', ';', '{', '[') | |
776 | temp_tokens.insert(0, token) | |
777 | if last_token.name == '(': | |
778 | # If there is an assignment before the paren, | |
779 | # this is an expression, not a method. | |
780 | expr = bool([e for e in temp_tokens if e.name == '=']) | |
781 | if expr: | |
782 | new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
783 | temp_tokens.append(last_token) | |
784 | temp_tokens.extend(new_temp) | |
785 | last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0) | |
786 | ||
787 | if last_token.name == '[': | |
788 | # Handle array, this isn't a method, unless it's an operator. | |
789 | # TODO(nnorwitz): keep the size somewhere. | |
790 | # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']') | |
791 | temp_tokens.append(last_token) | |
792 | if temp_tokens[-2].name == 'operator': | |
793 | temp_tokens.append(self._GetNextToken()) | |
794 | else: | |
795 | temp_tokens2, last_token = \ | |
796 | self._GetVarTokensUpTo(tokenize.SYNTAX, ';') | |
797 | temp_tokens.extend(temp_tokens2) | |
798 | ||
799 | if last_token.name == ';': | |
800 | # Handle data, this isn't a method. | |
801 | parts = self.converter.DeclarationToParts(temp_tokens, True) | |
802 | (name, type_name, templated_types, modifiers, default, | |
803 | unused_other_tokens) = parts | |
804 | ||
805 | t0 = temp_tokens[0] | |
806 | names = [t.name for t in temp_tokens] | |
807 | if templated_types: | |
808 | start, end = self.converter.GetTemplateIndices(names) | |
809 | names = names[:start] + names[end:] | |
810 | default = ''.join([t.name for t in default]) | |
811 | return self._CreateVariable(t0, name, type_name, modifiers, | |
812 | names, templated_types, default) | |
813 | if last_token.name == '{': | |
814 | self._AddBackTokens(temp_tokens[1:]) | |
815 | self._AddBackToken(last_token) | |
816 | method_name = temp_tokens[0].name | |
817 | method = getattr(self, 'handle_' + method_name, None) | |
818 | if not method: | |
819 | # Must be declaring a variable. | |
820 | # TODO(nnorwitz): handle the declaration. | |
821 | return None | |
822 | return method() | |
823 | return self._GetMethod(temp_tokens, 0, None, False) | |
824 | elif token.token_type == tokenize.SYNTAX: | |
825 | if token.name == '~' and self.in_class: | |
826 | # Must be a dtor (probably not in method body). | |
827 | token = self._GetNextToken() | |
828 | # self.in_class can contain A::Name, but the dtor will only | |
829 | # be Name. Make sure to compare against the right value. | |
830 | if (token.token_type == tokenize.NAME and | |
831 | token.name == self.in_class_name_only): | |
832 | return self._GetMethod([token], FUNCTION_DTOR, None, True) | |
833 | # TODO(nnorwitz): handle a lot more syntax. | |
834 | elif token.token_type == tokenize.PREPROCESSOR: | |
835 | # TODO(nnorwitz): handle more preprocessor directives. | |
836 | # token starts with a #, so remove it and strip whitespace. | |
837 | name = token.name[1:].lstrip() | |
838 | if name.startswith('include'): | |
839 | # Remove "include". | |
840 | name = name[7:].strip() | |
841 | assert name | |
842 | # Handle #include \<newline> "header-on-second-line.h". | |
843 | if name.startswith('\\'): | |
844 | name = name[1:].strip() | |
845 | assert name[0] in '<"', token | |
846 | assert name[-1] in '>"', token | |
847 | system = name[0] == '<' | |
848 | filename = name[1:-1] | |
849 | return Include(token.start, token.end, filename, system) | |
850 | if name.startswith('define'): | |
851 | # Remove "define". | |
852 | name = name[6:].strip() | |
853 | assert name | |
854 | value = '' | |
855 | for i, c in enumerate(name): | |
856 | if c.isspace(): | |
857 | value = name[i:].lstrip() | |
858 | name = name[:i] | |
859 | break | |
860 | return Define(token.start, token.end, name, value) | |
861 | if name.startswith('if') and name[2:3].isspace(): | |
862 | condition = name[3:].strip() | |
863 | if condition.startswith('0') or condition.startswith('(0)'): | |
864 | self._SkipIf0Blocks() | |
865 | return None | |
866 | ||
867 | def _GetTokensUpTo(self, expected_token_type, expected_token): | |
868 | return self._GetVarTokensUpTo(expected_token_type, expected_token)[0] | |
869 | ||
870 | def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens): | |
871 | last_token = self._GetNextToken() | |
872 | tokens = [] | |
873 | while (last_token.token_type != expected_token_type or | |
874 | last_token.name not in expected_tokens): | |
875 | tokens.append(last_token) | |
876 | last_token = self._GetNextToken() | |
877 | return tokens, last_token | |
878 | ||
879 | # Same as _GetVarTokensUpTo, but skips over '<...>' which could contain an | |
880 | # expected token. | |
881 | def _GetVarTokensUpToIgnoringTemplates(self, expected_token_type, | |
882 | *expected_tokens): | |
883 | last_token = self._GetNextToken() | |
884 | tokens = [] | |
885 | nesting = 0 | |
886 | while (nesting > 0 or | |
887 | last_token.token_type != expected_token_type or | |
888 | last_token.name not in expected_tokens): | |
889 | tokens.append(last_token) | |
890 | last_token = self._GetNextToken() | |
891 | if last_token.name == '<': | |
892 | nesting += 1 | |
893 | elif last_token.name == '>': | |
894 | nesting -= 1 | |
895 | return tokens, last_token | |
896 | ||
897 | # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necesary. | |
898 | def _IgnoreUpTo(self, token_type, token): | |
899 | unused_tokens = self._GetTokensUpTo(token_type, token) | |
900 | ||
901 | def _SkipIf0Blocks(self): | |
902 | count = 1 | |
903 | while 1: | |
904 | token = self._GetNextToken() | |
905 | if token.token_type != tokenize.PREPROCESSOR: | |
906 | continue | |
907 | ||
908 | name = token.name[1:].lstrip() | |
909 | if name.startswith('endif'): | |
910 | count -= 1 | |
911 | if count == 0: | |
912 | break | |
913 | elif name.startswith('if'): | |
914 | count += 1 | |
915 | ||
916 | def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None): | |
917 | if GetNextToken is None: | |
918 | GetNextToken = self._GetNextToken | |
919 | # Assumes the current token is open_paren and we will consume | |
920 | # and return up to the close_paren. | |
921 | count = 1 | |
922 | token = GetNextToken() | |
923 | while 1: | |
924 | if token.token_type == tokenize.SYNTAX: | |
925 | if token.name == open_paren: | |
926 | count += 1 | |
927 | elif token.name == close_paren: | |
928 | count -= 1 | |
929 | if count == 0: | |
930 | break | |
931 | yield token | |
932 | token = GetNextToken() | |
933 | yield token | |
934 | ||
935 | def _GetParameters(self): | |
936 | return self._GetMatchingChar('(', ')') | |
937 | ||
938 | def GetScope(self): | |
939 | return self._GetMatchingChar('{', '}') | |
940 | ||
941 | def _GetNextToken(self): | |
942 | if self.token_queue: | |
943 | return self.token_queue.pop() | |
944 | try: | |
945 | return next(self.tokens) | |
946 | except StopIteration: | |
947 | return | |
948 | ||
949 | def _AddBackToken(self, token): | |
950 | if token.whence == tokenize.WHENCE_STREAM: | |
951 | token.whence = tokenize.WHENCE_QUEUE | |
952 | self.token_queue.insert(0, token) | |
953 | else: | |
954 | assert token.whence == tokenize.WHENCE_QUEUE, token | |
955 | self.token_queue.append(token) | |
956 | ||
957 | def _AddBackTokens(self, tokens): | |
958 | if tokens: | |
959 | if tokens[-1].whence == tokenize.WHENCE_STREAM: | |
960 | for token in tokens: | |
961 | token.whence = tokenize.WHENCE_QUEUE | |
962 | self.token_queue[:0] = reversed(tokens) | |
963 | else: | |
964 | assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens | |
965 | self.token_queue.extend(reversed(tokens)) | |
966 | ||
967 | def GetName(self, seq=None): | |
968 | """Returns ([tokens], next_token_info).""" | |
969 | GetNextToken = self._GetNextToken | |
970 | if seq is not None: | |
971 | it = iter(seq) | |
972 | GetNextToken = lambda: next(it) | |
973 | next_token = GetNextToken() | |
974 | tokens = [] | |
975 | last_token_was_name = False | |
976 | while (next_token.token_type == tokenize.NAME or | |
977 | (next_token.token_type == tokenize.SYNTAX and | |
978 | next_token.name in ('::', '<'))): | |
979 | # Two NAMEs in a row means the identifier should terminate. | |
980 | # It's probably some sort of variable declaration. | |
981 | if last_token_was_name and next_token.token_type == tokenize.NAME: | |
982 | break | |
983 | last_token_was_name = next_token.token_type == tokenize.NAME | |
984 | tokens.append(next_token) | |
985 | # Handle templated names. | |
986 | if next_token.name == '<': | |
987 | tokens.extend(self._GetMatchingChar('<', '>', GetNextToken)) | |
988 | last_token_was_name = True | |
989 | next_token = GetNextToken() | |
990 | return tokens, next_token | |
991 | ||
992 | def GetMethod(self, modifiers, templated_types): | |
993 | return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') | |
994 | assert len(return_type_and_name) >= 1 | |
995 | return self._GetMethod(return_type_and_name, modifiers, templated_types, | |
996 | False) | |
997 | ||
998 | def _GetMethod(self, return_type_and_name, modifiers, templated_types, | |
999 | get_paren): | |
1000 | template_portion = None | |
1001 | if get_paren: | |
1002 | token = self._GetNextToken() | |
1003 | assert token.token_type == tokenize.SYNTAX, token | |
1004 | if token.name == '<': | |
1005 | # Handle templatized dtors. | |
1006 | template_portion = [token] | |
1007 | template_portion.extend(self._GetMatchingChar('<', '>')) | |
1008 | token = self._GetNextToken() | |
1009 | assert token.token_type == tokenize.SYNTAX, token | |
1010 | assert token.name == '(', token | |
1011 | ||
1012 | name = return_type_and_name.pop() | |
1013 | # Handle templatized ctors. | |
1014 | if name.name == '>': | |
1015 | index = 1 | |
1016 | while return_type_and_name[index].name != '<': | |
1017 | index += 1 | |
1018 | template_portion = return_type_and_name[index:] + [name] | |
1019 | del return_type_and_name[index:] | |
1020 | name = return_type_and_name.pop() | |
1021 | elif name.name == ']': | |
1022 | rt = return_type_and_name | |
1023 | assert rt[-1].name == '[', return_type_and_name | |
1024 | assert rt[-2].name == 'operator', return_type_and_name | |
1025 | name_seq = return_type_and_name[-2:] | |
1026 | del return_type_and_name[-2:] | |
1027 | name = tokenize.Token(tokenize.NAME, 'operator[]', | |
1028 | name_seq[0].start, name.end) | |
1029 | # Get the open paren so _GetParameters() below works. | |
1030 | unused_open_paren = self._GetNextToken() | |
1031 | ||
1032 | # TODO(nnorwitz): store template_portion. | |
1033 | return_type = return_type_and_name | |
1034 | indices = name | |
1035 | if return_type: | |
1036 | indices = return_type[0] | |
1037 | ||
1038 | # Force ctor for templatized ctors. | |
1039 | if name.name == self.in_class and not modifiers: | |
1040 | modifiers |= FUNCTION_CTOR | |
1041 | parameters = list(self._GetParameters()) | |
1042 | del parameters[-1] # Remove trailing ')'. | |
1043 | ||
1044 | # Handling operator() is especially weird. | |
1045 | if name.name == 'operator' and not parameters: | |
1046 | token = self._GetNextToken() | |
1047 | assert token.name == '(', token | |
1048 | parameters = list(self._GetParameters()) | |
1049 | del parameters[-1] # Remove trailing ')'. | |
1050 | ||
1051 | token = self._GetNextToken() | |
1052 | while token.token_type == tokenize.NAME: | |
1053 | modifier_token = token | |
1054 | token = self._GetNextToken() | |
1055 | if modifier_token.name == 'const': | |
1056 | modifiers |= FUNCTION_CONST | |
1057 | elif modifier_token.name == '__attribute__': | |
1058 | # TODO(nnorwitz): handle more __attribute__ details. | |
1059 | modifiers |= FUNCTION_ATTRIBUTE | |
1060 | assert token.name == '(', token | |
1061 | # Consume everything between the (parens). | |
1062 | unused_tokens = list(self._GetMatchingChar('(', ')')) | |
1063 | token = self._GetNextToken() | |
1064 | elif modifier_token.name == 'throw': | |
1065 | modifiers |= FUNCTION_THROW | |
1066 | assert token.name == '(', token | |
1067 | # Consume everything between the (parens). | |
1068 | unused_tokens = list(self._GetMatchingChar('(', ')')) | |
1069 | token = self._GetNextToken() | |
1070 | elif modifier_token.name == 'override': | |
1071 | modifiers |= FUNCTION_OVERRIDE | |
1072 | elif modifier_token.name == modifier_token.name.upper(): | |
1073 | # HACK(nnorwitz): assume that all upper-case names | |
1074 | # are some macro we aren't expanding. | |
1075 | modifiers |= FUNCTION_UNKNOWN_ANNOTATION | |
1076 | else: | |
1077 | self.HandleError('unexpected token', modifier_token) | |
1078 | ||
1079 | assert token.token_type == tokenize.SYNTAX, token | |
1080 | # Handle ctor initializers. | |
1081 | if token.name == ':': | |
1082 | # TODO(nnorwitz): anything else to handle for initializer list? | |
1083 | while token.name != ';' and token.name != '{': | |
1084 | token = self._GetNextToken() | |
1085 | ||
1086 | # Handle pointer to functions that are really data but look | |
1087 | # like method declarations. | |
1088 | if token.name == '(': | |
1089 | if parameters[0].name == '*': | |
1090 | # name contains the return type. | |
1091 | name = parameters.pop() | |
1092 | # parameters contains the name of the data. | |
1093 | modifiers = [p.name for p in parameters] | |
1094 | # Already at the ( to open the parameter list. | |
1095 | function_parameters = list(self._GetMatchingChar('(', ')')) | |
1096 | del function_parameters[-1] # Remove trailing ')'. | |
1097 | # TODO(nnorwitz): store the function_parameters. | |
1098 | token = self._GetNextToken() | |
1099 | assert token.token_type == tokenize.SYNTAX, token | |
1100 | assert token.name == ';', token | |
1101 | return self._CreateVariable(indices, name.name, indices.name, | |
1102 | modifiers, '', None) | |
1103 | # At this point, we got something like: | |
1104 | # return_type (type::*name_)(params); | |
1105 | # This is a data member called name_ that is a function pointer. | |
1106 | # With this code: void (sq_type::*field_)(string&); | |
1107 | # We get: name=void return_type=[] parameters=sq_type ... field_ | |
1108 | # TODO(nnorwitz): is return_type always empty? | |
1109 | # TODO(nnorwitz): this isn't even close to being correct. | |
1110 | # Just put in something so we don't crash and can move on. | |
1111 | real_name = parameters[-1] | |
1112 | modifiers = [p.name for p in self._GetParameters()] | |
1113 | del modifiers[-1] # Remove trailing ')'. | |
1114 | return self._CreateVariable(indices, real_name.name, indices.name, | |
1115 | modifiers, '', None) | |
1116 | ||
1117 | if token.name == '{': | |
1118 | body = list(self.GetScope()) | |
1119 | del body[-1] # Remove trailing '}'. | |
1120 | else: | |
1121 | body = None | |
1122 | if token.name == '=': | |
1123 | token = self._GetNextToken() | |
1124 | ||
1125 | if token.name == 'default' or token.name == 'delete': | |
1126 | # Ignore explicitly defaulted and deleted special members | |
1127 | # in C++11. | |
1128 | token = self._GetNextToken() | |
1129 | else: | |
1130 | # Handle pure-virtual declarations. | |
1131 | assert token.token_type == tokenize.CONSTANT, token | |
1132 | assert token.name == '0', token | |
1133 | modifiers |= FUNCTION_PURE_VIRTUAL | |
1134 | token = self._GetNextToken() | |
1135 | ||
1136 | if token.name == '[': | |
1137 | # TODO(nnorwitz): store tokens and improve parsing. | |
1138 | # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N]; | |
1139 | tokens = list(self._GetMatchingChar('[', ']')) | |
1140 | token = self._GetNextToken() | |
1141 | ||
1142 | assert token.name == ';', (token, return_type_and_name, parameters) | |
1143 | ||
1144 | # Looks like we got a method, not a function. | |
1145 | if len(return_type) > 2 and return_type[-1].name == '::': | |
1146 | return_type, in_class = \ | |
1147 | self._GetReturnTypeAndClassName(return_type) | |
1148 | return Method(indices.start, indices.end, name.name, in_class, | |
1149 | return_type, parameters, modifiers, templated_types, | |
1150 | body, self.namespace_stack) | |
1151 | return Function(indices.start, indices.end, name.name, return_type, | |
1152 | parameters, modifiers, templated_types, body, | |
1153 | self.namespace_stack) | |
1154 | ||
1155 | def _GetReturnTypeAndClassName(self, token_seq): | |
1156 | # Splitting the return type from the class name in a method | |
1157 | # can be tricky. For example, Return::Type::Is::Hard::To::Find(). | |
1158 | # Where is the return type and where is the class name? | |
1159 | # The heuristic used is to pull the last name as the class name. | |
1160 | # This includes all the templated type info. | |
1161 | # TODO(nnorwitz): if there is only One name like in the | |
1162 | # example above, punt and assume the last bit is the class name. | |
1163 | ||
1164 | # Ignore a :: prefix, if exists so we can find the first real name. | |
1165 | i = 0 | |
1166 | if token_seq[0].name == '::': | |
1167 | i = 1 | |
1168 | # Ignore a :: suffix, if exists. | |
1169 | end = len(token_seq) - 1 | |
1170 | if token_seq[end-1].name == '::': | |
1171 | end -= 1 | |
1172 | ||
1173 | # Make a copy of the sequence so we can append a sentinel | |
1174 | # value. This is required for GetName will has to have some | |
1175 | # terminating condition beyond the last name. | |
1176 | seq_copy = token_seq[i:end] | |
1177 | seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0)) | |
1178 | names = [] | |
1179 | while i < end: | |
1180 | # Iterate through the sequence parsing out each name. | |
1181 | new_name, next = self.GetName(seq_copy[i:]) | |
1182 | assert new_name, 'Got empty new_name, next=%s' % next | |
1183 | # We got a pointer or ref. Add it to the name. | |
1184 | if next and next.token_type == tokenize.SYNTAX: | |
1185 | new_name.append(next) | |
1186 | names.append(new_name) | |
1187 | i += len(new_name) | |
1188 | ||
1189 | # Now that we have the names, it's time to undo what we did. | |
1190 | ||
1191 | # Remove the sentinel value. | |
1192 | names[-1].pop() | |
1193 | # Flatten the token sequence for the return type. | |
1194 | return_type = [e for seq in names[:-1] for e in seq] | |
1195 | # The class name is the last name. | |
1196 | class_name = names[-1] | |
1197 | return return_type, class_name | |
1198 | ||
1199 | def handle_bool(self): | |
1200 | pass | |
1201 | ||
1202 | def handle_char(self): | |
1203 | pass | |
1204 | ||
1205 | def handle_int(self): | |
1206 | pass | |
1207 | ||
1208 | def handle_long(self): | |
1209 | pass | |
1210 | ||
1211 | def handle_short(self): | |
1212 | pass | |
1213 | ||
1214 | def handle_double(self): | |
1215 | pass | |
1216 | ||
1217 | def handle_float(self): | |
1218 | pass | |
1219 | ||
1220 | def handle_void(self): | |
1221 | pass | |
1222 | ||
1223 | def handle_wchar_t(self): | |
1224 | pass | |
1225 | ||
1226 | def handle_unsigned(self): | |
1227 | pass | |
1228 | ||
1229 | def handle_signed(self): | |
1230 | pass | |
1231 | ||
1232 | def _GetNestedType(self, ctor): | |
1233 | name = None | |
1234 | name_tokens, token = self.GetName() | |
1235 | if name_tokens: | |
1236 | name = ''.join([t.name for t in name_tokens]) | |
1237 | ||
1238 | # Handle forward declarations. | |
1239 | if token.token_type == tokenize.SYNTAX and token.name == ';': | |
1240 | return ctor(token.start, token.end, name, None, | |
1241 | self.namespace_stack) | |
1242 | ||
1243 | if token.token_type == tokenize.NAME and self._handling_typedef: | |
1244 | self._AddBackToken(token) | |
1245 | return ctor(token.start, token.end, name, None, | |
1246 | self.namespace_stack) | |
1247 | ||
1248 | # Must be the type declaration. | |
1249 | fields = list(self._GetMatchingChar('{', '}')) | |
1250 | del fields[-1] # Remove trailing '}'. | |
1251 | if token.token_type == tokenize.SYNTAX and token.name == '{': | |
1252 | next = self._GetNextToken() | |
1253 | new_type = ctor(token.start, token.end, name, fields, | |
1254 | self.namespace_stack) | |
1255 | # A name means this is an anonymous type and the name | |
1256 | # is the variable declaration. | |
1257 | if next.token_type != tokenize.NAME: | |
1258 | return new_type | |
1259 | name = new_type | |
1260 | token = next | |
1261 | ||
1262 | # Must be variable declaration using the type prefixed with keyword. | |
1263 | assert token.token_type == tokenize.NAME, token | |
1264 | return self._CreateVariable(token, token.name, name, [], '', None) | |
1265 | ||
1266 | def handle_struct(self): | |
1267 | # Special case the handling typedef/aliasing of structs here. | |
1268 | # It would be a pain to handle in the class code. | |
1269 | name_tokens, var_token = self.GetName() | |
1270 | if name_tokens: | |
1271 | next_token = self._GetNextToken() | |
1272 | is_syntax = (var_token.token_type == tokenize.SYNTAX and | |
1273 | var_token.name[0] in '*&') | |
1274 | is_variable = (var_token.token_type == tokenize.NAME and | |
1275 | next_token.name == ';') | |
1276 | variable = var_token | |
1277 | if is_syntax and not is_variable: | |
1278 | variable = next_token | |
1279 | temp = self._GetNextToken() | |
1280 | if temp.token_type == tokenize.SYNTAX and temp.name == '(': | |
1281 | # Handle methods declared to return a struct. | |
1282 | t0 = name_tokens[0] | |
1283 | struct = tokenize.Token(tokenize.NAME, 'struct', | |
1284 | t0.start-7, t0.start-2) | |
1285 | type_and_name = [struct] | |
1286 | type_and_name.extend(name_tokens) | |
1287 | type_and_name.extend((var_token, next_token)) | |
1288 | return self._GetMethod(type_and_name, 0, None, False) | |
1289 | assert temp.name == ';', (temp, name_tokens, var_token) | |
1290 | if is_syntax or (is_variable and not self._handling_typedef): | |
1291 | modifiers = ['struct'] | |
1292 | type_name = ''.join([t.name for t in name_tokens]) | |
1293 | position = name_tokens[0] | |
1294 | return self._CreateVariable(position, variable.name, type_name, | |
1295 | modifiers, var_token.name, None) | |
1296 | name_tokens.extend((var_token, next_token)) | |
1297 | self._AddBackTokens(name_tokens) | |
1298 | else: | |
1299 | self._AddBackToken(var_token) | |
1300 | return self._GetClass(Struct, VISIBILITY_PUBLIC, None) | |
1301 | ||
1302 | def handle_union(self): | |
1303 | return self._GetNestedType(Union) | |
1304 | ||
1305 | def handle_enum(self): | |
1306 | return self._GetNestedType(Enum) | |
1307 | ||
1308 | def handle_auto(self): | |
1309 | # TODO(nnorwitz): warn about using auto? Probably not since it | |
1310 | # will be reclaimed and useful for C++0x. | |
1311 | pass | |
1312 | ||
1313 | def handle_register(self): | |
1314 | pass | |
1315 | ||
1316 | def handle_const(self): | |
1317 | pass | |
1318 | ||
1319 | def handle_inline(self): | |
1320 | pass | |
1321 | ||
1322 | def handle_extern(self): | |
1323 | pass | |
1324 | ||
1325 | def handle_static(self): | |
1326 | pass | |
1327 | ||
1328 | def handle_virtual(self): | |
1329 | # What follows must be a method. | |
1330 | token = token2 = self._GetNextToken() | |
1331 | if token.name == 'inline': | |
1332 | # HACK(nnorwitz): handle inline dtors by ignoring 'inline'. | |
1333 | token2 = self._GetNextToken() | |
1334 | if token2.token_type == tokenize.SYNTAX and token2.name == '~': | |
1335 | return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None) | |
1336 | assert token.token_type == tokenize.NAME or token.name == '::', token | |
1337 | return_type_and_name, _ = self._GetVarTokensUpToIgnoringTemplates( | |
1338 | tokenize.SYNTAX, '(') # ) | |
1339 | return_type_and_name.insert(0, token) | |
1340 | if token2 is not token: | |
1341 | return_type_and_name.insert(1, token2) | |
1342 | return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL, | |
1343 | None, False) | |
1344 | ||
1345 | def handle_volatile(self): | |
1346 | pass | |
1347 | ||
1348 | def handle_mutable(self): | |
1349 | pass | |
1350 | ||
1351 | def handle_public(self): | |
1352 | assert self.in_class | |
1353 | self.visibility = VISIBILITY_PUBLIC | |
1354 | ||
1355 | def handle_protected(self): | |
1356 | assert self.in_class | |
1357 | self.visibility = VISIBILITY_PROTECTED | |
1358 | ||
1359 | def handle_private(self): | |
1360 | assert self.in_class | |
1361 | self.visibility = VISIBILITY_PRIVATE | |
1362 | ||
1363 | def handle_friend(self): | |
1364 | tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
1365 | assert tokens | |
1366 | t0 = tokens[0] | |
1367 | return Friend(t0.start, t0.end, tokens, self.namespace_stack) | |
1368 | ||
1369 | def handle_static_cast(self): | |
1370 | pass | |
1371 | ||
1372 | def handle_const_cast(self): | |
1373 | pass | |
1374 | ||
1375 | def handle_dynamic_cast(self): | |
1376 | pass | |
1377 | ||
1378 | def handle_reinterpret_cast(self): | |
1379 | pass | |
1380 | ||
1381 | def handle_new(self): | |
1382 | pass | |
1383 | ||
1384 | def handle_delete(self): | |
1385 | tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
1386 | assert tokens | |
1387 | return Delete(tokens[0].start, tokens[0].end, tokens) | |
1388 | ||
1389 | def handle_typedef(self): | |
1390 | token = self._GetNextToken() | |
1391 | if (token.token_type == tokenize.NAME and | |
1392 | keywords.IsKeyword(token.name)): | |
1393 | # Token must be struct/enum/union/class. | |
1394 | method = getattr(self, 'handle_' + token.name) | |
1395 | self._handling_typedef = True | |
1396 | tokens = [method()] | |
1397 | self._handling_typedef = False | |
1398 | else: | |
1399 | tokens = [token] | |
1400 | ||
1401 | # Get the remainder of the typedef up to the semi-colon. | |
1402 | tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';')) | |
1403 | ||
1404 | # TODO(nnorwitz): clean all this up. | |
1405 | assert tokens | |
1406 | name = tokens.pop() | |
1407 | indices = name | |
1408 | if tokens: | |
1409 | indices = tokens[0] | |
1410 | if not indices: | |
1411 | indices = token | |
1412 | if name.name == ')': | |
1413 | # HACK(nnorwitz): Handle pointers to functions "properly". | |
1414 | if (len(tokens) >= 4 and | |
1415 | tokens[1].name == '(' and tokens[2].name == '*'): | |
1416 | tokens.append(name) | |
1417 | name = tokens[3] | |
1418 | elif name.name == ']': | |
1419 | # HACK(nnorwitz): Handle arrays properly. | |
1420 | if len(tokens) >= 2: | |
1421 | tokens.append(name) | |
1422 | name = tokens[1] | |
1423 | new_type = tokens | |
1424 | if tokens and isinstance(tokens[0], tokenize.Token): | |
1425 | new_type = self.converter.ToType(tokens)[0] | |
1426 | return Typedef(indices.start, indices.end, name.name, | |
1427 | new_type, self.namespace_stack) | |
1428 | ||
1429 | def handle_typeid(self): | |
1430 | pass # Not needed yet. | |
1431 | ||
1432 | def handle_typename(self): | |
1433 | pass # Not needed yet. | |
1434 | ||
1435 | def _GetTemplatedTypes(self): | |
1436 | result = {} | |
1437 | tokens = list(self._GetMatchingChar('<', '>')) | |
1438 | len_tokens = len(tokens) - 1 # Ignore trailing '>'. | |
1439 | i = 0 | |
1440 | while i < len_tokens: | |
1441 | key = tokens[i].name | |
1442 | i += 1 | |
1443 | if keywords.IsKeyword(key) or key == ',': | |
1444 | continue | |
1445 | type_name = default = None | |
1446 | if i < len_tokens: | |
1447 | i += 1 | |
1448 | if tokens[i-1].name == '=': | |
1449 | assert i < len_tokens, '%s %s' % (i, tokens) | |
1450 | default, unused_next_token = self.GetName(tokens[i:]) | |
1451 | i += len(default) | |
1452 | else: | |
1453 | if tokens[i-1].name != ',': | |
1454 | # We got something like: Type variable. | |
1455 | # Re-adjust the key (variable) and type_name (Type). | |
1456 | key = tokens[i-1].name | |
1457 | type_name = tokens[i-2] | |
1458 | ||
1459 | result[key] = (type_name, default) | |
1460 | return result | |
1461 | ||
1462 | def handle_template(self): | |
1463 | token = self._GetNextToken() | |
1464 | assert token.token_type == tokenize.SYNTAX, token | |
1465 | assert token.name == '<', token | |
1466 | templated_types = self._GetTemplatedTypes() | |
1467 | # TODO(nnorwitz): for now, just ignore the template params. | |
1468 | token = self._GetNextToken() | |
1469 | if token.token_type == tokenize.NAME: | |
1470 | if token.name == 'class': | |
1471 | return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types) | |
1472 | elif token.name == 'struct': | |
1473 | return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types) | |
1474 | elif token.name == 'friend': | |
1475 | return self.handle_friend() | |
1476 | self._AddBackToken(token) | |
1477 | tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';') | |
1478 | tokens.append(last) | |
1479 | self._AddBackTokens(tokens) | |
1480 | if last.name == '(': | |
1481 | return self.GetMethod(FUNCTION_NONE, templated_types) | |
1482 | # Must be a variable definition. | |
1483 | return None | |
1484 | ||
1485 | def handle_true(self): | |
1486 | pass # Nothing to do. | |
1487 | ||
1488 | def handle_false(self): | |
1489 | pass # Nothing to do. | |
1490 | ||
1491 | def handle_asm(self): | |
1492 | pass # Not needed yet. | |
1493 | ||
1494 | def handle_class(self): | |
1495 | return self._GetClass(Class, VISIBILITY_PRIVATE, None) | |
1496 | ||
1497 | def _GetBases(self): | |
1498 | # Get base classes. | |
1499 | bases = [] | |
1500 | while 1: | |
1501 | token = self._GetNextToken() | |
1502 | assert token.token_type == tokenize.NAME, token | |
1503 | # TODO(nnorwitz): store kind of inheritance...maybe. | |
1504 | if token.name not in ('public', 'protected', 'private'): | |
1505 | # If inheritance type is not specified, it is private. | |
1506 | # Just put the token back so we can form a name. | |
1507 | # TODO(nnorwitz): it would be good to warn about this. | |
1508 | self._AddBackToken(token) | |
1509 | else: | |
1510 | # Check for virtual inheritance. | |
1511 | token = self._GetNextToken() | |
1512 | if token.name != 'virtual': | |
1513 | self._AddBackToken(token) | |
1514 | else: | |
1515 | # TODO(nnorwitz): store that we got virtual for this base. | |
1516 | pass | |
1517 | base, next_token = self.GetName() | |
1518 | bases_ast = self.converter.ToType(base) | |
1519 | assert len(bases_ast) == 1, bases_ast | |
1520 | bases.append(bases_ast[0]) | |
1521 | assert next_token.token_type == tokenize.SYNTAX, next_token | |
1522 | if next_token.name == '{': | |
1523 | token = next_token | |
1524 | break | |
1525 | # Support multiple inheritance. | |
1526 | assert next_token.name == ',', next_token | |
1527 | return bases, token | |
1528 | ||
1529 | def _GetClass(self, class_type, visibility, templated_types): | |
1530 | class_name = None | |
1531 | class_token = self._GetNextToken() | |
1532 | if class_token.token_type != tokenize.NAME: | |
1533 | assert class_token.token_type == tokenize.SYNTAX, class_token | |
1534 | token = class_token | |
1535 | else: | |
1536 | # Skip any macro (e.g. storage class specifiers) after the | |
1537 | # 'class' keyword. | |
1538 | next_token = self._GetNextToken() | |
1539 | if next_token.token_type == tokenize.NAME: | |
1540 | self._AddBackToken(next_token) | |
1541 | else: | |
1542 | self._AddBackTokens([class_token, next_token]) | |
1543 | name_tokens, token = self.GetName() | |
1544 | class_name = ''.join([t.name for t in name_tokens]) | |
1545 | bases = None | |
1546 | if token.token_type == tokenize.SYNTAX: | |
1547 | if token.name == ';': | |
1548 | # Forward declaration. | |
1549 | return class_type(class_token.start, class_token.end, | |
1550 | class_name, None, templated_types, None, | |
1551 | self.namespace_stack) | |
1552 | if token.name in '*&': | |
1553 | # Inline forward declaration. Could be method or data. | |
1554 | name_token = self._GetNextToken() | |
1555 | next_token = self._GetNextToken() | |
1556 | if next_token.name == ';': | |
1557 | # Handle data | |
1558 | modifiers = ['class'] | |
1559 | return self._CreateVariable(class_token, name_token.name, | |
1560 | class_name, | |
1561 | modifiers, token.name, None) | |
1562 | else: | |
1563 | # Assume this is a method. | |
1564 | tokens = (class_token, token, name_token, next_token) | |
1565 | self._AddBackTokens(tokens) | |
1566 | return self.GetMethod(FUNCTION_NONE, None) | |
1567 | if token.name == ':': | |
1568 | bases, token = self._GetBases() | |
1569 | ||
1570 | body = None | |
1571 | if token.token_type == tokenize.SYNTAX and token.name == '{': | |
1572 | assert token.token_type == tokenize.SYNTAX, token | |
1573 | assert token.name == '{', token | |
1574 | ||
1575 | ast = AstBuilder(self.GetScope(), self.filename, class_name, | |
1576 | visibility, self.namespace_stack) | |
1577 | body = list(ast.Generate()) | |
1578 | ||
1579 | if not self._handling_typedef: | |
1580 | token = self._GetNextToken() | |
1581 | if token.token_type != tokenize.NAME: | |
1582 | assert token.token_type == tokenize.SYNTAX, token | |
1583 | assert token.name == ';', token | |
1584 | else: | |
1585 | new_class = class_type(class_token.start, class_token.end, | |
1586 | class_name, bases, None, | |
1587 | body, self.namespace_stack) | |
1588 | ||
1589 | modifiers = [] | |
1590 | return self._CreateVariable(class_token, | |
1591 | token.name, new_class, | |
1592 | modifiers, token.name, None) | |
1593 | else: | |
1594 | if not self._handling_typedef: | |
1595 | self.HandleError('non-typedef token', token) | |
1596 | self._AddBackToken(token) | |
1597 | ||
1598 | return class_type(class_token.start, class_token.end, class_name, | |
1599 | bases, templated_types, body, self.namespace_stack) | |
1600 | ||
1601 | def handle_namespace(self): | |
1602 | # Support anonymous namespaces. | |
1603 | name = None | |
1604 | name_tokens, token = self.GetName() | |
1605 | if name_tokens: | |
1606 | name = ''.join([t.name for t in name_tokens]) | |
1607 | self.namespace_stack.append(name) | |
1608 | assert token.token_type == tokenize.SYNTAX, token | |
1609 | # Create an internal token that denotes when the namespace is complete. | |
1610 | internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP, | |
1611 | None, None) | |
1612 | internal_token.whence = token.whence | |
1613 | if token.name == '=': | |
1614 | # TODO(nnorwitz): handle aliasing namespaces. | |
1615 | name, next_token = self.GetName() | |
1616 | assert next_token.name == ';', next_token | |
1617 | self._AddBackToken(internal_token) | |
1618 | else: | |
1619 | assert token.name == '{', token | |
1620 | tokens = list(self.GetScope()) | |
1621 | # Replace the trailing } with the internal namespace pop token. | |
1622 | tokens[-1] = internal_token | |
1623 | # Handle namespace with nothing in it. | |
1624 | self._AddBackTokens(tokens) | |
1625 | return None | |
1626 | ||
1627 | def handle_using(self): | |
1628 | tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
1629 | assert tokens | |
1630 | return Using(tokens[0].start, tokens[0].end, tokens) | |
1631 | ||
1632 | def handle_explicit(self): | |
1633 | assert self.in_class | |
1634 | # Nothing much to do. | |
1635 | # TODO(nnorwitz): maybe verify the method name == class name. | |
1636 | # This must be a ctor. | |
1637 | return self.GetMethod(FUNCTION_CTOR, None) | |
1638 | ||
1639 | def handle_this(self): | |
1640 | pass # Nothing to do. | |
1641 | ||
1642 | def handle_operator(self): | |
1643 | # Pull off the next token(s?) and make that part of the method name. | |
1644 | pass | |
1645 | ||
1646 | def handle_sizeof(self): | |
1647 | pass | |
1648 | ||
1649 | def handle_case(self): | |
1650 | pass | |
1651 | ||
1652 | def handle_switch(self): | |
1653 | pass | |
1654 | ||
1655 | def handle_default(self): | |
1656 | token = self._GetNextToken() | |
1657 | assert token.token_type == tokenize.SYNTAX | |
1658 | assert token.name == ':' | |
1659 | ||
1660 | def handle_if(self): | |
1661 | pass | |
1662 | ||
1663 | def handle_else(self): | |
1664 | pass | |
1665 | ||
1666 | def handle_return(self): | |
1667 | tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
1668 | if not tokens: | |
1669 | return Return(self.current_token.start, self.current_token.end, None) | |
1670 | return Return(tokens[0].start, tokens[0].end, tokens) | |
1671 | ||
1672 | def handle_goto(self): | |
1673 | tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
1674 | assert len(tokens) == 1, str(tokens) | |
1675 | return Goto(tokens[0].start, tokens[0].end, tokens[0].name) | |
1676 | ||
1677 | def handle_try(self): | |
1678 | pass # Not needed yet. | |
1679 | ||
1680 | def handle_catch(self): | |
1681 | pass # Not needed yet. | |
1682 | ||
1683 | def handle_throw(self): | |
1684 | pass # Not needed yet. | |
1685 | ||
1686 | def handle_while(self): | |
1687 | pass | |
1688 | ||
1689 | def handle_do(self): | |
1690 | pass | |
1691 | ||
1692 | def handle_for(self): | |
1693 | pass | |
1694 | ||
1695 | def handle_break(self): | |
1696 | self._IgnoreUpTo(tokenize.SYNTAX, ';') | |
1697 | ||
1698 | def handle_continue(self): | |
1699 | self._IgnoreUpTo(tokenize.SYNTAX, ';') | |
1700 | ||
1701 | ||
1702 | def BuilderFromSource(source, filename): | |
1703 | """Utility method that returns an AstBuilder from source code. | |
1704 | ||
1705 | Args: | |
1706 | source: 'C++ source code' | |
1707 | filename: 'file1' | |
1708 | ||
1709 | Returns: | |
1710 | AstBuilder | |
1711 | """ | |
1712 | return AstBuilder(tokenize.GetTokens(source), filename) | |
1713 | ||
1714 | ||
1715 | def PrintIndentifiers(filename, should_print): | |
1716 | """Prints all identifiers for a C++ source file. | |
1717 | ||
1718 | Args: | |
1719 | filename: 'file1' | |
1720 | should_print: predicate with signature: bool Function(token) | |
1721 | """ | |
1722 | source = utils.ReadFile(filename, False) | |
1723 | if source is None: | |
1724 | sys.stderr.write('Unable to find: %s\n' % filename) | |
1725 | return | |
1726 | ||
1727 | #print('Processing %s' % actual_filename) | |
1728 | builder = BuilderFromSource(source, filename) | |
1729 | try: | |
1730 | for node in builder.Generate(): | |
1731 | if should_print(node): | |
1732 | print(node.name) | |
1733 | except KeyboardInterrupt: | |
1734 | return | |
1735 | except: | |
1736 | pass | |
1737 | ||
1738 | ||
1739 | def PrintAllIndentifiers(filenames, should_print): | |
1740 | """Prints all identifiers for each C++ source file in filenames. | |
1741 | ||
1742 | Args: | |
1743 | filenames: ['file1', 'file2', ...] | |
1744 | should_print: predicate with signature: bool Function(token) | |
1745 | """ | |
1746 | for path in filenames: | |
1747 | PrintIndentifiers(path, should_print) | |
1748 | ||
1749 | ||
1750 | def main(argv): | |
1751 | for filename in argv[1:]: | |
1752 | source = utils.ReadFile(filename) | |
1753 | if source is None: | |
1754 | continue | |
1755 | ||
1756 | print('Processing %s' % filename) | |
1757 | builder = BuilderFromSource(source, filename) | |
1758 | try: | |
1759 | entire_ast = filter(None, builder.Generate()) | |
1760 | except KeyboardInterrupt: | |
1761 | return | |
1762 | except: | |
1763 | # Already printed a warning, print the traceback and continue. | |
1764 | traceback.print_exc() | |
1765 | else: | |
1766 | if utils.DEBUG: | |
1767 | for ast in entire_ast: | |
1768 | print(ast) | |
1769 | ||
1770 | ||
1771 | if __name__ == '__main__': | |
1772 | main(sys.argv) |