]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | #!/usr/bin/env python |
2 | # | |
3 | # Copyright 2007 Neal Norwitz | |
4 | # Portions Copyright 2007 Google Inc. | |
5 | # | |
6 | # Licensed under the Apache License, Version 2.0 (the "License"); | |
7 | # you may not use this file except in compliance with the License. | |
8 | # You may obtain a copy of the License at | |
9 | # | |
10 | # http://www.apache.org/licenses/LICENSE-2.0 | |
11 | # | |
12 | # Unless required by applicable law or agreed to in writing, software | |
13 | # distributed under the License is distributed on an "AS IS" BASIS, | |
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
15 | # See the License for the specific language governing permissions and | |
16 | # limitations under the License. | |
17 | ||
18 | """Tokenize C++ source code.""" | |
19 | ||
20 | __author__ = 'nnorwitz@google.com (Neal Norwitz)' | |
21 | ||
22 | ||
23 | try: | |
24 | # Python 3.x | |
25 | import builtins | |
26 | except ImportError: | |
27 | # Python 2.x | |
28 | import __builtin__ as builtins | |
29 | ||
30 | ||
31 | import sys | |
32 | ||
33 | from cpp import utils | |
34 | ||
35 | ||
36 | if not hasattr(builtins, 'set'): | |
37 | # Nominal support for Python 2.3. | |
38 | from sets import Set as set | |
39 | ||
40 | ||
41 | # Add $ as a valid identifier char since so much code uses it. | |
42 | _letters = 'abcdefghijklmnopqrstuvwxyz' | |
43 | VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$') | |
44 | HEX_DIGITS = set('0123456789abcdefABCDEF') | |
45 | INT_OR_FLOAT_DIGITS = set('01234567890eE-+') | |
46 | ||
47 | ||
48 | # C++0x string preffixes. | |
49 | _STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR')) | |
50 | ||
51 | ||
52 | # Token types. | |
53 | UNKNOWN = 'UNKNOWN' | |
54 | SYNTAX = 'SYNTAX' | |
55 | CONSTANT = 'CONSTANT' | |
56 | NAME = 'NAME' | |
57 | PREPROCESSOR = 'PREPROCESSOR' | |
58 | ||
59 | # Where the token originated from. This can be used for backtracking. | |
60 | # It is always set to WHENCE_STREAM in this code. | |
61 | WHENCE_STREAM, WHENCE_QUEUE = range(2) | |
62 | ||
63 | ||
64 | class Token(object): | |
65 | """Data container to represent a C++ token. | |
66 | ||
67 | Tokens can be identifiers, syntax char(s), constants, or | |
68 | pre-processor directives. | |
69 | ||
70 | start contains the index of the first char of the token in the source | |
71 | end contains the index of the last char of the token in the source | |
72 | """ | |
73 | ||
74 | def __init__(self, token_type, name, start, end): | |
75 | self.token_type = token_type | |
76 | self.name = name | |
77 | self.start = start | |
78 | self.end = end | |
79 | self.whence = WHENCE_STREAM | |
80 | ||
81 | def __str__(self): | |
82 | if not utils.DEBUG: | |
83 | return 'Token(%r)' % self.name | |
84 | return 'Token(%r, %s, %s)' % (self.name, self.start, self.end) | |
85 | ||
86 | __repr__ = __str__ | |
87 | ||
88 | ||
89 | def _GetString(source, start, i): | |
90 | i = source.find('"', i+1) | |
91 | while source[i-1] == '\\': | |
92 | # Count the trailing backslashes. | |
93 | backslash_count = 1 | |
94 | j = i - 2 | |
95 | while source[j] == '\\': | |
96 | backslash_count += 1 | |
97 | j -= 1 | |
98 | # When trailing backslashes are even, they escape each other. | |
99 | if (backslash_count % 2) == 0: | |
100 | break | |
101 | i = source.find('"', i+1) | |
102 | return i + 1 | |
103 | ||
104 | ||
105 | def _GetChar(source, start, i): | |
106 | # NOTE(nnorwitz): may not be quite correct, should be good enough. | |
107 | i = source.find("'", i+1) | |
108 | while source[i-1] == '\\': | |
109 | # Need to special case '\\'. | |
110 | if (i - 2) > start and source[i-2] == '\\': | |
111 | break | |
112 | i = source.find("'", i+1) | |
113 | # Try to handle unterminated single quotes (in a #if 0 block). | |
114 | if i < 0: | |
115 | i = start | |
116 | return i + 1 | |
117 | ||
118 | ||
119 | def GetTokens(source): | |
120 | """Returns a sequence of Tokens. | |
121 | ||
122 | Args: | |
123 | source: string of C++ source code. | |
124 | ||
125 | Yields: | |
126 | Token that represents the next token in the source. | |
127 | """ | |
128 | # Cache various valid character sets for speed. | |
129 | valid_identifier_chars = VALID_IDENTIFIER_CHARS | |
130 | hex_digits = HEX_DIGITS | |
131 | int_or_float_digits = INT_OR_FLOAT_DIGITS | |
132 | int_or_float_digits2 = int_or_float_digits | set('.') | |
133 | ||
134 | # Only ignore errors while in a #if 0 block. | |
135 | ignore_errors = False | |
136 | count_ifs = 0 | |
137 | ||
138 | i = 0 | |
139 | end = len(source) | |
140 | while i < end: | |
141 | # Skip whitespace. | |
142 | while i < end and source[i].isspace(): | |
143 | i += 1 | |
144 | if i >= end: | |
145 | return | |
146 | ||
147 | token_type = UNKNOWN | |
148 | start = i | |
149 | c = source[i] | |
150 | if c.isalpha() or c == '_': # Find a string token. | |
151 | token_type = NAME | |
152 | while source[i] in valid_identifier_chars: | |
153 | i += 1 | |
154 | # String and character constants can look like a name if | |
155 | # they are something like L"". | |
156 | if (source[i] == "'" and (i - start) == 1 and | |
157 | source[start:i] in 'uUL'): | |
158 | # u, U, and L are valid C++0x character preffixes. | |
159 | token_type = CONSTANT | |
160 | i = _GetChar(source, start, i) | |
161 | elif source[i] == "'" and source[start:i] in _STR_PREFIXES: | |
162 | token_type = CONSTANT | |
163 | i = _GetString(source, start, i) | |
164 | elif c == '/' and source[i+1] == '/': # Find // comments. | |
165 | i = source.find('\n', i) | |
166 | if i == -1: # Handle EOF. | |
167 | i = end | |
168 | continue | |
169 | elif c == '/' and source[i+1] == '*': # Find /* comments. */ | |
170 | i = source.find('*/', i) + 2 | |
171 | continue | |
172 | elif c in ':+-<>&|*=': # : or :: (plus other chars). | |
173 | token_type = SYNTAX | |
174 | i += 1 | |
175 | new_ch = source[i] | |
176 | if new_ch == c and c != '>': # Treat ">>" as two tokens. | |
177 | i += 1 | |
178 | elif c == '-' and new_ch == '>': | |
179 | i += 1 | |
180 | elif new_ch == '=': | |
181 | i += 1 | |
182 | elif c in '()[]{}~!?^%;/.,': # Handle single char tokens. | |
183 | token_type = SYNTAX | |
184 | i += 1 | |
185 | if c == '.' and source[i].isdigit(): | |
186 | token_type = CONSTANT | |
187 | i += 1 | |
188 | while source[i] in int_or_float_digits: | |
189 | i += 1 | |
190 | # Handle float suffixes. | |
191 | for suffix in ('l', 'f'): | |
192 | if suffix == source[i:i+1].lower(): | |
193 | i += 1 | |
194 | break | |
195 | elif c.isdigit(): # Find integer. | |
196 | token_type = CONSTANT | |
197 | if c == '0' and source[i+1] in 'xX': | |
198 | # Handle hex digits. | |
199 | i += 2 | |
200 | while source[i] in hex_digits: | |
201 | i += 1 | |
202 | else: | |
203 | while source[i] in int_or_float_digits2: | |
204 | i += 1 | |
205 | # Handle integer (and float) suffixes. | |
206 | for suffix in ('ull', 'll', 'ul', 'l', 'f', 'u'): | |
207 | size = len(suffix) | |
208 | if suffix == source[i:i+size].lower(): | |
209 | i += size | |
210 | break | |
211 | elif c == '"': # Find string. | |
212 | token_type = CONSTANT | |
213 | i = _GetString(source, start, i) | |
214 | elif c == "'": # Find char. | |
215 | token_type = CONSTANT | |
216 | i = _GetChar(source, start, i) | |
217 | elif c == '#': # Find pre-processor command. | |
218 | token_type = PREPROCESSOR | |
219 | got_if = source[i:i+3] == '#if' and source[i+3:i+4].isspace() | |
220 | if got_if: | |
221 | count_ifs += 1 | |
222 | elif source[i:i+6] == '#endif': | |
223 | count_ifs -= 1 | |
224 | if count_ifs == 0: | |
225 | ignore_errors = False | |
226 | ||
227 | # TODO(nnorwitz): handle preprocessor statements (\ continuations). | |
228 | while 1: | |
229 | i1 = source.find('\n', i) | |
230 | i2 = source.find('//', i) | |
231 | i3 = source.find('/*', i) | |
232 | i4 = source.find('"', i) | |
233 | # NOTE(nnorwitz): doesn't handle comments in #define macros. | |
234 | # Get the first important symbol (newline, comment, EOF/end). | |
235 | i = min([x for x in (i1, i2, i3, i4, end) if x != -1]) | |
236 | ||
237 | # Handle #include "dir//foo.h" properly. | |
238 | if source[i] == '"': | |
239 | i = source.find('"', i+1) + 1 | |
240 | assert i > 0 | |
241 | continue | |
242 | # Keep going if end of the line and the line ends with \. | |
243 | if not (i == i1 and source[i-1] == '\\'): | |
244 | if got_if: | |
245 | condition = source[start+4:i].lstrip() | |
246 | if (condition.startswith('0') or | |
247 | condition.startswith('(0)')): | |
248 | ignore_errors = True | |
249 | break | |
250 | i += 1 | |
251 | elif c == '\\': # Handle \ in code. | |
252 | # This is different from the pre-processor \ handling. | |
253 | i += 1 | |
254 | continue | |
255 | elif ignore_errors: | |
256 | # The tokenizer seems to be in pretty good shape. This | |
257 | # raise is conditionally disabled so that bogus code | |
258 | # in an #if 0 block can be handled. Since we will ignore | |
259 | # it anyways, this is probably fine. So disable the | |
260 | # exception and return the bogus char. | |
261 | i += 1 | |
262 | else: | |
263 | sys.stderr.write('Got invalid token in %s @ %d token:%s: %r\n' % | |
264 | ('?', i, c, source[i-10:i+10])) | |
265 | raise RuntimeError('unexpected token') | |
266 | ||
267 | if i <= 0: | |
268 | print('Invalid index, exiting now.') | |
269 | return | |
270 | yield Token(token_type, source[start:i], start, i) | |
271 | ||
272 | ||
273 | if __name__ == '__main__': | |
274 | def main(argv): | |
275 | """Driver mostly for testing purposes.""" | |
276 | for filename in argv[1:]: | |
277 | source = utils.ReadFile(filename) | |
278 | if source is None: | |
279 | continue | |
280 | ||
281 | for token in GetTokens(source): | |
282 | print('%-12s: %s' % (token.token_type, token.name)) | |
283 | # print('\r%6.2f%%' % (100.0 * index / token.end),) | |
284 | sys.stdout.write('\n') | |
285 | ||
286 | ||
287 | main(sys.argv) |