2 # preprocess source file
4 # Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
23 from CLexer
import CLexer
24 from CParser
import CParser
27 from CodeFragment
import PP_Directive
28 from ParserWarning
import Warning
31 ##define T_CHAR_SPACE ' '
32 ##define T_CHAR_NULL '\0'
33 ##define T_CHAR_CR '\r'
34 ##define T_CHAR_TAB '\t'
35 ##define T_CHAR_LF '\n'
36 ##define T_CHAR_SLASH '/'
37 ##define T_CHAR_BACKSLASH '\\'
38 ##define T_CHAR_DOUBLE_QUOTE '\"'
39 ##define T_CHAR_SINGLE_QUOTE '\''
40 ##define T_CHAR_STAR '*'
41 ##define T_CHAR_HASH '#'
43 (T_CHAR_SPACE
, T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_TAB
, T_CHAR_LF
, T_CHAR_SLASH
, \
44 T_CHAR_BACKSLASH
, T_CHAR_DOUBLE_QUOTE
, T_CHAR_SINGLE_QUOTE
, T_CHAR_STAR
, T_CHAR_HASH
) = \
45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
47 SEPERATOR_TUPLE
= ('=', '|', ',', '{', '}')
49 (T_COMMENT_TWO_SLASH
, T_COMMENT_SLASH_STAR
) = (0, 1)
51 (T_PP_INCLUDE
, T_PP_DEFINE
, T_PP_OTHERS
) = (0, 1, 2)
53 ## The collector for source code fragments.
55 # PreprocessFile method should be called prior to ParseFile
57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
58 # Get*** procedures mean these procedures will make judgement on current token only.
60 class CodeFragmentCollector
:
63 # @param self The object pointer
64 # @param FileName The file that to be parsed
66 def __init__(self
, FileName
):
67 self
.Profile
= FileProfile
.FileProfile(FileName
)
68 self
.Profile
.FileLinesList
.append(T_CHAR_LF
)
69 self
.FileName
= FileName
70 self
.CurrentLineNumber
= 1
71 self
.CurrentOffsetWithinLine
= 0
74 self
.__SkippedChars
= ""
76 ## __IsWhiteSpace() method
78 # Whether char at current FileBufferPos is whitespace
80 # @param self The object pointer
81 # @param Char The char to test
82 # @retval True The char is a kind of white space
83 # @retval False The char is NOT a kind of white space
85 def __IsWhiteSpace(self
, Char
):
86 if Char
in (T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_SPACE
, T_CHAR_TAB
, T_CHAR_LF
):
91 ## __SkipWhiteSpace() method
93 # Skip white spaces from current char, return number of chars skipped
95 # @param self The object pointer
96 # @retval Count The number of chars skipped
98 def __SkipWhiteSpace(self
):
100 while not self
.__EndOfFile
():
102 if self
.__CurrentChar
() in (T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_LF
, T_CHAR_SPACE
, T_CHAR_TAB
):
103 self
.__SkippedChars
+= str(self
.__CurrentChar
())
110 ## __EndOfFile() method
112 # Judge current buffer pos is at file end
114 # @param self The object pointer
115 # @retval True Current File buffer position is at file end
116 # @retval False Current File buffer position is NOT at file end
118 def __EndOfFile(self
):
119 NumberOfLines
= len(self
.Profile
.FileLinesList
)
120 SizeOfLastLine
= len(self
.Profile
.FileLinesList
[-1])
121 if self
.CurrentLineNumber
== NumberOfLines
and self
.CurrentOffsetWithinLine
>= SizeOfLastLine
- 1:
123 elif self
.CurrentLineNumber
> NumberOfLines
:
128 ## __EndOfLine() method
130 # Judge current buffer pos is at line end
132 # @param self The object pointer
133 # @retval True Current File buffer position is at line end
134 # @retval False Current File buffer position is NOT at line end
136 def __EndOfLine(self
):
137 SizeOfCurrentLine
= len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1])
138 if self
.CurrentOffsetWithinLine
>= SizeOfCurrentLine
- 1:
145 # Reset file data buffer to the initial state
147 # @param self The object pointer
150 self
.CurrentLineNumber
= 1
151 self
.CurrentOffsetWithinLine
= 0
153 ## __UndoOneChar() method
155 # Go back one char in the file buffer
157 # @param self The object pointer
158 # @retval True Successfully go back one char
159 # @retval False Not able to go back one char as file beginning reached
161 def __UndoOneChar(self
):
163 if self
.CurrentLineNumber
== 1 and self
.CurrentOffsetWithinLine
== 0:
165 elif self
.CurrentOffsetWithinLine
== 0:
166 self
.CurrentLineNumber
-= 1
167 self
.CurrentOffsetWithinLine
= len(self
.__CurrentLine
()) - 1
169 self
.CurrentOffsetWithinLine
-= 1
172 ## __GetOneChar() method
174 # Move forward one char in the file buffer
176 # @param self The object pointer
178 def __GetOneChar(self
):
179 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
180 self
.CurrentLineNumber
+= 1
181 self
.CurrentOffsetWithinLine
= 0
183 self
.CurrentOffsetWithinLine
+= 1
185 ## __CurrentChar() method
187 # Get the char pointed to by the file buffer pointer
189 # @param self The object pointer
190 # @retval Char Current char
192 def __CurrentChar(self
):
193 CurrentChar
= self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
]
197 ## __NextChar() method
199 # Get the one char pass the char pointed to by the file buffer pointer
201 # @param self The object pointer
202 # @retval Char Next char
204 def __NextChar(self
):
205 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
206 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
][0]
208 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
+ 1]
210 ## __SetCurrentCharValue() method
212 # Modify the value of current char
214 # @param self The object pointer
215 # @param Value The new value of current char
217 def __SetCurrentCharValue(self
, Value
):
218 self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
] = Value
220 ## __SetCharValue() method
222 # Modify the value of current char
224 # @param self The object pointer
225 # @param Value The new value of current char
227 def __SetCharValue(self
, Line
, Offset
, Value
):
228 self
.Profile
.FileLinesList
[Line
- 1][Offset
] = Value
230 ## __CurrentLine() method
232 # Get the list that contains current line contents
234 # @param self The object pointer
235 # @retval List current line contents
237 def __CurrentLine(self
):
238 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]
240 ## __InsertComma() method
242 # Insert ',' to replace PP
244 # @param self The object pointer
245 # @retval List current line contents
247 def __InsertComma(self
, Line
):
250 if self
.Profile
.FileLinesList
[Line
- 1][0] != T_CHAR_HASH
:
251 BeforeHashPart
= str(self
.Profile
.FileLinesList
[Line
- 1]).split(T_CHAR_HASH
)[0]
252 if BeforeHashPart
.rstrip().endswith(T_CHAR_COMMA
) or BeforeHashPart
.rstrip().endswith(';'):
255 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(','):
258 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(';'):
261 if str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(',') or str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(';'):
264 self
.Profile
.FileLinesList
[Line
- 1].insert(self
.CurrentOffsetWithinLine
, ',')
266 ## PreprocessFileWithClear() method
268 # Run a preprocess for the file to clean all comments
270 # @param self The object pointer
272 def PreprocessFileWithClear(self
):
276 DoubleSlashComment
= False
279 PPDirectiveObj
= None
280 # HashComment in quoted string " " is ignored.
282 InCharLiteral
= False
284 self
.Profile
.FileLinesList
= [list(s
) for s
in self
.Profile
.FileLinesListFromFile
]
285 while not self
.__EndOfFile
():
287 if not InComment
and self
.__CurrentChar
() == T_CHAR_DOUBLE_QUOTE
:
288 InString
= not InString
290 if not InComment
and self
.__CurrentChar
() == T_CHAR_SINGLE_QUOTE
:
291 InCharLiteral
= not InCharLiteral
292 # meet new line, then no longer in a comment for // and '#'
293 if self
.__CurrentChar
() == T_CHAR_LF
:
294 if HashComment
and PPDirectiveObj
!= None:
295 if PPDirectiveObj
.Content
.rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
296 PPDirectiveObj
.Content
+= T_CHAR_LF
301 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
303 if InComment
and DoubleSlashComment
:
305 DoubleSlashComment
= False
307 if InComment
and HashComment
and not PPExtend
:
310 PPDirectiveObj
.Content
+= T_CHAR_LF
311 PPDirectiveObj
.EndPos
= EndLinePos
312 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
313 PPDirectiveObj
= None
315 if InString
or InCharLiteral
:
316 CurrentLine
= "".join(self
.__CurrentLine
())
317 if CurrentLine
.rstrip(T_CHAR_LF
).rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
318 SlashIndex
= CurrentLine
.rindex(T_CHAR_BACKSLASH
)
319 self
.__SetCharValue
(self
.CurrentLineNumber
, SlashIndex
, T_CHAR_SPACE
)
321 self
.CurrentLineNumber
+= 1
322 self
.CurrentOffsetWithinLine
= 0
323 # check for */ comment end
324 elif InComment
and not DoubleSlashComment
and not HashComment
and self
.__CurrentChar
() == T_CHAR_STAR
and self
.__NextChar
() == T_CHAR_SLASH
:
326 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
328 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
331 # set comments to spaces
334 # // follows hash PP directive
335 if self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
338 PPDirectiveObj
.EndPos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
- 1)
339 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
340 PPDirectiveObj
= None
343 PPDirectiveObj
.Content
+= self
.__CurrentChar
()
345 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
347 # check for // comment
348 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
350 DoubleSlashComment
= True
352 # check for '#' comment
353 elif self
.__CurrentChar
() == T_CHAR_HASH
and not InString
and not InCharLiteral
:
356 PPDirectiveObj
= PP_Directive('', (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
), None)
357 # check for /* comment start
358 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_STAR
:
360 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
362 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
368 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
370 if InComment
and HashComment
and not PPExtend
:
371 PPDirectiveObj
.EndPos
= EndLinePos
372 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
375 ## ParseFile() method
377 # Parse the file profile buffer to extract fd, fv ... information
378 # Exception will be raised if syntax error found
380 # @param self The object pointer
383 self
.PreprocessFileWithClear()
384 # restore from ListOfList to ListOfString
385 self
.Profile
.FileLinesList
= ["".join(list) for list in self
.Profile
.FileLinesList
]
386 FileStringContents
= ''
387 for fileLine
in self
.Profile
.FileLinesList
:
388 FileStringContents
+= fileLine
389 cStream
= antlr3
.StringStream(FileStringContents
)
390 lexer
= CLexer(cStream
)
391 tStream
= antlr3
.CommonTokenStream(lexer
)
392 parser
= CParser(tStream
)
393 parser
.translation_unit()
395 ## CleanFileProfileBuffer() method
397 # Reset all contents of the profile of a file
399 def CleanFileProfileBuffer(self
):
401 FileProfile
.PPDirectiveList
= []
402 FileProfile
.AssignmentExpressionList
= []
403 FileProfile
.FunctionDefinitionList
= []
404 FileProfile
.VariableDeclarationList
= []
405 FileProfile
.EnumerationDefinitionList
= []
406 FileProfile
.StructUnionDefinitionList
= []
407 FileProfile
.TypedefDefinitionList
= []
408 FileProfile
.FunctionCallingList
= []
410 ## PrintFragments() method
412 # Print the contents of the profile of a file
414 def PrintFragments(self
):
416 print '################# ' + self
.FileName
+ '#####################'
418 print '/****************************************/'
419 print '/*************** ASSIGNMENTS ***************/'
420 print '/****************************************/'
421 for asign
in FileProfile
.AssignmentExpressionList
:
422 print str(asign
.StartPos
) + asign
.Name
+ asign
.Operator
+ asign
.Value
424 print '/****************************************/'
425 print '/********* PREPROCESS DIRECTIVES ********/'
426 print '/****************************************/'
427 for pp
in FileProfile
.PPDirectiveList
:
428 print str(pp
.StartPos
) + pp
.Content
430 print '/****************************************/'
431 print '/********* VARIABLE DECLARATIONS ********/'
432 print '/****************************************/'
433 for var
in FileProfile
.VariableDeclarationList
:
434 print str(var
.StartPos
) + var
.Modifier
+ ' '+ var
.Declarator
436 print '/****************************************/'
437 print '/********* FUNCTION DEFINITIONS *********/'
438 print '/****************************************/'
439 for func
in FileProfile
.FunctionDefinitionList
:
440 print str(func
.StartPos
) + func
.Modifier
+ ' '+ func
.Declarator
+ ' ' + str(func
.NamePos
)
442 print '/****************************************/'
443 print '/************ ENUMERATIONS **************/'
444 print '/****************************************/'
445 for enum
in FileProfile
.EnumerationDefinitionList
:
446 print str(enum
.StartPos
) + enum
.Content
448 print '/****************************************/'
449 print '/*********** STRUCTS/UNIONS *************/'
450 print '/****************************************/'
451 for su
in FileProfile
.StructUnionDefinitionList
:
452 print str(su
.StartPos
) + su
.Content
454 print '/****************************************/'
455 print '/************** TYPEDEFS ****************/'
456 print '/****************************************/'
457 for typedef
in FileProfile
.TypedefDefinitionList
:
458 print str(typedef
.StartPos
) + typedef
.ToType
462 # This acts like the main() function for the script, unless it is 'import'ed into another
465 if __name__
== "__main__":