2 # preprocess source file
4 # Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
19 import Common
.LongFilePathOs
as os
23 from CLexer
import CLexer
24 from CParser
import CParser
27 from CodeFragment
import PP_Directive
28 from ParserWarning
import Warning
31 ##define T_CHAR_SPACE ' '
32 ##define T_CHAR_NULL '\0'
33 ##define T_CHAR_CR '\r'
34 ##define T_CHAR_TAB '\t'
35 ##define T_CHAR_LF '\n'
36 ##define T_CHAR_SLASH '/'
37 ##define T_CHAR_BACKSLASH '\\'
38 ##define T_CHAR_DOUBLE_QUOTE '\"'
39 ##define T_CHAR_SINGLE_QUOTE '\''
40 ##define T_CHAR_STAR '*'
41 ##define T_CHAR_HASH '#'
43 (T_CHAR_SPACE
, T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_TAB
, T_CHAR_LF
, T_CHAR_SLASH
, \
44 T_CHAR_BACKSLASH
, T_CHAR_DOUBLE_QUOTE
, T_CHAR_SINGLE_QUOTE
, T_CHAR_STAR
, T_CHAR_HASH
) = \
45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
47 SEPERATOR_TUPLE
= ('=', '|', ',', '{', '}')
49 (T_COMMENT_TWO_SLASH
, T_COMMENT_SLASH_STAR
) = (0, 1)
51 (T_PP_INCLUDE
, T_PP_DEFINE
, T_PP_OTHERS
) = (0, 1, 2)
53 ## The collector for source code fragments.
55 # PreprocessFile method should be called prior to ParseFile
57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
58 # Get*** procedures mean these procedures will make judgement on current token only.
60 class CodeFragmentCollector
:
63 # @param self The object pointer
64 # @param FileName The file that to be parsed
66 def __init__(self
, FileName
):
67 self
.Profile
= FileProfile
.FileProfile(FileName
)
68 self
.Profile
.FileLinesList
.append(T_CHAR_LF
)
69 self
.FileName
= FileName
70 self
.CurrentLineNumber
= 1
71 self
.CurrentOffsetWithinLine
= 0
74 self
.__SkippedChars
= ""
76 ## __IsWhiteSpace() method
78 # Whether char at current FileBufferPos is whitespace
80 # @param self The object pointer
81 # @param Char The char to test
82 # @retval True The char is a kind of white space
83 # @retval False The char is NOT a kind of white space
85 def __IsWhiteSpace(self
, Char
):
86 if Char
in (T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_SPACE
, T_CHAR_TAB
, T_CHAR_LF
):
91 ## __SkipWhiteSpace() method
93 # Skip white spaces from current char, return number of chars skipped
95 # @param self The object pointer
96 # @retval Count The number of chars skipped
98 def __SkipWhiteSpace(self
):
100 while not self
.__EndOfFile
():
102 if self
.__CurrentChar
() in (T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_LF
, T_CHAR_SPACE
, T_CHAR_TAB
):
103 self
.__SkippedChars
+= str(self
.__CurrentChar
())
110 ## __EndOfFile() method
112 # Judge current buffer pos is at file end
114 # @param self The object pointer
115 # @retval True Current File buffer position is at file end
116 # @retval False Current File buffer position is NOT at file end
118 def __EndOfFile(self
):
119 NumberOfLines
= len(self
.Profile
.FileLinesList
)
120 SizeOfLastLine
= len(self
.Profile
.FileLinesList
[-1])
121 if self
.CurrentLineNumber
== NumberOfLines
and self
.CurrentOffsetWithinLine
>= SizeOfLastLine
- 1:
123 elif self
.CurrentLineNumber
> NumberOfLines
:
128 ## __EndOfLine() method
130 # Judge current buffer pos is at line end
132 # @param self The object pointer
133 # @retval True Current File buffer position is at line end
134 # @retval False Current File buffer position is NOT at line end
136 def __EndOfLine(self
):
137 SizeOfCurrentLine
= len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1])
138 if self
.CurrentOffsetWithinLine
>= SizeOfCurrentLine
- 1:
145 # Reset file data buffer to the initial state
147 # @param self The object pointer
150 self
.CurrentLineNumber
= 1
151 self
.CurrentOffsetWithinLine
= 0
153 ## __UndoOneChar() method
155 # Go back one char in the file buffer
157 # @param self The object pointer
158 # @retval True Successfully go back one char
159 # @retval False Not able to go back one char as file beginning reached
161 def __UndoOneChar(self
):
163 if self
.CurrentLineNumber
== 1 and self
.CurrentOffsetWithinLine
== 0:
165 elif self
.CurrentOffsetWithinLine
== 0:
166 self
.CurrentLineNumber
-= 1
167 self
.CurrentOffsetWithinLine
= len(self
.__CurrentLine
()) - 1
169 self
.CurrentOffsetWithinLine
-= 1
172 ## __GetOneChar() method
174 # Move forward one char in the file buffer
176 # @param self The object pointer
178 def __GetOneChar(self
):
179 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
180 self
.CurrentLineNumber
+= 1
181 self
.CurrentOffsetWithinLine
= 0
183 self
.CurrentOffsetWithinLine
+= 1
185 ## __CurrentChar() method
187 # Get the char pointed to by the file buffer pointer
189 # @param self The object pointer
190 # @retval Char Current char
192 def __CurrentChar(self
):
193 CurrentChar
= self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
]
197 ## __NextChar() method
199 # Get the one char pass the char pointed to by the file buffer pointer
201 # @param self The object pointer
202 # @retval Char Next char
204 def __NextChar(self
):
205 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
206 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
][0]
208 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
+ 1]
210 ## __SetCurrentCharValue() method
212 # Modify the value of current char
214 # @param self The object pointer
215 # @param Value The new value of current char
217 def __SetCurrentCharValue(self
, Value
):
218 self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
] = Value
220 ## __SetCharValue() method
222 # Modify the value of current char
224 # @param self The object pointer
225 # @param Value The new value of current char
227 def __SetCharValue(self
, Line
, Offset
, Value
):
228 self
.Profile
.FileLinesList
[Line
- 1][Offset
] = Value
230 ## __CurrentLine() method
232 # Get the list that contains current line contents
234 # @param self The object pointer
235 # @retval List current line contents
237 def __CurrentLine(self
):
238 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]
240 ## __InsertComma() method
242 # Insert ',' to replace PP
244 # @param self The object pointer
245 # @retval List current line contents
247 def __InsertComma(self
, Line
):
250 if self
.Profile
.FileLinesList
[Line
- 1][0] != T_CHAR_HASH
:
251 BeforeHashPart
= str(self
.Profile
.FileLinesList
[Line
- 1]).split(T_CHAR_HASH
)[0]
252 if BeforeHashPart
.rstrip().endswith(T_CHAR_COMMA
) or BeforeHashPart
.rstrip().endswith(';'):
255 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(','):
258 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(';'):
261 if str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(',') or str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(';'):
264 self
.Profile
.FileLinesList
[Line
- 1].insert(self
.CurrentOffsetWithinLine
, ',')
266 ## PreprocessFileWithClear() method
268 # Run a preprocess for the file to clean all comments
270 # @param self The object pointer
272 def PreprocessFileWithClear(self
):
276 DoubleSlashComment
= False
279 PPDirectiveObj
= None
280 # HashComment in quoted string " " is ignored.
282 InCharLiteral
= False
284 self
.Profile
.FileLinesList
= [list(s
) for s
in self
.Profile
.FileLinesListFromFile
]
285 while not self
.__EndOfFile
():
287 if not InComment
and self
.__CurrentChar
() == T_CHAR_DOUBLE_QUOTE
:
288 InString
= not InString
290 if not InComment
and self
.__CurrentChar
() == T_CHAR_SINGLE_QUOTE
:
291 InCharLiteral
= not InCharLiteral
292 # meet new line, then no longer in a comment for // and '#'
293 if self
.__CurrentChar
() == T_CHAR_LF
:
294 if HashComment
and PPDirectiveObj
!= None:
295 if PPDirectiveObj
.Content
.rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
296 PPDirectiveObj
.Content
+= T_CHAR_LF
301 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
303 if InComment
and DoubleSlashComment
:
305 DoubleSlashComment
= False
307 if InComment
and HashComment
and not PPExtend
:
310 PPDirectiveObj
.Content
+= T_CHAR_LF
311 PPDirectiveObj
.EndPos
= EndLinePos
312 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
313 PPDirectiveObj
= None
315 if InString
or InCharLiteral
:
316 CurrentLine
= "".join(self
.__CurrentLine
())
317 if CurrentLine
.rstrip(T_CHAR_LF
).rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
318 SlashIndex
= CurrentLine
.rindex(T_CHAR_BACKSLASH
)
319 self
.__SetCharValue
(self
.CurrentLineNumber
, SlashIndex
, T_CHAR_SPACE
)
321 self
.CurrentLineNumber
+= 1
322 self
.CurrentOffsetWithinLine
= 0
323 # check for */ comment end
324 elif InComment
and not DoubleSlashComment
and not HashComment
and self
.__CurrentChar
() == T_CHAR_STAR
and self
.__NextChar
() == T_CHAR_SLASH
:
326 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
328 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
331 # set comments to spaces
334 # // follows hash PP directive
335 if self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
338 PPDirectiveObj
.EndPos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
- 1)
339 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
340 PPDirectiveObj
= None
343 PPDirectiveObj
.Content
+= self
.__CurrentChar
()
345 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
347 # check for // comment
348 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
350 DoubleSlashComment
= True
352 # check for '#' comment
353 elif self
.__CurrentChar
() == T_CHAR_HASH
and not InString
and not InCharLiteral
:
356 PPDirectiveObj
= PP_Directive('', (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
), None)
357 # check for /* comment start
358 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_STAR
:
360 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
362 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
368 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
370 if InComment
and HashComment
and not PPExtend
:
371 PPDirectiveObj
.EndPos
= EndLinePos
372 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
375 ## ParseFile() method
377 # Parse the file profile buffer to extract fd, fv ... information
378 # Exception will be raised if syntax error found
380 # @param self The object pointer
383 self
.PreprocessFileWithClear()
384 # restore from ListOfList to ListOfString
385 self
.Profile
.FileLinesList
= ["".join(list) for list in self
.Profile
.FileLinesList
]
386 FileStringContents
= ''
387 for fileLine
in self
.Profile
.FileLinesList
:
388 FileStringContents
+= fileLine
389 cStream
= antlr3
.StringStream(FileStringContents
)
390 lexer
= CLexer(cStream
)
391 tStream
= antlr3
.CommonTokenStream(lexer
)
392 parser
= CParser(tStream
)
393 parser
.translation_unit()
395 ## CleanFileProfileBuffer() method
397 # Reset all contents of the profile of a file
399 def CleanFileProfileBuffer(self
):
401 FileProfile
.PPDirectiveList
= []
402 FileProfile
.AssignmentExpressionList
= []
403 FileProfile
.FunctionDefinitionList
= []
404 FileProfile
.VariableDeclarationList
= []
405 FileProfile
.EnumerationDefinitionList
= []
406 FileProfile
.StructUnionDefinitionList
= []
407 FileProfile
.TypedefDefinitionList
= []
408 FileProfile
.FunctionCallingList
= []
410 ## PrintFragments() method
412 # Print the contents of the profile of a file
414 def PrintFragments(self
):
416 print '################# ' + self
.FileName
+ '#####################'
418 print '/****************************************/'
419 print '/*************** ASSIGNMENTS ***************/'
420 print '/****************************************/'
421 for asign
in FileProfile
.AssignmentExpressionList
:
422 print str(asign
.StartPos
) + asign
.Name
+ asign
.Operator
+ asign
.Value
424 print '/****************************************/'
425 print '/********* PREPROCESS DIRECTIVES ********/'
426 print '/****************************************/'
427 for pp
in FileProfile
.PPDirectiveList
:
428 print str(pp
.StartPos
) + pp
.Content
430 print '/****************************************/'
431 print '/********* VARIABLE DECLARATIONS ********/'
432 print '/****************************************/'
433 for var
in FileProfile
.VariableDeclarationList
:
434 print str(var
.StartPos
) + var
.Modifier
+ ' '+ var
.Declarator
436 print '/****************************************/'
437 print '/********* FUNCTION DEFINITIONS *********/'
438 print '/****************************************/'
439 for func
in FileProfile
.FunctionDefinitionList
:
440 print str(func
.StartPos
) + func
.Modifier
+ ' '+ func
.Declarator
+ ' ' + str(func
.NamePos
)
442 print '/****************************************/'
443 print '/************ ENUMERATIONS **************/'
444 print '/****************************************/'
445 for enum
in FileProfile
.EnumerationDefinitionList
:
446 print str(enum
.StartPos
) + enum
.Content
448 print '/****************************************/'
449 print '/*********** STRUCTS/UNIONS *************/'
450 print '/****************************************/'
451 for su
in FileProfile
.StructUnionDefinitionList
:
452 print str(su
.StartPos
) + su
.Content
454 print '/****************************************/'
455 print '/************** TYPEDEFS ****************/'
456 print '/****************************************/'
457 for typedef
in FileProfile
.TypedefDefinitionList
:
458 print str(typedef
.StartPos
) + typedef
.ToType
462 # This acts like the main() function for the script, unless it is 'import'ed into another
465 if __name__
== "__main__":