2 # preprocess source file
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
19 import Common
.LongFilePathOs
as os
23 from CLexer
import CLexer
24 from CParser
import CParser
27 from CodeFragment
import PP_Directive
28 from ParserWarning
import Warning
31 ##define T_CHAR_SPACE ' '
32 ##define T_CHAR_NULL '\0'
33 ##define T_CHAR_CR '\r'
34 ##define T_CHAR_TAB '\t'
35 ##define T_CHAR_LF '\n'
36 ##define T_CHAR_SLASH '/'
37 ##define T_CHAR_BACKSLASH '\\'
38 ##define T_CHAR_DOUBLE_QUOTE '\"'
39 ##define T_CHAR_SINGLE_QUOTE '\''
40 ##define T_CHAR_STAR '*'
41 ##define T_CHAR_HASH '#'
43 (T_CHAR_SPACE
, T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_TAB
, T_CHAR_LF
, T_CHAR_SLASH
, \
44 T_CHAR_BACKSLASH
, T_CHAR_DOUBLE_QUOTE
, T_CHAR_SINGLE_QUOTE
, T_CHAR_STAR
, T_CHAR_HASH
) = \
45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
47 SEPERATOR_TUPLE
= ('=', '|', ',', '{', '}')
49 (T_COMMENT_TWO_SLASH
, T_COMMENT_SLASH_STAR
) = (0, 1)
51 (T_PP_INCLUDE
, T_PP_DEFINE
, T_PP_OTHERS
) = (0, 1, 2)
53 ## The collector for source code fragments.
55 # PreprocessFile method should be called prior to ParseFile
57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
58 # Get*** procedures mean these procedures will make judgement on current token only.
60 class CodeFragmentCollector
:
63 # @param self The object pointer
64 # @param FileName The file that to be parsed
66 def __init__(self
, FileName
):
67 self
.Profile
= FileProfile
.FileProfile(FileName
)
68 self
.Profile
.FileLinesList
.append(T_CHAR_LF
)
69 self
.FileName
= FileName
70 self
.CurrentLineNumber
= 1
71 self
.CurrentOffsetWithinLine
= 0
74 self
.__SkippedChars
= ""
76 ## __SkipWhiteSpace() method
78 # Skip white spaces from current char, return number of chars skipped
80 # @param self The object pointer
81 # @retval Count The number of chars skipped
83 def __SkipWhiteSpace(self
):
85 while not self
.__EndOfFile
():
87 if self
.__CurrentChar
() in (T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_LF
, T_CHAR_SPACE
, T_CHAR_TAB
):
88 self
.__SkippedChars
+= str(self
.__CurrentChar
())
95 ## __EndOfFile() method
97 # Judge current buffer pos is at file end
99 # @param self The object pointer
100 # @retval True Current File buffer position is at file end
101 # @retval False Current File buffer position is NOT at file end
103 def __EndOfFile(self
):
104 NumberOfLines
= len(self
.Profile
.FileLinesList
)
105 SizeOfLastLine
= len(self
.Profile
.FileLinesList
[-1])
106 if self
.CurrentLineNumber
== NumberOfLines
and self
.CurrentOffsetWithinLine
>= SizeOfLastLine
- 1:
108 elif self
.CurrentLineNumber
> NumberOfLines
:
113 ## __EndOfLine() method
115 # Judge current buffer pos is at line end
117 # @param self The object pointer
118 # @retval True Current File buffer position is at line end
119 # @retval False Current File buffer position is NOT at line end
121 def __EndOfLine(self
):
122 SizeOfCurrentLine
= len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1])
123 if self
.CurrentOffsetWithinLine
>= SizeOfCurrentLine
- 1:
130 # Reset file data buffer to the initial state
132 # @param self The object pointer
135 self
.CurrentLineNumber
= 1
136 self
.CurrentOffsetWithinLine
= 0
138 ## __UndoOneChar() method
140 # Go back one char in the file buffer
142 # @param self The object pointer
143 # @retval True Successfully go back one char
144 # @retval False Not able to go back one char as file beginning reached
146 def __UndoOneChar(self
):
148 if self
.CurrentLineNumber
== 1 and self
.CurrentOffsetWithinLine
== 0:
150 elif self
.CurrentOffsetWithinLine
== 0:
151 self
.CurrentLineNumber
-= 1
152 self
.CurrentOffsetWithinLine
= len(self
.__CurrentLine
()) - 1
154 self
.CurrentOffsetWithinLine
-= 1
157 ## __GetOneChar() method
159 # Move forward one char in the file buffer
161 # @param self The object pointer
163 def __GetOneChar(self
):
164 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
165 self
.CurrentLineNumber
+= 1
166 self
.CurrentOffsetWithinLine
= 0
168 self
.CurrentOffsetWithinLine
+= 1
170 ## __CurrentChar() method
172 # Get the char pointed to by the file buffer pointer
174 # @param self The object pointer
175 # @retval Char Current char
177 def __CurrentChar(self
):
178 CurrentChar
= self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
]
182 ## __NextChar() method
184 # Get the one char pass the char pointed to by the file buffer pointer
186 # @param self The object pointer
187 # @retval Char Next char
189 def __NextChar(self
):
190 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
191 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
][0]
193 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
+ 1]
195 ## __SetCurrentCharValue() method
197 # Modify the value of current char
199 # @param self The object pointer
200 # @param Value The new value of current char
202 def __SetCurrentCharValue(self
, Value
):
203 self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
] = Value
205 ## __SetCharValue() method
207 # Modify the value of current char
209 # @param self The object pointer
210 # @param Value The new value of current char
212 def __SetCharValue(self
, Line
, Offset
, Value
):
213 self
.Profile
.FileLinesList
[Line
- 1][Offset
] = Value
215 ## __CurrentLine() method
217 # Get the list that contains current line contents
219 # @param self The object pointer
220 # @retval List current line contents
222 def __CurrentLine(self
):
223 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]
225 ## __InsertComma() method
227 # Insert ',' to replace PP
229 # @param self The object pointer
230 # @retval List current line contents
232 def __InsertComma(self
, Line
):
235 if self
.Profile
.FileLinesList
[Line
- 1][0] != T_CHAR_HASH
:
236 BeforeHashPart
= str(self
.Profile
.FileLinesList
[Line
- 1]).split(T_CHAR_HASH
)[0]
237 if BeforeHashPart
.rstrip().endswith(T_CHAR_COMMA
) or BeforeHashPart
.rstrip().endswith(';'):
240 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(','):
243 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(';'):
246 if str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(',') or str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(';'):
249 self
.Profile
.FileLinesList
[Line
- 1].insert(self
.CurrentOffsetWithinLine
, ',')
251 ## PreprocessFileWithClear() method
253 # Run a preprocess for the file to clean all comments
255 # @param self The object pointer
257 def PreprocessFileWithClear(self
):
261 DoubleSlashComment
= False
264 PPDirectiveObj
= None
265 # HashComment in quoted string " " is ignored.
267 InCharLiteral
= False
269 self
.Profile
.FileLinesList
= [list(s
) for s
in self
.Profile
.FileLinesListFromFile
]
270 while not self
.__EndOfFile
():
272 if not InComment
and self
.__CurrentChar
() == T_CHAR_DOUBLE_QUOTE
:
273 InString
= not InString
275 if not InComment
and self
.__CurrentChar
() == T_CHAR_SINGLE_QUOTE
:
276 InCharLiteral
= not InCharLiteral
277 # meet new line, then no longer in a comment for // and '#'
278 if self
.__CurrentChar
() == T_CHAR_LF
:
279 if HashComment
and PPDirectiveObj
is not None:
280 if PPDirectiveObj
.Content
.rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
281 PPDirectiveObj
.Content
+= T_CHAR_LF
286 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
288 if InComment
and DoubleSlashComment
:
290 DoubleSlashComment
= False
292 if InComment
and HashComment
and not PPExtend
:
295 PPDirectiveObj
.Content
+= T_CHAR_LF
296 PPDirectiveObj
.EndPos
= EndLinePos
297 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
298 PPDirectiveObj
= None
300 if InString
or InCharLiteral
:
301 CurrentLine
= "".join(self
.__CurrentLine
())
302 if CurrentLine
.rstrip(T_CHAR_LF
).rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
303 SlashIndex
= CurrentLine
.rindex(T_CHAR_BACKSLASH
)
304 self
.__SetCharValue
(self
.CurrentLineNumber
, SlashIndex
, T_CHAR_SPACE
)
306 self
.CurrentLineNumber
+= 1
307 self
.CurrentOffsetWithinLine
= 0
308 # check for */ comment end
309 elif InComment
and not DoubleSlashComment
and not HashComment
and self
.__CurrentChar
() == T_CHAR_STAR
and self
.__NextChar
() == T_CHAR_SLASH
:
311 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
313 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
316 # set comments to spaces
319 # // follows hash PP directive
320 if self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
323 PPDirectiveObj
.EndPos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
- 1)
324 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
325 PPDirectiveObj
= None
328 PPDirectiveObj
.Content
+= self
.__CurrentChar
()
330 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
332 # check for // comment
333 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
335 DoubleSlashComment
= True
337 # check for '#' comment
338 elif self
.__CurrentChar
() == T_CHAR_HASH
and not InString
and not InCharLiteral
:
341 PPDirectiveObj
= PP_Directive('', (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
), None)
342 # check for /* comment start
343 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_STAR
:
345 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
347 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
353 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
355 if InComment
and HashComment
and not PPExtend
:
356 PPDirectiveObj
.EndPos
= EndLinePos
357 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
360 ## ParseFile() method
362 # Parse the file profile buffer to extract fd, fv ... information
363 # Exception will be raised if syntax error found
365 # @param self The object pointer
368 self
.PreprocessFileWithClear()
369 # restore from ListOfList to ListOfString
370 self
.Profile
.FileLinesList
= ["".join(list) for list in self
.Profile
.FileLinesList
]
371 FileStringContents
= ''
372 for fileLine
in self
.Profile
.FileLinesList
:
373 FileStringContents
+= fileLine
374 cStream
= antlr3
.StringStream(FileStringContents
)
375 lexer
= CLexer(cStream
)
376 tStream
= antlr3
.CommonTokenStream(lexer
)
377 parser
= CParser(tStream
)
378 parser
.translation_unit()
380 ## CleanFileProfileBuffer() method
382 # Reset all contents of the profile of a file
384 def CleanFileProfileBuffer(self
):
386 FileProfile
.PPDirectiveList
= []
387 FileProfile
.AssignmentExpressionList
= []
388 FileProfile
.FunctionDefinitionList
= []
389 FileProfile
.VariableDeclarationList
= []
390 FileProfile
.EnumerationDefinitionList
= []
391 FileProfile
.StructUnionDefinitionList
= []
392 FileProfile
.TypedefDefinitionList
= []
393 FileProfile
.FunctionCallingList
= []
395 ## PrintFragments() method
397 # Print the contents of the profile of a file
399 def PrintFragments(self
):
401 print '################# ' + self
.FileName
+ '#####################'
403 print '/****************************************/'
404 print '/*************** ASSIGNMENTS ***************/'
405 print '/****************************************/'
406 for asign
in FileProfile
.AssignmentExpressionList
:
407 print str(asign
.StartPos
) + asign
.Name
+ asign
.Operator
+ asign
.Value
409 print '/****************************************/'
410 print '/********* PREPROCESS DIRECTIVES ********/'
411 print '/****************************************/'
412 for pp
in FileProfile
.PPDirectiveList
:
413 print str(pp
.StartPos
) + pp
.Content
415 print '/****************************************/'
416 print '/********* VARIABLE DECLARATIONS ********/'
417 print '/****************************************/'
418 for var
in FileProfile
.VariableDeclarationList
:
419 print str(var
.StartPos
) + var
.Modifier
+ ' '+ var
.Declarator
421 print '/****************************************/'
422 print '/********* FUNCTION DEFINITIONS *********/'
423 print '/****************************************/'
424 for func
in FileProfile
.FunctionDefinitionList
:
425 print str(func
.StartPos
) + func
.Modifier
+ ' '+ func
.Declarator
+ ' ' + str(func
.NamePos
)
427 print '/****************************************/'
428 print '/************ ENUMERATIONS **************/'
429 print '/****************************************/'
430 for enum
in FileProfile
.EnumerationDefinitionList
:
431 print str(enum
.StartPos
) + enum
.Content
433 print '/****************************************/'
434 print '/*********** STRUCTS/UNIONS *************/'
435 print '/****************************************/'
436 for su
in FileProfile
.StructUnionDefinitionList
:
437 print str(su
.StartPos
) + su
.Content
439 print '/****************************************/'
440 print '/************** TYPEDEFS ****************/'
441 print '/****************************************/'
442 for typedef
in FileProfile
.TypedefDefinitionList
:
443 print str(typedef
.StartPos
) + typedef
.ToType
447 # This acts like the main() function for the script, unless it is 'import'ed into another
450 if __name__
== "__main__":