2 # preprocess source file
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
19 import Common
.LongFilePathOs
as os
23 from CLexer
import CLexer
24 from CParser
import CParser
27 from CodeFragment
import PP_Directive
28 from ParserWarning
import Warning
31 ##define T_CHAR_SPACE ' '
32 ##define T_CHAR_NULL '\0'
33 ##define T_CHAR_CR '\r'
34 ##define T_CHAR_TAB '\t'
35 ##define T_CHAR_LF '\n'
36 ##define T_CHAR_SLASH '/'
37 ##define T_CHAR_BACKSLASH '\\'
38 ##define T_CHAR_DOUBLE_QUOTE '\"'
39 ##define T_CHAR_SINGLE_QUOTE '\''
40 ##define T_CHAR_STAR '*'
41 ##define T_CHAR_HASH '#'
43 (T_CHAR_SPACE
, T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_TAB
, T_CHAR_LF
, T_CHAR_SLASH
, \
44 T_CHAR_BACKSLASH
, T_CHAR_DOUBLE_QUOTE
, T_CHAR_SINGLE_QUOTE
, T_CHAR_STAR
, T_CHAR_HASH
) = \
45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
47 SEPERATOR_TUPLE
= ('=', '|', ',', '{', '}')
49 (T_COMMENT_TWO_SLASH
, T_COMMENT_SLASH_STAR
) = (0, 1)
51 (T_PP_INCLUDE
, T_PP_DEFINE
, T_PP_OTHERS
) = (0, 1, 2)
53 ## The collector for source code fragments.
55 # PreprocessFile method should be called prior to ParseFile
57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
58 # Get*** procedures mean these procedures will make judgement on current token only.
60 class CodeFragmentCollector
:
63 # @param self The object pointer
64 # @param FileName The file that to be parsed
66 def __init__(self
, FileName
):
67 self
.Profile
= FileProfile
.FileProfile(FileName
)
68 self
.Profile
.FileLinesList
.append(T_CHAR_LF
)
69 self
.FileName
= FileName
70 self
.CurrentLineNumber
= 1
71 self
.CurrentOffsetWithinLine
= 0
74 self
.__SkippedChars
= ""
76 ## __EndOfFile() method
78 # Judge current buffer pos is at file end
80 # @param self The object pointer
81 # @retval True Current File buffer position is at file end
82 # @retval False Current File buffer position is NOT at file end
84 def __EndOfFile(self
):
85 NumberOfLines
= len(self
.Profile
.FileLinesList
)
86 SizeOfLastLine
= len(self
.Profile
.FileLinesList
[-1])
87 if self
.CurrentLineNumber
== NumberOfLines
and self
.CurrentOffsetWithinLine
>= SizeOfLastLine
- 1:
89 elif self
.CurrentLineNumber
> NumberOfLines
:
94 ## __EndOfLine() method
96 # Judge current buffer pos is at line end
98 # @param self The object pointer
99 # @retval True Current File buffer position is at line end
100 # @retval False Current File buffer position is NOT at line end
102 def __EndOfLine(self
):
103 SizeOfCurrentLine
= len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1])
104 if self
.CurrentOffsetWithinLine
>= SizeOfCurrentLine
- 1:
111 # Reset file data buffer to the initial state
113 # @param self The object pointer
116 self
.CurrentLineNumber
= 1
117 self
.CurrentOffsetWithinLine
= 0
119 ## __UndoOneChar() method
121 # Go back one char in the file buffer
123 # @param self The object pointer
124 # @retval True Successfully go back one char
125 # @retval False Not able to go back one char as file beginning reached
127 def __UndoOneChar(self
):
129 if self
.CurrentLineNumber
== 1 and self
.CurrentOffsetWithinLine
== 0:
131 elif self
.CurrentOffsetWithinLine
== 0:
132 self
.CurrentLineNumber
-= 1
133 self
.CurrentOffsetWithinLine
= len(self
.__CurrentLine
()) - 1
135 self
.CurrentOffsetWithinLine
-= 1
138 ## __GetOneChar() method
140 # Move forward one char in the file buffer
142 # @param self The object pointer
144 def __GetOneChar(self
):
145 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
146 self
.CurrentLineNumber
+= 1
147 self
.CurrentOffsetWithinLine
= 0
149 self
.CurrentOffsetWithinLine
+= 1
151 ## __CurrentChar() method
153 # Get the char pointed to by the file buffer pointer
155 # @param self The object pointer
156 # @retval Char Current char
158 def __CurrentChar(self
):
159 CurrentChar
= self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
]
163 ## __NextChar() method
165 # Get the one char pass the char pointed to by the file buffer pointer
167 # @param self The object pointer
168 # @retval Char Next char
170 def __NextChar(self
):
171 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
172 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
][0]
174 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
+ 1]
176 ## __SetCurrentCharValue() method
178 # Modify the value of current char
180 # @param self The object pointer
181 # @param Value The new value of current char
183 def __SetCurrentCharValue(self
, Value
):
184 self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
] = Value
186 ## __SetCharValue() method
188 # Modify the value of current char
190 # @param self The object pointer
191 # @param Value The new value of current char
193 def __SetCharValue(self
, Line
, Offset
, Value
):
194 self
.Profile
.FileLinesList
[Line
- 1][Offset
] = Value
196 ## __CurrentLine() method
198 # Get the list that contains current line contents
200 # @param self The object pointer
201 # @retval List current line contents
203 def __CurrentLine(self
):
204 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]
206 ## __InsertComma() method
208 # Insert ',' to replace PP
210 # @param self The object pointer
211 # @retval List current line contents
213 def __InsertComma(self
, Line
):
216 if self
.Profile
.FileLinesList
[Line
- 1][0] != T_CHAR_HASH
:
217 BeforeHashPart
= str(self
.Profile
.FileLinesList
[Line
- 1]).split(T_CHAR_HASH
)[0]
218 if BeforeHashPart
.rstrip().endswith(T_CHAR_COMMA
) or BeforeHashPart
.rstrip().endswith(';'):
221 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(','):
224 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(';'):
227 if str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(',') or str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(';'):
230 self
.Profile
.FileLinesList
[Line
- 1].insert(self
.CurrentOffsetWithinLine
, ',')
232 ## PreprocessFileWithClear() method
234 # Run a preprocess for the file to clean all comments
236 # @param self The object pointer
238 def PreprocessFileWithClear(self
):
242 DoubleSlashComment
= False
245 PPDirectiveObj
= None
246 # HashComment in quoted string " " is ignored.
248 InCharLiteral
= False
250 self
.Profile
.FileLinesList
= [list(s
) for s
in self
.Profile
.FileLinesListFromFile
]
251 while not self
.__EndOfFile
():
253 if not InComment
and self
.__CurrentChar
() == T_CHAR_DOUBLE_QUOTE
:
254 InString
= not InString
256 if not InComment
and self
.__CurrentChar
() == T_CHAR_SINGLE_QUOTE
:
257 InCharLiteral
= not InCharLiteral
258 # meet new line, then no longer in a comment for // and '#'
259 if self
.__CurrentChar
() == T_CHAR_LF
:
260 if HashComment
and PPDirectiveObj
is not None:
261 if PPDirectiveObj
.Content
.rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
262 PPDirectiveObj
.Content
+= T_CHAR_LF
267 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
269 if InComment
and DoubleSlashComment
:
271 DoubleSlashComment
= False
273 if InComment
and HashComment
and not PPExtend
:
276 PPDirectiveObj
.Content
+= T_CHAR_LF
277 PPDirectiveObj
.EndPos
= EndLinePos
278 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
279 PPDirectiveObj
= None
281 if InString
or InCharLiteral
:
282 CurrentLine
= "".join(self
.__CurrentLine
())
283 if CurrentLine
.rstrip(T_CHAR_LF
).rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
284 SlashIndex
= CurrentLine
.rindex(T_CHAR_BACKSLASH
)
285 self
.__SetCharValue
(self
.CurrentLineNumber
, SlashIndex
, T_CHAR_SPACE
)
287 self
.CurrentLineNumber
+= 1
288 self
.CurrentOffsetWithinLine
= 0
289 # check for */ comment end
290 elif InComment
and not DoubleSlashComment
and not HashComment
and self
.__CurrentChar
() == T_CHAR_STAR
and self
.__NextChar
() == T_CHAR_SLASH
:
292 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
294 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
297 # set comments to spaces
300 # // follows hash PP directive
301 if self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
304 PPDirectiveObj
.EndPos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
- 1)
305 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
306 PPDirectiveObj
= None
309 PPDirectiveObj
.Content
+= self
.__CurrentChar
()
311 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
313 # check for // comment
314 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
316 DoubleSlashComment
= True
318 # check for '#' comment
319 elif self
.__CurrentChar
() == T_CHAR_HASH
and not InString
and not InCharLiteral
:
322 PPDirectiveObj
= PP_Directive('', (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
), None)
323 # check for /* comment start
324 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_STAR
:
326 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
328 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
334 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
336 if InComment
and HashComment
and not PPExtend
:
337 PPDirectiveObj
.EndPos
= EndLinePos
338 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
341 ## ParseFile() method
343 # Parse the file profile buffer to extract fd, fv ... information
344 # Exception will be raised if syntax error found
346 # @param self The object pointer
349 self
.PreprocessFileWithClear()
350 # restore from ListOfList to ListOfString
351 self
.Profile
.FileLinesList
= ["".join(list) for list in self
.Profile
.FileLinesList
]
352 FileStringContents
= ''
353 for fileLine
in self
.Profile
.FileLinesList
:
354 FileStringContents
+= fileLine
355 cStream
= antlr3
.StringStream(FileStringContents
)
356 lexer
= CLexer(cStream
)
357 tStream
= antlr3
.CommonTokenStream(lexer
)
358 parser
= CParser(tStream
)
359 parser
.translation_unit()
361 ## CleanFileProfileBuffer() method
363 # Reset all contents of the profile of a file
365 def CleanFileProfileBuffer(self
):
367 FileProfile
.PPDirectiveList
= []
368 FileProfile
.AssignmentExpressionList
= []
369 FileProfile
.FunctionDefinitionList
= []
370 FileProfile
.VariableDeclarationList
= []
371 FileProfile
.EnumerationDefinitionList
= []
372 FileProfile
.StructUnionDefinitionList
= []
373 FileProfile
.TypedefDefinitionList
= []
374 FileProfile
.FunctionCallingList
= []
376 ## PrintFragments() method
378 # Print the contents of the profile of a file
380 def PrintFragments(self
):
382 print '################# ' + self
.FileName
+ '#####################'
384 print '/****************************************/'
385 print '/*************** ASSIGNMENTS ***************/'
386 print '/****************************************/'
387 for asign
in FileProfile
.AssignmentExpressionList
:
388 print str(asign
.StartPos
) + asign
.Name
+ asign
.Operator
+ asign
.Value
390 print '/****************************************/'
391 print '/********* PREPROCESS DIRECTIVES ********/'
392 print '/****************************************/'
393 for pp
in FileProfile
.PPDirectiveList
:
394 print str(pp
.StartPos
) + pp
.Content
396 print '/****************************************/'
397 print '/********* VARIABLE DECLARATIONS ********/'
398 print '/****************************************/'
399 for var
in FileProfile
.VariableDeclarationList
:
400 print str(var
.StartPos
) + var
.Modifier
+ ' '+ var
.Declarator
402 print '/****************************************/'
403 print '/********* FUNCTION DEFINITIONS *********/'
404 print '/****************************************/'
405 for func
in FileProfile
.FunctionDefinitionList
:
406 print str(func
.StartPos
) + func
.Modifier
+ ' '+ func
.Declarator
+ ' ' + str(func
.NamePos
)
408 print '/****************************************/'
409 print '/************ ENUMERATIONS **************/'
410 print '/****************************************/'
411 for enum
in FileProfile
.EnumerationDefinitionList
:
412 print str(enum
.StartPos
) + enum
.Content
414 print '/****************************************/'
415 print '/*********** STRUCTS/UNIONS *************/'
416 print '/****************************************/'
417 for su
in FileProfile
.StructUnionDefinitionList
:
418 print str(su
.StartPos
) + su
.Content
420 print '/****************************************/'
421 print '/************** TYPEDEFS ****************/'
422 print '/****************************************/'
423 for typedef
in FileProfile
.TypedefDefinitionList
:
424 print str(typedef
.StartPos
) + typedef
.ToType
428 # This acts like the main() function for the script, unless it is 'import'ed into another
431 if __name__
== "__main__":