2 # preprocess source file
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
18 from __future__
import print_function
19 from __future__
import absolute_import
21 import Common
.LongFilePathOs
as os
24 if sys
.version_info
.major
== 3:
25 import antlr4
as antlr
26 from Eot
.CParser4
.CLexer
import CLexer
27 from Eot
.CParser4
.CParser
import CParser
29 import antlr3
as antlr
30 antlr
.InputStream
= antlr
.StringStream
31 from Eot
.CParser3
.CLexer
import CLexer
32 from Eot
.CParser3
.CParser
import CParser
34 from Eot
import FileProfile
35 from Eot
.CodeFragment
import PP_Directive
36 from Eot
.ParserWarning
import Warning
39 ##define T_CHAR_SPACE ' '
40 ##define T_CHAR_NULL '\0'
41 ##define T_CHAR_CR '\r'
42 ##define T_CHAR_TAB '\t'
43 ##define T_CHAR_LF '\n'
44 ##define T_CHAR_SLASH '/'
45 ##define T_CHAR_BACKSLASH '\\'
46 ##define T_CHAR_DOUBLE_QUOTE '\"'
47 ##define T_CHAR_SINGLE_QUOTE '\''
48 ##define T_CHAR_STAR '*'
49 ##define T_CHAR_HASH '#'
51 (T_CHAR_SPACE
, T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_TAB
, T_CHAR_LF
, T_CHAR_SLASH
, \
52 T_CHAR_BACKSLASH
, T_CHAR_DOUBLE_QUOTE
, T_CHAR_SINGLE_QUOTE
, T_CHAR_STAR
, T_CHAR_HASH
) = \
53 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
55 SEPERATOR_TUPLE
= ('=', '|', ',', '{', '}')
57 (T_COMMENT_TWO_SLASH
, T_COMMENT_SLASH_STAR
) = (0, 1)
59 (T_PP_INCLUDE
, T_PP_DEFINE
, T_PP_OTHERS
) = (0, 1, 2)
61 ## The collector for source code fragments.
63 # PreprocessFile method should be called prior to ParseFile
65 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
66 # Get*** procedures mean these procedures will make judgement on current token only.
68 class CodeFragmentCollector
:
71 # @param self The object pointer
72 # @param FileName The file that to be parsed
74 def __init__(self
, FileName
):
75 self
.Profile
= FileProfile
.FileProfile(FileName
)
76 self
.Profile
.FileLinesList
.append(T_CHAR_LF
)
77 self
.FileName
= FileName
78 self
.CurrentLineNumber
= 1
79 self
.CurrentOffsetWithinLine
= 0
82 self
.__SkippedChars
= ""
84 ## __EndOfFile() method
86 # Judge current buffer pos is at file end
88 # @param self The object pointer
89 # @retval True Current File buffer position is at file end
90 # @retval False Current File buffer position is NOT at file end
92 def __EndOfFile(self
):
93 NumberOfLines
= len(self
.Profile
.FileLinesList
)
94 SizeOfLastLine
= len(self
.Profile
.FileLinesList
[-1])
95 if self
.CurrentLineNumber
== NumberOfLines
and self
.CurrentOffsetWithinLine
>= SizeOfLastLine
- 1:
97 elif self
.CurrentLineNumber
> NumberOfLines
:
102 ## __EndOfLine() method
104 # Judge current buffer pos is at line end
106 # @param self The object pointer
107 # @retval True Current File buffer position is at line end
108 # @retval False Current File buffer position is NOT at line end
110 def __EndOfLine(self
):
111 SizeOfCurrentLine
= len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1])
112 if self
.CurrentOffsetWithinLine
>= SizeOfCurrentLine
- 1:
119 # Reset file data buffer to the initial state
121 # @param self The object pointer
124 self
.CurrentLineNumber
= 1
125 self
.CurrentOffsetWithinLine
= 0
127 ## __UndoOneChar() method
129 # Go back one char in the file buffer
131 # @param self The object pointer
132 # @retval True Successfully go back one char
133 # @retval False Not able to go back one char as file beginning reached
135 def __UndoOneChar(self
):
137 if self
.CurrentLineNumber
== 1 and self
.CurrentOffsetWithinLine
== 0:
139 elif self
.CurrentOffsetWithinLine
== 0:
140 self
.CurrentLineNumber
-= 1
141 self
.CurrentOffsetWithinLine
= len(self
.__CurrentLine
()) - 1
143 self
.CurrentOffsetWithinLine
-= 1
146 ## __GetOneChar() method
148 # Move forward one char in the file buffer
150 # @param self The object pointer
152 def __GetOneChar(self
):
153 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
154 self
.CurrentLineNumber
+= 1
155 self
.CurrentOffsetWithinLine
= 0
157 self
.CurrentOffsetWithinLine
+= 1
159 ## __CurrentChar() method
161 # Get the char pointed to by the file buffer pointer
163 # @param self The object pointer
164 # @retval Char Current char
166 def __CurrentChar(self
):
167 CurrentChar
= self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
]
171 ## __NextChar() method
173 # Get the one char pass the char pointed to by the file buffer pointer
175 # @param self The object pointer
176 # @retval Char Next char
178 def __NextChar(self
):
179 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
180 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
][0]
182 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
+ 1]
184 ## __SetCurrentCharValue() method
186 # Modify the value of current char
188 # @param self The object pointer
189 # @param Value The new value of current char
191 def __SetCurrentCharValue(self
, Value
):
192 self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
] = Value
194 ## __SetCharValue() method
196 # Modify the value of current char
198 # @param self The object pointer
199 # @param Value The new value of current char
201 def __SetCharValue(self
, Line
, Offset
, Value
):
202 self
.Profile
.FileLinesList
[Line
- 1][Offset
] = Value
204 ## __CurrentLine() method
206 # Get the list that contains current line contents
208 # @param self The object pointer
209 # @retval List current line contents
211 def __CurrentLine(self
):
212 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]
214 ## __InsertComma() method
216 # Insert ',' to replace PP
218 # @param self The object pointer
219 # @retval List current line contents
221 def __InsertComma(self
, Line
):
224 if self
.Profile
.FileLinesList
[Line
- 1][0] != T_CHAR_HASH
:
225 BeforeHashPart
= str(self
.Profile
.FileLinesList
[Line
- 1]).split(T_CHAR_HASH
)[0]
226 if BeforeHashPart
.rstrip().endswith(T_CHAR_COMMA
) or BeforeHashPart
.rstrip().endswith(';'):
229 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(','):
232 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(';'):
235 if str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(',') or str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(';'):
238 self
.Profile
.FileLinesList
[Line
- 1].insert(self
.CurrentOffsetWithinLine
, ',')
240 ## PreprocessFileWithClear() method
242 # Run a preprocess for the file to clean all comments
244 # @param self The object pointer
246 def PreprocessFileWithClear(self
):
250 DoubleSlashComment
= False
253 PPDirectiveObj
= None
254 # HashComment in quoted string " " is ignored.
256 InCharLiteral
= False
258 self
.Profile
.FileLinesList
= [list(s
) for s
in self
.Profile
.FileLinesListFromFile
]
259 while not self
.__EndOfFile
():
261 if not InComment
and self
.__CurrentChar
() == T_CHAR_DOUBLE_QUOTE
:
262 InString
= not InString
264 if not InComment
and self
.__CurrentChar
() == T_CHAR_SINGLE_QUOTE
:
265 InCharLiteral
= not InCharLiteral
266 # meet new line, then no longer in a comment for // and '#'
267 if self
.__CurrentChar
() == T_CHAR_LF
:
268 if HashComment
and PPDirectiveObj
is not None:
269 if PPDirectiveObj
.Content
.rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
270 PPDirectiveObj
.Content
+= T_CHAR_LF
275 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
277 if InComment
and DoubleSlashComment
:
279 DoubleSlashComment
= False
281 if InComment
and HashComment
and not PPExtend
:
284 PPDirectiveObj
.Content
+= T_CHAR_LF
285 PPDirectiveObj
.EndPos
= EndLinePos
286 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
287 PPDirectiveObj
= None
289 if InString
or InCharLiteral
:
290 CurrentLine
= "".join(self
.__CurrentLine
())
291 if CurrentLine
.rstrip(T_CHAR_LF
).rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
292 SlashIndex
= CurrentLine
.rindex(T_CHAR_BACKSLASH
)
293 self
.__SetCharValue
(self
.CurrentLineNumber
, SlashIndex
, T_CHAR_SPACE
)
295 self
.CurrentLineNumber
+= 1
296 self
.CurrentOffsetWithinLine
= 0
297 # check for */ comment end
298 elif InComment
and not DoubleSlashComment
and not HashComment
and self
.__CurrentChar
() == T_CHAR_STAR
and self
.__NextChar
() == T_CHAR_SLASH
:
300 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
302 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
305 # set comments to spaces
308 # // follows hash PP directive
309 if self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
312 PPDirectiveObj
.EndPos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
- 1)
313 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
314 PPDirectiveObj
= None
317 PPDirectiveObj
.Content
+= self
.__CurrentChar
()
319 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
321 # check for // comment
322 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
324 DoubleSlashComment
= True
326 # check for '#' comment
327 elif self
.__CurrentChar
() == T_CHAR_HASH
and not InString
and not InCharLiteral
:
330 PPDirectiveObj
= PP_Directive('', (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
), None)
331 # check for /* comment start
332 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_STAR
:
334 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
336 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
342 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
344 if InComment
and HashComment
and not PPExtend
:
345 PPDirectiveObj
.EndPos
= EndLinePos
346 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
349 ## ParseFile() method
351 # Parse the file profile buffer to extract fd, fv ... information
352 # Exception will be raised if syntax error found
354 # @param self The object pointer
357 self
.PreprocessFileWithClear()
358 # restore from ListOfList to ListOfString
359 self
.Profile
.FileLinesList
= ["".join(list) for list in self
.Profile
.FileLinesList
]
360 FileStringContents
= ''
361 for fileLine
in self
.Profile
.FileLinesList
:
362 FileStringContents
+= fileLine
363 cStream
= antlr
.InputStream(FileStringContents
)
364 lexer
= CLexer(cStream
)
365 tStream
= antlr
.CommonTokenStream(lexer
)
366 parser
= CParser(tStream
)
367 parser
.translation_unit()
369 ## CleanFileProfileBuffer() method
371 # Reset all contents of the profile of a file
373 def CleanFileProfileBuffer(self
):
375 FileProfile
.PPDirectiveList
= []
376 FileProfile
.AssignmentExpressionList
= []
377 FileProfile
.FunctionDefinitionList
= []
378 FileProfile
.VariableDeclarationList
= []
379 FileProfile
.EnumerationDefinitionList
= []
380 FileProfile
.StructUnionDefinitionList
= []
381 FileProfile
.TypedefDefinitionList
= []
382 FileProfile
.FunctionCallingList
= []
384 ## PrintFragments() method
386 # Print the contents of the profile of a file
388 def PrintFragments(self
):
390 print('################# ' + self
.FileName
+ '#####################')
392 print('/****************************************/')
393 print('/*************** ASSIGNMENTS ***************/')
394 print('/****************************************/')
395 for asign
in FileProfile
.AssignmentExpressionList
:
396 print(str(asign
.StartPos
) + asign
.Name
+ asign
.Operator
+ asign
.Value
)
398 print('/****************************************/')
399 print('/********* PREPROCESS DIRECTIVES ********/')
400 print('/****************************************/')
401 for pp
in FileProfile
.PPDirectiveList
:
402 print(str(pp
.StartPos
) + pp
.Content
)
404 print('/****************************************/')
405 print('/********* VARIABLE DECLARATIONS ********/')
406 print('/****************************************/')
407 for var
in FileProfile
.VariableDeclarationList
:
408 print(str(var
.StartPos
) + var
.Modifier
+ ' '+ var
.Declarator
)
410 print('/****************************************/')
411 print('/********* FUNCTION DEFINITIONS *********/')
412 print('/****************************************/')
413 for func
in FileProfile
.FunctionDefinitionList
:
414 print(str(func
.StartPos
) + func
.Modifier
+ ' '+ func
.Declarator
+ ' ' + str(func
.NamePos
))
416 print('/****************************************/')
417 print('/************ ENUMERATIONS **************/')
418 print('/****************************************/')
419 for enum
in FileProfile
.EnumerationDefinitionList
:
420 print(str(enum
.StartPos
) + enum
.Content
)
422 print('/****************************************/')
423 print('/*********** STRUCTS/UNIONS *************/')
424 print('/****************************************/')
425 for su
in FileProfile
.StructUnionDefinitionList
:
426 print(str(su
.StartPos
) + su
.Content
)
428 print('/****************************************/')
429 print('/************** TYPEDEFS ****************/')
430 print('/****************************************/')
431 for typedef
in FileProfile
.TypedefDefinitionList
:
432 print(str(typedef
.StartPos
) + typedef
.ToType
)
436 # This acts like the main() function for the script, unless it is 'import'ed into another
439 if __name__
== "__main__":