2 # preprocess source file
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials
7 # are licensed and made available under the terms and conditions of the BSD License
8 # which accompanies this distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
18 from __future__
import print_function
20 import Common
.LongFilePathOs
as os
24 from CLexer
import CLexer
25 from CParser
import CParser
28 from CodeFragment
import PP_Directive
29 from ParserWarning
import Warning
32 ##define T_CHAR_SPACE ' '
33 ##define T_CHAR_NULL '\0'
34 ##define T_CHAR_CR '\r'
35 ##define T_CHAR_TAB '\t'
36 ##define T_CHAR_LF '\n'
37 ##define T_CHAR_SLASH '/'
38 ##define T_CHAR_BACKSLASH '\\'
39 ##define T_CHAR_DOUBLE_QUOTE '\"'
40 ##define T_CHAR_SINGLE_QUOTE '\''
41 ##define T_CHAR_STAR '*'
42 ##define T_CHAR_HASH '#'
44 (T_CHAR_SPACE
, T_CHAR_NULL
, T_CHAR_CR
, T_CHAR_TAB
, T_CHAR_LF
, T_CHAR_SLASH
, \
45 T_CHAR_BACKSLASH
, T_CHAR_DOUBLE_QUOTE
, T_CHAR_SINGLE_QUOTE
, T_CHAR_STAR
, T_CHAR_HASH
) = \
46 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
48 SEPERATOR_TUPLE
= ('=', '|', ',', '{', '}')
50 (T_COMMENT_TWO_SLASH
, T_COMMENT_SLASH_STAR
) = (0, 1)
52 (T_PP_INCLUDE
, T_PP_DEFINE
, T_PP_OTHERS
) = (0, 1, 2)
54 ## The collector for source code fragments.
56 # PreprocessFile method should be called prior to ParseFile
58 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
59 # Get*** procedures mean these procedures will make judgement on current token only.
61 class CodeFragmentCollector
:
64 # @param self The object pointer
65 # @param FileName The file that to be parsed
67 def __init__(self
, FileName
):
68 self
.Profile
= FileProfile
.FileProfile(FileName
)
69 self
.Profile
.FileLinesList
.append(T_CHAR_LF
)
70 self
.FileName
= FileName
71 self
.CurrentLineNumber
= 1
72 self
.CurrentOffsetWithinLine
= 0
75 self
.__SkippedChars
= ""
77 ## __EndOfFile() method
79 # Judge current buffer pos is at file end
81 # @param self The object pointer
82 # @retval True Current File buffer position is at file end
83 # @retval False Current File buffer position is NOT at file end
85 def __EndOfFile(self
):
86 NumberOfLines
= len(self
.Profile
.FileLinesList
)
87 SizeOfLastLine
= len(self
.Profile
.FileLinesList
[-1])
88 if self
.CurrentLineNumber
== NumberOfLines
and self
.CurrentOffsetWithinLine
>= SizeOfLastLine
- 1:
90 elif self
.CurrentLineNumber
> NumberOfLines
:
95 ## __EndOfLine() method
97 # Judge current buffer pos is at line end
99 # @param self The object pointer
100 # @retval True Current File buffer position is at line end
101 # @retval False Current File buffer position is NOT at line end
103 def __EndOfLine(self
):
104 SizeOfCurrentLine
= len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1])
105 if self
.CurrentOffsetWithinLine
>= SizeOfCurrentLine
- 1:
112 # Reset file data buffer to the initial state
114 # @param self The object pointer
117 self
.CurrentLineNumber
= 1
118 self
.CurrentOffsetWithinLine
= 0
120 ## __UndoOneChar() method
122 # Go back one char in the file buffer
124 # @param self The object pointer
125 # @retval True Successfully go back one char
126 # @retval False Not able to go back one char as file beginning reached
128 def __UndoOneChar(self
):
130 if self
.CurrentLineNumber
== 1 and self
.CurrentOffsetWithinLine
== 0:
132 elif self
.CurrentOffsetWithinLine
== 0:
133 self
.CurrentLineNumber
-= 1
134 self
.CurrentOffsetWithinLine
= len(self
.__CurrentLine
()) - 1
136 self
.CurrentOffsetWithinLine
-= 1
139 ## __GetOneChar() method
141 # Move forward one char in the file buffer
143 # @param self The object pointer
145 def __GetOneChar(self
):
146 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
147 self
.CurrentLineNumber
+= 1
148 self
.CurrentOffsetWithinLine
= 0
150 self
.CurrentOffsetWithinLine
+= 1
152 ## __CurrentChar() method
154 # Get the char pointed to by the file buffer pointer
156 # @param self The object pointer
157 # @retval Char Current char
159 def __CurrentChar(self
):
160 CurrentChar
= self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
]
164 ## __NextChar() method
166 # Get the one char pass the char pointed to by the file buffer pointer
168 # @param self The object pointer
169 # @retval Char Next char
171 def __NextChar(self
):
172 if self
.CurrentOffsetWithinLine
== len(self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]) - 1:
173 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
][0]
175 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
+ 1]
177 ## __SetCurrentCharValue() method
179 # Modify the value of current char
181 # @param self The object pointer
182 # @param Value The new value of current char
184 def __SetCurrentCharValue(self
, Value
):
185 self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1][self
.CurrentOffsetWithinLine
] = Value
187 ## __SetCharValue() method
189 # Modify the value of current char
191 # @param self The object pointer
192 # @param Value The new value of current char
194 def __SetCharValue(self
, Line
, Offset
, Value
):
195 self
.Profile
.FileLinesList
[Line
- 1][Offset
] = Value
197 ## __CurrentLine() method
199 # Get the list that contains current line contents
201 # @param self The object pointer
202 # @retval List current line contents
204 def __CurrentLine(self
):
205 return self
.Profile
.FileLinesList
[self
.CurrentLineNumber
- 1]
207 ## __InsertComma() method
209 # Insert ',' to replace PP
211 # @param self The object pointer
212 # @retval List current line contents
214 def __InsertComma(self
, Line
):
217 if self
.Profile
.FileLinesList
[Line
- 1][0] != T_CHAR_HASH
:
218 BeforeHashPart
= str(self
.Profile
.FileLinesList
[Line
- 1]).split(T_CHAR_HASH
)[0]
219 if BeforeHashPart
.rstrip().endswith(T_CHAR_COMMA
) or BeforeHashPart
.rstrip().endswith(';'):
222 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(','):
225 if Line
- 2 >= 0 and str(self
.Profile
.FileLinesList
[Line
- 2]).rstrip().endswith(';'):
228 if str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(',') or str(self
.Profile
.FileLinesList
[Line
]).lstrip().startswith(';'):
231 self
.Profile
.FileLinesList
[Line
- 1].insert(self
.CurrentOffsetWithinLine
, ',')
233 ## PreprocessFileWithClear() method
235 # Run a preprocess for the file to clean all comments
237 # @param self The object pointer
239 def PreprocessFileWithClear(self
):
243 DoubleSlashComment
= False
246 PPDirectiveObj
= None
247 # HashComment in quoted string " " is ignored.
249 InCharLiteral
= False
251 self
.Profile
.FileLinesList
= [list(s
) for s
in self
.Profile
.FileLinesListFromFile
]
252 while not self
.__EndOfFile
():
254 if not InComment
and self
.__CurrentChar
() == T_CHAR_DOUBLE_QUOTE
:
255 InString
= not InString
257 if not InComment
and self
.__CurrentChar
() == T_CHAR_SINGLE_QUOTE
:
258 InCharLiteral
= not InCharLiteral
259 # meet new line, then no longer in a comment for // and '#'
260 if self
.__CurrentChar
() == T_CHAR_LF
:
261 if HashComment
and PPDirectiveObj
is not None:
262 if PPDirectiveObj
.Content
.rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
263 PPDirectiveObj
.Content
+= T_CHAR_LF
268 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
270 if InComment
and DoubleSlashComment
:
272 DoubleSlashComment
= False
274 if InComment
and HashComment
and not PPExtend
:
277 PPDirectiveObj
.Content
+= T_CHAR_LF
278 PPDirectiveObj
.EndPos
= EndLinePos
279 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
280 PPDirectiveObj
= None
282 if InString
or InCharLiteral
:
283 CurrentLine
= "".join(self
.__CurrentLine
())
284 if CurrentLine
.rstrip(T_CHAR_LF
).rstrip(T_CHAR_CR
).endswith(T_CHAR_BACKSLASH
):
285 SlashIndex
= CurrentLine
.rindex(T_CHAR_BACKSLASH
)
286 self
.__SetCharValue
(self
.CurrentLineNumber
, SlashIndex
, T_CHAR_SPACE
)
288 self
.CurrentLineNumber
+= 1
289 self
.CurrentOffsetWithinLine
= 0
290 # check for */ comment end
291 elif InComment
and not DoubleSlashComment
and not HashComment
and self
.__CurrentChar
() == T_CHAR_STAR
and self
.__NextChar
() == T_CHAR_SLASH
:
293 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
295 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
298 # set comments to spaces
301 # // follows hash PP directive
302 if self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
305 PPDirectiveObj
.EndPos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
- 1)
306 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
307 PPDirectiveObj
= None
310 PPDirectiveObj
.Content
+= self
.__CurrentChar
()
312 self
.__SetCurrentCharValue
(T_CHAR_SPACE
)
314 # check for // comment
315 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_SLASH
:
317 DoubleSlashComment
= True
319 # check for '#' comment
320 elif self
.__CurrentChar
() == T_CHAR_HASH
and not InString
and not InCharLiteral
:
323 PPDirectiveObj
= PP_Directive('', (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
), None)
324 # check for /* comment start
325 elif self
.__CurrentChar
() == T_CHAR_SLASH
and self
.__NextChar
() == T_CHAR_STAR
:
327 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
329 self
.__SetCurrentCharValue
( T_CHAR_SPACE
)
335 EndLinePos
= (self
.CurrentLineNumber
, self
.CurrentOffsetWithinLine
)
337 if InComment
and HashComment
and not PPExtend
:
338 PPDirectiveObj
.EndPos
= EndLinePos
339 FileProfile
.PPDirectiveList
.append(PPDirectiveObj
)
342 ## ParseFile() method
344 # Parse the file profile buffer to extract fd, fv ... information
345 # Exception will be raised if syntax error found
347 # @param self The object pointer
350 self
.PreprocessFileWithClear()
351 # restore from ListOfList to ListOfString
352 self
.Profile
.FileLinesList
= ["".join(list) for list in self
.Profile
.FileLinesList
]
353 FileStringContents
= ''
354 for fileLine
in self
.Profile
.FileLinesList
:
355 FileStringContents
+= fileLine
356 cStream
= antlr3
.StringStream(FileStringContents
)
357 lexer
= CLexer(cStream
)
358 tStream
= antlr3
.CommonTokenStream(lexer
)
359 parser
= CParser(tStream
)
360 parser
.translation_unit()
362 ## CleanFileProfileBuffer() method
364 # Reset all contents of the profile of a file
366 def CleanFileProfileBuffer(self
):
368 FileProfile
.PPDirectiveList
= []
369 FileProfile
.AssignmentExpressionList
= []
370 FileProfile
.FunctionDefinitionList
= []
371 FileProfile
.VariableDeclarationList
= []
372 FileProfile
.EnumerationDefinitionList
= []
373 FileProfile
.StructUnionDefinitionList
= []
374 FileProfile
.TypedefDefinitionList
= []
375 FileProfile
.FunctionCallingList
= []
377 ## PrintFragments() method
379 # Print the contents of the profile of a file
381 def PrintFragments(self
):
383 print('################# ' + self
.FileName
+ '#####################')
385 print('/****************************************/')
386 print('/*************** ASSIGNMENTS ***************/')
387 print('/****************************************/')
388 for asign
in FileProfile
.AssignmentExpressionList
:
389 print(str(asign
.StartPos
) + asign
.Name
+ asign
.Operator
+ asign
.Value
)
391 print('/****************************************/')
392 print('/********* PREPROCESS DIRECTIVES ********/')
393 print('/****************************************/')
394 for pp
in FileProfile
.PPDirectiveList
:
395 print(str(pp
.StartPos
) + pp
.Content
)
397 print('/****************************************/')
398 print('/********* VARIABLE DECLARATIONS ********/')
399 print('/****************************************/')
400 for var
in FileProfile
.VariableDeclarationList
:
401 print(str(var
.StartPos
) + var
.Modifier
+ ' '+ var
.Declarator
)
403 print('/****************************************/')
404 print('/********* FUNCTION DEFINITIONS *********/')
405 print('/****************************************/')
406 for func
in FileProfile
.FunctionDefinitionList
:
407 print(str(func
.StartPos
) + func
.Modifier
+ ' '+ func
.Declarator
+ ' ' + str(func
.NamePos
))
409 print('/****************************************/')
410 print('/************ ENUMERATIONS **************/')
411 print('/****************************************/')
412 for enum
in FileProfile
.EnumerationDefinitionList
:
413 print(str(enum
.StartPos
) + enum
.Content
)
415 print('/****************************************/')
416 print('/*********** STRUCTS/UNIONS *************/')
417 print('/****************************************/')
418 for su
in FileProfile
.StructUnionDefinitionList
:
419 print(str(su
.StartPos
) + su
.Content
)
421 print('/****************************************/')
422 print('/************** TYPEDEFS ****************/')
423 print('/****************************************/')
424 for typedef
in FileProfile
.TypedefDefinitionList
:
425 print(str(typedef
.StartPos
) + typedef
.ToType
)
429 # This acts like the main() function for the script, unless it is 'import'ed into another
432 if __name__
== "__main__":