]>
Commit | Line | Data |
---|---|---|
52302d4d LG |
1 | ## @file\r |
2 | # preprocess source file\r | |
3 | #\r | |
231fdbb2 | 4 | # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>\r |
52302d4d | 5 | #\r |
40d841f6 | 6 | # This program and the accompanying materials\r |
52302d4d LG |
7 | # are licensed and made available under the terms and conditions of the BSD License\r |
8 | # which accompanies this distribution. The full text of the license may be found at\r | |
9 | # http://opensource.org/licenses/bsd-license.php\r | |
10 | #\r | |
11 | # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r | |
12 | # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r | |
13 | #\r | |
14 | \r | |
15 | ##\r | |
16 | # Import Modules\r | |
17 | #\r | |
18 | import re\r | |
1be2ed90 | 19 | import Common.LongFilePathOs as os\r |
52302d4d LG |
20 | import sys\r |
21 | \r | |
22 | import antlr3\r | |
23 | from CLexer import CLexer\r | |
24 | from CParser import CParser\r | |
25 | \r | |
26 | import FileProfile\r | |
27 | from CodeFragment import PP_Directive\r | |
28 | from ParserWarning import Warning\r | |
29 | \r | |
30 | \r | |
31 | ##define T_CHAR_SPACE ' '\r | |
32 | ##define T_CHAR_NULL '\0'\r | |
33 | ##define T_CHAR_CR '\r'\r | |
34 | ##define T_CHAR_TAB '\t'\r | |
35 | ##define T_CHAR_LF '\n'\r | |
36 | ##define T_CHAR_SLASH '/'\r | |
37 | ##define T_CHAR_BACKSLASH '\\'\r | |
38 | ##define T_CHAR_DOUBLE_QUOTE '\"'\r | |
39 | ##define T_CHAR_SINGLE_QUOTE '\''\r | |
40 | ##define T_CHAR_STAR '*'\r | |
41 | ##define T_CHAR_HASH '#'\r | |
42 | \r | |
43 | (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \\r | |
44 | T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \\r | |
45 | (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')\r | |
46 | \r | |
47 | SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')\r | |
48 | \r | |
49 | (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)\r | |
50 | \r | |
51 | (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)\r | |
52 | \r | |
53 | ## The collector for source code fragments.\r | |
54 | #\r | |
55 | # PreprocessFile method should be called prior to ParseFile\r | |
56 | #\r | |
57 | # GetNext*** procedures mean these procedures will get next token first, then make judgement.\r | |
58 | # Get*** procedures mean these procedures will make judgement on current token only.\r | |
59 | #\r | |
60 | class CodeFragmentCollector:\r | |
61 | ## The constructor\r | |
62 | #\r | |
63 | # @param self The object pointer\r | |
64 | # @param FileName The file that to be parsed\r | |
65 | #\r | |
66 | def __init__(self, FileName):\r | |
67 | self.Profile = FileProfile.FileProfile(FileName)\r | |
68 | self.Profile.FileLinesList.append(T_CHAR_LF)\r | |
69 | self.FileName = FileName\r | |
70 | self.CurrentLineNumber = 1\r | |
71 | self.CurrentOffsetWithinLine = 0\r | |
72 | \r | |
73 | self.__Token = ""\r | |
74 | self.__SkippedChars = ""\r | |
75 | \r | |
52302d4d LG |
76 | ## __EndOfFile() method\r |
77 | #\r | |
78 | # Judge current buffer pos is at file end\r | |
79 | #\r | |
80 | # @param self The object pointer\r | |
81 | # @retval True Current File buffer position is at file end\r | |
82 | # @retval False Current File buffer position is NOT at file end\r | |
83 | #\r | |
84 | def __EndOfFile(self):\r | |
85 | NumberOfLines = len(self.Profile.FileLinesList)\r | |
86 | SizeOfLastLine = len(self.Profile.FileLinesList[-1])\r | |
87 | if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:\r | |
88 | return True\r | |
89 | elif self.CurrentLineNumber > NumberOfLines:\r | |
90 | return True\r | |
91 | else:\r | |
92 | return False\r | |
93 | \r | |
94 | ## __EndOfLine() method\r | |
95 | #\r | |
96 | # Judge current buffer pos is at line end\r | |
97 | #\r | |
98 | # @param self The object pointer\r | |
99 | # @retval True Current File buffer position is at line end\r | |
100 | # @retval False Current File buffer position is NOT at line end\r | |
101 | #\r | |
102 | def __EndOfLine(self):\r | |
103 | SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])\r | |
104 | if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:\r | |
105 | return True\r | |
106 | else:\r | |
107 | return False\r | |
108 | \r | |
109 | ## Rewind() method\r | |
110 | #\r | |
111 | # Reset file data buffer to the initial state\r | |
112 | #\r | |
113 | # @param self The object pointer\r | |
114 | #\r | |
115 | def Rewind(self):\r | |
116 | self.CurrentLineNumber = 1\r | |
117 | self.CurrentOffsetWithinLine = 0\r | |
118 | \r | |
119 | ## __UndoOneChar() method\r | |
120 | #\r | |
121 | # Go back one char in the file buffer\r | |
122 | #\r | |
123 | # @param self The object pointer\r | |
124 | # @retval True Successfully go back one char\r | |
125 | # @retval False Not able to go back one char as file beginning reached\r | |
126 | #\r | |
127 | def __UndoOneChar(self):\r | |
128 | \r | |
129 | if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:\r | |
130 | return False\r | |
131 | elif self.CurrentOffsetWithinLine == 0:\r | |
132 | self.CurrentLineNumber -= 1\r | |
133 | self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1\r | |
134 | else:\r | |
135 | self.CurrentOffsetWithinLine -= 1\r | |
136 | return True\r | |
137 | \r | |
138 | ## __GetOneChar() method\r | |
139 | #\r | |
140 | # Move forward one char in the file buffer\r | |
141 | #\r | |
142 | # @param self The object pointer\r | |
143 | #\r | |
144 | def __GetOneChar(self):\r | |
145 | if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r | |
146 | self.CurrentLineNumber += 1\r | |
147 | self.CurrentOffsetWithinLine = 0\r | |
148 | else:\r | |
149 | self.CurrentOffsetWithinLine += 1\r | |
150 | \r | |
151 | ## __CurrentChar() method\r | |
152 | #\r | |
153 | # Get the char pointed to by the file buffer pointer\r | |
154 | #\r | |
155 | # @param self The object pointer\r | |
156 | # @retval Char Current char\r | |
157 | #\r | |
158 | def __CurrentChar(self):\r | |
159 | CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]\r | |
160 | \r | |
161 | return CurrentChar\r | |
162 | \r | |
163 | ## __NextChar() method\r | |
164 | #\r | |
165 | # Get the one char pass the char pointed to by the file buffer pointer\r | |
166 | #\r | |
167 | # @param self The object pointer\r | |
168 | # @retval Char Next char\r | |
169 | #\r | |
170 | def __NextChar(self):\r | |
171 | if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r | |
172 | return self.Profile.FileLinesList[self.CurrentLineNumber][0]\r | |
173 | else:\r | |
174 | return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]\r | |
175 | \r | |
176 | ## __SetCurrentCharValue() method\r | |
177 | #\r | |
178 | # Modify the value of current char\r | |
179 | #\r | |
180 | # @param self The object pointer\r | |
181 | # @param Value The new value of current char\r | |
182 | #\r | |
183 | def __SetCurrentCharValue(self, Value):\r | |
184 | self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value\r | |
185 | \r | |
186 | ## __SetCharValue() method\r | |
187 | #\r | |
188 | # Modify the value of current char\r | |
189 | #\r | |
190 | # @param self The object pointer\r | |
191 | # @param Value The new value of current char\r | |
192 | #\r | |
193 | def __SetCharValue(self, Line, Offset, Value):\r | |
194 | self.Profile.FileLinesList[Line - 1][Offset] = Value\r | |
195 | \r | |
196 | ## __CurrentLine() method\r | |
197 | #\r | |
198 | # Get the list that contains current line contents\r | |
199 | #\r | |
200 | # @param self The object pointer\r | |
201 | # @retval List current line contents\r | |
202 | #\r | |
203 | def __CurrentLine(self):\r | |
204 | return self.Profile.FileLinesList[self.CurrentLineNumber - 1]\r | |
205 | \r | |
206 | ## __InsertComma() method\r | |
207 | #\r | |
208 | # Insert ',' to replace PP\r | |
209 | #\r | |
210 | # @param self The object pointer\r | |
211 | # @retval List current line contents\r | |
212 | #\r | |
213 | def __InsertComma(self, Line):\r | |
214 | \r | |
215 | \r | |
216 | if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:\r | |
217 | BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]\r | |
218 | if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):\r | |
219 | return\r | |
220 | \r | |
221 | if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):\r | |
222 | return\r | |
223 | \r | |
224 | if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):\r | |
225 | return\r | |
226 | \r | |
227 | if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):\r | |
228 | return\r | |
229 | \r | |
230 | self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')\r | |
231 | \r | |
232 | ## PreprocessFileWithClear() method\r | |
233 | #\r | |
234 | # Run a preprocess for the file to clean all comments\r | |
235 | #\r | |
236 | # @param self The object pointer\r | |
237 | #\r | |
238 | def PreprocessFileWithClear(self):\r | |
239 | \r | |
240 | self.Rewind()\r | |
241 | InComment = False\r | |
242 | DoubleSlashComment = False\r | |
243 | HashComment = False\r | |
244 | PPExtend = False\r | |
245 | PPDirectiveObj = None\r | |
246 | # HashComment in quoted string " " is ignored.\r | |
247 | InString = False\r | |
248 | InCharLiteral = False\r | |
249 | \r | |
250 | self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]\r | |
251 | while not self.__EndOfFile():\r | |
252 | \r | |
253 | if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:\r | |
254 | InString = not InString\r | |
255 | \r | |
256 | if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:\r | |
257 | InCharLiteral = not InCharLiteral\r | |
258 | # meet new line, then no longer in a comment for // and '#'\r | |
259 | if self.__CurrentChar() == T_CHAR_LF:\r | |
4231a819 | 260 | if HashComment and PPDirectiveObj is not None:\r |
52302d4d LG |
261 | if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r |
262 | PPDirectiveObj.Content += T_CHAR_LF\r | |
263 | PPExtend = True\r | |
264 | else:\r | |
265 | PPExtend = False\r | |
266 | \r | |
267 | EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r | |
268 | \r | |
269 | if InComment and DoubleSlashComment:\r | |
270 | InComment = False\r | |
271 | DoubleSlashComment = False\r | |
272 | \r | |
273 | if InComment and HashComment and not PPExtend:\r | |
274 | InComment = False\r | |
275 | HashComment = False\r | |
276 | PPDirectiveObj.Content += T_CHAR_LF\r | |
277 | PPDirectiveObj.EndPos = EndLinePos\r | |
278 | FileProfile.PPDirectiveList.append(PPDirectiveObj)\r | |
279 | PPDirectiveObj = None\r | |
280 | \r | |
281 | if InString or InCharLiteral:\r | |
282 | CurrentLine = "".join(self.__CurrentLine())\r | |
283 | if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r | |
284 | SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)\r | |
285 | self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)\r | |
286 | \r | |
287 | self.CurrentLineNumber += 1\r | |
288 | self.CurrentOffsetWithinLine = 0\r | |
289 | # check for */ comment end\r | |
290 | elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:\r | |
291 | \r | |
292 | self.__SetCurrentCharValue(T_CHAR_SPACE)\r | |
293 | self.__GetOneChar()\r | |
294 | self.__SetCurrentCharValue(T_CHAR_SPACE)\r | |
295 | self.__GetOneChar()\r | |
296 | InComment = False\r | |
297 | # set comments to spaces\r | |
298 | elif InComment:\r | |
299 | if HashComment:\r | |
300 | # // follows hash PP directive\r | |
301 | if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r | |
302 | InComment = False\r | |
303 | HashComment = False\r | |
304 | PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)\r | |
305 | FileProfile.PPDirectiveList.append(PPDirectiveObj)\r | |
306 | PPDirectiveObj = None\r | |
307 | continue\r | |
308 | else:\r | |
309 | PPDirectiveObj.Content += self.__CurrentChar()\r | |
310 | \r | |
311 | self.__SetCurrentCharValue(T_CHAR_SPACE)\r | |
312 | self.__GetOneChar()\r | |
313 | # check for // comment\r | |
314 | elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r | |
315 | InComment = True\r | |
316 | DoubleSlashComment = True\r | |
317 | \r | |
318 | # check for '#' comment\r | |
319 | elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:\r | |
320 | InComment = True\r | |
321 | HashComment = True\r | |
322 | PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)\r | |
323 | # check for /* comment start\r | |
324 | elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:\r | |
325 | \r | |
326 | self.__SetCurrentCharValue( T_CHAR_SPACE)\r | |
327 | self.__GetOneChar()\r | |
328 | self.__SetCurrentCharValue( T_CHAR_SPACE)\r | |
329 | self.__GetOneChar()\r | |
330 | InComment = True\r | |
331 | else:\r | |
332 | self.__GetOneChar()\r | |
333 | \r | |
334 | EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r | |
335 | \r | |
336 | if InComment and HashComment and not PPExtend:\r | |
337 | PPDirectiveObj.EndPos = EndLinePos\r | |
338 | FileProfile.PPDirectiveList.append(PPDirectiveObj)\r | |
339 | self.Rewind()\r | |
340 | \r | |
341 | ## ParseFile() method\r | |
342 | #\r | |
343 | # Parse the file profile buffer to extract fd, fv ... information\r | |
344 | # Exception will be raised if syntax error found\r | |
345 | #\r | |
346 | # @param self The object pointer\r | |
347 | #\r | |
348 | def ParseFile(self):\r | |
349 | self.PreprocessFileWithClear()\r | |
350 | # restore from ListOfList to ListOfString\r | |
351 | self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]\r | |
352 | FileStringContents = ''\r | |
353 | for fileLine in self.Profile.FileLinesList:\r | |
354 | FileStringContents += fileLine\r | |
355 | cStream = antlr3.StringStream(FileStringContents)\r | |
356 | lexer = CLexer(cStream)\r | |
357 | tStream = antlr3.CommonTokenStream(lexer)\r | |
358 | parser = CParser(tStream)\r | |
359 | parser.translation_unit()\r | |
360 | \r | |
361 | ## CleanFileProfileBuffer() method\r | |
362 | #\r | |
363 | # Reset all contents of the profile of a file\r | |
364 | #\r | |
365 | def CleanFileProfileBuffer(self):\r | |
366 | \r | |
367 | FileProfile.PPDirectiveList = []\r | |
368 | FileProfile.AssignmentExpressionList = []\r | |
369 | FileProfile.FunctionDefinitionList = []\r | |
370 | FileProfile.VariableDeclarationList = []\r | |
371 | FileProfile.EnumerationDefinitionList = []\r | |
372 | FileProfile.StructUnionDefinitionList = []\r | |
373 | FileProfile.TypedefDefinitionList = []\r | |
374 | FileProfile.FunctionCallingList = []\r | |
375 | \r | |
376 | ## PrintFragments() method\r | |
377 | #\r | |
378 | # Print the contents of the profile of a file\r | |
379 | #\r | |
380 | def PrintFragments(self):\r | |
381 | \r | |
382 | print '################# ' + self.FileName + '#####################'\r | |
383 | \r | |
384 | print '/****************************************/'\r | |
385 | print '/*************** ASSIGNMENTS ***************/'\r | |
386 | print '/****************************************/'\r | |
387 | for asign in FileProfile.AssignmentExpressionList:\r | |
388 | print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value\r | |
389 | \r | |
390 | print '/****************************************/'\r | |
391 | print '/********* PREPROCESS DIRECTIVES ********/'\r | |
392 | print '/****************************************/'\r | |
393 | for pp in FileProfile.PPDirectiveList:\r | |
394 | print str(pp.StartPos) + pp.Content\r | |
395 | \r | |
396 | print '/****************************************/'\r | |
397 | print '/********* VARIABLE DECLARATIONS ********/'\r | |
398 | print '/****************************************/'\r | |
399 | for var in FileProfile.VariableDeclarationList:\r | |
400 | print str(var.StartPos) + var.Modifier + ' '+ var.Declarator\r | |
401 | \r | |
402 | print '/****************************************/'\r | |
403 | print '/********* FUNCTION DEFINITIONS *********/'\r | |
404 | print '/****************************************/'\r | |
405 | for func in FileProfile.FunctionDefinitionList:\r | |
406 | print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)\r | |
407 | \r | |
408 | print '/****************************************/'\r | |
409 | print '/************ ENUMERATIONS **************/'\r | |
410 | print '/****************************************/'\r | |
411 | for enum in FileProfile.EnumerationDefinitionList:\r | |
412 | print str(enum.StartPos) + enum.Content\r | |
413 | \r | |
414 | print '/****************************************/'\r | |
415 | print '/*********** STRUCTS/UNIONS *************/'\r | |
416 | print '/****************************************/'\r | |
417 | for su in FileProfile.StructUnionDefinitionList:\r | |
418 | print str(su.StartPos) + su.Content\r | |
419 | \r | |
420 | print '/****************************************/'\r | |
421 | print '/************** TYPEDEFS ****************/'\r | |
422 | print '/****************************************/'\r | |
423 | for typedef in FileProfile.TypedefDefinitionList:\r | |
424 | print str(typedef.StartPos) + typedef.ToType\r | |
425 | \r | |
426 | ##\r | |
427 | #\r | |
428 | # This acts like the main() function for the script, unless it is 'import'ed into another\r | |
429 | # script.\r | |
430 | #\r | |
431 | if __name__ == "__main__":\r | |
432 | \r | |
433 | print "For Test."\r |