]> git.proxmox.com Git - mirror_edk2.git/blame - BaseTools/Source/Python/Eot/CodeFragmentCollector.py
BaseTools: Fix indentation in CParser.py file
[mirror_edk2.git] / BaseTools / Source / Python / Eot / CodeFragmentCollector.py
CommitLineData
52302d4d
LG
1## @file\r
2# preprocess source file\r
3#\r
1be2ed90 4# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>\r
52302d4d 5#\r
40d841f6 6# This program and the accompanying materials\r
52302d4d
LG
7# are licensed and made available under the terms and conditions of the BSD License\r
8# which accompanies this distribution. The full text of the license may be found at\r
9# http://opensource.org/licenses/bsd-license.php\r
10#\r
11# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
12# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
13#\r
14\r
15##\r
16# Import Modules\r
17#\r
18import re\r
1be2ed90 19import Common.LongFilePathOs as os\r
52302d4d
LG
20import sys\r
21\r
22import antlr3\r
23from CLexer import CLexer\r
24from CParser import CParser\r
25\r
26import FileProfile\r
27from CodeFragment import PP_Directive\r
28from ParserWarning import Warning\r
29\r
30\r
31##define T_CHAR_SPACE ' '\r
32##define T_CHAR_NULL '\0'\r
33##define T_CHAR_CR '\r'\r
34##define T_CHAR_TAB '\t'\r
35##define T_CHAR_LF '\n'\r
36##define T_CHAR_SLASH '/'\r
37##define T_CHAR_BACKSLASH '\\'\r
38##define T_CHAR_DOUBLE_QUOTE '\"'\r
39##define T_CHAR_SINGLE_QUOTE '\''\r
40##define T_CHAR_STAR '*'\r
41##define T_CHAR_HASH '#'\r
42\r
43(T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \\r
44T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \\r
45(' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')\r
46\r
47SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')\r
48\r
49(T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)\r
50\r
51(T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)\r
52\r
53## The collector for source code fragments.\r
54#\r
55# PreprocessFile method should be called prior to ParseFile\r
56#\r
57# GetNext*** procedures mean these procedures will get next token first, then make judgement.\r
58# Get*** procedures mean these procedures will make judgement on current token only.\r
59#\r
60class CodeFragmentCollector:\r
61 ## The constructor\r
62 #\r
63 # @param self The object pointer\r
64 # @param FileName The file that to be parsed\r
65 #\r
66 def __init__(self, FileName):\r
67 self.Profile = FileProfile.FileProfile(FileName)\r
68 self.Profile.FileLinesList.append(T_CHAR_LF)\r
69 self.FileName = FileName\r
70 self.CurrentLineNumber = 1\r
71 self.CurrentOffsetWithinLine = 0\r
72\r
73 self.__Token = ""\r
74 self.__SkippedChars = ""\r
75\r
76 ## __IsWhiteSpace() method\r
77 #\r
78 # Whether char at current FileBufferPos is whitespace\r
79 #\r
80 # @param self The object pointer\r
81 # @param Char The char to test\r
82 # @retval True The char is a kind of white space\r
83 # @retval False The char is NOT a kind of white space\r
84 #\r
85 def __IsWhiteSpace(self, Char):\r
86 if Char in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_SPACE, T_CHAR_TAB, T_CHAR_LF):\r
87 return True\r
88 else:\r
89 return False\r
90\r
91 ## __SkipWhiteSpace() method\r
92 #\r
93 # Skip white spaces from current char, return number of chars skipped\r
94 #\r
95 # @param self The object pointer\r
96 # @retval Count The number of chars skipped\r
97 #\r
98 def __SkipWhiteSpace(self):\r
99 Count = 0\r
100 while not self.__EndOfFile():\r
101 Count += 1\r
102 if self.__CurrentChar() in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_LF, T_CHAR_SPACE, T_CHAR_TAB):\r
103 self.__SkippedChars += str(self.__CurrentChar())\r
104 self.__GetOneChar()\r
105\r
106 else:\r
107 Count = Count - 1\r
108 return Count\r
109\r
110 ## __EndOfFile() method\r
111 #\r
112 # Judge current buffer pos is at file end\r
113 #\r
114 # @param self The object pointer\r
115 # @retval True Current File buffer position is at file end\r
116 # @retval False Current File buffer position is NOT at file end\r
117 #\r
118 def __EndOfFile(self):\r
119 NumberOfLines = len(self.Profile.FileLinesList)\r
120 SizeOfLastLine = len(self.Profile.FileLinesList[-1])\r
121 if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:\r
122 return True\r
123 elif self.CurrentLineNumber > NumberOfLines:\r
124 return True\r
125 else:\r
126 return False\r
127\r
128 ## __EndOfLine() method\r
129 #\r
130 # Judge current buffer pos is at line end\r
131 #\r
132 # @param self The object pointer\r
133 # @retval True Current File buffer position is at line end\r
134 # @retval False Current File buffer position is NOT at line end\r
135 #\r
136 def __EndOfLine(self):\r
137 SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])\r
138 if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:\r
139 return True\r
140 else:\r
141 return False\r
142\r
143 ## Rewind() method\r
144 #\r
145 # Reset file data buffer to the initial state\r
146 #\r
147 # @param self The object pointer\r
148 #\r
149 def Rewind(self):\r
150 self.CurrentLineNumber = 1\r
151 self.CurrentOffsetWithinLine = 0\r
152\r
153 ## __UndoOneChar() method\r
154 #\r
155 # Go back one char in the file buffer\r
156 #\r
157 # @param self The object pointer\r
158 # @retval True Successfully go back one char\r
159 # @retval False Not able to go back one char as file beginning reached\r
160 #\r
161 def __UndoOneChar(self):\r
162\r
163 if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:\r
164 return False\r
165 elif self.CurrentOffsetWithinLine == 0:\r
166 self.CurrentLineNumber -= 1\r
167 self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1\r
168 else:\r
169 self.CurrentOffsetWithinLine -= 1\r
170 return True\r
171\r
172 ## __GetOneChar() method\r
173 #\r
174 # Move forward one char in the file buffer\r
175 #\r
176 # @param self The object pointer\r
177 #\r
178 def __GetOneChar(self):\r
179 if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r
180 self.CurrentLineNumber += 1\r
181 self.CurrentOffsetWithinLine = 0\r
182 else:\r
183 self.CurrentOffsetWithinLine += 1\r
184\r
185 ## __CurrentChar() method\r
186 #\r
187 # Get the char pointed to by the file buffer pointer\r
188 #\r
189 # @param self The object pointer\r
190 # @retval Char Current char\r
191 #\r
192 def __CurrentChar(self):\r
193 CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]\r
194\r
195 return CurrentChar\r
196\r
197 ## __NextChar() method\r
198 #\r
199 # Get the one char pass the char pointed to by the file buffer pointer\r
200 #\r
201 # @param self The object pointer\r
202 # @retval Char Next char\r
203 #\r
204 def __NextChar(self):\r
205 if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r
206 return self.Profile.FileLinesList[self.CurrentLineNumber][0]\r
207 else:\r
208 return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]\r
209\r
210 ## __SetCurrentCharValue() method\r
211 #\r
212 # Modify the value of current char\r
213 #\r
214 # @param self The object pointer\r
215 # @param Value The new value of current char\r
216 #\r
217 def __SetCurrentCharValue(self, Value):\r
218 self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value\r
219\r
220 ## __SetCharValue() method\r
221 #\r
222 # Modify the value of current char\r
223 #\r
224 # @param self The object pointer\r
225 # @param Value The new value of current char\r
226 #\r
227 def __SetCharValue(self, Line, Offset, Value):\r
228 self.Profile.FileLinesList[Line - 1][Offset] = Value\r
229\r
230 ## __CurrentLine() method\r
231 #\r
232 # Get the list that contains current line contents\r
233 #\r
234 # @param self The object pointer\r
235 # @retval List current line contents\r
236 #\r
237 def __CurrentLine(self):\r
238 return self.Profile.FileLinesList[self.CurrentLineNumber - 1]\r
239\r
240 ## __InsertComma() method\r
241 #\r
242 # Insert ',' to replace PP\r
243 #\r
244 # @param self The object pointer\r
245 # @retval List current line contents\r
246 #\r
247 def __InsertComma(self, Line):\r
248\r
249\r
250 if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:\r
251 BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]\r
252 if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):\r
253 return\r
254\r
255 if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):\r
256 return\r
257\r
258 if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):\r
259 return\r
260\r
261 if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):\r
262 return\r
263\r
264 self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')\r
265\r
266 ## PreprocessFileWithClear() method\r
267 #\r
268 # Run a preprocess for the file to clean all comments\r
269 #\r
270 # @param self The object pointer\r
271 #\r
272 def PreprocessFileWithClear(self):\r
273\r
274 self.Rewind()\r
275 InComment = False\r
276 DoubleSlashComment = False\r
277 HashComment = False\r
278 PPExtend = False\r
279 PPDirectiveObj = None\r
280 # HashComment in quoted string " " is ignored.\r
281 InString = False\r
282 InCharLiteral = False\r
283\r
284 self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]\r
285 while not self.__EndOfFile():\r
286\r
287 if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:\r
288 InString = not InString\r
289\r
290 if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:\r
291 InCharLiteral = not InCharLiteral\r
292 # meet new line, then no longer in a comment for // and '#'\r
293 if self.__CurrentChar() == T_CHAR_LF:\r
294 if HashComment and PPDirectiveObj != None:\r
295 if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r
296 PPDirectiveObj.Content += T_CHAR_LF\r
297 PPExtend = True\r
298 else:\r
299 PPExtend = False\r
300\r
301 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r
302\r
303 if InComment and DoubleSlashComment:\r
304 InComment = False\r
305 DoubleSlashComment = False\r
306\r
307 if InComment and HashComment and not PPExtend:\r
308 InComment = False\r
309 HashComment = False\r
310 PPDirectiveObj.Content += T_CHAR_LF\r
311 PPDirectiveObj.EndPos = EndLinePos\r
312 FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
313 PPDirectiveObj = None\r
314\r
315 if InString or InCharLiteral:\r
316 CurrentLine = "".join(self.__CurrentLine())\r
317 if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r
318 SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)\r
319 self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)\r
320\r
321 self.CurrentLineNumber += 1\r
322 self.CurrentOffsetWithinLine = 0\r
323 # check for */ comment end\r
324 elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:\r
325\r
326 self.__SetCurrentCharValue(T_CHAR_SPACE)\r
327 self.__GetOneChar()\r
328 self.__SetCurrentCharValue(T_CHAR_SPACE)\r
329 self.__GetOneChar()\r
330 InComment = False\r
331 # set comments to spaces\r
332 elif InComment:\r
333 if HashComment:\r
334 # // follows hash PP directive\r
335 if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r
336 InComment = False\r
337 HashComment = False\r
338 PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)\r
339 FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
340 PPDirectiveObj = None\r
341 continue\r
342 else:\r
343 PPDirectiveObj.Content += self.__CurrentChar()\r
344\r
345 self.__SetCurrentCharValue(T_CHAR_SPACE)\r
346 self.__GetOneChar()\r
347 # check for // comment\r
348 elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r
349 InComment = True\r
350 DoubleSlashComment = True\r
351\r
352 # check for '#' comment\r
353 elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:\r
354 InComment = True\r
355 HashComment = True\r
356 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)\r
357 # check for /* comment start\r
358 elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:\r
359\r
360 self.__SetCurrentCharValue( T_CHAR_SPACE)\r
361 self.__GetOneChar()\r
362 self.__SetCurrentCharValue( T_CHAR_SPACE)\r
363 self.__GetOneChar()\r
364 InComment = True\r
365 else:\r
366 self.__GetOneChar()\r
367\r
368 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r
369\r
370 if InComment and HashComment and not PPExtend:\r
371 PPDirectiveObj.EndPos = EndLinePos\r
372 FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
373 self.Rewind()\r
374\r
375 ## ParseFile() method\r
376 #\r
377 # Parse the file profile buffer to extract fd, fv ... information\r
378 # Exception will be raised if syntax error found\r
379 #\r
380 # @param self The object pointer\r
381 #\r
382 def ParseFile(self):\r
383 self.PreprocessFileWithClear()\r
384 # restore from ListOfList to ListOfString\r
385 self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]\r
386 FileStringContents = ''\r
387 for fileLine in self.Profile.FileLinesList:\r
388 FileStringContents += fileLine\r
389 cStream = antlr3.StringStream(FileStringContents)\r
390 lexer = CLexer(cStream)\r
391 tStream = antlr3.CommonTokenStream(lexer)\r
392 parser = CParser(tStream)\r
393 parser.translation_unit()\r
394\r
395 ## CleanFileProfileBuffer() method\r
396 #\r
397 # Reset all contents of the profile of a file\r
398 #\r
399 def CleanFileProfileBuffer(self):\r
400\r
401 FileProfile.PPDirectiveList = []\r
402 FileProfile.AssignmentExpressionList = []\r
403 FileProfile.FunctionDefinitionList = []\r
404 FileProfile.VariableDeclarationList = []\r
405 FileProfile.EnumerationDefinitionList = []\r
406 FileProfile.StructUnionDefinitionList = []\r
407 FileProfile.TypedefDefinitionList = []\r
408 FileProfile.FunctionCallingList = []\r
409\r
410 ## PrintFragments() method\r
411 #\r
412 # Print the contents of the profile of a file\r
413 #\r
414 def PrintFragments(self):\r
415\r
416 print '################# ' + self.FileName + '#####################'\r
417\r
418 print '/****************************************/'\r
419 print '/*************** ASSIGNMENTS ***************/'\r
420 print '/****************************************/'\r
421 for asign in FileProfile.AssignmentExpressionList:\r
422 print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value\r
423\r
424 print '/****************************************/'\r
425 print '/********* PREPROCESS DIRECTIVES ********/'\r
426 print '/****************************************/'\r
427 for pp in FileProfile.PPDirectiveList:\r
428 print str(pp.StartPos) + pp.Content\r
429\r
430 print '/****************************************/'\r
431 print '/********* VARIABLE DECLARATIONS ********/'\r
432 print '/****************************************/'\r
433 for var in FileProfile.VariableDeclarationList:\r
434 print str(var.StartPos) + var.Modifier + ' '+ var.Declarator\r
435\r
436 print '/****************************************/'\r
437 print '/********* FUNCTION DEFINITIONS *********/'\r
438 print '/****************************************/'\r
439 for func in FileProfile.FunctionDefinitionList:\r
440 print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)\r
441\r
442 print '/****************************************/'\r
443 print '/************ ENUMERATIONS **************/'\r
444 print '/****************************************/'\r
445 for enum in FileProfile.EnumerationDefinitionList:\r
446 print str(enum.StartPos) + enum.Content\r
447\r
448 print '/****************************************/'\r
449 print '/*********** STRUCTS/UNIONS *************/'\r
450 print '/****************************************/'\r
451 for su in FileProfile.StructUnionDefinitionList:\r
452 print str(su.StartPos) + su.Content\r
453\r
454 print '/****************************************/'\r
455 print '/************** TYPEDEFS ****************/'\r
456 print '/****************************************/'\r
457 for typedef in FileProfile.TypedefDefinitionList:\r
458 print str(typedef.StartPos) + typedef.ToType\r
459\r
460##\r
461#\r
462# This acts like the main() function for the script, unless it is 'import'ed into another\r
463# script.\r
464#\r
465if __name__ == "__main__":\r
466\r
467 print "For Test."\r