BaseTools: Use absolute import in Eot
[mirror_edk2.git] / BaseTools / Source / Python / Eot / CodeFragmentCollector.py
CommitLineData
52302d4d
LG
1## @file\r
2# preprocess source file\r
3#\r
231fdbb2 4# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>\r
52302d4d 5#\r
40d841f6 6# This program and the accompanying materials\r
52302d4d
LG
7# are licensed and made available under the terms and conditions of the BSD License\r
8# which accompanies this distribution. The full text of the license may be found at\r
9# http://opensource.org/licenses/bsd-license.php\r
10#\r
11# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
12# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
13#\r
14\r
15##\r
16# Import Modules\r
17#\r
72443dd2 18from __future__ import print_function\r
64429fbd 19from __future__ import absolute_import\r
52302d4d 20import re\r
1be2ed90 21import Common.LongFilePathOs as os\r
52302d4d
LG
22import sys\r
23\r
24import antlr3\r
64429fbd
GL
25from .CLexer import CLexer\r
26from .CParser import CParser\r
52302d4d 27\r
64429fbd
GL
28from . import FileProfile\r
29from .CodeFragment import PP_Directive\r
30from .ParserWarning import Warning\r
52302d4d
LG
31\r
32\r
33##define T_CHAR_SPACE ' '\r
34##define T_CHAR_NULL '\0'\r
35##define T_CHAR_CR '\r'\r
36##define T_CHAR_TAB '\t'\r
37##define T_CHAR_LF '\n'\r
38##define T_CHAR_SLASH '/'\r
39##define T_CHAR_BACKSLASH '\\'\r
40##define T_CHAR_DOUBLE_QUOTE '\"'\r
41##define T_CHAR_SINGLE_QUOTE '\''\r
42##define T_CHAR_STAR '*'\r
43##define T_CHAR_HASH '#'\r
44\r
45(T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \\r
46T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \\r
47(' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')\r
48\r
49SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')\r
50\r
51(T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)\r
52\r
53(T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)\r
54\r
55## The collector for source code fragments.\r
56#\r
57# PreprocessFile method should be called prior to ParseFile\r
58#\r
59# GetNext*** procedures mean these procedures will get next token first, then make judgement.\r
60# Get*** procedures mean these procedures will make judgement on current token only.\r
61#\r
62class CodeFragmentCollector:\r
63 ## The constructor\r
64 #\r
65 # @param self The object pointer\r
66 # @param FileName The file that to be parsed\r
67 #\r
68 def __init__(self, FileName):\r
69 self.Profile = FileProfile.FileProfile(FileName)\r
70 self.Profile.FileLinesList.append(T_CHAR_LF)\r
71 self.FileName = FileName\r
72 self.CurrentLineNumber = 1\r
73 self.CurrentOffsetWithinLine = 0\r
74\r
75 self.__Token = ""\r
76 self.__SkippedChars = ""\r
77\r
52302d4d
LG
78 ## __EndOfFile() method\r
79 #\r
80 # Judge current buffer pos is at file end\r
81 #\r
82 # @param self The object pointer\r
83 # @retval True Current File buffer position is at file end\r
84 # @retval False Current File buffer position is NOT at file end\r
85 #\r
86 def __EndOfFile(self):\r
87 NumberOfLines = len(self.Profile.FileLinesList)\r
88 SizeOfLastLine = len(self.Profile.FileLinesList[-1])\r
89 if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:\r
90 return True\r
91 elif self.CurrentLineNumber > NumberOfLines:\r
92 return True\r
93 else:\r
94 return False\r
95\r
96 ## __EndOfLine() method\r
97 #\r
98 # Judge current buffer pos is at line end\r
99 #\r
100 # @param self The object pointer\r
101 # @retval True Current File buffer position is at line end\r
102 # @retval False Current File buffer position is NOT at line end\r
103 #\r
104 def __EndOfLine(self):\r
105 SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])\r
106 if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:\r
107 return True\r
108 else:\r
109 return False\r
110\r
111 ## Rewind() method\r
112 #\r
113 # Reset file data buffer to the initial state\r
114 #\r
115 # @param self The object pointer\r
116 #\r
117 def Rewind(self):\r
118 self.CurrentLineNumber = 1\r
119 self.CurrentOffsetWithinLine = 0\r
120\r
121 ## __UndoOneChar() method\r
122 #\r
123 # Go back one char in the file buffer\r
124 #\r
125 # @param self The object pointer\r
126 # @retval True Successfully go back one char\r
127 # @retval False Not able to go back one char as file beginning reached\r
128 #\r
129 def __UndoOneChar(self):\r
130\r
131 if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:\r
132 return False\r
133 elif self.CurrentOffsetWithinLine == 0:\r
134 self.CurrentLineNumber -= 1\r
135 self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1\r
136 else:\r
137 self.CurrentOffsetWithinLine -= 1\r
138 return True\r
139\r
140 ## __GetOneChar() method\r
141 #\r
142 # Move forward one char in the file buffer\r
143 #\r
144 # @param self The object pointer\r
145 #\r
146 def __GetOneChar(self):\r
147 if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r
148 self.CurrentLineNumber += 1\r
149 self.CurrentOffsetWithinLine = 0\r
150 else:\r
151 self.CurrentOffsetWithinLine += 1\r
152\r
153 ## __CurrentChar() method\r
154 #\r
155 # Get the char pointed to by the file buffer pointer\r
156 #\r
157 # @param self The object pointer\r
158 # @retval Char Current char\r
159 #\r
160 def __CurrentChar(self):\r
161 CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]\r
162\r
163 return CurrentChar\r
164\r
165 ## __NextChar() method\r
166 #\r
167 # Get the one char pass the char pointed to by the file buffer pointer\r
168 #\r
169 # @param self The object pointer\r
170 # @retval Char Next char\r
171 #\r
172 def __NextChar(self):\r
173 if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r
174 return self.Profile.FileLinesList[self.CurrentLineNumber][0]\r
175 else:\r
176 return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]\r
177\r
178 ## __SetCurrentCharValue() method\r
179 #\r
180 # Modify the value of current char\r
181 #\r
182 # @param self The object pointer\r
183 # @param Value The new value of current char\r
184 #\r
185 def __SetCurrentCharValue(self, Value):\r
186 self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value\r
187\r
188 ## __SetCharValue() method\r
189 #\r
190 # Modify the value of current char\r
191 #\r
192 # @param self The object pointer\r
193 # @param Value The new value of current char\r
194 #\r
195 def __SetCharValue(self, Line, Offset, Value):\r
196 self.Profile.FileLinesList[Line - 1][Offset] = Value\r
197\r
198 ## __CurrentLine() method\r
199 #\r
200 # Get the list that contains current line contents\r
201 #\r
202 # @param self The object pointer\r
203 # @retval List current line contents\r
204 #\r
205 def __CurrentLine(self):\r
206 return self.Profile.FileLinesList[self.CurrentLineNumber - 1]\r
207\r
208 ## __InsertComma() method\r
209 #\r
210 # Insert ',' to replace PP\r
211 #\r
212 # @param self The object pointer\r
213 # @retval List current line contents\r
214 #\r
215 def __InsertComma(self, Line):\r
216\r
217\r
218 if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:\r
219 BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]\r
220 if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):\r
221 return\r
222\r
223 if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):\r
224 return\r
225\r
226 if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):\r
227 return\r
228\r
229 if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):\r
230 return\r
231\r
232 self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')\r
233\r
234 ## PreprocessFileWithClear() method\r
235 #\r
236 # Run a preprocess for the file to clean all comments\r
237 #\r
238 # @param self The object pointer\r
239 #\r
240 def PreprocessFileWithClear(self):\r
241\r
242 self.Rewind()\r
243 InComment = False\r
244 DoubleSlashComment = False\r
245 HashComment = False\r
246 PPExtend = False\r
247 PPDirectiveObj = None\r
248 # HashComment in quoted string " " is ignored.\r
249 InString = False\r
250 InCharLiteral = False\r
251\r
252 self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]\r
253 while not self.__EndOfFile():\r
254\r
255 if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:\r
256 InString = not InString\r
257\r
258 if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:\r
259 InCharLiteral = not InCharLiteral\r
260 # meet new line, then no longer in a comment for // and '#'\r
261 if self.__CurrentChar() == T_CHAR_LF:\r
4231a819 262 if HashComment and PPDirectiveObj is not None:\r
52302d4d
LG
263 if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r
264 PPDirectiveObj.Content += T_CHAR_LF\r
265 PPExtend = True\r
266 else:\r
267 PPExtend = False\r
268\r
269 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r
270\r
271 if InComment and DoubleSlashComment:\r
272 InComment = False\r
273 DoubleSlashComment = False\r
274\r
275 if InComment and HashComment and not PPExtend:\r
276 InComment = False\r
277 HashComment = False\r
278 PPDirectiveObj.Content += T_CHAR_LF\r
279 PPDirectiveObj.EndPos = EndLinePos\r
280 FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
281 PPDirectiveObj = None\r
282\r
283 if InString or InCharLiteral:\r
284 CurrentLine = "".join(self.__CurrentLine())\r
285 if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r
286 SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)\r
287 self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)\r
288\r
289 self.CurrentLineNumber += 1\r
290 self.CurrentOffsetWithinLine = 0\r
291 # check for */ comment end\r
292 elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:\r
293\r
294 self.__SetCurrentCharValue(T_CHAR_SPACE)\r
295 self.__GetOneChar()\r
296 self.__SetCurrentCharValue(T_CHAR_SPACE)\r
297 self.__GetOneChar()\r
298 InComment = False\r
299 # set comments to spaces\r
300 elif InComment:\r
301 if HashComment:\r
302 # // follows hash PP directive\r
303 if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r
304 InComment = False\r
305 HashComment = False\r
306 PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)\r
307 FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
308 PPDirectiveObj = None\r
309 continue\r
310 else:\r
311 PPDirectiveObj.Content += self.__CurrentChar()\r
312\r
313 self.__SetCurrentCharValue(T_CHAR_SPACE)\r
314 self.__GetOneChar()\r
315 # check for // comment\r
316 elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r
317 InComment = True\r
318 DoubleSlashComment = True\r
319\r
320 # check for '#' comment\r
321 elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:\r
322 InComment = True\r
323 HashComment = True\r
324 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)\r
325 # check for /* comment start\r
326 elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:\r
327\r
328 self.__SetCurrentCharValue( T_CHAR_SPACE)\r
329 self.__GetOneChar()\r
330 self.__SetCurrentCharValue( T_CHAR_SPACE)\r
331 self.__GetOneChar()\r
332 InComment = True\r
333 else:\r
334 self.__GetOneChar()\r
335\r
336 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r
337\r
338 if InComment and HashComment and not PPExtend:\r
339 PPDirectiveObj.EndPos = EndLinePos\r
340 FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
341 self.Rewind()\r
342\r
343 ## ParseFile() method\r
344 #\r
345 # Parse the file profile buffer to extract fd, fv ... information\r
346 # Exception will be raised if syntax error found\r
347 #\r
348 # @param self The object pointer\r
349 #\r
350 def ParseFile(self):\r
351 self.PreprocessFileWithClear()\r
352 # restore from ListOfList to ListOfString\r
353 self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]\r
354 FileStringContents = ''\r
355 for fileLine in self.Profile.FileLinesList:\r
356 FileStringContents += fileLine\r
357 cStream = antlr3.StringStream(FileStringContents)\r
358 lexer = CLexer(cStream)\r
359 tStream = antlr3.CommonTokenStream(lexer)\r
360 parser = CParser(tStream)\r
361 parser.translation_unit()\r
362\r
363 ## CleanFileProfileBuffer() method\r
364 #\r
365 # Reset all contents of the profile of a file\r
366 #\r
367 def CleanFileProfileBuffer(self):\r
368\r
369 FileProfile.PPDirectiveList = []\r
370 FileProfile.AssignmentExpressionList = []\r
371 FileProfile.FunctionDefinitionList = []\r
372 FileProfile.VariableDeclarationList = []\r
373 FileProfile.EnumerationDefinitionList = []\r
374 FileProfile.StructUnionDefinitionList = []\r
375 FileProfile.TypedefDefinitionList = []\r
376 FileProfile.FunctionCallingList = []\r
377\r
378 ## PrintFragments() method\r
379 #\r
380 # Print the contents of the profile of a file\r
381 #\r
382 def PrintFragments(self):\r
383\r
72443dd2 384 print('################# ' + self.FileName + '#####################')\r
52302d4d 385\r
72443dd2
GL
386 print('/****************************************/')\r
387 print('/*************** ASSIGNMENTS ***************/')\r
388 print('/****************************************/')\r
52302d4d 389 for asign in FileProfile.AssignmentExpressionList:\r
72443dd2 390 print(str(asign.StartPos) + asign.Name + asign.Operator + asign.Value)\r
52302d4d 391\r
72443dd2
GL
392 print('/****************************************/')\r
393 print('/********* PREPROCESS DIRECTIVES ********/')\r
394 print('/****************************************/')\r
52302d4d 395 for pp in FileProfile.PPDirectiveList:\r
72443dd2 396 print(str(pp.StartPos) + pp.Content)\r
52302d4d 397\r
72443dd2
GL
398 print('/****************************************/')\r
399 print('/********* VARIABLE DECLARATIONS ********/')\r
400 print('/****************************************/')\r
52302d4d 401 for var in FileProfile.VariableDeclarationList:\r
72443dd2 402 print(str(var.StartPos) + var.Modifier + ' '+ var.Declarator)\r
52302d4d 403\r
72443dd2
GL
404 print('/****************************************/')\r
405 print('/********* FUNCTION DEFINITIONS *********/')\r
406 print('/****************************************/')\r
52302d4d 407 for func in FileProfile.FunctionDefinitionList:\r
72443dd2 408 print(str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos))\r
52302d4d 409\r
72443dd2
GL
410 print('/****************************************/')\r
411 print('/************ ENUMERATIONS **************/')\r
412 print('/****************************************/')\r
52302d4d 413 for enum in FileProfile.EnumerationDefinitionList:\r
72443dd2 414 print(str(enum.StartPos) + enum.Content)\r
52302d4d 415\r
72443dd2
GL
416 print('/****************************************/')\r
417 print('/*********** STRUCTS/UNIONS *************/')\r
418 print('/****************************************/')\r
52302d4d 419 for su in FileProfile.StructUnionDefinitionList:\r
72443dd2 420 print(str(su.StartPos) + su.Content)\r
52302d4d 421\r
72443dd2
GL
422 print('/****************************************/')\r
423 print('/************** TYPEDEFS ****************/')\r
424 print('/****************************************/')\r
52302d4d 425 for typedef in FileProfile.TypedefDefinitionList:\r
72443dd2 426 print(str(typedef.StartPos) + typedef.ToType)\r
52302d4d
LG
427\r
428##\r
429#\r
430# This acts like the main() function for the script, unless it is 'import'ed into another\r
431# script.\r
432#\r
433if __name__ == "__main__":\r
434\r
72443dd2 435 print("For Test.")\r