BaseTools/Source/Python/Eot/CodeFragmentCollector.py

   1 ## @file
   2 # preprocess source file
   3 #
   4 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 from __future__ import print_function
  19 import re
  20 import Common.LongFilePathOs as os
  21 import sys
  22
  23 import antlr3
  24 from CLexer import CLexer
  25 from CParser import CParser
  26
  27 import FileProfile
  28 from CodeFragment import PP_Directive
  29 from ParserWarning import Warning
  30
  31
  32 ##define T_CHAR_SPACE                ' '
  33 ##define T_CHAR_NULL                 '\0'
  34 ##define T_CHAR_CR                   '\r'
  35 ##define T_CHAR_TAB                  '\t'
  36 ##define T_CHAR_LF                   '\n'
  37 ##define T_CHAR_SLASH                '/'
  38 ##define T_CHAR_BACKSLASH            '\\'
  39 ##define T_CHAR_DOUBLE_QUOTE         '\"'
  40 ##define T_CHAR_SINGLE_QUOTE         '\''
  41 ##define T_CHAR_STAR                 '*'
  42 ##define T_CHAR_HASH                 '#'
  43
  44 (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
  45 T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
  46 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
  47
  48 SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
  49
  50 (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
  51
  52 (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
  53
  54 ## The collector for source code fragments.
  55 #
  56 # PreprocessFile method should be called prior to ParseFile
  57 #
  58 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
  59 # Get*** procedures mean these procedures will make judgement on current token only.
  60 #
  61 class CodeFragmentCollector:
  62     ## The constructor
  63     #
  64     #   @param  self        The object pointer
  65     #   @param  FileName    The file that to be parsed
  66     #
  67     def __init__(self, FileName):
  68         self.Profile = FileProfile.FileProfile(FileName)
  69         self.Profile.FileLinesList.append(T_CHAR_LF)
  70         self.FileName = FileName
  71         self.CurrentLineNumber = 1
  72         self.CurrentOffsetWithinLine = 0
  73
  74         self.__Token = ""
  75         self.__SkippedChars = ""
  76
  77     ## __EndOfFile() method
  78     #
  79     #   Judge current buffer pos is at file end
  80     #
  81     #   @param  self        The object pointer
  82     #   @retval True        Current File buffer position is at file end
  83     #   @retval False       Current File buffer position is NOT at file end
  84     #
  85     def __EndOfFile(self):
  86         NumberOfLines = len(self.Profile.FileLinesList)
  87         SizeOfLastLine = len(self.Profile.FileLinesList[-1])
  88         if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
  89             return True
  90         elif self.CurrentLineNumber > NumberOfLines:
  91             return True
  92         else:
  93             return False
  94
  95     ## __EndOfLine() method
  96     #
  97     #   Judge current buffer pos is at line end
  98     #
  99     #   @param  self        The object pointer
 100     #   @retval True        Current File buffer position is at line end
 101     #   @retval False       Current File buffer position is NOT at line end
 102     #
 103     def __EndOfLine(self):
 104         SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
 105         if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
 106             return True
 107         else:
 108             return False
 109
 110     ## Rewind() method
 111     #
 112     #   Reset file data buffer to the initial state
 113     #
 114     #   @param  self        The object pointer
 115     #
 116     def Rewind(self):
 117         self.CurrentLineNumber = 1
 118         self.CurrentOffsetWithinLine = 0
 119
 120     ## __UndoOneChar() method
 121     #
 122     #   Go back one char in the file buffer
 123     #
 124     #   @param  self        The object pointer
 125     #   @retval True        Successfully go back one char
 126     #   @retval False       Not able to go back one char as file beginning reached
 127     #
 128     def __UndoOneChar(self):
 129
 130         if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
 131             return False
 132         elif self.CurrentOffsetWithinLine == 0:
 133             self.CurrentLineNumber -= 1
 134             self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
 135         else:
 136             self.CurrentOffsetWithinLine -= 1
 137         return True
 138
 139     ## __GetOneChar() method
 140     #
 141     #   Move forward one char in the file buffer
 142     #
 143     #   @param  self        The object pointer
 144     #
 145     def __GetOneChar(self):
 146         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 147                 self.CurrentLineNumber += 1
 148                 self.CurrentOffsetWithinLine = 0
 149         else:
 150                 self.CurrentOffsetWithinLine += 1
 151
 152     ## __CurrentChar() method
 153     #
 154     #   Get the char pointed to by the file buffer pointer
 155     #
 156     #   @param  self        The object pointer
 157     #   @retval Char        Current char
 158     #
 159     def __CurrentChar(self):
 160         CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
 161
 162         return CurrentChar
 163
 164     ## __NextChar() method
 165     #
 166     #   Get the one char pass the char pointed to by the file buffer pointer
 167     #
 168     #   @param  self        The object pointer
 169     #   @retval Char        Next char
 170     #
 171     def __NextChar(self):
 172         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 173             return self.Profile.FileLinesList[self.CurrentLineNumber][0]
 174         else:
 175             return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
 176
 177     ## __SetCurrentCharValue() method
 178     #
 179     #   Modify the value of current char
 180     #
 181     #   @param  self        The object pointer
 182     #   @param  Value       The new value of current char
 183     #
 184     def __SetCurrentCharValue(self, Value):
 185         self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
 186
 187     ## __SetCharValue() method
 188     #
 189     #   Modify the value of current char
 190     #
 191     #   @param  self        The object pointer
 192     #   @param  Value       The new value of current char
 193     #
 194     def __SetCharValue(self, Line, Offset, Value):
 195         self.Profile.FileLinesList[Line - 1][Offset] = Value
 196
 197     ## __CurrentLine() method
 198     #
 199     #   Get the list that contains current line contents
 200     #
 201     #   @param  self        The object pointer
 202     #   @retval List        current line contents
 203     #
 204     def __CurrentLine(self):
 205         return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
 206
 207     ## __InsertComma() method
 208     #
 209     #   Insert ',' to replace PP
 210     #
 211     #   @param  self        The object pointer
 212     #   @retval List        current line contents
 213     #
 214     def __InsertComma(self, Line):
 215
 216
 217         if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
 218             BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
 219             if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
 220                 return
 221
 222         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
 223             return
 224
 225         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
 226             return
 227
 228         if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
 229             return
 230
 231         self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
 232
 233     ## PreprocessFileWithClear() method
 234     #
 235     # Run a preprocess for the file to clean all comments
 236     #
 237     # @param  self        The object pointer
 238     #
 239     def PreprocessFileWithClear(self):
 240
 241         self.Rewind()
 242         InComment = False
 243         DoubleSlashComment = False
 244         HashComment = False
 245         PPExtend = False
 246         PPDirectiveObj = None
 247         # HashComment in quoted string " " is ignored.
 248         InString = False
 249         InCharLiteral = False
 250
 251         self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
 252         while not self.__EndOfFile():
 253
 254             if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
 255                 InString = not InString
 256
 257             if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
 258                 InCharLiteral = not InCharLiteral
 259             # meet new line, then no longer in a comment for // and '#'
 260             if self.__CurrentChar() == T_CHAR_LF:
 261                 if HashComment and PPDirectiveObj is not None:
 262                     if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 263                         PPDirectiveObj.Content += T_CHAR_LF
 264                         PPExtend = True
 265                     else:
 266                         PPExtend = False
 267
 268                 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 269
 270                 if InComment and DoubleSlashComment:
 271                     InComment = False
 272                     DoubleSlashComment = False
 273
 274                 if InComment and HashComment and not PPExtend:
 275                     InComment = False
 276                     HashComment = False
 277                     PPDirectiveObj.Content += T_CHAR_LF
 278                     PPDirectiveObj.EndPos = EndLinePos
 279                     FileProfile.PPDirectiveList.append(PPDirectiveObj)
 280                     PPDirectiveObj = None
 281
 282                 if InString or InCharLiteral:
 283                     CurrentLine = "".join(self.__CurrentLine())
 284                     if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 285                         SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
 286                         self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
 287
 288                 self.CurrentLineNumber += 1
 289                 self.CurrentOffsetWithinLine = 0
 290             # check for */ comment end
 291             elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
 292
 293                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 294                 self.__GetOneChar()
 295                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 296                 self.__GetOneChar()
 297                 InComment = False
 298             # set comments to spaces
 299             elif InComment:
 300                 if HashComment:
 301                     # // follows hash PP directive
 302                     if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 303                         InComment = False
 304                         HashComment = False
 305                         PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
 306                         FileProfile.PPDirectiveList.append(PPDirectiveObj)
 307                         PPDirectiveObj = None
 308                         continue
 309                     else:
 310                         PPDirectiveObj.Content += self.__CurrentChar()
 311
 312                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 313                 self.__GetOneChar()
 314             # check for // comment
 315             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 316                 InComment = True
 317                 DoubleSlashComment = True
 318
 319             # check for '#' comment
 320             elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
 321                 InComment = True
 322                 HashComment = True
 323                 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
 324             # check for /* comment start
 325             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
 326
 327                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 328                 self.__GetOneChar()
 329                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 330                 self.__GetOneChar()
 331                 InComment = True
 332             else:
 333                 self.__GetOneChar()
 334
 335         EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 336
 337         if InComment and HashComment and not PPExtend:
 338             PPDirectiveObj.EndPos = EndLinePos
 339             FileProfile.PPDirectiveList.append(PPDirectiveObj)
 340         self.Rewind()
 341
 342     ## ParseFile() method
 343     #
 344     #   Parse the file profile buffer to extract fd, fv ... information
 345     #   Exception will be raised if syntax error found
 346     #
 347     #   @param  self        The object pointer
 348     #
 349     def ParseFile(self):
 350         self.PreprocessFileWithClear()
 351         # restore from ListOfList to ListOfString
 352         self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
 353         FileStringContents = ''
 354         for fileLine in self.Profile.FileLinesList:
 355             FileStringContents += fileLine
 356         cStream = antlr3.StringStream(FileStringContents)
 357         lexer = CLexer(cStream)
 358         tStream = antlr3.CommonTokenStream(lexer)
 359         parser = CParser(tStream)
 360         parser.translation_unit()
 361
 362     ## CleanFileProfileBuffer() method
 363     #
 364     #   Reset all contents of the profile of a file
 365     #
 366     def CleanFileProfileBuffer(self):
 367
 368         FileProfile.PPDirectiveList = []
 369         FileProfile.AssignmentExpressionList = []
 370         FileProfile.FunctionDefinitionList = []
 371         FileProfile.VariableDeclarationList = []
 372         FileProfile.EnumerationDefinitionList = []
 373         FileProfile.StructUnionDefinitionList = []
 374         FileProfile.TypedefDefinitionList = []
 375         FileProfile.FunctionCallingList = []
 376
 377     ## PrintFragments() method
 378     #
 379     #   Print the contents of the profile of a file
 380     #
 381     def PrintFragments(self):
 382
 383         print('################# ' + self.FileName + '#####################')
 384
 385         print('/****************************************/')
 386         print('/*************** ASSIGNMENTS ***************/')
 387         print('/****************************************/')
 388         for asign in FileProfile.AssignmentExpressionList:
 389             print(str(asign.StartPos) + asign.Name + asign.Operator + asign.Value)
 390
 391         print('/****************************************/')
 392         print('/********* PREPROCESS DIRECTIVES ********/')
 393         print('/****************************************/')
 394         for pp in FileProfile.PPDirectiveList:
 395             print(str(pp.StartPos) + pp.Content)
 396
 397         print('/****************************************/')
 398         print('/********* VARIABLE DECLARATIONS ********/')
 399         print('/****************************************/')
 400         for var in FileProfile.VariableDeclarationList:
 401             print(str(var.StartPos) + var.Modifier + ' '+ var.Declarator)
 402
 403         print('/****************************************/')
 404         print('/********* FUNCTION DEFINITIONS *********/')
 405         print('/****************************************/')
 406         for func in FileProfile.FunctionDefinitionList:
 407             print(str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos))
 408
 409         print('/****************************************/')
 410         print('/************ ENUMERATIONS **************/')
 411         print('/****************************************/')
 412         for enum in FileProfile.EnumerationDefinitionList:
 413             print(str(enum.StartPos) + enum.Content)
 414
 415         print('/****************************************/')
 416         print('/*********** STRUCTS/UNIONS *************/')
 417         print('/****************************************/')
 418         for su in FileProfile.StructUnionDefinitionList:
 419             print(str(su.StartPos) + su.Content)
 420
 421         print('/****************************************/')
 422         print('/************** TYPEDEFS ****************/')
 423         print('/****************************************/')
 424         for typedef in FileProfile.TypedefDefinitionList:
 425             print(str(typedef.StartPos) + typedef.ToType)
 426
 427 ##
 428 #
 429 # This acts like the main() function for the script, unless it is 'import'ed into another
 430 # script.
 431 #
 432 if __name__ == "__main__":
 433
 434     print("For Test.")