BaseTools/Source/Python/Eot/CodeFragmentCollector.py

   1 ## @file
   2 # preprocess source file
   3 #
   4 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8
   9 ##
  10 # Import Modules
  11 #
  12 from __future__ import print_function
  13 from __future__ import absolute_import
  14 import re
  15 import Common.LongFilePathOs as os
  16 import sys
  17
  18 if sys.version_info.major == 3:
  19     import antlr4 as antlr
  20     from Eot.CParser4.CLexer import CLexer
  21     from Eot.CParser4.CParser import CParser
  22 else:
  23     import antlr3 as antlr
  24     antlr.InputStream = antlr.StringStream
  25     from Eot.CParser3.CLexer import CLexer
  26     from Eot.CParser3.CParser import CParser
  27
  28 from Eot import FileProfile
  29 from Eot.CodeFragment import PP_Directive
  30 from Eot.ParserWarning import Warning
  31
  32
  33 ##define T_CHAR_SPACE                ' '
  34 ##define T_CHAR_NULL                 '\0'
  35 ##define T_CHAR_CR                   '\r'
  36 ##define T_CHAR_TAB                  '\t'
  37 ##define T_CHAR_LF                   '\n'
  38 ##define T_CHAR_SLASH                '/'
  39 ##define T_CHAR_BACKSLASH            '\\'
  40 ##define T_CHAR_DOUBLE_QUOTE         '\"'
  41 ##define T_CHAR_SINGLE_QUOTE         '\''
  42 ##define T_CHAR_STAR                 '*'
  43 ##define T_CHAR_HASH                 '#'
  44
  45 (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
  46 T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
  47 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
  48
  49 SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
  50
  51 (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
  52
  53 (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
  54
  55 ## The collector for source code fragments.
  56 #
  57 # PreprocessFile method should be called prior to ParseFile
  58 #
  59 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
  60 # Get*** procedures mean these procedures will make judgement on current token only.
  61 #
  62 class CodeFragmentCollector:
  63     ## The constructor
  64     #
  65     #   @param  self        The object pointer
  66     #   @param  FileName    The file that to be parsed
  67     #
  68     def __init__(self, FileName):
  69         self.Profile = FileProfile.FileProfile(FileName)
  70         self.Profile.FileLinesList.append(T_CHAR_LF)
  71         self.FileName = FileName
  72         self.CurrentLineNumber = 1
  73         self.CurrentOffsetWithinLine = 0
  74
  75         self.__Token = ""
  76         self.__SkippedChars = ""
  77
  78     ## __EndOfFile() method
  79     #
  80     #   Judge current buffer pos is at file end
  81     #
  82     #   @param  self        The object pointer
  83     #   @retval True        Current File buffer position is at file end
  84     #   @retval False       Current File buffer position is NOT at file end
  85     #
  86     def __EndOfFile(self):
  87         NumberOfLines = len(self.Profile.FileLinesList)
  88         SizeOfLastLine = len(self.Profile.FileLinesList[-1])
  89         if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
  90             return True
  91         elif self.CurrentLineNumber > NumberOfLines:
  92             return True
  93         else:
  94             return False
  95
  96     ## __EndOfLine() method
  97     #
  98     #   Judge current buffer pos is at line end
  99     #
 100     #   @param  self        The object pointer
 101     #   @retval True        Current File buffer position is at line end
 102     #   @retval False       Current File buffer position is NOT at line end
 103     #
 104     def __EndOfLine(self):
 105         SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
 106         if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
 107             return True
 108         else:
 109             return False
 110
 111     ## Rewind() method
 112     #
 113     #   Reset file data buffer to the initial state
 114     #
 115     #   @param  self        The object pointer
 116     #
 117     def Rewind(self):
 118         self.CurrentLineNumber = 1
 119         self.CurrentOffsetWithinLine = 0
 120
 121     ## __UndoOneChar() method
 122     #
 123     #   Go back one char in the file buffer
 124     #
 125     #   @param  self        The object pointer
 126     #   @retval True        Successfully go back one char
 127     #   @retval False       Not able to go back one char as file beginning reached
 128     #
 129     def __UndoOneChar(self):
 130
 131         if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
 132             return False
 133         elif self.CurrentOffsetWithinLine == 0:
 134             self.CurrentLineNumber -= 1
 135             self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
 136         else:
 137             self.CurrentOffsetWithinLine -= 1
 138         return True
 139
 140     ## __GetOneChar() method
 141     #
 142     #   Move forward one char in the file buffer
 143     #
 144     #   @param  self        The object pointer
 145     #
 146     def __GetOneChar(self):
 147         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 148                 self.CurrentLineNumber += 1
 149                 self.CurrentOffsetWithinLine = 0
 150         else:
 151                 self.CurrentOffsetWithinLine += 1
 152
 153     ## __CurrentChar() method
 154     #
 155     #   Get the char pointed to by the file buffer pointer
 156     #
 157     #   @param  self        The object pointer
 158     #   @retval Char        Current char
 159     #
 160     def __CurrentChar(self):
 161         CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
 162
 163         return CurrentChar
 164
 165     ## __NextChar() method
 166     #
 167     #   Get the one char pass the char pointed to by the file buffer pointer
 168     #
 169     #   @param  self        The object pointer
 170     #   @retval Char        Next char
 171     #
 172     def __NextChar(self):
 173         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 174             return self.Profile.FileLinesList[self.CurrentLineNumber][0]
 175         else:
 176             return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
 177
 178     ## __SetCurrentCharValue() method
 179     #
 180     #   Modify the value of current char
 181     #
 182     #   @param  self        The object pointer
 183     #   @param  Value       The new value of current char
 184     #
 185     def __SetCurrentCharValue(self, Value):
 186         self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
 187
 188     ## __SetCharValue() method
 189     #
 190     #   Modify the value of current char
 191     #
 192     #   @param  self        The object pointer
 193     #   @param  Value       The new value of current char
 194     #
 195     def __SetCharValue(self, Line, Offset, Value):
 196         self.Profile.FileLinesList[Line - 1][Offset] = Value
 197
 198     ## __CurrentLine() method
 199     #
 200     #   Get the list that contains current line contents
 201     #
 202     #   @param  self        The object pointer
 203     #   @retval List        current line contents
 204     #
 205     def __CurrentLine(self):
 206         return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
 207
 208     ## __InsertComma() method
 209     #
 210     #   Insert ',' to replace PP
 211     #
 212     #   @param  self        The object pointer
 213     #   @retval List        current line contents
 214     #
 215     def __InsertComma(self, Line):
 216
 217
 218         if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
 219             BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
 220             if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
 221                 return
 222
 223         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
 224             return
 225
 226         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
 227             return
 228
 229         if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
 230             return
 231
 232         self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
 233
 234     ## PreprocessFileWithClear() method
 235     #
 236     # Run a preprocess for the file to clean all comments
 237     #
 238     # @param  self        The object pointer
 239     #
 240     def PreprocessFileWithClear(self):
 241
 242         self.Rewind()
 243         InComment = False
 244         DoubleSlashComment = False
 245         HashComment = False
 246         PPExtend = False
 247         PPDirectiveObj = None
 248         # HashComment in quoted string " " is ignored.
 249         InString = False
 250         InCharLiteral = False
 251
 252         self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
 253         while not self.__EndOfFile():
 254
 255             if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
 256                 InString = not InString
 257
 258             if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
 259                 InCharLiteral = not InCharLiteral
 260             # meet new line, then no longer in a comment for // and '#'
 261             if self.__CurrentChar() == T_CHAR_LF:
 262                 if HashComment and PPDirectiveObj is not None:
 263                     if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 264                         PPDirectiveObj.Content += T_CHAR_LF
 265                         PPExtend = True
 266                     else:
 267                         PPExtend = False
 268
 269                 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 270
 271                 if InComment and DoubleSlashComment:
 272                     InComment = False
 273                     DoubleSlashComment = False
 274
 275                 if InComment and HashComment and not PPExtend:
 276                     InComment = False
 277                     HashComment = False
 278                     PPDirectiveObj.Content += T_CHAR_LF
 279                     PPDirectiveObj.EndPos = EndLinePos
 280                     FileProfile.PPDirectiveList.append(PPDirectiveObj)
 281                     PPDirectiveObj = None
 282
 283                 if InString or InCharLiteral:
 284                     CurrentLine = "".join(self.__CurrentLine())
 285                     if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 286                         SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
 287                         self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
 288
 289                 self.CurrentLineNumber += 1
 290                 self.CurrentOffsetWithinLine = 0
 291             # check for */ comment end
 292             elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
 293
 294                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 295                 self.__GetOneChar()
 296                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 297                 self.__GetOneChar()
 298                 InComment = False
 299             # set comments to spaces
 300             elif InComment:
 301                 if HashComment:
 302                     # // follows hash PP directive
 303                     if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 304                         InComment = False
 305                         HashComment = False
 306                         PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
 307                         FileProfile.PPDirectiveList.append(PPDirectiveObj)
 308                         PPDirectiveObj = None
 309                         continue
 310                     else:
 311                         PPDirectiveObj.Content += self.__CurrentChar()
 312
 313                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 314                 self.__GetOneChar()
 315             # check for // comment
 316             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 317                 InComment = True
 318                 DoubleSlashComment = True
 319
 320             # check for '#' comment
 321             elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
 322                 InComment = True
 323                 HashComment = True
 324                 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
 325             # check for /* comment start
 326             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
 327
 328                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 329                 self.__GetOneChar()
 330                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 331                 self.__GetOneChar()
 332                 InComment = True
 333             else:
 334                 self.__GetOneChar()
 335
 336         EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 337
 338         if InComment and HashComment and not PPExtend:
 339             PPDirectiveObj.EndPos = EndLinePos
 340             FileProfile.PPDirectiveList.append(PPDirectiveObj)
 341         self.Rewind()
 342
 343     ## ParseFile() method
 344     #
 345     #   Parse the file profile buffer to extract fd, fv ... information
 346     #   Exception will be raised if syntax error found
 347     #
 348     #   @param  self        The object pointer
 349     #
 350     def ParseFile(self):
 351         self.PreprocessFileWithClear()
 352         # restore from ListOfList to ListOfString
 353         self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
 354         FileStringContents = ''
 355         for fileLine in self.Profile.FileLinesList:
 356             FileStringContents += fileLine
 357         cStream = antlr.InputStream(FileStringContents)
 358         lexer = CLexer(cStream)
 359         tStream = antlr.CommonTokenStream(lexer)
 360         parser = CParser(tStream)
 361         parser.translation_unit()
 362
 363     ## CleanFileProfileBuffer() method
 364     #
 365     #   Reset all contents of the profile of a file
 366     #
 367     def CleanFileProfileBuffer(self):
 368
 369         FileProfile.PPDirectiveList = []
 370         FileProfile.AssignmentExpressionList = []
 371         FileProfile.FunctionDefinitionList = []
 372         FileProfile.VariableDeclarationList = []
 373         FileProfile.EnumerationDefinitionList = []
 374         FileProfile.StructUnionDefinitionList = []
 375         FileProfile.TypedefDefinitionList = []
 376         FileProfile.FunctionCallingList = []
 377
 378     ## PrintFragments() method
 379     #
 380     #   Print the contents of the profile of a file
 381     #
 382     def PrintFragments(self):
 383
 384         print('################# ' + self.FileName + '#####################')
 385
 386         print('/****************************************/')
 387         print('/************** ASSIGNMENTS *************/')
 388         print('/****************************************/')
 389         for assign in FileProfile.AssignmentExpressionList:
 390             print(str(assign.StartPos) + assign.Name + assign.Operator + assign.Value)
 391
 392         print('/****************************************/')
 393         print('/********* PREPROCESS DIRECTIVES ********/')
 394         print('/****************************************/')
 395         for pp in FileProfile.PPDirectiveList:
 396             print(str(pp.StartPos) + pp.Content)
 397
 398         print('/****************************************/')
 399         print('/********* VARIABLE DECLARATIONS ********/')
 400         print('/****************************************/')
 401         for var in FileProfile.VariableDeclarationList:
 402             print(str(var.StartPos) + var.Modifier + ' '+ var.Declarator)
 403
 404         print('/****************************************/')
 405         print('/********* FUNCTION DEFINITIONS *********/')
 406         print('/****************************************/')
 407         for func in FileProfile.FunctionDefinitionList:
 408             print(str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos))
 409
 410         print('/****************************************/')
 411         print('/************ ENUMERATIONS **************/')
 412         print('/****************************************/')
 413         for enum in FileProfile.EnumerationDefinitionList:
 414             print(str(enum.StartPos) + enum.Content)
 415
 416         print('/****************************************/')
 417         print('/*********** STRUCTS/UNIONS *************/')
 418         print('/****************************************/')
 419         for su in FileProfile.StructUnionDefinitionList:
 420             print(str(su.StartPos) + su.Content)
 421
 422         print('/****************************************/')
 423         print('/************** TYPEDEFS ****************/')
 424         print('/****************************************/')
 425         for typedef in FileProfile.TypedefDefinitionList:
 426             print(str(typedef.StartPos) + typedef.ToType)
 427
 428 ##
 429 #
 430 # This acts like the main() function for the script, unless it is 'import'ed into another
 431 # script.
 432 #
 433 if __name__ == "__main__":
 434
 435     print("For Test.")