BaseTools/Source/Python/Eot/CodeFragmentCollector.py

   1 ## @file
   2 # preprocess source file
   3 #
   4 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 import re
  19 import Common.LongFilePathOs as os
  20 import sys
  21
  22 import antlr3
  23 from CLexer import CLexer
  24 from CParser import CParser
  25
  26 import FileProfile
  27 from CodeFragment import PP_Directive
  28 from ParserWarning import Warning
  29
  30
  31 ##define T_CHAR_SPACE                ' '
  32 ##define T_CHAR_NULL                 '\0'
  33 ##define T_CHAR_CR                   '\r'
  34 ##define T_CHAR_TAB                  '\t'
  35 ##define T_CHAR_LF                   '\n'
  36 ##define T_CHAR_SLASH                '/'
  37 ##define T_CHAR_BACKSLASH            '\\'
  38 ##define T_CHAR_DOUBLE_QUOTE         '\"'
  39 ##define T_CHAR_SINGLE_QUOTE         '\''
  40 ##define T_CHAR_STAR                 '*'
  41 ##define T_CHAR_HASH                 '#'
  42
  43 (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
  44 T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
  45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
  46
  47 SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
  48
  49 (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
  50
  51 (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
  52
  53 ## The collector for source code fragments.
  54 #
  55 # PreprocessFile method should be called prior to ParseFile
  56 #
  57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
  58 # Get*** procedures mean these procedures will make judgement on current token only.
  59 #
  60 class CodeFragmentCollector:
  61     ## The constructor
  62     #
  63     #   @param  self        The object pointer
  64     #   @param  FileName    The file that to be parsed
  65     #
  66     def __init__(self, FileName):
  67         self.Profile = FileProfile.FileProfile(FileName)
  68         self.Profile.FileLinesList.append(T_CHAR_LF)
  69         self.FileName = FileName
  70         self.CurrentLineNumber = 1
  71         self.CurrentOffsetWithinLine = 0
  72
  73         self.__Token = ""
  74         self.__SkippedChars = ""
  75
  76     ## __SkipWhiteSpace() method
  77     #
  78     #   Skip white spaces from current char, return number of chars skipped
  79     #
  80     #   @param  self        The object pointer
  81     #   @retval Count       The number of chars skipped
  82     #
  83     def __SkipWhiteSpace(self):
  84         Count = 0
  85         while not self.__EndOfFile():
  86             Count += 1
  87             if self.__CurrentChar() in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_LF, T_CHAR_SPACE, T_CHAR_TAB):
  88                 self.__SkippedChars += str(self.__CurrentChar())
  89                 self.__GetOneChar()
  90
  91             else:
  92                 Count = Count - 1
  93                 return Count
  94
  95     ## __EndOfFile() method
  96     #
  97     #   Judge current buffer pos is at file end
  98     #
  99     #   @param  self        The object pointer
 100     #   @retval True        Current File buffer position is at file end
 101     #   @retval False       Current File buffer position is NOT at file end
 102     #
 103     def __EndOfFile(self):
 104         NumberOfLines = len(self.Profile.FileLinesList)
 105         SizeOfLastLine = len(self.Profile.FileLinesList[-1])
 106         if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
 107             return True
 108         elif self.CurrentLineNumber > NumberOfLines:
 109             return True
 110         else:
 111             return False
 112
 113     ## __EndOfLine() method
 114     #
 115     #   Judge current buffer pos is at line end
 116     #
 117     #   @param  self        The object pointer
 118     #   @retval True        Current File buffer position is at line end
 119     #   @retval False       Current File buffer position is NOT at line end
 120     #
 121     def __EndOfLine(self):
 122         SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
 123         if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
 124             return True
 125         else:
 126             return False
 127
 128     ## Rewind() method
 129     #
 130     #   Reset file data buffer to the initial state
 131     #
 132     #   @param  self        The object pointer
 133     #
 134     def Rewind(self):
 135         self.CurrentLineNumber = 1
 136         self.CurrentOffsetWithinLine = 0
 137
 138     ## __UndoOneChar() method
 139     #
 140     #   Go back one char in the file buffer
 141     #
 142     #   @param  self        The object pointer
 143     #   @retval True        Successfully go back one char
 144     #   @retval False       Not able to go back one char as file beginning reached
 145     #
 146     def __UndoOneChar(self):
 147
 148         if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
 149             return False
 150         elif self.CurrentOffsetWithinLine == 0:
 151             self.CurrentLineNumber -= 1
 152             self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
 153         else:
 154             self.CurrentOffsetWithinLine -= 1
 155         return True
 156
 157     ## __GetOneChar() method
 158     #
 159     #   Move forward one char in the file buffer
 160     #
 161     #   @param  self        The object pointer
 162     #
 163     def __GetOneChar(self):
 164         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 165                 self.CurrentLineNumber += 1
 166                 self.CurrentOffsetWithinLine = 0
 167         else:
 168                 self.CurrentOffsetWithinLine += 1
 169
 170     ## __CurrentChar() method
 171     #
 172     #   Get the char pointed to by the file buffer pointer
 173     #
 174     #   @param  self        The object pointer
 175     #   @retval Char        Current char
 176     #
 177     def __CurrentChar(self):
 178         CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
 179
 180         return CurrentChar
 181
 182     ## __NextChar() method
 183     #
 184     #   Get the one char pass the char pointed to by the file buffer pointer
 185     #
 186     #   @param  self        The object pointer
 187     #   @retval Char        Next char
 188     #
 189     def __NextChar(self):
 190         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 191             return self.Profile.FileLinesList[self.CurrentLineNumber][0]
 192         else:
 193             return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
 194
 195     ## __SetCurrentCharValue() method
 196     #
 197     #   Modify the value of current char
 198     #
 199     #   @param  self        The object pointer
 200     #   @param  Value       The new value of current char
 201     #
 202     def __SetCurrentCharValue(self, Value):
 203         self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
 204
 205     ## __SetCharValue() method
 206     #
 207     #   Modify the value of current char
 208     #
 209     #   @param  self        The object pointer
 210     #   @param  Value       The new value of current char
 211     #
 212     def __SetCharValue(self, Line, Offset, Value):
 213         self.Profile.FileLinesList[Line - 1][Offset] = Value
 214
 215     ## __CurrentLine() method
 216     #
 217     #   Get the list that contains current line contents
 218     #
 219     #   @param  self        The object pointer
 220     #   @retval List        current line contents
 221     #
 222     def __CurrentLine(self):
 223         return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
 224
 225     ## __InsertComma() method
 226     #
 227     #   Insert ',' to replace PP
 228     #
 229     #   @param  self        The object pointer
 230     #   @retval List        current line contents
 231     #
 232     def __InsertComma(self, Line):
 233
 234
 235         if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
 236             BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
 237             if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
 238                 return
 239
 240         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
 241             return
 242
 243         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
 244             return
 245
 246         if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
 247             return
 248
 249         self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
 250
 251     ## PreprocessFileWithClear() method
 252     #
 253     # Run a preprocess for the file to clean all comments
 254     #
 255     # @param  self        The object pointer
 256     #
 257     def PreprocessFileWithClear(self):
 258
 259         self.Rewind()
 260         InComment = False
 261         DoubleSlashComment = False
 262         HashComment = False
 263         PPExtend = False
 264         PPDirectiveObj = None
 265         # HashComment in quoted string " " is ignored.
 266         InString = False
 267         InCharLiteral = False
 268
 269         self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
 270         while not self.__EndOfFile():
 271
 272             if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
 273                 InString = not InString
 274
 275             if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
 276                 InCharLiteral = not InCharLiteral
 277             # meet new line, then no longer in a comment for // and '#'
 278             if self.__CurrentChar() == T_CHAR_LF:
 279                 if HashComment and PPDirectiveObj is not None:
 280                     if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 281                         PPDirectiveObj.Content += T_CHAR_LF
 282                         PPExtend = True
 283                     else:
 284                         PPExtend = False
 285
 286                 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 287
 288                 if InComment and DoubleSlashComment:
 289                     InComment = False
 290                     DoubleSlashComment = False
 291
 292                 if InComment and HashComment and not PPExtend:
 293                     InComment = False
 294                     HashComment = False
 295                     PPDirectiveObj.Content += T_CHAR_LF
 296                     PPDirectiveObj.EndPos = EndLinePos
 297                     FileProfile.PPDirectiveList.append(PPDirectiveObj)
 298                     PPDirectiveObj = None
 299
 300                 if InString or InCharLiteral:
 301                     CurrentLine = "".join(self.__CurrentLine())
 302                     if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 303                         SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
 304                         self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
 305
 306                 self.CurrentLineNumber += 1
 307                 self.CurrentOffsetWithinLine = 0
 308             # check for */ comment end
 309             elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
 310
 311                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 312                 self.__GetOneChar()
 313                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 314                 self.__GetOneChar()
 315                 InComment = False
 316             # set comments to spaces
 317             elif InComment:
 318                 if HashComment:
 319                     # // follows hash PP directive
 320                     if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 321                         InComment = False
 322                         HashComment = False
 323                         PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
 324                         FileProfile.PPDirectiveList.append(PPDirectiveObj)
 325                         PPDirectiveObj = None
 326                         continue
 327                     else:
 328                         PPDirectiveObj.Content += self.__CurrentChar()
 329
 330                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 331                 self.__GetOneChar()
 332             # check for // comment
 333             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 334                 InComment = True
 335                 DoubleSlashComment = True
 336
 337             # check for '#' comment
 338             elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
 339                 InComment = True
 340                 HashComment = True
 341                 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
 342             # check for /* comment start
 343             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
 344
 345                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 346                 self.__GetOneChar()
 347                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 348                 self.__GetOneChar()
 349                 InComment = True
 350             else:
 351                 self.__GetOneChar()
 352
 353         EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 354
 355         if InComment and HashComment and not PPExtend:
 356             PPDirectiveObj.EndPos = EndLinePos
 357             FileProfile.PPDirectiveList.append(PPDirectiveObj)
 358         self.Rewind()
 359
 360     ## ParseFile() method
 361     #
 362     #   Parse the file profile buffer to extract fd, fv ... information
 363     #   Exception will be raised if syntax error found
 364     #
 365     #   @param  self        The object pointer
 366     #
 367     def ParseFile(self):
 368         self.PreprocessFileWithClear()
 369         # restore from ListOfList to ListOfString
 370         self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
 371         FileStringContents = ''
 372         for fileLine in self.Profile.FileLinesList:
 373             FileStringContents += fileLine
 374         cStream = antlr3.StringStream(FileStringContents)
 375         lexer = CLexer(cStream)
 376         tStream = antlr3.CommonTokenStream(lexer)
 377         parser = CParser(tStream)
 378         parser.translation_unit()
 379
 380     ## CleanFileProfileBuffer() method
 381     #
 382     #   Reset all contents of the profile of a file
 383     #
 384     def CleanFileProfileBuffer(self):
 385
 386         FileProfile.PPDirectiveList = []
 387         FileProfile.AssignmentExpressionList = []
 388         FileProfile.FunctionDefinitionList = []
 389         FileProfile.VariableDeclarationList = []
 390         FileProfile.EnumerationDefinitionList = []
 391         FileProfile.StructUnionDefinitionList = []
 392         FileProfile.TypedefDefinitionList = []
 393         FileProfile.FunctionCallingList = []
 394
 395     ## PrintFragments() method
 396     #
 397     #   Print the contents of the profile of a file
 398     #
 399     def PrintFragments(self):
 400
 401         print '################# ' + self.FileName + '#####################'
 402
 403         print '/****************************************/'
 404         print '/*************** ASSIGNMENTS ***************/'
 405         print '/****************************************/'
 406         for asign in FileProfile.AssignmentExpressionList:
 407             print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value
 408
 409         print '/****************************************/'
 410         print '/********* PREPROCESS DIRECTIVES ********/'
 411         print '/****************************************/'
 412         for pp in FileProfile.PPDirectiveList:
 413             print str(pp.StartPos) + pp.Content
 414
 415         print '/****************************************/'
 416         print '/********* VARIABLE DECLARATIONS ********/'
 417         print '/****************************************/'
 418         for var in FileProfile.VariableDeclarationList:
 419             print str(var.StartPos) + var.Modifier + ' '+ var.Declarator
 420
 421         print '/****************************************/'
 422         print '/********* FUNCTION DEFINITIONS *********/'
 423         print '/****************************************/'
 424         for func in FileProfile.FunctionDefinitionList:
 425             print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)
 426
 427         print '/****************************************/'
 428         print '/************ ENUMERATIONS **************/'
 429         print '/****************************************/'
 430         for enum in FileProfile.EnumerationDefinitionList:
 431             print str(enum.StartPos) + enum.Content
 432
 433         print '/****************************************/'
 434         print '/*********** STRUCTS/UNIONS *************/'
 435         print '/****************************************/'
 436         for su in FileProfile.StructUnionDefinitionList:
 437             print str(su.StartPos) + su.Content
 438
 439         print '/****************************************/'
 440         print '/************** TYPEDEFS ****************/'
 441         print '/****************************************/'
 442         for typedef in FileProfile.TypedefDefinitionList:
 443             print str(typedef.StartPos) + typedef.ToType
 444
 445 ##
 446 #
 447 # This acts like the main() function for the script, unless it is 'import'ed into another
 448 # script.
 449 #
 450 if __name__ == "__main__":
 451
 452     print "For Test."