BaseTools/Source/Python/Eot/CodeFragmentCollector.py

   1 ## @file
   2 # preprocess source file
   3 #
   4 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 import re
  19 import Common.LongFilePathOs as os
  20 import sys
  21
  22 import antlr3
  23 from CLexer import CLexer
  24 from CParser import CParser
  25
  26 import FileProfile
  27 from CodeFragment import PP_Directive
  28 from ParserWarning import Warning
  29
  30
  31 ##define T_CHAR_SPACE                ' '
  32 ##define T_CHAR_NULL                 '\0'
  33 ##define T_CHAR_CR                   '\r'
  34 ##define T_CHAR_TAB                  '\t'
  35 ##define T_CHAR_LF                   '\n'
  36 ##define T_CHAR_SLASH                '/'
  37 ##define T_CHAR_BACKSLASH            '\\'
  38 ##define T_CHAR_DOUBLE_QUOTE         '\"'
  39 ##define T_CHAR_SINGLE_QUOTE         '\''
  40 ##define T_CHAR_STAR                 '*'
  41 ##define T_CHAR_HASH                 '#'
  42
  43 (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
  44 T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
  45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
  46
  47 SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
  48
  49 (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
  50
  51 (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
  52
  53 ## The collector for source code fragments.
  54 #
  55 # PreprocessFile method should be called prior to ParseFile
  56 #
  57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
  58 # Get*** procedures mean these procedures will make judgement on current token only.
  59 #
  60 class CodeFragmentCollector:
  61     ## The constructor
  62     #
  63     #   @param  self        The object pointer
  64     #   @param  FileName    The file that to be parsed
  65     #
  66     def __init__(self, FileName):
  67         self.Profile = FileProfile.FileProfile(FileName)
  68         self.Profile.FileLinesList.append(T_CHAR_LF)
  69         self.FileName = FileName
  70         self.CurrentLineNumber = 1
  71         self.CurrentOffsetWithinLine = 0
  72
  73         self.__Token = ""
  74         self.__SkippedChars = ""
  75
  76     ## __EndOfFile() method
  77     #
  78     #   Judge current buffer pos is at file end
  79     #
  80     #   @param  self        The object pointer
  81     #   @retval True        Current File buffer position is at file end
  82     #   @retval False       Current File buffer position is NOT at file end
  83     #
  84     def __EndOfFile(self):
  85         NumberOfLines = len(self.Profile.FileLinesList)
  86         SizeOfLastLine = len(self.Profile.FileLinesList[-1])
  87         if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
  88             return True
  89         elif self.CurrentLineNumber > NumberOfLines:
  90             return True
  91         else:
  92             return False
  93
  94     ## __EndOfLine() method
  95     #
  96     #   Judge current buffer pos is at line end
  97     #
  98     #   @param  self        The object pointer
  99     #   @retval True        Current File buffer position is at line end
 100     #   @retval False       Current File buffer position is NOT at line end
 101     #
 102     def __EndOfLine(self):
 103         SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
 104         if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
 105             return True
 106         else:
 107             return False
 108
 109     ## Rewind() method
 110     #
 111     #   Reset file data buffer to the initial state
 112     #
 113     #   @param  self        The object pointer
 114     #
 115     def Rewind(self):
 116         self.CurrentLineNumber = 1
 117         self.CurrentOffsetWithinLine = 0
 118
 119     ## __UndoOneChar() method
 120     #
 121     #   Go back one char in the file buffer
 122     #
 123     #   @param  self        The object pointer
 124     #   @retval True        Successfully go back one char
 125     #   @retval False       Not able to go back one char as file beginning reached
 126     #
 127     def __UndoOneChar(self):
 128
 129         if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
 130             return False
 131         elif self.CurrentOffsetWithinLine == 0:
 132             self.CurrentLineNumber -= 1
 133             self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
 134         else:
 135             self.CurrentOffsetWithinLine -= 1
 136         return True
 137
 138     ## __GetOneChar() method
 139     #
 140     #   Move forward one char in the file buffer
 141     #
 142     #   @param  self        The object pointer
 143     #
 144     def __GetOneChar(self):
 145         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 146                 self.CurrentLineNumber += 1
 147                 self.CurrentOffsetWithinLine = 0
 148         else:
 149                 self.CurrentOffsetWithinLine += 1
 150
 151     ## __CurrentChar() method
 152     #
 153     #   Get the char pointed to by the file buffer pointer
 154     #
 155     #   @param  self        The object pointer
 156     #   @retval Char        Current char
 157     #
 158     def __CurrentChar(self):
 159         CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
 160
 161         return CurrentChar
 162
 163     ## __NextChar() method
 164     #
 165     #   Get the one char pass the char pointed to by the file buffer pointer
 166     #
 167     #   @param  self        The object pointer
 168     #   @retval Char        Next char
 169     #
 170     def __NextChar(self):
 171         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 172             return self.Profile.FileLinesList[self.CurrentLineNumber][0]
 173         else:
 174             return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
 175
 176     ## __SetCurrentCharValue() method
 177     #
 178     #   Modify the value of current char
 179     #
 180     #   @param  self        The object pointer
 181     #   @param  Value       The new value of current char
 182     #
 183     def __SetCurrentCharValue(self, Value):
 184         self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
 185
 186     ## __SetCharValue() method
 187     #
 188     #   Modify the value of current char
 189     #
 190     #   @param  self        The object pointer
 191     #   @param  Value       The new value of current char
 192     #
 193     def __SetCharValue(self, Line, Offset, Value):
 194         self.Profile.FileLinesList[Line - 1][Offset] = Value
 195
 196     ## __CurrentLine() method
 197     #
 198     #   Get the list that contains current line contents
 199     #
 200     #   @param  self        The object pointer
 201     #   @retval List        current line contents
 202     #
 203     def __CurrentLine(self):
 204         return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
 205
 206     ## __InsertComma() method
 207     #
 208     #   Insert ',' to replace PP
 209     #
 210     #   @param  self        The object pointer
 211     #   @retval List        current line contents
 212     #
 213     def __InsertComma(self, Line):
 214
 215
 216         if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
 217             BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
 218             if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
 219                 return
 220
 221         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
 222             return
 223
 224         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
 225             return
 226
 227         if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
 228             return
 229
 230         self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
 231
 232     ## PreprocessFileWithClear() method
 233     #
 234     # Run a preprocess for the file to clean all comments
 235     #
 236     # @param  self        The object pointer
 237     #
 238     def PreprocessFileWithClear(self):
 239
 240         self.Rewind()
 241         InComment = False
 242         DoubleSlashComment = False
 243         HashComment = False
 244         PPExtend = False
 245         PPDirectiveObj = None
 246         # HashComment in quoted string " " is ignored.
 247         InString = False
 248         InCharLiteral = False
 249
 250         self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
 251         while not self.__EndOfFile():
 252
 253             if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
 254                 InString = not InString
 255
 256             if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
 257                 InCharLiteral = not InCharLiteral
 258             # meet new line, then no longer in a comment for // and '#'
 259             if self.__CurrentChar() == T_CHAR_LF:
 260                 if HashComment and PPDirectiveObj is not None:
 261                     if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 262                         PPDirectiveObj.Content += T_CHAR_LF
 263                         PPExtend = True
 264                     else:
 265                         PPExtend = False
 266
 267                 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 268
 269                 if InComment and DoubleSlashComment:
 270                     InComment = False
 271                     DoubleSlashComment = False
 272
 273                 if InComment and HashComment and not PPExtend:
 274                     InComment = False
 275                     HashComment = False
 276                     PPDirectiveObj.Content += T_CHAR_LF
 277                     PPDirectiveObj.EndPos = EndLinePos
 278                     FileProfile.PPDirectiveList.append(PPDirectiveObj)
 279                     PPDirectiveObj = None
 280
 281                 if InString or InCharLiteral:
 282                     CurrentLine = "".join(self.__CurrentLine())
 283                     if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 284                         SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
 285                         self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
 286
 287                 self.CurrentLineNumber += 1
 288                 self.CurrentOffsetWithinLine = 0
 289             # check for */ comment end
 290             elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
 291
 292                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 293                 self.__GetOneChar()
 294                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 295                 self.__GetOneChar()
 296                 InComment = False
 297             # set comments to spaces
 298             elif InComment:
 299                 if HashComment:
 300                     # // follows hash PP directive
 301                     if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 302                         InComment = False
 303                         HashComment = False
 304                         PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
 305                         FileProfile.PPDirectiveList.append(PPDirectiveObj)
 306                         PPDirectiveObj = None
 307                         continue
 308                     else:
 309                         PPDirectiveObj.Content += self.__CurrentChar()
 310
 311                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 312                 self.__GetOneChar()
 313             # check for // comment
 314             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 315                 InComment = True
 316                 DoubleSlashComment = True
 317
 318             # check for '#' comment
 319             elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
 320                 InComment = True
 321                 HashComment = True
 322                 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
 323             # check for /* comment start
 324             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
 325
 326                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 327                 self.__GetOneChar()
 328                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 329                 self.__GetOneChar()
 330                 InComment = True
 331             else:
 332                 self.__GetOneChar()
 333
 334         EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 335
 336         if InComment and HashComment and not PPExtend:
 337             PPDirectiveObj.EndPos = EndLinePos
 338             FileProfile.PPDirectiveList.append(PPDirectiveObj)
 339         self.Rewind()
 340
 341     ## ParseFile() method
 342     #
 343     #   Parse the file profile buffer to extract fd, fv ... information
 344     #   Exception will be raised if syntax error found
 345     #
 346     #   @param  self        The object pointer
 347     #
 348     def ParseFile(self):
 349         self.PreprocessFileWithClear()
 350         # restore from ListOfList to ListOfString
 351         self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
 352         FileStringContents = ''
 353         for fileLine in self.Profile.FileLinesList:
 354             FileStringContents += fileLine
 355         cStream = antlr3.StringStream(FileStringContents)
 356         lexer = CLexer(cStream)
 357         tStream = antlr3.CommonTokenStream(lexer)
 358         parser = CParser(tStream)
 359         parser.translation_unit()
 360
 361     ## CleanFileProfileBuffer() method
 362     #
 363     #   Reset all contents of the profile of a file
 364     #
 365     def CleanFileProfileBuffer(self):
 366
 367         FileProfile.PPDirectiveList = []
 368         FileProfile.AssignmentExpressionList = []
 369         FileProfile.FunctionDefinitionList = []
 370         FileProfile.VariableDeclarationList = []
 371         FileProfile.EnumerationDefinitionList = []
 372         FileProfile.StructUnionDefinitionList = []
 373         FileProfile.TypedefDefinitionList = []
 374         FileProfile.FunctionCallingList = []
 375
 376     ## PrintFragments() method
 377     #
 378     #   Print the contents of the profile of a file
 379     #
 380     def PrintFragments(self):
 381
 382         print '################# ' + self.FileName + '#####################'
 383
 384         print '/****************************************/'
 385         print '/*************** ASSIGNMENTS ***************/'
 386         print '/****************************************/'
 387         for asign in FileProfile.AssignmentExpressionList:
 388             print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value
 389
 390         print '/****************************************/'
 391         print '/********* PREPROCESS DIRECTIVES ********/'
 392         print '/****************************************/'
 393         for pp in FileProfile.PPDirectiveList:
 394             print str(pp.StartPos) + pp.Content
 395
 396         print '/****************************************/'
 397         print '/********* VARIABLE DECLARATIONS ********/'
 398         print '/****************************************/'
 399         for var in FileProfile.VariableDeclarationList:
 400             print str(var.StartPos) + var.Modifier + ' '+ var.Declarator
 401
 402         print '/****************************************/'
 403         print '/********* FUNCTION DEFINITIONS *********/'
 404         print '/****************************************/'
 405         for func in FileProfile.FunctionDefinitionList:
 406             print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)
 407
 408         print '/****************************************/'
 409         print '/************ ENUMERATIONS **************/'
 410         print '/****************************************/'
 411         for enum in FileProfile.EnumerationDefinitionList:
 412             print str(enum.StartPos) + enum.Content
 413
 414         print '/****************************************/'
 415         print '/*********** STRUCTS/UNIONS *************/'
 416         print '/****************************************/'
 417         for su in FileProfile.StructUnionDefinitionList:
 418             print str(su.StartPos) + su.Content
 419
 420         print '/****************************************/'
 421         print '/************** TYPEDEFS ****************/'
 422         print '/****************************************/'
 423         for typedef in FileProfile.TypedefDefinitionList:
 424             print str(typedef.StartPos) + typedef.ToType
 425
 426 ##
 427 #
 428 # This acts like the main() function for the script, unless it is 'import'ed into another
 429 # script.
 430 #
 431 if __name__ == "__main__":
 432
 433     print "For Test."