BaseTools/Source/Python/Eot/CodeFragmentCollector.py

   1 ## @file
   2 # preprocess source file
   3 #
   4 #  Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 import re
  19 import os
  20 import sys
  21
  22 import antlr3
  23 from CLexer import CLexer
  24 from CParser import CParser
  25
  26 import FileProfile
  27 from CodeFragment import PP_Directive
  28 from ParserWarning import Warning
  29
  30
  31 ##define T_CHAR_SPACE                ' '
  32 ##define T_CHAR_NULL                 '\0'
  33 ##define T_CHAR_CR                   '\r'
  34 ##define T_CHAR_TAB                  '\t'
  35 ##define T_CHAR_LF                   '\n'
  36 ##define T_CHAR_SLASH                '/'
  37 ##define T_CHAR_BACKSLASH            '\\'
  38 ##define T_CHAR_DOUBLE_QUOTE         '\"'
  39 ##define T_CHAR_SINGLE_QUOTE         '\''
  40 ##define T_CHAR_STAR                 '*'
  41 ##define T_CHAR_HASH                 '#'
  42
  43 (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
  44 T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
  45 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
  46
  47 SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
  48
  49 (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
  50
  51 (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
  52
  53 ## The collector for source code fragments.
  54 #
  55 # PreprocessFile method should be called prior to ParseFile
  56 #
  57 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
  58 # Get*** procedures mean these procedures will make judgement on current token only.
  59 #
  60 class CodeFragmentCollector:
  61     ## The constructor
  62     #
  63     #   @param  self        The object pointer
  64     #   @param  FileName    The file that to be parsed
  65     #
  66     def __init__(self, FileName):
  67         self.Profile = FileProfile.FileProfile(FileName)
  68         self.Profile.FileLinesList.append(T_CHAR_LF)
  69         self.FileName = FileName
  70         self.CurrentLineNumber = 1
  71         self.CurrentOffsetWithinLine = 0
  72
  73         self.__Token = ""
  74         self.__SkippedChars = ""
  75
  76     ## __IsWhiteSpace() method
  77     #
  78     #   Whether char at current FileBufferPos is whitespace
  79     #
  80     #   @param  self        The object pointer
  81     #   @param  Char        The char to test
  82     #   @retval True        The char is a kind of white space
  83     #   @retval False       The char is NOT a kind of white space
  84     #
  85     def __IsWhiteSpace(self, Char):
  86         if Char in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_SPACE, T_CHAR_TAB, T_CHAR_LF):
  87             return True
  88         else:
  89             return False
  90
  91     ## __SkipWhiteSpace() method
  92     #
  93     #   Skip white spaces from current char, return number of chars skipped
  94     #
  95     #   @param  self        The object pointer
  96     #   @retval Count       The number of chars skipped
  97     #
  98     def __SkipWhiteSpace(self):
  99         Count = 0
 100         while not self.__EndOfFile():
 101             Count += 1
 102             if self.__CurrentChar() in (T_CHAR_NULL, T_CHAR_CR, T_CHAR_LF, T_CHAR_SPACE, T_CHAR_TAB):
 103                 self.__SkippedChars += str(self.__CurrentChar())
 104                 self.__GetOneChar()
 105
 106             else:
 107                 Count = Count - 1
 108                 return Count
 109
 110     ## __EndOfFile() method
 111     #
 112     #   Judge current buffer pos is at file end
 113     #
 114     #   @param  self        The object pointer
 115     #   @retval True        Current File buffer position is at file end
 116     #   @retval False       Current File buffer position is NOT at file end
 117     #
 118     def __EndOfFile(self):
 119         NumberOfLines = len(self.Profile.FileLinesList)
 120         SizeOfLastLine = len(self.Profile.FileLinesList[-1])
 121         if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
 122             return True
 123         elif self.CurrentLineNumber > NumberOfLines:
 124             return True
 125         else:
 126             return False
 127
 128     ## __EndOfLine() method
 129     #
 130     #   Judge current buffer pos is at line end
 131     #
 132     #   @param  self        The object pointer
 133     #   @retval True        Current File buffer position is at line end
 134     #   @retval False       Current File buffer position is NOT at line end
 135     #
 136     def __EndOfLine(self):
 137         SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
 138         if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
 139             return True
 140         else:
 141             return False
 142
 143     ## Rewind() method
 144     #
 145     #   Reset file data buffer to the initial state
 146     #
 147     #   @param  self        The object pointer
 148     #
 149     def Rewind(self):
 150         self.CurrentLineNumber = 1
 151         self.CurrentOffsetWithinLine = 0
 152
 153     ## __UndoOneChar() method
 154     #
 155     #   Go back one char in the file buffer
 156     #
 157     #   @param  self        The object pointer
 158     #   @retval True        Successfully go back one char
 159     #   @retval False       Not able to go back one char as file beginning reached
 160     #
 161     def __UndoOneChar(self):
 162
 163         if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
 164             return False
 165         elif self.CurrentOffsetWithinLine == 0:
 166             self.CurrentLineNumber -= 1
 167             self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
 168         else:
 169             self.CurrentOffsetWithinLine -= 1
 170         return True
 171
 172     ## __GetOneChar() method
 173     #
 174     #   Move forward one char in the file buffer
 175     #
 176     #   @param  self        The object pointer
 177     #
 178     def __GetOneChar(self):
 179         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 180                 self.CurrentLineNumber += 1
 181                 self.CurrentOffsetWithinLine = 0
 182         else:
 183                 self.CurrentOffsetWithinLine += 1
 184
 185     ## __CurrentChar() method
 186     #
 187     #   Get the char pointed to by the file buffer pointer
 188     #
 189     #   @param  self        The object pointer
 190     #   @retval Char        Current char
 191     #
 192     def __CurrentChar(self):
 193         CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
 194
 195         return CurrentChar
 196
 197     ## __NextChar() method
 198     #
 199     #   Get the one char pass the char pointed to by the file buffer pointer
 200     #
 201     #   @param  self        The object pointer
 202     #   @retval Char        Next char
 203     #
 204     def __NextChar(self):
 205         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 206             return self.Profile.FileLinesList[self.CurrentLineNumber][0]
 207         else:
 208             return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
 209
 210     ## __SetCurrentCharValue() method
 211     #
 212     #   Modify the value of current char
 213     #
 214     #   @param  self        The object pointer
 215     #   @param  Value       The new value of current char
 216     #
 217     def __SetCurrentCharValue(self, Value):
 218         self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
 219
 220     ## __SetCharValue() method
 221     #
 222     #   Modify the value of current char
 223     #
 224     #   @param  self        The object pointer
 225     #   @param  Value       The new value of current char
 226     #
 227     def __SetCharValue(self, Line, Offset, Value):
 228         self.Profile.FileLinesList[Line - 1][Offset] = Value
 229
 230     ## __CurrentLine() method
 231     #
 232     #   Get the list that contains current line contents
 233     #
 234     #   @param  self        The object pointer
 235     #   @retval List        current line contents
 236     #
 237     def __CurrentLine(self):
 238         return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
 239
 240     ## __InsertComma() method
 241     #
 242     #   Insert ',' to replace PP
 243     #
 244     #   @param  self        The object pointer
 245     #   @retval List        current line contents
 246     #
 247     def __InsertComma(self, Line):
 248
 249
 250         if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
 251             BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
 252             if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
 253                 return
 254
 255         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
 256             return
 257
 258         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
 259             return
 260
 261         if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
 262             return
 263
 264         self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
 265
 266     ## PreprocessFileWithClear() method
 267     #
 268     # Run a preprocess for the file to clean all comments
 269     #
 270     # @param  self        The object pointer
 271     #
 272     def PreprocessFileWithClear(self):
 273
 274         self.Rewind()
 275         InComment = False
 276         DoubleSlashComment = False
 277         HashComment = False
 278         PPExtend = False
 279         PPDirectiveObj = None
 280         # HashComment in quoted string " " is ignored.
 281         InString = False
 282         InCharLiteral = False
 283
 284         self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
 285         while not self.__EndOfFile():
 286
 287             if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
 288                 InString = not InString
 289
 290             if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
 291                 InCharLiteral = not InCharLiteral
 292             # meet new line, then no longer in a comment for // and '#'
 293             if self.__CurrentChar() == T_CHAR_LF:
 294                 if HashComment and PPDirectiveObj != None:
 295                     if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 296                         PPDirectiveObj.Content += T_CHAR_LF
 297                         PPExtend = True
 298                     else:
 299                         PPExtend = False
 300
 301                 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 302
 303                 if InComment and DoubleSlashComment:
 304                     InComment = False
 305                     DoubleSlashComment = False
 306
 307                 if InComment and HashComment and not PPExtend:
 308                     InComment = False
 309                     HashComment = False
 310                     PPDirectiveObj.Content += T_CHAR_LF
 311                     PPDirectiveObj.EndPos = EndLinePos
 312                     FileProfile.PPDirectiveList.append(PPDirectiveObj)
 313                     PPDirectiveObj = None
 314
 315                 if InString or InCharLiteral:
 316                     CurrentLine = "".join(self.__CurrentLine())
 317                     if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 318                         SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
 319                         self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
 320
 321                 self.CurrentLineNumber += 1
 322                 self.CurrentOffsetWithinLine = 0
 323             # check for */ comment end
 324             elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
 325
 326                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 327                 self.__GetOneChar()
 328                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 329                 self.__GetOneChar()
 330                 InComment = False
 331             # set comments to spaces
 332             elif InComment:
 333                 if HashComment:
 334                     # // follows hash PP directive
 335                     if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 336                         InComment = False
 337                         HashComment = False
 338                         PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
 339                         FileProfile.PPDirectiveList.append(PPDirectiveObj)
 340                         PPDirectiveObj = None
 341                         continue
 342                     else:
 343                         PPDirectiveObj.Content += self.__CurrentChar()
 344
 345                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 346                 self.__GetOneChar()
 347             # check for // comment
 348             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 349                 InComment = True
 350                 DoubleSlashComment = True
 351
 352             # check for '#' comment
 353             elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
 354                 InComment = True
 355                 HashComment = True
 356                 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
 357             # check for /* comment start
 358             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
 359
 360                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 361                 self.__GetOneChar()
 362                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 363                 self.__GetOneChar()
 364                 InComment = True
 365             else:
 366                 self.__GetOneChar()
 367
 368         EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 369
 370         if InComment and HashComment and not PPExtend:
 371             PPDirectiveObj.EndPos = EndLinePos
 372             FileProfile.PPDirectiveList.append(PPDirectiveObj)
 373         self.Rewind()
 374
 375     ## ParseFile() method
 376     #
 377     #   Parse the file profile buffer to extract fd, fv ... information
 378     #   Exception will be raised if syntax error found
 379     #
 380     #   @param  self        The object pointer
 381     #
 382     def ParseFile(self):
 383         self.PreprocessFileWithClear()
 384         # restore from ListOfList to ListOfString
 385         self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
 386         FileStringContents = ''
 387         for fileLine in self.Profile.FileLinesList:
 388             FileStringContents += fileLine
 389         cStream = antlr3.StringStream(FileStringContents)
 390         lexer = CLexer(cStream)
 391         tStream = antlr3.CommonTokenStream(lexer)
 392         parser = CParser(tStream)
 393         parser.translation_unit()
 394
 395     ## CleanFileProfileBuffer() method
 396     #
 397     #   Reset all contents of the profile of a file
 398     #
 399     def CleanFileProfileBuffer(self):
 400
 401         FileProfile.PPDirectiveList = []
 402         FileProfile.AssignmentExpressionList = []
 403         FileProfile.FunctionDefinitionList = []
 404         FileProfile.VariableDeclarationList = []
 405         FileProfile.EnumerationDefinitionList = []
 406         FileProfile.StructUnionDefinitionList = []
 407         FileProfile.TypedefDefinitionList = []
 408         FileProfile.FunctionCallingList = []
 409
 410     ## PrintFragments() method
 411     #
 412     #   Print the contents of the profile of a file
 413     #
 414     def PrintFragments(self):
 415
 416         print '################# ' + self.FileName + '#####################'
 417
 418         print '/****************************************/'
 419         print '/*************** ASSIGNMENTS ***************/'
 420         print '/****************************************/'
 421         for asign in FileProfile.AssignmentExpressionList:
 422             print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value
 423
 424         print '/****************************************/'
 425         print '/********* PREPROCESS DIRECTIVES ********/'
 426         print '/****************************************/'
 427         for pp in FileProfile.PPDirectiveList:
 428             print str(pp.StartPos) + pp.Content
 429
 430         print '/****************************************/'
 431         print '/********* VARIABLE DECLARATIONS ********/'
 432         print '/****************************************/'
 433         for var in FileProfile.VariableDeclarationList:
 434             print str(var.StartPos) + var.Modifier + ' '+ var.Declarator
 435
 436         print '/****************************************/'
 437         print '/********* FUNCTION DEFINITIONS *********/'
 438         print '/****************************************/'
 439         for func in FileProfile.FunctionDefinitionList:
 440             print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)
 441
 442         print '/****************************************/'
 443         print '/************ ENUMERATIONS **************/'
 444         print '/****************************************/'
 445         for enum in FileProfile.EnumerationDefinitionList:
 446             print str(enum.StartPos) + enum.Content
 447
 448         print '/****************************************/'
 449         print '/*********** STRUCTS/UNIONS *************/'
 450         print '/****************************************/'
 451         for su in FileProfile.StructUnionDefinitionList:
 452             print str(su.StartPos) + su.Content
 453
 454         print '/****************************************/'
 455         print '/************** TYPEDEFS ****************/'
 456         print '/****************************************/'
 457         for typedef in FileProfile.TypedefDefinitionList:
 458             print str(typedef.StartPos) + typedef.ToType
 459
 460 ##
 461 #
 462 # This acts like the main() function for the script, unless it is 'import'ed into another
 463 # script.
 464 #
 465 if __name__ == "__main__":
 466
 467     print "For Test."