BaseTools/Source/Python/Eot/CodeFragmentCollector.py

   1 ## @file
   2 # preprocess source file
   3 #
   4 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 #
   6 #  This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 from __future__ import print_function
  19 from __future__ import absolute_import
  20 import re
  21 import Common.LongFilePathOs as os
  22 import sys
  23
  24 if sys.version_info.major == 3:
  25     import antlr4 as antlr
  26     from Eot.CParser4.CLexer import CLexer
  27     from Eot.CParser4.CParser import CParser
  28 else:
  29     import antlr3 as antlr
  30     antlr.InputStream = antlr.StringStream
  31     from Eot.CParser3.CLexer import CLexer
  32     from Eot.CParser3.CParser import CParser
  33
  34 from Eot import FileProfile
  35 from Eot.CodeFragment import PP_Directive
  36 from Eot.ParserWarning import Warning
  37
  38
  39 ##define T_CHAR_SPACE                ' '
  40 ##define T_CHAR_NULL                 '\0'
  41 ##define T_CHAR_CR                   '\r'
  42 ##define T_CHAR_TAB                  '\t'
  43 ##define T_CHAR_LF                   '\n'
  44 ##define T_CHAR_SLASH                '/'
  45 ##define T_CHAR_BACKSLASH            '\\'
  46 ##define T_CHAR_DOUBLE_QUOTE         '\"'
  47 ##define T_CHAR_SINGLE_QUOTE         '\''
  48 ##define T_CHAR_STAR                 '*'
  49 ##define T_CHAR_HASH                 '#'
  50
  51 (T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \
  52 T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \
  53 (' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')
  54
  55 SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')
  56
  57 (T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)
  58
  59 (T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)
  60
  61 ## The collector for source code fragments.
  62 #
  63 # PreprocessFile method should be called prior to ParseFile
  64 #
  65 # GetNext*** procedures mean these procedures will get next token first, then make judgement.
  66 # Get*** procedures mean these procedures will make judgement on current token only.
  67 #
  68 class CodeFragmentCollector:
  69     ## The constructor
  70     #
  71     #   @param  self        The object pointer
  72     #   @param  FileName    The file that to be parsed
  73     #
  74     def __init__(self, FileName):
  75         self.Profile = FileProfile.FileProfile(FileName)
  76         self.Profile.FileLinesList.append(T_CHAR_LF)
  77         self.FileName = FileName
  78         self.CurrentLineNumber = 1
  79         self.CurrentOffsetWithinLine = 0
  80
  81         self.__Token = ""
  82         self.__SkippedChars = ""
  83
  84     ## __EndOfFile() method
  85     #
  86     #   Judge current buffer pos is at file end
  87     #
  88     #   @param  self        The object pointer
  89     #   @retval True        Current File buffer position is at file end
  90     #   @retval False       Current File buffer position is NOT at file end
  91     #
  92     def __EndOfFile(self):
  93         NumberOfLines = len(self.Profile.FileLinesList)
  94         SizeOfLastLine = len(self.Profile.FileLinesList[-1])
  95         if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:
  96             return True
  97         elif self.CurrentLineNumber > NumberOfLines:
  98             return True
  99         else:
 100             return False
 101
 102     ## __EndOfLine() method
 103     #
 104     #   Judge current buffer pos is at line end
 105     #
 106     #   @param  self        The object pointer
 107     #   @retval True        Current File buffer position is at line end
 108     #   @retval False       Current File buffer position is NOT at line end
 109     #
 110     def __EndOfLine(self):
 111         SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])
 112         if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:
 113             return True
 114         else:
 115             return False
 116
 117     ## Rewind() method
 118     #
 119     #   Reset file data buffer to the initial state
 120     #
 121     #   @param  self        The object pointer
 122     #
 123     def Rewind(self):
 124         self.CurrentLineNumber = 1
 125         self.CurrentOffsetWithinLine = 0
 126
 127     ## __UndoOneChar() method
 128     #
 129     #   Go back one char in the file buffer
 130     #
 131     #   @param  self        The object pointer
 132     #   @retval True        Successfully go back one char
 133     #   @retval False       Not able to go back one char as file beginning reached
 134     #
 135     def __UndoOneChar(self):
 136
 137         if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:
 138             return False
 139         elif self.CurrentOffsetWithinLine == 0:
 140             self.CurrentLineNumber -= 1
 141             self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1
 142         else:
 143             self.CurrentOffsetWithinLine -= 1
 144         return True
 145
 146     ## __GetOneChar() method
 147     #
 148     #   Move forward one char in the file buffer
 149     #
 150     #   @param  self        The object pointer
 151     #
 152     def __GetOneChar(self):
 153         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 154                 self.CurrentLineNumber += 1
 155                 self.CurrentOffsetWithinLine = 0
 156         else:
 157                 self.CurrentOffsetWithinLine += 1
 158
 159     ## __CurrentChar() method
 160     #
 161     #   Get the char pointed to by the file buffer pointer
 162     #
 163     #   @param  self        The object pointer
 164     #   @retval Char        Current char
 165     #
 166     def __CurrentChar(self):
 167         CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]
 168
 169         return CurrentChar
 170
 171     ## __NextChar() method
 172     #
 173     #   Get the one char pass the char pointed to by the file buffer pointer
 174     #
 175     #   @param  self        The object pointer
 176     #   @retval Char        Next char
 177     #
 178     def __NextChar(self):
 179         if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:
 180             return self.Profile.FileLinesList[self.CurrentLineNumber][0]
 181         else:
 182             return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]
 183
 184     ## __SetCurrentCharValue() method
 185     #
 186     #   Modify the value of current char
 187     #
 188     #   @param  self        The object pointer
 189     #   @param  Value       The new value of current char
 190     #
 191     def __SetCurrentCharValue(self, Value):
 192         self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value
 193
 194     ## __SetCharValue() method
 195     #
 196     #   Modify the value of current char
 197     #
 198     #   @param  self        The object pointer
 199     #   @param  Value       The new value of current char
 200     #
 201     def __SetCharValue(self, Line, Offset, Value):
 202         self.Profile.FileLinesList[Line - 1][Offset] = Value
 203
 204     ## __CurrentLine() method
 205     #
 206     #   Get the list that contains current line contents
 207     #
 208     #   @param  self        The object pointer
 209     #   @retval List        current line contents
 210     #
 211     def __CurrentLine(self):
 212         return self.Profile.FileLinesList[self.CurrentLineNumber - 1]
 213
 214     ## __InsertComma() method
 215     #
 216     #   Insert ',' to replace PP
 217     #
 218     #   @param  self        The object pointer
 219     #   @retval List        current line contents
 220     #
 221     def __InsertComma(self, Line):
 222
 223
 224         if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:
 225             BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]
 226             if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):
 227                 return
 228
 229         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):
 230             return
 231
 232         if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):
 233             return
 234
 235         if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):
 236             return
 237
 238         self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')
 239
 240     ## PreprocessFileWithClear() method
 241     #
 242     # Run a preprocess for the file to clean all comments
 243     #
 244     # @param  self        The object pointer
 245     #
 246     def PreprocessFileWithClear(self):
 247
 248         self.Rewind()
 249         InComment = False
 250         DoubleSlashComment = False
 251         HashComment = False
 252         PPExtend = False
 253         PPDirectiveObj = None
 254         # HashComment in quoted string " " is ignored.
 255         InString = False
 256         InCharLiteral = False
 257
 258         self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]
 259         while not self.__EndOfFile():
 260
 261             if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:
 262                 InString = not InString
 263
 264             if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:
 265                 InCharLiteral = not InCharLiteral
 266             # meet new line, then no longer in a comment for // and '#'
 267             if self.__CurrentChar() == T_CHAR_LF:
 268                 if HashComment and PPDirectiveObj is not None:
 269                     if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 270                         PPDirectiveObj.Content += T_CHAR_LF
 271                         PPExtend = True
 272                     else:
 273                         PPExtend = False
 274
 275                 EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 276
 277                 if InComment and DoubleSlashComment:
 278                     InComment = False
 279                     DoubleSlashComment = False
 280
 281                 if InComment and HashComment and not PPExtend:
 282                     InComment = False
 283                     HashComment = False
 284                     PPDirectiveObj.Content += T_CHAR_LF
 285                     PPDirectiveObj.EndPos = EndLinePos
 286                     FileProfile.PPDirectiveList.append(PPDirectiveObj)
 287                     PPDirectiveObj = None
 288
 289                 if InString or InCharLiteral:
 290                     CurrentLine = "".join(self.__CurrentLine())
 291                     if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):
 292                         SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)
 293                         self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)
 294
 295                 self.CurrentLineNumber += 1
 296                 self.CurrentOffsetWithinLine = 0
 297             # check for */ comment end
 298             elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:
 299
 300                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 301                 self.__GetOneChar()
 302                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 303                 self.__GetOneChar()
 304                 InComment = False
 305             # set comments to spaces
 306             elif InComment:
 307                 if HashComment:
 308                     # // follows hash PP directive
 309                     if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 310                         InComment = False
 311                         HashComment = False
 312                         PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)
 313                         FileProfile.PPDirectiveList.append(PPDirectiveObj)
 314                         PPDirectiveObj = None
 315                         continue
 316                     else:
 317                         PPDirectiveObj.Content += self.__CurrentChar()
 318
 319                 self.__SetCurrentCharValue(T_CHAR_SPACE)
 320                 self.__GetOneChar()
 321             # check for // comment
 322             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:
 323                 InComment = True
 324                 DoubleSlashComment = True
 325
 326             # check for '#' comment
 327             elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:
 328                 InComment = True
 329                 HashComment = True
 330                 PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)
 331             # check for /* comment start
 332             elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:
 333
 334                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 335                 self.__GetOneChar()
 336                 self.__SetCurrentCharValue( T_CHAR_SPACE)
 337                 self.__GetOneChar()
 338                 InComment = True
 339             else:
 340                 self.__GetOneChar()
 341
 342         EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)
 343
 344         if InComment and HashComment and not PPExtend:
 345             PPDirectiveObj.EndPos = EndLinePos
 346             FileProfile.PPDirectiveList.append(PPDirectiveObj)
 347         self.Rewind()
 348
 349     ## ParseFile() method
 350     #
 351     #   Parse the file profile buffer to extract fd, fv ... information
 352     #   Exception will be raised if syntax error found
 353     #
 354     #   @param  self        The object pointer
 355     #
 356     def ParseFile(self):
 357         self.PreprocessFileWithClear()
 358         # restore from ListOfList to ListOfString
 359         self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]
 360         FileStringContents = ''
 361         for fileLine in self.Profile.FileLinesList:
 362             FileStringContents += fileLine
 363         cStream = antlr.InputStream(FileStringContents)
 364         lexer = CLexer(cStream)
 365         tStream = antlr.CommonTokenStream(lexer)
 366         parser = CParser(tStream)
 367         parser.translation_unit()
 368
 369     ## CleanFileProfileBuffer() method
 370     #
 371     #   Reset all contents of the profile of a file
 372     #
 373     def CleanFileProfileBuffer(self):
 374
 375         FileProfile.PPDirectiveList = []
 376         FileProfile.AssignmentExpressionList = []
 377         FileProfile.FunctionDefinitionList = []
 378         FileProfile.VariableDeclarationList = []
 379         FileProfile.EnumerationDefinitionList = []
 380         FileProfile.StructUnionDefinitionList = []
 381         FileProfile.TypedefDefinitionList = []
 382         FileProfile.FunctionCallingList = []
 383
 384     ## PrintFragments() method
 385     #
 386     #   Print the contents of the profile of a file
 387     #
 388     def PrintFragments(self):
 389
 390         print('################# ' + self.FileName + '#####################')
 391
 392         print('/****************************************/')
 393         print('/************** ASSIGNMENTS *************/')
 394         print('/****************************************/')
 395         for assign in FileProfile.AssignmentExpressionList:
 396             print(str(assign.StartPos) + assign.Name + assign.Operator + assign.Value)
 397
 398         print('/****************************************/')
 399         print('/********* PREPROCESS DIRECTIVES ********/')
 400         print('/****************************************/')
 401         for pp in FileProfile.PPDirectiveList:
 402             print(str(pp.StartPos) + pp.Content)
 403
 404         print('/****************************************/')
 405         print('/********* VARIABLE DECLARATIONS ********/')
 406         print('/****************************************/')
 407         for var in FileProfile.VariableDeclarationList:
 408             print(str(var.StartPos) + var.Modifier + ' '+ var.Declarator)
 409
 410         print('/****************************************/')
 411         print('/********* FUNCTION DEFINITIONS *********/')
 412         print('/****************************************/')
 413         for func in FileProfile.FunctionDefinitionList:
 414             print(str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos))
 415
 416         print('/****************************************/')
 417         print('/************ ENUMERATIONS **************/')
 418         print('/****************************************/')
 419         for enum in FileProfile.EnumerationDefinitionList:
 420             print(str(enum.StartPos) + enum.Content)
 421
 422         print('/****************************************/')
 423         print('/*********** STRUCTS/UNIONS *************/')
 424         print('/****************************************/')
 425         for su in FileProfile.StructUnionDefinitionList:
 426             print(str(su.StartPos) + su.Content)
 427
 428         print('/****************************************/')
 429         print('/************** TYPEDEFS ****************/')
 430         print('/****************************************/')
 431         for typedef in FileProfile.TypedefDefinitionList:
 432             print(str(typedef.StartPos) + typedef.ToType)
 433
 434 ##
 435 #
 436 # This acts like the main() function for the script, unless it is 'import'ed into another
 437 # script.
 438 #
 439 if __name__ == "__main__":
 440
 441     print("For Test.")