BaseTools: use built in dict instead of custom version.

[mirror_edk2.git] / BaseTools / Source / Python / Eot / CodeFragmentCollector.py

## @file\r
# preprocess source file\r
#\r
#  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>\r
#\r
#  This program and the accompanying materials\r
#  are licensed and made available under the terms and conditions of the BSD License\r
#  which accompanies this distribution.  The full text of the license may be found at\r
#  http://opensource.org/licenses/bsd-license.php\r
#\r
#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
#  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
#\r
\r
##\r
# Import Modules\r
#\r
import re\r
import Common.LongFilePathOs as os\r
import sys\r
\r
import antlr3\r
from CLexer import CLexer\r
from CParser import CParser\r
\r
import FileProfile\r
from CodeFragment import PP_Directive\r
from ParserWarning import Warning\r
\r
\r
##define T_CHAR_SPACE                ' '\r
##define T_CHAR_NULL                 '\0'\r
##define T_CHAR_CR                   '\r'\r
##define T_CHAR_TAB                  '\t'\r
##define T_CHAR_LF                   '\n'\r
##define T_CHAR_SLASH                '/'\r
##define T_CHAR_BACKSLASH            '\\'\r
##define T_CHAR_DOUBLE_QUOTE         '\"'\r
##define T_CHAR_SINGLE_QUOTE         '\''\r
##define T_CHAR_STAR                 '*'\r
##define T_CHAR_HASH                 '#'\r
\r
(T_CHAR_SPACE, T_CHAR_NULL, T_CHAR_CR, T_CHAR_TAB, T_CHAR_LF, T_CHAR_SLASH, \\r
T_CHAR_BACKSLASH, T_CHAR_DOUBLE_QUOTE, T_CHAR_SINGLE_QUOTE, T_CHAR_STAR, T_CHAR_HASH) = \\r
(' ', '\0', '\r', '\t', '\n', '/', '\\', '\"', '\'', '*', '#')\r
\r
SEPERATOR_TUPLE = ('=', '|', ',', '{', '}')\r
\r
(T_COMMENT_TWO_SLASH, T_COMMENT_SLASH_STAR) = (0, 1)\r
\r
(T_PP_INCLUDE, T_PP_DEFINE, T_PP_OTHERS) = (0, 1, 2)\r
\r
## The collector for source code fragments.\r
#\r
# PreprocessFile method should be called prior to ParseFile\r
#\r
# GetNext*** procedures mean these procedures will get next token first, then make judgement.\r
# Get*** procedures mean these procedures will make judgement on current token only.\r
#\r
class CodeFragmentCollector:\r
    ## The constructor\r
    #\r
    #   @param  self        The object pointer\r
    #   @param  FileName    The file that to be parsed\r
    #\r
    def __init__(self, FileName):\r
        self.Profile = FileProfile.FileProfile(FileName)\r
        self.Profile.FileLinesList.append(T_CHAR_LF)\r
        self.FileName = FileName\r
        self.CurrentLineNumber = 1\r
        self.CurrentOffsetWithinLine = 0\r
\r
        self.__Token = ""\r
        self.__SkippedChars = ""\r
\r
    ## __EndOfFile() method\r
    #\r
    #   Judge current buffer pos is at file end\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval True        Current File buffer position is at file end\r
    #   @retval False       Current File buffer position is NOT at file end\r
    #\r
    def __EndOfFile(self):\r
        NumberOfLines = len(self.Profile.FileLinesList)\r
        SizeOfLastLine = len(self.Profile.FileLinesList[-1])\r
        if self.CurrentLineNumber == NumberOfLines and self.CurrentOffsetWithinLine >= SizeOfLastLine - 1:\r
            return True\r
        elif self.CurrentLineNumber > NumberOfLines:\r
            return True\r
        else:\r
            return False\r
\r
    ## __EndOfLine() method\r
    #\r
    #   Judge current buffer pos is at line end\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval True        Current File buffer position is at line end\r
    #   @retval False       Current File buffer position is NOT at line end\r
    #\r
    def __EndOfLine(self):\r
        SizeOfCurrentLine = len(self.Profile.FileLinesList[self.CurrentLineNumber - 1])\r
        if self.CurrentOffsetWithinLine >= SizeOfCurrentLine - 1:\r
            return True\r
        else:\r
            return False\r
\r
    ## Rewind() method\r
    #\r
    #   Reset file data buffer to the initial state\r
    #\r
    #   @param  self        The object pointer\r
    #\r
    def Rewind(self):\r
        self.CurrentLineNumber = 1\r
        self.CurrentOffsetWithinLine = 0\r
\r
    ## __UndoOneChar() method\r
    #\r
    #   Go back one char in the file buffer\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval True        Successfully go back one char\r
    #   @retval False       Not able to go back one char as file beginning reached\r
    #\r
    def __UndoOneChar(self):\r
\r
        if self.CurrentLineNumber == 1 and self.CurrentOffsetWithinLine == 0:\r
            return False\r
        elif self.CurrentOffsetWithinLine == 0:\r
            self.CurrentLineNumber -= 1\r
            self.CurrentOffsetWithinLine = len(self.__CurrentLine()) - 1\r
        else:\r
            self.CurrentOffsetWithinLine -= 1\r
        return True\r
\r
    ## __GetOneChar() method\r
    #\r
    #   Move forward one char in the file buffer\r
    #\r
    #   @param  self        The object pointer\r
    #\r
    def __GetOneChar(self):\r
        if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r
                self.CurrentLineNumber += 1\r
                self.CurrentOffsetWithinLine = 0\r
        else:\r
                self.CurrentOffsetWithinLine += 1\r
\r
    ## __CurrentChar() method\r
    #\r
    #   Get the char pointed to by the file buffer pointer\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval Char        Current char\r
    #\r
    def __CurrentChar(self):\r
        CurrentChar = self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine]\r
\r
        return CurrentChar\r
\r
    ## __NextChar() method\r
    #\r
    #   Get the one char pass the char pointed to by the file buffer pointer\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval Char        Next char\r
    #\r
    def __NextChar(self):\r
        if self.CurrentOffsetWithinLine == len(self.Profile.FileLinesList[self.CurrentLineNumber - 1]) - 1:\r
            return self.Profile.FileLinesList[self.CurrentLineNumber][0]\r
        else:\r
            return self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine + 1]\r
\r
    ## __SetCurrentCharValue() method\r
    #\r
    #   Modify the value of current char\r
    #\r
    #   @param  self        The object pointer\r
    #   @param  Value       The new value of current char\r
    #\r
    def __SetCurrentCharValue(self, Value):\r
        self.Profile.FileLinesList[self.CurrentLineNumber - 1][self.CurrentOffsetWithinLine] = Value\r
\r
    ## __SetCharValue() method\r
    #\r
    #   Modify the value of current char\r
    #\r
    #   @param  self        The object pointer\r
    #   @param  Value       The new value of current char\r
    #\r
    def __SetCharValue(self, Line, Offset, Value):\r
        self.Profile.FileLinesList[Line - 1][Offset] = Value\r
\r
    ## __CurrentLine() method\r
    #\r
    #   Get the list that contains current line contents\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval List        current line contents\r
    #\r
    def __CurrentLine(self):\r
        return self.Profile.FileLinesList[self.CurrentLineNumber - 1]\r
\r
    ## __InsertComma() method\r
    #\r
    #   Insert ',' to replace PP\r
    #\r
    #   @param  self        The object pointer\r
    #   @retval List        current line contents\r
    #\r
    def __InsertComma(self, Line):\r
\r
\r
        if self.Profile.FileLinesList[Line - 1][0] != T_CHAR_HASH:\r
            BeforeHashPart = str(self.Profile.FileLinesList[Line - 1]).split(T_CHAR_HASH)[0]\r
            if BeforeHashPart.rstrip().endswith(T_CHAR_COMMA) or BeforeHashPart.rstrip().endswith(';'):\r
                return\r
\r
        if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(','):\r
            return\r
\r
        if Line - 2 >= 0 and str(self.Profile.FileLinesList[Line - 2]).rstrip().endswith(';'):\r
            return\r
\r
        if str(self.Profile.FileLinesList[Line]).lstrip().startswith(',') or str(self.Profile.FileLinesList[Line]).lstrip().startswith(';'):\r
            return\r
\r
        self.Profile.FileLinesList[Line - 1].insert(self.CurrentOffsetWithinLine, ',')\r
\r
    ## PreprocessFileWithClear() method\r
    #\r
    # Run a preprocess for the file to clean all comments\r
    #\r
    # @param  self        The object pointer\r
    #\r
    def PreprocessFileWithClear(self):\r
\r
        self.Rewind()\r
        InComment = False\r
        DoubleSlashComment = False\r
        HashComment = False\r
        PPExtend = False\r
        PPDirectiveObj = None\r
        # HashComment in quoted string " " is ignored.\r
        InString = False\r
        InCharLiteral = False\r
\r
        self.Profile.FileLinesList = [list(s) for s in self.Profile.FileLinesListFromFile]\r
        while not self.__EndOfFile():\r
\r
            if not InComment and self.__CurrentChar() == T_CHAR_DOUBLE_QUOTE:\r
                InString = not InString\r
\r
            if not InComment and self.__CurrentChar() == T_CHAR_SINGLE_QUOTE:\r
                InCharLiteral = not InCharLiteral\r
            # meet new line, then no longer in a comment for // and '#'\r
            if self.__CurrentChar() == T_CHAR_LF:\r
                if HashComment and PPDirectiveObj is not None:\r
                    if PPDirectiveObj.Content.rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r
                        PPDirectiveObj.Content += T_CHAR_LF\r
                        PPExtend = True\r
                    else:\r
                        PPExtend = False\r
\r
                EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r
\r
                if InComment and DoubleSlashComment:\r
                    InComment = False\r
                    DoubleSlashComment = False\r
\r
                if InComment and HashComment and not PPExtend:\r
                    InComment = False\r
                    HashComment = False\r
                    PPDirectiveObj.Content += T_CHAR_LF\r
                    PPDirectiveObj.EndPos = EndLinePos\r
                    FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
                    PPDirectiveObj = None\r
\r
                if InString or InCharLiteral:\r
                    CurrentLine = "".join(self.__CurrentLine())\r
                    if CurrentLine.rstrip(T_CHAR_LF).rstrip(T_CHAR_CR).endswith(T_CHAR_BACKSLASH):\r
                        SlashIndex = CurrentLine.rindex(T_CHAR_BACKSLASH)\r
                        self.__SetCharValue(self.CurrentLineNumber, SlashIndex, T_CHAR_SPACE)\r
\r
                self.CurrentLineNumber += 1\r
                self.CurrentOffsetWithinLine = 0\r
            # check for */ comment end\r
            elif InComment and not DoubleSlashComment and not HashComment and self.__CurrentChar() == T_CHAR_STAR and self.__NextChar() == T_CHAR_SLASH:\r
\r
                self.__SetCurrentCharValue(T_CHAR_SPACE)\r
                self.__GetOneChar()\r
                self.__SetCurrentCharValue(T_CHAR_SPACE)\r
                self.__GetOneChar()\r
                InComment = False\r
            # set comments to spaces\r
            elif InComment:\r
                if HashComment:\r
                    # // follows hash PP directive\r
                    if self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r
                        InComment = False\r
                        HashComment = False\r
                        PPDirectiveObj.EndPos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine - 1)\r
                        FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
                        PPDirectiveObj = None\r
                        continue\r
                    else:\r
                        PPDirectiveObj.Content += self.__CurrentChar()\r
\r
                self.__SetCurrentCharValue(T_CHAR_SPACE)\r
                self.__GetOneChar()\r
            # check for // comment\r
            elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_SLASH:\r
                InComment = True\r
                DoubleSlashComment = True\r
\r
            # check for '#' comment\r
            elif self.__CurrentChar() == T_CHAR_HASH and not InString and not InCharLiteral:\r
                InComment = True\r
                HashComment = True\r
                PPDirectiveObj = PP_Directive('', (self.CurrentLineNumber, self.CurrentOffsetWithinLine), None)\r
            # check for /* comment start\r
            elif self.__CurrentChar() == T_CHAR_SLASH and self.__NextChar() == T_CHAR_STAR:\r
\r
                self.__SetCurrentCharValue( T_CHAR_SPACE)\r
                self.__GetOneChar()\r
                self.__SetCurrentCharValue( T_CHAR_SPACE)\r
                self.__GetOneChar()\r
                InComment = True\r
            else:\r
                self.__GetOneChar()\r
\r
        EndLinePos = (self.CurrentLineNumber, self.CurrentOffsetWithinLine)\r
\r
        if InComment and HashComment and not PPExtend:\r
            PPDirectiveObj.EndPos = EndLinePos\r
            FileProfile.PPDirectiveList.append(PPDirectiveObj)\r
        self.Rewind()\r
\r
    ## ParseFile() method\r
    #\r
    #   Parse the file profile buffer to extract fd, fv ... information\r
    #   Exception will be raised if syntax error found\r
    #\r
    #   @param  self        The object pointer\r
    #\r
    def ParseFile(self):\r
        self.PreprocessFileWithClear()\r
        # restore from ListOfList to ListOfString\r
        self.Profile.FileLinesList = ["".join(list) for list in self.Profile.FileLinesList]\r
        FileStringContents = ''\r
        for fileLine in self.Profile.FileLinesList:\r
            FileStringContents += fileLine\r
        cStream = antlr3.StringStream(FileStringContents)\r
        lexer = CLexer(cStream)\r
        tStream = antlr3.CommonTokenStream(lexer)\r
        parser = CParser(tStream)\r
        parser.translation_unit()\r
\r
    ## CleanFileProfileBuffer() method\r
    #\r
    #   Reset all contents of the profile of a file\r
    #\r
    def CleanFileProfileBuffer(self):\r
\r
        FileProfile.PPDirectiveList = []\r
        FileProfile.AssignmentExpressionList = []\r
        FileProfile.FunctionDefinitionList = []\r
        FileProfile.VariableDeclarationList = []\r
        FileProfile.EnumerationDefinitionList = []\r
        FileProfile.StructUnionDefinitionList = []\r
        FileProfile.TypedefDefinitionList = []\r
        FileProfile.FunctionCallingList = []\r
\r
    ## PrintFragments() method\r
    #\r
    #   Print the contents of the profile of a file\r
    #\r
    def PrintFragments(self):\r
\r
        print '################# ' + self.FileName + '#####################'\r
\r
        print '/****************************************/'\r
        print '/*************** ASSIGNMENTS ***************/'\r
        print '/****************************************/'\r
        for asign in FileProfile.AssignmentExpressionList:\r
            print str(asign.StartPos) + asign.Name + asign.Operator + asign.Value\r
\r
        print '/****************************************/'\r
        print '/********* PREPROCESS DIRECTIVES ********/'\r
        print '/****************************************/'\r
        for pp in FileProfile.PPDirectiveList:\r
            print str(pp.StartPos) + pp.Content\r
\r
        print '/****************************************/'\r
        print '/********* VARIABLE DECLARATIONS ********/'\r
        print '/****************************************/'\r
        for var in FileProfile.VariableDeclarationList:\r
            print str(var.StartPos) + var.Modifier + ' '+ var.Declarator\r
\r
        print '/****************************************/'\r
        print '/********* FUNCTION DEFINITIONS *********/'\r
        print '/****************************************/'\r
        for func in FileProfile.FunctionDefinitionList:\r
            print str(func.StartPos) + func.Modifier + ' '+ func.Declarator + ' ' + str(func.NamePos)\r
\r
        print '/****************************************/'\r
        print '/************ ENUMERATIONS **************/'\r
        print '/****************************************/'\r
        for enum in FileProfile.EnumerationDefinitionList:\r
            print str(enum.StartPos) + enum.Content\r
\r
        print '/****************************************/'\r
        print '/*********** STRUCTS/UNIONS *************/'\r
        print '/****************************************/'\r
        for su in FileProfile.StructUnionDefinitionList:\r
            print str(su.StartPos) + su.Content\r
\r
        print '/****************************************/'\r
        print '/************** TYPEDEFS ****************/'\r
        print '/****************************************/'\r
        for typedef in FileProfile.TypedefDefinitionList:\r
            print str(typedef.StartPos) + typedef.ToType\r
\r
##\r
#\r
# This acts like the main() function for the script, unless it is 'import'ed into another\r
# script.\r
#\r
if __name__ == "__main__":\r
\r
    print "For Test."\r