Add PCI PMC and PMCSR register definitions from PCI Power Management Interface Specif...

[mirror_edk2.git] / BaseTools / Source / Python / UPT / Library / CommentParsing.py

## @file\r
# This file is used to define comment parsing interface\r
#\r
# Copyright (c) 2011, Intel Corporation. All rights reserved.<BR>\r
#\r
# This program and the accompanying materials are licensed and made available \r
# under the terms and conditions of the BSD License which accompanies this \r
# distribution. The full text of the license may be found at \r
# http://opensource.org/licenses/bsd-license.php\r
#\r
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
#\r
\r
'''\r
CommentParsing\r
'''\r
\r
##\r
# Import Modules\r
#\r
import re\r
\r
from Library.String import GetSplitValueList\r
from Library.String import CleanString2\r
from Library.DataType import HEADER_COMMENT_NOT_STARTED\r
from Library.DataType import TAB_COMMENT_SPLIT\r
from Library.DataType import HEADER_COMMENT_LICENSE\r
from Library.DataType import HEADER_COMMENT_ABSTRACT\r
from Library.DataType import HEADER_COMMENT_COPYRIGHT\r
from Library.DataType import HEADER_COMMENT_DESCRIPTION\r
from Library.DataType import TAB_SPACE_SPLIT\r
from Library.DataType import TAB_COMMA_SPLIT\r
from Library.DataType import SUP_MODULE_LIST\r
from Object.POM.CommonObject import TextObject\r
from Object.POM.CommonObject import PcdErrorObject\r
import Logger.Log as Logger\r
from Logger.ToolError import FORMAT_INVALID\r
from Logger.ToolError import FORMAT_NOT_SUPPORTED\r
from Logger import StringTable as ST\r
\r
## ParseHeaderCommentSection\r
#\r
# Parse Header comment section lines, extract Abstract, Description, Copyright\r
# , License lines\r
#\r
# @param CommentList:   List of (Comment, LineNumber)\r
# @param FileName:      FileName of the comment\r
#\r
def ParseHeaderCommentSection(CommentList, FileName = None):\r
    Abstract = ''\r
    Description = ''\r
    Copyright = ''\r
    License = ''\r
    EndOfLine = "\n"\r
    STR_HEADER_COMMENT_START = "@file"\r
    HeaderCommentStage = HEADER_COMMENT_NOT_STARTED\r
    \r
    #\r
    # first find the last copyright line\r
    #\r
    Last = 0\r
    for Index in xrange(len(CommentList)-1, 0, -1):\r
        Line = CommentList[Index][0]\r
        if _IsCopyrightLine(Line):\r
            Last = Index\r
            break\r
    \r
    for Item in CommentList:\r
        Line = Item[0]\r
        LineNo = Item[1]\r
        \r
        if not Line.startswith(TAB_COMMENT_SPLIT) and Line:\r
            Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])\r
        Comment = CleanString2(Line)[1]\r
        Comment = Comment.strip()\r
        #\r
        # if there are blank lines between License or Description, keep them as they would be \r
        # indication of different block; or in the position that Abstract should be, also keep it\r
        # as it indicates that no abstract\r
        #\r
        if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \\r
                                                      HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:\r
            continue\r
        \r
        if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:\r
            if Comment.startswith(STR_HEADER_COMMENT_START):\r
                HeaderCommentStage = HEADER_COMMENT_ABSTRACT\r
            else:\r
                License += Comment + EndOfLine\r
        else:\r
            if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:\r
                #\r
                # in case there is no abstract and description\r
                #\r
                if not Comment:\r
                    Abstract = ''\r
                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
                elif _IsCopyrightLine(Comment):\r
                    Result, ErrMsg = _ValidateCopyright(Comment)\r
                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)\r
                    Copyright += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
                else:                    \r
                    Abstract += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
            elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:\r
                #\r
                # in case there is no description\r
                #                \r
                if _IsCopyrightLine(Comment):\r
                    Result, ErrMsg = _ValidateCopyright(Comment)\r
                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)\r
                    Copyright += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
                else:\r
                    Description += Comment + EndOfLine                \r
            elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:\r
                if _IsCopyrightLine(Comment):\r
                    Result, ErrMsg = _ValidateCopyright(Comment)\r
                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)\r
                    Copyright += Comment + EndOfLine\r
                else:\r
                    #\r
                    # Contents after copyright line are license, those non-copyright lines in between\r
                    # copyright line will be discarded \r
                    #\r
                    if LineNo > Last:\r
                        if License:\r
                            License += EndOfLine\r
                        License += Comment + EndOfLine\r
                        HeaderCommentStage = HEADER_COMMENT_LICENSE                \r
            else:\r
                if not Comment and not License:\r
                    continue\r
                License += Comment + EndOfLine\r
    \r
    if not Copyright:\r
        Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_COPYRIGHT_MISSING, \\r
                     FileName)\r
\r
    if not License:\r
        Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_LICENSE_MISSING, FileName)\r
                     \r
    return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()\r
\r
## _IsCopyrightLine\r
# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright" \r
# followed by zero or more white space characters followed by a "(" character \r
#\r
# @param LineContent:  the line need to be checked\r
# @return: True if current line is copyright line, False else\r
#\r
def _IsCopyrightLine (LineContent):\r
    LineContent = LineContent.upper()\r
    Result = False\r
    \r
    ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)\r
    if ReIsCopyrightRe.search(LineContent):\r
        Result = True\r
        \r
    return Result\r
\r
## ParseGenericComment\r
#\r
# @param GenericComment: Generic comment list, element of \r
#                        (CommentLine, LineNum)\r
# @param ContainerFile:  Input value for filename of Dec file\r
# \r
def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):\r
    if ContainerFile:\r
        pass\r
    HelpTxt = None         \r
    HelpStr = '' \r
        \r
    for Item in GenericComment:\r
        CommentLine = Item[0]\r
        Comment = CleanString2(CommentLine)[1]\r
        if SkipTag is not None and Comment.startswith(SkipTag):\r
            Comment = Comment.replace(SkipTag, '', 1)\r
        HelpStr += Comment + '\n'\r
        \r
    if HelpStr:\r
        HelpTxt = TextObject()\r
        if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':\r
            HelpStr = HelpStr[:-1]\r
        HelpTxt.SetString(HelpStr)\r
\r
    return HelpTxt\r
\r
    \r
## ParseDecPcdGenericComment\r
#\r
# @param GenericComment: Generic comment list, element of (CommentLine, \r
#                         LineNum)\r
# @param ContainerFile:  Input value for filename of Dec file\r
# \r
def ParseDecPcdGenericComment (GenericComment, ContainerFile):       \r
    HelpStr = '' \r
    PcdErr = None\r
        \r
    for (CommentLine, LineNum) in GenericComment:\r
        Comment = CleanString2(CommentLine)[1]\r
        if Comment.startswith("@ValidRange"):\r
            if PcdErr:\r
                Logger.Error('Parser', \r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_RANGES,\r
                             File = ContainerFile, \r
                             Line = LineNum)\r
            ValidRange = Comment.replace("@ValidRange", "", 1)\r
            if _CheckRangeExpression(ValidRange):\r
                PcdErr = PcdErrorObject()\r
                PcdErr.SetValidValueRange(ValidRange)\r
        elif Comment.startswith("@ValidList"):\r
            if PcdErr:\r
                Logger.Error('Parser', \r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_RANGES,\r
                             File = ContainerFile, \r
                             Line = LineNum)\r
            ValidValue = Comment.replace("@ValidList", "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)\r
            PcdErr = PcdErrorObject()\r
            PcdErr.SetValidValue(ValidValue)\r
        elif Comment.startswith("@Expression"):\r
            if PcdErr:\r
                Logger.Error('Parser', \r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_RANGES,\r
                             File = ContainerFile, \r
                             Line = LineNum)\r
            Expression = Comment.replace("@Expression", "", 1)\r
            if _CheckRangeExpression(Expression):\r
                PcdErr = PcdErrorObject()\r
                PcdErr.SetExpression(Expression)\r
        else:\r
            HelpStr += Comment + '\n'\r
    \r
    #\r
    # remove the last EOL if the comment is of format 'FOO\n'\r
    #\r
    if HelpStr.endswith('\n'):\r
        if HelpStr != '\n' and not HelpStr.endswith('\n\n'):\r
            HelpStr = HelpStr[:-1]\r
\r
    return HelpStr, PcdErr\r
\r
## ParseDecPcdTailComment\r
#\r
# @param TailCommentList:    Tail comment list of Pcd, item of format (Comment, LineNum)\r
# @param ContainerFile:      Input value for filename of Dec file\r
# @retVal SupModuleList:  The supported module type list detected\r
# @retVal HelpStr:  The generic help text string detected\r
#\r
def ParseDecPcdTailComment (TailCommentList, ContainerFile):\r
    assert(len(TailCommentList) == 1)\r
    TailComment = TailCommentList[0][0]\r
    LineNum = TailCommentList[0][1]\r
\r
    Comment = TailComment.lstrip(" #")\r
    \r
    ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)\r
    \r
    #\r
    # get first word and compare with SUP_MODULE_LIST\r
    #\r
    MatchObject = ReFindFirstWordRe.match(Comment)\r
    if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):\r
        return None, Comment\r
\r
    #\r
    # parse line, it must have supported module type specified\r
    #\r
    if Comment.find(TAB_COMMENT_SPLIT) == -1:\r
        Comment += TAB_COMMENT_SPLIT    \r
    SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)\r
    SupModuleList = []\r
    for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):\r
        if not Mod:\r
            continue\r
        elif Mod not in SUP_MODULE_LIST:\r
            Logger.Error("UPT",\r
                         FORMAT_INVALID,\r
                         ST.WRN_INVALID_MODULE_TYPE%Mod, \r
                         ContainerFile, \r
                         LineNum)\r
        else:\r
            SupModuleList.append(Mod)\r
\r
    return SupModuleList, HelpStr\r
\r
\r
## _CheckRangeExpression\r
#\r
# @param Expression:    Pcd range expression\r
#          \r
def _CheckRangeExpression(Expression):\r
    #\r
    # check grammar for Pcd range expression is not required yet\r
    #\r
    if Expression:\r
        pass\r
    return True\r
\r
## ValidateCopyright\r
#\r
#\r
#\r
def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):\r
    if not Result:\r
        Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg) \r
\r
## _ValidateCopyright\r
#\r
# @param Line:    Line that contains copyright information, # stripped\r
# \r
# @retval Result: True if line is conformed to Spec format, False else\r
# @retval ErrMsg: the detailed error description\r
#  \r
def _ValidateCopyright(Line):\r
    if Line:\r
        pass\r
    Result = True\r
    ErrMsg = ''\r
    \r
    return Result, ErrMsg\r
\r
def GenerateTokenList (Comment):\r
    #\r
    # Tokenize Comment using '#' and ' ' as token seperators\r
    #\r
    RelplacedComment = None    \r
    while Comment != RelplacedComment:\r
        RelplacedComment = Comment\r
        Comment = Comment.replace('##', '#').replace('  ', ' ').replace(' ', '#').strip('# ')\r
    return Comment.split('#')\r
\r
\r
#\r
# Comment       - Comment to parse\r
# TypeTokens    - A dictionary of type token synonyms\r
# RemoveTokens  - A list of tokens to remove from help text\r
# ParseVariable - True for parsing [Guids].  Otherwise False\r
#\r
def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):\r
    #\r
    # Initialize return values\r
    #\r
    Usage = None\r
    Type = None\r
    String = None\r
    HelpText = None\r
    \r
    Comment = Comment[0]\r
    \r
    NumTokens = 2  \r
    if ParseVariable:\r
        # \r
        # Remove white space around first instance of ':' from Comment if 'Variable' \r
        # is in front of ':' and Variable is the 1st or 2nd token in Comment.\r
        #\r
        List = Comment.split(':', 1)    \r
        if len(List) > 1:\r
            SubList = GenerateTokenList (List[0].strip())\r
            if len(SubList) in [1, 2] and SubList[-1] == 'Variable':\r
                if List[1].strip().find('L"') == 0:      \r
                    Comment = List[0].strip() + ':' + List[1].strip()\r
        \r
        # \r
        # Remove first instance of L"<VariableName> from Comment and put into String\r
        # if and only if L"<VariableName>" is the 1st token, the 2nd token.  Or \r
        # L"<VariableName>" is the third token immediately following 'Variable:'.\r
        #\r
        End = -1\r
        Start = Comment.find('Variable:L"')\r
        if Start >= 0:\r
            String = Comment[Start + 9:]\r
            End = String[2:].find('"')\r
        else:\r
            Start = Comment.find('L"')\r
            if Start >= 0:\r
                String = Comment[Start:]\r
                End = String[2:].find('"')\r
        if End >= 0:\r
            SubList = GenerateTokenList (Comment[:Start])\r
            if len(SubList) < 2: \r
                Comment = Comment[:Start] + String[End + 3:]\r
                String = String[:End + 3]\r
                Type = 'Variable'\r
                NumTokens = 1  \r
    \r
    #\r
    # Initialze HelpText to Comment.  \r
    # Content will be remove from HelpText as matching tokens are found\r
    #  \r
    HelpText = Comment\r
    \r
    #\r
    # Tokenize Comment using '#' and ' ' as token seperators\r
    #\r
    List = GenerateTokenList (Comment)\r
    \r
    #\r
    # Search first two tokens for Usage and Type and remove any matching tokens \r
    # from HelpText\r
    #\r
    for Token in List[0:NumTokens]:\r
        if Usage == None and Token in UsageTokens:\r
            Usage = UsageTokens[Token]\r
            HelpText = HelpText.replace(Token, '')\r
    if Usage != None or not ParseVariable:\r
        for Token in List[0:NumTokens]:\r
            if Type == None and Token in TypeTokens:\r
                Type = TypeTokens[Token]\r
                HelpText = HelpText.replace(Token, '')\r
            if Usage != None:    \r
                for Token in List[0:NumTokens]:\r
                    if Token in RemoveTokens:\r
                        HelpText = HelpText.replace(Token, '')\r
    \r
    #\r
    # If no Usage token is present and set Usage to UNDEFINED\r
    #  \r
    if Usage == None:\r
        Usage = 'UNDEFINED'\r
    \r
    #\r
    # If no Type token is present and set Type to UNDEFINED\r
    #  \r
    if Type == None:\r
        Type = 'UNDEFINED'\r
    \r
    #\r
    # If Type is not 'Variable:', then set String to None\r
    #  \r
    if Type != 'Variable':\r
        String = None  \r
    \r
    #\r
    # Strip ' ' and '#' from the beginning of HelpText\r
    # If HelpText is an empty string after all parsing is \r
    # complete then set HelpText to None\r
    #  \r
    HelpText = HelpText.lstrip('# ')\r
    if HelpText == '':\r
        HelpText = None\r
      \r
    #\r
    # Return parsing results\r
    #  \r
    return Usage, Type, String, HelpText  \r