[mirror_edk2.git] / BaseTools / Source / Python / UPT / Library / CommentParsing.py

## @file\r
# This file is used to define comment parsing interface\r
#\r
# Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>\r
#\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
#\r
\r
'''\r
CommentParsing\r
'''\r
\r
##\r
# Import Modules\r
#\r
import re\r
\r
from Library.StringUtils import GetSplitValueList\r
from Library.StringUtils import CleanString2\r
from Library.DataType import HEADER_COMMENT_NOT_STARTED\r
from Library.DataType import TAB_COMMENT_SPLIT\r
from Library.DataType import HEADER_COMMENT_LICENSE\r
from Library.DataType import HEADER_COMMENT_ABSTRACT\r
from Library.DataType import HEADER_COMMENT_COPYRIGHT\r
from Library.DataType import HEADER_COMMENT_DESCRIPTION\r
from Library.DataType import TAB_SPACE_SPLIT\r
from Library.DataType import TAB_COMMA_SPLIT\r
from Library.DataType import SUP_MODULE_LIST\r
from Library.DataType import TAB_VALUE_SPLIT\r
from Library.DataType import TAB_PCD_VALIDRANGE\r
from Library.DataType import TAB_PCD_VALIDLIST\r
from Library.DataType import TAB_PCD_EXPRESSION\r
from Library.DataType import TAB_PCD_PROMPT\r
from Library.DataType import TAB_CAPHEX_START\r
from Library.DataType import TAB_HEX_START\r
from Library.DataType import PCD_ERR_CODE_MAX_SIZE\r
from Library.ExpressionValidate import IsValidRangeExpr\r
from Library.ExpressionValidate import IsValidListExpr\r
from Library.ExpressionValidate import IsValidLogicalExpr\r
from Object.POM.CommonObject import TextObject\r
from Object.POM.CommonObject import PcdErrorObject\r
import Logger.Log as Logger\r
from Logger.ToolError import FORMAT_INVALID\r
from Logger.ToolError import FORMAT_NOT_SUPPORTED\r
from Logger import StringTable as ST\r
\r
## ParseHeaderCommentSection\r
#\r
# Parse Header comment section lines, extract Abstract, Description, Copyright\r
# , License lines\r
#\r
# @param CommentList:   List of (Comment, LineNumber)\r
# @param FileName:      FileName of the comment\r
#\r
def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False):\r
    Abstract = ''\r
    Description = ''\r
    Copyright = ''\r
    License = ''\r
    EndOfLine = "\n"\r
    if IsBinaryHeader:\r
        STR_HEADER_COMMENT_START = "@BinaryHeader"\r
    else:\r
        STR_HEADER_COMMENT_START = "@file"\r
    HeaderCommentStage = HEADER_COMMENT_NOT_STARTED\r
\r
    #\r
    # first find the last copyright line\r
    #\r
    Last = 0\r
    for Index in range(len(CommentList)-1, 0, -1):\r
        Line = CommentList[Index][0]\r
        if _IsCopyrightLine(Line):\r
            Last = Index\r
            break\r
\r
    for Item in CommentList:\r
        Line = Item[0]\r
        LineNo = Item[1]\r
\r
        if not Line.startswith(TAB_COMMENT_SPLIT) and Line:\r
            Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])\r
        Comment = CleanString2(Line)[1]\r
        Comment = Comment.strip()\r
        #\r
        # if there are blank lines between License or Description, keep them as they would be\r
        # indication of different block; or in the position that Abstract should be, also keep it\r
        # as it indicates that no abstract\r
        #\r
        if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \\r
                                                      HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:\r
            continue\r
\r
        if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:\r
            if Comment.startswith(STR_HEADER_COMMENT_START):\r
                HeaderCommentStage = HEADER_COMMENT_ABSTRACT\r
            else:\r
                License += Comment + EndOfLine\r
        else:\r
            if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:\r
                #\r
                # in case there is no abstract and description\r
                #\r
                if not Comment:\r
                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
                elif _IsCopyrightLine(Comment):\r
                    Result, ErrMsg = _ValidateCopyright(Comment)\r
                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)\r
                    Copyright += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
                else:\r
                    Abstract += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
            elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:\r
                #\r
                # in case there is no description\r
                #\r
                if _IsCopyrightLine(Comment):\r
                    Result, ErrMsg = _ValidateCopyright(Comment)\r
                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)\r
                    Copyright += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
                else:\r
                    Description += Comment + EndOfLine\r
            elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:\r
                if _IsCopyrightLine(Comment):\r
                    Result, ErrMsg = _ValidateCopyright(Comment)\r
                    ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)\r
                    Copyright += Comment + EndOfLine\r
                else:\r
                    #\r
                    # Contents after copyright line are license, those non-copyright lines in between\r
                    # copyright line will be discarded\r
                    #\r
                    if LineNo > Last:\r
                        if License:\r
                            License += EndOfLine\r
                        License += Comment + EndOfLine\r
                        HeaderCommentStage = HEADER_COMMENT_LICENSE\r
            else:\r
                if not Comment and not License:\r
                    continue\r
                License += Comment + EndOfLine\r
\r
    return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()\r
\r
## _IsCopyrightLine\r
# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"\r
# followed by zero or more white space characters followed by a "(" character\r
#\r
# @param LineContent:  the line need to be checked\r
# @return: True if current line is copyright line, False else\r
#\r
def _IsCopyrightLine (LineContent):\r
    LineContent = LineContent.upper()\r
    Result = False\r
\r
    ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)\r
    if ReIsCopyrightRe.search(LineContent):\r
        Result = True\r
\r
    return Result\r
\r
## ParseGenericComment\r
#\r
# @param GenericComment: Generic comment list, element of\r
#                        (CommentLine, LineNum)\r
# @param ContainerFile:  Input value for filename of Dec file\r
#\r
def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):\r
    if ContainerFile:\r
        pass\r
    HelpTxt = None\r
    HelpStr = ''\r
\r
    for Item in GenericComment:\r
        CommentLine = Item[0]\r
        Comment = CleanString2(CommentLine)[1]\r
        if SkipTag is not None and Comment.startswith(SkipTag):\r
            Comment = Comment.replace(SkipTag, '', 1)\r
        HelpStr += Comment + '\n'\r
\r
    if HelpStr:\r
        HelpTxt = TextObject()\r
        if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':\r
            HelpStr = HelpStr[:-1]\r
        HelpTxt.SetString(HelpStr)\r
\r
    return HelpTxt\r
\r
## ParsePcdErrorCode\r
#\r
# @param Value: original ErrorCode value\r
# @param ContainerFile: Input value for filename of Dec file\r
# @param LineNum: Line Num\r
#\r
def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None):\r
    try:\r
        if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)):\r
            Base = 16\r
        else:\r
            Base = 10\r
        ErrorCode = int(Value, Base)\r
        if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0:\r
            Logger.Error('Parser',\r
                        FORMAT_NOT_SUPPORTED,\r
                        "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,\r
                        File = ContainerFile,\r
                        Line = LineNum)\r
        ErrorCode = '0x%x' % ErrorCode\r
        return ErrorCode\r
    except ValueError as XStr:\r
        if XStr:\r
            pass\r
        Logger.Error('Parser',\r
                    FORMAT_NOT_SUPPORTED,\r
                    "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,\r
                    File = ContainerFile,\r
                    Line = LineNum)\r
\r
## ParseDecPcdGenericComment\r
#\r
# @param GenericComment: Generic comment list, element of (CommentLine,\r
#                         LineNum)\r
# @param ContainerFile:  Input value for filename of Dec file\r
#\r
def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict):\r
    HelpStr = ''\r
    PromptStr = ''\r
    PcdErr = None\r
    PcdErrList = []\r
    ValidValueNum = 0\r
    ValidRangeNum = 0\r
    ExpressionNum = 0\r
\r
    for (CommentLine, LineNum) in GenericComment:\r
        Comment = CleanString2(CommentLine)[1]\r
        #\r
        # To replace Macro\r
        #\r
        MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'\r
        MatchedStrs =  re.findall(MACRO_PATTERN, Comment)\r
        for MatchedStr in MatchedStrs:\r
            if MatchedStr:\r
                Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip()\r
                if Macro in MacroReplaceDict:\r
                    Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro])\r
        if Comment.startswith(TAB_PCD_VALIDRANGE):\r
            if ValidValueNum > 0 or ExpressionNum > 0:\r
                Logger.Error('Parser',\r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_RANGES,\r
                             File = ContainerFile,\r
                             Line = LineNum)\r
            else:\r
                PcdErr = PcdErrorObject()\r
                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)\r
                PcdErr.SetCName(CName)\r
                PcdErr.SetFileLine(Comment)\r
                PcdErr.SetLineNum(LineNum)\r
                ValidRangeNum += 1\r
            ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip()\r
            Valid, Cause = _CheckRangeExpression(ValidRange)\r
            if Valid:\r
                ValueList = ValidRange.split(TAB_VALUE_SPLIT)\r
                if len(ValueList) > 1:\r
                    PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())\r
                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))\r
                else:\r
                    PcdErr.SetValidValueRange(ValidRange)\r
                PcdErrList.append(PcdErr)\r
            else:\r
                Logger.Error("Parser",\r
                         FORMAT_NOT_SUPPORTED,\r
                         Cause,\r
                         ContainerFile,\r
                         LineNum)\r
        elif Comment.startswith(TAB_PCD_VALIDLIST):\r
            if ValidRangeNum > 0 or ExpressionNum > 0:\r
                Logger.Error('Parser',\r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_RANGES,\r
                             File = ContainerFile,\r
                             Line = LineNum)\r
            elif ValidValueNum > 0:\r
                Logger.Error('Parser',\r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_VALIDVALUE,\r
                             File = ContainerFile,\r
                             Line = LineNum)\r
            else:\r
                PcdErr = PcdErrorObject()\r
                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)\r
                PcdErr.SetCName(CName)\r
                PcdErr.SetFileLine(Comment)\r
                PcdErr.SetLineNum(LineNum)\r
                ValidValueNum += 1\r
                ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip()\r
            Valid, Cause = _CheckListExpression(ValidValueExpr)\r
            if Valid:\r
                ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)\r
                ValueList = ValidValue.split(TAB_VALUE_SPLIT)\r
                if len(ValueList) > 1:\r
                    PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())\r
                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))\r
                else:\r
                    PcdErr.SetValidValue(ValidValue)\r
                PcdErrList.append(PcdErr)\r
            else:\r
                Logger.Error("Parser",\r
                         FORMAT_NOT_SUPPORTED,\r
                         Cause,\r
                         ContainerFile,\r
                         LineNum)\r
        elif Comment.startswith(TAB_PCD_EXPRESSION):\r
            if ValidRangeNum > 0 or ValidValueNum > 0:\r
                Logger.Error('Parser',\r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_RANGES,\r
                             File = ContainerFile,\r
                             Line = LineNum)\r
            else:\r
                PcdErr = PcdErrorObject()\r
                PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)\r
                PcdErr.SetCName(CName)\r
                PcdErr.SetFileLine(Comment)\r
                PcdErr.SetLineNum(LineNum)\r
                ExpressionNum += 1\r
            Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip()\r
            Valid, Cause = _CheckExpression(Expression)\r
            if Valid:\r
                ValueList = Expression.split(TAB_VALUE_SPLIT)\r
                if len(ValueList) > 1:\r
                    PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())\r
                    PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))\r
                else:\r
                    PcdErr.SetExpression(Expression)\r
                PcdErrList.append(PcdErr)\r
            else:\r
                Logger.Error("Parser",\r
                         FORMAT_NOT_SUPPORTED,\r
                         Cause,\r
                         ContainerFile,\r
                         LineNum)\r
        elif Comment.startswith(TAB_PCD_PROMPT):\r
            if PromptStr:\r
                Logger.Error('Parser',\r
                             FORMAT_NOT_SUPPORTED,\r
                             ST.WRN_MULTI_PCD_PROMPT,\r
                             File = ContainerFile,\r
                             Line = LineNum)\r
            PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip()\r
        else:\r
            if Comment:\r
                HelpStr += Comment + '\n'\r
\r
    #\r
    # remove the last EOL if the comment is of format 'FOO\n'\r
    #\r
    if HelpStr.endswith('\n'):\r
        if HelpStr != '\n' and not HelpStr.endswith('\n\n'):\r
            HelpStr = HelpStr[:-1]\r
\r
    return HelpStr, PcdErrList, PromptStr\r
\r
## ParseDecPcdTailComment\r
#\r
# @param TailCommentList:    Tail comment list of Pcd, item of format (Comment, LineNum)\r
# @param ContainerFile:      Input value for filename of Dec file\r
# @retVal SupModuleList:  The supported module type list detected\r
# @retVal HelpStr:  The generic help text string detected\r
#\r
def ParseDecPcdTailComment (TailCommentList, ContainerFile):\r
    assert(len(TailCommentList) == 1)\r
    TailComment = TailCommentList[0][0]\r
    LineNum = TailCommentList[0][1]\r
\r
    Comment = TailComment.lstrip(" #")\r
\r
    ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)\r
\r
    #\r
    # get first word and compare with SUP_MODULE_LIST\r
    #\r
    MatchObject = ReFindFirstWordRe.match(Comment)\r
    if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):\r
        return None, Comment\r
\r
    #\r
    # parse line, it must have supported module type specified\r
    #\r
    if Comment.find(TAB_COMMENT_SPLIT) == -1:\r
        Comment += TAB_COMMENT_SPLIT\r
    SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)\r
    SupModuleList = []\r
    for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):\r
        if not Mod:\r
            continue\r
        elif Mod not in SUP_MODULE_LIST:\r
            Logger.Error("UPT",\r
                         FORMAT_INVALID,\r
                         ST.WRN_INVALID_MODULE_TYPE%Mod,\r
                         ContainerFile,\r
                         LineNum)\r
        else:\r
            SupModuleList.append(Mod)\r
\r
    return SupModuleList, HelpStr\r
\r
## _CheckListExpression\r
#\r
# @param Expression: Pcd value list expression\r
#\r
def _CheckListExpression(Expression):\r
    ListExpr = ''\r
    if TAB_VALUE_SPLIT in Expression:\r
        ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]\r
    else:\r
        ListExpr = Expression\r
\r
    return IsValidListExpr(ListExpr)\r
\r
## _CheckExpression\r
#\r
# @param Expression: Pcd value expression\r
#\r
def _CheckExpression(Expression):\r
    Expr = ''\r
    if TAB_VALUE_SPLIT in Expression:\r
        Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]\r
    else:\r
        Expr = Expression\r
    return IsValidLogicalExpr(Expr, True)\r
\r
## _CheckRangeExpression\r
#\r
# @param Expression:    Pcd range expression\r
#\r
def _CheckRangeExpression(Expression):\r
    RangeExpr = ''\r
    if TAB_VALUE_SPLIT in Expression:\r
        RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]\r
    else:\r
        RangeExpr = Expression\r
\r
    return IsValidRangeExpr(RangeExpr)\r
\r
## ValidateCopyright\r
#\r
#\r
#\r
def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):\r
    if not Result:\r
        Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg)\r
\r
## _ValidateCopyright\r
#\r
# @param Line:    Line that contains copyright information, # stripped\r
#\r
# @retval Result: True if line is conformed to Spec format, False else\r
# @retval ErrMsg: the detailed error description\r
#\r
def _ValidateCopyright(Line):\r
    if Line:\r
        pass\r
    Result = True\r
    ErrMsg = ''\r
\r
    return Result, ErrMsg\r
\r
def GenerateTokenList (Comment):\r
    #\r
    # Tokenize Comment using '#' and ' ' as token separators\r
    #\r
    ReplacedComment = None\r
    while Comment != ReplacedComment:\r
        ReplacedComment = Comment\r
        Comment = Comment.replace('##', '#').replace('  ', ' ').replace(' ', '#').strip('# ')\r
    return Comment.split('#')\r
\r
\r
#\r
# Comment       - Comment to parse\r
# TypeTokens    - A dictionary of type token synonyms\r
# RemoveTokens  - A list of tokens to remove from help text\r
# ParseVariable - True for parsing [Guids].  Otherwise False\r
#\r
def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):\r
    #\r
    # Initialize return values\r
    #\r
    Usage = None\r
    Type = None\r
    String = None\r
\r
    Comment = Comment[0]\r
\r
    NumTokens = 2\r
    if ParseVariable:\r
        #\r
        # Remove white space around first instance of ':' from Comment if 'Variable'\r
        # is in front of ':' and Variable is the 1st or 2nd token in Comment.\r
        #\r
        List = Comment.split(':', 1)\r
        if len(List) > 1:\r
            SubList = GenerateTokenList (List[0].strip())\r
            if len(SubList) in [1, 2] and SubList[-1] == 'Variable':\r
                if List[1].strip().find('L"') == 0:\r
                    Comment = List[0].strip() + ':' + List[1].strip()\r
\r
        #\r
        # Remove first instance of L"<VariableName> from Comment and put into String\r
        # if and only if L"<VariableName>" is the 1st token, the 2nd token.  Or\r
        # L"<VariableName>" is the third token immediately following 'Variable:'.\r
        #\r
        End = -1\r
        Start = Comment.find('Variable:L"')\r
        if Start >= 0:\r
            String = Comment[Start + 9:]\r
            End = String[2:].find('"')\r
        else:\r
            Start = Comment.find('L"')\r
            if Start >= 0:\r
                String = Comment[Start:]\r
                End = String[2:].find('"')\r
        if End >= 0:\r
            SubList = GenerateTokenList (Comment[:Start])\r
            if len(SubList) < 2:\r
                Comment = Comment[:Start] + String[End + 3:]\r
                String = String[:End + 3]\r
                Type = 'Variable'\r
                NumTokens = 1\r
\r
    #\r
    # Initialize HelpText to Comment.\r
    # Content will be remove from HelpText as matching tokens are found\r
    #\r
    HelpText = Comment\r
\r
    #\r
    # Tokenize Comment using '#' and ' ' as token separators\r
    #\r
    List = GenerateTokenList (Comment)\r
\r
    #\r
    # Search first two tokens for Usage and Type and remove any matching tokens\r
    # from HelpText\r
    #\r
    for Token in List[0:NumTokens]:\r
        if Usage is None and Token in UsageTokens:\r
            Usage = UsageTokens[Token]\r
            HelpText = HelpText.replace(Token, '')\r
    if Usage is not None or not ParseVariable:\r
        for Token in List[0:NumTokens]:\r
            if Type is None and Token in TypeTokens:\r
                Type = TypeTokens[Token]\r
                HelpText = HelpText.replace(Token, '')\r
            if Usage is not None:\r
                for Token in List[0:NumTokens]:\r
                    if Token in RemoveTokens:\r
                        HelpText = HelpText.replace(Token, '')\r
\r
    #\r
    # If no Usage token is present and set Usage to UNDEFINED\r
    #\r
    if Usage is None:\r
        Usage = 'UNDEFINED'\r
\r
    #\r
    # If no Type token is present and set Type to UNDEFINED\r
    #\r
    if Type is None:\r
        Type = 'UNDEFINED'\r
\r
    #\r
    # If Type is not 'Variable:', then set String to None\r
    #\r
    if Type != 'Variable':\r
        String = None\r
\r
    #\r
    # Strip ' ' and '#' from the beginning of HelpText\r
    # If HelpText is an empty string after all parsing is\r
    # complete then set HelpText to None\r
    #\r
    HelpText = HelpText.lstrip('# ')\r
    if HelpText == '':\r
        HelpText = None\r
\r
    #\r
    # Return parsing results\r
    #\r
    return Usage, Type, String, HelpText\r