BaseTools/Source/Python/UPT/Library/CommentParsing.py

   1 ## @file
   2 # This file is used to define comment parsing interface
   3 #
   4 # Copyright (c) 2011, Intel Corporation. All rights reserved.<BR>
   5 #
   6 # This program and the accompanying materials are licensed and made available
   7 # under the terms and conditions of the BSD License which accompanies this
   8 # distribution. The full text of the license may be found at
   9 # http://opensource.org/licenses/bsd-license.php
  10 #
  11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 '''
  16 CommentParsing
  17 '''
  18
  19 ##
  20 # Import Modules
  21 #
  22 import re
  23
  24 from Library.String import GetSplitValueList
  25 from Library.String import CleanString2
  26 from Library.DataType import HEADER_COMMENT_NOT_STARTED
  27 from Library.DataType import TAB_COMMENT_SPLIT
  28 from Library.DataType import HEADER_COMMENT_LICENSE
  29 from Library.DataType import HEADER_COMMENT_ABSTRACT
  30 from Library.DataType import HEADER_COMMENT_COPYRIGHT
  31 from Library.DataType import HEADER_COMMENT_DESCRIPTION
  32 from Library.DataType import TAB_SPACE_SPLIT
  33 from Library.DataType import TAB_COMMA_SPLIT
  34 from Library.DataType import SUP_MODULE_LIST
  35 from Object.POM.CommonObject import TextObject
  36 from Object.POM.CommonObject import PcdErrorObject
  37 import Logger.Log as Logger
  38 from Logger.ToolError import FORMAT_INVALID
  39 from Logger.ToolError import FORMAT_NOT_SUPPORTED
  40 from Logger import StringTable as ST
  41
  42 ## ParseHeaderCommentSection
  43 #
  44 # Parse Header comment section lines, extract Abstract, Description, Copyright
  45 # , License lines
  46 #
  47 # @param CommentList:   List of (Comment, LineNumber)
  48 # @param FileName:      FileName of the comment
  49 #
  50 def ParseHeaderCommentSection(CommentList, FileName = None):
  51     Abstract = ''
  52     Description = ''
  53     Copyright = ''
  54     License = ''
  55     EndOfLine = "\n"
  56     STR_HEADER_COMMENT_START = "@file"
  57     HeaderCommentStage = HEADER_COMMENT_NOT_STARTED
  58
  59     #
  60     # first find the last copyright line
  61     #
  62     Last = 0
  63     for Index in xrange(len(CommentList)-1, 0, -1):
  64         Line = CommentList[Index][0]
  65         if _IsCopyrightLine(Line):
  66             Last = Index
  67             break
  68
  69     for Item in CommentList:
  70         Line = Item[0]
  71         LineNo = Item[1]
  72
  73         if not Line.startswith(TAB_COMMENT_SPLIT) and Line:
  74             Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])
  75         Comment = CleanString2(Line)[1]
  76         Comment = Comment.strip()
  77         #
  78         # if there are blank lines between License or Description, keep them as they would be
  79         # indication of different block; or in the position that Abstract should be, also keep it
  80         # as it indicates that no abstract
  81         #
  82         if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \
  83                                                       HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:
  84             continue
  85
  86         if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:
  87             if Comment.startswith(STR_HEADER_COMMENT_START):
  88                 HeaderCommentStage = HEADER_COMMENT_ABSTRACT
  89             else:
  90                 License += Comment + EndOfLine
  91         else:
  92             if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:
  93                 #
  94                 # in case there is no abstract and description
  95                 #
  96                 if not Comment:
  97                     Abstract = ''
  98                     HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
  99                 elif _IsCopyrightLine(Comment):
 100                     Result, ErrMsg = _ValidateCopyright(Comment)
 101                     ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
 102                     Copyright += Comment + EndOfLine
 103                     HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
 104                 else:
 105                     Abstract += Comment + EndOfLine
 106                     HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
 107             elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:
 108                 #
 109                 # in case there is no description
 110                 #
 111                 if _IsCopyrightLine(Comment):
 112                     Result, ErrMsg = _ValidateCopyright(Comment)
 113                     ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
 114                     Copyright += Comment + EndOfLine
 115                     HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
 116                 else:
 117                     Description += Comment + EndOfLine
 118             elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:
 119                 if _IsCopyrightLine(Comment):
 120                     Result, ErrMsg = _ValidateCopyright(Comment)
 121                     ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
 122                     Copyright += Comment + EndOfLine
 123                 else:
 124                     #
 125                     # Contents after copyright line are license, those non-copyright lines in between
 126                     # copyright line will be discarded
 127                     #
 128                     if LineNo > Last:
 129                         if License:
 130                             License += EndOfLine
 131                         License += Comment + EndOfLine
 132                         HeaderCommentStage = HEADER_COMMENT_LICENSE
 133             else:
 134                 if not Comment and not License:
 135                     continue
 136                 License += Comment + EndOfLine
 137
 138     if not Copyright:
 139         Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_COPYRIGHT_MISSING, \
 140                      FileName)
 141
 142     if not License:
 143         Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_LICENSE_MISSING, FileName)
 144
 145     return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()
 146
 147 ## _IsCopyrightLine
 148 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
 149 # followed by zero or more white space characters followed by a "(" character
 150 #
 151 # @param LineContent:  the line need to be checked
 152 # @return: True if current line is copyright line, False else
 153 #
 154 def _IsCopyrightLine (LineContent):
 155     LineContent = LineContent.upper()
 156     Result = False
 157
 158     ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)
 159     if ReIsCopyrightRe.search(LineContent):
 160         Result = True
 161
 162     return Result
 163
 164 ## ParseGenericComment
 165 #
 166 # @param GenericComment: Generic comment list, element of
 167 #                        (CommentLine, LineNum)
 168 # @param ContainerFile:  Input value for filename of Dec file
 169 #
 170 def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):
 171     if ContainerFile:
 172         pass
 173     HelpTxt = None
 174     HelpStr = ''
 175
 176     for Item in GenericComment:
 177         CommentLine = Item[0]
 178         Comment = CleanString2(CommentLine)[1]
 179         if SkipTag is not None and Comment.startswith(SkipTag):
 180             Comment = Comment.replace(SkipTag, '', 1)
 181         HelpStr += Comment + '\n'
 182
 183     if HelpStr:
 184         HelpTxt = TextObject()
 185         if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':
 186             HelpStr = HelpStr[:-1]
 187         HelpTxt.SetString(HelpStr)
 188
 189     return HelpTxt
 190
 191
 192 ## ParseDecPcdGenericComment
 193 #
 194 # @param GenericComment: Generic comment list, element of (CommentLine,
 195 #                         LineNum)
 196 # @param ContainerFile:  Input value for filename of Dec file
 197 #
 198 def ParseDecPcdGenericComment (GenericComment, ContainerFile):
 199     HelpStr = ''
 200     PcdErr = None
 201
 202     for (CommentLine, LineNum) in GenericComment:
 203         Comment = CleanString2(CommentLine)[1]
 204         if Comment.startswith("@ValidRange"):
 205             if PcdErr:
 206                 Logger.Error('Parser',
 207                              FORMAT_NOT_SUPPORTED,
 208                              ST.WRN_MULTI_PCD_RANGES,
 209                              File = ContainerFile,
 210                              Line = LineNum)
 211             ValidRange = Comment.replace("@ValidRange", "", 1)
 212             if _CheckRangeExpression(ValidRange):
 213                 PcdErr = PcdErrorObject()
 214                 PcdErr.SetValidValueRange(ValidRange)
 215         elif Comment.startswith("@ValidList"):
 216             if PcdErr:
 217                 Logger.Error('Parser',
 218                              FORMAT_NOT_SUPPORTED,
 219                              ST.WRN_MULTI_PCD_RANGES,
 220                              File = ContainerFile,
 221                              Line = LineNum)
 222             ValidValue = Comment.replace("@ValidList", "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)
 223             PcdErr = PcdErrorObject()
 224             PcdErr.SetValidValue(ValidValue)
 225         elif Comment.startswith("@Expression"):
 226             if PcdErr:
 227                 Logger.Error('Parser',
 228                              FORMAT_NOT_SUPPORTED,
 229                              ST.WRN_MULTI_PCD_RANGES,
 230                              File = ContainerFile,
 231                              Line = LineNum)
 232             Expression = Comment.replace("@Expression", "", 1)
 233             if _CheckRangeExpression(Expression):
 234                 PcdErr = PcdErrorObject()
 235                 PcdErr.SetExpression(Expression)
 236         else:
 237             HelpStr += Comment + '\n'
 238
 239     #
 240     # remove the last EOL if the comment is of format 'FOO\n'
 241     #
 242     if HelpStr.endswith('\n'):
 243         if HelpStr != '\n' and not HelpStr.endswith('\n\n'):
 244             HelpStr = HelpStr[:-1]
 245
 246     return HelpStr, PcdErr
 247
 248 ## ParseDecPcdTailComment
 249 #
 250 # @param TailCommentList:    Tail comment list of Pcd, item of format (Comment, LineNum)
 251 # @param ContainerFile:      Input value for filename of Dec file
 252 # @retVal SupModuleList:  The supported module type list detected
 253 # @retVal HelpStr:  The generic help text string detected
 254 #
 255 def ParseDecPcdTailComment (TailCommentList, ContainerFile):
 256     assert(len(TailCommentList) == 1)
 257     TailComment = TailCommentList[0][0]
 258     LineNum = TailCommentList[0][1]
 259
 260     Comment = TailComment.lstrip(" #")
 261
 262     ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)
 263
 264     #
 265     # get first word and compare with SUP_MODULE_LIST
 266     #
 267     MatchObject = ReFindFirstWordRe.match(Comment)
 268     if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):
 269         return None, Comment
 270
 271     #
 272     # parse line, it must have supported module type specified
 273     #
 274     if Comment.find(TAB_COMMENT_SPLIT) == -1:
 275         Comment += TAB_COMMENT_SPLIT
 276     SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)
 277     SupModuleList = []
 278     for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):
 279         if not Mod:
 280             continue
 281         elif Mod not in SUP_MODULE_LIST:
 282             Logger.Error("UPT",
 283                          FORMAT_INVALID,
 284                          ST.WRN_INVALID_MODULE_TYPE%Mod,
 285                          ContainerFile,
 286                          LineNum)
 287         else:
 288             SupModuleList.append(Mod)
 289
 290     return SupModuleList, HelpStr
 291
 292
 293 ## _CheckRangeExpression
 294 #
 295 # @param Expression:    Pcd range expression
 296 #
 297 def _CheckRangeExpression(Expression):
 298     #
 299     # check grammar for Pcd range expression is not required yet
 300     #
 301     if Expression:
 302         pass
 303     return True
 304
 305 ## ValidateCopyright
 306 #
 307 #
 308 #
 309 def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):
 310     if not Result:
 311         Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg)
 312
 313 ## _ValidateCopyright
 314 #
 315 # @param Line:    Line that contains copyright information, # stripped
 316 #
 317 # @retval Result: True if line is conformed to Spec format, False else
 318 # @retval ErrMsg: the detailed error description
 319 #
 320 def _ValidateCopyright(Line):
 321     if Line:
 322         pass
 323     Result = True
 324     ErrMsg = ''
 325
 326     return Result, ErrMsg
 327
 328 def GenerateTokenList (Comment):
 329     #
 330     # Tokenize Comment using '#' and ' ' as token seperators
 331     #
 332     RelplacedComment = None
 333     while Comment != RelplacedComment:
 334         RelplacedComment = Comment
 335         Comment = Comment.replace('##', '#').replace('  ', ' ').replace(' ', '#').strip('# ')
 336     return Comment.split('#')
 337
 338
 339 #
 340 # Comment       - Comment to parse
 341 # TypeTokens    - A dictionary of type token synonyms
 342 # RemoveTokens  - A list of tokens to remove from help text
 343 # ParseVariable - True for parsing [Guids].  Otherwise False
 344 #
 345 def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):
 346     #
 347     # Initialize return values
 348     #
 349     Usage = None
 350     Type = None
 351     String = None
 352     HelpText = None
 353
 354     Comment = Comment[0]
 355
 356     NumTokens = 2
 357     if ParseVariable:
 358         #
 359         # Remove white space around first instance of ':' from Comment if 'Variable'
 360         # is in front of ':' and Variable is the 1st or 2nd token in Comment.
 361         #
 362         List = Comment.split(':', 1)
 363         if len(List) > 1:
 364             SubList = GenerateTokenList (List[0].strip())
 365             if len(SubList) in [1, 2] and SubList[-1] == 'Variable':
 366                 if List[1].strip().find('L"') == 0:
 367                     Comment = List[0].strip() + ':' + List[1].strip()
 368
 369         #
 370         # Remove first instance of L"<VariableName> from Comment and put into String
 371         # if and only if L"<VariableName>" is the 1st token, the 2nd token.  Or
 372         # L"<VariableName>" is the third token immediately following 'Variable:'.
 373         #
 374         End = -1
 375         Start = Comment.find('Variable:L"')
 376         if Start >= 0:
 377             String = Comment[Start + 9:]
 378             End = String[2:].find('"')
 379         else:
 380             Start = Comment.find('L"')
 381             if Start >= 0:
 382                 String = Comment[Start:]
 383                 End = String[2:].find('"')
 384         if End >= 0:
 385             SubList = GenerateTokenList (Comment[:Start])
 386             if len(SubList) < 2:
 387                 Comment = Comment[:Start] + String[End + 3:]
 388                 String = String[:End + 3]
 389                 Type = 'Variable'
 390                 NumTokens = 1
 391
 392     #
 393     # Initialze HelpText to Comment.
 394     # Content will be remove from HelpText as matching tokens are found
 395     #
 396     HelpText = Comment
 397
 398     #
 399     # Tokenize Comment using '#' and ' ' as token seperators
 400     #
 401     List = GenerateTokenList (Comment)
 402
 403     #
 404     # Search first two tokens for Usage and Type and remove any matching tokens
 405     # from HelpText
 406     #
 407     for Token in List[0:NumTokens]:
 408         if Usage == None and Token in UsageTokens:
 409             Usage = UsageTokens[Token]
 410             HelpText = HelpText.replace(Token, '')
 411     if Usage != None or not ParseVariable:
 412         for Token in List[0:NumTokens]:
 413             if Type == None and Token in TypeTokens:
 414                 Type = TypeTokens[Token]
 415                 HelpText = HelpText.replace(Token, '')
 416             if Usage != None:
 417                 for Token in List[0:NumTokens]:
 418                     if Token in RemoveTokens:
 419                         HelpText = HelpText.replace(Token, '')
 420
 421     #
 422     # If no Usage token is present and set Usage to UNDEFINED
 423     #
 424     if Usage == None:
 425         Usage = 'UNDEFINED'
 426
 427     #
 428     # If no Type token is present and set Type to UNDEFINED
 429     #
 430     if Type == None:
 431         Type = 'UNDEFINED'
 432
 433     #
 434     # If Type is not 'Variable:', then set String to None
 435     #
 436     if Type != 'Variable':
 437         String = None
 438
 439     #
 440     # Strip ' ' and '#' from the beginning of HelpText
 441     # If HelpText is an empty string after all parsing is
 442     # complete then set HelpText to None
 443     #
 444     HelpText = HelpText.lstrip('# ')
 445     if HelpText == '':
 446         HelpText = None
 447
 448     #
 449     # Return parsing results
 450     #
 451     return Usage, Type, String, HelpText