2 # This file is used to define comment parsing interface
4 # Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
6 # SPDX-License-Identifier: BSD-2-Clause-Patent
18 from Library
.StringUtils
import GetSplitValueList
19 from Library
.StringUtils
import CleanString2
20 from Library
.DataType
import HEADER_COMMENT_NOT_STARTED
21 from Library
.DataType
import TAB_COMMENT_SPLIT
22 from Library
.DataType
import HEADER_COMMENT_LICENSE
23 from Library
.DataType
import HEADER_COMMENT_ABSTRACT
24 from Library
.DataType
import HEADER_COMMENT_COPYRIGHT
25 from Library
.DataType
import HEADER_COMMENT_DESCRIPTION
26 from Library
.DataType
import TAB_SPACE_SPLIT
27 from Library
.DataType
import TAB_COMMA_SPLIT
28 from Library
.DataType
import SUP_MODULE_LIST
29 from Library
.DataType
import TAB_VALUE_SPLIT
30 from Library
.DataType
import TAB_PCD_VALIDRANGE
31 from Library
.DataType
import TAB_PCD_VALIDLIST
32 from Library
.DataType
import TAB_PCD_EXPRESSION
33 from Library
.DataType
import TAB_PCD_PROMPT
34 from Library
.DataType
import TAB_CAPHEX_START
35 from Library
.DataType
import TAB_HEX_START
36 from Library
.DataType
import PCD_ERR_CODE_MAX_SIZE
37 from Library
.ExpressionValidate
import IsValidRangeExpr
38 from Library
.ExpressionValidate
import IsValidListExpr
39 from Library
.ExpressionValidate
import IsValidLogicalExpr
40 from Object
.POM
.CommonObject
import TextObject
41 from Object
.POM
.CommonObject
import PcdErrorObject
42 import Logger
.Log
as Logger
43 from Logger
.ToolError
import FORMAT_INVALID
44 from Logger
.ToolError
import FORMAT_NOT_SUPPORTED
45 from Logger
import StringTable
as ST
47 ## ParseHeaderCommentSection
49 # Parse Header comment section lines, extract Abstract, Description, Copyright
52 # @param CommentList: List of (Comment, LineNumber)
53 # @param FileName: FileName of the comment
55 def ParseHeaderCommentSection(CommentList
, FileName
= None, IsBinaryHeader
= False):
62 STR_HEADER_COMMENT_START
= "@BinaryHeader"
64 STR_HEADER_COMMENT_START
= "@file"
65 HeaderCommentStage
= HEADER_COMMENT_NOT_STARTED
68 # first find the last copyright line
71 for Index
in range(len(CommentList
)-1, 0, -1):
72 Line
= CommentList
[Index
][0]
73 if _IsCopyrightLine(Line
):
77 for Item
in CommentList
:
81 if not Line
.startswith(TAB_COMMENT_SPLIT
) and Line
:
82 Logger
.Error("\nUPT", FORMAT_INVALID
, ST
.ERR_INVALID_COMMENT_FORMAT
, FileName
, Item
[1])
83 Comment
= CleanString2(Line
)[1]
84 Comment
= Comment
.strip()
86 # if there are blank lines between License or Description, keep them as they would be
87 # indication of different block; or in the position that Abstract should be, also keep it
88 # as it indicates that no abstract
90 if not Comment
and HeaderCommentStage
not in [HEADER_COMMENT_LICENSE
, \
91 HEADER_COMMENT_DESCRIPTION
, HEADER_COMMENT_ABSTRACT
]:
94 if HeaderCommentStage
== HEADER_COMMENT_NOT_STARTED
:
95 if Comment
.startswith(STR_HEADER_COMMENT_START
):
96 HeaderCommentStage
= HEADER_COMMENT_ABSTRACT
98 License
+= Comment
+ EndOfLine
100 if HeaderCommentStage
== HEADER_COMMENT_ABSTRACT
:
102 # in case there is no abstract and description
105 HeaderCommentStage
= HEADER_COMMENT_DESCRIPTION
106 elif _IsCopyrightLine(Comment
):
107 Result
, ErrMsg
= _ValidateCopyright(Comment
)
108 ValidateCopyright(Result
, ST
.WRN_INVALID_COPYRIGHT
, FileName
, LineNo
, ErrMsg
)
109 Copyright
+= Comment
+ EndOfLine
110 HeaderCommentStage
= HEADER_COMMENT_COPYRIGHT
112 Abstract
+= Comment
+ EndOfLine
113 HeaderCommentStage
= HEADER_COMMENT_DESCRIPTION
114 elif HeaderCommentStage
== HEADER_COMMENT_DESCRIPTION
:
116 # in case there is no description
118 if _IsCopyrightLine(Comment
):
119 Result
, ErrMsg
= _ValidateCopyright(Comment
)
120 ValidateCopyright(Result
, ST
.WRN_INVALID_COPYRIGHT
, FileName
, LineNo
, ErrMsg
)
121 Copyright
+= Comment
+ EndOfLine
122 HeaderCommentStage
= HEADER_COMMENT_COPYRIGHT
124 Description
+= Comment
+ EndOfLine
125 elif HeaderCommentStage
== HEADER_COMMENT_COPYRIGHT
:
126 if _IsCopyrightLine(Comment
):
127 Result
, ErrMsg
= _ValidateCopyright(Comment
)
128 ValidateCopyright(Result
, ST
.WRN_INVALID_COPYRIGHT
, FileName
, LineNo
, ErrMsg
)
129 Copyright
+= Comment
+ EndOfLine
132 # Contents after copyright line are license, those non-copyright lines in between
133 # copyright line will be discarded
138 License
+= Comment
+ EndOfLine
139 HeaderCommentStage
= HEADER_COMMENT_LICENSE
141 if not Comment
and not License
:
143 License
+= Comment
+ EndOfLine
145 return Abstract
.strip(), Description
.strip(), Copyright
.strip(), License
.strip()
148 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
149 # followed by zero or more white space characters followed by a "(" character
151 # @param LineContent: the line need to be checked
152 # @return: True if current line is copyright line, False else
154 def _IsCopyrightLine (LineContent
):
155 LineContent
= LineContent
.upper()
158 ReIsCopyrightRe
= re
.compile(r
"""(^|\s)COPYRIGHT *\(""", re
.DOTALL
)
159 if ReIsCopyrightRe
.search(LineContent
):
164 ## ParseGenericComment
166 # @param GenericComment: Generic comment list, element of
167 # (CommentLine, LineNum)
168 # @param ContainerFile: Input value for filename of Dec file
170 def ParseGenericComment (GenericComment
, ContainerFile
=None, SkipTag
=None):
176 for Item
in GenericComment
:
177 CommentLine
= Item
[0]
178 Comment
= CleanString2(CommentLine
)[1]
179 if SkipTag
is not None and Comment
.startswith(SkipTag
):
180 Comment
= Comment
.replace(SkipTag
, '', 1)
181 HelpStr
+= Comment
+ '\n'
184 HelpTxt
= TextObject()
185 if HelpStr
.endswith('\n') and not HelpStr
.endswith('\n\n') and HelpStr
!= '\n':
186 HelpStr
= HelpStr
[:-1]
187 HelpTxt
.SetString(HelpStr
)
193 # @param Value: original ErrorCode value
194 # @param ContainerFile: Input value for filename of Dec file
195 # @param LineNum: Line Num
197 def ParsePcdErrorCode (Value
= None, ContainerFile
= None, LineNum
= None):
199 if Value
.strip().startswith((TAB_HEX_START
, TAB_CAPHEX_START
)):
203 ErrorCode
= int(Value
, Base
)
204 if ErrorCode
> PCD_ERR_CODE_MAX_SIZE
or ErrorCode
< 0:
205 Logger
.Error('Parser',
206 FORMAT_NOT_SUPPORTED
,
207 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value
,
208 File
= ContainerFile
,
210 ErrorCode
= '0x%x' % ErrorCode
212 except ValueError as XStr
:
215 Logger
.Error('Parser',
216 FORMAT_NOT_SUPPORTED
,
217 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value
,
218 File
= ContainerFile
,
221 ## ParseDecPcdGenericComment
223 # @param GenericComment: Generic comment list, element of (CommentLine,
225 # @param ContainerFile: Input value for filename of Dec file
227 def ParseDecPcdGenericComment (GenericComment
, ContainerFile
, TokenSpaceGuidCName
, CName
, MacroReplaceDict
):
236 for (CommentLine
, LineNum
) in GenericComment
:
237 Comment
= CleanString2(CommentLine
)[1]
241 MACRO_PATTERN
= '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'
242 MatchedStrs
= re
.findall(MACRO_PATTERN
, Comment
)
243 for MatchedStr
in MatchedStrs
:
245 Macro
= MatchedStr
.strip().lstrip('$(').rstrip(')').strip()
246 if Macro
in MacroReplaceDict
:
247 Comment
= Comment
.replace(MatchedStr
, MacroReplaceDict
[Macro
])
248 if Comment
.startswith(TAB_PCD_VALIDRANGE
):
249 if ValidValueNum
> 0 or ExpressionNum
> 0:
250 Logger
.Error('Parser',
251 FORMAT_NOT_SUPPORTED
,
252 ST
.WRN_MULTI_PCD_RANGES
,
253 File
= ContainerFile
,
256 PcdErr
= PcdErrorObject()
257 PcdErr
.SetTokenSpaceGuidCName(TokenSpaceGuidCName
)
258 PcdErr
.SetCName(CName
)
259 PcdErr
.SetFileLine(Comment
)
260 PcdErr
.SetLineNum(LineNum
)
262 ValidRange
= Comment
.replace(TAB_PCD_VALIDRANGE
, "", 1).strip()
263 Valid
, Cause
= _CheckRangeExpression(ValidRange
)
265 ValueList
= ValidRange
.split(TAB_VALUE_SPLIT
)
266 if len(ValueList
) > 1:
267 PcdErr
.SetValidValueRange((TAB_VALUE_SPLIT
.join(ValueList
[1:])).strip())
268 PcdErr
.SetErrorNumber(ParsePcdErrorCode(ValueList
[0], ContainerFile
, LineNum
))
270 PcdErr
.SetValidValueRange(ValidRange
)
271 PcdErrList
.append(PcdErr
)
273 Logger
.Error("Parser",
274 FORMAT_NOT_SUPPORTED
,
278 elif Comment
.startswith(TAB_PCD_VALIDLIST
):
279 if ValidRangeNum
> 0 or ExpressionNum
> 0:
280 Logger
.Error('Parser',
281 FORMAT_NOT_SUPPORTED
,
282 ST
.WRN_MULTI_PCD_RANGES
,
283 File
= ContainerFile
,
285 elif ValidValueNum
> 0:
286 Logger
.Error('Parser',
287 FORMAT_NOT_SUPPORTED
,
288 ST
.WRN_MULTI_PCD_VALIDVALUE
,
289 File
= ContainerFile
,
292 PcdErr
= PcdErrorObject()
293 PcdErr
.SetTokenSpaceGuidCName(TokenSpaceGuidCName
)
294 PcdErr
.SetCName(CName
)
295 PcdErr
.SetFileLine(Comment
)
296 PcdErr
.SetLineNum(LineNum
)
298 ValidValueExpr
= Comment
.replace(TAB_PCD_VALIDLIST
, "", 1).strip()
299 Valid
, Cause
= _CheckListExpression(ValidValueExpr
)
301 ValidValue
= Comment
.replace(TAB_PCD_VALIDLIST
, "", 1).replace(TAB_COMMA_SPLIT
, TAB_SPACE_SPLIT
)
302 ValueList
= ValidValue
.split(TAB_VALUE_SPLIT
)
303 if len(ValueList
) > 1:
304 PcdErr
.SetValidValue((TAB_VALUE_SPLIT
.join(ValueList
[1:])).strip())
305 PcdErr
.SetErrorNumber(ParsePcdErrorCode(ValueList
[0], ContainerFile
, LineNum
))
307 PcdErr
.SetValidValue(ValidValue
)
308 PcdErrList
.append(PcdErr
)
310 Logger
.Error("Parser",
311 FORMAT_NOT_SUPPORTED
,
315 elif Comment
.startswith(TAB_PCD_EXPRESSION
):
316 if ValidRangeNum
> 0 or ValidValueNum
> 0:
317 Logger
.Error('Parser',
318 FORMAT_NOT_SUPPORTED
,
319 ST
.WRN_MULTI_PCD_RANGES
,
320 File
= ContainerFile
,
323 PcdErr
= PcdErrorObject()
324 PcdErr
.SetTokenSpaceGuidCName(TokenSpaceGuidCName
)
325 PcdErr
.SetCName(CName
)
326 PcdErr
.SetFileLine(Comment
)
327 PcdErr
.SetLineNum(LineNum
)
329 Expression
= Comment
.replace(TAB_PCD_EXPRESSION
, "", 1).strip()
330 Valid
, Cause
= _CheckExpression(Expression
)
332 ValueList
= Expression
.split(TAB_VALUE_SPLIT
)
333 if len(ValueList
) > 1:
334 PcdErr
.SetExpression((TAB_VALUE_SPLIT
.join(ValueList
[1:])).strip())
335 PcdErr
.SetErrorNumber(ParsePcdErrorCode(ValueList
[0], ContainerFile
, LineNum
))
337 PcdErr
.SetExpression(Expression
)
338 PcdErrList
.append(PcdErr
)
340 Logger
.Error("Parser",
341 FORMAT_NOT_SUPPORTED
,
345 elif Comment
.startswith(TAB_PCD_PROMPT
):
347 Logger
.Error('Parser',
348 FORMAT_NOT_SUPPORTED
,
349 ST
.WRN_MULTI_PCD_PROMPT
,
350 File
= ContainerFile
,
352 PromptStr
= Comment
.replace(TAB_PCD_PROMPT
, "", 1).strip()
355 HelpStr
+= Comment
+ '\n'
358 # remove the last EOL if the comment is of format 'FOO\n'
360 if HelpStr
.endswith('\n'):
361 if HelpStr
!= '\n' and not HelpStr
.endswith('\n\n'):
362 HelpStr
= HelpStr
[:-1]
364 return HelpStr
, PcdErrList
, PromptStr
366 ## ParseDecPcdTailComment
368 # @param TailCommentList: Tail comment list of Pcd, item of format (Comment, LineNum)
369 # @param ContainerFile: Input value for filename of Dec file
370 # @retVal SupModuleList: The supported module type list detected
371 # @retVal HelpStr: The generic help text string detected
373 def ParseDecPcdTailComment (TailCommentList
, ContainerFile
):
374 assert(len(TailCommentList
) == 1)
375 TailComment
= TailCommentList
[0][0]
376 LineNum
= TailCommentList
[0][1]
378 Comment
= TailComment
.lstrip(" #")
380 ReFindFirstWordRe
= re
.compile(r
"""^([^ #]*)""", re
.DOTALL
)
383 # get first word and compare with SUP_MODULE_LIST
385 MatchObject
= ReFindFirstWordRe
.match(Comment
)
386 if not (MatchObject
and MatchObject
.group(1) in SUP_MODULE_LIST
):
390 # parse line, it must have supported module type specified
392 if Comment
.find(TAB_COMMENT_SPLIT
) == -1:
393 Comment
+= TAB_COMMENT_SPLIT
394 SupMode
, HelpStr
= GetSplitValueList(Comment
, TAB_COMMENT_SPLIT
, 1)
396 for Mod
in GetSplitValueList(SupMode
, TAB_SPACE_SPLIT
):
399 elif Mod
not in SUP_MODULE_LIST
:
402 ST
.WRN_INVALID_MODULE_TYPE
%Mod
,
406 SupModuleList
.append(Mod
)
408 return SupModuleList
, HelpStr
410 ## _CheckListExpression
412 # @param Expression: Pcd value list expression
414 def _CheckListExpression(Expression
):
416 if TAB_VALUE_SPLIT
in Expression
:
417 ListExpr
= Expression
[Expression
.find(TAB_VALUE_SPLIT
)+1:]
419 ListExpr
= Expression
421 return IsValidListExpr(ListExpr
)
425 # @param Expression: Pcd value expression
427 def _CheckExpression(Expression
):
429 if TAB_VALUE_SPLIT
in Expression
:
430 Expr
= Expression
[Expression
.find(TAB_VALUE_SPLIT
)+1:]
433 return IsValidLogicalExpr(Expr
, True)
435 ## _CheckRangeExpression
437 # @param Expression: Pcd range expression
439 def _CheckRangeExpression(Expression
):
441 if TAB_VALUE_SPLIT
in Expression
:
442 RangeExpr
= Expression
[Expression
.find(TAB_VALUE_SPLIT
)+1:]
444 RangeExpr
= Expression
446 return IsValidRangeExpr(RangeExpr
)
452 def ValidateCopyright(Result
, ErrType
, FileName
, LineNo
, ErrMsg
):
454 Logger
.Warn("\nUPT", ErrType
, FileName
, LineNo
, ErrMsg
)
456 ## _ValidateCopyright
458 # @param Line: Line that contains copyright information, # stripped
460 # @retval Result: True if line is conformed to Spec format, False else
461 # @retval ErrMsg: the detailed error description
463 def _ValidateCopyright(Line
):
469 return Result
, ErrMsg
471 def GenerateTokenList (Comment
):
473 # Tokenize Comment using '#' and ' ' as token separators
475 ReplacedComment
= None
476 while Comment
!= ReplacedComment
:
477 ReplacedComment
= Comment
478 Comment
= Comment
.replace('##', '#').replace(' ', ' ').replace(' ', '#').strip('# ')
479 return Comment
.split('#')
483 # Comment - Comment to parse
484 # TypeTokens - A dictionary of type token synonyms
485 # RemoveTokens - A list of tokens to remove from help text
486 # ParseVariable - True for parsing [Guids]. Otherwise False
488 def ParseComment (Comment
, UsageTokens
, TypeTokens
, RemoveTokens
, ParseVariable
):
490 # Initialize return values
501 # Remove white space around first instance of ':' from Comment if 'Variable'
502 # is in front of ':' and Variable is the 1st or 2nd token in Comment.
504 List
= Comment
.split(':', 1)
506 SubList
= GenerateTokenList (List
[0].strip())
507 if len(SubList
) in [1, 2] and SubList
[-1] == 'Variable':
508 if List
[1].strip().find('L"') == 0:
509 Comment
= List
[0].strip() + ':' + List
[1].strip()
512 # Remove first instance of L"<VariableName> from Comment and put into String
513 # if and only if L"<VariableName>" is the 1st token, the 2nd token. Or
514 # L"<VariableName>" is the third token immediately following 'Variable:'.
517 Start
= Comment
.find('Variable:L"')
519 String
= Comment
[Start
+ 9:]
520 End
= String
[2:].find('"')
522 Start
= Comment
.find('L"')
524 String
= Comment
[Start
:]
525 End
= String
[2:].find('"')
527 SubList
= GenerateTokenList (Comment
[:Start
])
529 Comment
= Comment
[:Start
] + String
[End
+ 3:]
530 String
= String
[:End
+ 3]
535 # Initialize HelpText to Comment.
536 # Content will be remove from HelpText as matching tokens are found
541 # Tokenize Comment using '#' and ' ' as token separators
543 List
= GenerateTokenList (Comment
)
546 # Search first two tokens for Usage and Type and remove any matching tokens
549 for Token
in List
[0:NumTokens
]:
550 if Usage
is None and Token
in UsageTokens
:
551 Usage
= UsageTokens
[Token
]
552 HelpText
= HelpText
.replace(Token
, '')
553 if Usage
is not None or not ParseVariable
:
554 for Token
in List
[0:NumTokens
]:
555 if Type
is None and Token
in TypeTokens
:
556 Type
= TypeTokens
[Token
]
557 HelpText
= HelpText
.replace(Token
, '')
558 if Usage
is not None:
559 for Token
in List
[0:NumTokens
]:
560 if Token
in RemoveTokens
:
561 HelpText
= HelpText
.replace(Token
, '')
564 # If no Usage token is present and set Usage to UNDEFINED
570 # If no Type token is present and set Type to UNDEFINED
576 # If Type is not 'Variable:', then set String to None
578 if Type
!= 'Variable':
582 # Strip ' ' and '#' from the beginning of HelpText
583 # If HelpText is an empty string after all parsing is
584 # complete then set HelpText to None
586 HelpText
= HelpText
.lstrip('# ')
591 # Return parsing results
593 return Usage
, Type
, String
, HelpText