2 # This file is used to define comment parsing interface
4 # Copyright (c) 2011 - 2014, Intel Corporation. All rights reserved.<BR>
6 # This program and the accompanying materials are licensed and made available
7 # under the terms and conditions of the BSD License which accompanies this
8 # distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
24 from Library
.String
import GetSplitValueList
25 from Library
.String
import CleanString2
26 from Library
.DataType
import HEADER_COMMENT_NOT_STARTED
27 from Library
.DataType
import TAB_COMMENT_SPLIT
28 from Library
.DataType
import HEADER_COMMENT_LICENSE
29 from Library
.DataType
import HEADER_COMMENT_ABSTRACT
30 from Library
.DataType
import HEADER_COMMENT_COPYRIGHT
31 from Library
.DataType
import HEADER_COMMENT_DESCRIPTION
32 from Library
.DataType
import TAB_SPACE_SPLIT
33 from Library
.DataType
import TAB_COMMA_SPLIT
34 from Library
.DataType
import SUP_MODULE_LIST
35 from Library
.DataType
import TAB_VALUE_SPLIT
36 from Library
.DataType
import TAB_PCD_VALIDRANGE
37 from Library
.DataType
import TAB_PCD_VALIDLIST
38 from Library
.DataType
import TAB_PCD_EXPRESSION
39 from Library
.DataType
import TAB_PCD_PROMPT
40 from Library
.DataType
import TAB_CAPHEX_START
41 from Library
.DataType
import TAB_HEX_START
42 from Library
.DataType
import PCD_ERR_CODE_MAX_SIZE
43 from Library
.ExpressionValidate
import IsValidRangeExpr
44 from Library
.ExpressionValidate
import IsValidListExpr
45 from Library
.ExpressionValidate
import IsValidLogicalExpr
46 from Object
.POM
.CommonObject
import TextObject
47 from Object
.POM
.CommonObject
import PcdErrorObject
48 import Logger
.Log
as Logger
49 from Logger
.ToolError
import FORMAT_INVALID
50 from Logger
.ToolError
import FORMAT_NOT_SUPPORTED
51 from Logger
import StringTable
as ST
53 ## ParseHeaderCommentSection
55 # Parse Header comment section lines, extract Abstract, Description, Copyright
58 # @param CommentList: List of (Comment, LineNumber)
59 # @param FileName: FileName of the comment
61 def ParseHeaderCommentSection(CommentList
, FileName
= None, IsBinaryHeader
= False):
68 STR_HEADER_COMMENT_START
= "@BinaryHeader"
70 STR_HEADER_COMMENT_START
= "@file"
71 HeaderCommentStage
= HEADER_COMMENT_NOT_STARTED
74 # first find the last copyright line
77 for Index
in xrange(len(CommentList
)-1, 0, -1):
78 Line
= CommentList
[Index
][0]
79 if _IsCopyrightLine(Line
):
83 for Item
in CommentList
:
87 if not Line
.startswith(TAB_COMMENT_SPLIT
) and Line
:
88 Logger
.Error("\nUPT", FORMAT_INVALID
, ST
.ERR_INVALID_COMMENT_FORMAT
, FileName
, Item
[1])
89 Comment
= CleanString2(Line
)[1]
90 Comment
= Comment
.strip()
92 # if there are blank lines between License or Description, keep them as they would be
93 # indication of different block; or in the position that Abstract should be, also keep it
94 # as it indicates that no abstract
96 if not Comment
and HeaderCommentStage
not in [HEADER_COMMENT_LICENSE
, \
97 HEADER_COMMENT_DESCRIPTION
, HEADER_COMMENT_ABSTRACT
]:
100 if HeaderCommentStage
== HEADER_COMMENT_NOT_STARTED
:
101 if Comment
.startswith(STR_HEADER_COMMENT_START
):
102 HeaderCommentStage
= HEADER_COMMENT_ABSTRACT
104 License
+= Comment
+ EndOfLine
106 if HeaderCommentStage
== HEADER_COMMENT_ABSTRACT
:
108 # in case there is no abstract and description
111 HeaderCommentStage
= HEADER_COMMENT_DESCRIPTION
112 elif _IsCopyrightLine(Comment
):
113 Result
, ErrMsg
= _ValidateCopyright(Comment
)
114 ValidateCopyright(Result
, ST
.WRN_INVALID_COPYRIGHT
, FileName
, LineNo
, ErrMsg
)
115 Copyright
+= Comment
+ EndOfLine
116 HeaderCommentStage
= HEADER_COMMENT_COPYRIGHT
118 Abstract
+= Comment
+ EndOfLine
119 HeaderCommentStage
= HEADER_COMMENT_DESCRIPTION
120 elif HeaderCommentStage
== HEADER_COMMENT_DESCRIPTION
:
122 # in case there is no description
124 if _IsCopyrightLine(Comment
):
125 Result
, ErrMsg
= _ValidateCopyright(Comment
)
126 ValidateCopyright(Result
, ST
.WRN_INVALID_COPYRIGHT
, FileName
, LineNo
, ErrMsg
)
127 Copyright
+= Comment
+ EndOfLine
128 HeaderCommentStage
= HEADER_COMMENT_COPYRIGHT
130 Description
+= Comment
+ EndOfLine
131 elif HeaderCommentStage
== HEADER_COMMENT_COPYRIGHT
:
132 if _IsCopyrightLine(Comment
):
133 Result
, ErrMsg
= _ValidateCopyright(Comment
)
134 ValidateCopyright(Result
, ST
.WRN_INVALID_COPYRIGHT
, FileName
, LineNo
, ErrMsg
)
135 Copyright
+= Comment
+ EndOfLine
138 # Contents after copyright line are license, those non-copyright lines in between
139 # copyright line will be discarded
144 License
+= Comment
+ EndOfLine
145 HeaderCommentStage
= HEADER_COMMENT_LICENSE
147 if not Comment
and not License
:
149 License
+= Comment
+ EndOfLine
151 return Abstract
.strip(), Description
.strip(), Copyright
.strip(), License
.strip()
154 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
155 # followed by zero or more white space characters followed by a "(" character
157 # @param LineContent: the line need to be checked
158 # @return: True if current line is copyright line, False else
160 def _IsCopyrightLine (LineContent
):
161 LineContent
= LineContent
.upper()
164 ReIsCopyrightRe
= re
.compile(r
"""(^|\s)COPYRIGHT *\(""", re
.DOTALL
)
165 if ReIsCopyrightRe
.search(LineContent
):
170 ## ParseGenericComment
172 # @param GenericComment: Generic comment list, element of
173 # (CommentLine, LineNum)
174 # @param ContainerFile: Input value for filename of Dec file
176 def ParseGenericComment (GenericComment
, ContainerFile
=None, SkipTag
=None):
182 for Item
in GenericComment
:
183 CommentLine
= Item
[0]
184 Comment
= CleanString2(CommentLine
)[1]
185 if SkipTag
is not None and Comment
.startswith(SkipTag
):
186 Comment
= Comment
.replace(SkipTag
, '', 1)
187 HelpStr
+= Comment
+ '\n'
190 HelpTxt
= TextObject()
191 if HelpStr
.endswith('\n') and not HelpStr
.endswith('\n\n') and HelpStr
!= '\n':
192 HelpStr
= HelpStr
[:-1]
193 HelpTxt
.SetString(HelpStr
)
199 # @param Value: original ErrorCode value
200 # @param ContainerFile: Input value for filename of Dec file
201 # @param LineNum: Line Num
203 def ParsePcdErrorCode (Value
= None, ContainerFile
= None, LineNum
= None):
205 if Value
.strip().startswith((TAB_HEX_START
, TAB_CAPHEX_START
)):
209 ErrorCode
= long(Value
, Base
)
210 if ErrorCode
> PCD_ERR_CODE_MAX_SIZE
or ErrorCode
< 0:
211 Logger
.Error('Parser',
212 FORMAT_NOT_SUPPORTED
,
213 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value
,
214 File
= ContainerFile
,
217 # To delete the tailing 'L'
219 return hex(ErrorCode
)[:-1]
220 except ValueError, XStr
:
223 Logger
.Error('Parser',
224 FORMAT_NOT_SUPPORTED
,
225 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value
,
226 File
= ContainerFile
,
229 ## ParseDecPcdGenericComment
231 # @param GenericComment: Generic comment list, element of (CommentLine,
233 # @param ContainerFile: Input value for filename of Dec file
235 def ParseDecPcdGenericComment (GenericComment
, ContainerFile
, TokenSpaceGuidCName
, CName
, MacroReplaceDict
):
244 for (CommentLine
, LineNum
) in GenericComment
:
245 Comment
= CleanString2(CommentLine
)[1]
249 MACRO_PATTERN
= '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'
250 MatchedStrs
= re
.findall(MACRO_PATTERN
, Comment
)
251 for MatchedStr
in MatchedStrs
:
253 Macro
= MatchedStr
.strip().lstrip('$(').rstrip(')').strip()
254 if Macro
in MacroReplaceDict
:
255 Comment
= Comment
.replace(MatchedStr
, MacroReplaceDict
[Macro
])
256 if Comment
.startswith(TAB_PCD_VALIDRANGE
):
257 if ValidValueNum
> 0 or ExpressionNum
> 0:
258 Logger
.Error('Parser',
259 FORMAT_NOT_SUPPORTED
,
260 ST
.WRN_MULTI_PCD_RANGES
,
261 File
= ContainerFile
,
264 PcdErr
= PcdErrorObject()
265 PcdErr
.SetTokenSpaceGuidCName(TokenSpaceGuidCName
)
266 PcdErr
.SetCName(CName
)
267 PcdErr
.SetFileLine(Comment
)
268 PcdErr
.SetLineNum(LineNum
)
270 ValidRange
= Comment
.replace(TAB_PCD_VALIDRANGE
, "", 1).strip()
271 Valid
, Cause
= _CheckRangeExpression(ValidRange
)
273 ValueList
= ValidRange
.split(TAB_VALUE_SPLIT
)
274 if len(ValueList
) > 1:
275 PcdErr
.SetValidValueRange((TAB_VALUE_SPLIT
.join(ValueList
[1:])).strip())
276 PcdErr
.SetErrorNumber(ParsePcdErrorCode(ValueList
[0], ContainerFile
, LineNum
))
278 PcdErr
.SetValidValueRange(ValidRange
)
279 PcdErrList
.append(PcdErr
)
281 Logger
.Error("Parser",
282 FORMAT_NOT_SUPPORTED
,
286 elif Comment
.startswith(TAB_PCD_VALIDLIST
):
287 if ValidRangeNum
> 0 or ExpressionNum
> 0:
288 Logger
.Error('Parser',
289 FORMAT_NOT_SUPPORTED
,
290 ST
.WRN_MULTI_PCD_RANGES
,
291 File
= ContainerFile
,
293 elif ValidValueNum
> 0:
294 Logger
.Error('Parser',
295 FORMAT_NOT_SUPPORTED
,
296 ST
.WRN_MULTI_PCD_VALIDVALUE
,
297 File
= ContainerFile
,
300 PcdErr
= PcdErrorObject()
301 PcdErr
.SetTokenSpaceGuidCName(TokenSpaceGuidCName
)
302 PcdErr
.SetCName(CName
)
303 PcdErr
.SetFileLine(Comment
)
304 PcdErr
.SetLineNum(LineNum
)
306 ValidValueExpr
= Comment
.replace(TAB_PCD_VALIDLIST
, "", 1).strip()
307 Valid
, Cause
= _CheckListExpression(ValidValueExpr
)
309 ValidValue
= Comment
.replace(TAB_PCD_VALIDLIST
, "", 1).replace(TAB_COMMA_SPLIT
, TAB_SPACE_SPLIT
)
310 ValueList
= ValidValue
.split(TAB_VALUE_SPLIT
)
311 if len(ValueList
) > 1:
312 PcdErr
.SetValidValue((TAB_VALUE_SPLIT
.join(ValueList
[1:])).strip())
313 PcdErr
.SetErrorNumber(ParsePcdErrorCode(ValueList
[0], ContainerFile
, LineNum
))
315 PcdErr
.SetValidValue(ValidValue
)
316 PcdErrList
.append(PcdErr
)
318 Logger
.Error("Parser",
319 FORMAT_NOT_SUPPORTED
,
323 elif Comment
.startswith(TAB_PCD_EXPRESSION
):
324 if ValidRangeNum
> 0 or ValidValueNum
> 0:
325 Logger
.Error('Parser',
326 FORMAT_NOT_SUPPORTED
,
327 ST
.WRN_MULTI_PCD_RANGES
,
328 File
= ContainerFile
,
331 PcdErr
= PcdErrorObject()
332 PcdErr
.SetTokenSpaceGuidCName(TokenSpaceGuidCName
)
333 PcdErr
.SetCName(CName
)
334 PcdErr
.SetFileLine(Comment
)
335 PcdErr
.SetLineNum(LineNum
)
337 Expression
= Comment
.replace(TAB_PCD_EXPRESSION
, "", 1).strip()
338 Valid
, Cause
= _CheckExpression(Expression
)
340 ValueList
= Expression
.split(TAB_VALUE_SPLIT
)
341 if len(ValueList
) > 1:
342 PcdErr
.SetExpression((TAB_VALUE_SPLIT
.join(ValueList
[1:])).strip())
343 PcdErr
.SetErrorNumber(ParsePcdErrorCode(ValueList
[0], ContainerFile
, LineNum
))
345 PcdErr
.SetExpression(Expression
)
346 PcdErrList
.append(PcdErr
)
348 Logger
.Error("Parser",
349 FORMAT_NOT_SUPPORTED
,
353 elif Comment
.startswith(TAB_PCD_PROMPT
):
355 Logger
.Error('Parser',
356 FORMAT_NOT_SUPPORTED
,
357 ST
.WRN_MULTI_PCD_PROMPT
,
358 File
= ContainerFile
,
360 PromptStr
= Comment
.replace(TAB_PCD_PROMPT
, "", 1).strip()
363 HelpStr
+= Comment
+ '\n'
366 # remove the last EOL if the comment is of format 'FOO\n'
368 if HelpStr
.endswith('\n'):
369 if HelpStr
!= '\n' and not HelpStr
.endswith('\n\n'):
370 HelpStr
= HelpStr
[:-1]
372 return HelpStr
, PcdErrList
, PromptStr
374 ## ParseDecPcdTailComment
376 # @param TailCommentList: Tail comment list of Pcd, item of format (Comment, LineNum)
377 # @param ContainerFile: Input value for filename of Dec file
378 # @retVal SupModuleList: The supported module type list detected
379 # @retVal HelpStr: The generic help text string detected
381 def ParseDecPcdTailComment (TailCommentList
, ContainerFile
):
382 assert(len(TailCommentList
) == 1)
383 TailComment
= TailCommentList
[0][0]
384 LineNum
= TailCommentList
[0][1]
386 Comment
= TailComment
.lstrip(" #")
388 ReFindFirstWordRe
= re
.compile(r
"""^([^ #]*)""", re
.DOTALL
)
391 # get first word and compare with SUP_MODULE_LIST
393 MatchObject
= ReFindFirstWordRe
.match(Comment
)
394 if not (MatchObject
and MatchObject
.group(1) in SUP_MODULE_LIST
):
398 # parse line, it must have supported module type specified
400 if Comment
.find(TAB_COMMENT_SPLIT
) == -1:
401 Comment
+= TAB_COMMENT_SPLIT
402 SupMode
, HelpStr
= GetSplitValueList(Comment
, TAB_COMMENT_SPLIT
, 1)
404 for Mod
in GetSplitValueList(SupMode
, TAB_SPACE_SPLIT
):
407 elif Mod
not in SUP_MODULE_LIST
:
410 ST
.WRN_INVALID_MODULE_TYPE
%Mod
,
414 SupModuleList
.append(Mod
)
416 return SupModuleList
, HelpStr
418 ## _CheckListExpression
420 # @param Expression: Pcd value list expression
422 def _CheckListExpression(Expression
):
424 if TAB_VALUE_SPLIT
in Expression
:
425 ListExpr
= Expression
[Expression
.find(TAB_VALUE_SPLIT
)+1:]
427 ListExpr
= Expression
429 return IsValidListExpr(ListExpr
)
433 # @param Expression: Pcd value expression
435 def _CheckExpression(Expression
):
437 if TAB_VALUE_SPLIT
in Expression
:
438 Expr
= Expression
[Expression
.find(TAB_VALUE_SPLIT
)+1:]
441 return IsValidLogicalExpr(Expr
, True)
443 ## _CheckRangeExpression
445 # @param Expression: Pcd range expression
447 def _CheckRangeExpression(Expression
):
449 if TAB_VALUE_SPLIT
in Expression
:
450 RangeExpr
= Expression
[Expression
.find(TAB_VALUE_SPLIT
)+1:]
452 RangeExpr
= Expression
454 return IsValidRangeExpr(RangeExpr
)
460 def ValidateCopyright(Result
, ErrType
, FileName
, LineNo
, ErrMsg
):
462 Logger
.Warn("\nUPT", ErrType
, FileName
, LineNo
, ErrMsg
)
464 ## _ValidateCopyright
466 # @param Line: Line that contains copyright information, # stripped
468 # @retval Result: True if line is conformed to Spec format, False else
469 # @retval ErrMsg: the detailed error description
471 def _ValidateCopyright(Line
):
477 return Result
, ErrMsg
479 def GenerateTokenList (Comment
):
481 # Tokenize Comment using '#' and ' ' as token seperators
483 RelplacedComment
= None
484 while Comment
!= RelplacedComment
:
485 RelplacedComment
= Comment
486 Comment
= Comment
.replace('##', '#').replace(' ', ' ').replace(' ', '#').strip('# ')
487 return Comment
.split('#')
491 # Comment - Comment to parse
492 # TypeTokens - A dictionary of type token synonyms
493 # RemoveTokens - A list of tokens to remove from help text
494 # ParseVariable - True for parsing [Guids]. Otherwise False
496 def ParseComment (Comment
, UsageTokens
, TypeTokens
, RemoveTokens
, ParseVariable
):
498 # Initialize return values
509 # Remove white space around first instance of ':' from Comment if 'Variable'
510 # is in front of ':' and Variable is the 1st or 2nd token in Comment.
512 List
= Comment
.split(':', 1)
514 SubList
= GenerateTokenList (List
[0].strip())
515 if len(SubList
) in [1, 2] and SubList
[-1] == 'Variable':
516 if List
[1].strip().find('L"') == 0:
517 Comment
= List
[0].strip() + ':' + List
[1].strip()
520 # Remove first instance of L"<VariableName> from Comment and put into String
521 # if and only if L"<VariableName>" is the 1st token, the 2nd token. Or
522 # L"<VariableName>" is the third token immediately following 'Variable:'.
525 Start
= Comment
.find('Variable:L"')
527 String
= Comment
[Start
+ 9:]
528 End
= String
[2:].find('"')
530 Start
= Comment
.find('L"')
532 String
= Comment
[Start
:]
533 End
= String
[2:].find('"')
535 SubList
= GenerateTokenList (Comment
[:Start
])
537 Comment
= Comment
[:Start
] + String
[End
+ 3:]
538 String
= String
[:End
+ 3]
543 # Initialze HelpText to Comment.
544 # Content will be remove from HelpText as matching tokens are found
549 # Tokenize Comment using '#' and ' ' as token seperators
551 List
= GenerateTokenList (Comment
)
554 # Search first two tokens for Usage and Type and remove any matching tokens
557 for Token
in List
[0:NumTokens
]:
558 if Usage
is None and Token
in UsageTokens
:
559 Usage
= UsageTokens
[Token
]
560 HelpText
= HelpText
.replace(Token
, '')
561 if Usage
is not None or not ParseVariable
:
562 for Token
in List
[0:NumTokens
]:
563 if Type
is None and Token
in TypeTokens
:
564 Type
= TypeTokens
[Token
]
565 HelpText
= HelpText
.replace(Token
, '')
566 if Usage
is not None:
567 for Token
in List
[0:NumTokens
]:
568 if Token
in RemoveTokens
:
569 HelpText
= HelpText
.replace(Token
, '')
572 # If no Usage token is present and set Usage to UNDEFINED
578 # If no Type token is present and set Type to UNDEFINED
584 # If Type is not 'Variable:', then set String to None
586 if Type
!= 'Variable':
590 # Strip ' ' and '#' from the beginning of HelpText
591 # If HelpText is an empty string after all parsing is
592 # complete then set HelpText to None
594 HelpText
= HelpText
.lstrip('# ')
599 # Return parsing results
601 return Usage
, Type
, String
, HelpText