]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/UPT/Library/CommentParsing.py
UefiCpuPkg: Move AsmRelocateApLoopStart from Mpfuncs.nasm to AmdSev.nasm
[mirror_edk2.git] / BaseTools / Source / Python / UPT / Library / CommentParsing.py
1 ## @file
2 # This file is used to define comment parsing interface
3 #
4 # Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
5 #
6 # SPDX-License-Identifier: BSD-2-Clause-Patent
7 #
8
9 '''
10 CommentParsing
11 '''
12
13 ##
14 # Import Modules
15 #
16 import re
17
18 from Library.StringUtils import GetSplitValueList
19 from Library.StringUtils import CleanString2
20 from Library.DataType import HEADER_COMMENT_NOT_STARTED
21 from Library.DataType import TAB_COMMENT_SPLIT
22 from Library.DataType import HEADER_COMMENT_LICENSE
23 from Library.DataType import HEADER_COMMENT_ABSTRACT
24 from Library.DataType import HEADER_COMMENT_COPYRIGHT
25 from Library.DataType import HEADER_COMMENT_DESCRIPTION
26 from Library.DataType import TAB_SPACE_SPLIT
27 from Library.DataType import TAB_COMMA_SPLIT
28 from Library.DataType import SUP_MODULE_LIST
29 from Library.DataType import TAB_VALUE_SPLIT
30 from Library.DataType import TAB_PCD_VALIDRANGE
31 from Library.DataType import TAB_PCD_VALIDLIST
32 from Library.DataType import TAB_PCD_EXPRESSION
33 from Library.DataType import TAB_PCD_PROMPT
34 from Library.DataType import TAB_CAPHEX_START
35 from Library.DataType import TAB_HEX_START
36 from Library.DataType import PCD_ERR_CODE_MAX_SIZE
37 from Library.ExpressionValidate import IsValidRangeExpr
38 from Library.ExpressionValidate import IsValidListExpr
39 from Library.ExpressionValidate import IsValidLogicalExpr
40 from Object.POM.CommonObject import TextObject
41 from Object.POM.CommonObject import PcdErrorObject
42 import Logger.Log as Logger
43 from Logger.ToolError import FORMAT_INVALID
44 from Logger.ToolError import FORMAT_NOT_SUPPORTED
45 from Logger import StringTable as ST
46
47 ## ParseHeaderCommentSection
48 #
49 # Parse Header comment section lines, extract Abstract, Description, Copyright
50 # , License lines
51 #
52 # @param CommentList: List of (Comment, LineNumber)
53 # @param FileName: FileName of the comment
54 #
55 def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False):
56 Abstract = ''
57 Description = ''
58 Copyright = ''
59 License = ''
60 EndOfLine = "\n"
61 if IsBinaryHeader:
62 STR_HEADER_COMMENT_START = "@BinaryHeader"
63 else:
64 STR_HEADER_COMMENT_START = "@file"
65 HeaderCommentStage = HEADER_COMMENT_NOT_STARTED
66
67 #
68 # first find the last copyright line
69 #
70 Last = 0
71 for Index in range(len(CommentList)-1, 0, -1):
72 Line = CommentList[Index][0]
73 if _IsCopyrightLine(Line):
74 Last = Index
75 break
76
77 for Item in CommentList:
78 Line = Item[0]
79 LineNo = Item[1]
80
81 if not Line.startswith(TAB_COMMENT_SPLIT) and Line:
82 Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])
83 Comment = CleanString2(Line)[1]
84 Comment = Comment.strip()
85 #
86 # if there are blank lines between License or Description, keep them as they would be
87 # indication of different block; or in the position that Abstract should be, also keep it
88 # as it indicates that no abstract
89 #
90 if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \
91 HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:
92 continue
93
94 if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:
95 if Comment.startswith(STR_HEADER_COMMENT_START):
96 HeaderCommentStage = HEADER_COMMENT_ABSTRACT
97 else:
98 License += Comment + EndOfLine
99 else:
100 if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:
101 #
102 # in case there is no abstract and description
103 #
104 if not Comment:
105 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
106 elif _IsCopyrightLine(Comment):
107 Result, ErrMsg = _ValidateCopyright(Comment)
108 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
109 Copyright += Comment + EndOfLine
110 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
111 else:
112 Abstract += Comment + EndOfLine
113 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
114 elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:
115 #
116 # in case there is no description
117 #
118 if _IsCopyrightLine(Comment):
119 Result, ErrMsg = _ValidateCopyright(Comment)
120 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
121 Copyright += Comment + EndOfLine
122 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
123 else:
124 Description += Comment + EndOfLine
125 elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:
126 if _IsCopyrightLine(Comment):
127 Result, ErrMsg = _ValidateCopyright(Comment)
128 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
129 Copyright += Comment + EndOfLine
130 else:
131 #
132 # Contents after copyright line are license, those non-copyright lines in between
133 # copyright line will be discarded
134 #
135 if LineNo > Last:
136 if License:
137 License += EndOfLine
138 License += Comment + EndOfLine
139 HeaderCommentStage = HEADER_COMMENT_LICENSE
140 else:
141 if not Comment and not License:
142 continue
143 License += Comment + EndOfLine
144
145 return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()
146
147 ## _IsCopyrightLine
148 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
149 # followed by zero or more white space characters followed by a "(" character
150 #
151 # @param LineContent: the line need to be checked
152 # @return: True if current line is copyright line, False else
153 #
154 def _IsCopyrightLine (LineContent):
155 LineContent = LineContent.upper()
156 Result = False
157
158 ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)
159 if ReIsCopyrightRe.search(LineContent):
160 Result = True
161
162 return Result
163
164 ## ParseGenericComment
165 #
166 # @param GenericComment: Generic comment list, element of
167 # (CommentLine, LineNum)
168 # @param ContainerFile: Input value for filename of Dec file
169 #
170 def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):
171 if ContainerFile:
172 pass
173 HelpTxt = None
174 HelpStr = ''
175
176 for Item in GenericComment:
177 CommentLine = Item[0]
178 Comment = CleanString2(CommentLine)[1]
179 if SkipTag is not None and Comment.startswith(SkipTag):
180 Comment = Comment.replace(SkipTag, '', 1)
181 HelpStr += Comment + '\n'
182
183 if HelpStr:
184 HelpTxt = TextObject()
185 if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':
186 HelpStr = HelpStr[:-1]
187 HelpTxt.SetString(HelpStr)
188
189 return HelpTxt
190
191 ## ParsePcdErrorCode
192 #
193 # @param Value: original ErrorCode value
194 # @param ContainerFile: Input value for filename of Dec file
195 # @param LineNum: Line Num
196 #
197 def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None):
198 try:
199 if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)):
200 Base = 16
201 else:
202 Base = 10
203 ErrorCode = int(Value, Base)
204 if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0:
205 Logger.Error('Parser',
206 FORMAT_NOT_SUPPORTED,
207 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
208 File = ContainerFile,
209 Line = LineNum)
210 ErrorCode = '0x%x' % ErrorCode
211 return ErrorCode
212 except ValueError as XStr:
213 if XStr:
214 pass
215 Logger.Error('Parser',
216 FORMAT_NOT_SUPPORTED,
217 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
218 File = ContainerFile,
219 Line = LineNum)
220
221 ## ParseDecPcdGenericComment
222 #
223 # @param GenericComment: Generic comment list, element of (CommentLine,
224 # LineNum)
225 # @param ContainerFile: Input value for filename of Dec file
226 #
227 def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict):
228 HelpStr = ''
229 PromptStr = ''
230 PcdErr = None
231 PcdErrList = []
232 ValidValueNum = 0
233 ValidRangeNum = 0
234 ExpressionNum = 0
235
236 for (CommentLine, LineNum) in GenericComment:
237 Comment = CleanString2(CommentLine)[1]
238 #
239 # To replace Macro
240 #
241 MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'
242 MatchedStrs = re.findall(MACRO_PATTERN, Comment)
243 for MatchedStr in MatchedStrs:
244 if MatchedStr:
245 Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip()
246 if Macro in MacroReplaceDict:
247 Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro])
248 if Comment.startswith(TAB_PCD_VALIDRANGE):
249 if ValidValueNum > 0 or ExpressionNum > 0:
250 Logger.Error('Parser',
251 FORMAT_NOT_SUPPORTED,
252 ST.WRN_MULTI_PCD_RANGES,
253 File = ContainerFile,
254 Line = LineNum)
255 else:
256 PcdErr = PcdErrorObject()
257 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
258 PcdErr.SetCName(CName)
259 PcdErr.SetFileLine(Comment)
260 PcdErr.SetLineNum(LineNum)
261 ValidRangeNum += 1
262 ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip()
263 Valid, Cause = _CheckRangeExpression(ValidRange)
264 if Valid:
265 ValueList = ValidRange.split(TAB_VALUE_SPLIT)
266 if len(ValueList) > 1:
267 PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
268 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
269 else:
270 PcdErr.SetValidValueRange(ValidRange)
271 PcdErrList.append(PcdErr)
272 else:
273 Logger.Error("Parser",
274 FORMAT_NOT_SUPPORTED,
275 Cause,
276 ContainerFile,
277 LineNum)
278 elif Comment.startswith(TAB_PCD_VALIDLIST):
279 if ValidRangeNum > 0 or ExpressionNum > 0:
280 Logger.Error('Parser',
281 FORMAT_NOT_SUPPORTED,
282 ST.WRN_MULTI_PCD_RANGES,
283 File = ContainerFile,
284 Line = LineNum)
285 elif ValidValueNum > 0:
286 Logger.Error('Parser',
287 FORMAT_NOT_SUPPORTED,
288 ST.WRN_MULTI_PCD_VALIDVALUE,
289 File = ContainerFile,
290 Line = LineNum)
291 else:
292 PcdErr = PcdErrorObject()
293 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
294 PcdErr.SetCName(CName)
295 PcdErr.SetFileLine(Comment)
296 PcdErr.SetLineNum(LineNum)
297 ValidValueNum += 1
298 ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip()
299 Valid, Cause = _CheckListExpression(ValidValueExpr)
300 if Valid:
301 ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)
302 ValueList = ValidValue.split(TAB_VALUE_SPLIT)
303 if len(ValueList) > 1:
304 PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
305 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
306 else:
307 PcdErr.SetValidValue(ValidValue)
308 PcdErrList.append(PcdErr)
309 else:
310 Logger.Error("Parser",
311 FORMAT_NOT_SUPPORTED,
312 Cause,
313 ContainerFile,
314 LineNum)
315 elif Comment.startswith(TAB_PCD_EXPRESSION):
316 if ValidRangeNum > 0 or ValidValueNum > 0:
317 Logger.Error('Parser',
318 FORMAT_NOT_SUPPORTED,
319 ST.WRN_MULTI_PCD_RANGES,
320 File = ContainerFile,
321 Line = LineNum)
322 else:
323 PcdErr = PcdErrorObject()
324 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
325 PcdErr.SetCName(CName)
326 PcdErr.SetFileLine(Comment)
327 PcdErr.SetLineNum(LineNum)
328 ExpressionNum += 1
329 Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip()
330 Valid, Cause = _CheckExpression(Expression)
331 if Valid:
332 ValueList = Expression.split(TAB_VALUE_SPLIT)
333 if len(ValueList) > 1:
334 PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
335 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
336 else:
337 PcdErr.SetExpression(Expression)
338 PcdErrList.append(PcdErr)
339 else:
340 Logger.Error("Parser",
341 FORMAT_NOT_SUPPORTED,
342 Cause,
343 ContainerFile,
344 LineNum)
345 elif Comment.startswith(TAB_PCD_PROMPT):
346 if PromptStr:
347 Logger.Error('Parser',
348 FORMAT_NOT_SUPPORTED,
349 ST.WRN_MULTI_PCD_PROMPT,
350 File = ContainerFile,
351 Line = LineNum)
352 PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip()
353 else:
354 if Comment:
355 HelpStr += Comment + '\n'
356
357 #
358 # remove the last EOL if the comment is of format 'FOO\n'
359 #
360 if HelpStr.endswith('\n'):
361 if HelpStr != '\n' and not HelpStr.endswith('\n\n'):
362 HelpStr = HelpStr[:-1]
363
364 return HelpStr, PcdErrList, PromptStr
365
366 ## ParseDecPcdTailComment
367 #
368 # @param TailCommentList: Tail comment list of Pcd, item of format (Comment, LineNum)
369 # @param ContainerFile: Input value for filename of Dec file
370 # @retVal SupModuleList: The supported module type list detected
371 # @retVal HelpStr: The generic help text string detected
372 #
373 def ParseDecPcdTailComment (TailCommentList, ContainerFile):
374 assert(len(TailCommentList) == 1)
375 TailComment = TailCommentList[0][0]
376 LineNum = TailCommentList[0][1]
377
378 Comment = TailComment.lstrip(" #")
379
380 ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)
381
382 #
383 # get first word and compare with SUP_MODULE_LIST
384 #
385 MatchObject = ReFindFirstWordRe.match(Comment)
386 if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):
387 return None, Comment
388
389 #
390 # parse line, it must have supported module type specified
391 #
392 if Comment.find(TAB_COMMENT_SPLIT) == -1:
393 Comment += TAB_COMMENT_SPLIT
394 SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)
395 SupModuleList = []
396 for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):
397 if not Mod:
398 continue
399 elif Mod not in SUP_MODULE_LIST:
400 Logger.Error("UPT",
401 FORMAT_INVALID,
402 ST.WRN_INVALID_MODULE_TYPE%Mod,
403 ContainerFile,
404 LineNum)
405 else:
406 SupModuleList.append(Mod)
407
408 return SupModuleList, HelpStr
409
410 ## _CheckListExpression
411 #
412 # @param Expression: Pcd value list expression
413 #
414 def _CheckListExpression(Expression):
415 ListExpr = ''
416 if TAB_VALUE_SPLIT in Expression:
417 ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
418 else:
419 ListExpr = Expression
420
421 return IsValidListExpr(ListExpr)
422
423 ## _CheckExpression
424 #
425 # @param Expression: Pcd value expression
426 #
427 def _CheckExpression(Expression):
428 Expr = ''
429 if TAB_VALUE_SPLIT in Expression:
430 Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
431 else:
432 Expr = Expression
433 return IsValidLogicalExpr(Expr, True)
434
435 ## _CheckRangeExpression
436 #
437 # @param Expression: Pcd range expression
438 #
439 def _CheckRangeExpression(Expression):
440 RangeExpr = ''
441 if TAB_VALUE_SPLIT in Expression:
442 RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
443 else:
444 RangeExpr = Expression
445
446 return IsValidRangeExpr(RangeExpr)
447
448 ## ValidateCopyright
449 #
450 #
451 #
452 def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):
453 if not Result:
454 Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg)
455
456 ## _ValidateCopyright
457 #
458 # @param Line: Line that contains copyright information, # stripped
459 #
460 # @retval Result: True if line is conformed to Spec format, False else
461 # @retval ErrMsg: the detailed error description
462 #
463 def _ValidateCopyright(Line):
464 if Line:
465 pass
466 Result = True
467 ErrMsg = ''
468
469 return Result, ErrMsg
470
471 def GenerateTokenList (Comment):
472 #
473 # Tokenize Comment using '#' and ' ' as token separators
474 #
475 ReplacedComment = None
476 while Comment != ReplacedComment:
477 ReplacedComment = Comment
478 Comment = Comment.replace('##', '#').replace(' ', ' ').replace(' ', '#').strip('# ')
479 return Comment.split('#')
480
481
482 #
483 # Comment - Comment to parse
484 # TypeTokens - A dictionary of type token synonyms
485 # RemoveTokens - A list of tokens to remove from help text
486 # ParseVariable - True for parsing [Guids]. Otherwise False
487 #
488 def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):
489 #
490 # Initialize return values
491 #
492 Usage = None
493 Type = None
494 String = None
495
496 Comment = Comment[0]
497
498 NumTokens = 2
499 if ParseVariable:
500 #
501 # Remove white space around first instance of ':' from Comment if 'Variable'
502 # is in front of ':' and Variable is the 1st or 2nd token in Comment.
503 #
504 List = Comment.split(':', 1)
505 if len(List) > 1:
506 SubList = GenerateTokenList (List[0].strip())
507 if len(SubList) in [1, 2] and SubList[-1] == 'Variable':
508 if List[1].strip().find('L"') == 0:
509 Comment = List[0].strip() + ':' + List[1].strip()
510
511 #
512 # Remove first instance of L"<VariableName> from Comment and put into String
513 # if and only if L"<VariableName>" is the 1st token, the 2nd token. Or
514 # L"<VariableName>" is the third token immediately following 'Variable:'.
515 #
516 End = -1
517 Start = Comment.find('Variable:L"')
518 if Start >= 0:
519 String = Comment[Start + 9:]
520 End = String[2:].find('"')
521 else:
522 Start = Comment.find('L"')
523 if Start >= 0:
524 String = Comment[Start:]
525 End = String[2:].find('"')
526 if End >= 0:
527 SubList = GenerateTokenList (Comment[:Start])
528 if len(SubList) < 2:
529 Comment = Comment[:Start] + String[End + 3:]
530 String = String[:End + 3]
531 Type = 'Variable'
532 NumTokens = 1
533
534 #
535 # Initialize HelpText to Comment.
536 # Content will be remove from HelpText as matching tokens are found
537 #
538 HelpText = Comment
539
540 #
541 # Tokenize Comment using '#' and ' ' as token separators
542 #
543 List = GenerateTokenList (Comment)
544
545 #
546 # Search first two tokens for Usage and Type and remove any matching tokens
547 # from HelpText
548 #
549 for Token in List[0:NumTokens]:
550 if Usage is None and Token in UsageTokens:
551 Usage = UsageTokens[Token]
552 HelpText = HelpText.replace(Token, '')
553 if Usage is not None or not ParseVariable:
554 for Token in List[0:NumTokens]:
555 if Type is None and Token in TypeTokens:
556 Type = TypeTokens[Token]
557 HelpText = HelpText.replace(Token, '')
558 if Usage is not None:
559 for Token in List[0:NumTokens]:
560 if Token in RemoveTokens:
561 HelpText = HelpText.replace(Token, '')
562
563 #
564 # If no Usage token is present and set Usage to UNDEFINED
565 #
566 if Usage is None:
567 Usage = 'UNDEFINED'
568
569 #
570 # If no Type token is present and set Type to UNDEFINED
571 #
572 if Type is None:
573 Type = 'UNDEFINED'
574
575 #
576 # If Type is not 'Variable:', then set String to None
577 #
578 if Type != 'Variable':
579 String = None
580
581 #
582 # Strip ' ' and '#' from the beginning of HelpText
583 # If HelpText is an empty string after all parsing is
584 # complete then set HelpText to None
585 #
586 HelpText = HelpText.lstrip('# ')
587 if HelpText == '':
588 HelpText = None
589
590 #
591 # Return parsing results
592 #
593 return Usage, Type, String, HelpText