]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/UPT/Library/CommentParsing.py
BaseTools: Clean up source files
[mirror_edk2.git] / BaseTools / Source / Python / UPT / Library / CommentParsing.py
1 ## @file
2 # This file is used to define comment parsing interface
3 #
4 # Copyright (c) 2011 - 2018, Intel Corporation. All rights reserved.<BR>
5 #
6 # This program and the accompanying materials are licensed and made available
7 # under the terms and conditions of the BSD License which accompanies this
8 # distribution. The full text of the license may be found at
9 # http://opensource.org/licenses/bsd-license.php
10 #
11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
13 #
14
15 '''
16 CommentParsing
17 '''
18
19 ##
20 # Import Modules
21 #
22 import re
23
24 from Library.StringUtils import GetSplitValueList
25 from Library.StringUtils import CleanString2
26 from Library.DataType import HEADER_COMMENT_NOT_STARTED
27 from Library.DataType import TAB_COMMENT_SPLIT
28 from Library.DataType import HEADER_COMMENT_LICENSE
29 from Library.DataType import HEADER_COMMENT_ABSTRACT
30 from Library.DataType import HEADER_COMMENT_COPYRIGHT
31 from Library.DataType import HEADER_COMMENT_DESCRIPTION
32 from Library.DataType import TAB_SPACE_SPLIT
33 from Library.DataType import TAB_COMMA_SPLIT
34 from Library.DataType import SUP_MODULE_LIST
35 from Library.DataType import TAB_VALUE_SPLIT
36 from Library.DataType import TAB_PCD_VALIDRANGE
37 from Library.DataType import TAB_PCD_VALIDLIST
38 from Library.DataType import TAB_PCD_EXPRESSION
39 from Library.DataType import TAB_PCD_PROMPT
40 from Library.DataType import TAB_CAPHEX_START
41 from Library.DataType import TAB_HEX_START
42 from Library.DataType import PCD_ERR_CODE_MAX_SIZE
43 from Library.ExpressionValidate import IsValidRangeExpr
44 from Library.ExpressionValidate import IsValidListExpr
45 from Library.ExpressionValidate import IsValidLogicalExpr
46 from Object.POM.CommonObject import TextObject
47 from Object.POM.CommonObject import PcdErrorObject
48 import Logger.Log as Logger
49 from Logger.ToolError import FORMAT_INVALID
50 from Logger.ToolError import FORMAT_NOT_SUPPORTED
51 from Logger import StringTable as ST
52
53 ## ParseHeaderCommentSection
54 #
55 # Parse Header comment section lines, extract Abstract, Description, Copyright
56 # , License lines
57 #
58 # @param CommentList: List of (Comment, LineNumber)
59 # @param FileName: FileName of the comment
60 #
61 def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False):
62 Abstract = ''
63 Description = ''
64 Copyright = ''
65 License = ''
66 EndOfLine = "\n"
67 if IsBinaryHeader:
68 STR_HEADER_COMMENT_START = "@BinaryHeader"
69 else:
70 STR_HEADER_COMMENT_START = "@file"
71 HeaderCommentStage = HEADER_COMMENT_NOT_STARTED
72
73 #
74 # first find the last copyright line
75 #
76 Last = 0
77 for Index in xrange(len(CommentList)-1, 0, -1):
78 Line = CommentList[Index][0]
79 if _IsCopyrightLine(Line):
80 Last = Index
81 break
82
83 for Item in CommentList:
84 Line = Item[0]
85 LineNo = Item[1]
86
87 if not Line.startswith(TAB_COMMENT_SPLIT) and Line:
88 Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1])
89 Comment = CleanString2(Line)[1]
90 Comment = Comment.strip()
91 #
92 # if there are blank lines between License or Description, keep them as they would be
93 # indication of different block; or in the position that Abstract should be, also keep it
94 # as it indicates that no abstract
95 #
96 if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \
97 HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:
98 continue
99
100 if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:
101 if Comment.startswith(STR_HEADER_COMMENT_START):
102 HeaderCommentStage = HEADER_COMMENT_ABSTRACT
103 else:
104 License += Comment + EndOfLine
105 else:
106 if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:
107 #
108 # in case there is no abstract and description
109 #
110 if not Comment:
111 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
112 elif _IsCopyrightLine(Comment):
113 Result, ErrMsg = _ValidateCopyright(Comment)
114 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
115 Copyright += Comment + EndOfLine
116 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
117 else:
118 Abstract += Comment + EndOfLine
119 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION
120 elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:
121 #
122 # in case there is no description
123 #
124 if _IsCopyrightLine(Comment):
125 Result, ErrMsg = _ValidateCopyright(Comment)
126 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
127 Copyright += Comment + EndOfLine
128 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT
129 else:
130 Description += Comment + EndOfLine
131 elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:
132 if _IsCopyrightLine(Comment):
133 Result, ErrMsg = _ValidateCopyright(Comment)
134 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg)
135 Copyright += Comment + EndOfLine
136 else:
137 #
138 # Contents after copyright line are license, those non-copyright lines in between
139 # copyright line will be discarded
140 #
141 if LineNo > Last:
142 if License:
143 License += EndOfLine
144 License += Comment + EndOfLine
145 HeaderCommentStage = HEADER_COMMENT_LICENSE
146 else:
147 if not Comment and not License:
148 continue
149 License += Comment + EndOfLine
150
151 return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()
152
153 ## _IsCopyrightLine
154 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright"
155 # followed by zero or more white space characters followed by a "(" character
156 #
157 # @param LineContent: the line need to be checked
158 # @return: True if current line is copyright line, False else
159 #
160 def _IsCopyrightLine (LineContent):
161 LineContent = LineContent.upper()
162 Result = False
163
164 ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)
165 if ReIsCopyrightRe.search(LineContent):
166 Result = True
167
168 return Result
169
170 ## ParseGenericComment
171 #
172 # @param GenericComment: Generic comment list, element of
173 # (CommentLine, LineNum)
174 # @param ContainerFile: Input value for filename of Dec file
175 #
176 def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None):
177 if ContainerFile:
178 pass
179 HelpTxt = None
180 HelpStr = ''
181
182 for Item in GenericComment:
183 CommentLine = Item[0]
184 Comment = CleanString2(CommentLine)[1]
185 if SkipTag is not None and Comment.startswith(SkipTag):
186 Comment = Comment.replace(SkipTag, '', 1)
187 HelpStr += Comment + '\n'
188
189 if HelpStr:
190 HelpTxt = TextObject()
191 if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n':
192 HelpStr = HelpStr[:-1]
193 HelpTxt.SetString(HelpStr)
194
195 return HelpTxt
196
197 ## ParsePcdErrorCode
198 #
199 # @param Value: original ErrorCode value
200 # @param ContainerFile: Input value for filename of Dec file
201 # @param LineNum: Line Num
202 #
203 def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None):
204 try:
205 if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)):
206 Base = 16
207 else:
208 Base = 10
209 ErrorCode = long(Value, Base)
210 if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0:
211 Logger.Error('Parser',
212 FORMAT_NOT_SUPPORTED,
213 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
214 File = ContainerFile,
215 Line = LineNum)
216 #
217 # To delete the tailing 'L'
218 #
219 return hex(ErrorCode)[:-1]
220 except ValueError as XStr:
221 if XStr:
222 pass
223 Logger.Error('Parser',
224 FORMAT_NOT_SUPPORTED,
225 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value,
226 File = ContainerFile,
227 Line = LineNum)
228
229 ## ParseDecPcdGenericComment
230 #
231 # @param GenericComment: Generic comment list, element of (CommentLine,
232 # LineNum)
233 # @param ContainerFile: Input value for filename of Dec file
234 #
235 def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict):
236 HelpStr = ''
237 PromptStr = ''
238 PcdErr = None
239 PcdErrList = []
240 ValidValueNum = 0
241 ValidRangeNum = 0
242 ExpressionNum = 0
243
244 for (CommentLine, LineNum) in GenericComment:
245 Comment = CleanString2(CommentLine)[1]
246 #
247 # To replace Macro
248 #
249 MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)'
250 MatchedStrs = re.findall(MACRO_PATTERN, Comment)
251 for MatchedStr in MatchedStrs:
252 if MatchedStr:
253 Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip()
254 if Macro in MacroReplaceDict:
255 Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro])
256 if Comment.startswith(TAB_PCD_VALIDRANGE):
257 if ValidValueNum > 0 or ExpressionNum > 0:
258 Logger.Error('Parser',
259 FORMAT_NOT_SUPPORTED,
260 ST.WRN_MULTI_PCD_RANGES,
261 File = ContainerFile,
262 Line = LineNum)
263 else:
264 PcdErr = PcdErrorObject()
265 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
266 PcdErr.SetCName(CName)
267 PcdErr.SetFileLine(Comment)
268 PcdErr.SetLineNum(LineNum)
269 ValidRangeNum += 1
270 ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip()
271 Valid, Cause = _CheckRangeExpression(ValidRange)
272 if Valid:
273 ValueList = ValidRange.split(TAB_VALUE_SPLIT)
274 if len(ValueList) > 1:
275 PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
276 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
277 else:
278 PcdErr.SetValidValueRange(ValidRange)
279 PcdErrList.append(PcdErr)
280 else:
281 Logger.Error("Parser",
282 FORMAT_NOT_SUPPORTED,
283 Cause,
284 ContainerFile,
285 LineNum)
286 elif Comment.startswith(TAB_PCD_VALIDLIST):
287 if ValidRangeNum > 0 or ExpressionNum > 0:
288 Logger.Error('Parser',
289 FORMAT_NOT_SUPPORTED,
290 ST.WRN_MULTI_PCD_RANGES,
291 File = ContainerFile,
292 Line = LineNum)
293 elif ValidValueNum > 0:
294 Logger.Error('Parser',
295 FORMAT_NOT_SUPPORTED,
296 ST.WRN_MULTI_PCD_VALIDVALUE,
297 File = ContainerFile,
298 Line = LineNum)
299 else:
300 PcdErr = PcdErrorObject()
301 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
302 PcdErr.SetCName(CName)
303 PcdErr.SetFileLine(Comment)
304 PcdErr.SetLineNum(LineNum)
305 ValidValueNum += 1
306 ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip()
307 Valid, Cause = _CheckListExpression(ValidValueExpr)
308 if Valid:
309 ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT)
310 ValueList = ValidValue.split(TAB_VALUE_SPLIT)
311 if len(ValueList) > 1:
312 PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
313 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
314 else:
315 PcdErr.SetValidValue(ValidValue)
316 PcdErrList.append(PcdErr)
317 else:
318 Logger.Error("Parser",
319 FORMAT_NOT_SUPPORTED,
320 Cause,
321 ContainerFile,
322 LineNum)
323 elif Comment.startswith(TAB_PCD_EXPRESSION):
324 if ValidRangeNum > 0 or ValidValueNum > 0:
325 Logger.Error('Parser',
326 FORMAT_NOT_SUPPORTED,
327 ST.WRN_MULTI_PCD_RANGES,
328 File = ContainerFile,
329 Line = LineNum)
330 else:
331 PcdErr = PcdErrorObject()
332 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName)
333 PcdErr.SetCName(CName)
334 PcdErr.SetFileLine(Comment)
335 PcdErr.SetLineNum(LineNum)
336 ExpressionNum += 1
337 Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip()
338 Valid, Cause = _CheckExpression(Expression)
339 if Valid:
340 ValueList = Expression.split(TAB_VALUE_SPLIT)
341 if len(ValueList) > 1:
342 PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip())
343 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum))
344 else:
345 PcdErr.SetExpression(Expression)
346 PcdErrList.append(PcdErr)
347 else:
348 Logger.Error("Parser",
349 FORMAT_NOT_SUPPORTED,
350 Cause,
351 ContainerFile,
352 LineNum)
353 elif Comment.startswith(TAB_PCD_PROMPT):
354 if PromptStr:
355 Logger.Error('Parser',
356 FORMAT_NOT_SUPPORTED,
357 ST.WRN_MULTI_PCD_PROMPT,
358 File = ContainerFile,
359 Line = LineNum)
360 PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip()
361 else:
362 if Comment:
363 HelpStr += Comment + '\n'
364
365 #
366 # remove the last EOL if the comment is of format 'FOO\n'
367 #
368 if HelpStr.endswith('\n'):
369 if HelpStr != '\n' and not HelpStr.endswith('\n\n'):
370 HelpStr = HelpStr[:-1]
371
372 return HelpStr, PcdErrList, PromptStr
373
374 ## ParseDecPcdTailComment
375 #
376 # @param TailCommentList: Tail comment list of Pcd, item of format (Comment, LineNum)
377 # @param ContainerFile: Input value for filename of Dec file
378 # @retVal SupModuleList: The supported module type list detected
379 # @retVal HelpStr: The generic help text string detected
380 #
381 def ParseDecPcdTailComment (TailCommentList, ContainerFile):
382 assert(len(TailCommentList) == 1)
383 TailComment = TailCommentList[0][0]
384 LineNum = TailCommentList[0][1]
385
386 Comment = TailComment.lstrip(" #")
387
388 ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL)
389
390 #
391 # get first word and compare with SUP_MODULE_LIST
392 #
393 MatchObject = ReFindFirstWordRe.match(Comment)
394 if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST):
395 return None, Comment
396
397 #
398 # parse line, it must have supported module type specified
399 #
400 if Comment.find(TAB_COMMENT_SPLIT) == -1:
401 Comment += TAB_COMMENT_SPLIT
402 SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1)
403 SupModuleList = []
404 for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT):
405 if not Mod:
406 continue
407 elif Mod not in SUP_MODULE_LIST:
408 Logger.Error("UPT",
409 FORMAT_INVALID,
410 ST.WRN_INVALID_MODULE_TYPE%Mod,
411 ContainerFile,
412 LineNum)
413 else:
414 SupModuleList.append(Mod)
415
416 return SupModuleList, HelpStr
417
418 ## _CheckListExpression
419 #
420 # @param Expression: Pcd value list expression
421 #
422 def _CheckListExpression(Expression):
423 ListExpr = ''
424 if TAB_VALUE_SPLIT in Expression:
425 ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
426 else:
427 ListExpr = Expression
428
429 return IsValidListExpr(ListExpr)
430
431 ## _CheckExpreesion
432 #
433 # @param Expression: Pcd value expression
434 #
435 def _CheckExpression(Expression):
436 Expr = ''
437 if TAB_VALUE_SPLIT in Expression:
438 Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
439 else:
440 Expr = Expression
441 return IsValidLogicalExpr(Expr, True)
442
443 ## _CheckRangeExpression
444 #
445 # @param Expression: Pcd range expression
446 #
447 def _CheckRangeExpression(Expression):
448 RangeExpr = ''
449 if TAB_VALUE_SPLIT in Expression:
450 RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:]
451 else:
452 RangeExpr = Expression
453
454 return IsValidRangeExpr(RangeExpr)
455
456 ## ValidateCopyright
457 #
458 #
459 #
460 def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg):
461 if not Result:
462 Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg)
463
464 ## _ValidateCopyright
465 #
466 # @param Line: Line that contains copyright information, # stripped
467 #
468 # @retval Result: True if line is conformed to Spec format, False else
469 # @retval ErrMsg: the detailed error description
470 #
471 def _ValidateCopyright(Line):
472 if Line:
473 pass
474 Result = True
475 ErrMsg = ''
476
477 return Result, ErrMsg
478
479 def GenerateTokenList (Comment):
480 #
481 # Tokenize Comment using '#' and ' ' as token seperators
482 #
483 RelplacedComment = None
484 while Comment != RelplacedComment:
485 RelplacedComment = Comment
486 Comment = Comment.replace('##', '#').replace(' ', ' ').replace(' ', '#').strip('# ')
487 return Comment.split('#')
488
489
490 #
491 # Comment - Comment to parse
492 # TypeTokens - A dictionary of type token synonyms
493 # RemoveTokens - A list of tokens to remove from help text
494 # ParseVariable - True for parsing [Guids]. Otherwise False
495 #
496 def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable):
497 #
498 # Initialize return values
499 #
500 Usage = None
501 Type = None
502 String = None
503
504 Comment = Comment[0]
505
506 NumTokens = 2
507 if ParseVariable:
508 #
509 # Remove white space around first instance of ':' from Comment if 'Variable'
510 # is in front of ':' and Variable is the 1st or 2nd token in Comment.
511 #
512 List = Comment.split(':', 1)
513 if len(List) > 1:
514 SubList = GenerateTokenList (List[0].strip())
515 if len(SubList) in [1, 2] and SubList[-1] == 'Variable':
516 if List[1].strip().find('L"') == 0:
517 Comment = List[0].strip() + ':' + List[1].strip()
518
519 #
520 # Remove first instance of L"<VariableName> from Comment and put into String
521 # if and only if L"<VariableName>" is the 1st token, the 2nd token. Or
522 # L"<VariableName>" is the third token immediately following 'Variable:'.
523 #
524 End = -1
525 Start = Comment.find('Variable:L"')
526 if Start >= 0:
527 String = Comment[Start + 9:]
528 End = String[2:].find('"')
529 else:
530 Start = Comment.find('L"')
531 if Start >= 0:
532 String = Comment[Start:]
533 End = String[2:].find('"')
534 if End >= 0:
535 SubList = GenerateTokenList (Comment[:Start])
536 if len(SubList) < 2:
537 Comment = Comment[:Start] + String[End + 3:]
538 String = String[:End + 3]
539 Type = 'Variable'
540 NumTokens = 1
541
542 #
543 # Initialze HelpText to Comment.
544 # Content will be remove from HelpText as matching tokens are found
545 #
546 HelpText = Comment
547
548 #
549 # Tokenize Comment using '#' and ' ' as token seperators
550 #
551 List = GenerateTokenList (Comment)
552
553 #
554 # Search first two tokens for Usage and Type and remove any matching tokens
555 # from HelpText
556 #
557 for Token in List[0:NumTokens]:
558 if Usage is None and Token in UsageTokens:
559 Usage = UsageTokens[Token]
560 HelpText = HelpText.replace(Token, '')
561 if Usage is not None or not ParseVariable:
562 for Token in List[0:NumTokens]:
563 if Type is None and Token in TypeTokens:
564 Type = TypeTokens[Token]
565 HelpText = HelpText.replace(Token, '')
566 if Usage is not None:
567 for Token in List[0:NumTokens]:
568 if Token in RemoveTokens:
569 HelpText = HelpText.replace(Token, '')
570
571 #
572 # If no Usage token is present and set Usage to UNDEFINED
573 #
574 if Usage is None:
575 Usage = 'UNDEFINED'
576
577 #
578 # If no Type token is present and set Type to UNDEFINED
579 #
580 if Type is None:
581 Type = 'UNDEFINED'
582
583 #
584 # If Type is not 'Variable:', then set String to None
585 #
586 if Type != 'Variable':
587 String = None
588
589 #
590 # Strip ' ' and '#' from the beginning of HelpText
591 # If HelpText is an empty string after all parsing is
592 # complete then set HelpText to None
593 #
594 HelpText = HelpText.lstrip('# ')
595 if HelpText == '':
596 HelpText = None
597
598 #
599 # Return parsing results
600 #
601 return Usage, Type, String, HelpText