]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/Common/String.py
Check In tool source code based on Build tool project revision r1655.
[mirror_edk2.git] / BaseTools / Source / Python / Common / String.py
1 ## @file
2 # This file is used to define common string related functions used in parsing process
3 #
4 # Copyright (c) 2007 ~ 2008, Intel Corporation
5 # All rights reserved. This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import re
18 import DataType
19 import os.path
20 import string
21 import EdkLogger as EdkLogger
22
23 from GlobalData import *
24 from BuildToolError import *
25
26 ## GetSplitValueList
27 #
28 # Get a value list from a string with multiple values splited with SplitTag
29 # The default SplitTag is DataType.TAB_VALUE_SPLIT
30 # 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC']
31 #
32 # @param String: The input string to be splitted
33 # @param SplitTag: The split key, default is DataType.TAB_VALUE_SPLIT
34 # @param MaxSplit: The max number of split values, default is -1
35 #
36 # @retval list() A list for splitted string
37 #
38 def GetSplitValueList(String, SplitTag = DataType.TAB_VALUE_SPLIT, MaxSplit = -1):
39 return map(lambda l: l.strip(), String.split(SplitTag, MaxSplit))
40
41 ## MergeArches
42 #
43 # Find a key's all arches in dict, add the new arch to the list
44 # If not exist any arch, set the arch directly
45 #
46 # @param Dict: The input value for Dict
47 # @param Key: The input value for Key
48 # @param Arch: The Arch to be added or merged
49 #
50 def MergeArches(Dict, Key, Arch):
51 if Key in Dict.keys():
52 Dict[Key].append(Arch)
53 else:
54 Dict[Key] = Arch.split()
55
56 ## GenDefines
57 #
58 # Parse a string with format "DEFINE <VarName> = <PATH>"
59 # Generate a map Defines[VarName] = PATH
60 # Return False if invalid format
61 #
62 # @param String: String with DEFINE statement
63 # @param Arch: Supportted Arch
64 # @param Defines: DEFINE statement to be parsed
65 #
66 # @retval 0 DEFINE statement found, and valid
67 # @retval 1 DEFINE statement found, but not valid
68 # @retval -1 DEFINE statement not found
69 #
70 def GenDefines(String, Arch, Defines):
71 if String.find(DataType.TAB_DEFINE + ' ') > -1:
72 List = String.replace(DataType.TAB_DEFINE + ' ', '').split(DataType.TAB_EQUAL_SPLIT)
73 if len(List) == 2:
74 Defines[(CleanString(List[0]), Arch)] = CleanString(List[1])
75 return 0
76 else:
77 return -1
78
79 return 1
80
81 ## GenInclude
82 #
83 # Parse a string with format "!include <Filename>"
84 # Return the file path
85 # Return False if invalid format or NOT FOUND
86 #
87 # @param String: String with INCLUDE statement
88 # @param IncludeFiles: INCLUDE statement to be parsed
89 # @param Arch: Supportted Arch
90 #
91 # @retval True
92 # @retval False
93 #
94 def GenInclude(String, IncludeFiles, Arch):
95 if String.upper().find(DataType.TAB_INCLUDE.upper() + ' ') > -1:
96 IncludeFile = CleanString(String[String.upper().find(DataType.TAB_INCLUDE.upper() + ' ') + len(DataType.TAB_INCLUDE + ' ') : ])
97 MergeArches(IncludeFiles, IncludeFile, Arch)
98 return True
99 else:
100 return False
101
102 ## GetLibraryClassesWithModuleType
103 #
104 # Get Library Class definition when no module type defined
105 #
106 # @param Lines: The content to be parsed
107 # @param Key: Reserved
108 # @param KeyValues: To store data after parsing
109 # @param CommentCharacter: Comment char, used to ignore comment content
110 #
111 # @retval True Get library classes successfully
112 #
113 def GetLibraryClassesWithModuleType(Lines, Key, KeyValues, CommentCharacter):
114 newKey = SplitModuleType(Key)
115 Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1]
116 LineList = Lines.splitlines()
117 for Line in LineList:
118 Line = CleanString(Line, CommentCharacter)
119 if Line != '' and Line[0] != CommentCharacter:
120 KeyValues.append([CleanString(Line, CommentCharacter), newKey[1]])
121
122 return True
123
124 ## GetDynamics
125 #
126 # Get Dynamic Pcds
127 #
128 # @param Lines: The content to be parsed
129 # @param Key: Reserved
130 # @param KeyValues: To store data after parsing
131 # @param CommentCharacter: Comment char, used to ignore comment content
132 #
133 # @retval True Get Dynamic Pcds successfully
134 #
135 def GetDynamics(Lines, Key, KeyValues, CommentCharacter):
136 #
137 # Get SkuId Name List
138 #
139 SkuIdNameList = SplitModuleType(Key)
140
141 Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1]
142 LineList = Lines.splitlines()
143 for Line in LineList:
144 Line = CleanString(Line, CommentCharacter)
145 if Line != '' and Line[0] != CommentCharacter:
146 KeyValues.append([CleanString(Line, CommentCharacter), SkuIdNameList[1]])
147
148 return True
149
150 ## SplitModuleType
151 #
152 # Split ModuleType out of section defien to get key
153 # [LibraryClass.Arch.ModuleType|ModuleType|ModuleType] -> [ 'LibraryClass.Arch', ['ModuleType', 'ModuleType', 'ModuleType'] ]
154 #
155 # @param Key: String to be parsed
156 #
157 # @retval ReturnValue A list for module types
158 #
159 def SplitModuleType(Key):
160 KeyList = Key.split(DataType.TAB_SPLIT)
161 #
162 # Fill in for arch
163 #
164 KeyList.append('')
165 #
166 # Fill in for moduletype
167 #
168 KeyList.append('')
169 ReturnValue = []
170 KeyValue = KeyList[0]
171 if KeyList[1] != '':
172 KeyValue = KeyValue + DataType.TAB_SPLIT + KeyList[1]
173 ReturnValue.append(KeyValue)
174 ReturnValue.append(GetSplitValueList(KeyList[2]))
175
176 return ReturnValue
177
178 ## Replace macro in strings list
179 #
180 # This method replace macros used in a given string list. The macros are
181 # given in a dictionary.
182 #
183 # @param StringList StringList to be processed
184 # @param MacroDefinitions The macro definitions in the form of dictionary
185 # @param SelfReplacement To decide whether replace un-defined macro to ''
186 #
187 # @retval NewList A new string list whose macros are replaced
188 #
189 def ReplaceMacros(StringList, MacroDefinitions={}, SelfReplacement = False):
190 NewList = []
191 for String in StringList:
192 if type(String) == type(''):
193 NewList.append(ReplaceMacro(String, MacroDefinitions, SelfReplacement))
194 else:
195 NewList.append(String)
196
197 return NewList
198
199 ## Replace macro in string
200 #
201 # This method replace macros used in given string. The macros are given in a
202 # dictionary.
203 #
204 # @param String String to be processed
205 # @param MacroDefinitions The macro definitions in the form of dictionary
206 # @param SelfReplacement To decide whether replace un-defined macro to ''
207 #
208 # @retval string The string whose macros are replaced
209 #
210 def ReplaceMacro(String, MacroDefinitions={}, SelfReplacement = False):
211 LastString = String
212 while MacroDefinitions:
213 MacroUsed = gMacroPattern.findall(String)
214 # no macro found in String, stop replacing
215 if len(MacroUsed) == 0:
216 break
217
218 for Macro in MacroUsed:
219 if Macro not in MacroDefinitions:
220 if SelfReplacement:
221 String = String.replace("$(%s)" % Macro, '')
222 continue
223 String = String.replace("$(%s)" % Macro, MacroDefinitions[Macro])
224 # in case there's macro not defined
225 if String == LastString:
226 break
227 LastString = String
228
229 return String
230
231 ## NormPath
232 #
233 # Create a normal path
234 # And replace DFEINE in the path
235 #
236 # @param Path: The input value for Path to be converted
237 # @param Defines: A set for DEFINE statement
238 #
239 # @retval Path Formatted path
240 #
241 def NormPath(Path, Defines = {}):
242 IsRelativePath = False
243 if Path:
244 if Path[0] == '.':
245 IsRelativePath = True
246 #
247 # Replace with Define
248 #
249 if Defines:
250 Path = ReplaceMacro(Path, Defines)
251 #
252 # To local path format
253 #
254 Path = os.path.normpath(Path)
255
256 if IsRelativePath and Path[0] != '.':
257 Path = os.path.join('.', Path)
258
259 return Path
260
261 ## CleanString
262 #
263 # Remove comments in a string
264 # Remove spaces
265 #
266 # @param Line: The string to be cleaned
267 # @param CommentCharacter: Comment char, used to ignore comment content, default is DataType.TAB_COMMENT_SPLIT
268 #
269 # @retval Path Formatted path
270 #
271 def CleanString(Line, CommentCharacter = DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False):
272 #
273 # remove whitespace
274 #
275 Line = Line.strip();
276 #
277 # Replace R8's comment character
278 #
279 if AllowCppStyleComment:
280 Line = Line.replace(DataType.TAB_COMMENT_R8_SPLIT, CommentCharacter)
281 #
282 # remove comments
283 #
284 Line = Line.split(CommentCharacter, 1)[0];
285 #
286 # remove whitespace again
287 #
288 Line = Line.strip();
289
290 return Line
291
292 ## GetMultipleValuesOfKeyFromLines
293 #
294 # Parse multiple strings to clean comment and spaces
295 # The result is saved to KeyValues
296 #
297 # @param Lines: The content to be parsed
298 # @param Key: Reserved
299 # @param KeyValues: To store data after parsing
300 # @param CommentCharacter: Comment char, used to ignore comment content
301 #
302 # @retval True Successfully executed
303 #
304 def GetMultipleValuesOfKeyFromLines(Lines, Key, KeyValues, CommentCharacter):
305 Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1]
306 LineList = Lines.split('\n')
307 for Line in LineList:
308 Line = CleanString(Line, CommentCharacter)
309 if Line != '' and Line[0] != CommentCharacter:
310 KeyValues += [Line]
311
312 return True
313
314 ## GetDefineValue
315 #
316 # Parse a DEFINE statement to get defined value
317 # DEFINE Key Value
318 #
319 # @param String: The content to be parsed
320 # @param Key: The key of DEFINE statement
321 # @param CommentCharacter: Comment char, used to ignore comment content
322 #
323 # @retval string The defined value
324 #
325 def GetDefineValue(String, Key, CommentCharacter):
326 String = CleanString(String)
327 return String[String.find(Key + ' ') + len(Key + ' ') : ]
328
329 ## GetSingleValueOfKeyFromLines
330 #
331 # Parse multiple strings as below to get value of each definition line
332 # Key1 = Value1
333 # Key2 = Value2
334 # The result is saved to Dictionary
335 #
336 # @param Lines: The content to be parsed
337 # @param Dictionary: To store data after parsing
338 # @param CommentCharacter: Comment char, be used to ignore comment content
339 # @param KeySplitCharacter: Key split char, between key name and key value. Key1 = Value1, '=' is the key split char
340 # @param ValueSplitFlag: Value split flag, be used to decide if has multiple values
341 # @param ValueSplitCharacter: Value split char, be used to split multiple values. Key1 = Value1|Value2, '|' is the value split char
342 #
343 # @retval True Successfully executed
344 #
345 def GetSingleValueOfKeyFromLines(Lines, Dictionary, CommentCharacter, KeySplitCharacter, ValueSplitFlag, ValueSplitCharacter):
346 Lines = Lines.split('\n')
347 Keys = []
348 Value = ''
349 DefineValues = ['']
350 SpecValues = ['']
351
352 for Line in Lines:
353 #
354 # Handle DEFINE and SPEC
355 #
356 if Line.find(DataType.TAB_INF_DEFINES_DEFINE + ' ') > -1:
357 if '' in DefineValues:
358 DefineValues.remove('')
359 DefineValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_DEFINE, CommentCharacter))
360 continue
361 if Line.find(DataType.TAB_INF_DEFINES_SPEC + ' ') > -1:
362 if '' in SpecValues:
363 SpecValues.remove('')
364 SpecValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_SPEC, CommentCharacter))
365 continue
366
367 #
368 # Handle Others
369 #
370 LineList = Line.split(KeySplitCharacter, 1)
371 if len(LineList) >= 2:
372 Key = LineList[0].split()
373 if len(Key) == 1 and Key[0][0] != CommentCharacter:
374 #
375 # Remove comments and white spaces
376 #
377 LineList[1] = CleanString(LineList[1], CommentCharacter)
378 if ValueSplitFlag:
379 Value = map(string.strip, LineList[1].split(ValueSplitCharacter))
380 else:
381 Value = CleanString(LineList[1], CommentCharacter).splitlines()
382
383 if Key[0] in Dictionary:
384 if Key[0] not in Keys:
385 Dictionary[Key[0]] = Value
386 Keys.append(Key[0])
387 else:
388 Dictionary[Key[0]].extend(Value)
389 else:
390 Dictionary[DataType.TAB_INF_DEFINES_MACRO][Key[0]] = Value[0]
391
392 if DefineValues == []:
393 DefineValues = ['']
394 if SpecValues == []:
395 SpecValues = ['']
396 Dictionary[DataType.TAB_INF_DEFINES_DEFINE] = DefineValues
397 Dictionary[DataType.TAB_INF_DEFINES_SPEC] = SpecValues
398
399 return True
400
401 ## The content to be parsed
402 #
403 # Do pre-check for a file before it is parsed
404 # Check $()
405 # Check []
406 #
407 # @param FileName: Used for error report
408 # @param FileContent: File content to be parsed
409 # @param SupSectionTag: Used for error report
410 #
411 def PreCheck(FileName, FileContent, SupSectionTag):
412 LineNo = 0
413 IsFailed = False
414 NewFileContent = ''
415 for Line in FileContent.splitlines():
416 LineNo = LineNo + 1
417 #
418 # Clean current line
419 #
420 Line = CleanString(Line)
421
422 #
423 # Remove commented line
424 #
425 if Line.find(DataType.TAB_COMMA_SPLIT) == 0:
426 Line = ''
427 #
428 # Check $()
429 #
430 if Line.find('$') > -1:
431 if Line.find('$(') < 0 or Line.find(')') < 0:
432 EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError = EdkLogger.IsRaiseError)
433
434 #
435 # Check []
436 #
437 if Line.find('[') > -1 or Line.find(']') > -1:
438 #
439 # Only get one '[' or one ']'
440 #
441 if not (Line.find('[') > -1 and Line.find(']') > -1):
442 EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError = EdkLogger.IsRaiseError)
443
444 #
445 # Regenerate FileContent
446 #
447 NewFileContent = NewFileContent + Line + '\r\n'
448
449 if IsFailed:
450 EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError = EdkLogger.IsRaiseError)
451
452 return NewFileContent
453
454 ## CheckFileType
455 #
456 # Check if the Filename is including ExtName
457 # Return True if it exists
458 # Raise a error message if it not exists
459 #
460 # @param CheckFilename: Name of the file to be checked
461 # @param ExtName: Ext name of the file to be checked
462 # @param ContainerFilename: The container file which describes the file to be checked, used for error report
463 # @param SectionName: Used for error report
464 # @param Line: The line in container file which defines the file to be checked
465 #
466 # @retval True The file type is correct
467 #
468 def CheckFileType(CheckFilename, ExtName, ContainerFilename, SectionName, Line, LineNo = -1):
469 if CheckFilename != '' and CheckFilename != None:
470 (Root, Ext) = os.path.splitext(CheckFilename)
471 if Ext.upper() != ExtName.upper():
472 ContainerFile = open(ContainerFilename, 'r').read()
473 if LineNo == -1:
474 LineNo = GetLineNo(ContainerFile, Line)
475 ErrorMsg = "Invalid %s. '%s' is found, but '%s' file is needed" % (SectionName, CheckFilename, ExtName)
476 EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, Line=LineNo,
477 File=ContainerFilename, RaiseError = EdkLogger.IsRaiseError)
478
479 return True
480
481 ## CheckFileExist
482 #
483 # Check if the file exists
484 # Return True if it exists
485 # Raise a error message if it not exists
486 #
487 # @param CheckFilename: Name of the file to be checked
488 # @param WorkspaceDir: Current workspace dir
489 # @param ContainerFilename: The container file which describes the file to be checked, used for error report
490 # @param SectionName: Used for error report
491 # @param Line: The line in container file which defines the file to be checked
492 #
493 # @retval The file full path if the file exists
494 #
495 def CheckFileExist(WorkspaceDir, CheckFilename, ContainerFilename, SectionName, Line, LineNo = -1):
496 CheckFile = ''
497 if CheckFilename != '' and CheckFilename != None:
498 CheckFile = WorkspaceFile(WorkspaceDir, CheckFilename)
499 if not os.path.isfile(CheckFile):
500 ContainerFile = open(ContainerFilename, 'r').read()
501 if LineNo == -1:
502 LineNo = GetLineNo(ContainerFile, Line)
503 ErrorMsg = "Can't find file '%s' defined in section '%s'" % (CheckFile, SectionName)
504 EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg,
505 File=ContainerFilename, Line = LineNo, RaiseError = EdkLogger.IsRaiseError)
506
507 return CheckFile
508
509 ## GetLineNo
510 #
511 # Find the index of a line in a file
512 #
513 # @param FileContent: Search scope
514 # @param Line: Search key
515 #
516 # @retval int Index of the line
517 # @retval -1 The line is not found
518 #
519 def GetLineNo(FileContent, Line, IsIgnoreComment = True):
520 LineList = FileContent.splitlines()
521 for Index in range(len(LineList)):
522 if LineList[Index].find(Line) > -1:
523 #
524 # Ignore statement in comment
525 #
526 if IsIgnoreComment:
527 if LineList[Index].strip()[0] == DataType.TAB_COMMENT_SPLIT:
528 continue
529 return Index + 1
530
531 return -1
532
533 ## RaiseParserError
534 #
535 # Raise a parser error
536 #
537 # @param Line: String which has error
538 # @param Section: Used for error report
539 # @param File: File which has the string
540 # @param Format: Correct format
541 #
542 def RaiseParserError(Line, Section, File, Format = '', LineNo = -1):
543 if LineNo == -1:
544 LineNo = GetLineNo(open(os.path.normpath(File), 'r').read(), Line)
545 ErrorMsg = "Invalid statement '%s' is found in section '%s'" % (Line, Section)
546 if Format != '':
547 Format = "Correct format is " + Format
548 EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, File=File, Line=LineNo, ExtraData=Format, RaiseError = EdkLogger.IsRaiseError)
549
550 ## WorkspaceFile
551 #
552 # Return a full path with workspace dir
553 #
554 # @param WorkspaceDir: Workspace dir
555 # @param Filename: Relative file name
556 #
557 # @retval string A full path
558 #
559 def WorkspaceFile(WorkspaceDir, Filename):
560 return os.path.join(NormPath(WorkspaceDir), NormPath(Filename))
561
562 ## Split string
563 #
564 # Revmove '"' which startswith and endswith string
565 #
566 # @param String: The string need to be splited
567 #
568 # @retval String: The string after removed '""'
569 #
570 def SplitString(String):
571 if String.startswith('\"'):
572 String = String[1:]
573 if String.endswith('\"'):
574 String = String[:-1]
575
576 return String
577
578 ## Convert To Sql String
579 #
580 # 1. Replace "'" with "''" in each item of StringList
581 #
582 # @param StringList: A list for strings to be converted
583 #
584 def ConvertToSqlString(StringList):
585 return map(lambda s: s.replace("'", "''") , StringList)
586
587 ## Convert To Sql String
588 #
589 # 1. Replace "'" with "''" in the String
590 #
591 # @param String: A String to be converted
592 #
593 def ConvertToSqlString2(String):
594 return String.replace("'", "''")
595
596 #
597 # Remove comment block
598 #
599 def RemoveBlockComment(Lines):
600 IsFindBlockComment = False
601 IsFindBlockCode = False
602 ReservedLine = ''
603 NewLines = []
604
605 for Line in Lines:
606 Line = Line.strip()
607 #
608 # Remove comment block
609 #
610 if Line.find(DataType.TAB_COMMENT_R8_START) > -1:
611 ReservedLine = GetSplitValueList(Line, DataType.TAB_COMMENT_R8_START, 1)[0]
612 IsFindBlockComment = True
613 if Line.find(DataType.TAB_COMMENT_R8_END) > -1:
614 Line = ReservedLine + GetSplitValueList(Line, DataType.TAB_COMMENT_R8_END, 1)[1]
615 ReservedLine = ''
616 IsFindBlockComment = False
617 if IsFindBlockComment:
618 NewLines.append('')
619 continue
620
621 NewLines.append(Line)
622 return NewLines
623
624 #
625 # Get String of a List
626 #
627 def GetStringOfList(List, Split = ' '):
628 if type(List) != type([]):
629 return List
630 Str = ''
631 for Item in List:
632 Str = Str + Item + Split
633
634 return Str.strip()
635
636 #
637 # Get HelpTextList from HelpTextClassList
638 #
639 def GetHelpTextList(HelpTextClassList):
640 List = []
641 if HelpTextClassList:
642 for HelpText in HelpTextClassList:
643 if HelpText.String.endswith('\n'):
644 HelpText.String = HelpText.String[0: len(HelpText.String) - len('\n')]
645 List.extend(HelpText.String.split('\n'))
646
647 return List
648
649 def StringToArray(String):
650 if isinstance(String, unicode):
651 if len(unicode) ==0:
652 return "{0x00, 0x00}"
653 return "{%s, 0x00, 0x00}" % ", ".join(["0x%02x, 0x00" % ord(C) for C in String])
654 elif String.startswith('L"'):
655 if String == "L\"\"":
656 return "{0x00, 0x00}"
657 else:
658 return "{%s, 0x00, 0x00}" % ", ".join(["0x%02x, 0x00" % ord(C) for C in String[2:-1]])
659 elif String.startswith('"'):
660 if String == "\"\"":
661 return "{0x00}";
662 else:
663 return "{%s, 0x00}" % ", ".join(["0x%02x" % ord(C) for C in String[1:-1]])
664 else:
665 return '{%s, 0}' % ', '.join(String.split())
666
667 def StringArrayLength(String):
668 if isinstance(String, unicode):
669 return (len(String) + 1) * 2 + 1;
670 elif String.startswith('L"'):
671 return (len(String) - 3 + 1) * 2
672 elif String.startswith('"'):
673 return (len(String) - 2 + 1)
674 else:
675 return len(String.split()) + 1
676
677 def RemoveDupOption(OptionString, Which="/I", Against=None):
678 OptionList = OptionString.split()
679 ValueList = []
680 if Against:
681 ValueList += Against
682 for Index in range(len(OptionList)):
683 Opt = OptionList[Index]
684 if not Opt.startswith(Which):
685 continue
686 if len(Opt) > len(Which):
687 Val = Opt[len(Which):]
688 else:
689 Val = ""
690 if Val in ValueList:
691 OptionList[Index] = ""
692 else:
693 ValueList.append(Val)
694 return " ".join(OptionList)
695
696 ##
697 #
698 # This acts like the main() function for the script, unless it is 'import'ed into another
699 # script.
700 #
701 if __name__ == '__main__':
702 pass
703