From: Marvin.Haeuser@outlook.com Date: Sat, 19 May 2018 10:50:25 +0000 (+0800) Subject: BaseTools: Rename String to StringUtils. X-Git-Tag: edk2-stable201903~1671 X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=commitdiff_plain;h=5a57246eab80f00ae2481970d12a2abc345a2730 BaseTools: Rename String to StringUtils. For case-insensitive file systems, edk2 String.py collides with the Python string.py, which results in build errors. This,for example, applies to building via the Windows Subsystem for Linux from a DriveFS file system. This patch renames String to StringUtils to prevent conflicts for case-insensitive file systems. Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: Marvin Haeuser Reviewed-by: Liming Gao --- diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index cf58abf251..9a57b9abd9 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -33,7 +33,7 @@ from Common.LongFilePathSupport import CopyLongFilePath from Common.BuildToolError import * from Common.DataType import * from Common.Misc import * -from Common.String import * +from Common.StringUtils import * import Common.GlobalData as GlobalData from GenFds.FdfParser import * from CommonDataClass.CommonClass import SkuInfoClass diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py index dd6301b01f..ad1919442e 100644 --- a/BaseTools/Source/Python/AutoGen/BuildEngine.py +++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py @@ -23,7 +23,7 @@ from Common.LongFilePathSupport import OpenLongFilePath as open from Common.GlobalData import * from Common.BuildToolError import * from Common.Misc import tdict, PathClass -from Common.String import NormPath +from Common.StringUtils import NormPath from Common.DataType import * import Common.EdkLogger as EdkLogger diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py index 40a343ca10..1be27d2b89 100644 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -21,7 +21,7 @@ from Common import EdkLogger from Common.BuildToolError import * from Common.DataType import * from Common.Misc import * -from Common.String import StringToArray +from Common.StringUtils import StringToArray from StrGather import * from GenPcdDb import CreatePcdDatabaseCode from IdfClassObject import * diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index a373507422..6e83b3d73e 100644 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -22,7 +22,7 @@ from Common.LongFilePathSupport import OpenLongFilePath as open from Common.MultipleWorkspace import MultipleWorkspace as mws from Common.BuildToolError import * from Common.Misc import * -from Common.String import * +from Common.StringUtils import * from BuildEngine import * import Common.GlobalData as GlobalData from collections import OrderedDict diff --git a/BaseTools/Source/Python/AutoGen/GenPcdDb.py b/BaseTools/Source/Python/AutoGen/GenPcdDb.py index d2d42fe9d0..2e05b77e14 100644 --- a/BaseTools/Source/Python/AutoGen/GenPcdDb.py +++ b/BaseTools/Source/Python/AutoGen/GenPcdDb.py @@ -12,7 +12,7 @@ # from StringIO import StringIO from Common.Misc import * -from Common.String import StringToArray +from Common.StringUtils import StringToArray from struct import pack from ValidCheckingInfoObject import VAR_CHECK_PCD_VARIABLE_TAB_CONTAINER from ValidCheckingInfoObject import VAR_CHECK_PCD_VARIABLE_TAB diff --git a/BaseTools/Source/Python/AutoGen/IdfClassObject.py b/BaseTools/Source/Python/AutoGen/IdfClassObject.py index 769790d965..e5b933c203 100644 --- a/BaseTools/Source/Python/AutoGen/IdfClassObject.py +++ b/BaseTools/Source/Python/AutoGen/IdfClassObject.py @@ -16,7 +16,7 @@ import Common.EdkLogger as EdkLogger import StringIO from Common.BuildToolError import * -from Common.String import GetLineNo +from Common.StringUtils import GetLineNo from Common.Misc import PathClass from Common.LongFilePathSupport import LongFilePath import re diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py index 54b6fb22a0..5a3c254778 100644 --- a/BaseTools/Source/Python/AutoGen/UniClassObject.py +++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py @@ -21,7 +21,7 @@ import distutils.util import Common.EdkLogger as EdkLogger import StringIO from Common.BuildToolError import * -from Common.String import GetLineNo +from Common.StringUtils import GetLineNo from Common.Misc import PathClass from Common.LongFilePathSupport import LongFilePath from Common.GlobalData import * diff --git a/BaseTools/Source/Python/Common/Database.py b/BaseTools/Source/Python/Common/Database.py index a81a44731f..d3340f5a90 100644 --- a/BaseTools/Source/Python/Common/Database.py +++ b/BaseTools/Source/Python/Common/Database.py @@ -19,7 +19,7 @@ import Common.LongFilePathOs as os import EdkLogger as EdkLogger from CommonDataClass.DataClass import * -from String import * +from StringUtils import * from DataType import * from Table.TableDataModel import TableDataModel diff --git a/BaseTools/Source/Python/Common/Parsing.py b/BaseTools/Source/Python/Common/Parsing.py index 453c2039e3..527852a50c 100644 --- a/BaseTools/Source/Python/Common/Parsing.py +++ b/BaseTools/Source/Python/Common/Parsing.py @@ -14,7 +14,7 @@ ## # Import Modules # -from String import * +from StringUtils import * from CommonDataClass.DataClass import * from DataType import * diff --git a/BaseTools/Source/Python/Common/String.py b/BaseTools/Source/Python/Common/String.py deleted file mode 100644 index 34361ecdd5..0000000000 --- a/BaseTools/Source/Python/Common/String.py +++ /dev/null @@ -1,885 +0,0 @@ -## @file -# This file is used to define common string related functions used in parsing process -# -# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
-# This program and the accompanying materials -# are licensed and made available under the terms and conditions of the BSD License -# which accompanies this distribution. The full text of the license may be found at -# http://opensource.org/licenses/bsd-license.php -# -# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, -# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. -# - -## -# Import Modules -# -import re -import DataType -import Common.LongFilePathOs as os -import string -import EdkLogger as EdkLogger - -import GlobalData -from BuildToolError import * -from CommonDataClass.Exceptions import * -from Common.LongFilePathSupport import OpenLongFilePath as open -from Common.MultipleWorkspace import MultipleWorkspace as mws - -gHexVerPatt = re.compile('0x[a-f0-9]{4}[a-f0-9]{4}$', re.IGNORECASE) -gHumanReadableVerPatt = re.compile(r'([1-9][0-9]*|0)\.[0-9]{1,2}$') - -## GetSplitValueList -# -# Get a value list from a string with multiple values splited with SplitTag -# The default SplitTag is DataType.TAB_VALUE_SPLIT -# 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC'] -# -# @param String: The input string to be splitted -# @param SplitTag: The split key, default is DataType.TAB_VALUE_SPLIT -# @param MaxSplit: The max number of split values, default is -1 -# -# @retval list() A list for splitted string -# -def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): - ValueList = [] - Last = 0 - Escaped = False - InSingleQuoteString = False - InDoubleQuoteString = False - InParenthesis = 0 - for Index in range(0, len(String)): - Char = String[Index] - - if not Escaped: - # Found a splitter not in a string, split it - if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag: - ValueList.append(String[Last:Index].strip()) - Last = Index + 1 - if MaxSplit > 0 and len(ValueList) >= MaxSplit: - break - - if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString): - Escaped = True - elif Char == '"' and not InSingleQuoteString: - if not InDoubleQuoteString: - InDoubleQuoteString = True - else: - InDoubleQuoteString = False - elif Char == "'" and not InDoubleQuoteString: - if not InSingleQuoteString: - InSingleQuoteString = True - else: - InSingleQuoteString = False - elif Char == '(': - InParenthesis = InParenthesis + 1 - elif Char == ')': - InParenthesis = InParenthesis - 1 - else: - Escaped = False - - if Last < len(String): - ValueList.append(String[Last:].strip()) - elif Last == len(String): - ValueList.append('') - - return ValueList - -## GetSplitList -# -# Get a value list from a string with multiple values splited with SplitString -# The default SplitTag is DataType.TAB_VALUE_SPLIT -# 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC'] -# -# @param String: The input string to be splitted -# @param SplitStr: The split key, default is DataType.TAB_VALUE_SPLIT -# @param MaxSplit: The max number of split values, default is -1 -# -# @retval list() A list for splitted string -# -def GetSplitList(String, SplitStr=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): - return map(lambda l: l.strip(), String.split(SplitStr, MaxSplit)) - -## MergeArches -# -# Find a key's all arches in dict, add the new arch to the list -# If not exist any arch, set the arch directly -# -# @param Dict: The input value for Dict -# @param Key: The input value for Key -# @param Arch: The Arch to be added or merged -# -def MergeArches(Dict, Key, Arch): - if Key in Dict: - Dict[Key].append(Arch) - else: - Dict[Key] = Arch.split() - -## GenDefines -# -# Parse a string with format "DEFINE = " -# Generate a map Defines[VarName] = PATH -# Return False if invalid format -# -# @param String: String with DEFINE statement -# @param Arch: Supportted Arch -# @param Defines: DEFINE statement to be parsed -# -# @retval 0 DEFINE statement found, and valid -# @retval 1 DEFINE statement found, but not valid -# @retval -1 DEFINE statement not found -# -def GenDefines(String, Arch, Defines): - if String.find(DataType.TAB_DEFINE + ' ') > -1: - List = String.replace(DataType.TAB_DEFINE + ' ', '').split(DataType.TAB_EQUAL_SPLIT) - if len(List) == 2: - Defines[(CleanString(List[0]), Arch)] = CleanString(List[1]) - return 0 - else: - return -1 - - return 1 - -## GenInclude -# -# Parse a string with format "!include " -# Return the file path -# Return False if invalid format or NOT FOUND -# -# @param String: String with INCLUDE statement -# @param IncludeFiles: INCLUDE statement to be parsed -# @param Arch: Supportted Arch -# -# @retval True -# @retval False -# -def GenInclude(String, IncludeFiles, Arch): - if String.upper().find(DataType.TAB_INCLUDE.upper() + ' ') > -1: - IncludeFile = CleanString(String[String.upper().find(DataType.TAB_INCLUDE.upper() + ' ') + len(DataType.TAB_INCLUDE + ' ') : ]) - MergeArches(IncludeFiles, IncludeFile, Arch) - return True - else: - return False - -## GetLibraryClassesWithModuleType -# -# Get Library Class definition when no module type defined -# -# @param Lines: The content to be parsed -# @param Key: Reserved -# @param KeyValues: To store data after parsing -# @param CommentCharacter: Comment char, used to ignore comment content -# -# @retval True Get library classes successfully -# -def GetLibraryClassesWithModuleType(Lines, Key, KeyValues, CommentCharacter): - newKey = SplitModuleType(Key) - Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] - LineList = Lines.splitlines() - for Line in LineList: - Line = CleanString(Line, CommentCharacter) - if Line != '' and Line[0] != CommentCharacter: - KeyValues.append([CleanString(Line, CommentCharacter), newKey[1]]) - - return True - -## GetDynamics -# -# Get Dynamic Pcds -# -# @param Lines: The content to be parsed -# @param Key: Reserved -# @param KeyValues: To store data after parsing -# @param CommentCharacter: Comment char, used to ignore comment content -# -# @retval True Get Dynamic Pcds successfully -# -def GetDynamics(Lines, Key, KeyValues, CommentCharacter): - # - # Get SkuId Name List - # - SkuIdNameList = SplitModuleType(Key) - - Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] - LineList = Lines.splitlines() - for Line in LineList: - Line = CleanString(Line, CommentCharacter) - if Line != '' and Line[0] != CommentCharacter: - KeyValues.append([CleanString(Line, CommentCharacter), SkuIdNameList[1]]) - - return True - -## SplitModuleType -# -# Split ModuleType out of section defien to get key -# [LibraryClass.Arch.ModuleType|ModuleType|ModuleType] -> [ 'LibraryClass.Arch', ['ModuleType', 'ModuleType', 'ModuleType'] ] -# -# @param Key: String to be parsed -# -# @retval ReturnValue A list for module types -# -def SplitModuleType(Key): - KeyList = Key.split(DataType.TAB_SPLIT) - # - # Fill in for arch - # - KeyList.append('') - # - # Fill in for moduletype - # - KeyList.append('') - ReturnValue = [] - KeyValue = KeyList[0] - if KeyList[1] != '': - KeyValue = KeyValue + DataType.TAB_SPLIT + KeyList[1] - ReturnValue.append(KeyValue) - ReturnValue.append(GetSplitValueList(KeyList[2])) - - return ReturnValue - -## Replace macro in strings list -# -# This method replace macros used in a given string list. The macros are -# given in a dictionary. -# -# @param StringList StringList to be processed -# @param MacroDefinitions The macro definitions in the form of dictionary -# @param SelfReplacement To decide whether replace un-defined macro to '' -# -# @retval NewList A new string list whose macros are replaced -# -def ReplaceMacros(StringList, MacroDefinitions={}, SelfReplacement=False): - NewList = [] - for String in StringList: - if type(String) == type(''): - NewList.append(ReplaceMacro(String, MacroDefinitions, SelfReplacement)) - else: - NewList.append(String) - - return NewList - -## Replace macro in string -# -# This method replace macros used in given string. The macros are given in a -# dictionary. -# -# @param String String to be processed -# @param MacroDefinitions The macro definitions in the form of dictionary -# @param SelfReplacement To decide whether replace un-defined macro to '' -# -# @retval string The string whose macros are replaced -# -def ReplaceMacro(String, MacroDefinitions={}, SelfReplacement=False, RaiseError=False): - LastString = String - while String and MacroDefinitions: - MacroUsed = GlobalData.gMacroRefPattern.findall(String) - # no macro found in String, stop replacing - if len(MacroUsed) == 0: - break - - for Macro in MacroUsed: - if Macro not in MacroDefinitions: - if RaiseError: - raise SymbolNotFound("%s not defined" % Macro) - if SelfReplacement: - String = String.replace("$(%s)" % Macro, '') - continue - if "$(%s)" % Macro not in MacroDefinitions[Macro]: - String = String.replace("$(%s)" % Macro, MacroDefinitions[Macro]) - # in case there's macro not defined - if String == LastString: - break - LastString = String - - return String - -## NormPath -# -# Create a normal path -# And replace DFEINE in the path -# -# @param Path: The input value for Path to be converted -# @param Defines: A set for DEFINE statement -# -# @retval Path Formatted path -# -def NormPath(Path, Defines={}): - IsRelativePath = False - if Path: - if Path[0] == '.': - IsRelativePath = True - # - # Replace with Define - # - if Defines: - Path = ReplaceMacro(Path, Defines) - # - # To local path format - # - Path = os.path.normpath(Path) - if Path.startswith(GlobalData.gWorkspace) and not Path.startswith(GlobalData.gBuildDirectory) and not os.path.exists(Path): - Path = Path[len (GlobalData.gWorkspace):] - if Path[0] == os.path.sep: - Path = Path[1:] - Path = mws.join(GlobalData.gWorkspace, Path) - - if IsRelativePath and Path[0] != '.': - Path = os.path.join('.', Path) - - return Path - -## CleanString -# -# Remove comments in a string -# Remove spaces -# -# @param Line: The string to be cleaned -# @param CommentCharacter: Comment char, used to ignore comment content, default is DataType.TAB_COMMENT_SPLIT -# -# @retval Path Formatted path -# -def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False, BuildOption=False): - # - # remove whitespace - # - Line = Line.strip(); - # - # Replace Edk's comment character - # - if AllowCppStyleComment: - Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter) - # - # remove comments, but we should escape comment character in string - # - InDoubleQuoteString = False - InSingleQuoteString = False - CommentInString = False - for Index in range(0, len(Line)): - if Line[Index] == '"' and not InSingleQuoteString: - InDoubleQuoteString = not InDoubleQuoteString - elif Line[Index] == "'" and not InDoubleQuoteString: - InSingleQuoteString = not InSingleQuoteString - elif Line[Index] == CommentCharacter and (InSingleQuoteString or InDoubleQuoteString): - CommentInString = True - elif Line[Index] == CommentCharacter and not (InSingleQuoteString or InDoubleQuoteString): - Line = Line[0: Index] - break - - if CommentInString and BuildOption: - Line = Line.replace('"', '') - ChIndex = Line.find('#') - while ChIndex >= 0: - if GlobalData.gIsWindows: - if ChIndex == 0 or Line[ChIndex - 1] != '^': - Line = Line[0:ChIndex] + '^' + Line[ChIndex:] - ChIndex = Line.find('#', ChIndex + 2) - else: - ChIndex = Line.find('#', ChIndex + 1) - else: - if ChIndex == 0 or Line[ChIndex - 1] != '\\': - Line = Line[0:ChIndex] + '\\' + Line[ChIndex:] - ChIndex = Line.find('#', ChIndex + 2) - else: - ChIndex = Line.find('#', ChIndex + 1) - # - # remove whitespace again - # - Line = Line.strip(); - - return Line - -## CleanString2 -# -# Split statement with comments in a string -# Remove spaces -# -# @param Line: The string to be cleaned -# @param CommentCharacter: Comment char, used to ignore comment content, default is DataType.TAB_COMMENT_SPLIT -# -# @retval Path Formatted path -# -def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False): - # - # remove whitespace - # - Line = Line.strip(); - # - # Replace Edk's comment character - # - if AllowCppStyleComment: - Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter) - # - # separate comments and statements, but we should escape comment character in string - # - InDoubleQuoteString = False - InSingleQuoteString = False - CommentInString = False - Comment = '' - for Index in range(0, len(Line)): - if Line[Index] == '"' and not InSingleQuoteString: - InDoubleQuoteString = not InDoubleQuoteString - elif Line[Index] == "'" and not InDoubleQuoteString: - InSingleQuoteString = not InSingleQuoteString - elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString): - CommentInString = True - elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString): - Comment = Line[Index:].strip() - Line = Line[0:Index].strip() - break - - return Line, Comment - -## GetMultipleValuesOfKeyFromLines -# -# Parse multiple strings to clean comment and spaces -# The result is saved to KeyValues -# -# @param Lines: The content to be parsed -# @param Key: Reserved -# @param KeyValues: To store data after parsing -# @param CommentCharacter: Comment char, used to ignore comment content -# -# @retval True Successfully executed -# -def GetMultipleValuesOfKeyFromLines(Lines, Key, KeyValues, CommentCharacter): - Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] - LineList = Lines.split('\n') - for Line in LineList: - Line = CleanString(Line, CommentCharacter) - if Line != '' and Line[0] != CommentCharacter: - KeyValues += [Line] - - return True - -## GetDefineValue -# -# Parse a DEFINE statement to get defined value -# DEFINE Key Value -# -# @param String: The content to be parsed -# @param Key: The key of DEFINE statement -# @param CommentCharacter: Comment char, used to ignore comment content -# -# @retval string The defined value -# -def GetDefineValue(String, Key, CommentCharacter): - String = CleanString(String) - return String[String.find(Key + ' ') + len(Key + ' ') : ] - -## GetHexVerValue -# -# Get a Hex Version Value -# -# @param VerString: The version string to be parsed -# -# -# @retval: If VerString is incorrectly formatted, return "None" which will break the build. -# If VerString is correctly formatted, return a Hex value of the Version Number (0xmmmmnnnn) -# where mmmm is the major number and nnnn is the adjusted minor number. -# -def GetHexVerValue(VerString): - VerString = CleanString(VerString) - - if gHumanReadableVerPatt.match(VerString): - ValueList = VerString.split('.') - Major = ValueList[0] - Minor = ValueList[1] - if len(Minor) == 1: - Minor += '0' - DeciValue = (int(Major) << 16) + int(Minor); - return "0x%08x" % DeciValue - elif gHexVerPatt.match(VerString): - return VerString - else: - return None - - -## GetSingleValueOfKeyFromLines -# -# Parse multiple strings as below to get value of each definition line -# Key1 = Value1 -# Key2 = Value2 -# The result is saved to Dictionary -# -# @param Lines: The content to be parsed -# @param Dictionary: To store data after parsing -# @param CommentCharacter: Comment char, be used to ignore comment content -# @param KeySplitCharacter: Key split char, between key name and key value. Key1 = Value1, '=' is the key split char -# @param ValueSplitFlag: Value split flag, be used to decide if has multiple values -# @param ValueSplitCharacter: Value split char, be used to split multiple values. Key1 = Value1|Value2, '|' is the value split char -# -# @retval True Successfully executed -# -def GetSingleValueOfKeyFromLines(Lines, Dictionary, CommentCharacter, KeySplitCharacter, ValueSplitFlag, ValueSplitCharacter): - Lines = Lines.split('\n') - Keys = [] - Value = '' - DefineValues = [''] - SpecValues = [''] - - for Line in Lines: - # - # Handle DEFINE and SPEC - # - if Line.find(DataType.TAB_INF_DEFINES_DEFINE + ' ') > -1: - if '' in DefineValues: - DefineValues.remove('') - DefineValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_DEFINE, CommentCharacter)) - continue - if Line.find(DataType.TAB_INF_DEFINES_SPEC + ' ') > -1: - if '' in SpecValues: - SpecValues.remove('') - SpecValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_SPEC, CommentCharacter)) - continue - - # - # Handle Others - # - LineList = Line.split(KeySplitCharacter, 1) - if len(LineList) >= 2: - Key = LineList[0].split() - if len(Key) == 1 and Key[0][0] != CommentCharacter: - # - # Remove comments and white spaces - # - LineList[1] = CleanString(LineList[1], CommentCharacter) - if ValueSplitFlag: - Value = map(string.strip, LineList[1].split(ValueSplitCharacter)) - else: - Value = CleanString(LineList[1], CommentCharacter).splitlines() - - if Key[0] in Dictionary: - if Key[0] not in Keys: - Dictionary[Key[0]] = Value - Keys.append(Key[0]) - else: - Dictionary[Key[0]].extend(Value) - else: - Dictionary[DataType.TAB_INF_DEFINES_MACRO][Key[0]] = Value[0] - - if DefineValues == []: - DefineValues = [''] - if SpecValues == []: - SpecValues = [''] - Dictionary[DataType.TAB_INF_DEFINES_DEFINE] = DefineValues - Dictionary[DataType.TAB_INF_DEFINES_SPEC] = SpecValues - - return True - -## The content to be parsed -# -# Do pre-check for a file before it is parsed -# Check $() -# Check [] -# -# @param FileName: Used for error report -# @param FileContent: File content to be parsed -# @param SupSectionTag: Used for error report -# -def PreCheck(FileName, FileContent, SupSectionTag): - LineNo = 0 - IsFailed = False - NewFileContent = '' - for Line in FileContent.splitlines(): - LineNo = LineNo + 1 - # - # Clean current line - # - Line = CleanString(Line) - - # - # Remove commented line - # - if Line.find(DataType.TAB_COMMA_SPLIT) == 0: - Line = '' - # - # Check $() - # - if Line.find('$') > -1: - if Line.find('$(') < 0 or Line.find(')') < 0: - EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) - - # - # Check [] - # - if Line.find('[') > -1 or Line.find(']') > -1: - # - # Only get one '[' or one ']' - # - if not (Line.find('[') > -1 and Line.find(']') > -1): - EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) - - # - # Regenerate FileContent - # - NewFileContent = NewFileContent + Line + '\r\n' - - if IsFailed: - EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) - - return NewFileContent - -## CheckFileType -# -# Check if the Filename is including ExtName -# Return True if it exists -# Raise a error message if it not exists -# -# @param CheckFilename: Name of the file to be checked -# @param ExtName: Ext name of the file to be checked -# @param ContainerFilename: The container file which describes the file to be checked, used for error report -# @param SectionName: Used for error report -# @param Line: The line in container file which defines the file to be checked -# -# @retval True The file type is correct -# -def CheckFileType(CheckFilename, ExtName, ContainerFilename, SectionName, Line, LineNo= -1): - if CheckFilename != '' and CheckFilename is not None: - (Root, Ext) = os.path.splitext(CheckFilename) - if Ext.upper() != ExtName.upper(): - ContainerFile = open(ContainerFilename, 'r').read() - if LineNo == -1: - LineNo = GetLineNo(ContainerFile, Line) - ErrorMsg = "Invalid %s. '%s' is found, but '%s' file is needed" % (SectionName, CheckFilename, ExtName) - EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, Line=LineNo, - File=ContainerFilename, RaiseError=EdkLogger.IsRaiseError) - - return True - -## CheckFileExist -# -# Check if the file exists -# Return True if it exists -# Raise a error message if it not exists -# -# @param CheckFilename: Name of the file to be checked -# @param WorkspaceDir: Current workspace dir -# @param ContainerFilename: The container file which describes the file to be checked, used for error report -# @param SectionName: Used for error report -# @param Line: The line in container file which defines the file to be checked -# -# @retval The file full path if the file exists -# -def CheckFileExist(WorkspaceDir, CheckFilename, ContainerFilename, SectionName, Line, LineNo= -1): - CheckFile = '' - if CheckFilename != '' and CheckFilename is not None: - CheckFile = WorkspaceFile(WorkspaceDir, CheckFilename) - if not os.path.isfile(CheckFile): - ContainerFile = open(ContainerFilename, 'r').read() - if LineNo == -1: - LineNo = GetLineNo(ContainerFile, Line) - ErrorMsg = "Can't find file '%s' defined in section '%s'" % (CheckFile, SectionName) - EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, - File=ContainerFilename, Line=LineNo, RaiseError=EdkLogger.IsRaiseError) - - return CheckFile - -## GetLineNo -# -# Find the index of a line in a file -# -# @param FileContent: Search scope -# @param Line: Search key -# -# @retval int Index of the line -# @retval -1 The line is not found -# -def GetLineNo(FileContent, Line, IsIgnoreComment=True): - LineList = FileContent.splitlines() - for Index in range(len(LineList)): - if LineList[Index].find(Line) > -1: - # - # Ignore statement in comment - # - if IsIgnoreComment: - if LineList[Index].strip()[0] == DataType.TAB_COMMENT_SPLIT: - continue - return Index + 1 - - return -1 - -## RaiseParserError -# -# Raise a parser error -# -# @param Line: String which has error -# @param Section: Used for error report -# @param File: File which has the string -# @param Format: Correct format -# -def RaiseParserError(Line, Section, File, Format='', LineNo= -1): - if LineNo == -1: - LineNo = GetLineNo(open(os.path.normpath(File), 'r').read(), Line) - ErrorMsg = "Invalid statement '%s' is found in section '%s'" % (Line, Section) - if Format != '': - Format = "Correct format is " + Format - EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, File=File, Line=LineNo, ExtraData=Format, RaiseError=EdkLogger.IsRaiseError) - -## WorkspaceFile -# -# Return a full path with workspace dir -# -# @param WorkspaceDir: Workspace dir -# @param Filename: Relative file name -# -# @retval string A full path -# -def WorkspaceFile(WorkspaceDir, Filename): - return mws.join(NormPath(WorkspaceDir), NormPath(Filename)) - -## Split string -# -# Revmove '"' which startswith and endswith string -# -# @param String: The string need to be splited -# -# @retval String: The string after removed '""' -# -def SplitString(String): - if String.startswith('\"'): - String = String[1:] - if String.endswith('\"'): - String = String[:-1] - - return String - -## Convert To Sql String -# -# 1. Replace "'" with "''" in each item of StringList -# -# @param StringList: A list for strings to be converted -# -def ConvertToSqlString(StringList): - return map(lambda s: s.replace("'", "''") , StringList) - -## Convert To Sql String -# -# 1. Replace "'" with "''" in the String -# -# @param String: A String to be converted -# -def ConvertToSqlString2(String): - return String.replace("'", "''") - -# -# Remove comment block -# -def RemoveBlockComment(Lines): - IsFindBlockComment = False - IsFindBlockCode = False - ReservedLine = '' - NewLines = [] - - for Line in Lines: - Line = Line.strip() - # - # Remove comment block - # - if Line.find(DataType.TAB_COMMENT_EDK_START) > -1: - ReservedLine = GetSplitList(Line, DataType.TAB_COMMENT_EDK_START, 1)[0] - IsFindBlockComment = True - if Line.find(DataType.TAB_COMMENT_EDK_END) > -1: - Line = ReservedLine + GetSplitList(Line, DataType.TAB_COMMENT_EDK_END, 1)[1] - ReservedLine = '' - IsFindBlockComment = False - if IsFindBlockComment: - NewLines.append('') - continue - - NewLines.append(Line) - return NewLines - -# -# Get String of a List -# -def GetStringOfList(List, Split=' '): - if type(List) != type([]): - return List - Str = '' - for Item in List: - Str = Str + Item + Split - - return Str.strip() - -# -# Get HelpTextList from HelpTextClassList -# -def GetHelpTextList(HelpTextClassList): - List = [] - if HelpTextClassList: - for HelpText in HelpTextClassList: - if HelpText.String.endswith('\n'): - HelpText.String = HelpText.String[0: len(HelpText.String) - len('\n')] - List.extend(HelpText.String.split('\n')) - - return List - -def StringToArray(String): - if isinstance(String, unicode): - if len(unicode) == 0: - return "{0x00,0x00}" - return "{%s,0x00,0x00}" % ",".join("0x%02x,0x00" % ord(C) for C in String) - elif String.startswith('L"'): - if String == "L\"\"": - return "{0x00,0x00}" - else: - return "{%s,0x00,0x00}" % ",".join("0x%02x,0x00" % ord(C) for C in String[2:-1]) - elif String.startswith('"'): - if String == "\"\"": - return "{0x00,0x00}" - else: - StringLen = len(String[1:-1]) - if StringLen % 2: - return "{%s,0x00}" % ",".join("0x%02x" % ord(C) for C in String[1:-1]) - else: - return "{%s,0x00,0x00}" % ",".join("0x%02x" % ord(C) for C in String[1:-1]) - elif String.startswith('{'): - StringLen = len(String.split(",")) - if StringLen % 2: - return "{%s,0x00}" % ",".join(C.strip() for C in String[1:-1].split(',')) - else: - return "{%s}" % ",".join(C.strip() for C in String[1:-1].split(',')) - - else: - if len(String.split()) % 2: - return '{%s,0}' % ','.join(String.split()) - else: - return '{%s,0,0}' % ','.join(String.split()) - -def StringArrayLength(String): - if isinstance(String, unicode): - return (len(String) + 1) * 2 + 1; - elif String.startswith('L"'): - return (len(String) - 3 + 1) * 2 - elif String.startswith('"'): - return (len(String) - 2 + 1) - else: - return len(String.split()) + 1 - -def RemoveDupOption(OptionString, Which="/I", Against=None): - OptionList = OptionString.split() - ValueList = [] - if Against: - ValueList += Against - for Index in range(len(OptionList)): - Opt = OptionList[Index] - if not Opt.startswith(Which): - continue - if len(Opt) > len(Which): - Val = Opt[len(Which):] - else: - Val = "" - if Val in ValueList: - OptionList[Index] = "" - else: - ValueList.append(Val) - return " ".join(OptionList) - -## -# -# This acts like the main() function for the script, unless it is 'import'ed into another -# script. -# -if __name__ == '__main__': - pass - diff --git a/BaseTools/Source/Python/Common/StringUtils.py b/BaseTools/Source/Python/Common/StringUtils.py new file mode 100644 index 0000000000..34361ecdd5 --- /dev/null +++ b/BaseTools/Source/Python/Common/StringUtils.py @@ -0,0 +1,885 @@ +## @file +# This file is used to define common string related functions used in parsing process +# +# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
+# This program and the accompanying materials +# are licensed and made available under the terms and conditions of the BSD License +# which accompanies this distribution. The full text of the license may be found at +# http://opensource.org/licenses/bsd-license.php +# +# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +# + +## +# Import Modules +# +import re +import DataType +import Common.LongFilePathOs as os +import string +import EdkLogger as EdkLogger + +import GlobalData +from BuildToolError import * +from CommonDataClass.Exceptions import * +from Common.LongFilePathSupport import OpenLongFilePath as open +from Common.MultipleWorkspace import MultipleWorkspace as mws + +gHexVerPatt = re.compile('0x[a-f0-9]{4}[a-f0-9]{4}$', re.IGNORECASE) +gHumanReadableVerPatt = re.compile(r'([1-9][0-9]*|0)\.[0-9]{1,2}$') + +## GetSplitValueList +# +# Get a value list from a string with multiple values splited with SplitTag +# The default SplitTag is DataType.TAB_VALUE_SPLIT +# 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC'] +# +# @param String: The input string to be splitted +# @param SplitTag: The split key, default is DataType.TAB_VALUE_SPLIT +# @param MaxSplit: The max number of split values, default is -1 +# +# @retval list() A list for splitted string +# +def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): + ValueList = [] + Last = 0 + Escaped = False + InSingleQuoteString = False + InDoubleQuoteString = False + InParenthesis = 0 + for Index in range(0, len(String)): + Char = String[Index] + + if not Escaped: + # Found a splitter not in a string, split it + if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag: + ValueList.append(String[Last:Index].strip()) + Last = Index + 1 + if MaxSplit > 0 and len(ValueList) >= MaxSplit: + break + + if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString): + Escaped = True + elif Char == '"' and not InSingleQuoteString: + if not InDoubleQuoteString: + InDoubleQuoteString = True + else: + InDoubleQuoteString = False + elif Char == "'" and not InDoubleQuoteString: + if not InSingleQuoteString: + InSingleQuoteString = True + else: + InSingleQuoteString = False + elif Char == '(': + InParenthesis = InParenthesis + 1 + elif Char == ')': + InParenthesis = InParenthesis - 1 + else: + Escaped = False + + if Last < len(String): + ValueList.append(String[Last:].strip()) + elif Last == len(String): + ValueList.append('') + + return ValueList + +## GetSplitList +# +# Get a value list from a string with multiple values splited with SplitString +# The default SplitTag is DataType.TAB_VALUE_SPLIT +# 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC'] +# +# @param String: The input string to be splitted +# @param SplitStr: The split key, default is DataType.TAB_VALUE_SPLIT +# @param MaxSplit: The max number of split values, default is -1 +# +# @retval list() A list for splitted string +# +def GetSplitList(String, SplitStr=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): + return map(lambda l: l.strip(), String.split(SplitStr, MaxSplit)) + +## MergeArches +# +# Find a key's all arches in dict, add the new arch to the list +# If not exist any arch, set the arch directly +# +# @param Dict: The input value for Dict +# @param Key: The input value for Key +# @param Arch: The Arch to be added or merged +# +def MergeArches(Dict, Key, Arch): + if Key in Dict: + Dict[Key].append(Arch) + else: + Dict[Key] = Arch.split() + +## GenDefines +# +# Parse a string with format "DEFINE = " +# Generate a map Defines[VarName] = PATH +# Return False if invalid format +# +# @param String: String with DEFINE statement +# @param Arch: Supportted Arch +# @param Defines: DEFINE statement to be parsed +# +# @retval 0 DEFINE statement found, and valid +# @retval 1 DEFINE statement found, but not valid +# @retval -1 DEFINE statement not found +# +def GenDefines(String, Arch, Defines): + if String.find(DataType.TAB_DEFINE + ' ') > -1: + List = String.replace(DataType.TAB_DEFINE + ' ', '').split(DataType.TAB_EQUAL_SPLIT) + if len(List) == 2: + Defines[(CleanString(List[0]), Arch)] = CleanString(List[1]) + return 0 + else: + return -1 + + return 1 + +## GenInclude +# +# Parse a string with format "!include " +# Return the file path +# Return False if invalid format or NOT FOUND +# +# @param String: String with INCLUDE statement +# @param IncludeFiles: INCLUDE statement to be parsed +# @param Arch: Supportted Arch +# +# @retval True +# @retval False +# +def GenInclude(String, IncludeFiles, Arch): + if String.upper().find(DataType.TAB_INCLUDE.upper() + ' ') > -1: + IncludeFile = CleanString(String[String.upper().find(DataType.TAB_INCLUDE.upper() + ' ') + len(DataType.TAB_INCLUDE + ' ') : ]) + MergeArches(IncludeFiles, IncludeFile, Arch) + return True + else: + return False + +## GetLibraryClassesWithModuleType +# +# Get Library Class definition when no module type defined +# +# @param Lines: The content to be parsed +# @param Key: Reserved +# @param KeyValues: To store data after parsing +# @param CommentCharacter: Comment char, used to ignore comment content +# +# @retval True Get library classes successfully +# +def GetLibraryClassesWithModuleType(Lines, Key, KeyValues, CommentCharacter): + newKey = SplitModuleType(Key) + Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] + LineList = Lines.splitlines() + for Line in LineList: + Line = CleanString(Line, CommentCharacter) + if Line != '' and Line[0] != CommentCharacter: + KeyValues.append([CleanString(Line, CommentCharacter), newKey[1]]) + + return True + +## GetDynamics +# +# Get Dynamic Pcds +# +# @param Lines: The content to be parsed +# @param Key: Reserved +# @param KeyValues: To store data after parsing +# @param CommentCharacter: Comment char, used to ignore comment content +# +# @retval True Get Dynamic Pcds successfully +# +def GetDynamics(Lines, Key, KeyValues, CommentCharacter): + # + # Get SkuId Name List + # + SkuIdNameList = SplitModuleType(Key) + + Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] + LineList = Lines.splitlines() + for Line in LineList: + Line = CleanString(Line, CommentCharacter) + if Line != '' and Line[0] != CommentCharacter: + KeyValues.append([CleanString(Line, CommentCharacter), SkuIdNameList[1]]) + + return True + +## SplitModuleType +# +# Split ModuleType out of section defien to get key +# [LibraryClass.Arch.ModuleType|ModuleType|ModuleType] -> [ 'LibraryClass.Arch', ['ModuleType', 'ModuleType', 'ModuleType'] ] +# +# @param Key: String to be parsed +# +# @retval ReturnValue A list for module types +# +def SplitModuleType(Key): + KeyList = Key.split(DataType.TAB_SPLIT) + # + # Fill in for arch + # + KeyList.append('') + # + # Fill in for moduletype + # + KeyList.append('') + ReturnValue = [] + KeyValue = KeyList[0] + if KeyList[1] != '': + KeyValue = KeyValue + DataType.TAB_SPLIT + KeyList[1] + ReturnValue.append(KeyValue) + ReturnValue.append(GetSplitValueList(KeyList[2])) + + return ReturnValue + +## Replace macro in strings list +# +# This method replace macros used in a given string list. The macros are +# given in a dictionary. +# +# @param StringList StringList to be processed +# @param MacroDefinitions The macro definitions in the form of dictionary +# @param SelfReplacement To decide whether replace un-defined macro to '' +# +# @retval NewList A new string list whose macros are replaced +# +def ReplaceMacros(StringList, MacroDefinitions={}, SelfReplacement=False): + NewList = [] + for String in StringList: + if type(String) == type(''): + NewList.append(ReplaceMacro(String, MacroDefinitions, SelfReplacement)) + else: + NewList.append(String) + + return NewList + +## Replace macro in string +# +# This method replace macros used in given string. The macros are given in a +# dictionary. +# +# @param String String to be processed +# @param MacroDefinitions The macro definitions in the form of dictionary +# @param SelfReplacement To decide whether replace un-defined macro to '' +# +# @retval string The string whose macros are replaced +# +def ReplaceMacro(String, MacroDefinitions={}, SelfReplacement=False, RaiseError=False): + LastString = String + while String and MacroDefinitions: + MacroUsed = GlobalData.gMacroRefPattern.findall(String) + # no macro found in String, stop replacing + if len(MacroUsed) == 0: + break + + for Macro in MacroUsed: + if Macro not in MacroDefinitions: + if RaiseError: + raise SymbolNotFound("%s not defined" % Macro) + if SelfReplacement: + String = String.replace("$(%s)" % Macro, '') + continue + if "$(%s)" % Macro not in MacroDefinitions[Macro]: + String = String.replace("$(%s)" % Macro, MacroDefinitions[Macro]) + # in case there's macro not defined + if String == LastString: + break + LastString = String + + return String + +## NormPath +# +# Create a normal path +# And replace DFEINE in the path +# +# @param Path: The input value for Path to be converted +# @param Defines: A set for DEFINE statement +# +# @retval Path Formatted path +# +def NormPath(Path, Defines={}): + IsRelativePath = False + if Path: + if Path[0] == '.': + IsRelativePath = True + # + # Replace with Define + # + if Defines: + Path = ReplaceMacro(Path, Defines) + # + # To local path format + # + Path = os.path.normpath(Path) + if Path.startswith(GlobalData.gWorkspace) and not Path.startswith(GlobalData.gBuildDirectory) and not os.path.exists(Path): + Path = Path[len (GlobalData.gWorkspace):] + if Path[0] == os.path.sep: + Path = Path[1:] + Path = mws.join(GlobalData.gWorkspace, Path) + + if IsRelativePath and Path[0] != '.': + Path = os.path.join('.', Path) + + return Path + +## CleanString +# +# Remove comments in a string +# Remove spaces +# +# @param Line: The string to be cleaned +# @param CommentCharacter: Comment char, used to ignore comment content, default is DataType.TAB_COMMENT_SPLIT +# +# @retval Path Formatted path +# +def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False, BuildOption=False): + # + # remove whitespace + # + Line = Line.strip(); + # + # Replace Edk's comment character + # + if AllowCppStyleComment: + Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter) + # + # remove comments, but we should escape comment character in string + # + InDoubleQuoteString = False + InSingleQuoteString = False + CommentInString = False + for Index in range(0, len(Line)): + if Line[Index] == '"' and not InSingleQuoteString: + InDoubleQuoteString = not InDoubleQuoteString + elif Line[Index] == "'" and not InDoubleQuoteString: + InSingleQuoteString = not InSingleQuoteString + elif Line[Index] == CommentCharacter and (InSingleQuoteString or InDoubleQuoteString): + CommentInString = True + elif Line[Index] == CommentCharacter and not (InSingleQuoteString or InDoubleQuoteString): + Line = Line[0: Index] + break + + if CommentInString and BuildOption: + Line = Line.replace('"', '') + ChIndex = Line.find('#') + while ChIndex >= 0: + if GlobalData.gIsWindows: + if ChIndex == 0 or Line[ChIndex - 1] != '^': + Line = Line[0:ChIndex] + '^' + Line[ChIndex:] + ChIndex = Line.find('#', ChIndex + 2) + else: + ChIndex = Line.find('#', ChIndex + 1) + else: + if ChIndex == 0 or Line[ChIndex - 1] != '\\': + Line = Line[0:ChIndex] + '\\' + Line[ChIndex:] + ChIndex = Line.find('#', ChIndex + 2) + else: + ChIndex = Line.find('#', ChIndex + 1) + # + # remove whitespace again + # + Line = Line.strip(); + + return Line + +## CleanString2 +# +# Split statement with comments in a string +# Remove spaces +# +# @param Line: The string to be cleaned +# @param CommentCharacter: Comment char, used to ignore comment content, default is DataType.TAB_COMMENT_SPLIT +# +# @retval Path Formatted path +# +def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False): + # + # remove whitespace + # + Line = Line.strip(); + # + # Replace Edk's comment character + # + if AllowCppStyleComment: + Line = Line.replace(DataType.TAB_COMMENT_EDK_SPLIT, CommentCharacter) + # + # separate comments and statements, but we should escape comment character in string + # + InDoubleQuoteString = False + InSingleQuoteString = False + CommentInString = False + Comment = '' + for Index in range(0, len(Line)): + if Line[Index] == '"' and not InSingleQuoteString: + InDoubleQuoteString = not InDoubleQuoteString + elif Line[Index] == "'" and not InDoubleQuoteString: + InSingleQuoteString = not InSingleQuoteString + elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString): + CommentInString = True + elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString): + Comment = Line[Index:].strip() + Line = Line[0:Index].strip() + break + + return Line, Comment + +## GetMultipleValuesOfKeyFromLines +# +# Parse multiple strings to clean comment and spaces +# The result is saved to KeyValues +# +# @param Lines: The content to be parsed +# @param Key: Reserved +# @param KeyValues: To store data after parsing +# @param CommentCharacter: Comment char, used to ignore comment content +# +# @retval True Successfully executed +# +def GetMultipleValuesOfKeyFromLines(Lines, Key, KeyValues, CommentCharacter): + Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] + LineList = Lines.split('\n') + for Line in LineList: + Line = CleanString(Line, CommentCharacter) + if Line != '' and Line[0] != CommentCharacter: + KeyValues += [Line] + + return True + +## GetDefineValue +# +# Parse a DEFINE statement to get defined value +# DEFINE Key Value +# +# @param String: The content to be parsed +# @param Key: The key of DEFINE statement +# @param CommentCharacter: Comment char, used to ignore comment content +# +# @retval string The defined value +# +def GetDefineValue(String, Key, CommentCharacter): + String = CleanString(String) + return String[String.find(Key + ' ') + len(Key + ' ') : ] + +## GetHexVerValue +# +# Get a Hex Version Value +# +# @param VerString: The version string to be parsed +# +# +# @retval: If VerString is incorrectly formatted, return "None" which will break the build. +# If VerString is correctly formatted, return a Hex value of the Version Number (0xmmmmnnnn) +# where mmmm is the major number and nnnn is the adjusted minor number. +# +def GetHexVerValue(VerString): + VerString = CleanString(VerString) + + if gHumanReadableVerPatt.match(VerString): + ValueList = VerString.split('.') + Major = ValueList[0] + Minor = ValueList[1] + if len(Minor) == 1: + Minor += '0' + DeciValue = (int(Major) << 16) + int(Minor); + return "0x%08x" % DeciValue + elif gHexVerPatt.match(VerString): + return VerString + else: + return None + + +## GetSingleValueOfKeyFromLines +# +# Parse multiple strings as below to get value of each definition line +# Key1 = Value1 +# Key2 = Value2 +# The result is saved to Dictionary +# +# @param Lines: The content to be parsed +# @param Dictionary: To store data after parsing +# @param CommentCharacter: Comment char, be used to ignore comment content +# @param KeySplitCharacter: Key split char, between key name and key value. Key1 = Value1, '=' is the key split char +# @param ValueSplitFlag: Value split flag, be used to decide if has multiple values +# @param ValueSplitCharacter: Value split char, be used to split multiple values. Key1 = Value1|Value2, '|' is the value split char +# +# @retval True Successfully executed +# +def GetSingleValueOfKeyFromLines(Lines, Dictionary, CommentCharacter, KeySplitCharacter, ValueSplitFlag, ValueSplitCharacter): + Lines = Lines.split('\n') + Keys = [] + Value = '' + DefineValues = [''] + SpecValues = [''] + + for Line in Lines: + # + # Handle DEFINE and SPEC + # + if Line.find(DataType.TAB_INF_DEFINES_DEFINE + ' ') > -1: + if '' in DefineValues: + DefineValues.remove('') + DefineValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_DEFINE, CommentCharacter)) + continue + if Line.find(DataType.TAB_INF_DEFINES_SPEC + ' ') > -1: + if '' in SpecValues: + SpecValues.remove('') + SpecValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_SPEC, CommentCharacter)) + continue + + # + # Handle Others + # + LineList = Line.split(KeySplitCharacter, 1) + if len(LineList) >= 2: + Key = LineList[0].split() + if len(Key) == 1 and Key[0][0] != CommentCharacter: + # + # Remove comments and white spaces + # + LineList[1] = CleanString(LineList[1], CommentCharacter) + if ValueSplitFlag: + Value = map(string.strip, LineList[1].split(ValueSplitCharacter)) + else: + Value = CleanString(LineList[1], CommentCharacter).splitlines() + + if Key[0] in Dictionary: + if Key[0] not in Keys: + Dictionary[Key[0]] = Value + Keys.append(Key[0]) + else: + Dictionary[Key[0]].extend(Value) + else: + Dictionary[DataType.TAB_INF_DEFINES_MACRO][Key[0]] = Value[0] + + if DefineValues == []: + DefineValues = [''] + if SpecValues == []: + SpecValues = [''] + Dictionary[DataType.TAB_INF_DEFINES_DEFINE] = DefineValues + Dictionary[DataType.TAB_INF_DEFINES_SPEC] = SpecValues + + return True + +## The content to be parsed +# +# Do pre-check for a file before it is parsed +# Check $() +# Check [] +# +# @param FileName: Used for error report +# @param FileContent: File content to be parsed +# @param SupSectionTag: Used for error report +# +def PreCheck(FileName, FileContent, SupSectionTag): + LineNo = 0 + IsFailed = False + NewFileContent = '' + for Line in FileContent.splitlines(): + LineNo = LineNo + 1 + # + # Clean current line + # + Line = CleanString(Line) + + # + # Remove commented line + # + if Line.find(DataType.TAB_COMMA_SPLIT) == 0: + Line = '' + # + # Check $() + # + if Line.find('$') > -1: + if Line.find('$(') < 0 or Line.find(')') < 0: + EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) + + # + # Check [] + # + if Line.find('[') > -1 or Line.find(']') > -1: + # + # Only get one '[' or one ']' + # + if not (Line.find('[') > -1 and Line.find(']') > -1): + EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) + + # + # Regenerate FileContent + # + NewFileContent = NewFileContent + Line + '\r\n' + + if IsFailed: + EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError) + + return NewFileContent + +## CheckFileType +# +# Check if the Filename is including ExtName +# Return True if it exists +# Raise a error message if it not exists +# +# @param CheckFilename: Name of the file to be checked +# @param ExtName: Ext name of the file to be checked +# @param ContainerFilename: The container file which describes the file to be checked, used for error report +# @param SectionName: Used for error report +# @param Line: The line in container file which defines the file to be checked +# +# @retval True The file type is correct +# +def CheckFileType(CheckFilename, ExtName, ContainerFilename, SectionName, Line, LineNo= -1): + if CheckFilename != '' and CheckFilename is not None: + (Root, Ext) = os.path.splitext(CheckFilename) + if Ext.upper() != ExtName.upper(): + ContainerFile = open(ContainerFilename, 'r').read() + if LineNo == -1: + LineNo = GetLineNo(ContainerFile, Line) + ErrorMsg = "Invalid %s. '%s' is found, but '%s' file is needed" % (SectionName, CheckFilename, ExtName) + EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, Line=LineNo, + File=ContainerFilename, RaiseError=EdkLogger.IsRaiseError) + + return True + +## CheckFileExist +# +# Check if the file exists +# Return True if it exists +# Raise a error message if it not exists +# +# @param CheckFilename: Name of the file to be checked +# @param WorkspaceDir: Current workspace dir +# @param ContainerFilename: The container file which describes the file to be checked, used for error report +# @param SectionName: Used for error report +# @param Line: The line in container file which defines the file to be checked +# +# @retval The file full path if the file exists +# +def CheckFileExist(WorkspaceDir, CheckFilename, ContainerFilename, SectionName, Line, LineNo= -1): + CheckFile = '' + if CheckFilename != '' and CheckFilename is not None: + CheckFile = WorkspaceFile(WorkspaceDir, CheckFilename) + if not os.path.isfile(CheckFile): + ContainerFile = open(ContainerFilename, 'r').read() + if LineNo == -1: + LineNo = GetLineNo(ContainerFile, Line) + ErrorMsg = "Can't find file '%s' defined in section '%s'" % (CheckFile, SectionName) + EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, + File=ContainerFilename, Line=LineNo, RaiseError=EdkLogger.IsRaiseError) + + return CheckFile + +## GetLineNo +# +# Find the index of a line in a file +# +# @param FileContent: Search scope +# @param Line: Search key +# +# @retval int Index of the line +# @retval -1 The line is not found +# +def GetLineNo(FileContent, Line, IsIgnoreComment=True): + LineList = FileContent.splitlines() + for Index in range(len(LineList)): + if LineList[Index].find(Line) > -1: + # + # Ignore statement in comment + # + if IsIgnoreComment: + if LineList[Index].strip()[0] == DataType.TAB_COMMENT_SPLIT: + continue + return Index + 1 + + return -1 + +## RaiseParserError +# +# Raise a parser error +# +# @param Line: String which has error +# @param Section: Used for error report +# @param File: File which has the string +# @param Format: Correct format +# +def RaiseParserError(Line, Section, File, Format='', LineNo= -1): + if LineNo == -1: + LineNo = GetLineNo(open(os.path.normpath(File), 'r').read(), Line) + ErrorMsg = "Invalid statement '%s' is found in section '%s'" % (Line, Section) + if Format != '': + Format = "Correct format is " + Format + EdkLogger.error("Parser", PARSER_ERROR, ErrorMsg, File=File, Line=LineNo, ExtraData=Format, RaiseError=EdkLogger.IsRaiseError) + +## WorkspaceFile +# +# Return a full path with workspace dir +# +# @param WorkspaceDir: Workspace dir +# @param Filename: Relative file name +# +# @retval string A full path +# +def WorkspaceFile(WorkspaceDir, Filename): + return mws.join(NormPath(WorkspaceDir), NormPath(Filename)) + +## Split string +# +# Revmove '"' which startswith and endswith string +# +# @param String: The string need to be splited +# +# @retval String: The string after removed '""' +# +def SplitString(String): + if String.startswith('\"'): + String = String[1:] + if String.endswith('\"'): + String = String[:-1] + + return String + +## Convert To Sql String +# +# 1. Replace "'" with "''" in each item of StringList +# +# @param StringList: A list for strings to be converted +# +def ConvertToSqlString(StringList): + return map(lambda s: s.replace("'", "''") , StringList) + +## Convert To Sql String +# +# 1. Replace "'" with "''" in the String +# +# @param String: A String to be converted +# +def ConvertToSqlString2(String): + return String.replace("'", "''") + +# +# Remove comment block +# +def RemoveBlockComment(Lines): + IsFindBlockComment = False + IsFindBlockCode = False + ReservedLine = '' + NewLines = [] + + for Line in Lines: + Line = Line.strip() + # + # Remove comment block + # + if Line.find(DataType.TAB_COMMENT_EDK_START) > -1: + ReservedLine = GetSplitList(Line, DataType.TAB_COMMENT_EDK_START, 1)[0] + IsFindBlockComment = True + if Line.find(DataType.TAB_COMMENT_EDK_END) > -1: + Line = ReservedLine + GetSplitList(Line, DataType.TAB_COMMENT_EDK_END, 1)[1] + ReservedLine = '' + IsFindBlockComment = False + if IsFindBlockComment: + NewLines.append('') + continue + + NewLines.append(Line) + return NewLines + +# +# Get String of a List +# +def GetStringOfList(List, Split=' '): + if type(List) != type([]): + return List + Str = '' + for Item in List: + Str = Str + Item + Split + + return Str.strip() + +# +# Get HelpTextList from HelpTextClassList +# +def GetHelpTextList(HelpTextClassList): + List = [] + if HelpTextClassList: + for HelpText in HelpTextClassList: + if HelpText.String.endswith('\n'): + HelpText.String = HelpText.String[0: len(HelpText.String) - len('\n')] + List.extend(HelpText.String.split('\n')) + + return List + +def StringToArray(String): + if isinstance(String, unicode): + if len(unicode) == 0: + return "{0x00,0x00}" + return "{%s,0x00,0x00}" % ",".join("0x%02x,0x00" % ord(C) for C in String) + elif String.startswith('L"'): + if String == "L\"\"": + return "{0x00,0x00}" + else: + return "{%s,0x00,0x00}" % ",".join("0x%02x,0x00" % ord(C) for C in String[2:-1]) + elif String.startswith('"'): + if String == "\"\"": + return "{0x00,0x00}" + else: + StringLen = len(String[1:-1]) + if StringLen % 2: + return "{%s,0x00}" % ",".join("0x%02x" % ord(C) for C in String[1:-1]) + else: + return "{%s,0x00,0x00}" % ",".join("0x%02x" % ord(C) for C in String[1:-1]) + elif String.startswith('{'): + StringLen = len(String.split(",")) + if StringLen % 2: + return "{%s,0x00}" % ",".join(C.strip() for C in String[1:-1].split(',')) + else: + return "{%s}" % ",".join(C.strip() for C in String[1:-1].split(',')) + + else: + if len(String.split()) % 2: + return '{%s,0}' % ','.join(String.split()) + else: + return '{%s,0,0}' % ','.join(String.split()) + +def StringArrayLength(String): + if isinstance(String, unicode): + return (len(String) + 1) * 2 + 1; + elif String.startswith('L"'): + return (len(String) - 3 + 1) * 2 + elif String.startswith('"'): + return (len(String) - 2 + 1) + else: + return len(String.split()) + 1 + +def RemoveDupOption(OptionString, Which="/I", Against=None): + OptionList = OptionString.split() + ValueList = [] + if Against: + ValueList += Against + for Index in range(len(OptionList)): + Opt = OptionList[Index] + if not Opt.startswith(Which): + continue + if len(Opt) > len(Which): + Val = Opt[len(Which):] + else: + Val = "" + if Val in ValueList: + OptionList[Index] = "" + else: + ValueList.append(Val) + return " ".join(OptionList) + +## +# +# This acts like the main() function for the script, unless it is 'import'ed into another +# script. +# +if __name__ == '__main__': + pass + diff --git a/BaseTools/Source/Python/Common/ToolDefClassObject.py b/BaseTools/Source/Python/Common/ToolDefClassObject.py index 83359586b9..dd985ab303 100644 --- a/BaseTools/Source/Python/Common/ToolDefClassObject.py +++ b/BaseTools/Source/Python/Common/ToolDefClassObject.py @@ -22,7 +22,7 @@ from BuildToolError import * from TargetTxtClassObject import * from Common.LongFilePathSupport import OpenLongFilePath as open from Common.Misc import PathClass -from Common.String import NormPath +from Common.StringUtils import NormPath import Common.GlobalData as GlobalData from Common import GlobalData from Common.MultipleWorkspace import MultipleWorkspace as mws diff --git a/BaseTools/Source/Python/Ecc/Configuration.py b/BaseTools/Source/Python/Ecc/Configuration.py index fee7ecb970..217b60f4f3 100644 --- a/BaseTools/Source/Python/Ecc/Configuration.py +++ b/BaseTools/Source/Python/Ecc/Configuration.py @@ -17,7 +17,7 @@ import Common.LongFilePathOs as os import Common.EdkLogger as EdkLogger from Common.DataType import * -from Common.String import * +from Common.StringUtils import * from Common.LongFilePathSupport import OpenLongFilePath as open _ConfigFileToInternalTranslation = { diff --git a/BaseTools/Source/Python/Ecc/Ecc.py b/BaseTools/Source/Python/Ecc/Ecc.py index 60dfc00260..e78d70372e 100644 --- a/BaseTools/Source/Python/Ecc/Ecc.py +++ b/BaseTools/Source/Python/Ecc/Ecc.py @@ -24,7 +24,7 @@ from Configuration import Configuration from Check import Check import Common.GlobalData as GlobalData -from Common.String import NormPath +from Common.StringUtils import NormPath from Common.BuildVersion import gBUILD_VERSION from Common import BuildToolError from Common.Misc import PathClass diff --git a/BaseTools/Source/Python/Ecc/MetaFileWorkspace/MetaFileParser.py b/BaseTools/Source/Python/Ecc/MetaFileWorkspace/MetaFileParser.py index 4d61cd1cea..3749f6a269 100644 --- a/BaseTools/Source/Python/Ecc/MetaFileWorkspace/MetaFileParser.py +++ b/BaseTools/Source/Python/Ecc/MetaFileWorkspace/MetaFileParser.py @@ -26,7 +26,7 @@ import EccToolError from CommonDataClass.DataClass import * from Common.DataType import * -from Common.String import * +from Common.StringUtils import * from Common.Misc import GuidStructureStringToGuidString, CheckPcdDatum, PathClass, AnalyzePcdData from Common.Expression import * from CommonDataClass.Exceptions import * diff --git a/BaseTools/Source/Python/Eot/Eot.py b/BaseTools/Source/Python/Eot/Eot.py index 786868d55d..dfd1146af7 100644 --- a/BaseTools/Source/Python/Eot/Eot.py +++ b/BaseTools/Source/Python/Eot/Eot.py @@ -18,7 +18,7 @@ import Common.LongFilePathOs as os, time, glob import Common.EdkLogger as EdkLogger import EotGlobalData from optparse import OptionParser -from Common.String import NormPath +from Common.StringUtils import NormPath from Common import BuildToolError from Common.Misc import GuidStructureStringToGuidString, sdict from InfParserLite import * diff --git a/BaseTools/Source/Python/Eot/InfParserLite.py b/BaseTools/Source/Python/Eot/InfParserLite.py index 8867bb5dc2..584a95d6f3 100644 --- a/BaseTools/Source/Python/Eot/InfParserLite.py +++ b/BaseTools/Source/Python/Eot/InfParserLite.py @@ -19,7 +19,7 @@ import Common.EdkLogger as EdkLogger from Common.DataType import * from CommonDataClass.DataClass import * from Common.Identification import * -from Common.String import * +from Common.StringUtils import * from Parser import * import Database diff --git a/BaseTools/Source/Python/Eot/Parser.py b/BaseTools/Source/Python/Eot/Parser.py index 14c287588a..ff88e957ad 100644 --- a/BaseTools/Source/Python/Eot/Parser.py +++ b/BaseTools/Source/Python/Eot/Parser.py @@ -19,9 +19,9 @@ import Common.LongFilePathOs as os, re import Common.EdkLogger as EdkLogger from Common.DataType import * from CommonDataClass.DataClass import * -from Common.String import CleanString, GetSplitValueList, ReplaceMacro +from Common.StringUtils import CleanString, GetSplitValueList, ReplaceMacro import EotGlobalData -from Common.String import GetSplitList +from Common.StringUtils import GetSplitList from Common.LongFilePathSupport import OpenLongFilePath as open ## PreProcess() method diff --git a/BaseTools/Source/Python/GenFds/AprioriSection.py b/BaseTools/Source/Python/GenFds/AprioriSection.py index 7cf792b5b2..6b81b42620 100644 --- a/BaseTools/Source/Python/GenFds/AprioriSection.py +++ b/BaseTools/Source/Python/GenFds/AprioriSection.py @@ -21,7 +21,7 @@ import StringIO import FfsFileStatement from GenFdsGlobalVariable import GenFdsGlobalVariable from CommonDataClass.FdfClass import AprioriSectionClassObject -from Common.String import * +from Common.StringUtils import * from Common.Misc import SaveFileOnChange,PathClass from Common import EdkLogger from Common.BuildToolError import * diff --git a/BaseTools/Source/Python/GenFds/FdfParser.py b/BaseTools/Source/Python/GenFds/FdfParser.py index 4518d8f2af..ddc5c930d4 100644 --- a/BaseTools/Source/Python/GenFds/FdfParser.py +++ b/BaseTools/Source/Python/GenFds/FdfParser.py @@ -48,12 +48,12 @@ from GenFdsGlobalVariable import GenFdsGlobalVariable from Common.BuildToolError import * from Common import EdkLogger from Common.Misc import PathClass -from Common.String import NormPath +from Common.StringUtils import NormPath import Common.GlobalData as GlobalData from Common.Expression import * from Common import GlobalData from Common.DataType import * -from Common.String import ReplaceMacro +from Common.StringUtils import ReplaceMacro import uuid from Common.Misc import tdict from Common.MultipleWorkspace import MultipleWorkspace as mws diff --git a/BaseTools/Source/Python/GenFds/FfsInfStatement.py b/BaseTools/Source/Python/GenFds/FfsInfStatement.py index c332eee607..b26821b290 100644 --- a/BaseTools/Source/Python/GenFds/FfsInfStatement.py +++ b/BaseTools/Source/Python/GenFds/FfsInfStatement.py @@ -29,7 +29,7 @@ import RuleSimpleFile import RuleComplexFile from CommonDataClass.FdfClass import FfsInfStatementClassObject from Common.MultipleWorkspace import MultipleWorkspace as mws -from Common.String import * +from Common.StringUtils import * from Common.Misc import PathClass from Common.Misc import GuidStructureByteArrayToGuidString from Common.Misc import ProcessDuplicatedInf diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py index f0b51e25df..339b998673 100644 --- a/BaseTools/Source/Python/GenFds/GenFds.py +++ b/BaseTools/Source/Python/GenFds/GenFds.py @@ -32,7 +32,7 @@ import Common.ToolDefClassObject as ToolDefClassObject from Common.DataType import * import Common.GlobalData as GlobalData from Common import EdkLogger -from Common.String import * +from Common.StringUtils import * from Common.Misc import DirCache, PathClass from Common.Misc import SaveFileOnChange from Common.Misc import ClearDuplicatedInf diff --git a/BaseTools/Source/Python/GenFds/OptRomInfStatement.py b/BaseTools/Source/Python/GenFds/OptRomInfStatement.py index a865ac4436..79891b3b82 100644 --- a/BaseTools/Source/Python/GenFds/OptRomInfStatement.py +++ b/BaseTools/Source/Python/GenFds/OptRomInfStatement.py @@ -22,7 +22,7 @@ import OptionRom import Common.GlobalData as GlobalData from Common.DataType import * -from Common.String import * +from Common.StringUtils import * from FfsInfStatement import FfsInfStatement from GenFdsGlobalVariable import GenFdsGlobalVariable diff --git a/BaseTools/Source/Python/Makefile b/BaseTools/Source/Python/Makefile index 8466f6ebab..d78b12d505 100644 --- a/BaseTools/Source/Python/Makefile +++ b/BaseTools/Source/Python/Makefile @@ -46,7 +46,7 @@ COMMON_PYTHON=$(BASE_TOOLS_PATH)\Source\Python\Common\BuildToolError.py \ $(BASE_TOOLS_PATH)\Source\Python\Common\LongFilePathSupport.py \ $(BASE_TOOLS_PATH)\Source\Python\Common\Misc.py \ $(BASE_TOOLS_PATH)\Source\Python\Common\Parsing.py \ - $(BASE_TOOLS_PATH)\Source\Python\Common\String.py \ + $(BASE_TOOLS_PATH)\Source\Python\Common\StringUtils.py \ $(BASE_TOOLS_PATH)\Source\Python\Common\TargetTxtClassObject.py \ $(BASE_TOOLS_PATH)\Source\Python\Common\ToolDefClassObject.py \ $(BASE_TOOLS_PATH)\Source\Python\Common\VpdInfoFile.py \ @@ -167,7 +167,7 @@ CMD_UPT=$(BASE_TOOLS_PATH)\Source\Python\UPT\Core\DependencyRules.py \ $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\Misc.py \ $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\ParserValidate.py \ $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\Parsing.py \ - $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\String.py \ + $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\StringUtils.py \ $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\UniClassObject.py \ $(BASE_TOOLS_PATH)\Source\Python\UPT\Library\Xml\XmlRoutines.py \ $(BASE_TOOLS_PATH)\Source\Python\UPT\Logger\Log.py \ diff --git a/BaseTools/Source/Python/Table/TableDataModel.py b/BaseTools/Source/Python/Table/TableDataModel.py index 9c3d7bd934..2c37592fc6 100644 --- a/BaseTools/Source/Python/Table/TableDataModel.py +++ b/BaseTools/Source/Python/Table/TableDataModel.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import CommonDataClass.DataClass as DataClass from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TableDataModel # diff --git a/BaseTools/Source/Python/Table/TableDec.py b/BaseTools/Source/Python/Table/TableDec.py index 6b7d22c938..97139c58d8 100644 --- a/BaseTools/Source/Python/Table/TableDec.py +++ b/BaseTools/Source/Python/Table/TableDec.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import CommonDataClass.DataClass as DataClass from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TableDec # diff --git a/BaseTools/Source/Python/Table/TableDsc.py b/BaseTools/Source/Python/Table/TableDsc.py index 69477d544d..4ac54933aa 100644 --- a/BaseTools/Source/Python/Table/TableDsc.py +++ b/BaseTools/Source/Python/Table/TableDsc.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import CommonDataClass.DataClass as DataClass from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TableDsc # diff --git a/BaseTools/Source/Python/Table/TableEotReport.py b/BaseTools/Source/Python/Table/TableEotReport.py index 740105c8f9..bccf25ca45 100644 --- a/BaseTools/Source/Python/Table/TableEotReport.py +++ b/BaseTools/Source/Python/Table/TableEotReport.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import Common.LongFilePathOs as os, time from Table import Table -from Common.String import ConvertToSqlString2 +from Common.StringUtils import ConvertToSqlString2 import Eot.EotToolError as EotToolError import Eot.EotGlobalData as EotGlobalData diff --git a/BaseTools/Source/Python/Table/TableFdf.py b/BaseTools/Source/Python/Table/TableFdf.py index 927b5d1a3b..eea8e9404d 100644 --- a/BaseTools/Source/Python/Table/TableFdf.py +++ b/BaseTools/Source/Python/Table/TableFdf.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import CommonDataClass.DataClass as DataClass from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TableFdf # diff --git a/BaseTools/Source/Python/Table/TableFile.py b/BaseTools/Source/Python/Table/TableFile.py index caf749e9d3..ac762ea7fc 100644 --- a/BaseTools/Source/Python/Table/TableFile.py +++ b/BaseTools/Source/Python/Table/TableFile.py @@ -16,7 +16,7 @@ # import Common.EdkLogger as EdkLogger from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString import Common.LongFilePathOs as os from CommonDataClass.DataClass import FileClass diff --git a/BaseTools/Source/Python/Table/TableFunction.py b/BaseTools/Source/Python/Table/TableFunction.py index 3d7c2d0ea5..3d40bd61f6 100644 --- a/BaseTools/Source/Python/Table/TableFunction.py +++ b/BaseTools/Source/Python/Table/TableFunction.py @@ -16,7 +16,7 @@ # import Common.EdkLogger as EdkLogger from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TableFunction # diff --git a/BaseTools/Source/Python/Table/TableIdentifier.py b/BaseTools/Source/Python/Table/TableIdentifier.py index bcd6d6e1c1..0ec8b3c162 100644 --- a/BaseTools/Source/Python/Table/TableIdentifier.py +++ b/BaseTools/Source/Python/Table/TableIdentifier.py @@ -15,7 +15,7 @@ # Import Modules # import Common.EdkLogger as EdkLogger -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString from Table import Table ## TableIdentifier diff --git a/BaseTools/Source/Python/Table/TableInf.py b/BaseTools/Source/Python/Table/TableInf.py index b6e300b150..478b77776e 100644 --- a/BaseTools/Source/Python/Table/TableInf.py +++ b/BaseTools/Source/Python/Table/TableInf.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import CommonDataClass.DataClass as DataClass from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TableInf # diff --git a/BaseTools/Source/Python/Table/TablePcd.py b/BaseTools/Source/Python/Table/TablePcd.py index 19623f98f4..ca1c0f0c8a 100644 --- a/BaseTools/Source/Python/Table/TablePcd.py +++ b/BaseTools/Source/Python/Table/TablePcd.py @@ -16,7 +16,7 @@ # import Common.EdkLogger as EdkLogger from Table import Table -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString ## TablePcd # diff --git a/BaseTools/Source/Python/Table/TableQuery.py b/BaseTools/Source/Python/Table/TableQuery.py index e1d2537394..f377b10561 100644 --- a/BaseTools/Source/Python/Table/TableQuery.py +++ b/BaseTools/Source/Python/Table/TableQuery.py @@ -15,7 +15,7 @@ # Import Modules # import Common.EdkLogger as EdkLogger -from Common.String import ConvertToSqlString +from Common.StringUtils import ConvertToSqlString from Table import Table ## TableQuery diff --git a/BaseTools/Source/Python/Table/TableReport.py b/BaseTools/Source/Python/Table/TableReport.py index 4af0e98d86..9ce1d0aa25 100644 --- a/BaseTools/Source/Python/Table/TableReport.py +++ b/BaseTools/Source/Python/Table/TableReport.py @@ -17,7 +17,7 @@ import Common.EdkLogger as EdkLogger import Common.LongFilePathOs as os, time from Table import Table -from Common.String import ConvertToSqlString2 +from Common.StringUtils import ConvertToSqlString2 import EccToolError as EccToolError import EccGlobalData as EccGlobalData from Common.LongFilePathSupport import OpenLongFilePath as open diff --git a/BaseTools/Source/Python/UPT/Library/String.py b/BaseTools/Source/Python/UPT/Library/String.py deleted file mode 100644 index b79891ea14..0000000000 --- a/BaseTools/Source/Python/UPT/Library/String.py +++ /dev/null @@ -1,990 +0,0 @@ -## @file -# This file is used to define common string related functions used in parsing -# process -# -# Copyright (c) 2011 - 2016, Intel Corporation. All rights reserved.
-# -# This program and the accompanying materials are licensed and made available -# under the terms and conditions of the BSD License which accompanies this -# distribution. The full text of the license may be found at -# http://opensource.org/licenses/bsd-license.php -# -# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, -# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. -# -''' -String -''' -## -# Import Modules -# -import re -import os.path -from string import strip -import Logger.Log as Logger -import Library.DataType as DataType -from Logger.ToolError import FORMAT_INVALID -from Logger.ToolError import PARSER_ERROR -from Logger import StringTable as ST - -# -# Regular expression for matching macro used in DSC/DEC/INF file inclusion -# -gMACRO_PATTERN = re.compile("\$\(([_A-Z][_A-Z0-9]*)\)", re.UNICODE) - -## GetSplitValueList -# -# Get a value list from a string with multiple values splited with SplitTag -# The default SplitTag is DataType.TAB_VALUE_SPLIT -# 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC'] -# -# @param String: The input string to be splitted -# @param SplitTag: The split key, default is DataType.TAB_VALUE_SPLIT -# @param MaxSplit: The max number of split values, default is -1 -# -# -def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): - return map(lambda l: l.strip(), String.split(SplitTag, MaxSplit)) - -## MergeArches -# -# Find a key's all arches in dict, add the new arch to the list -# If not exist any arch, set the arch directly -# -# @param Dict: The input value for Dict -# @param Key: The input value for Key -# @param Arch: The Arch to be added or merged -# -def MergeArches(Dict, Key, Arch): - if Key in Dict.keys(): - Dict[Key].append(Arch) - else: - Dict[Key] = Arch.split() - -## GenDefines -# -# Parse a string with format "DEFINE = " -# Generate a map Defines[VarName] = PATH -# Return False if invalid format -# -# @param String: String with DEFINE statement -# @param Arch: Supportted Arch -# @param Defines: DEFINE statement to be parsed -# -def GenDefines(String, Arch, Defines): - if String.find(DataType.TAB_DEFINE + ' ') > -1: - List = String.replace(DataType.TAB_DEFINE + ' ', '').\ - split(DataType.TAB_EQUAL_SPLIT) - if len(List) == 2: - Defines[(CleanString(List[0]), Arch)] = CleanString(List[1]) - return 0 - else: - return -1 - return 1 - -## GetLibraryClassesWithModuleType -# -# Get Library Class definition when no module type defined -# -# @param Lines: The content to be parsed -# @param Key: Reserved -# @param KeyValues: To store data after parsing -# @param CommentCharacter: Comment char, used to ignore comment content -# -def GetLibraryClassesWithModuleType(Lines, Key, KeyValues, CommentCharacter): - NewKey = SplitModuleType(Key) - Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] - LineList = Lines.splitlines() - for Line in LineList: - Line = CleanString(Line, CommentCharacter) - if Line != '' and Line[0] != CommentCharacter: - KeyValues.append([CleanString(Line, CommentCharacter), NewKey[1]]) - - return True - -## GetDynamics -# -# Get Dynamic Pcds -# -# @param Lines: The content to be parsed -# @param Key: Reserved -# @param KeyValues: To store data after parsing -# @param CommentCharacter: Comment char, used to ignore comment content -# -def GetDynamics(Lines, Key, KeyValues, CommentCharacter): - # - # Get SkuId Name List - # - SkuIdNameList = SplitModuleType(Key) - - Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] - LineList = Lines.splitlines() - for Line in LineList: - Line = CleanString(Line, CommentCharacter) - if Line != '' and Line[0] != CommentCharacter: - KeyValues.append([CleanString(Line, CommentCharacter), SkuIdNameList[1]]) - - return True - -## SplitModuleType -# -# Split ModuleType out of section defien to get key -# [LibraryClass.Arch.ModuleType|ModuleType|ModuleType] -> [ -# 'LibraryClass.Arch', ['ModuleType', 'ModuleType', 'ModuleType'] ] -# -# @param Key: String to be parsed -# -def SplitModuleType(Key): - KeyList = Key.split(DataType.TAB_SPLIT) - # - # Fill in for arch - # - KeyList.append('') - # - # Fill in for moduletype - # - KeyList.append('') - ReturnValue = [] - KeyValue = KeyList[0] - if KeyList[1] != '': - KeyValue = KeyValue + DataType.TAB_SPLIT + KeyList[1] - ReturnValue.append(KeyValue) - ReturnValue.append(GetSplitValueList(KeyList[2])) - - return ReturnValue - -## Replace macro in string -# -# This method replace macros used in given string. The macros are given in a -# dictionary. -# -# @param String String to be processed -# @param MacroDefinitions The macro definitions in the form of dictionary -# @param SelfReplacement To decide whether replace un-defined macro to '' -# @param Line: The content contain line string and line number -# @param FileName: The meta-file file name -# -def ReplaceMacro(String, MacroDefinitions=None, SelfReplacement=False, Line=None, FileName=None, Flag=False): - LastString = String - if MacroDefinitions is None: - MacroDefinitions = {} - while MacroDefinitions: - QuotedStringList = [] - HaveQuotedMacroFlag = False - if not Flag: - MacroUsed = gMACRO_PATTERN.findall(String) - else: - ReQuotedString = re.compile('\"') - QuotedStringList = ReQuotedString.split(String) - if len(QuotedStringList) >= 3: - HaveQuotedMacroFlag = True - Count = 0 - MacroString = "" - for QuotedStringItem in QuotedStringList: - Count += 1 - if Count % 2 != 0: - MacroString += QuotedStringItem - - if Count == len(QuotedStringList) and Count % 2 == 0: - MacroString += QuotedStringItem - - MacroUsed = gMACRO_PATTERN.findall(MacroString) - # - # no macro found in String, stop replacing - # - if len(MacroUsed) == 0: - break - for Macro in MacroUsed: - if Macro not in MacroDefinitions: - if SelfReplacement: - String = String.replace("$(%s)" % Macro, '') - Logger.Debug(5, "Delete undefined MACROs in file %s line %d: %s!" % (FileName, Line[1], Line[0])) - continue - if not HaveQuotedMacroFlag: - String = String.replace("$(%s)" % Macro, MacroDefinitions[Macro]) - else: - Count = 0 - for QuotedStringItem in QuotedStringList: - Count += 1 - if Count % 2 != 0: - QuotedStringList[Count - 1] = QuotedStringList[Count - 1].replace("$(%s)" % Macro, - MacroDefinitions[Macro]) - elif Count == len(QuotedStringList) and Count % 2 == 0: - QuotedStringList[Count - 1] = QuotedStringList[Count - 1].replace("$(%s)" % Macro, - MacroDefinitions[Macro]) - - RetString = '' - if HaveQuotedMacroFlag: - Count = 0 - for QuotedStringItem in QuotedStringList: - Count += 1 - if Count != len(QuotedStringList): - RetString += QuotedStringList[Count - 1] + "\"" - else: - RetString += QuotedStringList[Count - 1] - - String = RetString - - # - # in case there's macro not defined - # - if String == LastString: - break - LastString = String - - return String - -## NormPath -# -# Create a normal path -# And replace DFEINE in the path -# -# @param Path: The input value for Path to be converted -# @param Defines: A set for DEFINE statement -# -def NormPath(Path, Defines=None): - IsRelativePath = False - if Defines is None: - Defines = {} - if Path: - if Path[0] == '.': - IsRelativePath = True - # - # Replace with Define - # - if Defines: - Path = ReplaceMacro(Path, Defines) - # - # To local path format - # - Path = os.path.normpath(Path) - - if IsRelativePath and Path[0] != '.': - Path = os.path.join('.', Path) - return Path - -## CleanString -# -# Remove comments in a string -# Remove spaces -# -# @param Line: The string to be cleaned -# @param CommentCharacter: Comment char, used to ignore comment content, -# default is DataType.TAB_COMMENT_SPLIT -# -def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False): - # - # remove whitespace - # - Line = Line.strip() - # - # Replace EDK1's comment character - # - if AllowCppStyleComment: - Line = Line.replace(DataType.TAB_COMMENT_EDK1_SPLIT, CommentCharacter) - # - # remove comments, but we should escape comment character in string - # - InString = False - for Index in range(0, len(Line)): - if Line[Index] == '"': - InString = not InString - elif Line[Index] == CommentCharacter and not InString: - Line = Line[0: Index] - break - # - # remove whitespace again - # - Line = Line.strip() - - return Line - -## CleanString2 -# -# Split comments in a string -# Remove spaces -# -# @param Line: The string to be cleaned -# @param CommentCharacter: Comment char, used to ignore comment content, -# default is DataType.TAB_COMMENT_SPLIT -# -def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False): - # - # remove whitespace - # - Line = Line.strip() - # - # Replace EDK1's comment character - # - if AllowCppStyleComment: - Line = Line.replace(DataType.TAB_COMMENT_EDK1_SPLIT, CommentCharacter) - # - # separate comments and statements - # - LineParts = Line.split(CommentCharacter, 1) - # - # remove whitespace again - # - Line = LineParts[0].strip() - if len(LineParts) > 1: - Comment = LineParts[1].strip() - # - # Remove prefixed and trailing comment characters - # - Start = 0 - End = len(Comment) - while Start < End and Comment.startswith(CommentCharacter, Start, End): - Start += 1 - while End >= 0 and Comment.endswith(CommentCharacter, Start, End): - End -= 1 - Comment = Comment[Start:End] - Comment = Comment.strip() - else: - Comment = '' - - return Line, Comment - -## GetMultipleValuesOfKeyFromLines -# -# Parse multiple strings to clean comment and spaces -# The result is saved to KeyValues -# -# @param Lines: The content to be parsed -# @param Key: Reserved -# @param KeyValues: To store data after parsing -# @param CommentCharacter: Comment char, used to ignore comment content -# -def GetMultipleValuesOfKeyFromLines(Lines, Key, KeyValues, CommentCharacter): - if Key: - pass - if KeyValues: - pass - Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] - LineList = Lines.split('\n') - for Line in LineList: - Line = CleanString(Line, CommentCharacter) - if Line != '' and Line[0] != CommentCharacter: - KeyValues += [Line] - return True - -## GetDefineValue -# -# Parse a DEFINE statement to get defined value -# DEFINE Key Value -# -# @param String: The content to be parsed -# @param Key: The key of DEFINE statement -# @param CommentCharacter: Comment char, used to ignore comment content -# -def GetDefineValue(String, Key, CommentCharacter): - if CommentCharacter: - pass - String = CleanString(String) - return String[String.find(Key + ' ') + len(Key + ' ') : ] - -## GetSingleValueOfKeyFromLines -# -# Parse multiple strings as below to get value of each definition line -# Key1 = Value1 -# Key2 = Value2 -# The result is saved to Dictionary -# -# @param Lines: The content to be parsed -# @param Dictionary: To store data after parsing -# @param CommentCharacter: Comment char, be used to ignore comment content -# @param KeySplitCharacter: Key split char, between key name and key value. -# Key1 = Value1, '=' is the key split char -# @param ValueSplitFlag: Value split flag, be used to decide if has -# multiple values -# @param ValueSplitCharacter: Value split char, be used to split multiple -# values. Key1 = Value1|Value2, '|' is the value -# split char -# -def GetSingleValueOfKeyFromLines(Lines, Dictionary, CommentCharacter, KeySplitCharacter, \ - ValueSplitFlag, ValueSplitCharacter): - Lines = Lines.split('\n') - Keys = [] - Value = '' - DefineValues = [''] - SpecValues = [''] - - for Line in Lines: - # - # Handle DEFINE and SPEC - # - if Line.find(DataType.TAB_INF_DEFINES_DEFINE + ' ') > -1: - if '' in DefineValues: - DefineValues.remove('') - DefineValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_DEFINE, CommentCharacter)) - continue - if Line.find(DataType.TAB_INF_DEFINES_SPEC + ' ') > -1: - if '' in SpecValues: - SpecValues.remove('') - SpecValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_SPEC, CommentCharacter)) - continue - - # - # Handle Others - # - LineList = Line.split(KeySplitCharacter, 1) - if len(LineList) >= 2: - Key = LineList[0].split() - if len(Key) == 1 and Key[0][0] != CommentCharacter: - # - # Remove comments and white spaces - # - LineList[1] = CleanString(LineList[1], CommentCharacter) - if ValueSplitFlag: - Value = map(strip, LineList[1].split(ValueSplitCharacter)) - else: - Value = CleanString(LineList[1], CommentCharacter).splitlines() - - if Key[0] in Dictionary: - if Key[0] not in Keys: - Dictionary[Key[0]] = Value - Keys.append(Key[0]) - else: - Dictionary[Key[0]].extend(Value) - else: - Dictionary[DataType.TAB_INF_DEFINES_MACRO][Key[0]] = Value[0] - - if DefineValues == []: - DefineValues = [''] - if SpecValues == []: - SpecValues = [''] - Dictionary[DataType.TAB_INF_DEFINES_DEFINE] = DefineValues - Dictionary[DataType.TAB_INF_DEFINES_SPEC] = SpecValues - - return True - -## The content to be parsed -# -# Do pre-check for a file before it is parsed -# Check $() -# Check [] -# -# @param FileName: Used for error report -# @param FileContent: File content to be parsed -# @param SupSectionTag: Used for error report -# -def PreCheck(FileName, FileContent, SupSectionTag): - if SupSectionTag: - pass - LineNo = 0 - IsFailed = False - NewFileContent = '' - for Line in FileContent.splitlines(): - LineNo = LineNo + 1 - # - # Clean current line - # - Line = CleanString(Line) - # - # Remove commented line - # - if Line.find(DataType.TAB_COMMA_SPLIT) == 0: - Line = '' - # - # Check $() - # - if Line.find('$') > -1: - if Line.find('$(') < 0 or Line.find(')') < 0: - Logger.Error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=Logger.IS_RAISE_ERROR) - # - # Check [] - # - if Line.find('[') > -1 or Line.find(']') > -1: - # - # Only get one '[' or one ']' - # - if not (Line.find('[') > -1 and Line.find(']') > -1): - Logger.Error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=Logger.IS_RAISE_ERROR) - # - # Regenerate FileContent - # - NewFileContent = NewFileContent + Line + '\r\n' - - if IsFailed: - Logger.Error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=Logger.IS_RAISE_ERROR) - - return NewFileContent - -## CheckFileType -# -# Check if the Filename is including ExtName -# Return True if it exists -# Raise a error message if it not exists -# -# @param CheckFilename: Name of the file to be checked -# @param ExtName: Ext name of the file to be checked -# @param ContainerFilename: The container file which describes the file to be -# checked, used for error report -# @param SectionName: Used for error report -# @param Line: The line in container file which defines the file -# to be checked -# -def CheckFileType(CheckFilename, ExtName, ContainerFilename, SectionName, Line, LineNo= -1): - if CheckFilename != '' and CheckFilename is not None: - (Root, Ext) = os.path.splitext(CheckFilename) - if Ext.upper() != ExtName.upper() and Root: - ContainerFile = open(ContainerFilename, 'r').read() - if LineNo == -1: - LineNo = GetLineNo(ContainerFile, Line) - ErrorMsg = ST.ERR_SECTIONNAME_INVALID % (SectionName, CheckFilename, ExtName) - Logger.Error("Parser", PARSER_ERROR, ErrorMsg, Line=LineNo, \ - File=ContainerFilename, RaiseError=Logger.IS_RAISE_ERROR) - - return True - -## CheckFileExist -# -# Check if the file exists -# Return True if it exists -# Raise a error message if it not exists -# -# @param CheckFilename: Name of the file to be checked -# @param WorkspaceDir: Current workspace dir -# @param ContainerFilename: The container file which describes the file to -# be checked, used for error report -# @param SectionName: Used for error report -# @param Line: The line in container file which defines the -# file to be checked -# -def CheckFileExist(WorkspaceDir, CheckFilename, ContainerFilename, SectionName, Line, LineNo= -1): - CheckFile = '' - if CheckFilename != '' and CheckFilename is not None: - CheckFile = WorkspaceFile(WorkspaceDir, CheckFilename) - if not os.path.isfile(CheckFile): - ContainerFile = open(ContainerFilename, 'r').read() - if LineNo == -1: - LineNo = GetLineNo(ContainerFile, Line) - ErrorMsg = ST.ERR_CHECKFILE_NOTFOUND % (CheckFile, SectionName) - Logger.Error("Parser", PARSER_ERROR, ErrorMsg, - File=ContainerFilename, Line=LineNo, RaiseError=Logger.IS_RAISE_ERROR) - return CheckFile - -## GetLineNo -# -# Find the index of a line in a file -# -# @param FileContent: Search scope -# @param Line: Search key -# -def GetLineNo(FileContent, Line, IsIgnoreComment=True): - LineList = FileContent.splitlines() - for Index in range(len(LineList)): - if LineList[Index].find(Line) > -1: - # - # Ignore statement in comment - # - if IsIgnoreComment: - if LineList[Index].strip()[0] == DataType.TAB_COMMENT_SPLIT: - continue - return Index + 1 - - return -1 - -## RaiseParserError -# -# Raise a parser error -# -# @param Line: String which has error -# @param Section: Used for error report -# @param File: File which has the string -# @param Format: Correct format -# -def RaiseParserError(Line, Section, File, Format='', LineNo= -1): - if LineNo == -1: - LineNo = GetLineNo(open(os.path.normpath(File), 'r').read(), Line) - ErrorMsg = ST.ERR_INVALID_NOTFOUND % (Line, Section) - if Format != '': - Format = "Correct format is " + Format - Logger.Error("Parser", PARSER_ERROR, ErrorMsg, File=File, Line=LineNo, \ - ExtraData=Format, RaiseError=Logger.IS_RAISE_ERROR) - -## WorkspaceFile -# -# Return a full path with workspace dir -# -# @param WorkspaceDir: Workspace dir -# @param Filename: Relative file name -# -def WorkspaceFile(WorkspaceDir, Filename): - return os.path.join(NormPath(WorkspaceDir), NormPath(Filename)) - -## Split string -# -# Revmove '"' which startswith and endswith string -# -# @param String: The string need to be splited -# -def SplitString(String): - if String.startswith('\"'): - String = String[1:] - if String.endswith('\"'): - String = String[:-1] - return String - -## Convert To Sql String -# -# Replace "'" with "''" in each item of StringList -# -# @param StringList: A list for strings to be converted -# -def ConvertToSqlString(StringList): - return map(lambda s: s.replace("'", "''") , StringList) - -## Convert To Sql String -# -# Replace "'" with "''" in the String -# -# @param String: A String to be converted -# -def ConvertToSqlString2(String): - return String.replace("'", "''") - -## GetStringOfList -# -# Get String of a List -# -# @param Lines: string list -# @param Split: split character -# -def GetStringOfList(List, Split=' '): - if type(List) != type([]): - return List - Str = '' - for Item in List: - Str = Str + Item + Split - return Str.strip() - -## Get HelpTextList -# -# Get HelpTextList from HelpTextClassList -# -# @param HelpTextClassList: Help Text Class List -# -def GetHelpTextList(HelpTextClassList): - List = [] - if HelpTextClassList: - for HelpText in HelpTextClassList: - if HelpText.String.endswith('\n'): - HelpText.String = HelpText.String[0: len(HelpText.String) - len('\n')] - List.extend(HelpText.String.split('\n')) - return List - -## Get String Array Length -# -# Get String Array Length -# -# @param String: the source string -# -def StringArrayLength(String): - if isinstance(String, unicode): - return (len(String) + 1) * 2 + 1 - elif String.startswith('L"'): - return (len(String) - 3 + 1) * 2 - elif String.startswith('"'): - return (len(String) - 2 + 1) - else: - return len(String.split()) + 1 - -## RemoveDupOption -# -# Remove Dup Option -# -# @param OptionString: the option string -# @param Which: Which flag -# @param Against: Against flag -# -def RemoveDupOption(OptionString, Which="/I", Against=None): - OptionList = OptionString.split() - ValueList = [] - if Against: - ValueList += Against - for Index in range(len(OptionList)): - Opt = OptionList[Index] - if not Opt.startswith(Which): - continue - if len(Opt) > len(Which): - Val = Opt[len(Which):] - else: - Val = "" - if Val in ValueList: - OptionList[Index] = "" - else: - ValueList.append(Val) - return " ".join(OptionList) - -## Check if the string is HexDgit -# -# Return true if all characters in the string are digits and there is at -# least one character -# or valid Hexs (started with 0x, following by hexdigit letters) -# , false otherwise. -# @param string: input string -# -def IsHexDigit(Str): - try: - int(Str, 10) - return True - except ValueError: - if len(Str) > 2 and Str.upper().startswith('0X'): - try: - int(Str, 16) - return True - except ValueError: - return False - return False - -## Check if the string is HexDgit and its interger value within limit of UINT32 -# -# Return true if all characters in the string are digits and there is at -# least one character -# or valid Hexs (started with 0x, following by hexdigit letters) -# , false otherwise. -# @param string: input string -# -def IsHexDigitUINT32(Str): - try: - Value = int(Str, 10) - if (Value <= 0xFFFFFFFF) and (Value >= 0): - return True - except ValueError: - if len(Str) > 2 and Str.upper().startswith('0X'): - try: - Value = int(Str, 16) - if (Value <= 0xFFFFFFFF) and (Value >= 0): - return True - except ValueError: - return False - return False - -## CleanSpecialChar -# -# The ASCII text files of type INF, DEC, INI are edited by developers, -# and may contain characters that cannot be directly translated to strings that -# are conformant with the UDP XML Schema. Any characters in this category -# (0x00-0x08, TAB [0x09], 0x0B, 0x0C, 0x0E-0x1F, 0x80-0xFF) -# must be converted to a space character[0x20] as part of the parsing process. -# -def ConvertSpecialChar(Lines): - RetLines = [] - for line in Lines: - ReMatchSpecialChar = re.compile(r"[\x00-\x08]|\x09|\x0b|\x0c|[\x0e-\x1f]|[\x7f-\xff]") - RetLines.append(ReMatchSpecialChar.sub(' ', line)) - - return RetLines - -## __GetTokenList -# -# Assume Str is a valid feature flag expression. -# Return a list which contains tokens: alpha numeric token and other token -# Whitespace are not stripped -# -def __GetTokenList(Str): - InQuote = False - Token = '' - TokenOP = '' - PreChar = '' - List = [] - for Char in Str: - if InQuote: - Token += Char - if Char == '"' and PreChar != '\\': - InQuote = not InQuote - List.append(Token) - Token = '' - continue - if Char == '"': - if Token and Token != 'L': - List.append(Token) - Token = '' - if TokenOP: - List.append(TokenOP) - TokenOP = '' - InQuote = not InQuote - Token += Char - continue - - if not (Char.isalnum() or Char in '_'): - TokenOP += Char - if Token: - List.append(Token) - Token = '' - else: - Token += Char - if TokenOP: - List.append(TokenOP) - TokenOP = '' - - if PreChar == '\\' and Char == '\\': - PreChar = '' - else: - PreChar = Char - if Token: - List.append(Token) - if TokenOP: - List.append(TokenOP) - return List - -## ConvertNEToNOTEQ -# -# Convert NE operator to NOT EQ -# For example: 1 NE 2 -> 1 NOT EQ 2 -# -# @param Expr: Feature flag expression to be converted -# -def ConvertNEToNOTEQ(Expr): - List = __GetTokenList(Expr) - for Index in range(len(List)): - if List[Index] == 'NE': - List[Index] = 'NOT EQ' - return ''.join(List) - -## ConvertNOTEQToNE -# -# Convert NOT EQ operator to NE -# For example: 1 NOT NE 2 -> 1 NE 2 -# -# @param Expr: Feature flag expression to be converted -# -def ConvertNOTEQToNE(Expr): - List = __GetTokenList(Expr) - HasNOT = False - RetList = [] - for Token in List: - if HasNOT and Token == 'EQ': - # At least, 'NOT' is in the list - while not RetList[-1].strip(): - RetList.pop() - RetList[-1] = 'NE' - HasNOT = False - continue - if Token == 'NOT': - HasNOT = True - elif Token.strip(): - HasNOT = False - RetList.append(Token) - - return ''.join(RetList) - -## SplitPcdEntry -# -# Split an PCD entry string to Token.CName and PCD value and FFE. -# NOTE: PCD Value and FFE can contain "|" in it's expression. And in INF specification, have below rule. -# When using the characters "|" or "||" in an expression, the expression must be encapsulated in -# open "(" and close ")" parenthesis. -# -# @param String An PCD entry string need to be split. -# -# @return List [PcdTokenCName, Value, FFE] -# -def SplitPcdEntry(String): - if not String: - return ['', '', ''], False - - PcdTokenCName = '' - PcdValue = '' - PcdFeatureFlagExp = '' - - ValueList = GetSplitValueList(String, "|", 1) - - # - # Only contain TokenCName - # - if len(ValueList) == 1: - return [ValueList[0]], True - - NewValueList = [] - - if len(ValueList) == 2: - PcdTokenCName = ValueList[0] - - InQuote = False - InParenthesis = False - StrItem = '' - for StrCh in ValueList[1]: - if StrCh == '"': - InQuote = not InQuote - elif StrCh == '(' or StrCh == ')': - InParenthesis = not InParenthesis - - if StrCh == '|': - if not InQuote or not InParenthesis: - NewValueList.append(StrItem.strip()) - StrItem = ' ' - continue - - StrItem += StrCh - - NewValueList.append(StrItem.strip()) - - if len(NewValueList) == 1: - PcdValue = NewValueList[0] - return [PcdTokenCName, PcdValue], True - elif len(NewValueList) == 2: - PcdValue = NewValueList[0] - PcdFeatureFlagExp = NewValueList[1] - return [PcdTokenCName, PcdValue, PcdFeatureFlagExp], True - else: - return ['', '', ''], False - - return ['', '', ''], False - -## Check if two arches matched? -# -# @param Arch1 -# @param Arch2 -# -def IsMatchArch(Arch1, Arch2): - if 'COMMON' in Arch1 or 'COMMON' in Arch2: - return True - if isinstance(Arch1, basestring) and isinstance(Arch2, basestring): - if Arch1 == Arch2: - return True - - if isinstance(Arch1, basestring) and isinstance(Arch2, list): - return Arch1 in Arch2 - - if isinstance(Arch2, basestring) and isinstance(Arch1, list): - return Arch2 in Arch1 - - if isinstance(Arch1, list) and isinstance(Arch2, list): - for Item1 in Arch1: - for Item2 in Arch2: - if Item1 == Item2: - return True - - return False - -# Search all files in FilePath to find the FileName with the largest index -# Return the FileName with index +1 under the FilePath -# -def GetUniFileName(FilePath, FileName): - Files = [] - try: - Files = os.listdir(FilePath) - except: - pass - - LargestIndex = -1 - IndexNotFound = True - for File in Files: - if File.upper().startswith(FileName.upper()) and File.upper().endswith('.UNI'): - Index = File.upper().replace(FileName.upper(), '').replace('.UNI', '') - if Index: - try: - Index = int(Index) - except Exception: - Index = -1 - else: - IndexNotFound = False - Index = 0 - if Index > LargestIndex: - LargestIndex = Index + 1 - - if LargestIndex > -1 and not IndexNotFound: - return os.path.normpath(os.path.join(FilePath, FileName + str(LargestIndex) + '.uni')) - else: - return os.path.normpath(os.path.join(FilePath, FileName + '.uni')) diff --git a/BaseTools/Source/Python/UPT/Library/StringUtils.py b/BaseTools/Source/Python/UPT/Library/StringUtils.py new file mode 100644 index 0000000000..b79891ea14 --- /dev/null +++ b/BaseTools/Source/Python/UPT/Library/StringUtils.py @@ -0,0 +1,990 @@ +## @file +# This file is used to define common string related functions used in parsing +# process +# +# Copyright (c) 2011 - 2016, Intel Corporation. All rights reserved.
+# +# This program and the accompanying materials are licensed and made available +# under the terms and conditions of the BSD License which accompanies this +# distribution. The full text of the license may be found at +# http://opensource.org/licenses/bsd-license.php +# +# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +# +''' +String +''' +## +# Import Modules +# +import re +import os.path +from string import strip +import Logger.Log as Logger +import Library.DataType as DataType +from Logger.ToolError import FORMAT_INVALID +from Logger.ToolError import PARSER_ERROR +from Logger import StringTable as ST + +# +# Regular expression for matching macro used in DSC/DEC/INF file inclusion +# +gMACRO_PATTERN = re.compile("\$\(([_A-Z][_A-Z0-9]*)\)", re.UNICODE) + +## GetSplitValueList +# +# Get a value list from a string with multiple values splited with SplitTag +# The default SplitTag is DataType.TAB_VALUE_SPLIT +# 'AAA|BBB|CCC' -> ['AAA', 'BBB', 'CCC'] +# +# @param String: The input string to be splitted +# @param SplitTag: The split key, default is DataType.TAB_VALUE_SPLIT +# @param MaxSplit: The max number of split values, default is -1 +# +# +def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1): + return map(lambda l: l.strip(), String.split(SplitTag, MaxSplit)) + +## MergeArches +# +# Find a key's all arches in dict, add the new arch to the list +# If not exist any arch, set the arch directly +# +# @param Dict: The input value for Dict +# @param Key: The input value for Key +# @param Arch: The Arch to be added or merged +# +def MergeArches(Dict, Key, Arch): + if Key in Dict.keys(): + Dict[Key].append(Arch) + else: + Dict[Key] = Arch.split() + +## GenDefines +# +# Parse a string with format "DEFINE = " +# Generate a map Defines[VarName] = PATH +# Return False if invalid format +# +# @param String: String with DEFINE statement +# @param Arch: Supportted Arch +# @param Defines: DEFINE statement to be parsed +# +def GenDefines(String, Arch, Defines): + if String.find(DataType.TAB_DEFINE + ' ') > -1: + List = String.replace(DataType.TAB_DEFINE + ' ', '').\ + split(DataType.TAB_EQUAL_SPLIT) + if len(List) == 2: + Defines[(CleanString(List[0]), Arch)] = CleanString(List[1]) + return 0 + else: + return -1 + return 1 + +## GetLibraryClassesWithModuleType +# +# Get Library Class definition when no module type defined +# +# @param Lines: The content to be parsed +# @param Key: Reserved +# @param KeyValues: To store data after parsing +# @param CommentCharacter: Comment char, used to ignore comment content +# +def GetLibraryClassesWithModuleType(Lines, Key, KeyValues, CommentCharacter): + NewKey = SplitModuleType(Key) + Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] + LineList = Lines.splitlines() + for Line in LineList: + Line = CleanString(Line, CommentCharacter) + if Line != '' and Line[0] != CommentCharacter: + KeyValues.append([CleanString(Line, CommentCharacter), NewKey[1]]) + + return True + +## GetDynamics +# +# Get Dynamic Pcds +# +# @param Lines: The content to be parsed +# @param Key: Reserved +# @param KeyValues: To store data after parsing +# @param CommentCharacter: Comment char, used to ignore comment content +# +def GetDynamics(Lines, Key, KeyValues, CommentCharacter): + # + # Get SkuId Name List + # + SkuIdNameList = SplitModuleType(Key) + + Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] + LineList = Lines.splitlines() + for Line in LineList: + Line = CleanString(Line, CommentCharacter) + if Line != '' and Line[0] != CommentCharacter: + KeyValues.append([CleanString(Line, CommentCharacter), SkuIdNameList[1]]) + + return True + +## SplitModuleType +# +# Split ModuleType out of section defien to get key +# [LibraryClass.Arch.ModuleType|ModuleType|ModuleType] -> [ +# 'LibraryClass.Arch', ['ModuleType', 'ModuleType', 'ModuleType'] ] +# +# @param Key: String to be parsed +# +def SplitModuleType(Key): + KeyList = Key.split(DataType.TAB_SPLIT) + # + # Fill in for arch + # + KeyList.append('') + # + # Fill in for moduletype + # + KeyList.append('') + ReturnValue = [] + KeyValue = KeyList[0] + if KeyList[1] != '': + KeyValue = KeyValue + DataType.TAB_SPLIT + KeyList[1] + ReturnValue.append(KeyValue) + ReturnValue.append(GetSplitValueList(KeyList[2])) + + return ReturnValue + +## Replace macro in string +# +# This method replace macros used in given string. The macros are given in a +# dictionary. +# +# @param String String to be processed +# @param MacroDefinitions The macro definitions in the form of dictionary +# @param SelfReplacement To decide whether replace un-defined macro to '' +# @param Line: The content contain line string and line number +# @param FileName: The meta-file file name +# +def ReplaceMacro(String, MacroDefinitions=None, SelfReplacement=False, Line=None, FileName=None, Flag=False): + LastString = String + if MacroDefinitions is None: + MacroDefinitions = {} + while MacroDefinitions: + QuotedStringList = [] + HaveQuotedMacroFlag = False + if not Flag: + MacroUsed = gMACRO_PATTERN.findall(String) + else: + ReQuotedString = re.compile('\"') + QuotedStringList = ReQuotedString.split(String) + if len(QuotedStringList) >= 3: + HaveQuotedMacroFlag = True + Count = 0 + MacroString = "" + for QuotedStringItem in QuotedStringList: + Count += 1 + if Count % 2 != 0: + MacroString += QuotedStringItem + + if Count == len(QuotedStringList) and Count % 2 == 0: + MacroString += QuotedStringItem + + MacroUsed = gMACRO_PATTERN.findall(MacroString) + # + # no macro found in String, stop replacing + # + if len(MacroUsed) == 0: + break + for Macro in MacroUsed: + if Macro not in MacroDefinitions: + if SelfReplacement: + String = String.replace("$(%s)" % Macro, '') + Logger.Debug(5, "Delete undefined MACROs in file %s line %d: %s!" % (FileName, Line[1], Line[0])) + continue + if not HaveQuotedMacroFlag: + String = String.replace("$(%s)" % Macro, MacroDefinitions[Macro]) + else: + Count = 0 + for QuotedStringItem in QuotedStringList: + Count += 1 + if Count % 2 != 0: + QuotedStringList[Count - 1] = QuotedStringList[Count - 1].replace("$(%s)" % Macro, + MacroDefinitions[Macro]) + elif Count == len(QuotedStringList) and Count % 2 == 0: + QuotedStringList[Count - 1] = QuotedStringList[Count - 1].replace("$(%s)" % Macro, + MacroDefinitions[Macro]) + + RetString = '' + if HaveQuotedMacroFlag: + Count = 0 + for QuotedStringItem in QuotedStringList: + Count += 1 + if Count != len(QuotedStringList): + RetString += QuotedStringList[Count - 1] + "\"" + else: + RetString += QuotedStringList[Count - 1] + + String = RetString + + # + # in case there's macro not defined + # + if String == LastString: + break + LastString = String + + return String + +## NormPath +# +# Create a normal path +# And replace DFEINE in the path +# +# @param Path: The input value for Path to be converted +# @param Defines: A set for DEFINE statement +# +def NormPath(Path, Defines=None): + IsRelativePath = False + if Defines is None: + Defines = {} + if Path: + if Path[0] == '.': + IsRelativePath = True + # + # Replace with Define + # + if Defines: + Path = ReplaceMacro(Path, Defines) + # + # To local path format + # + Path = os.path.normpath(Path) + + if IsRelativePath and Path[0] != '.': + Path = os.path.join('.', Path) + return Path + +## CleanString +# +# Remove comments in a string +# Remove spaces +# +# @param Line: The string to be cleaned +# @param CommentCharacter: Comment char, used to ignore comment content, +# default is DataType.TAB_COMMENT_SPLIT +# +def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False): + # + # remove whitespace + # + Line = Line.strip() + # + # Replace EDK1's comment character + # + if AllowCppStyleComment: + Line = Line.replace(DataType.TAB_COMMENT_EDK1_SPLIT, CommentCharacter) + # + # remove comments, but we should escape comment character in string + # + InString = False + for Index in range(0, len(Line)): + if Line[Index] == '"': + InString = not InString + elif Line[Index] == CommentCharacter and not InString: + Line = Line[0: Index] + break + # + # remove whitespace again + # + Line = Line.strip() + + return Line + +## CleanString2 +# +# Split comments in a string +# Remove spaces +# +# @param Line: The string to be cleaned +# @param CommentCharacter: Comment char, used to ignore comment content, +# default is DataType.TAB_COMMENT_SPLIT +# +def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyleComment=False): + # + # remove whitespace + # + Line = Line.strip() + # + # Replace EDK1's comment character + # + if AllowCppStyleComment: + Line = Line.replace(DataType.TAB_COMMENT_EDK1_SPLIT, CommentCharacter) + # + # separate comments and statements + # + LineParts = Line.split(CommentCharacter, 1) + # + # remove whitespace again + # + Line = LineParts[0].strip() + if len(LineParts) > 1: + Comment = LineParts[1].strip() + # + # Remove prefixed and trailing comment characters + # + Start = 0 + End = len(Comment) + while Start < End and Comment.startswith(CommentCharacter, Start, End): + Start += 1 + while End >= 0 and Comment.endswith(CommentCharacter, Start, End): + End -= 1 + Comment = Comment[Start:End] + Comment = Comment.strip() + else: + Comment = '' + + return Line, Comment + +## GetMultipleValuesOfKeyFromLines +# +# Parse multiple strings to clean comment and spaces +# The result is saved to KeyValues +# +# @param Lines: The content to be parsed +# @param Key: Reserved +# @param KeyValues: To store data after parsing +# @param CommentCharacter: Comment char, used to ignore comment content +# +def GetMultipleValuesOfKeyFromLines(Lines, Key, KeyValues, CommentCharacter): + if Key: + pass + if KeyValues: + pass + Lines = Lines.split(DataType.TAB_SECTION_END, 1)[1] + LineList = Lines.split('\n') + for Line in LineList: + Line = CleanString(Line, CommentCharacter) + if Line != '' and Line[0] != CommentCharacter: + KeyValues += [Line] + return True + +## GetDefineValue +# +# Parse a DEFINE statement to get defined value +# DEFINE Key Value +# +# @param String: The content to be parsed +# @param Key: The key of DEFINE statement +# @param CommentCharacter: Comment char, used to ignore comment content +# +def GetDefineValue(String, Key, CommentCharacter): + if CommentCharacter: + pass + String = CleanString(String) + return String[String.find(Key + ' ') + len(Key + ' ') : ] + +## GetSingleValueOfKeyFromLines +# +# Parse multiple strings as below to get value of each definition line +# Key1 = Value1 +# Key2 = Value2 +# The result is saved to Dictionary +# +# @param Lines: The content to be parsed +# @param Dictionary: To store data after parsing +# @param CommentCharacter: Comment char, be used to ignore comment content +# @param KeySplitCharacter: Key split char, between key name and key value. +# Key1 = Value1, '=' is the key split char +# @param ValueSplitFlag: Value split flag, be used to decide if has +# multiple values +# @param ValueSplitCharacter: Value split char, be used to split multiple +# values. Key1 = Value1|Value2, '|' is the value +# split char +# +def GetSingleValueOfKeyFromLines(Lines, Dictionary, CommentCharacter, KeySplitCharacter, \ + ValueSplitFlag, ValueSplitCharacter): + Lines = Lines.split('\n') + Keys = [] + Value = '' + DefineValues = [''] + SpecValues = [''] + + for Line in Lines: + # + # Handle DEFINE and SPEC + # + if Line.find(DataType.TAB_INF_DEFINES_DEFINE + ' ') > -1: + if '' in DefineValues: + DefineValues.remove('') + DefineValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_DEFINE, CommentCharacter)) + continue + if Line.find(DataType.TAB_INF_DEFINES_SPEC + ' ') > -1: + if '' in SpecValues: + SpecValues.remove('') + SpecValues.append(GetDefineValue(Line, DataType.TAB_INF_DEFINES_SPEC, CommentCharacter)) + continue + + # + # Handle Others + # + LineList = Line.split(KeySplitCharacter, 1) + if len(LineList) >= 2: + Key = LineList[0].split() + if len(Key) == 1 and Key[0][0] != CommentCharacter: + # + # Remove comments and white spaces + # + LineList[1] = CleanString(LineList[1], CommentCharacter) + if ValueSplitFlag: + Value = map(strip, LineList[1].split(ValueSplitCharacter)) + else: + Value = CleanString(LineList[1], CommentCharacter).splitlines() + + if Key[0] in Dictionary: + if Key[0] not in Keys: + Dictionary[Key[0]] = Value + Keys.append(Key[0]) + else: + Dictionary[Key[0]].extend(Value) + else: + Dictionary[DataType.TAB_INF_DEFINES_MACRO][Key[0]] = Value[0] + + if DefineValues == []: + DefineValues = [''] + if SpecValues == []: + SpecValues = [''] + Dictionary[DataType.TAB_INF_DEFINES_DEFINE] = DefineValues + Dictionary[DataType.TAB_INF_DEFINES_SPEC] = SpecValues + + return True + +## The content to be parsed +# +# Do pre-check for a file before it is parsed +# Check $() +# Check [] +# +# @param FileName: Used for error report +# @param FileContent: File content to be parsed +# @param SupSectionTag: Used for error report +# +def PreCheck(FileName, FileContent, SupSectionTag): + if SupSectionTag: + pass + LineNo = 0 + IsFailed = False + NewFileContent = '' + for Line in FileContent.splitlines(): + LineNo = LineNo + 1 + # + # Clean current line + # + Line = CleanString(Line) + # + # Remove commented line + # + if Line.find(DataType.TAB_COMMA_SPLIT) == 0: + Line = '' + # + # Check $() + # + if Line.find('$') > -1: + if Line.find('$(') < 0 or Line.find(')') < 0: + Logger.Error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=Logger.IS_RAISE_ERROR) + # + # Check [] + # + if Line.find('[') > -1 or Line.find(']') > -1: + # + # Only get one '[' or one ']' + # + if not (Line.find('[') > -1 and Line.find(']') > -1): + Logger.Error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=Logger.IS_RAISE_ERROR) + # + # Regenerate FileContent + # + NewFileContent = NewFileContent + Line + '\r\n' + + if IsFailed: + Logger.Error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=Logger.IS_RAISE_ERROR) + + return NewFileContent + +## CheckFileType +# +# Check if the Filename is including ExtName +# Return True if it exists +# Raise a error message if it not exists +# +# @param CheckFilename: Name of the file to be checked +# @param ExtName: Ext name of the file to be checked +# @param ContainerFilename: The container file which describes the file to be +# checked, used for error report +# @param SectionName: Used for error report +# @param Line: The line in container file which defines the file +# to be checked +# +def CheckFileType(CheckFilename, ExtName, ContainerFilename, SectionName, Line, LineNo= -1): + if CheckFilename != '' and CheckFilename is not None: + (Root, Ext) = os.path.splitext(CheckFilename) + if Ext.upper() != ExtName.upper() and Root: + ContainerFile = open(ContainerFilename, 'r').read() + if LineNo == -1: + LineNo = GetLineNo(ContainerFile, Line) + ErrorMsg = ST.ERR_SECTIONNAME_INVALID % (SectionName, CheckFilename, ExtName) + Logger.Error("Parser", PARSER_ERROR, ErrorMsg, Line=LineNo, \ + File=ContainerFilename, RaiseError=Logger.IS_RAISE_ERROR) + + return True + +## CheckFileExist +# +# Check if the file exists +# Return True if it exists +# Raise a error message if it not exists +# +# @param CheckFilename: Name of the file to be checked +# @param WorkspaceDir: Current workspace dir +# @param ContainerFilename: The container file which describes the file to +# be checked, used for error report +# @param SectionName: Used for error report +# @param Line: The line in container file which defines the +# file to be checked +# +def CheckFileExist(WorkspaceDir, CheckFilename, ContainerFilename, SectionName, Line, LineNo= -1): + CheckFile = '' + if CheckFilename != '' and CheckFilename is not None: + CheckFile = WorkspaceFile(WorkspaceDir, CheckFilename) + if not os.path.isfile(CheckFile): + ContainerFile = open(ContainerFilename, 'r').read() + if LineNo == -1: + LineNo = GetLineNo(ContainerFile, Line) + ErrorMsg = ST.ERR_CHECKFILE_NOTFOUND % (CheckFile, SectionName) + Logger.Error("Parser", PARSER_ERROR, ErrorMsg, + File=ContainerFilename, Line=LineNo, RaiseError=Logger.IS_RAISE_ERROR) + return CheckFile + +## GetLineNo +# +# Find the index of a line in a file +# +# @param FileContent: Search scope +# @param Line: Search key +# +def GetLineNo(FileContent, Line, IsIgnoreComment=True): + LineList = FileContent.splitlines() + for Index in range(len(LineList)): + if LineList[Index].find(Line) > -1: + # + # Ignore statement in comment + # + if IsIgnoreComment: + if LineList[Index].strip()[0] == DataType.TAB_COMMENT_SPLIT: + continue + return Index + 1 + + return -1 + +## RaiseParserError +# +# Raise a parser error +# +# @param Line: String which has error +# @param Section: Used for error report +# @param File: File which has the string +# @param Format: Correct format +# +def RaiseParserError(Line, Section, File, Format='', LineNo= -1): + if LineNo == -1: + LineNo = GetLineNo(open(os.path.normpath(File), 'r').read(), Line) + ErrorMsg = ST.ERR_INVALID_NOTFOUND % (Line, Section) + if Format != '': + Format = "Correct format is " + Format + Logger.Error("Parser", PARSER_ERROR, ErrorMsg, File=File, Line=LineNo, \ + ExtraData=Format, RaiseError=Logger.IS_RAISE_ERROR) + +## WorkspaceFile +# +# Return a full path with workspace dir +# +# @param WorkspaceDir: Workspace dir +# @param Filename: Relative file name +# +def WorkspaceFile(WorkspaceDir, Filename): + return os.path.join(NormPath(WorkspaceDir), NormPath(Filename)) + +## Split string +# +# Revmove '"' which startswith and endswith string +# +# @param String: The string need to be splited +# +def SplitString(String): + if String.startswith('\"'): + String = String[1:] + if String.endswith('\"'): + String = String[:-1] + return String + +## Convert To Sql String +# +# Replace "'" with "''" in each item of StringList +# +# @param StringList: A list for strings to be converted +# +def ConvertToSqlString(StringList): + return map(lambda s: s.replace("'", "''") , StringList) + +## Convert To Sql String +# +# Replace "'" with "''" in the String +# +# @param String: A String to be converted +# +def ConvertToSqlString2(String): + return String.replace("'", "''") + +## GetStringOfList +# +# Get String of a List +# +# @param Lines: string list +# @param Split: split character +# +def GetStringOfList(List, Split=' '): + if type(List) != type([]): + return List + Str = '' + for Item in List: + Str = Str + Item + Split + return Str.strip() + +## Get HelpTextList +# +# Get HelpTextList from HelpTextClassList +# +# @param HelpTextClassList: Help Text Class List +# +def GetHelpTextList(HelpTextClassList): + List = [] + if HelpTextClassList: + for HelpText in HelpTextClassList: + if HelpText.String.endswith('\n'): + HelpText.String = HelpText.String[0: len(HelpText.String) - len('\n')] + List.extend(HelpText.String.split('\n')) + return List + +## Get String Array Length +# +# Get String Array Length +# +# @param String: the source string +# +def StringArrayLength(String): + if isinstance(String, unicode): + return (len(String) + 1) * 2 + 1 + elif String.startswith('L"'): + return (len(String) - 3 + 1) * 2 + elif String.startswith('"'): + return (len(String) - 2 + 1) + else: + return len(String.split()) + 1 + +## RemoveDupOption +# +# Remove Dup Option +# +# @param OptionString: the option string +# @param Which: Which flag +# @param Against: Against flag +# +def RemoveDupOption(OptionString, Which="/I", Against=None): + OptionList = OptionString.split() + ValueList = [] + if Against: + ValueList += Against + for Index in range(len(OptionList)): + Opt = OptionList[Index] + if not Opt.startswith(Which): + continue + if len(Opt) > len(Which): + Val = Opt[len(Which):] + else: + Val = "" + if Val in ValueList: + OptionList[Index] = "" + else: + ValueList.append(Val) + return " ".join(OptionList) + +## Check if the string is HexDgit +# +# Return true if all characters in the string are digits and there is at +# least one character +# or valid Hexs (started with 0x, following by hexdigit letters) +# , false otherwise. +# @param string: input string +# +def IsHexDigit(Str): + try: + int(Str, 10) + return True + except ValueError: + if len(Str) > 2 and Str.upper().startswith('0X'): + try: + int(Str, 16) + return True + except ValueError: + return False + return False + +## Check if the string is HexDgit and its interger value within limit of UINT32 +# +# Return true if all characters in the string are digits and there is at +# least one character +# or valid Hexs (started with 0x, following by hexdigit letters) +# , false otherwise. +# @param string: input string +# +def IsHexDigitUINT32(Str): + try: + Value = int(Str, 10) + if (Value <= 0xFFFFFFFF) and (Value >= 0): + return True + except ValueError: + if len(Str) > 2 and Str.upper().startswith('0X'): + try: + Value = int(Str, 16) + if (Value <= 0xFFFFFFFF) and (Value >= 0): + return True + except ValueError: + return False + return False + +## CleanSpecialChar +# +# The ASCII text files of type INF, DEC, INI are edited by developers, +# and may contain characters that cannot be directly translated to strings that +# are conformant with the UDP XML Schema. Any characters in this category +# (0x00-0x08, TAB [0x09], 0x0B, 0x0C, 0x0E-0x1F, 0x80-0xFF) +# must be converted to a space character[0x20] as part of the parsing process. +# +def ConvertSpecialChar(Lines): + RetLines = [] + for line in Lines: + ReMatchSpecialChar = re.compile(r"[\x00-\x08]|\x09|\x0b|\x0c|[\x0e-\x1f]|[\x7f-\xff]") + RetLines.append(ReMatchSpecialChar.sub(' ', line)) + + return RetLines + +## __GetTokenList +# +# Assume Str is a valid feature flag expression. +# Return a list which contains tokens: alpha numeric token and other token +# Whitespace are not stripped +# +def __GetTokenList(Str): + InQuote = False + Token = '' + TokenOP = '' + PreChar = '' + List = [] + for Char in Str: + if InQuote: + Token += Char + if Char == '"' and PreChar != '\\': + InQuote = not InQuote + List.append(Token) + Token = '' + continue + if Char == '"': + if Token and Token != 'L': + List.append(Token) + Token = '' + if TokenOP: + List.append(TokenOP) + TokenOP = '' + InQuote = not InQuote + Token += Char + continue + + if not (Char.isalnum() or Char in '_'): + TokenOP += Char + if Token: + List.append(Token) + Token = '' + else: + Token += Char + if TokenOP: + List.append(TokenOP) + TokenOP = '' + + if PreChar == '\\' and Char == '\\': + PreChar = '' + else: + PreChar = Char + if Token: + List.append(Token) + if TokenOP: + List.append(TokenOP) + return List + +## ConvertNEToNOTEQ +# +# Convert NE operator to NOT EQ +# For example: 1 NE 2 -> 1 NOT EQ 2 +# +# @param Expr: Feature flag expression to be converted +# +def ConvertNEToNOTEQ(Expr): + List = __GetTokenList(Expr) + for Index in range(len(List)): + if List[Index] == 'NE': + List[Index] = 'NOT EQ' + return ''.join(List) + +## ConvertNOTEQToNE +# +# Convert NOT EQ operator to NE +# For example: 1 NOT NE 2 -> 1 NE 2 +# +# @param Expr: Feature flag expression to be converted +# +def ConvertNOTEQToNE(Expr): + List = __GetTokenList(Expr) + HasNOT = False + RetList = [] + for Token in List: + if HasNOT and Token == 'EQ': + # At least, 'NOT' is in the list + while not RetList[-1].strip(): + RetList.pop() + RetList[-1] = 'NE' + HasNOT = False + continue + if Token == 'NOT': + HasNOT = True + elif Token.strip(): + HasNOT = False + RetList.append(Token) + + return ''.join(RetList) + +## SplitPcdEntry +# +# Split an PCD entry string to Token.CName and PCD value and FFE. +# NOTE: PCD Value and FFE can contain "|" in it's expression. And in INF specification, have below rule. +# When using the characters "|" or "||" in an expression, the expression must be encapsulated in +# open "(" and close ")" parenthesis. +# +# @param String An PCD entry string need to be split. +# +# @return List [PcdTokenCName, Value, FFE] +# +def SplitPcdEntry(String): + if not String: + return ['', '', ''], False + + PcdTokenCName = '' + PcdValue = '' + PcdFeatureFlagExp = '' + + ValueList = GetSplitValueList(String, "|", 1) + + # + # Only contain TokenCName + # + if len(ValueList) == 1: + return [ValueList[0]], True + + NewValueList = [] + + if len(ValueList) == 2: + PcdTokenCName = ValueList[0] + + InQuote = False + InParenthesis = False + StrItem = '' + for StrCh in ValueList[1]: + if StrCh == '"': + InQuote = not InQuote + elif StrCh == '(' or StrCh == ')': + InParenthesis = not InParenthesis + + if StrCh == '|': + if not InQuote or not InParenthesis: + NewValueList.append(StrItem.strip()) + StrItem = ' ' + continue + + StrItem += StrCh + + NewValueList.append(StrItem.strip()) + + if len(NewValueList) == 1: + PcdValue = NewValueList[0] + return [PcdTokenCName, PcdValue], True + elif len(NewValueList) == 2: + PcdValue = NewValueList[0] + PcdFeatureFlagExp = NewValueList[1] + return [PcdTokenCName, PcdValue, PcdFeatureFlagExp], True + else: + return ['', '', ''], False + + return ['', '', ''], False + +## Check if two arches matched? +# +# @param Arch1 +# @param Arch2 +# +def IsMatchArch(Arch1, Arch2): + if 'COMMON' in Arch1 or 'COMMON' in Arch2: + return True + if isinstance(Arch1, basestring) and isinstance(Arch2, basestring): + if Arch1 == Arch2: + return True + + if isinstance(Arch1, basestring) and isinstance(Arch2, list): + return Arch1 in Arch2 + + if isinstance(Arch2, basestring) and isinstance(Arch1, list): + return Arch2 in Arch1 + + if isinstance(Arch1, list) and isinstance(Arch2, list): + for Item1 in Arch1: + for Item2 in Arch2: + if Item1 == Item2: + return True + + return False + +# Search all files in FilePath to find the FileName with the largest index +# Return the FileName with index +1 under the FilePath +# +def GetUniFileName(FilePath, FileName): + Files = [] + try: + Files = os.listdir(FilePath) + except: + pass + + LargestIndex = -1 + IndexNotFound = True + for File in Files: + if File.upper().startswith(FileName.upper()) and File.upper().endswith('.UNI'): + Index = File.upper().replace(FileName.upper(), '').replace('.UNI', '') + if Index: + try: + Index = int(Index) + except Exception: + Index = -1 + else: + IndexNotFound = False + Index = 0 + if Index > LargestIndex: + LargestIndex = Index + 1 + + if LargestIndex > -1 and not IndexNotFound: + return os.path.normpath(os.path.join(FilePath, FileName + str(LargestIndex) + '.uni')) + else: + return os.path.normpath(os.path.join(FilePath, FileName + '.uni')) diff --git a/BaseTools/Source/Python/Workspace/DecBuildData.py b/BaseTools/Source/Python/Workspace/DecBuildData.py index 1fbd095f74..99257d0814 100644 --- a/BaseTools/Source/Python/Workspace/DecBuildData.py +++ b/BaseTools/Source/Python/Workspace/DecBuildData.py @@ -11,7 +11,7 @@ # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. # -from Common.String import * +from Common.StringUtils import * from Common.DataType import * from Common.Misc import * from types import * diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index 0262a91ed8..1ed7eb1c2c 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -17,7 +17,7 @@ # This class is used to retrieve information stored in database and convert them # into PlatformBuildClassObject form for easier use for AutoGen. # -from Common.String import * +from Common.StringUtils import * from Common.DataType import * from Common.Misc import * from types import * diff --git a/BaseTools/Source/Python/Workspace/InfBuildData.py b/BaseTools/Source/Python/Workspace/InfBuildData.py index bd1c841541..836140759f 100644 --- a/BaseTools/Source/Python/Workspace/InfBuildData.py +++ b/BaseTools/Source/Python/Workspace/InfBuildData.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. # -from Common.String import * +from Common.StringUtils import * from Common.DataType import * from Common.Misc import * from types import * diff --git a/BaseTools/Source/Python/Workspace/MetaFileParser.py b/BaseTools/Source/Python/Workspace/MetaFileParser.py index 36843643ed..8ab1dd23a0 100644 --- a/BaseTools/Source/Python/Workspace/MetaFileParser.py +++ b/BaseTools/Source/Python/Workspace/MetaFileParser.py @@ -26,7 +26,7 @@ import Common.GlobalData as GlobalData from CommonDataClass.DataClass import * from Common.DataType import * -from Common.String import * +from Common.StringUtils import * from Common.Misc import GuidStructureStringToGuidString, CheckPcdDatum, PathClass, AnalyzePcdData, AnalyzeDscPcd, AnalyzePcdExpression, ParseFieldValue from Common.Expression import * from CommonDataClass.Exceptions import * diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py index 14dcb1ae81..9e055fcfc4 100644 --- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py +++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py @@ -16,7 +16,7 @@ # Import Modules # import sqlite3 -from Common.String import * +from Common.StringUtils import * from Common.DataType import * from Common.Misc import * from types import * diff --git a/BaseTools/Source/Python/build/BuildReport.py b/BaseTools/Source/Python/build/BuildReport.py index cf45ef1734..3c495a6b1b 100644 --- a/BaseTools/Source/Python/build/BuildReport.py +++ b/BaseTools/Source/Python/build/BuildReport.py @@ -42,7 +42,7 @@ from Common.MultipleWorkspace import MultipleWorkspace as mws import Common.GlobalData as GlobalData from AutoGen.AutoGen import ModuleAutoGen from Common.Misc import PathClass -from Common.String import NormPath +from Common.StringUtils import NormPath from Common.DataType import * import collections from Common.Expression import *