X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FCommon%2FMisc.py;h=d082c58befda5b46aa3f40113c96f1566ded60fd;hp=c547c2f8e43a27cdff66faf2b7c0960a324664c3;hb=66b845ae06f1e48c693158ee0b8b4f610d5ada4a;hpb=42bd17508ec519689464175a794b6de54ebe95a0 diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py index c547c2f8e4..d082c58bef 100644 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -2,20 +2,14 @@ # Common routines used by all tools # # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.
-# This program and the accompanying materials -# are licensed and made available under the terms and conditions of the BSD License -# which accompanies this distribution. The full text of the license may be found at -# http://opensource.org/licenses/bsd-license.php -# -# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, -# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. +# SPDX-License-Identifier: BSD-2-Clause-Patent # ## # Import Modules # from __future__ import absolute_import -import Common.LongFilePathOs as os + import sys import string import threading @@ -26,22 +20,24 @@ import array import shutil from random import sample from struct import pack -from UserDict import IterableUserDict -from UserList import UserList +import uuid +import subprocess +import tempfile +from collections import OrderedDict +import Common.LongFilePathOs as os from Common import EdkLogger as EdkLogger from Common import GlobalData as GlobalData -from .DataType import * -from .BuildToolError import * +from Common.DataType import * +from Common.BuildToolError import * from CommonDataClass.DataClass import * -from .Parsing import GetSplitValueList +from Common.Parsing import GetSplitValueList from Common.LongFilePathSupport import OpenLongFilePath as open from Common.MultipleWorkspace import MultipleWorkspace as mws -import uuid from CommonDataClass.Exceptions import BadExpression from Common.caching import cached_property -import subprocess -from collections import OrderedDict + +ArrayIndex = re.compile("\[\s*[0-9a-fA-FxX]*\s*\]") ## Regular expression used to find out place holders in string template gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE) @@ -54,9 +50,6 @@ secReGeneral = re.compile('^([\da-fA-F]+):([\da-fA-F]+) +([\da-fA-F]+)[Hh]? +([. StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_]*$') -## Dictionary used to store file time stamp for quick re-access -gFileTimeStampCache = {} # {file path : file time stamp} - ## Dictionary used to store dependencies of files gDependencyDatabase = {} # arch : {file path : [dependent files list]} @@ -288,6 +281,7 @@ def ProcessDuplicatedInf(Path, BaseName, Workspace): # RtPath.Path = TempFullPath RtPath.BaseName = BaseName + RtPath.OriginalPath = Path # # If file exists, compare contents # @@ -459,15 +453,22 @@ def RemoveDirectory(Directory, Recursively=False): # @retval False If the file content is the same # def SaveFileOnChange(File, Content, IsBinaryFile=True): - if not IsBinaryFile: - Content = Content.replace("\n", os.linesep) if os.path.exists(File): - try: - if Content == open(File, "rb").read(): - return False - except: - EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File) + if IsBinaryFile: + try: + with open(File, "rb") as f: + if Content == f.read(): + return False + except: + EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File) + else: + try: + with open(File, "r") as f: + if Content == f.read(): + return False + except: + EdkLogger.error(None, FILE_OPEN_FAILURE, ExtraData=File) DirName = os.path.dirname(File) if not CreateDirectory(DirName): @@ -478,12 +479,26 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True): if not os.access(DirName, os.W_OK): EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName) - try: - Fd = open(File, "wb") - Fd.write(Content) - Fd.close() - except IOError as X: - EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X) + OpenMode = "w" + if IsBinaryFile: + OpenMode = "wb" + + if GlobalData.gIsWindows and not os.path.exists(File): + # write temp file, then rename the temp file to the real file + # to make sure the file be immediate saved to disk + with tempfile.NamedTemporaryFile(OpenMode, dir=os.path.dirname(File), delete=False) as tf: + tf.write(Content) + tempname = tf.name + try: + os.rename(tempname, File) + except: + EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X) + else: + try: + with open(File, OpenMode) as Fd: + Fd.write(Content) + except IOError as X: + EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X) return True @@ -577,13 +592,14 @@ def RealPath(File, Dir='', OverrideDir=''): # def GuidValue(CName, PackageList, Inffile = None): for P in PackageList: - GuidKeys = P.Guids.keys() + GuidKeys = list(P.Guids.keys()) if Inffile and P._PrivateGuids: if not Inffile.startswith(P.MetaFile.Dir): GuidKeys = [x for x in P.Guids if x not in P._PrivateGuids] if CName in GuidKeys: return P.Guids[CName] return None + return None ## A string template class # @@ -764,10 +780,10 @@ class Progressor: ## Constructor # - # @param OpenMessage The string printed before progress charaters - # @param CloseMessage The string printed after progress charaters - # @param ProgressChar The charater used to indicate the progress - # @param Interval The interval in seconds between two progress charaters + # @param OpenMessage The string printed before progress characters + # @param CloseMessage The string printed after progress characters + # @param ProgressChar The character used to indicate the progress + # @param Interval The interval in seconds between two progress characters # def __init__(self, OpenMessage="", CloseMessage="", ProgressChar='.', Interval=1.0): self.PromptMessage = OpenMessage @@ -777,9 +793,9 @@ class Progressor: if Progressor._StopFlag is None: Progressor._StopFlag = threading.Event() - ## Start to print progress charater + ## Start to print progress character # - # @param OpenMessage The string printed before progress charaters + # @param OpenMessage The string printed before progress characters # def Start(self, OpenMessage=None): if OpenMessage is not None: @@ -790,9 +806,9 @@ class Progressor: Progressor._ProgressThread.setDaemon(False) Progressor._ProgressThread.start() - ## Stop printing progress charater + ## Stop printing progress character # - # @param CloseMessage The string printed after progress charaters + # @param CloseMessage The string printed after progress characters # def Stop(self, CloseMessage=None): OriginalCodaMessage = self.CodaMessage @@ -825,165 +841,6 @@ class Progressor: Progressor._ProgressThread.join() Progressor._ProgressThread = None -## A dict which can access its keys and/or values orderly -# -# The class implements a new kind of dict which its keys or values can be -# accessed in the order they are added into the dict. It guarantees the order -# by making use of an internal list to keep a copy of keys. -# -class sdict(IterableUserDict): - ## Constructor - def __init__(self): - IterableUserDict.__init__(self) - self._key_list = [] - - ## [] operator - def __setitem__(self, key, value): - if key not in self._key_list: - self._key_list.append(key) - IterableUserDict.__setitem__(self, key, value) - - ## del operator - def __delitem__(self, key): - self._key_list.remove(key) - IterableUserDict.__delitem__(self, key) - - ## used in "for k in dict" loop to ensure the correct order - def __iter__(self): - return self.iterkeys() - - ## len() support - def __len__(self): - return len(self._key_list) - - ## "in" test support - def __contains__(self, key): - return key in self._key_list - - ## indexof support - def index(self, key): - return self._key_list.index(key) - - ## insert support - def insert(self, key, newkey, newvalue, order): - index = self._key_list.index(key) - if order == 'BEFORE': - self._key_list.insert(index, newkey) - IterableUserDict.__setitem__(self, newkey, newvalue) - elif order == 'AFTER': - self._key_list.insert(index + 1, newkey) - IterableUserDict.__setitem__(self, newkey, newvalue) - - ## append support - def append(self, sdict): - for key in sdict: - if key not in self._key_list: - self._key_list.append(key) - IterableUserDict.__setitem__(self, key, sdict[key]) - - def has_key(self, key): - return key in self._key_list - - ## Empty the dict - def clear(self): - self._key_list = [] - IterableUserDict.clear(self) - - ## Return a copy of keys - def keys(self): - keys = [] - for key in self._key_list: - keys.append(key) - return keys - - ## Return a copy of values - def values(self): - values = [] - for key in self._key_list: - values.append(self[key]) - return values - - ## Return a copy of (key, value) list - def items(self): - items = [] - for key in self._key_list: - items.append((key, self[key])) - return items - - ## Iteration support - def iteritems(self): - return iter(self.items()) - - ## Keys interation support - def iterkeys(self): - return iter(self.keys()) - - ## Values interation support - def itervalues(self): - return iter(self.values()) - - ## Return value related to a key, and remove the (key, value) from the dict - def pop(self, key, *dv): - value = None - if key in self._key_list: - value = self[key] - self.__delitem__(key) - elif len(dv) != 0 : - value = kv[0] - return value - - ## Return (key, value) pair, and remove the (key, value) from the dict - def popitem(self): - key = self._key_list[-1] - value = self[key] - self.__delitem__(key) - return key, value - - def update(self, dict=None, **kwargs): - if dict is not None: - for k, v in dict.items(): - self[k] = v - if len(kwargs): - for k, v in kwargs.items(): - self[k] = v - -## Dictionary with restricted keys -# -class rdict(dict): - ## Constructor - def __init__(self, KeyList): - for Key in KeyList: - dict.__setitem__(self, Key, "") - - ## []= operator - def __setitem__(self, key, value): - if key not in self: - EdkLogger.error("RestrictedDict", ATTRIBUTE_SET_FAILURE, "Key [%s] is not allowed" % key, - ExtraData=", ".join(dict.keys(self))) - dict.__setitem__(self, key, value) - - ## =[] operator - def __getitem__(self, key): - if key not in self: - return "" - return dict.__getitem__(self, key) - - ## del operator - def __delitem__(self, key): - EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="del") - - ## Empty the dict - def clear(self): - for Key in self: - self.__setitem__(Key, "") - - ## Return value related to a key, and remove the (key, value) from the dict - def pop(self, key, *dv): - EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="pop") - - ## Return (key, value) pair, and remove the (key, value) from the dict - def popitem(self): - EdkLogger.error("RestrictedDict", ATTRIBUTE_ACCESS_DENIED, ExtraData="popitem") ## Dictionary using prioritized list as key # @@ -1180,6 +1037,7 @@ def ParseFieldValue (Value): p.stderr.close() if err: raise BadExpression("DevicePath: %s" % str(err)) + out = out.decode(encoding='utf-8', errors='ignore') Size = len(out.split()) out = ','.join(out.split()) return '{' + out + '}', Size @@ -1187,7 +1045,7 @@ def ParseFieldValue (Value): if "{CODE(" in Value: return Value, len(Value.split(",")) if isinstance(Value, type(0)): - return Value, (Value.bit_length() + 7) / 8 + return Value, (Value.bit_length() + 7) // 8 if not isinstance(Value, type('')): raise BadExpression('Type %s is %s' %(Value, type(Value))) Value = Value.strip() @@ -1221,7 +1079,10 @@ def ParseFieldValue (Value): if Value[0] == '"' and Value[-1] == '"': Value = Value[1:-1] try: - Value = "'" + uuid.UUID(Value).get_bytes_le() + "'" + Value = str(uuid.UUID(Value).bytes_le) + if Value.startswith("b'"): + Value = Value[2:-1] + Value = "'" + Value + "'" except ValueError as Message: raise BadExpression(Message) Value, Size = ParseFieldValue(Value) @@ -1308,12 +1169,12 @@ def ParseFieldValue (Value): raise BadExpression("invalid hex value: %s" % Value) if Value == 0: return 0, 1 - return Value, (Value.bit_length() + 7) / 8 + return Value, (Value.bit_length() + 7) // 8 if Value[0].isdigit(): Value = int(Value, 10) if Value == 0: return 0, 1 - return Value, (Value.bit_length() + 7) / 8 + return Value, (Value.bit_length() + 7) // 8 if Value.lower() == 'true': return 1, 1 if Value.lower() == 'false': @@ -1472,10 +1333,12 @@ def CheckPcdDatum(Type, Value): return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\ ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type) elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]: - if Value and int(Value, 0) < 0: - return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type) + if Value.startswith('0') and not Value.lower().startswith('0x') and len(Value) > 1 and Value.lstrip('0'): + Value = Value.lstrip('0') try: - Value = long(Value, 0) + if Value and int(Value, 0) < 0: + return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type) + Value = int(Value, 0) if Value > MAX_VAL_TYPE[Type]: return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type) except: @@ -1489,7 +1352,7 @@ def CheckPcdDatum(Type, Value): def CommonPath(PathList): P1 = min(PathList).split(os.path.sep) P2 = max(PathList).split(os.path.sep) - for Index in xrange(min(len(P1), len(P2))): + for Index in range(min(len(P1), len(P2))): if P1[Index] != P2[Index]: return os.path.sep.join(P1[:Index]) return os.path.sep.join(P1) @@ -1543,6 +1406,7 @@ class PathClass(object): self.TagName = TagName self.ToolCode = ToolCode self.ToolChainFamily = ToolChainFamily + self.OriginalPath = self ## Convert the object of this class to a string # @@ -1565,7 +1429,7 @@ class PathClass(object): ## Override __cmp__ function # - # Customize the comparsion operation of two PathClass + # Customize the comparison operation of two PathClass # # @retval 0 The two PathClass are different # @retval -1 The first PathClass is less than the second PathClass @@ -1660,7 +1524,7 @@ class PathClass(object): self.Path = os.path.join(RealRoot, RealFile) return ErrorCode, ErrorInfo -## Parse PE image to get the required PE informaion. +## Parse PE image to get the required PE information. # class PeImageClass(): ## Constructor @@ -1695,7 +1559,7 @@ class PeImageClass(): ByteArray = array.array('B') ByteArray.fromfile(PeObject, 4) # PE signature should be 'PE\0\0' - if ByteArray.tostring() != 'PE\0\0': + if ByteArray.tostring() != b'PE\0\0': self.ErrorInfo = self.FileName + ' has no valid PE signature PE00' return @@ -1787,7 +1651,7 @@ class SkuClass(): ExtraData = "SKU-ID [%s] value %s exceeds the max value of UINT64" % (SkuName, SkuId)) - self.AvailableSkuIds = sdict() + self.AvailableSkuIds = OrderedDict() self.SkuIdSet = [] self.SkuIdNumberSet = [] self.SkuData = SkuIds @@ -1797,7 +1661,7 @@ class SkuClass(): self.SkuIdSet = ['DEFAULT'] self.SkuIdNumberSet = ['0U'] elif SkuIdentifier == 'ALL': - self.SkuIdSet = SkuIds.keys() + self.SkuIdSet = list(SkuIds.keys()) self.SkuIdNumberSet = [num[0].strip() + 'U' for num in SkuIds.values()] else: r = SkuIdentifier.split('|') @@ -1911,7 +1775,7 @@ class SkuClass(): # @retval Value The integer value that the input represents # def GetIntegerValue(Input): - if type(Input) in (int, long): + if not isinstance(Input, str): return Input String = Input if String.endswith("U"): @@ -1987,11 +1851,3 @@ def CopyDict(ori_dict): # def RemoveCComments(ctext): return re.sub('//.*?\n|/\*.*?\*/', '\n', ctext, flags=re.S) -## -# -# This acts like the main() function for the script, unless it is 'import'ed into another -# script. -# -if __name__ == '__main__': - pass -