From bf9e636605188e291d33ab694ff1c5926b6f0800 Mon Sep 17 00:00:00 2001 From: BobCF Date: Thu, 8 Nov 2018 14:03:38 +0800 Subject: [PATCH] BaseTools: Customize deepcopy function. https://bugzilla.tianocore.org/show_bug.cgi?id=1288 This patch is one of build tool performance improvement series patches. This patch is going to customize the deepcopy function for SkuClass, PcdClassObject and python dictionary. python deepcopy copy everything of a object, but for our current usage we just need to copy the data we care about recursively. By implementing __deepcopy__ for SkuClass, PcdClassObject, we can customize deepcopy function for them. Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: BobCF Cc: Liming Gao Cc: Jaben Carsey Reviewed-by: Jaben Carsey Reviewed-by: Liming Gao --- BaseTools/Source/Python/Common/Expression.py | 4 +- BaseTools/Source/Python/Common/Misc.py | 16 ++++++ .../Python/CommonDataClass/CommonClass.py | 15 ++++++ .../Python/Workspace/BuildClassObject.py | 52 +++++++++++++++++++ .../Source/Python/Workspace/DscBuildData.py | 6 +-- 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py index a5f085d7fb..19ea13b7fb 100644 --- a/BaseTools/Source/Python/Common/Expression.py +++ b/BaseTools/Source/Python/Common/Expression.py @@ -17,7 +17,7 @@ from __future__ import absolute_import from Common.GlobalData import * from CommonDataClass.Exceptions import BadExpression from CommonDataClass.Exceptions import WrnExpression -from .Misc import GuidStringToGuidStructureString, ParseFieldValue +from .Misc import GuidStringToGuidStructureString, ParseFieldValue,CopyDict import Common.EdkLogger as EdkLogger import copy from Common.DataType import * @@ -355,7 +355,7 @@ class ValueExpression(BaseExpression): # # The symbol table including PCD and macro mapping # - self._Symb = copy.deepcopy(SymbolTable) + self._Symb = CopyDict(SymbolTable) self._Symb.update(self.LogicalOperators) self._Idx = 0 self._Len = len(self._Expr) diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py index 84f3f16ee6..b063f064fb 100644 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -41,6 +41,7 @@ import uuid from CommonDataClass.Exceptions import BadExpression from Common.caching import cached_property import subprocess +from collections import OrderedDict ## Regular expression used to find out place holders in string template gPlaceholderPattern = re.compile("\$\{([^$()\s]+)\}", re.MULTILINE | re.UNICODE) @@ -2130,6 +2131,21 @@ def PackByteFormatGUID(Guid): Guid[10], ) +## DeepCopy dict/OrderedDict recusively +# +# @param ori_dict a nested dict or ordereddict +# +# @retval new dict or orderdict +# +def CopyDict(ori_dict): + dict_type = ori_dict.__class__ + new_dict = dict_type() + for key in ori_dict: + if isinstance(ori_dict[key],(dict,OrderedDict)): + new_dict[key] = CopyDict(ori_dict[key]) + else: + new_dict[key] = ori_dict[key] + return new_dict ## # # This acts like the main() function for the script, unless it is 'import'ed into another diff --git a/BaseTools/Source/Python/CommonDataClass/CommonClass.py b/BaseTools/Source/Python/CommonDataClass/CommonClass.py index a98cf8a7c5..336bb11671 100644 --- a/BaseTools/Source/Python/CommonDataClass/CommonClass.py +++ b/BaseTools/Source/Python/CommonDataClass/CommonClass.py @@ -80,3 +80,18 @@ class SkuInfoClass(object): 'VpdOffset = ' + str(self.VpdOffset) + "," + \ 'DefaultValue = ' + str(self.DefaultValue) + "," return Rtn + + def __deepcopy__(self,memo): + new_sku = SkuInfoClass() + new_sku.SkuIdName = self.SkuIdName + new_sku.SkuId = self.SkuId + new_sku.VariableName = self.VariableName + new_sku.VariableGuid = self.VariableGuid + new_sku.VariableGuidValue = self.VariableGuidValue + new_sku.VariableOffset = self.VariableOffset + new_sku.HiiDefaultValue = self.HiiDefaultValue + new_sku.VariableAttribute = self.VariableAttribute + new_sku.DefaultStoreDict = {key:value for key,value in self.DefaultStoreDict.items()} + new_sku.VpdOffset = self.VpdOffset + new_sku.DefaultValue = self.DefaultValue + return new_sku diff --git a/BaseTools/Source/Python/Workspace/BuildClassObject.py b/BaseTools/Source/Python/Workspace/BuildClassObject.py index 15bb822910..008eee1a16 100644 --- a/BaseTools/Source/Python/Workspace/BuildClassObject.py +++ b/BaseTools/Source/Python/Workspace/BuildClassObject.py @@ -16,6 +16,8 @@ from Common.DataType import * import collections import re from collections import OrderedDict +from Common.Misc import CopyDict +import copy StructPattern = re.compile(r'[_a-zA-Z][0-9A-Za-z_\[\]]*$') ArrayIndex = re.compile("\[\s*\d{0,1}\s*\]") ## PcdClassObject @@ -211,6 +213,37 @@ class PcdClassObject(object): def __hash__(self): return hash((self.TokenCName, self.TokenSpaceGuidCName)) + def sharedcopy(self,new_pcd): + new_pcd.TokenCName = self.TokenCName + new_pcd.TokenSpaceGuidCName = self.TokenSpaceGuidCName + new_pcd.TokenSpaceGuidValue = self.TokenSpaceGuidValue + new_pcd.Type = self.Type + new_pcd.DatumType = self.DatumType + new_pcd.DefaultValue = self.DefaultValue + new_pcd.TokenValue = self.TokenValue + new_pcd.MaxDatumSize = self.MaxDatumSize + + new_pcd.Phase = self.Phase + new_pcd.Pending = self.Pending + new_pcd.IsOverrided = self.IsOverrided + new_pcd.IsFromBinaryInf = self.IsFromBinaryInf + new_pcd.IsFromDsc = self.IsFromDsc + new_pcd.PcdValueFromComm = self.PcdValueFromComm + new_pcd.PcdValueFromFdf = self.PcdValueFromFdf + new_pcd.UserDefinedDefaultStoresFlag = self.UserDefinedDefaultStoresFlag + new_pcd.DscRawValue = self.DscRawValue + new_pcd.CustomAttribute = self.CustomAttribute + new_pcd.validateranges = [item for item in self.validateranges] + new_pcd.validlists = [item for item in self.validlists] + new_pcd.expressions = [item for item in self.expressions] + new_pcd.SkuInfoList = {key: copy.deepcopy(skuobj) for key,skuobj in self.SkuInfoList.items()} + return new_pcd + + def __deepcopy__(self,memo): + new_pcd = PcdClassObject() + self.sharedcopy(new_pcd) + return new_pcd + class StructurePcd(PcdClassObject): def __init__(self, StructuredPcdIncludeFile=None, Packages=None, Name=None, Guid=None, Type=None, DatumType=None, Value=None, Token=None, MaxDatumSize=None, SkuInfoList=None, IsOverrided=False, GuidValue=None, validateranges=None, validlists=None, expressions=None,default_store = TAB_DEFAULT_STORES_DEFAULT): if SkuInfoList is None: @@ -303,6 +336,25 @@ class StructurePcd(PcdClassObject): self.PcdFieldValueFromComm = PcdObject.PcdFieldValueFromComm if PcdObject.PcdFieldValueFromComm else self.PcdFieldValueFromComm self.PcdFieldValueFromFdf = PcdObject.PcdFieldValueFromFdf if PcdObject.PcdFieldValueFromFdf else self.PcdFieldValueFromFdf + def __deepcopy__(self,memo): + new_pcd = StructurePcd() + self.sharedcopy(new_pcd) + + new_pcd.DefaultValueFromDec = self.DefaultValueFromDec + new_pcd.PcdMode = self.PcdMode + new_pcd.StructName = self.DatumType + new_pcd.PcdDefineLineNo = self.PcdDefineLineNo + new_pcd.PkgPath = self.PkgPath + new_pcd.StructuredPcdIncludeFile = [item for item in self.StructuredPcdIncludeFile] + new_pcd.PackageDecs = [item for item in self.PackageDecs] + new_pcd.DefaultValues = CopyDict(self.DefaultValues) + new_pcd.DefaultFromDSC=CopyDict(self.DefaultFromDSC) + new_pcd.SkuOverrideValues = CopyDict(self.SkuOverrideValues) + new_pcd.PcdFieldValueFromComm = CopyDict(self.PcdFieldValueFromComm) + new_pcd.PcdFieldValueFromFdf = CopyDict(self.PcdFieldValueFromFdf) + new_pcd.ValueChain = {item for item in self.ValueChain} + return new_pcd + LibraryClassObject = namedtuple('LibraryClassObject', ['LibraryClass','SupModList'], verbose=False) ## ModuleBuildClassObject diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py index 28da5ea63b..59d118216d 100644 --- a/BaseTools/Source/Python/Workspace/DscBuildData.py +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py @@ -974,7 +974,7 @@ class DscBuildData(PlatformBuildClassObject): if TAB_DEFAULT_STORES_DEFAULT not in skuobj.DefaultStoreDict: PcdDefaultStoreSet = set(defaultstorename for defaultstorename in skuobj.DefaultStoreDict) mindefaultstorename = DefaultStoreMgr.GetMin(PcdDefaultStoreSet) - skuobj.DefaultStoreDict[TAB_DEFAULT_STORES_DEFAULT] = copy.deepcopy(skuobj.DefaultStoreDict[mindefaultstorename]) + skuobj.DefaultStoreDict[TAB_DEFAULT_STORES_DEFAULT] = CopyDict(skuobj.DefaultStoreDict[mindefaultstorename]) return Pcds def RecoverCommandLinePcd(self): @@ -1528,7 +1528,7 @@ class DscBuildData(PlatformBuildClassObject): for defaultstoreid in DefaultStores: if defaultstoreid not in stru_pcd.SkuOverrideValues[skuid]: - stru_pcd.SkuOverrideValues[skuid][defaultstoreid] = copy.deepcopy(stru_pcd.SkuOverrideValues[nextskuid][mindefaultstorename]) + stru_pcd.SkuOverrideValues[skuid][defaultstoreid] = CopyDict(stru_pcd.SkuOverrideValues[nextskuid][mindefaultstorename]) stru_pcd.ValueChain.add((skuid, defaultstoreid)) S_pcd_set = DscBuildData.OverrideByFdf(S_pcd_set,self.WorkspaceDir) S_pcd_set = DscBuildData.OverrideByComm(S_pcd_set) @@ -2769,7 +2769,7 @@ class DscBuildData(PlatformBuildClassObject): mindefaultstorename = DefaultStoreObj.GetMin(set(defaultstorename for defaultstorename in skuobj.DefaultStoreDict)) for defaultstorename in DefaultStores: if defaultstorename not in skuobj.DefaultStoreDict: - skuobj.DefaultStoreDict[defaultstorename] = copy.deepcopy(skuobj.DefaultStoreDict[mindefaultstorename]) + skuobj.DefaultStoreDict[defaultstorename] = CopyDict(skuobj.DefaultStoreDict[mindefaultstorename]) skuobj.HiiDefaultValue = skuobj.DefaultStoreDict[mindefaultstorename] for skuname, skuid in SkuIds.items(): if skuname not in PcdObj.SkuInfoList: -- 2.39.2