import threading\r
from subprocess import Popen,PIPE\r
from collections import OrderedDict, defaultdict\r
-from buildoptions import BuildOption,BuildTarget\r
+from Common.buildoptions import BuildOption,BuildTarget\r
from AutoGen.PlatformAutoGen import PlatformAutoGen\r
from AutoGen.ModuleAutoGen import ModuleAutoGen\r
from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen\r
#\r
def AddDependency(self, Dependency):\r
for Dep in Dependency:\r
- if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash():\r
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR):\r
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list\r
\r
## The thread wrapper of LaunchCommand function\r
self.FvList = BuildOptions.FvImage\r
self.CapList = BuildOptions.CapName\r
self.SilentMode = BuildOptions.SilentMode\r
- self.ThreadNumber = BuildOptions.ThreadNumber\r
+ self.ThreadNumber = 1\r
self.SkipAutoGen = BuildOptions.SkipAutoGen\r
self.Reparse = BuildOptions.Reparse\r
self.SkuId = BuildOptions.SkuId\r
self.AutoGenMgr = None\r
EdkLogger.info("")\r
os.chdir(self.WorkspaceDir)\r
- self.share_data = Manager().dict()\r
+ GlobalData.gCacheIR = Manager().dict()\r
self.log_q = log_q\r
def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):\r
try:\r
feedback_q = mp.Queue()\r
file_lock = mp.Lock()\r
error_event = mp.Event()\r
+ GlobalData.file_lock = file_lock\r
+ FfsCmd = DataPipe.Get("FfsCommand")\r
+ if FfsCmd is None:\r
+ FfsCmd = {}\r
+ GlobalData.FfsCmd = FfsCmd\r
+ GlobalData.libConstPcd = DataPipe.Get("LibConstPcd")\r
+ GlobalData.Refes = DataPipe.Get("REFS")\r
auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)\r
self.AutoGenMgr.start()\r
w.start()\r
if PcdMaList is not None:\r
for PcdMa in PcdMaList:\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ PcdMa.GenModuleFilesHash(share_data)\r
+ PcdMa.GenPreMakefileHash(share_data)\r
+ if PcdMa.CanSkipbyPreMakefileCache(share_data):\r
+ continue\r
+\r
PcdMa.CreateCodeFile(False)\r
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))\r
\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ PcdMa.GenMakeHeaderFilesHash(share_data)\r
+ PcdMa.GenMakeHash(share_data)\r
+ if PcdMa.CanSkipbyMakeCache(share_data):\r
+ continue\r
+\r
self.AutoGenMgr.join()\r
rt = self.AutoGenMgr.Status\r
return rt, 0\r
- except Exception as e:\r
- return False,e.errcode\r
+ except FatalError as e:\r
+ return False, e.args[0]\r
+ except:\r
+ return False, UNKNOWN_ERROR\r
\r
## Load configuration\r
#\r
ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])\r
self.ToolChainFamily = ToolChainFamily\r
\r
- if self.ThreadNumber is None:\r
- self.ThreadNumber = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]\r
- if self.ThreadNumber == '':\r
- self.ThreadNumber = 0\r
- else:\r
- self.ThreadNumber = int(self.ThreadNumber, 0)\r
-\r
- if self.ThreadNumber == 0:\r
- try:\r
- self.ThreadNumber = multiprocessing.cpu_count()\r
- except (ImportError, NotImplementedError):\r
- self.ThreadNumber = 1\r
-\r
if not self.PlatformFile:\r
PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]\r
if not PlatformFile:\r
ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")\r
\r
self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)\r
-\r
+ self.ThreadNumber = ThreadNum()\r
## Initialize build configuration\r
#\r
# This method will parse DSC file and merge the configurations from\r
mqueue.put(m)\r
\r
AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}\r
+ AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}\r
self.Progress.Start("Generating makefile and code")\r
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))\r
AutoGenObject.DataPipe.dump(data_pipe_file)\r
- autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)\r
+ autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)\r
self.Progress.Stop("done!")\r
if not autogen_rt:\r
self.AutoGenMgr.TerminateWorkers()\r
CmdListDict = None\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
+\r
+ # Add Platform and Package level hash in share_data for module hash calculation later\r
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
+ for PkgName in GlobalData.gPackageHash.keys():\r
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
+ GlobalData.file_lock = mp.Lock()\r
+ GlobalData.FfsCmd = CmdListDict\r
+\r
self.Progress.Stop("done!")\r
MaList = []\r
ExitFlag = threading.Event()\r
AutoGenStart = time.time()\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
+ GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd")\r
+ GlobalData.Refes = Pa.DataPipe.Get("REFS")\r
for Module in Pa.Platform.Modules:\r
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
if Ma is None:\r
continue\r
MaList.append(Ma)\r
- if Ma.CanSkipbyHash():\r
- self.HashSkipModules.append(Ma)\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
- continue\r
- else:\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ Ma.GenModuleFilesHash(GlobalData.gCacheIR)\r
+ Ma.GenPreMakefileHash(GlobalData.gCacheIR)\r
+ if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):\r
+ self.HashSkipModules.append(Ma)\r
+ EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ continue\r
+\r
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:\r
# for target which must generate AutoGen code and makefile\r
self.Progress.Stop("done!")\r
if self.Target == "genmake":\r
return True\r
+\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)\r
+ Ma.GenMakeHash(GlobalData.gCacheIR)\r
+ if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):\r
+ self.HashSkipModules.append(Ma)\r
+ EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ continue\r
+ else:\r
+ EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)\r
+\r
self.BuildModules.append(Ma)\r
# Initialize all modules in tracking to 'FAIL'\r
if Ma.Arch not in GlobalData.gModuleBuildTracking:\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
+ # Add Platform and Package level hash in share_data for module hash calculation later\r
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
+ for PkgName in GlobalData.gPackageHash.keys():\r
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
+\r
# multi-thread exit flag\r
ExitFlag = threading.Event()\r
ExitFlag.clear()\r
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))\r
self.BuildModules = []\r
+ TotalModules = []\r
for Arch in Wa.ArchList:\r
PcdMaList = []\r
AutoGenStart = time.time()\r
ModuleList.append(Inf)\r
Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}\r
Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}\r
+ Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}\r
for Module in ModuleList:\r
# Get ModuleAutoGen object to generate C code file and makefile\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
Ma.PlatformInfo = Pa\r
Ma.Workspace = Wa\r
PcdMaList.append(Ma)\r
- if Ma.CanSkipbyHash():\r
- self.HashSkipModules.append(Ma)\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
- continue\r
- else:\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
-\r
- # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
- # for target which must generate AutoGen code and makefile\r
-\r
- self.BuildModules.append(Ma)\r
+ TotalModules.append(Ma)\r
# Initialize all modules in tracking to 'FAIL'\r
if Ma.Arch not in GlobalData.gModuleBuildTracking:\r
GlobalData.gModuleBuildTracking[Ma.Arch] = dict()\r
if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:\r
GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'\r
+\r
mqueue = mp.Queue()\r
for m in Pa.GetAllModuleInfo:\r
mqueue.put(m)\r
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))\r
Pa.DataPipe.dump(data_pipe_file)\r
- autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)\r
- self.Progress.Stop("done!")\r
- self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)\r
+\r
+ # Skip cache hit modules\r
+ if GlobalData.gBinCacheSource:\r
+ for Ma in TotalModules:\r
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:\r
+ self.HashSkipModules.append(Ma)\r
+ continue\r
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:\r
+ self.HashSkipModules.append(Ma)\r
+ continue\r
+ self.BuildModules.append(Ma)\r
+ else:\r
+ self.BuildModules.extend(TotalModules)\r
+\r
if not autogen_rt:\r
self.AutoGenMgr.TerminateWorkers()\r
self.AutoGenMgr.join(0.1)\r
raise FatalError(errorcode)\r
+ self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
+ self.Progress.Stop("done!")\r
+\r
+ if GlobalData.gBinCacheSource:\r
+ EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))\r
+ CacheHitMa = set()\r
+ CacheNotHitMa = set()\r
+ for IR in GlobalData.gCacheIR.keys():\r
+ if 'PlatformHash' in IR or 'PackageHash' in IR:\r
+ continue\r
+ if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit:\r
+ CacheHitMa.add(IR)\r
+ else:\r
+ # There might be binary module or module which has .inc files, not count for cache miss\r
+ CacheNotHitMa.add(IR)\r
+ EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa)))\r
+\r
for Arch in Wa.ArchList:\r
MakeStart = time.time()\r
- for Ma in self.BuildModules:\r
+ for Ma in set(self.BuildModules):\r
# Generate build task for the module\r
if not Ma.IsBinaryModule:\r
Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))\r
return TimeDurStr\r
else:\r
return None\r
+def ThreadNum():\r
+ ThreadNumber = BuildOption.ThreadNumber\r
+ if ThreadNumber is None:\r
+ ThreadNumber = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]\r
+ if ThreadNumber == '':\r
+ ThreadNumber = 0\r
+ else:\r
+ ThreadNumber = int(ThreadNumber, 0)\r
\r
+ if ThreadNumber == 0:\r
+ try:\r
+ ThreadNumber = multiprocessing.cpu_count()\r
+ except (ImportError, NotImplementedError):\r
+ ThreadNumber = 1\r
+ return ThreadNumber\r
## Tool entrance method\r
#\r
# This method mainly dispatch specific methods per the command line options.\r
# @retval 0 Tool was successful\r
# @retval 1 Tool failed\r
#\r
+LogQMaxSize = ThreadNum() * 10\r
def Main():\r
StartTime = time.time()\r
\r
#\r
# Create a log Queue\r
#\r
- LogQ = mp.Queue()\r
+ LogQ = mp.Queue(LogQMaxSize)\r
# Initialize log system\r
EdkLogger.LogClientInitialize(LogQ)\r
GlobalData.gCommand = sys.argv[1:]\r
projects / mirror_edk2.git / blobdiff