from GenFds.GenFds import GenFds, GenFdsApi\r
import multiprocessing as mp\r
from multiprocessing import Manager\r
+from AutoGen.DataPipe import MemoryDataPipe\r
+from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo\r
+from GenFds.FdfParser import FdfParser\r
\r
\r
## standard targets of build command\r
#\r
def AddDependency(self, Dependency):\r
for Dep in Dependency:\r
- if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash():\r
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR):\r
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list\r
\r
## The thread wrapper of LaunchCommand function\r
\r
# Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'\r
# If Module or Lib is being tracked, it did not fail header check test, and built successfully\r
- if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking and\r
- self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and\r
- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] != 'FAIL_METAFILE' and\r
+ if (self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and\r
+ GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] != 'FAIL_METAFILE' and\r
not BuildTask._ErrorFlag.isSet()\r
):\r
- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] = 'SUCCESS'\r
+ GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = 'SUCCESS'\r
\r
# indicate there's a thread is available for another build task\r
BuildTask._RunningQueueLock.acquire()\r
GlobalData.gUseHashCache = BuildOptions.UseHashCache\r
GlobalData.gBinCacheDest = BuildOptions.BinCacheDest\r
GlobalData.gBinCacheSource = BuildOptions.BinCacheSource\r
- GlobalData.gEnableGenfdsMultiThread = BuildOptions.GenfdsMultiThread\r
+ GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread\r
GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck\r
\r
if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:\r
self.AutoGenMgr = None\r
EdkLogger.info("")\r
os.chdir(self.WorkspaceDir)\r
- self.share_data = Manager().dict()\r
+ GlobalData.gCacheIR = Manager().dict()\r
self.log_q = log_q\r
+ GlobalData.file_lock = mp.Lock()\r
+ GlobalData.cache_lock = mp.Lock()\r
def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):\r
try:\r
if SkipAutoGen:\r
return True,0\r
feedback_q = mp.Queue()\r
- file_lock = mp.Lock()\r
error_event = mp.Event()\r
- auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
+ FfsCmd = DataPipe.Get("FfsCommand")\r
+ if FfsCmd is None:\r
+ FfsCmd = {}\r
+ GlobalData.FfsCmd = FfsCmd\r
+ GlobalData.libConstPcd = DataPipe.Get("LibConstPcd")\r
+ GlobalData.Refes = DataPipe.Get("REFS")\r
+ auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,GlobalData.cache_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)\r
self.AutoGenMgr.start()\r
for w in auto_workers:\r
w.start()\r
if PcdMaList is not None:\r
for PcdMa in PcdMaList:\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ PcdMa.GenModuleFilesHash(share_data)\r
+ PcdMa.GenPreMakefileHash(share_data)\r
+ if PcdMa.CanSkipbyPreMakefileCache(share_data):\r
+ continue\r
+\r
PcdMa.CreateCodeFile(False)\r
- PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))\r
+ PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))\r
+\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ PcdMa.GenMakeHeaderFilesHash(share_data)\r
+ PcdMa.GenMakeHash(share_data)\r
+ if PcdMa.CanSkipbyMakeCache(share_data):\r
+ continue\r
\r
self.AutoGenMgr.join()\r
rt = self.AutoGenMgr.Status\r
return rt, 0\r
- except Exception as e:\r
- return False,e.errcode\r
+ except FatalError as e:\r
+ return False, e.args[0]\r
+ except:\r
+ return False, UNKNOWN_ERROR\r
\r
## Load configuration\r
#\r
return\r
\r
# GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash\r
- for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys():\r
- for moduleAutoGenObj in GlobalData.gModuleBuildTracking[moduleAutoGenObjArch].keys():\r
- # Skip invalidating for Successful Module/Lib builds\r
- if GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][moduleAutoGenObj] == 'SUCCESS':\r
- continue\r
+ for Ma in GlobalData.gModuleBuildTracking:\r
+ # Skip invalidating for Successful Module/Lib builds\r
+ if GlobalData.gModuleBuildTracking[Ma] == 'SUCCESS':\r
+ continue\r
\r
- # The module failed to build, failed to start building, or failed the header check test from this point on\r
+ # The module failed to build, failed to start building, or failed the header check test from this point on\r
\r
- # Remove .hash from build\r
- ModuleHashFile = os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash")\r
- if os.path.exists(ModuleHashFile):\r
- os.remove(ModuleHashFile)\r
+ # Remove .hash from build\r
+ ModuleHashFile = os.path.join(Ma.BuildDir, Ma.Name + ".hash")\r
+ if os.path.exists(ModuleHashFile):\r
+ os.remove(ModuleHashFile)\r
\r
- # Remove .hash file from cache\r
- if GlobalData.gBinCacheDest:\r
- FileDir = os.path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName)\r
- HashFile = os.path.join(FileDir, moduleAutoGenObj.Name + '.hash')\r
- if os.path.exists(HashFile):\r
- os.remove(HashFile)\r
+ # Remove .hash file from cache\r
+ if GlobalData.gBinCacheDest:\r
+ FileDir = os.path.join(GlobalData.gBinCacheDest, Ma.PlatformInfo.OutputDir, Ma.BuildTarget + "_" + Ma.ToolChain, Ma.Arch, Ma.SourceDir, Ma.MetaFile.BaseName)\r
+ HashFile = os.path.join(FileDir, Ma.Name + '.hash')\r
+ if os.path.exists(HashFile):\r
+ os.remove(HashFile)\r
\r
## Build a module or platform\r
#\r
for m in AutoGenObject.GetAllModuleInfo:\r
mqueue.put(m)\r
\r
- AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}\r
+ AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}\r
+ AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}\r
+ AutoGenObject.CreateLibModuelDirs()\r
+ AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}\r
+ AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}\r
+ AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}\r
self.Progress.Start("Generating makefile and code")\r
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))\r
AutoGenObject.DataPipe.dump(data_pipe_file)\r
- autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)\r
+ autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)\r
+ AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
+ with open(AutoGenIdFile,"w") as fw:\r
+ fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))\r
+ fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)\r
+ fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))\r
self.Progress.Stop("done!")\r
if not autogen_rt:\r
self.AutoGenMgr.TerminateWorkers()\r
- self.AutoGenMgr.join(0.1)\r
+ self.AutoGenMgr.join(1)\r
raise FatalError(errorcode)\r
AutoGenObject.CreateCodeFile(False)\r
AutoGenObject.CreateMakeFile(False)\r
Ma.Workspace = Wa\r
PcdMaList.append(Ma)\r
self.BuildModules.append(Ma)\r
+ Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}\r
+ Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}\r
self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)\r
\r
# Create MAP file when Load Fix Address is enabled.\r
# Save MAP buffer into MAP file.\r
#\r
self._SaveMapFile (MapBuffer, Wa)\r
+ self.CreateGuidedSectionToolsFile(Wa)\r
\r
## Build active module for different build targets, different tool chains and different archs\r
#\r
CmdListDict = None\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
+\r
+ # Add Platform and Package level hash in share_data for module hash calculation later\r
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
+ for PkgName in GlobalData.gPackageHash.keys():\r
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
+ GlobalData.file_lock = mp.Lock()\r
+ GlobalData.cache_lock = mp.Lock()\r
+ GlobalData.FfsCmd = CmdListDict\r
+\r
self.Progress.Stop("done!")\r
MaList = []\r
ExitFlag = threading.Event()\r
AutoGenStart = time.time()\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
+ GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd")\r
+ GlobalData.Refes = Pa.DataPipe.Get("REFS")\r
for Module in Pa.Platform.Modules:\r
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
if Ma is None:\r
continue\r
+ if Ma.PcdIsDriver:\r
+ Ma.PlatformInfo = Pa\r
+ Ma.Workspace = Wa\r
MaList.append(Ma)\r
- if Ma.CanSkipbyHash():\r
- self.HashSkipModules.append(Ma)\r
- if GlobalData.gBinCacheSource:\r
+\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ Ma.GenModuleFilesHash(GlobalData.gCacheIR)\r
+ Ma.GenPreMakefileHash(GlobalData.gCacheIR)\r
+ if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):\r
+ self.HashSkipModules.append(Ma)\r
EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
- continue\r
- else:\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ continue\r
+\r
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:\r
# for target which must generate AutoGen code and makefile\r
return True\r
if not self.SkipAutoGen or self.Target == 'genmake':\r
self.Progress.Start("Generating makefile")\r
- if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict:\r
- Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch])\r
- del CmdListDict[Module.File, Arch]\r
+ if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:\r
+ Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])\r
+ del CmdListDict[Module.Path, Arch]\r
else:\r
Ma.CreateMakeFile(True)\r
self.Progress.Stop("done!")\r
if self.Target == "genmake":\r
return True\r
+\r
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
+ Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)\r
+ Ma.GenMakeHash(GlobalData.gCacheIR)\r
+ if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):\r
+ self.HashSkipModules.append(Ma)\r
+ EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ continue\r
+ else:\r
+ EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)\r
+\r
self.BuildModules.append(Ma)\r
# Initialize all modules in tracking to 'FAIL'\r
- if Ma.Arch not in GlobalData.gModuleBuildTracking:\r
- GlobalData.gModuleBuildTracking[Ma.Arch] = dict()\r
- if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:\r
- GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'\r
+ GlobalData.gModuleBuildTracking[Ma] = 'FAIL'\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
MakeStart = time.time()\r
for Ma in self.BuildModules:\r
tmpInf, tmpArch = GenFfsDict[Cmd]\r
CmdSetDict[tmpInf, tmpArch].add(Cmd)\r
return CmdSetDict\r
+ def VerifyAutoGenFiles(self):\r
+ AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
+ try:\r
+ with open(AutoGenIdFile) as fd:\r
+ lines = fd.readlines()\r
+ except:\r
+ return None\r
+ for line in lines:\r
+ if "Arch" in line:\r
+ ArchList = line.strip().split("=")[1].split("|")\r
+ if "BuildDir" in line:\r
+ BuildDir = line.split("=")[1].strip()\r
+ if "PlatformGuid" in line:\r
+ PlatformGuid = line.split("=")[1].strip()\r
+ GlobalVarList = []\r
+ for arch in ArchList:\r
+ global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))\r
+ if not os.path.exists(global_var):\r
+ return None\r
+ GlobalVarList.append(global_var)\r
+ for global_var in GlobalVarList:\r
+ data_pipe = MemoryDataPipe()\r
+ data_pipe.load(global_var)\r
+ target = data_pipe.Get("P_Info").get("Target")\r
+ toolchain = data_pipe.Get("P_Info").get("ToolChain")\r
+ archlist = data_pipe.Get("P_Info").get("ArchList")\r
+ Arch = data_pipe.Get("P_Info").get("Arch")\r
+ active_p = data_pipe.Get("P_Info").get("ActivePlatform")\r
+ workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")\r
+ PackagesPath = os.getenv("PACKAGES_PATH")\r
+ mws.setWs(workspacedir, PackagesPath)\r
+ LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")\r
+ ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")\r
+\r
+ for m_build_dir in LibraryBuildDirectoryList:\r
+ if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):\r
+ return None\r
+ for m_build_dir in ModuleBuildDirectoryList:\r
+ if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):\r
+ return None\r
+ Wa = WorkSpaceInfo(\r
+ workspacedir,active_p,target,toolchain,archlist\r
+ )\r
+ Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)\r
+ Wa.AutoGenObjectList.append(Pa)\r
+ return Wa\r
+ def SetupMakeSetting(self,Wa):\r
+ BuildModules = []\r
+ for Pa in Wa.AutoGenObjectList:\r
+ for m in Pa._MbList:\r
+ ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)\r
+ BuildModules.append(ma)\r
+ fdf_file = Wa.FlashDefinition\r
+ if fdf_file:\r
+ Fdf = FdfParser(fdf_file.Path)\r
+ Fdf.ParseFile()\r
+ GlobalData.gFdfParser = Fdf\r
+ if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:\r
+ FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]\r
+ for FdRegion in FdDict.RegionList:\r
+ if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):\r
+ if int(FdRegion.Offset) % 8 != 0:\r
+ EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))\r
+ Wa.FdfProfile = Fdf.Profile\r
+ self.Fdf = Fdf\r
+ else:\r
+ self.Fdf = None\r
+ return BuildModules\r
\r
## Build a platform in multi-thread mode\r
#\r
+ def PerformAutoGen(self,BuildTarget,ToolChain):\r
+ WorkspaceAutoGenTime = time.time()\r
+ Wa = WorkspaceAutoGen(\r
+ self.WorkspaceDir,\r
+ self.PlatformFile,\r
+ BuildTarget,\r
+ ToolChain,\r
+ self.ArchList,\r
+ self.BuildDatabase,\r
+ self.TargetTxt,\r
+ self.ToolDef,\r
+ self.Fdf,\r
+ self.FdList,\r
+ self.FvList,\r
+ self.CapList,\r
+ self.SkuId,\r
+ self.UniFlag,\r
+ self.Progress\r
+ )\r
+ self.Fdf = Wa.FdfFile\r
+ self.LoadFixAddress = Wa.Platform.LoadFixAddress\r
+ self.BuildReport.AddPlatformReport(Wa)\r
+ Wa.CreateMakeFile(False)\r
+\r
+ # Add ffs build to makefile\r
+ CmdListDict = {}\r
+ if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
+\r
+ # Add Platform and Package level hash in share_data for module hash calculation later\r
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
+ for PkgName in GlobalData.gPackageHash.keys():\r
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
+\r
+ self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))\r
+ BuildModules = []\r
+ TotalModules = []\r
+ for Arch in Wa.ArchList:\r
+ PcdMaList = []\r
+ AutoGenStart = time.time()\r
+ GlobalData.gGlobalDefines['ARCH'] = Arch\r
+ Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
+ if Pa is None:\r
+ continue\r
+ ModuleList = []\r
+ for Inf in Pa.Platform.Modules:\r
+ ModuleList.append(Inf)\r
+ # Add the INF only list in FDF\r
+ if GlobalData.gFdfParser is not None:\r
+ for InfName in GlobalData.gFdfParser.Profile.InfList:\r
+ Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)\r
+ if Inf in Pa.Platform.Modules:\r
+ continue\r
+ ModuleList.append(Inf)\r
+ Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}\r
+ Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}\r
+ Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}\r
+ Pa.CreateLibModuelDirs()\r
+ Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}\r
+ Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}\r
+ Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}\r
+ ModuleCodaFile = {}\r
+ for ma in Pa.ModuleAutoGenList:\r
+ ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]\r
+ Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}\r
+ for Module in ModuleList:\r
+ # Get ModuleAutoGen object to generate C code file and makefile\r
+ Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
+\r
+ if Ma is None:\r
+ continue\r
+ if Ma.PcdIsDriver:\r
+ Ma.PlatformInfo = Pa\r
+ Ma.Workspace = Wa\r
+ PcdMaList.append(Ma)\r
+ TotalModules.append(Ma)\r
+ # Initialize all modules in tracking to 'FAIL'\r
+ GlobalData.gModuleBuildTracking[Ma] = 'FAIL'\r
+\r
+\r
+ mqueue = mp.Queue()\r
+ for m in Pa.GetAllModuleInfo:\r
+ mqueue.put(m)\r
+ data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))\r
+ Pa.DataPipe.dump(data_pipe_file)\r
+\r
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)\r
+\r
+ # Skip cache hit modules\r
+ if GlobalData.gBinCacheSource:\r
+ for Ma in TotalModules:\r
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:\r
+ self.HashSkipModules.append(Ma)\r
+ continue\r
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:\r
+ self.HashSkipModules.append(Ma)\r
+ continue\r
+ BuildModules.append(Ma)\r
+ else:\r
+ BuildModules.extend(TotalModules)\r
+\r
+ if not autogen_rt:\r
+ self.AutoGenMgr.TerminateWorkers()\r
+ self.AutoGenMgr.join(1)\r
+ raise FatalError(errorcode)\r
+ self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
+ AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
+ with open(AutoGenIdFile,"w") as fw:\r
+ fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))\r
+ fw.write("BuildDir=%s\n" % Wa.BuildDir)\r
+ fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))\r
+ self.Progress.Stop("done!")\r
+ return Wa, BuildModules\r
+\r
def _MultiThreadBuildPlatform(self):\r
SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)\r
for BuildTarget in self.BuildTargetList:\r
GlobalData.gGlobalDefines['TARGET'] = BuildTarget\r
index = 0\r
for ToolChain in self.ToolChainList:\r
- WorkspaceAutoGenTime = time.time()\r
GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain\r
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain\r
GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]\r
index += 1\r
- Wa = WorkspaceAutoGen(\r
- self.WorkspaceDir,\r
- self.PlatformFile,\r
- BuildTarget,\r
- ToolChain,\r
- self.ArchList,\r
- self.BuildDatabase,\r
- self.TargetTxt,\r
- self.ToolDef,\r
- self.Fdf,\r
- self.FdList,\r
- self.FvList,\r
- self.CapList,\r
- self.SkuId,\r
- self.UniFlag,\r
- self.Progress\r
- )\r
- self.Fdf = Wa.FdfFile\r
- self.LoadFixAddress = Wa.Platform.LoadFixAddress\r
- self.BuildReport.AddPlatformReport(Wa)\r
- Wa.CreateMakeFile(False)\r
-\r
- # Add ffs build to makefile\r
- CmdListDict = {}\r
- if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
- CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
-\r
- # multi-thread exit flag\r
ExitFlag = threading.Event()\r
ExitFlag.clear()\r
- self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))\r
- self.BuildModules = []\r
- for Arch in Wa.ArchList:\r
- PcdMaList = []\r
- AutoGenStart = time.time()\r
- GlobalData.gGlobalDefines['ARCH'] = Arch\r
- Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
- if Pa is None:\r
- continue\r
- ModuleList = []\r
- for Inf in Pa.Platform.Modules:\r
- ModuleList.append(Inf)\r
- # Add the INF only list in FDF\r
- if GlobalData.gFdfParser is not None:\r
- for InfName in GlobalData.gFdfParser.Profile.InfList:\r
- Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)\r
- if Inf in Pa.Platform.Modules:\r
- continue\r
- ModuleList.append(Inf)\r
- Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}\r
- Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}\r
- for Module in ModuleList:\r
- # Get ModuleAutoGen object to generate C code file and makefile\r
- Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
-\r
- if Ma is None:\r
- continue\r
- if Ma.PcdIsDriver:\r
- Ma.PlatformInfo = Pa\r
- Ma.Workspace = Wa\r
- PcdMaList.append(Ma)\r
- if Ma.CanSkipbyHash():\r
- self.HashSkipModules.append(Ma)\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ if self.SkipAutoGen:\r
+ Wa = self.VerifyAutoGenFiles()\r
+ if Wa is None:\r
+ self.SkipAutoGen = False\r
+ Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)\r
+ else:\r
+ GlobalData.gAutoGenPhase = True\r
+ self.BuildModules = self.SetupMakeSetting(Wa)\r
+ else:\r
+ Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)\r
+ Pa = Wa.AutoGenObjectList[0]\r
+ GlobalData.gAutoGenPhase = False\r
+\r
+ if GlobalData.gBinCacheSource:\r
+ EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))\r
+ CacheHitMa = set()\r
+ CacheNotHitMa = set()\r
+ for IR in GlobalData.gCacheIR.keys():\r
+ if 'PlatformHash' in IR or 'PackageHash' in IR:\r
continue\r
+ if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit:\r
+ CacheHitMa.add(IR)\r
else:\r
- if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
-\r
- # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
- # for target which must generate AutoGen code and makefile\r
+ # There might be binary module or module which has .inc files, not count for cache miss\r
+ CacheNotHitMa.add(IR)\r
+ EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa)))\r
\r
- self.BuildModules.append(Ma)\r
- # Initialize all modules in tracking to 'FAIL'\r
- if Ma.Arch not in GlobalData.gModuleBuildTracking:\r
- GlobalData.gModuleBuildTracking[Ma.Arch] = dict()\r
- if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:\r
- GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'\r
- mqueue = mp.Queue()\r
- for m in Pa.GetAllModuleInfo:\r
- mqueue.put(m)\r
- data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))\r
- Pa.DataPipe.dump(data_pipe_file)\r
- autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)\r
-\r
- if not autogen_rt:\r
- self.AutoGenMgr.TerminateWorkers()\r
- self.AutoGenMgr.join(0.1)\r
- raise FatalError(errorcode)\r
- self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
- self.Progress.Stop("done!")\r
for Arch in Wa.ArchList:\r
MakeStart = time.time()\r
- for Ma in self.BuildModules:\r
+ for Ma in set(self.BuildModules):\r
# Generate build task for the module\r
if not Ma.IsBinaryModule:\r
Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))\r
BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)\r
\r
# in case there's an interruption. we need a full version of makefile for platform\r
- Pa.CreateMakeFile(False)\r
+\r
if BuildTask.HasError():\r
self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
MakeContiue = time.time()\r
-\r
#\r
#\r
# All modules have been put in build tasks queue. Tell task scheduler\r
#\r
# Get Module List\r
#\r
- ModuleList = {}\r
- for Pa in Wa.AutoGenObjectList:\r
- for Ma in Pa.ModuleAutoGenList:\r
- if Ma is None:\r
- continue\r
- if not Ma.IsLibrary:\r
- ModuleList[Ma.Guid.upper()] = Ma\r
+ ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}\r
+\r
#\r
# Rebase module to the preferred memory address before GenFds\r
#\r
# Save MAP buffer into MAP file.\r
#\r
self._SaveMapFile(MapBuffer, Wa)\r
+ self.CreateGuidedSectionToolsFile(Wa)\r
self.invalidateHash()\r
-\r
## Generate GuidedSectionTools.txt in the FV directories.\r
#\r
- def CreateGuidedSectionToolsFile(self):\r
+ def CreateGuidedSectionToolsFile(self,Wa):\r
for BuildTarget in self.BuildTargetList:\r
for ToolChain in self.ToolChainList:\r
- Wa = WorkspaceAutoGen(\r
- self.WorkspaceDir,\r
- self.PlatformFile,\r
- BuildTarget,\r
- ToolChain,\r
- self.ArchList,\r
- self.BuildDatabase,\r
- self.TargetTxt,\r
- self.ToolDef,\r
- self.Fdf,\r
- self.FdList,\r
- self.FvList,\r
- self.CapList,\r
- self.SkuId,\r
- self.UniFlag\r
- )\r
FvDir = Wa.FvDir\r
if not os.path.exists(FvDir):\r
continue\r
self._BuildPlatform()\r
else:\r
self._MultiThreadBuildPlatform()\r
- self.CreateGuidedSectionToolsFile()\r
else:\r
self.SpawnMode = False\r
self._BuildModule()\r
projects / mirror_edk2.git / blobdiff