from AutoGen.DataPipe import MemoryDataPipe\r
from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo\r
from GenFds.FdfParser import FdfParser\r
-\r
+from AutoGen.IncludesAutoGen import IncludesAutoGen\r
+from GenFds.GenFds import resetFdsGlobalVariable\r
\r
## standard targets of build command\r
gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']\r
# @param To The stream message put on\r
# @param ExitFlag The flag used to indicate stopping reading\r
#\r
-def ReadMessage(From, To, ExitFlag):\r
+def ReadMessage(From, To, ExitFlag,MemTo=None):\r
while True:\r
# read one line a time\r
Line = From.readline()\r
# empty string means "end"\r
if Line is not None and Line != b"":\r
- To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))\r
+ LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')\r
+ if MemTo is not None:\r
+ if "Note: including file:" == LineStr.lstrip()[:21]:\r
+ MemTo.append(LineStr)\r
+ else:\r
+ To(LineStr)\r
+ MemTo.append(LineStr)\r
+ else:\r
+ To(LineStr)\r
else:\r
break\r
if ExitFlag.isSet():\r
break\r
\r
+class MakeSubProc(Popen):\r
+ def __init__(self,*args, **argv):\r
+ super(MakeSubProc,self).__init__(*args, **argv)\r
+ self.ProcOut = []\r
+\r
## Launch an external program\r
#\r
# This method will call subprocess.Popen to execute an external program with\r
# @param Command A list or string containing the call of the program\r
# @param WorkingDir The directory in which the program will be running\r
#\r
-def LaunchCommand(Command, WorkingDir):\r
+def LaunchCommand(Command, WorkingDir,ModuleAuto = None):\r
BeginTime = time.time()\r
# if working directory doesn't exist, Popen() will raise an exception\r
if not os.path.isdir(WorkingDir):\r
EndOfProcedure = None\r
try:\r
# launch the command\r
- Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)\r
+ Proc = MakeSubProc(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)\r
\r
# launch two threads to read the STDOUT and STDERR\r
EndOfProcedure = Event()\r
EndOfProcedure.clear()\r
if Proc.stdout:\r
- StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))\r
+ StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))\r
StdOutThread.setName("STDOUT-Redirector")\r
StdOutThread.setDaemon(False)\r
StdOutThread.start()\r
\r
if Proc.stderr:\r
- StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))\r
+ StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure,Proc.ProcOut))\r
StdErrThread.setName("STDERR-Redirector")\r
StdErrThread.setDaemon(False)\r
StdErrThread.start()\r
EdkLogger.info(RespContent)\r
\r
EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))\r
+ if ModuleAuto:\r
+ iau = IncludesAutoGen(WorkingDir,ModuleAuto)\r
+ if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:\r
+ iau.CreateDepsFileForMsvc(Proc.ProcOut)\r
+ else:\r
+ iau.UpdateDepsFileforNonMsvc()\r
+ iau.UpdateDepsFileforTrim()\r
+ iau.CreateModuleDeps()\r
+ iau.CreateDepsInclude()\r
return "%dms" % (int(round((time.time() - BeginTime) * 1000)))\r
\r
## The smallest unit that can be built in multi-thread build mode\r
#\r
def AddDependency(self, Dependency):\r
for Dep in Dependency:\r
- if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR):\r
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):\r
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list\r
\r
## The thread wrapper of LaunchCommand function\r
#\r
def _CommandThread(self, Command, WorkingDir):\r
try:\r
- self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)\r
+ self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)\r
self.CompleteFlag = True\r
\r
- # Run hash operation post dependency, to account for libs\r
- if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:\r
- HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")\r
- SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True)\r
+ # Run hash operation post dependency to account for libs\r
+ # Run if --hash or --binary-destination\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ self.BuildItem.BuildObject.GenModuleHash()\r
+ if GlobalData.gBinCacheDest:\r
+ self.BuildItem.BuildObject.GenCMakeHash()\r
+\r
except:\r
#\r
# TRICK: hide the output of threads left running, so that the user can\r
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \\r
(threading.currentThread().getName(), Command, WorkingDir)\r
\r
- # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'\r
- # If Module or Lib is being tracked, it did not fail header check test, and built successfully\r
- if (self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and\r
- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] != 'FAIL_METAFILE' and\r
- not BuildTask._ErrorFlag.isSet()\r
- ):\r
- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = 'SUCCESS'\r
-\r
# indicate there's a thread is available for another build task\r
BuildTask._RunningQueueLock.acquire()\r
BuildTask._RunningQueue.pop(self.BuildItem)\r
self.AutoGenMgr = None\r
EdkLogger.info("")\r
os.chdir(self.WorkspaceDir)\r
- GlobalData.gCacheIR = Manager().dict()\r
self.log_q = log_q\r
GlobalData.file_lock = mp.Lock()\r
- GlobalData.cache_lock = mp.Lock()\r
- def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):\r
+ # Init cache data for local only\r
+ GlobalData.gPackageHashFile = dict()\r
+ GlobalData.gModulePreMakeCacheStatus = dict()\r
+ GlobalData.gModuleMakeCacheStatus = dict()\r
+ GlobalData.gHashChainStatus = dict()\r
+ GlobalData.gCMakeHashFile = dict()\r
+ GlobalData.gModuleHashFile = dict()\r
+ GlobalData.gFileHashDict = dict()\r
+ GlobalData.gModuleAllCacheStatus = set()\r
+ GlobalData.gModuleCacheHit = set()\r
+\r
+ def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):\r
try:\r
if SkipAutoGen:\r
return True,0\r
if FfsCmd is None:\r
FfsCmd = {}\r
GlobalData.FfsCmd = FfsCmd\r
- GlobalData.libConstPcd = DataPipe.Get("LibConstPcd")\r
- GlobalData.Refes = DataPipe.Get("REFS")\r
- auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,GlobalData.cache_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
+ auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)\r
self.AutoGenMgr.start()\r
for w in auto_workers:\r
w.start()\r
if PcdMaList is not None:\r
for PcdMa in PcdMaList:\r
- if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- PcdMa.GenModuleFilesHash(share_data)\r
- PcdMa.GenPreMakefileHash(share_data)\r
- if PcdMa.CanSkipbyPreMakefileCache(share_data):\r
- continue\r
+ # SourceFileList calling sequence impact the makefile string sequence.\r
+ # Create cached SourceFileList here to unify its calling sequence for both\r
+ # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.\r
+ RetVal = PcdMa.SourceFileList\r
+ # Force cache miss for PCD driver\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:\r
+ cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))\r
\r
PcdMa.CreateCodeFile(False)\r
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))\r
\r
+ # Force cache miss for PCD driver\r
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- PcdMa.GenMakeHeaderFilesHash(share_data)\r
- PcdMa.GenMakeHash(share_data)\r
- if PcdMa.CanSkipbyMakeCache(share_data):\r
- continue\r
+ cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))\r
\r
self.AutoGenMgr.join()\r
rt = self.AutoGenMgr.Status\r
EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')\r
EdkLogger.info("\n- Postbuild Done -\n")\r
\r
- ## Error handling for hash feature\r
- #\r
- # On BuildTask error, iterate through the Module Build tracking\r
- # dictionary to determine wheather a module failed to build. Invalidate\r
- # the hash associated with that module by removing it from storage.\r
- #\r
- #\r
- def invalidateHash(self):\r
- # Only for hashing feature\r
- if not GlobalData.gUseHashCache:\r
- return\r
-\r
- # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash\r
- for Ma in GlobalData.gModuleBuildTracking:\r
- # Skip invalidating for Successful Module/Lib builds\r
- if GlobalData.gModuleBuildTracking[Ma] == 'SUCCESS':\r
- continue\r
-\r
- # The module failed to build, failed to start building, or failed the header check test from this point on\r
-\r
- # Remove .hash from build\r
- ModuleHashFile = os.path.join(Ma.BuildDir, Ma.Name + ".hash")\r
- if os.path.exists(ModuleHashFile):\r
- os.remove(ModuleHashFile)\r
-\r
- # Remove .hash file from cache\r
- if GlobalData.gBinCacheDest:\r
- FileDir = os.path.join(GlobalData.gBinCacheDest, Ma.PlatformInfo.OutputDir, Ma.BuildTarget + "_" + Ma.ToolChain, Ma.Arch, Ma.SourceDir, Ma.MetaFile.BaseName)\r
- HashFile = os.path.join(FileDir, Ma.Name + '.hash')\r
- if os.path.exists(HashFile):\r
- os.remove(HashFile)\r
-\r
## Build a module or platform\r
#\r
# Create autogen code and makefile for a module or platform, and the launch\r
self.Progress.Start("Generating makefile and code")\r
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))\r
AutoGenObject.DataPipe.dump(data_pipe_file)\r
- autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)\r
+ cqueue = mp.Queue()\r
+ autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)\r
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
with open(AutoGenIdFile,"w") as fw:\r
fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))\r
LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
return True\r
\r
# build library\r
if Target == 'libraries':\r
- for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
+ DirList = []\r
+ for Lib in AutoGenObject.LibraryAutoGenList:\r
+ if not Lib.IsBinaryModule:\r
+ DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))\r
+ for Lib, LibAutoGen in DirList:\r
NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']\r
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)\r
return True\r
\r
# build module\r
if Target == 'modules':\r
- for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
+ DirList = []\r
+ for Lib in AutoGenObject.LibraryAutoGenList:\r
+ if not Lib.IsBinaryModule:\r
+ DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))\r
+ for Lib, LibAutoGen in DirList:\r
NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']\r
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
- for Mod in AutoGenObject.ModuleBuildDirectoryList:\r
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)\r
+\r
+ DirList = []\r
+ for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:\r
+ if not ModuleAutoGen.IsBinaryModule:\r
+ DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))\r
+ for Mod,ModAutoGen in DirList:\r
NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']\r
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
return True\r
\r
AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
return True\r
\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
- # Add Platform and Package level hash in share_data for module hash calculation later\r
- if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
- GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
- for PkgName in GlobalData.gPackageHash.keys():\r
- GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
GlobalData.file_lock = mp.Lock()\r
- GlobalData.cache_lock = mp.Lock()\r
GlobalData.FfsCmd = CmdListDict\r
\r
self.Progress.Stop("done!")\r
AutoGenStart = time.time()\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
- GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd")\r
- GlobalData.Refes = Pa.DataPipe.Get("REFS")\r
for Module in Pa.Platform.Modules:\r
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
Ma.Workspace = Wa\r
MaList.append(Ma)\r
\r
- if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- Ma.GenModuleFilesHash(GlobalData.gCacheIR)\r
- Ma.GenPreMakefileHash(GlobalData.gCacheIR)\r
- if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):\r
- self.HashSkipModules.append(Ma)\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:\r
+ if Ma.CanSkipbyPreMakeCache():\r
continue\r
+ else:\r
+ self.PreMakeCacheMiss.add(Ma)\r
\r
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:\r
return True\r
\r
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)\r
- Ma.GenMakeHash(GlobalData.gCacheIR)\r
- if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):\r
- self.HashSkipModules.append(Ma)\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ if Ma.CanSkipbyMakeCache():\r
continue\r
else:\r
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
- Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)\r
+ self.MakeCacheMiss.add(Ma)\r
\r
self.BuildModules.append(Ma)\r
- # Initialize all modules in tracking to 'FAIL'\r
- GlobalData.gModuleBuildTracking[Ma] = 'FAIL'\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
MakeStart = time.time()\r
for Ma in self.BuildModules:\r
# we need a full version of makefile for platform\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
- self.invalidateHash()\r
Pa.CreateMakeFile(False)\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
# Start task scheduler\r
# in case there's an interruption. we need a full version of makefile for platform\r
Pa.CreateMakeFile(False)\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
BuildTask.WaitForComplete()\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
self.MakeTime += int(round((time.time() - MakeContiue)))\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
\r
self.BuildReport.AddPlatformReport(Wa, MaList)\r
# Save MAP buffer into MAP file.\r
#\r
self._SaveMapFile (MapBuffer, Wa)\r
- self.invalidateHash()\r
\r
def _GenFfsCmd(self,ArchList):\r
# convert dictionary of Cmd:(Inf,Arch)\r
if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:\r
FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]\r
for FdRegion in FdDict.RegionList:\r
- if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):\r
+ if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):\r
if int(FdRegion.Offset) % 8 != 0:\r
EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))\r
Wa.FdfProfile = Fdf.Profile\r
self.BuildReport.AddPlatformReport(Wa)\r
Wa.CreateMakeFile(False)\r
\r
- # Add ffs build to makefile\r
+ # Add ffs build to makefile\r
CmdListDict = {}\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
- # Add Platform and Package level hash in share_data for module hash calculation later\r
- if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
- GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
- for PkgName in GlobalData.gPackageHash.keys():\r
- GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
-\r
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))\r
BuildModules = []\r
- TotalModules = []\r
for Arch in Wa.ArchList:\r
PcdMaList = []\r
AutoGenStart = time.time()\r
ModuleList = []\r
for Inf in Pa.Platform.Modules:\r
ModuleList.append(Inf)\r
- # Add the INF only list in FDF\r
+ # Add the INF only list in FDF\r
if GlobalData.gFdfParser is not None:\r
for InfName in GlobalData.gFdfParser.Profile.InfList:\r
Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)\r
Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}\r
Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}\r
Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}\r
+ # Prepare the cache share data for multiprocessing\r
+ Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}\r
ModuleCodaFile = {}\r
for ma in Pa.ModuleAutoGenList:\r
ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]\r
Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}\r
+ # ModuleList contains all driver modules only\r
for Module in ModuleList:\r
- # Get ModuleAutoGen object to generate C code file and makefile\r
+ # Get ModuleAutoGen object to generate C code file and makefile\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
-\r
if Ma is None:\r
continue\r
if Ma.PcdIsDriver:\r
Ma.PlatformInfo = Pa\r
Ma.Workspace = Wa\r
PcdMaList.append(Ma)\r
- TotalModules.append(Ma)\r
- # Initialize all modules in tracking to 'FAIL'\r
- GlobalData.gModuleBuildTracking[Ma] = 'FAIL'\r
-\r
+ self.AllDrivers.add(Ma)\r
+ self.AllModules.add(Ma)\r
\r
mqueue = mp.Queue()\r
+ cqueue = mp.Queue()\r
for m in Pa.GetAllModuleInfo:\r
mqueue.put(m)\r
+ module_file,module_root,module_path,module_basename,\\r
+ module_originalpath,module_arch,IsLib = m\r
+ Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\\r
+ ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
+ self.AllModules.add(Ma)\r
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))\r
Pa.DataPipe.dump(data_pipe_file)\r
\r
- autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)\r
-\r
- # Skip cache hit modules\r
- if GlobalData.gBinCacheSource:\r
- for Ma in TotalModules:\r
- if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
- GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:\r
- self.HashSkipModules.append(Ma)\r
- continue\r
- if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
- GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:\r
- self.HashSkipModules.append(Ma)\r
- continue\r
- BuildModules.append(Ma)\r
- else:\r
- BuildModules.extend(TotalModules)\r
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)\r
\r
if not autogen_rt:\r
self.AutoGenMgr.TerminateWorkers()\r
self.AutoGenMgr.join(1)\r
raise FatalError(errorcode)\r
+\r
+ if GlobalData.gUseHashCache:\r
+ for item in GlobalData.gModuleAllCacheStatus:\r
+ (MetaFilePath, Arch, CacheStr, Status) = item\r
+ Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\\r
+ ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
+ if CacheStr == "PreMakeCache" and Status == False:\r
+ self.PreMakeCacheMiss.add(Ma)\r
+ if CacheStr == "PreMakeCache" and Status == True:\r
+ self.PreMakeCacheHit.add(Ma)\r
+ GlobalData.gModuleCacheHit.add(Ma)\r
+ if CacheStr == "MakeCache" and Status == False:\r
+ self.MakeCacheMiss.add(Ma)\r
+ if CacheStr == "MakeCache" and Status == True:\r
+ self.MakeCacheHit.add(Ma)\r
+ GlobalData.gModuleCacheHit.add(Ma)\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
with open(AutoGenIdFile,"w") as fw:\r
fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))\r
fw.write("BuildDir=%s\n" % Wa.BuildDir)\r
fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))\r
+\r
+ if GlobalData.gBinCacheSource:\r
+ BuildModules.extend(self.MakeCacheMiss)\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:\r
+ BuildModules.extend(self.PreMakeCacheMiss)\r
+ else:\r
+ BuildModules.extend(self.AllDrivers)\r
+\r
self.Progress.Stop("done!")\r
return Wa, BuildModules\r
\r
GlobalData.gGlobalDefines['TARGET'] = BuildTarget\r
index = 0\r
for ToolChain in self.ToolChainList:\r
+ resetFdsGlobalVariable()\r
GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain\r
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain\r
GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]\r
GlobalData.gAutoGenPhase = False\r
\r
if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))\r
- CacheHitMa = set()\r
- CacheNotHitMa = set()\r
- for IR in GlobalData.gCacheIR.keys():\r
- if 'PlatformHash' in IR or 'PackageHash' in IR:\r
- continue\r
- if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit:\r
- CacheHitMa.add(IR)\r
- else:\r
- # There might be binary module or module which has .inc files, not count for cache miss\r
- CacheNotHitMa.add(IR)\r
- EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa)))\r
+ EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))\r
+ EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))\r
+ EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))\r
\r
for Arch in Wa.ArchList:\r
MakeStart = time.time()\r
# we need a full version of makefile for platform\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
- self.invalidateHash()\r
Pa.CreateMakeFile(False)\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
# Start task scheduler\r
# in case there's an interruption. we need a full version of makefile for platform\r
\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
BuildTask.WaitForComplete()\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
+ #\r
+ # Get Module List\r
+ #\r
+ ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}\r
self.BuildModules = []\r
self.MakeTime += int(round((time.time() - MakeContiue)))\r
#\r
# has been signaled.\r
#\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
\r
# Create MAP file when Load Fix Address is enabled.\r
#\r
if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:\r
EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")\r
- #\r
- # Get Module List\r
- #\r
- ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}\r
\r
#\r
# Rebase module to the preferred memory address before GenFds\r
#\r
self._SaveMapFile(MapBuffer, Wa)\r
self.CreateGuidedSectionToolsFile(Wa)\r
- self.invalidateHash()\r
## Generate GuidedSectionTools.txt in the FV directories.\r
#\r
def CreateGuidedSectionToolsFile(self,Wa):\r
## Launch the module or platform build\r
#\r
def Launch(self):\r
+ self.AllDrivers = set()\r
+ self.AllModules = set()\r
+ self.PreMakeCacheMiss = set()\r
+ self.PreMakeCacheHit = set()\r
+ self.MakeCacheMiss = set()\r
+ self.MakeCacheHit = set()\r
if not self.ModuleFile:\r
if not self.SpawnMode or self.Target not in ["", "all"]:\r
self.SpawnMode = False\r
for Module in self.BuildModules:\r
Module.CreateAsBuiltInf()\r
\r
- def UpdateBuildCache(self):\r
- all_lib_set = set()\r
- all_mod_set = set()\r
- for Module in self.BuildModules:\r
- Module.CopyModuleToCache()\r
- all_mod_set.add(Module)\r
- for Module in self.HashSkipModules:\r
+ def GenDestCache(self):\r
+ for Module in self.AllModules:\r
+ Module.GenPreMakefileHashList()\r
+ Module.GenMakefileHashList()\r
Module.CopyModuleToCache()\r
- all_mod_set.add(Module)\r
- for Module in all_mod_set:\r
- for lib in Module.LibraryAutoGenList:\r
- all_lib_set.add(lib)\r
- for lib in all_lib_set:\r
- lib.CopyModuleToCache()\r
- all_lib_set.clear()\r
- all_mod_set.clear()\r
- self.HashSkipModules = []\r
+\r
+ def GenLocalPreMakeCache(self):\r
+ for Module in self.PreMakeCacheMiss:\r
+ Module.GenPreMakefileHashList()\r
+\r
## Do some clean-up works when error occurred\r
def Relinquish(self):\r
OldLogLevel = EdkLogger.GetLevel()\r
projects / mirror_edk2.git / blobdiff