X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;ds=sidebyside;f=BaseTools%2FSource%2FPython%2Fbuild%2Fbuild.py;h=b9305bf7b25a635d04d6b21cc0161d4f7bad2028;hb=3285fbda88238596b2b7e886e67e455f0626bb1f;hp=9e6e5c15b2a744b5747fa591add652533b8da848;hpb=b1e27d175abbb42c4bcc8565a3e0e622d643c40f;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 9e6e5c15b2..b9305bf7b2 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -12,42 +12,47 @@ # Import Modules # from __future__ import print_function -import Common.LongFilePathOs as os -import re +from __future__ import absolute_import +import os.path as path import sys +import os +import re import glob import time import platform import traceback -import encodings.ascii import multiprocessing - -from struct import * -from threading import * +from threading import Thread,Event,BoundedSemaphore import threading +from subprocess import Popen,PIPE +from collections import OrderedDict, defaultdict from optparse import OptionParser -from subprocess import * +from AutoGen.PlatformAutoGen import PlatformAutoGen +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager +from AutoGen import GenMake from Common import Misc as Utils -from Common.LongFilePathSupport import OpenLongFilePath as open -from Common.TargetTxtClassObject import TargetTxtClassObject -from Common.ToolDefClassObject import ToolDefClassObject +from Common.TargetTxtClassObject import TargetTxt +from Common.ToolDefClassObject import ToolDef +from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory +from Common.StringUtils import NormPath +from Common.MultipleWorkspace import MultipleWorkspace as mws +from Common.BuildToolError import * from Common.DataType import * +import Common.EdkLogger as EdkLogger from Common.BuildVersion import gBUILD_VERSION -from AutoGen.AutoGen import * -from Common.BuildToolError import * -from Workspace.WorkspaceDatabase import WorkspaceDatabase -from Common.MultipleWorkspace import MultipleWorkspace as mws +from Workspace.WorkspaceDatabase import BuildDB from BuildReport import BuildReport -from GenPatchPcdTable.GenPatchPcdTable import * -from PatchPcdValue.PatchPcdValue import * +from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile +from PatchPcdValue.PatchPcdValue import PatchBinaryFile -import Common.EdkLogger import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - -from collections import OrderedDict, defaultdict +import multiprocessing as mp +from multiprocessing import Manager # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION @@ -340,9 +345,9 @@ class ModuleMakeUnit(BuildUnit): # @param Obj The ModuleAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) if Target in [None, "", "all"]: self.Target = "tbuild" @@ -361,10 +366,10 @@ class PlatformMakeUnit(BuildUnit): # @param Obj The PlatformAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) ## The class representing the task of a module build or platform build # @@ -593,7 +598,7 @@ class BuildTask: # def AddDependency(self, Dependency): for Dep in Dependency: - if not Dep.BuildObject.IsBinaryModule: + if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash(): self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list ## The thread wrapper of LaunchCommand function @@ -605,6 +610,11 @@ class BuildTask: try: self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir) self.CompleteFlag = True + + # Run hash operation post dependency, to account for libs + if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary: + HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash") + SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True) except: # # TRICK: hide the output of threads left running, so that the user can @@ -620,6 +630,16 @@ class BuildTask: BuildTask._ErrorFlag.set() BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \ (threading.currentThread().getName(), Command, WorkingDir) + + # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS' + # If Module or Lib is being tracked, it did not fail header check test, and built successfully + if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking and + self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] != 'FAIL_METAFILE' and + not BuildTask._ErrorFlag.isSet() + ): + GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject] = 'SUCCESS' + # indicate there's a thread is available for another build task BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueue.pop(self.BuildItem) @@ -701,8 +721,8 @@ class Build(): self.ConfDirectory = BuildOptions.ConfDirectory self.SpawnMode = True self.BuildReport = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType) - self.TargetTxt = TargetTxtClassObject() - self.ToolDef = ToolDefClassObject() + self.TargetTxt = TargetTxt + self.ToolDef = ToolDef self.AutoGenTime = 0 self.MakeTime = 0 self.GenFdsTime = 0 @@ -758,8 +778,9 @@ class Build(): ConfDirectoryPath = mws.join(self.WorkspaceDir, 'Conf') GlobalData.gConfDirectory = ConfDirectoryPath GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath)) - - self.Db = WorkspaceDatabase() + if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')): + os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache')) + self.Db = BuildDB self.BuildDatabase = self.Db.BuildObject self.Platform = None self.ToolChainFamily = None @@ -800,36 +821,43 @@ class Build(): EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild)) if self.Prebuild: self.LaunchPrebuild() - self.TargetTxt = TargetTxtClassObject() - self.ToolDef = ToolDefClassObject() + self.TargetTxt = TargetTxt + self.ToolDef = ToolDef if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): self.InitBuild() + self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) + self.share_data = Manager().dict() + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data): + try: + if SkipAutoGen: + return True,0 + feedback_q = mp.Queue() + file_lock = mp.Lock() + error_event = mp.Event() + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,error_event) for _ in range(self.ThreadNumber)] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(False) + PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + + self.AutoGenMgr.join() + rt = self.AutoGenMgr.Status + return rt, 0 + except Exception as e: + return False,e.errcode ## Load configuration # # This method will parse target.txt and get the build configurations. # def LoadConfiguration(self): - # - # Check target.txt and tools_def.txt and Init them - # - BuildConfigurationFile = os.path.normpath(os.path.join(GlobalData.gConfDirectory, gBuildConfiguration)) - if os.path.isfile(BuildConfigurationFile) == True: - StatusCode = self.TargetTxt.LoadTargetTxtFile(BuildConfigurationFile) - - ToolDefinitionFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_CONF] - if ToolDefinitionFile == '': - ToolDefinitionFile = gToolsDefinition - ToolDefinitionFile = os.path.normpath(mws.join(self.WorkspaceDir, 'Conf', ToolDefinitionFile)) - if os.path.isfile(ToolDefinitionFile) == True: - StatusCode = self.ToolDef.LoadToolDefFile(ToolDefinitionFile) - else: - EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=ToolDefinitionFile) - else: - EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile) # if no ARCH given in command line, get it from target.txt if not self.ArchList: @@ -1138,6 +1166,40 @@ class Build(): if Process.returncode != 0 : EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!') EdkLogger.info("\n- Postbuild Done -\n") + + ## Error handling for hash feature + # + # On BuildTask error, iterate through the Module Build tracking + # dictionary to determine wheather a module failed to build. Invalidate + # the hash associated with that module by removing it from storage. + # + # + def invalidateHash(self): + # Only for hashing feature + if not GlobalData.gUseHashCache: + return + + # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash + for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys(): + for moduleAutoGenObj in GlobalData.gModuleBuildTracking[moduleAutoGenObjArch].keys(): + # Skip invalidating for Successful Module/Lib builds + if GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][moduleAutoGenObj] == 'SUCCESS': + continue + + # The module failed to build, failed to start building, or failed the header check test from this point on + + # Remove .hash from build + ModuleHashFile = os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash") + if os.path.exists(ModuleHashFile): + os.remove(ModuleHashFile) + + # Remove .hash file from cache + if GlobalData.gBinCacheDest: + FileDir = os.path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName) + HashFile = os.path.join(FileDir, moduleAutoGenObj.Name + '.hash') + if os.path.exists(HashFile): + os.remove(HashFile) + ## Build a module or platform # # Create autogen code and makefile for a module or platform, and the launch @@ -1154,30 +1216,34 @@ class Build(): # @param CreateDepModuleMakeFile Flag used to indicate creating makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand = {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target == 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target == "genc": - return True - - if not self.SkipAutoGen or Target == 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand) - self.Progress.Stop("done!") - if Target == "genmake": - return True - else: - # always recreate top/platform makefile when clean, just in case of inconsistency + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) + + AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) + self.Progress.Stop("done!") + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) + else: + # always recreate top/platform makefile when clean, just in case of inconsistency + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1194,10 +1260,6 @@ class Build(): # run if Target == 'run': - RunDir = os.path.normpath(os.path.join(AutoGenObject.BuildDir, GlobalData.gGlobalDefines['ARCH'])) - Command = '.\SecMain' - os.chdir(RunDir) - LaunchCommand(Command, RunDir) return True # build modules @@ -1205,6 +1267,9 @@ class Build(): BuildCommand = BuildCommand + [Target] LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.UpdateBuildCache() + self.BuildModules = [] return True # build library @@ -1223,6 +1288,9 @@ class Build(): NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild'] LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir) self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.UpdateBuildCache() + self.BuildModules = [] return True # cleanlib @@ -1296,8 +1364,8 @@ class Build(): return True else: # always recreate top/platform makefile when clean, just in case of inconsistency - AutoGenObject.CreateCodeFile(False) - AutoGenObject.CreateMakeFile(False) + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1316,6 +1384,9 @@ class Build(): BuildCommand = BuildCommand + [Target] AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.UpdateBuildCache() + self.BuildModules = [] return True # genfds @@ -1326,10 +1397,6 @@ class Build(): # run if Target == 'run': - RunDir = os.path.normpath(os.path.join(AutoGenObject.BuildDir, GlobalData.gGlobalDefines['ARCH'])) - Command = '.\SecMain' - os.chdir(RunDir) - LaunchCommand(Command, RunDir) return True # build library @@ -1449,7 +1516,7 @@ class Build(): if self.Fdf: # First get the XIP base address for FV map file. GuidPattern = re.compile("[-a-fA-F0-9]+") - GuidName = re.compile("\(GUID=[-a-fA-F0-9]+") + GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+") for FvName in Wa.FdfProfile.FvDict: FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map') if not os.path.exists(FvMapBuffer): @@ -1666,15 +1733,20 @@ class Build(): CmdListDict = self._GenFfsCmd(Wa.ArchList) for Arch in Wa.ArchList: + PcdMaList = [] GlobalData.gGlobalDefines['ARCH'] = Arch Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) for Module in Pa.Platform.Modules: # Get ModuleAutoGen object to generate C code file and makefile - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile) + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) if Ma is None: continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict) + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: @@ -1768,12 +1840,18 @@ class Build(): Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) for Module in Pa.Platform.Modules: if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name: - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile) - if Ma is None: continue + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) + if Ma is None: + continue MaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) + if GlobalData.gBinCacheSource: + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) continue + else: + if GlobalData.gBinCacheSource: + EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile @@ -1794,16 +1872,22 @@ class Build(): if self.Target == "genmake": return True self.BuildModules.append(Ma) + # Initialize all modules in tracking to 'FAIL' + if Ma.Arch not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma.Arch] = dict() + if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: + GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() + self.invalidateHash() Pa.CreateMakeFile(False) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) # Start task scheduler @@ -1813,6 +1897,7 @@ class Build(): # in case there's an interruption. we need a full version of makefile for platform Pa.CreateMakeFile(False) if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) self.MakeTime += int(round((time.time() - MakeStart))) @@ -1820,8 +1905,12 @@ class Build(): ExitFlag.set() BuildTask.WaitForComplete() self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.UpdateBuildCache() + self.BuildModules = [] self.MakeTime += int(round((time.time() - MakeContiue))) if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) self.BuildReport.AddPlatformReport(Wa, MaList) @@ -1874,6 +1963,7 @@ class Build(): # Save MAP buffer into MAP file. # self._SaveMapFile (MapBuffer, Wa) + self.invalidateHash() def _GenFfsCmd(self,ArchList): # convert dictionary of Cmd:(Inf,Arch) @@ -1921,7 +2011,7 @@ class Build(): Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = None + CmdListDict = {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) @@ -1929,7 +2019,9 @@ class Build(): ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + self.BuildModules = [] for Arch in Wa.ArchList: + PcdMaList = [] AutoGenStart = time.time() GlobalData.gGlobalDefines['ARCH'] = Arch Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) @@ -1945,45 +2037,60 @@ class Build(): if Inf in Pa.Platform.Modules: continue ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile) + Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) if Ma is None: continue + if Ma.PcdIsDriver: + Ma.PlatformInfo = Pa + Ma.Workspace = Wa + PcdMaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) + if GlobalData.gBinCacheSource: + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) continue + else: + if GlobalData.gBinCacheSource: + EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or self.Target == 'genc': - Ma.CreateCodeFile(True) - if self.Target == "genc": - continue - if not self.SkipAutoGen or self.Target == 'genmake': - if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target == "genmake": - continue self.BuildModules.append(Ma) + # Initialize all modules in tracking to 'FAIL' + if Ma.Arch not in GlobalData.gModuleBuildTracking: + GlobalData.gModuleBuildTracking[Ma.Arch] = dict() + if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: + GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) + for Arch in Wa.ArchList: MakeStart = time.time() for Ma in self.BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() + self.invalidateHash() Pa.CreateMakeFile(False) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) # Start task scheduler @@ -1993,6 +2100,7 @@ class Build(): # in case there's an interruption. we need a full version of makefile for platform Pa.CreateMakeFile(False) if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) self.MakeTime += int(round((time.time() - MakeStart))) @@ -2006,12 +2114,16 @@ class Build(): ExitFlag.set() BuildTask.WaitForComplete() self.CreateAsBuiltInf() + if GlobalData.gBinCacheDest: + self.UpdateBuildCache() + self.BuildModules = [] self.MakeTime += int(round((time.time() - MakeContiue))) # # Check for build error, and raise exception if one # has been signaled. # if BuildTask.HasError(): + self.invalidateHash() EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) # Create MAP file when Load Fix Address is enabled. @@ -2056,6 +2168,7 @@ class Build(): # Save MAP buffer into MAP file. # self._SaveMapFile(MapBuffer, Wa) + self.invalidateHash() ## Generate GuidedSectionTools.txt in the FV directories. # @@ -2145,9 +2258,23 @@ class Build(): def CreateAsBuiltInf(self): for Module in self.BuildModules: Module.CreateAsBuiltInf() + + def UpdateBuildCache(self): + all_lib_set = set() + all_mod_set = set() + for Module in self.BuildModules: + Module.CopyModuleToCache() + all_mod_set.add(Module) for Module in self.HashSkipModules: - Module.CreateAsBuiltInf(True) - self.BuildModules = [] + Module.CopyModuleToCache() + all_mod_set.add(Module) + for Module in all_mod_set: + for lib in Module.LibraryAutoGenList: + all_lib_set.add(lib) + for lib in all_lib_set: + lib.CopyModuleToCache() + all_lib_set.clear() + all_mod_set.clear() self.HashSkipModules = [] ## Do some clean-up works when error occurred def Relinquish(self): @@ -2404,6 +2531,10 @@ def Main(): EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) ReturnCode = FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode = ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) @@ -2423,7 +2554,7 @@ def Main(): "\nbuild", CODE_ERROR, "Unknown fatal error when processing [%s]" % MetaFile, - ExtraData="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n", + ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR, RaiseError=False ) EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) @@ -2460,6 +2591,10 @@ def Main(): return ReturnCode if __name__ == '__main__': + try: + mp.set_start_method('spawn') + except: + pass r = Main() ## 0-127 is a safe return range, and 1 is a standard default error if r < 0 or r > 127: r = 1