X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2Fbuild%2Fbuild.py;h=af1be8d281361ff32fd32b6a701cb88f92331bb6;hb=673d09a2dd;hp=3d083f4eaade6239f8b1df766f0fa4b288f05770;hpb=e8449e1d8e3b40186eb16ff25242397cffb00a63;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 3d083f4eaa..af1be8d281 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -30,6 +30,7 @@ from optparse import OptionParser from AutoGen.PlatformAutoGen import PlatformAutoGen from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager from AutoGen import GenMake from Common import Misc as Utils @@ -50,7 +51,7 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - +import multiprocessing as mp # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION @@ -343,9 +344,9 @@ class ModuleMakeUnit(BuildUnit): # @param Obj The ModuleAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) if Target in [None, "", "all"]: self.Target = "tbuild" @@ -364,10 +365,10 @@ class PlatformMakeUnit(BuildUnit): # @param Obj The PlatformAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) ## The class representing the task of a module build or platform build # @@ -824,8 +825,31 @@ class Build(): if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): self.InitBuild() + self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList): + try: + if SkipAutoGen: + return True,0 + feedback_q = mp.Queue() + file_lock = mp.Lock() + error_event = mp.Event() + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,error_event) for _ in range(self.ThreadNumber)] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(False) + PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + + self.AutoGenMgr.join() + rt = self.AutoGenMgr.Status + return rt, 0 + except Exception as e: + return False,e.errcode ## Load configuration # @@ -1190,30 +1214,34 @@ class Build(): # @param CreateDepModuleMakeFile Flag used to indicate creating makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand = {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target == 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target == "genc": - return True - - if not self.SkipAutoGen or Target == 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand) - self.Progress.Stop("done!") - if Target == "genmake": - return True - else: - # always recreate top/platform makefile when clean, just in case of inconsistency + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) + + AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList) + self.Progress.Stop("done!") + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) + else: + # always recreate top/platform makefile when clean, just in case of inconsistency + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1334,8 +1362,8 @@ class Build(): return True else: # always recreate top/platform makefile when clean, just in case of inconsistency - AutoGenObject.CreateCodeFile(False) - AutoGenObject.CreateMakeFile(False) + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1713,9 +1741,10 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict) + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: @@ -1850,7 +1879,7 @@ class Build(): MakeStart = time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform @@ -1980,7 +2009,7 @@ class Build(): Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = None + CmdListDict = {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) @@ -1988,6 +2017,7 @@ class Build(): ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + self.BuildModules = [] for Arch in Wa.ArchList: PcdMaList = [] AutoGenStart = time.time() @@ -2005,6 +2035,8 @@ class Build(): if Inf in Pa.Platform.Modules: continue ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) @@ -2013,6 +2045,7 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) @@ -2024,34 +2057,32 @@ class Build(): EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or self.Target == 'genc': - Ma.CreateCodeFile(True) - if self.Target == "genc": - continue - if not self.SkipAutoGen or self.Target == 'genmake': - if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target == "genmake": - continue self.BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] = dict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) + for Arch in Wa.ArchList: MakeStart = time.time() for Ma in self.BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform @@ -2498,6 +2529,10 @@ def Main(): EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) ReturnCode = FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode = ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) @@ -2554,6 +2589,10 @@ def Main(): return ReturnCode if __name__ == '__main__': + try: + mp.set_start_method('spawn') + except: + pass r = Main() ## 0-127 is a safe return range, and 1 is a standard default error if r < 0 or r > 127: r = 1