X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2Fbuild%2Fbuild.py;h=4daef5652f8d6b6a41a91874ec85161dbdaf1dc7;hb=636ed13a7f9339aea7fdb74de24be1703e9d482c;hp=3d083f4eaade6239f8b1df766f0fa4b288f05770;hpb=e8449e1d8e3b40186eb16ff25242397cffb00a63;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 3d083f4eaa..4daef5652f 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -30,6 +30,8 @@ from optparse import OptionParser from AutoGen.PlatformAutoGen import PlatformAutoGen from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\ + LogAgent from AutoGen import GenMake from Common import Misc as Utils @@ -50,7 +52,8 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - +import multiprocessing as mp +from multiprocessing import Manager # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION @@ -343,9 +346,9 @@ class ModuleMakeUnit(BuildUnit): # @param Obj The ModuleAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) if Target in [None, "", "all"]: self.Target = "tbuild" @@ -364,10 +367,10 @@ class PlatformMakeUnit(BuildUnit): # @param Obj The PlatformAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) ## The class representing the task of a module build or platform build # @@ -697,7 +700,7 @@ class Build(): # @param WorkspaceDir The directory of workspace # @param BuildOptions Build options passed from command line # - def __init__(self, Target, WorkspaceDir, BuildOptions): + def __init__(self, Target, WorkspaceDir, BuildOptions,log_q): self.WorkspaceDir = WorkspaceDir self.Target = Target self.PlatformFile = BuildOptions.PlatformFile @@ -824,8 +827,33 @@ class Build(): if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): self.InitBuild() + self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) + self.share_data = Manager().dict() + self.log_q = log_q + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data): + try: + if SkipAutoGen: + return True,0 + feedback_q = mp.Queue() + file_lock = mp.Lock() + error_event = mp.Event() + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(False) + PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + + self.AutoGenMgr.join() + rt = self.AutoGenMgr.Status + return rt, 0 + except Exception as e: + return False,e.errcode ## Load configuration # @@ -1190,30 +1218,34 @@ class Build(): # @param CreateDepModuleMakeFile Flag used to indicate creating makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand = {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target == 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target == "genc": - return True - - if not self.SkipAutoGen or Target == 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand) - self.Progress.Stop("done!") - if Target == "genmake": - return True - else: - # always recreate top/platform makefile when clean, just in case of inconsistency + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) + + AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) + self.Progress.Stop("done!") + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) + else: + # always recreate top/platform makefile when clean, just in case of inconsistency + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1334,8 +1366,8 @@ class Build(): return True else: # always recreate top/platform makefile when clean, just in case of inconsistency - AutoGenObject.CreateCodeFile(False) - AutoGenObject.CreateMakeFile(False) + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1713,9 +1745,10 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict) + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: @@ -1850,7 +1883,7 @@ class Build(): MakeStart = time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform @@ -1980,7 +2013,7 @@ class Build(): Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = None + CmdListDict = {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) @@ -1988,6 +2021,7 @@ class Build(): ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + self.BuildModules = [] for Arch in Wa.ArchList: PcdMaList = [] AutoGenStart = time.time() @@ -2005,6 +2039,8 @@ class Build(): if Inf in Pa.Platform.Modules: continue ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) @@ -2013,6 +2049,7 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) @@ -2024,34 +2061,32 @@ class Build(): EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or self.Target == 'genc': - Ma.CreateCodeFile(True) - if self.Target == "genc": - continue - if not self.SkipAutoGen or self.Target == 'genmake': - if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target == "genmake": - continue self.BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] = dict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) + for Arch in Wa.ArchList: MakeStart = time.time() for Ma in self.BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform @@ -2373,8 +2408,12 @@ def MyOptionParser(): def Main(): StartTime = time.time() + # + # Create a log Queue + # + LogQ = mp.Queue() # Initialize log system - EdkLogger.Initialize() + EdkLogger.LogClientInitialize(LogQ) GlobalData.gCommand = sys.argv[1:] # # Parse the options and args @@ -2384,20 +2423,23 @@ def Main(): GlobalData.gCaseInsensitive = Option.CaseInsensitive # Set log level + LogLevel = EdkLogger.INFO if Option.verbose is not None: EdkLogger.SetLevel(EdkLogger.VERBOSE) + LogLevel = EdkLogger.VERBOSE elif Option.quiet is not None: EdkLogger.SetLevel(EdkLogger.QUIET) + LogLevel = EdkLogger.QUIET elif Option.debug is not None: EdkLogger.SetLevel(Option.debug + 1) + LogLevel = Option.debug + 1 else: EdkLogger.SetLevel(EdkLogger.INFO) - if Option.LogFile is not None: - EdkLogger.SetLogFile(Option.LogFile) - if Option.WarningAsError == True: EdkLogger.SetWarningAsError() + Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile) + Log_Agent.start() if platform.platform().find("Windows") >= 0: GlobalData.gIsWindows = True @@ -2471,7 +2513,7 @@ def Main(): if Option.Flag is not None and Option.Flag not in ['-c', '-s']: EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s") - MyBuild = Build(Target, Workspace, Option) + MyBuild = Build(Target, Workspace, Option,LogQ) GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList) if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)): MyBuild.Launch() @@ -2498,6 +2540,10 @@ def Main(): EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) ReturnCode = FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode = ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) @@ -2551,9 +2597,15 @@ def Main(): EdkLogger.quiet("\n- %s -" % Conclusion) EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime())) EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr) + Log_Agent.kill() + Log_Agent.join() return ReturnCode if __name__ == '__main__': + try: + mp.set_start_method('spawn') + except: + pass r = Main() ## 0-127 is a safe return range, and 1 is a standard default error if r < 0 or r > 127: r = 1