From 673d09a2ddfacbc8414ca703519f32c1e48ca5b7 Mon Sep 17 00:00:00 2001 From: "Feng, Bob C" Date: Tue, 30 Jul 2019 17:15:31 +0800 Subject: [PATCH] BaseTools: Enable Multiple Process AutoGen BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875 Assign the Module AutoGen tasks into multiple sub process. Cc: Liming Gao Signed-off-by: Bob Feng Acked-by: Laszlo Ersek Tested-by: Laszlo Ersek Acked-by: Liming Gao --- .../Source/Python/AutoGen/AutoGenWorker.py | 191 ++++++++++++++++++ BaseTools/Source/Python/AutoGen/DataPipe.py | 17 +- BaseTools/Source/Python/AutoGen/GenC.py | 4 +- .../Source/Python/AutoGen/ModuleAutoGen.py | 2 +- .../Source/Python/AutoGen/PlatformAutoGen.py | 16 +- .../Source/Python/AutoGen/WorkspaceAutoGen.py | 6 +- BaseTools/Source/Python/build/build.py | 127 ++++++++---- 7 files changed, 307 insertions(+), 56 deletions(-) create mode 100644 BaseTools/Source/Python/AutoGen/AutoGenWorker.py diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py new file mode 100644 index 0000000000..6c6ef49647 --- /dev/null +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -0,0 +1,191 @@ +## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +import multiprocessing as mp +import threading +from Common.Misc import PathClass +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo +import Common.GlobalData as GlobalData +import Common.EdkLogger as EdkLogger +import os +from Common.MultipleWorkspace import MultipleWorkspace as mws +from AutoGen.AutoGen import AutoGen +from Workspace.WorkspaceDatabase import BuildDB +from queue import Empty +import traceback +import sys +from AutoGen.DataPipe import MemoryDataPipe +def clearQ(q): + try: + while True: + q.get_nowait() + except Empty: + pass +class AutoGenManager(threading.Thread): + def __init__(self,autogen_workers, feedback_q,error_event): + super(AutoGenManager,self).__init__() + self.autogen_workers = autogen_workers + self.feedback_q = feedback_q + self.terminate = False + self.Status = True + self.error_event = error_event + def run(self): + try: + fin_num = 0 + while True: + badnews = self.feedback_q.get() + if badnews is None: + break + if badnews == "Done": + fin_num += 1 + else: + self.Status = False + self.TerminateWorkers() + if fin_num == len(self.autogen_workers): + self.clearQueue() + for w in self.autogen_workers: + w.join() + break + except Exception: + return + + def clearQueue(self): + taskq = self.autogen_workers[0].module_queue + clearQ(taskq) + clearQ(self.feedback_q) + + def TerminateWorkers(self): + self.error_event.set() + def kill(self): + self.feedback_q.put(None) +class AutoGenWorkerInProcess(mp.Process): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,error_event): + mp.Process.__init__(self) + self.module_queue = module_queue + self.data_pipe_file_path =data_pipe_file_path + self.data_pipe = None + self.feedback_q = feedback_q + self.PlatformMetaFileSet = {} + self.file_lock = file_lock + self.error_event = error_event + def GetPlatformMetaFile(self,filepath,root): + try: + return self.PlatformMetaFileSet[(filepath,root)] + except: + self.PlatformMetaFileSet[(filepath,root)] = filepath + return self.PlatformMetaFileSet[(filepath,root)] + def run(self): + try: + taskname = "Init" + with self.file_lock: + if not os.path.exists(self.data_pipe_file_path): + self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) + self.data_pipe = MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + EdkLogger.Initialize() + loglevel = self.data_pipe.Get("LogLevel") + if not loglevel: + loglevel = EdkLogger.INFO + EdkLogger.SetLevel(loglevel) + logfile = self.data_pipe.Get("LogFile") + if logfile: + EdkLogger.SetLogFile(logfile) + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + archlist = self.data_pipe.Get("P_Info").get("ArchList") + + active_p = self.data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + self.Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp") + GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines") + GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines") + os.environ._data = self.data_pipe.Get("Env_Var") + GlobalData.gWorkspace = workspacedir + GlobalData.gDisableIncludePathCheck = False + GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") + GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + pcd_from_build_option = [] + for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): + pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) + if pcd_tuple[2].strip(): + pcd_id = ".".join((pcd_id,pcd_tuple[2])) + pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3]))) + GlobalData.BuildOptionPcd = pcd_from_build_option + module_count = 0 + FfsCmd = self.data_pipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), + self.data_pipe.Get("P_Info").get("WorkspaceDir")) + libConstPcd = self.data_pipe.Get("LibConstPcd") + Refes = self.data_pipe.Get("REFS") + while True: + if self.module_queue.empty(): + break + if self.error_event.is_set(): + break + module_count += 1 + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + modulefullpath = os.path.join(module_root,module_file) + taskname = " : ".join((modulefullpath,module_arch)) + module_metafile = PathClass(module_file,module_root) + if module_path: + module_metafile.Path = module_path + if module_basename: + module_metafile.BaseName = module_basename + if module_originalpath: + module_metafile.OriginalPath = PathClass(module_originalpath,module_root) + arch = module_arch + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) + Ma.IsLibrary = IsLib + if IsLib: + if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd: + Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] + if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: + Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] + Ma.CreateCodeFile(False) + Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) + except Empty: + pass + except: + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + finally: + self.feedback_q.put("Done") + def printStatus(self): + print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) + print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) + groupobj = {} + for buildobj in BuildDB.BuildObject.GetCache().values(): + if str(buildobj).lower().endswith("dec"): + try: + groupobj['dec'].append(str(buildobj)) + except: + groupobj['dec'] = [str(buildobj)] + if str(buildobj).lower().endswith("dsc"): + try: + groupobj['dsc'].append(str(buildobj)) + except: + groupobj['dsc'] = [str(buildobj)] + + if str(buildobj).lower().endswith("inf"): + try: + groupobj['inf'].append(str(buildobj)) + except: + groupobj['inf'] = [str(buildobj)] + + print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[])))) + print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[])))) + print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[])))) diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py b/BaseTools/Source/Python/AutoGen/DataPipe.py index 5bcc39bd38..6299208056 100644 --- a/BaseTools/Source/Python/AutoGen/DataPipe.py +++ b/BaseTools/Source/Python/AutoGen/DataPipe.py @@ -11,6 +11,7 @@ import Common.GlobalData as GlobalData import os import pickle from pickle import HIGHEST_PROTOCOL +from Common import EdkLogger class PCD_DATA(): def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInfoList,DefaultValue, @@ -34,6 +35,7 @@ class DataPipe(object): def __init__(self, BuildDir=None): self.data_container = {} self.BuildDir = BuildDir + self.dump_file = "" class MemoryDataPipe(DataPipe): @@ -41,6 +43,7 @@ class MemoryDataPipe(DataPipe): return self.data_container.get(key) def dump(self,file_path): + self.dump_file = file_path with open(file_path,'wb') as fd: pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL) @@ -71,7 +74,7 @@ class MemoryDataPipe(DataPipe): for m in PlatformInfo.Platform.Modules: m_pcds = PlatformInfo.Platform.Modules[m].Pcds if m_pcds: - ModulePcds[(m.File,m.Root)] = [PCD_DATA( + ModulePcds[(m.File,m.Root,m.Arch)] = [PCD_DATA( pcd.TokenCName,pcd.TokenSpaceGuidCName,pcd.Type, pcd.DatumType,pcd.SkuInfoList,pcd.DefaultValue, pcd.MaxDatumSize,pcd.UserDefinedDefaultStoresFlag,pcd.validateranges, @@ -83,11 +86,18 @@ class MemoryDataPipe(DataPipe): #Module's Library Instance ModuleLibs = {} + libModules = {} for m in PlatformInfo.Platform.Modules: module_obj = BuildDB.BuildObject[m,PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain] Libs = GetModuleLibInstances(module_obj, PlatformInfo.Platform, BuildDB.BuildObject, PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain) - ModuleLibs[(m.File,m.Root,module_obj.Arch)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch) for l in Libs] + for lib in Libs: + try: + libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)].append((m.File,m.Root,module_obj.Arch,m.Path)) + except: + libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)] = [(m.File,m.Root,module_obj.Arch,m.Path)] + ModuleLibs[(m.File,m.Root,module_obj.Arch,m.Path)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch,l.MetaFile.Path) for l in Libs] self.DataContainer = {"DEPS":ModuleLibs} + self.DataContainer = {"REFS":libModules} #Platform BuildOptions @@ -143,5 +153,8 @@ class MemoryDataPipe(DataPipe): self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict} + self.DataContainer = {"DatabasePath":GlobalData.gDatabasePath} self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False} + self.DataContainer = {"LogLevel": EdkLogger.GetLevel()} + self.DataContainer = {"LogFile": GlobalData.gOptions.LogFile if GlobalData.gOptions.LogFile is not None else ""} diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py index 4c3f4e3e55..910c8fe370 100644 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -1472,8 +1472,8 @@ def CreateModuleEntryPointCode(Info, AutoGenC, AutoGenH): if Info.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_DXE_CORE, SUP_MODULE_SMM_CORE, SUP_MODULE_MM_CORE_STANDALONE]: if Info.SourceFileList: - if NumEntryPoints != 1: - EdkLogger.error( + if NumEntryPoints != 1: + EdkLogger.error( "build", AUTOGEN_ERROR, '%s must have exactly one entry point' % Info.ModuleType, diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index d19c038620..ed6822334e 100644 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1686,6 +1686,7 @@ class ModuleAutoGen(AutoGen): if not self.IsLibrary and CreateLibraryMakeFile: for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateMakeFile() + # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return @@ -1729,7 +1730,6 @@ class ModuleAutoGen(AutoGen): if not self.IsLibrary and CreateLibraryCodeFile: for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateCodeFile() - # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping if not GlobalData.gUseHashCache and self.CanSkip(): return diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py index 6c947eca2b..4abfc6c29d 100644 --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py @@ -133,6 +133,12 @@ class PlatformAutoGen(AutoGen): self.DataPipe.FillData(self) return True + def FillData_LibConstPcd(self): + libConstPcd = {} + for LibAuto in self.LibraryAutoGenList: + if LibAuto.ConstPcd: + libConstPcd[(LibAuto.MetaFile.File,LibAuto.MetaFile.Root,LibAuto.Arch,LibAuto.MetaFile.Path)] = LibAuto.ConstPcd + self.DataPipe.DataContainer = {"LibConstPcd":libConstPcd} ## hash() operator of PlatformAutoGen # # The platform file path and arch string will be used to represent @@ -162,7 +168,7 @@ class PlatformAutoGen(AutoGen): return for Ma in self.ModuleAutoGenList: - Ma.CreateCodeFile(True) + Ma.CreateCodeFile(CreateModuleCodeFile) ## Generate Fds Command @cached_property @@ -179,9 +185,9 @@ class PlatformAutoGen(AutoGen): for Ma in self._MaList: key = (Ma.MetaFile.File, self.Arch) if key in FfsCommand: - Ma.CreateMakeFile(True, FfsCommand[key]) + Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key]) else: - Ma.CreateMakeFile(True) + Ma.CreateMakeFile(CreateModuleMakeFile) # no need to create makefile for the platform more than once if self.IsMakeFileCreated: @@ -1086,10 +1092,10 @@ class PlatformAutoGen(AutoGen): Libs = GetModuleLibInstances(module_obj, self.Platform, self.BuildDatabase, self.Arch,self.BuildTarget,self.ToolChain) else: Libs = [] - ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Arch,True) for l in Libs])) + ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.MetaFile.Path,l.MetaFile.BaseName,l.MetaFile.OriginalPath,l.Arch,True) for l in Libs])) if WithoutPcd and module_obj.PcdIsDriver: continue - ModuleLibs.add((m.File,m.Root,module_obj.Arch,False)) + ModuleLibs.add((m.File,m.Root,m.Path,m.BaseName,m.OriginalPath,module_obj.Arch,bool(module_obj.LibraryClass))) return ModuleLibs diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py index 22a7d996fd..ea0d8f8bfb 100644 --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py @@ -113,6 +113,8 @@ class WorkspaceAutoGen(AutoGen): self.ProcessMixedPcd() self.VerifyPcdsFromFDF() self.CollectAllPcds() + for Pa in self.AutoGenObjectList: + Pa.FillData_LibConstPcd() self.GeneratePkgLevelHash() # # Check PCDs token value conflict in each DEC file. @@ -881,7 +883,7 @@ class WorkspaceAutoGen(AutoGen): if not CreateDepsMakeFile: return for Pa in self.AutoGenObjectList: - Pa.CreateMakeFile(True) + Pa.CreateMakeFile(CreateDepsMakeFile) ## Create autogen code for platform and modules # @@ -895,7 +897,7 @@ class WorkspaceAutoGen(AutoGen): if not CreateDepsCodeFile: return for Pa in self.AutoGenObjectList: - Pa.CreateCodeFile(True) + Pa.CreateCodeFile(CreateDepsCodeFile) ## Create AsBuilt INF file the platform # diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index 3d083f4eaa..af1be8d281 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -30,6 +30,7 @@ from optparse import OptionParser from AutoGen.PlatformAutoGen import PlatformAutoGen from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen +from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager from AutoGen import GenMake from Common import Misc as Utils @@ -50,7 +51,7 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile import Common.GlobalData as GlobalData from GenFds.GenFds import GenFds, GenFdsApi - +import multiprocessing as mp # Version and Copyright VersionNumber = "0.60" + ' ' + gBUILD_VERSION @@ -343,9 +344,9 @@ class ModuleMakeUnit(BuildUnit): # @param Obj The ModuleAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList] - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand,Target): + Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList] + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) if Target in [None, "", "all"]: self.Target = "tbuild" @@ -364,10 +365,10 @@ class PlatformMakeUnit(BuildUnit): # @param Obj The PlatformAutoGen object the build is working on # @param Target The build target name, one of gSupportedTarget # - def __init__(self, Obj, Target): - Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList] - Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList]) - BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir) + def __init__(self, Obj, BuildCommand, Target): + Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList] + Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList]) + BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir) ## The class representing the task of a module build or platform build # @@ -824,8 +825,31 @@ class Build(): if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)): self.InitBuild() + self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) + def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList): + try: + if SkipAutoGen: + return True,0 + feedback_q = mp.Queue() + file_lock = mp.Lock() + error_event = mp.Event() + auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,error_event) for _ in range(self.ThreadNumber)] + self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) + self.AutoGenMgr.start() + for w in auto_workers: + w.start() + if PcdMaList is not None: + for PcdMa in PcdMaList: + PcdMa.CreateCodeFile(False) + PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + + self.AutoGenMgr.join() + rt = self.AutoGenMgr.Status + return rt, 0 + except Exception as e: + return False,e.errcode ## Load configuration # @@ -1190,30 +1214,34 @@ class Build(): # @param CreateDepModuleMakeFile Flag used to indicate creating makefile # for dependent modules/Libraries # - def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}): + def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None): if AutoGenObject is None: return False - + if FfsCommand is None: + FfsCommand = {} # skip file generation for cleanxxx targets, run and fds target if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or Target == 'genc': - self.Progress.Start("Generating code") - AutoGenObject.CreateCodeFile(CreateDepsCodeFile) - self.Progress.Stop("done!") - if Target == "genc": - return True - - if not self.SkipAutoGen or Target == 'genmake': - self.Progress.Start("Generating makefile") - AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand) - self.Progress.Stop("done!") - if Target == "genmake": - return True - else: - # always recreate top/platform makefile when clean, just in case of inconsistency + mqueue = mp.Queue() + for m in AutoGenObject.GetAllModuleInfo: + mqueue.put(m) + + AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + self.Progress.Start("Generating makefile and code") + data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) + AutoGenObject.DataPipe.dump(data_pipe_file) + autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList) + self.Progress.Stop("done!") + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) AutoGenObject.CreateCodeFile(False) AutoGenObject.CreateMakeFile(False) + else: + # always recreate top/platform makefile when clean, just in case of inconsistency + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1334,8 +1362,8 @@ class Build(): return True else: # always recreate top/platform makefile when clean, just in case of inconsistency - AutoGenObject.CreateCodeFile(False) - AutoGenObject.CreateMakeFile(False) + AutoGenObject.CreateCodeFile(True) + AutoGenObject.CreateMakeFile(True) if EdkLogger.GetLevel() == EdkLogger.QUIET: EdkLogger.quiet("Building ... %s" % repr(AutoGenObject)) @@ -1713,9 +1741,10 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) self.BuildModules.append(Ma) - self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict) + self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList) # Create MAP file when Load Fix Address is enabled. if self.Target in ["", "all", "fds"]: @@ -1850,7 +1879,7 @@ class Build(): MakeStart = time.time() for Ma in self.BuildModules: if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform @@ -1980,7 +2009,7 @@ class Build(): Wa.CreateMakeFile(False) # Add ffs build to makefile - CmdListDict = None + CmdListDict = {} if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) @@ -1988,6 +2017,7 @@ class Build(): ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) + self.BuildModules = [] for Arch in Wa.ArchList: PcdMaList = [] AutoGenStart = time.time() @@ -2005,6 +2035,8 @@ class Build(): if Inf in Pa.Platform.Modules: continue ModuleList.append(Inf) + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) @@ -2013,6 +2045,7 @@ class Build(): continue if Ma.PcdIsDriver: Ma.PlatformInfo = Pa + Ma.Workspace = Wa PcdMaList.append(Ma) if Ma.CanSkipbyHash(): self.HashSkipModules.append(Ma) @@ -2024,34 +2057,32 @@ class Build(): EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile - if not self.SkipAutoGen or self.Target == 'genc': - Ma.CreateCodeFile(True) - if self.Target == "genc": - continue - if not self.SkipAutoGen or self.Target == 'genmake': - if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict: - Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch]) - del CmdListDict[Module.File, Arch] - else: - Ma.CreateMakeFile(True) - if self.Target == "genmake": - continue self.BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] = dict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() + for m in Pa.GetAllModuleInfo: + mqueue.put(m) + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) + Pa.DataPipe.dump(data_pipe_file) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList) self.Progress.Stop("done!") self.AutoGenTime += int(round((time.time() - AutoGenStart))) + if not autogen_rt: + self.AutoGenMgr.TerminateWorkers() + self.AutoGenMgr.join(0.1) + raise FatalError(errorcode) + for Arch in Wa.ArchList: MakeStart = time.time() for Ma in self.BuildModules: # Generate build task for the module if not Ma.IsBinaryModule: - Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target)) + Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target)) # Break build if any build thread has error if BuildTask.HasError(): # we need a full version of makefile for platform @@ -2498,6 +2529,10 @@ def Main(): EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False) ReturnCode = FORMAT_INVALID except KeyboardInterrupt: + if MyBuild is not None: + + # for multi-thread build exits safely + MyBuild.Relinquish() ReturnCode = ABORT_ERROR if Option is not None and Option.debug is not None: EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc()) @@ -2554,6 +2589,10 @@ def Main(): return ReturnCode if __name__ == '__main__': + try: + mp.set_start_method('spawn') + except: + pass r = Main() ## 0-127 is a safe return range, and 1 is a standard default error if r < 0 or r > 127: r = 1 -- 2.39.2