X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FAutoGenWorker.py;h=0425f1ab0bd18108632491875eee53231b1ce05e;hb=5dc2699d101d924114f451d3ed7080e72d7130e2;hp=9d06b45ec03f4cb24a5d07bf42292b09ab4583a6;hpb=3285fbda88238596b2b7e886e67e455f0626bb1f;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py old mode 100644 new mode 100755 index 9d06b45ec0..0425f1ab0b --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -16,22 +16,91 @@ import os from Common.MultipleWorkspace import MultipleWorkspace as mws from AutoGen.AutoGen import AutoGen from Workspace.WorkspaceDatabase import BuildDB -from queue import Empty +try: + from queue import Empty +except: + from Queue import Empty import traceback import sys from AutoGen.DataPipe import MemoryDataPipe +import logging +import time + def clearQ(q): try: while True: q.get_nowait() except Empty: pass + +class LogAgent(threading.Thread): + def __init__(self,log_q,log_level,log_file=None): + super(LogAgent,self).__init__() + self.log_q = log_q + self.log_level = log_level + self.log_file = log_file + def InitLogger(self): + # For DEBUG level (All DEBUG_0~9 are applicable) + self._DebugLogger_agent = logging.getLogger("tool_debug_agent") + _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S") + self._DebugLogger_agent.setLevel(self.log_level) + _DebugChannel = logging.StreamHandler(sys.stdout) + _DebugChannel.setFormatter(_DebugFormatter) + self._DebugLogger_agent.addHandler(_DebugChannel) + + # For VERBOSE, INFO, WARN level + self._InfoLogger_agent = logging.getLogger("tool_info_agent") + _InfoFormatter = logging.Formatter("%(message)s") + self._InfoLogger_agent.setLevel(self.log_level) + _InfoChannel = logging.StreamHandler(sys.stdout) + _InfoChannel.setFormatter(_InfoFormatter) + self._InfoLogger_agent.addHandler(_InfoChannel) + + # For ERROR level + self._ErrorLogger_agent = logging.getLogger("tool_error_agent") + _ErrorFormatter = logging.Formatter("%(message)s") + self._ErrorLogger_agent.setLevel(self.log_level) + _ErrorCh = logging.StreamHandler(sys.stderr) + _ErrorCh.setFormatter(_ErrorFormatter) + self._ErrorLogger_agent.addHandler(_ErrorCh) + + if self.log_file: + if os.path.exists(self.log_file): + os.remove(self.log_file) + _Ch = logging.FileHandler(self.log_file) + _Ch.setFormatter(_DebugFormatter) + self._DebugLogger_agent.addHandler(_Ch) + + _Ch= logging.FileHandler(self.log_file) + _Ch.setFormatter(_InfoFormatter) + self._InfoLogger_agent.addHandler(_Ch) + + _Ch = logging.FileHandler(self.log_file) + _Ch.setFormatter(_ErrorFormatter) + self._ErrorLogger_agent.addHandler(_Ch) + + def run(self): + self.InitLogger() + while True: + log_message = self.log_q.get() + if log_message is None: + break + if log_message.name == "tool_error": + self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage()) + elif log_message.name == "tool_info": + self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) + elif log_message.name == "tool_debug": + self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage()) + else: + self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) + + def kill(self): + self.log_q.put(None) class AutoGenManager(threading.Thread): def __init__(self,autogen_workers, feedback_q,error_event): super(AutoGenManager,self).__init__() self.autogen_workers = autogen_workers self.feedback_q = feedback_q - self.terminate = False self.Status = True self.error_event = error_event def run(self): @@ -43,7 +112,11 @@ class AutoGenManager(threading.Thread): break if badnews == "Done": fin_num += 1 + elif badnews == "QueueEmpty": + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) + self.TerminateWorkers() else: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) self.Status = False self.TerminateWorkers() if fin_num == len(self.autogen_workers): @@ -56,15 +129,31 @@ class AutoGenManager(threading.Thread): def clearQueue(self): taskq = self.autogen_workers[0].module_queue + logq = self.autogen_workers[0].log_q clearQ(taskq) clearQ(self.feedback_q) + clearQ(logq) + # Copy the cache queue itmes to parent thread before clear + cacheq = self.autogen_workers[0].cache_q + try: + cache_num = 0 + while True: + item = cacheq.get() + if item == "CacheDone": + cache_num += 1 + else: + GlobalData.gModuleAllCacheStatus.add(item) + if cache_num == len(self.autogen_workers): + break + except: + print ("cache_q error") def TerminateWorkers(self): self.error_event.set() def kill(self): self.feedback_q.put(None) class AutoGenWorkerInProcess(mp.Process): - def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,error_event): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event): mp.Process.__init__(self) self.module_queue = module_queue self.data_pipe_file_path =data_pipe_file_path @@ -72,7 +161,8 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q = feedback_q self.PlatformMetaFileSet = {} self.file_lock = file_lock - self.share_data = share_data + self.cache_q = cache_q + self.log_q = log_q self.error_event = error_event def GetPlatformMetaFile(self,filepath,root): try: @@ -84,18 +174,16 @@ class AutoGenWorkerInProcess(mp.Process): try: taskname = "Init" with self.file_lock: - if not os.path.exists(self.data_pipe_file_path): + try: + self.data_pipe = MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + except: self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) - self.data_pipe = MemoryDataPipe() - self.data_pipe.load(self.data_pipe_file_path) - EdkLogger.Initialize() + EdkLogger.LogClientInitialize(self.log_q) loglevel = self.data_pipe.Get("LogLevel") if not loglevel: loglevel = EdkLogger.INFO EdkLogger.SetLevel(loglevel) - logfile = self.data_pipe.Get("LogFile") - if logfile: - EdkLogger.SetLogFile(logfile) target = self.data_pipe.Get("P_Info").get("Target") toolchain = self.data_pipe.Get("P_Info").get("ToolChain") archlist = self.data_pipe.Get("P_Info").get("ArchList") @@ -115,6 +203,20 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.gDisableIncludePathCheck = False GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + + GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache") + GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") + GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") + GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile") + GlobalData.gModulePreMakeCacheStatus = dict() + GlobalData.gModuleMakeCacheStatus = dict() + GlobalData.gHashChainStatus = dict() + GlobalData.gCMakeHashFile = dict() + GlobalData.gModuleHashFile = dict() + GlobalData.gFileHashDict = dict() + GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") + GlobalData.file_lock = self.file_lock + CommandTarget = self.data_pipe.Get("CommandTarget") pcd_from_build_option = [] for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) @@ -126,17 +228,25 @@ class AutoGenWorkerInProcess(mp.Process): FfsCmd = self.data_pipe.Get("FfsCommand") if FfsCmd is None: FfsCmd = {} + GlobalData.FfsCmd = FfsCmd PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), self.data_pipe.Get("P_Info").get("WorkspaceDir")) - libConstPcd = self.data_pipe.Get("LibConstPcd") - Refes = self.data_pipe.Get("REFS") while True: - if self.module_queue.empty(): - break if self.error_event.is_set(): break module_count += 1 - module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + try: + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + except Empty: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty.")) + time.sleep(0.01) + continue + if module_file is None: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue.")) + self.feedback_q.put("QueueEmpty") + time.sleep(0.01) + continue + modulefullpath = os.path.join(module_root,module_file) taskname = " : ".join((modulefullpath,module_arch)) module_metafile = PathClass(module_file,module_root) @@ -151,20 +261,47 @@ class AutoGenWorkerInProcess(mp.Process): toolchain = self.data_pipe.Get("P_Info").get("ToolChain") Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) Ma.IsLibrary = IsLib - if IsLib: - if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd: - Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] - if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: - Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] + # SourceFileList calling sequence impact the makefile string sequence. + # Create cached SourceFileList here to unify its calling sequence for both + # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. + RetVal = Ma.SourceFileList + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyPreMakeCache() + except: + CacheResult = False + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True)) + continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) + Ma.CreateCodeFile(False) - Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) - except Empty: - pass - except: - traceback.print_exc(file=sys.stdout) + Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) + Ma.CreateAsBuiltInf() + if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyMakeCache() + except: + CacheResult = False + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) + continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) + + except Exception as e: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e))) self.feedback_q.put(taskname) finally: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done")) self.feedback_q.put("Done") + self.cache_q.put("CacheDone") + def printStatus(self): print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))