X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FAutoGenWorker.py;h=40b448f5b2db31cf7ee103cc4a12c6adea075202;hb=fc8b8deac2d77524ff8cfe44acf95b5e1f59804e;hp=94ea61a4870b9c44300c7b7e444f43ee00a0f8bd;hpb=3bfbc915074a45f4d9c61aa2b698a62f1a24124e;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 94ea61a487..40b448f5b2 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -128,12 +128,27 @@ class AutoGenManager(threading.Thread): clearQ(taskq) clearQ(self.feedback_q) clearQ(logq) + # Copy the cache queue itmes to parent thread before clear + cacheq = self.autogen_workers[0].cache_q + try: + cache_num = 0 + while True: + item = cacheq.get() + if item == "CacheDone": + cache_num += 1 + else: + GlobalData.gModuleAllCacheStatus.add(item) + if cache_num == len(self.autogen_workers): + break + except: + print ("cache_q error") + def TerminateWorkers(self): self.error_event.set() def kill(self): self.feedback_q.put(None) class AutoGenWorkerInProcess(mp.Process): - def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_lock,share_data,log_q,error_event): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event): mp.Process.__init__(self) self.module_queue = module_queue self.data_pipe_file_path =data_pipe_file_path @@ -141,8 +156,7 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q = feedback_q self.PlatformMetaFileSet = {} self.file_lock = file_lock - self.cache_lock = cache_lock - self.share_data = share_data + self.cache_q = cache_q self.log_q = log_q self.error_event = error_event def GetPlatformMetaFile(self,filepath,root): @@ -184,12 +198,19 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.gDisableIncludePathCheck = False GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + + GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache") GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") - GlobalData.gCacheIR = self.share_data + GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile") + GlobalData.gModulePreMakeCacheStatus = dict() + GlobalData.gModuleMakeCacheStatus = dict() + GlobalData.gHashChainStatus = dict() + GlobalData.gCMakeHashFile = dict() + GlobalData.gModuleHashFile = dict() + GlobalData.gFileHashDict = dict() GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") GlobalData.file_lock = self.file_lock - GlobalData.cache_lock = self.cache_lock CommandTarget = self.data_pipe.Get("CommandTarget") pcd_from_build_option = [] for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): @@ -205,10 +226,6 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.FfsCmd = FfsCmd PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), self.data_pipe.Get("P_Info").get("WorkspaceDir")) - libConstPcd = self.data_pipe.Get("LibConstPcd") - Refes = self.data_pipe.Get("REFS") - GlobalData.libConstPcd = libConstPcd - GlobalData.Refes = Refes while True: if self.module_queue.empty(): break @@ -230,27 +247,41 @@ class AutoGenWorkerInProcess(mp.Process): toolchain = self.data_pipe.Get("P_Info").get("ToolChain") Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) Ma.IsLibrary = IsLib - if IsLib: - if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd: - Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] - if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: - Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] - if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: - Ma.GenModuleFilesHash(GlobalData.gCacheIR) - Ma.GenPreMakefileHash(GlobalData.gCacheIR) - if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): + # SourceFileList calling sequence impact the makefile string sequence. + # Create cached SourceFileList here to unify its calling sequence for both + # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. + RetVal = Ma.SourceFileList + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyPreMakeCache() + except: + CacheResult = False + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True)) continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: - Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR) - Ma.GenMakeHash(GlobalData.gCacheIR) - if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): + try: + CacheResult = Ma.CanSkipbyMakeCache() + except: + CacheResult = False + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) continue else: - Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR) + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) + except Empty: pass except: @@ -258,6 +289,8 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q.put(taskname) finally: self.feedback_q.put("Done") + self.cache_q.put("CacheDone") + def printStatus(self): print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))