import traceback\r
import sys\r
from AutoGen.DataPipe import MemoryDataPipe\r
+import logging\r
+import time\r
+\r
def clearQ(q):\r
try:\r
while True:\r
q.get_nowait()\r
except Empty:\r
pass\r
-import logging\r
\r
class LogAgent(threading.Thread):\r
def __init__(self,log_q,log_level,log_file=None):\r
break\r
if badnews == "Done":\r
fin_num += 1\r
+ elif badnews == "QueueEmpty":\r
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))\r
+ self.TerminateWorkers()\r
else:\r
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))\r
self.Status = False\r
self.TerminateWorkers()\r
if fin_num == len(self.autogen_workers):\r
\r
def clearQueue(self):\r
taskq = self.autogen_workers[0].module_queue\r
+ logq = self.autogen_workers[0].log_q\r
clearQ(taskq)\r
clearQ(self.feedback_q)\r
+ clearQ(logq)\r
+ # Copy the cache queue itmes to parent thread before clear\r
+ cacheq = self.autogen_workers[0].cache_q\r
+ try:\r
+ cache_num = 0\r
+ while True:\r
+ item = cacheq.get()\r
+ if item == "CacheDone":\r
+ cache_num += 1\r
+ else:\r
+ GlobalData.gModuleAllCacheStatus.add(item)\r
+ if cache_num == len(self.autogen_workers):\r
+ break\r
+ except:\r
+ print ("cache_q error")\r
\r
def TerminateWorkers(self):\r
self.error_event.set()\r
def kill(self):\r
self.feedback_q.put(None)\r
class AutoGenWorkerInProcess(mp.Process):\r
- def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,log_q,error_event):\r
+ def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):\r
mp.Process.__init__(self)\r
self.module_queue = module_queue\r
self.data_pipe_file_path =data_pipe_file_path\r
self.feedback_q = feedback_q\r
self.PlatformMetaFileSet = {}\r
self.file_lock = file_lock\r
- self.share_data = share_data\r
+ self.cache_q = cache_q\r
self.log_q = log_q\r
self.error_event = error_event\r
def GetPlatformMetaFile(self,filepath,root):\r
try:\r
taskname = "Init"\r
with self.file_lock:\r
- if not os.path.exists(self.data_pipe_file_path):\r
+ try:\r
+ self.data_pipe = MemoryDataPipe()\r
+ self.data_pipe.load(self.data_pipe_file_path)\r
+ except:\r
self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)\r
- self.data_pipe = MemoryDataPipe()\r
- self.data_pipe.load(self.data_pipe_file_path)\r
EdkLogger.LogClientInitialize(self.log_q)\r
loglevel = self.data_pipe.Get("LogLevel")\r
if not loglevel:\r
self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")\r
GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")\r
GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")\r
+ GlobalData.gCommandMaxLength = self.data_pipe.Get('gCommandMaxLength')\r
os.environ._data = self.data_pipe.Get("Env_Var")\r
GlobalData.gWorkspace = workspacedir\r
GlobalData.gDisableIncludePathCheck = False\r
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")\r
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")\r
+\r
+ GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")\r
+ GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")\r
+ GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")\r
+ GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")\r
+ GlobalData.gModulePreMakeCacheStatus = dict()\r
+ GlobalData.gModuleMakeCacheStatus = dict()\r
+ GlobalData.gHashChainStatus = dict()\r
+ GlobalData.gCMakeHashFile = dict()\r
+ GlobalData.gModuleHashFile = dict()\r
+ GlobalData.gFileHashDict = dict()\r
+ GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")\r
+ GlobalData.gPlatformFinalPcds = self.data_pipe.Get("gPlatformFinalPcds")\r
+ GlobalData.file_lock = self.file_lock\r
+ CommandTarget = self.data_pipe.Get("CommandTarget")\r
pcd_from_build_option = []\r
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):\r
pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))\r
FfsCmd = self.data_pipe.Get("FfsCommand")\r
if FfsCmd is None:\r
FfsCmd = {}\r
+ GlobalData.FfsCmd = FfsCmd\r
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),\r
self.data_pipe.Get("P_Info").get("WorkspaceDir"))\r
- libConstPcd = self.data_pipe.Get("LibConstPcd")\r
- Refes = self.data_pipe.Get("REFS")\r
while True:\r
- if self.module_queue.empty():\r
- break\r
if self.error_event.is_set():\r
break\r
module_count += 1\r
- module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()\r
+ try:\r
+ module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()\r
+ except Empty:\r
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))\r
+ time.sleep(0.01)\r
+ continue\r
+ if module_file is None:\r
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))\r
+ self.feedback_q.put("QueueEmpty")\r
+ time.sleep(0.01)\r
+ continue\r
+\r
modulefullpath = os.path.join(module_root,module_file)\r
taskname = " : ".join((modulefullpath,module_arch))\r
module_metafile = PathClass(module_file,module_root)\r
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)\r
Ma.IsLibrary = IsLib\r
- if IsLib:\r
- if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:\r
- Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r
- if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:\r
- Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r
+ # SourceFileList calling sequence impact the makefile string sequence.\r
+ # Create cached SourceFileList here to unify its calling sequence for both\r
+ # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.\r
+ RetVal = Ma.SourceFileList\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:\r
+ try:\r
+ CacheResult = Ma.CanSkipbyPreMakeCache()\r
+ except:\r
+ CacheResult = False\r
+ self.feedback_q.put(taskname)\r
+\r
+ if CacheResult:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))\r
+ continue\r
+ else:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))\r
+\r
Ma.CreateCodeFile(False)\r
- Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))\r
- except Empty:\r
- pass\r
- except:\r
- traceback.print_exc(file=sys.stdout)\r
+ Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))\r
+ Ma.CreateAsBuiltInf()\r
+ if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:\r
+ try:\r
+ CacheResult = Ma.CanSkipbyMakeCache()\r
+ except:\r
+ CacheResult = False\r
+ self.feedback_q.put(taskname)\r
+\r
+ if CacheResult:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))\r
+ continue\r
+ else:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))\r
+\r
+ except Exception as e:\r
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e)))\r
self.feedback_q.put(taskname)\r
finally:\r
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done"))\r
self.feedback_q.put("Done")\r
+ self.cache_q.put("CacheDone")\r
+\r
def printStatus(self):\r
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))\r
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))\r