from Common.MultipleWorkspace import MultipleWorkspace as mws\r
from AutoGen.AutoGen import AutoGen\r
from Workspace.WorkspaceDatabase import BuildDB\r
-from queue import Empty\r
+try:\r
+ from queue import Empty\r
+except:\r
+ from Queue import Empty\r
import traceback\r
import sys\r
from AutoGen.DataPipe import MemoryDataPipe\r
+import logging\r
+\r
def clearQ(q):\r
try:\r
while True:\r
q.get_nowait()\r
except Empty:\r
pass\r
+\r
+class LogAgent(threading.Thread):\r
+ def __init__(self,log_q,log_level,log_file=None):\r
+ super(LogAgent,self).__init__()\r
+ self.log_q = log_q\r
+ self.log_level = log_level\r
+ self.log_file = log_file\r
+ def InitLogger(self):\r
+ # For DEBUG level (All DEBUG_0~9 are applicable)\r
+ self._DebugLogger_agent = logging.getLogger("tool_debug_agent")\r
+ _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")\r
+ self._DebugLogger_agent.setLevel(self.log_level)\r
+ _DebugChannel = logging.StreamHandler(sys.stdout)\r
+ _DebugChannel.setFormatter(_DebugFormatter)\r
+ self._DebugLogger_agent.addHandler(_DebugChannel)\r
+\r
+ # For VERBOSE, INFO, WARN level\r
+ self._InfoLogger_agent = logging.getLogger("tool_info_agent")\r
+ _InfoFormatter = logging.Formatter("%(message)s")\r
+ self._InfoLogger_agent.setLevel(self.log_level)\r
+ _InfoChannel = logging.StreamHandler(sys.stdout)\r
+ _InfoChannel.setFormatter(_InfoFormatter)\r
+ self._InfoLogger_agent.addHandler(_InfoChannel)\r
+\r
+ # For ERROR level\r
+ self._ErrorLogger_agent = logging.getLogger("tool_error_agent")\r
+ _ErrorFormatter = logging.Formatter("%(message)s")\r
+ self._ErrorLogger_agent.setLevel(self.log_level)\r
+ _ErrorCh = logging.StreamHandler(sys.stderr)\r
+ _ErrorCh.setFormatter(_ErrorFormatter)\r
+ self._ErrorLogger_agent.addHandler(_ErrorCh)\r
+\r
+ if self.log_file:\r
+ if os.path.exists(self.log_file):\r
+ os.remove(self.log_file)\r
+ _Ch = logging.FileHandler(self.log_file)\r
+ _Ch.setFormatter(_DebugFormatter)\r
+ self._DebugLogger_agent.addHandler(_Ch)\r
+\r
+ _Ch= logging.FileHandler(self.log_file)\r
+ _Ch.setFormatter(_InfoFormatter)\r
+ self._InfoLogger_agent.addHandler(_Ch)\r
+\r
+ _Ch = logging.FileHandler(self.log_file)\r
+ _Ch.setFormatter(_ErrorFormatter)\r
+ self._ErrorLogger_agent.addHandler(_Ch)\r
+\r
+ def run(self):\r
+ self.InitLogger()\r
+ while True:\r
+ log_message = self.log_q.get()\r
+ if log_message is None:\r
+ break\r
+ if log_message.name == "tool_error":\r
+ self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())\r
+ elif log_message.name == "tool_info":\r
+ self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())\r
+ elif log_message.name == "tool_debug":\r
+ self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())\r
+ else:\r
+ self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())\r
+\r
+ def kill(self):\r
+ self.log_q.put(None)\r
class AutoGenManager(threading.Thread):\r
def __init__(self,autogen_workers, feedback_q,error_event):\r
super(AutoGenManager,self).__init__()\r
self.autogen_workers = autogen_workers\r
self.feedback_q = feedback_q\r
- self.terminate = False\r
self.Status = True\r
self.error_event = error_event\r
def run(self):\r
\r
def clearQueue(self):\r
taskq = self.autogen_workers[0].module_queue\r
+ logq = self.autogen_workers[0].log_q\r
clearQ(taskq)\r
clearQ(self.feedback_q)\r
+ clearQ(logq)\r
+ # Copy the cache queue itmes to parent thread before clear\r
+ cacheq = self.autogen_workers[0].cache_q\r
+ try:\r
+ cache_num = 0\r
+ while True:\r
+ item = cacheq.get()\r
+ if item == "CacheDone":\r
+ cache_num += 1\r
+ else:\r
+ GlobalData.gModuleAllCacheStatus.add(item)\r
+ if cache_num == len(self.autogen_workers):\r
+ break\r
+ except:\r
+ print ("cache_q error")\r
\r
def TerminateWorkers(self):\r
self.error_event.set()\r
def kill(self):\r
self.feedback_q.put(None)\r
class AutoGenWorkerInProcess(mp.Process):\r
- def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,error_event):\r
+ def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):\r
mp.Process.__init__(self)\r
self.module_queue = module_queue\r
self.data_pipe_file_path =data_pipe_file_path\r
self.feedback_q = feedback_q\r
self.PlatformMetaFileSet = {}\r
self.file_lock = file_lock\r
+ self.cache_q = cache_q\r
+ self.log_q = log_q\r
self.error_event = error_event\r
def GetPlatformMetaFile(self,filepath,root):\r
try:\r
try:\r
taskname = "Init"\r
with self.file_lock:\r
- if not os.path.exists(self.data_pipe_file_path):\r
+ try:\r
+ self.data_pipe = MemoryDataPipe()\r
+ self.data_pipe.load(self.data_pipe_file_path)\r
+ except:\r
self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)\r
- self.data_pipe = MemoryDataPipe()\r
- self.data_pipe.load(self.data_pipe_file_path)\r
- EdkLogger.Initialize()\r
+ EdkLogger.LogClientInitialize(self.log_q)\r
loglevel = self.data_pipe.Get("LogLevel")\r
if not loglevel:\r
loglevel = EdkLogger.INFO\r
EdkLogger.SetLevel(loglevel)\r
- logfile = self.data_pipe.Get("LogFile")\r
- if logfile:\r
- EdkLogger.SetLogFile(logfile)\r
target = self.data_pipe.Get("P_Info").get("Target")\r
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r
archlist = self.data_pipe.Get("P_Info").get("ArchList")\r
GlobalData.gDisableIncludePathCheck = False\r
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")\r
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")\r
+\r
+ GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")\r
+ GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")\r
+ GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")\r
+ GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")\r
+ GlobalData.gModulePreMakeCacheStatus = dict()\r
+ GlobalData.gModuleMakeCacheStatus = dict()\r
+ GlobalData.gHashChainStatus = dict()\r
+ GlobalData.gCMakeHashFile = dict()\r
+ GlobalData.gModuleHashFile = dict()\r
+ GlobalData.gFileHashDict = dict()\r
+ GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")\r
+ GlobalData.file_lock = self.file_lock\r
+ CommandTarget = self.data_pipe.Get("CommandTarget")\r
pcd_from_build_option = []\r
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):\r
pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))\r
FfsCmd = self.data_pipe.Get("FfsCommand")\r
if FfsCmd is None:\r
FfsCmd = {}\r
+ GlobalData.FfsCmd = FfsCmd\r
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),\r
self.data_pipe.Get("P_Info").get("WorkspaceDir"))\r
- libConstPcd = self.data_pipe.Get("LibConstPcd")\r
- Refes = self.data_pipe.Get("REFS")\r
while True:\r
if self.module_queue.empty():\r
break\r
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)\r
Ma.IsLibrary = IsLib\r
- if IsLib:\r
- if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:\r
- Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r
- if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:\r
- Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r
+ # SourceFileList calling sequence impact the makefile string sequence.\r
+ # Create cached SourceFileList here to unify its calling sequence for both\r
+ # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.\r
+ RetVal = Ma.SourceFileList\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:\r
+ try:\r
+ CacheResult = Ma.CanSkipbyPreMakeCache()\r
+ except:\r
+ CacheResult = False\r
+ self.feedback_q.put(taskname)\r
+\r
+ if CacheResult:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))\r
+ continue\r
+ else:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))\r
+\r
Ma.CreateCodeFile(False)\r
- Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))\r
+ Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))\r
+\r
+ if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:\r
+ try:\r
+ CacheResult = Ma.CanSkipbyMakeCache()\r
+ except:\r
+ CacheResult = False\r
+ self.feedback_q.put(taskname)\r
+\r
+ if CacheResult:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))\r
+ continue\r
+ else:\r
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))\r
+\r
except Empty:\r
pass\r
except:\r
- traceback.print_exc(file=sys.stdout)\r
self.feedback_q.put(taskname)\r
finally:\r
self.feedback_q.put("Done")\r
+ self.cache_q.put("CacheDone")\r
+\r
def printStatus(self):\r
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))\r
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))\r