BaseTools/Source/Python/AutoGen/AutoGenWorker.py

   1 ## @file
   2 # Create makefile for MS nmake and GNU make
   3 #
   4 # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7 from __future__ import absolute_import
   8 import multiprocessing as mp
   9 import threading
  10 from Common.Misc import PathClass
  11 from AutoGen.ModuleAutoGen import ModuleAutoGen
  12 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
  13 import Common.GlobalData as GlobalData
  14 import Common.EdkLogger as EdkLogger
  15 import os
  16 from Common.MultipleWorkspace import MultipleWorkspace as mws
  17 from AutoGen.AutoGen import AutoGen
  18 from Workspace.WorkspaceDatabase import BuildDB
  19 try:
  20     from queue import Empty
  21 except:
  22     from Queue import Empty
  23 import traceback
  24 import sys
  25 from AutoGen.DataPipe import MemoryDataPipe
  26 def clearQ(q):
  27     try:
  28         while True:
  29             q.get_nowait()
  30     except Empty:
  31         pass
  32 import logging
  33
  34 class LogAgent(threading.Thread):
  35     def __init__(self,log_q,log_level,log_file=None):
  36         super(LogAgent,self).__init__()
  37         self.log_q = log_q
  38         self.log_level = log_level
  39         self.log_file = log_file
  40     def InitLogger(self):
  41         # For DEBUG level (All DEBUG_0~9 are applicable)
  42         self._DebugLogger_agent = logging.getLogger("tool_debug_agent")
  43         _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")
  44         self._DebugLogger_agent.setLevel(self.log_level)
  45         _DebugChannel = logging.StreamHandler(sys.stdout)
  46         _DebugChannel.setFormatter(_DebugFormatter)
  47         self._DebugLogger_agent.addHandler(_DebugChannel)
  48
  49         # For VERBOSE, INFO, WARN level
  50         self._InfoLogger_agent = logging.getLogger("tool_info_agent")
  51         _InfoFormatter = logging.Formatter("%(message)s")
  52         self._InfoLogger_agent.setLevel(self.log_level)
  53         _InfoChannel = logging.StreamHandler(sys.stdout)
  54         _InfoChannel.setFormatter(_InfoFormatter)
  55         self._InfoLogger_agent.addHandler(_InfoChannel)
  56
  57         # For ERROR level
  58         self._ErrorLogger_agent = logging.getLogger("tool_error_agent")
  59         _ErrorFormatter = logging.Formatter("%(message)s")
  60         self._ErrorLogger_agent.setLevel(self.log_level)
  61         _ErrorCh = logging.StreamHandler(sys.stderr)
  62         _ErrorCh.setFormatter(_ErrorFormatter)
  63         self._ErrorLogger_agent.addHandler(_ErrorCh)
  64
  65         if self.log_file:
  66             if os.path.exists(self.log_file):
  67                 os.remove(self.log_file)
  68             _Ch = logging.FileHandler(self.log_file)
  69             _Ch.setFormatter(_DebugFormatter)
  70             self._DebugLogger_agent.addHandler(_Ch)
  71
  72             _Ch= logging.FileHandler(self.log_file)
  73             _Ch.setFormatter(_InfoFormatter)
  74             self._InfoLogger_agent.addHandler(_Ch)
  75
  76             _Ch = logging.FileHandler(self.log_file)
  77             _Ch.setFormatter(_ErrorFormatter)
  78             self._ErrorLogger_agent.addHandler(_Ch)
  79
  80     def run(self):
  81         self.InitLogger()
  82         while True:
  83             log_message = self.log_q.get()
  84             if log_message is None:
  85                 break
  86             if log_message.name == "tool_error":
  87                 self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())
  88             elif log_message.name == "tool_info":
  89                 self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
  90             elif log_message.name == "tool_debug":
  91                 self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())
  92             else:
  93                 self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
  94
  95     def kill(self):
  96         self.log_q.put(None)
  97 class AutoGenManager(threading.Thread):
  98     def __init__(self,autogen_workers, feedback_q,error_event):
  99         super(AutoGenManager,self).__init__()
 100         self.autogen_workers = autogen_workers
 101         self.feedback_q = feedback_q
 102         self.Status = True
 103         self.error_event = error_event
 104     def run(self):
 105         try:
 106             fin_num = 0
 107             while True:
 108                 badnews = self.feedback_q.get()
 109                 if badnews is None:
 110                     break
 111                 if badnews == "Done":
 112                     fin_num += 1
 113                 else:
 114                     self.Status = False
 115                     self.TerminateWorkers()
 116                 if fin_num == len(self.autogen_workers):
 117                     self.clearQueue()
 118                     for w in self.autogen_workers:
 119                         w.join()
 120                     break
 121         except Exception:
 122             return
 123
 124     def clearQueue(self):
 125         taskq = self.autogen_workers[0].module_queue
 126         clearQ(taskq)
 127         clearQ(self.feedback_q)
 128
 129     def TerminateWorkers(self):
 130         self.error_event.set()
 131     def kill(self):
 132         self.feedback_q.put(None)
 133 class AutoGenWorkerInProcess(mp.Process):
 134     def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,log_q,error_event):
 135         mp.Process.__init__(self)
 136         self.module_queue = module_queue
 137         self.data_pipe_file_path =data_pipe_file_path
 138         self.data_pipe = None
 139         self.feedback_q = feedback_q
 140         self.PlatformMetaFileSet = {}
 141         self.file_lock = file_lock
 142         self.share_data = share_data
 143         self.log_q = log_q
 144         self.error_event = error_event
 145     def GetPlatformMetaFile(self,filepath,root):
 146         try:
 147             return self.PlatformMetaFileSet[(filepath,root)]
 148         except:
 149             self.PlatformMetaFileSet[(filepath,root)]  = filepath
 150             return self.PlatformMetaFileSet[(filepath,root)]
 151     def run(self):
 152         try:
 153             taskname = "Init"
 154             with self.file_lock:
 155                 if not os.path.exists(self.data_pipe_file_path):
 156                     self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
 157                 self.data_pipe = MemoryDataPipe()
 158                 self.data_pipe.load(self.data_pipe_file_path)
 159             EdkLogger.LogClientInitialize(self.log_q)
 160             loglevel = self.data_pipe.Get("LogLevel")
 161             if not loglevel:
 162                 loglevel = EdkLogger.INFO
 163             EdkLogger.SetLevel(loglevel)
 164             target = self.data_pipe.Get("P_Info").get("Target")
 165             toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 166             archlist = self.data_pipe.Get("P_Info").get("ArchList")
 167
 168             active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
 169             workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
 170             PackagesPath = os.getenv("PACKAGES_PATH")
 171             mws.setWs(workspacedir, PackagesPath)
 172             self.Wa = WorkSpaceInfo(
 173                 workspacedir,active_p,target,toolchain,archlist
 174                 )
 175             self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
 176             GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
 177             GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
 178             os.environ._data = self.data_pipe.Get("Env_Var")
 179             GlobalData.gWorkspace = workspacedir
 180             GlobalData.gDisableIncludePathCheck = False
 181             GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
 182             GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
 183             pcd_from_build_option = []
 184             for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
 185                 pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
 186                 if pcd_tuple[2].strip():
 187                     pcd_id = ".".join((pcd_id,pcd_tuple[2]))
 188                 pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
 189             GlobalData.BuildOptionPcd = pcd_from_build_option
 190             module_count = 0
 191             FfsCmd = self.data_pipe.Get("FfsCommand")
 192             if FfsCmd is None:
 193                 FfsCmd = {}
 194             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
 195                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))
 196             libConstPcd = self.data_pipe.Get("LibConstPcd")
 197             Refes = self.data_pipe.Get("REFS")
 198             while True:
 199                 if self.module_queue.empty():
 200                     break
 201                 if self.error_event.is_set():
 202                     break
 203                 module_count += 1
 204                 module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
 205                 modulefullpath = os.path.join(module_root,module_file)
 206                 taskname = " : ".join((modulefullpath,module_arch))
 207                 module_metafile = PathClass(module_file,module_root)
 208                 if module_path:
 209                     module_metafile.Path = module_path
 210                 if module_basename:
 211                     module_metafile.BaseName = module_basename
 212                 if module_originalpath:
 213                     module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
 214                 arch = module_arch
 215                 target = self.data_pipe.Get("P_Info").get("Target")
 216                 toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 217                 Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
 218                 Ma.IsLibrary = IsLib
 219                 if IsLib:
 220                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
 221                         Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 222                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
 223                         Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 224                 Ma.CreateCodeFile(False)
 225                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
 226         except Empty:
 227             pass
 228         except:
 229             traceback.print_exc(file=sys.stdout)
 230             self.feedback_q.put(taskname)
 231         finally:
 232             self.feedback_q.put("Done")
 233     def printStatus(self):
 234         print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
 235         print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
 236         groupobj = {}
 237         for buildobj in BuildDB.BuildObject.GetCache().values():
 238             if str(buildobj).lower().endswith("dec"):
 239                 try:
 240                     groupobj['dec'].append(str(buildobj))
 241                 except:
 242                     groupobj['dec'] = [str(buildobj)]
 243             if str(buildobj).lower().endswith("dsc"):
 244                 try:
 245                     groupobj['dsc'].append(str(buildobj))
 246                 except:
 247                     groupobj['dsc'] = [str(buildobj)]
 248
 249             if str(buildobj).lower().endswith("inf"):
 250                 try:
 251                     groupobj['inf'].append(str(buildobj))
 252                 except:
 253                     groupobj['inf'] = [str(buildobj)]
 254
 255         print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
 256         print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
 257         print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))