BaseTools/Source/Python/AutoGen/AutoGenWorker.py

   1 ## @file
   2 # Create makefile for MS nmake and GNU make
   3 #
   4 # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7 from __future__ import absolute_import
   8 import multiprocessing as mp
   9 import threading
  10 from Common.Misc import PathClass
  11 from AutoGen.ModuleAutoGen import ModuleAutoGen
  12 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
  13 import Common.GlobalData as GlobalData
  14 import Common.EdkLogger as EdkLogger
  15 import os
  16 from Common.MultipleWorkspace import MultipleWorkspace as mws
  17 from AutoGen.AutoGen import AutoGen
  18 from Workspace.WorkspaceDatabase import BuildDB
  19 from queue import Empty
  20 import traceback
  21 import sys
  22 from AutoGen.DataPipe import MemoryDataPipe
  23 def clearQ(q):
  24     try:
  25         while True:
  26             q.get_nowait()
  27     except Empty:
  28         pass
  29 import logging
  30
  31 class LogAgent(threading.Thread):
  32     def __init__(self,log_q,log_level,log_file=None):
  33         super(LogAgent,self).__init__()
  34         self.log_q = log_q
  35         self.log_level = log_level
  36         self.log_file = log_file
  37     def InitLogger(self):
  38         # For DEBUG level (All DEBUG_0~9 are applicable)
  39         self._DebugLogger_agent = logging.getLogger("tool_debug_agent")
  40         _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")
  41         self._DebugLogger_agent.setLevel(self.log_level)
  42         _DebugChannel = logging.StreamHandler(sys.stdout)
  43         _DebugChannel.setFormatter(_DebugFormatter)
  44         self._DebugLogger_agent.addHandler(_DebugChannel)
  45
  46         # For VERBOSE, INFO, WARN level
  47         self._InfoLogger_agent = logging.getLogger("tool_info_agent")
  48         _InfoFormatter = logging.Formatter("%(message)s")
  49         self._InfoLogger_agent.setLevel(self.log_level)
  50         _InfoChannel = logging.StreamHandler(sys.stdout)
  51         _InfoChannel.setFormatter(_InfoFormatter)
  52         self._InfoLogger_agent.addHandler(_InfoChannel)
  53
  54         # For ERROR level
  55         self._ErrorLogger_agent = logging.getLogger("tool_error_agent")
  56         _ErrorFormatter = logging.Formatter("%(message)s")
  57         self._ErrorLogger_agent.setLevel(self.log_level)
  58         _ErrorCh = logging.StreamHandler(sys.stderr)
  59         _ErrorCh.setFormatter(_ErrorFormatter)
  60         self._ErrorLogger_agent.addHandler(_ErrorCh)
  61
  62         if self.log_file:
  63             if os.path.exists(self.log_file):
  64                 os.remove(self.log_file)
  65             _Ch = logging.FileHandler(self.log_file)
  66             _Ch.setFormatter(_DebugFormatter)
  67             self._DebugLogger_agent.addHandler(_Ch)
  68
  69             _Ch= logging.FileHandler(self.log_file)
  70             _Ch.setFormatter(_InfoFormatter)
  71             self._InfoLogger_agent.addHandler(_Ch)
  72
  73             _Ch = logging.FileHandler(self.log_file)
  74             _Ch.setFormatter(_ErrorFormatter)
  75             self._ErrorLogger_agent.addHandler(_Ch)
  76
  77     def run(self):
  78         self.InitLogger()
  79         while True:
  80             log_message = self.log_q.get()
  81             if log_message is None:
  82                 break
  83             if log_message.name == "tool_error":
  84                 self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())
  85             elif log_message.name == "tool_info":
  86                 self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
  87             elif log_message.name == "tool_debug":
  88                 self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())
  89             else:
  90                 self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
  91
  92     def kill(self):
  93         self.log_q.put(None)
  94 class AutoGenManager(threading.Thread):
  95     def __init__(self,autogen_workers, feedback_q,error_event):
  96         super(AutoGenManager,self).__init__()
  97         self.autogen_workers = autogen_workers
  98         self.feedback_q = feedback_q
  99         self.Status = True
 100         self.error_event = error_event
 101     def run(self):
 102         try:
 103             fin_num = 0
 104             while True:
 105                 badnews = self.feedback_q.get()
 106                 if badnews is None:
 107                     break
 108                 if badnews == "Done":
 109                     fin_num += 1
 110                 else:
 111                     self.Status = False
 112                     self.TerminateWorkers()
 113                 if fin_num == len(self.autogen_workers):
 114                     self.clearQueue()
 115                     for w in self.autogen_workers:
 116                         w.join()
 117                     break
 118         except Exception:
 119             return
 120
 121     def clearQueue(self):
 122         taskq = self.autogen_workers[0].module_queue
 123         clearQ(taskq)
 124         clearQ(self.feedback_q)
 125
 126     def TerminateWorkers(self):
 127         self.error_event.set()
 128     def kill(self):
 129         self.feedback_q.put(None)
 130 class AutoGenWorkerInProcess(mp.Process):
 131     def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,log_q,error_event):
 132         mp.Process.__init__(self)
 133         self.module_queue = module_queue
 134         self.data_pipe_file_path =data_pipe_file_path
 135         self.data_pipe = None
 136         self.feedback_q = feedback_q
 137         self.PlatformMetaFileSet = {}
 138         self.file_lock = file_lock
 139         self.share_data = share_data
 140         self.log_q = log_q
 141         self.error_event = error_event
 142     def GetPlatformMetaFile(self,filepath,root):
 143         try:
 144             return self.PlatformMetaFileSet[(filepath,root)]
 145         except:
 146             self.PlatformMetaFileSet[(filepath,root)]  = filepath
 147             return self.PlatformMetaFileSet[(filepath,root)]
 148     def run(self):
 149         try:
 150             taskname = "Init"
 151             with self.file_lock:
 152                 if not os.path.exists(self.data_pipe_file_path):
 153                     self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
 154                 self.data_pipe = MemoryDataPipe()
 155                 self.data_pipe.load(self.data_pipe_file_path)
 156             EdkLogger.LogClientInitialize(self.log_q)
 157             loglevel = self.data_pipe.Get("LogLevel")
 158             if not loglevel:
 159                 loglevel = EdkLogger.INFO
 160             EdkLogger.SetLevel(loglevel)
 161             target = self.data_pipe.Get("P_Info").get("Target")
 162             toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 163             archlist = self.data_pipe.Get("P_Info").get("ArchList")
 164
 165             active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
 166             workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
 167             PackagesPath = os.getenv("PACKAGES_PATH")
 168             mws.setWs(workspacedir, PackagesPath)
 169             self.Wa = WorkSpaceInfo(
 170                 workspacedir,active_p,target,toolchain,archlist
 171                 )
 172             self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
 173             GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
 174             GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
 175             os.environ._data = self.data_pipe.Get("Env_Var")
 176             GlobalData.gWorkspace = workspacedir
 177             GlobalData.gDisableIncludePathCheck = False
 178             GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
 179             GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
 180             pcd_from_build_option = []
 181             for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
 182                 pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
 183                 if pcd_tuple[2].strip():
 184                     pcd_id = ".".join((pcd_id,pcd_tuple[2]))
 185                 pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
 186             GlobalData.BuildOptionPcd = pcd_from_build_option
 187             module_count = 0
 188             FfsCmd = self.data_pipe.Get("FfsCommand")
 189             if FfsCmd is None:
 190                 FfsCmd = {}
 191             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
 192                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))
 193             libConstPcd = self.data_pipe.Get("LibConstPcd")
 194             Refes = self.data_pipe.Get("REFS")
 195             while True:
 196                 if self.module_queue.empty():
 197                     break
 198                 if self.error_event.is_set():
 199                     break
 200                 module_count += 1
 201                 module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
 202                 modulefullpath = os.path.join(module_root,module_file)
 203                 taskname = " : ".join((modulefullpath,module_arch))
 204                 module_metafile = PathClass(module_file,module_root)
 205                 if module_path:
 206                     module_metafile.Path = module_path
 207                 if module_basename:
 208                     module_metafile.BaseName = module_basename
 209                 if module_originalpath:
 210                     module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
 211                 arch = module_arch
 212                 target = self.data_pipe.Get("P_Info").get("Target")
 213                 toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 214                 Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
 215                 Ma.IsLibrary = IsLib
 216                 if IsLib:
 217                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
 218                         Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 219                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
 220                         Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 221                 Ma.CreateCodeFile(False)
 222                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
 223         except Empty:
 224             pass
 225         except:
 226             traceback.print_exc(file=sys.stdout)
 227             self.feedback_q.put(taskname)
 228         finally:
 229             self.feedback_q.put("Done")
 230     def printStatus(self):
 231         print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
 232         print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
 233         groupobj = {}
 234         for buildobj in BuildDB.BuildObject.GetCache().values():
 235             if str(buildobj).lower().endswith("dec"):
 236                 try:
 237                     groupobj['dec'].append(str(buildobj))
 238                 except:
 239                     groupobj['dec'] = [str(buildobj)]
 240             if str(buildobj).lower().endswith("dsc"):
 241                 try:
 242                     groupobj['dsc'].append(str(buildobj))
 243                 except:
 244                     groupobj['dsc'] = [str(buildobj)]
 245
 246             if str(buildobj).lower().endswith("inf"):
 247                 try:
 248                     groupobj['inf'].append(str(buildobj))
 249                 except:
 250                     groupobj['inf'] = [str(buildobj)]
 251
 252         print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
 253         print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
 254         print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))