BaseTools/Source/Python/AutoGen/AutoGenWorker.py

   1 ## @file
   2 # Create makefile for MS nmake and GNU make
   3 #
   4 # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7 from __future__ import absolute_import
   8 import multiprocessing as mp
   9 import threading
  10 from Common.Misc import PathClass
  11 from AutoGen.ModuleAutoGen import ModuleAutoGen
  12 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
  13 import Common.GlobalData as GlobalData
  14 import Common.EdkLogger as EdkLogger
  15 import os
  16 from Common.MultipleWorkspace import MultipleWorkspace as mws
  17 from AutoGen.AutoGen import AutoGen
  18 from Workspace.WorkspaceDatabase import BuildDB
  19 from queue import Empty
  20 import traceback
  21 import sys
  22 from AutoGen.DataPipe import MemoryDataPipe
  23 def clearQ(q):
  24     try:
  25         while True:
  26             q.get_nowait()
  27     except Empty:
  28         pass
  29 class AutoGenManager(threading.Thread):
  30     def __init__(self,autogen_workers, feedback_q,error_event):
  31         super(AutoGenManager,self).__init__()
  32         self.autogen_workers = autogen_workers
  33         self.feedback_q = feedback_q
  34         self.terminate = False
  35         self.Status = True
  36         self.error_event = error_event
  37     def run(self):
  38         try:
  39             fin_num = 0
  40             while True:
  41                 badnews = self.feedback_q.get()
  42                 if badnews is None:
  43                     break
  44                 if badnews == "Done":
  45                     fin_num += 1
  46                 else:
  47                     self.Status = False
  48                     self.TerminateWorkers()
  49                 if fin_num == len(self.autogen_workers):
  50                     self.clearQueue()
  51                     for w in self.autogen_workers:
  52                         w.join()
  53                     break
  54         except Exception:
  55             return
  56
  57     def clearQueue(self):
  58         taskq = self.autogen_workers[0].module_queue
  59         clearQ(taskq)
  60         clearQ(self.feedback_q)
  61
  62     def TerminateWorkers(self):
  63         self.error_event.set()
  64     def kill(self):
  65         self.feedback_q.put(None)
  66 class AutoGenWorkerInProcess(mp.Process):
  67     def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,error_event):
  68         mp.Process.__init__(self)
  69         self.module_queue = module_queue
  70         self.data_pipe_file_path =data_pipe_file_path
  71         self.data_pipe = None
  72         self.feedback_q = feedback_q
  73         self.PlatformMetaFileSet = {}
  74         self.file_lock = file_lock
  75         self.share_data = share_data
  76         self.error_event = error_event
  77     def GetPlatformMetaFile(self,filepath,root):
  78         try:
  79             return self.PlatformMetaFileSet[(filepath,root)]
  80         except:
  81             self.PlatformMetaFileSet[(filepath,root)]  = filepath
  82             return self.PlatformMetaFileSet[(filepath,root)]
  83     def run(self):
  84         try:
  85             taskname = "Init"
  86             with self.file_lock:
  87                 if not os.path.exists(self.data_pipe_file_path):
  88                     self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
  89                 self.data_pipe = MemoryDataPipe()
  90                 self.data_pipe.load(self.data_pipe_file_path)
  91             EdkLogger.Initialize()
  92             loglevel = self.data_pipe.Get("LogLevel")
  93             if not loglevel:
  94                 loglevel = EdkLogger.INFO
  95             EdkLogger.SetLevel(loglevel)
  96             logfile = self.data_pipe.Get("LogFile")
  97             if logfile:
  98                 EdkLogger.SetLogFile(logfile)
  99             target = self.data_pipe.Get("P_Info").get("Target")
 100             toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 101             archlist = self.data_pipe.Get("P_Info").get("ArchList")
 102
 103             active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
 104             workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
 105             PackagesPath = os.getenv("PACKAGES_PATH")
 106             mws.setWs(workspacedir, PackagesPath)
 107             self.Wa = WorkSpaceInfo(
 108                 workspacedir,active_p,target,toolchain,archlist
 109                 )
 110             self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
 111             GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
 112             GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
 113             os.environ._data = self.data_pipe.Get("Env_Var")
 114             GlobalData.gWorkspace = workspacedir
 115             GlobalData.gDisableIncludePathCheck = False
 116             GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
 117             GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
 118             pcd_from_build_option = []
 119             for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
 120                 pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
 121                 if pcd_tuple[2].strip():
 122                     pcd_id = ".".join((pcd_id,pcd_tuple[2]))
 123                 pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
 124             GlobalData.BuildOptionPcd = pcd_from_build_option
 125             module_count = 0
 126             FfsCmd = self.data_pipe.Get("FfsCommand")
 127             if FfsCmd is None:
 128                 FfsCmd = {}
 129             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
 130                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))
 131             libConstPcd = self.data_pipe.Get("LibConstPcd")
 132             Refes = self.data_pipe.Get("REFS")
 133             while True:
 134                 if self.module_queue.empty():
 135                     break
 136                 if self.error_event.is_set():
 137                     break
 138                 module_count += 1
 139                 module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
 140                 modulefullpath = os.path.join(module_root,module_file)
 141                 taskname = " : ".join((modulefullpath,module_arch))
 142                 module_metafile = PathClass(module_file,module_root)
 143                 if module_path:
 144                     module_metafile.Path = module_path
 145                 if module_basename:
 146                     module_metafile.BaseName = module_basename
 147                 if module_originalpath:
 148                     module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
 149                 arch = module_arch
 150                 target = self.data_pipe.Get("P_Info").get("Target")
 151                 toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 152                 Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
 153                 Ma.IsLibrary = IsLib
 154                 if IsLib:
 155                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
 156                         Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 157                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
 158                         Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 159                 Ma.CreateCodeFile(False)
 160                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
 161         except Empty:
 162             pass
 163         except:
 164             traceback.print_exc(file=sys.stdout)
 165             self.feedback_q.put(taskname)
 166         finally:
 167             self.feedback_q.put("Done")
 168     def printStatus(self):
 169         print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
 170         print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
 171         groupobj = {}
 172         for buildobj in BuildDB.BuildObject.GetCache().values():
 173             if str(buildobj).lower().endswith("dec"):
 174                 try:
 175                     groupobj['dec'].append(str(buildobj))
 176                 except:
 177                     groupobj['dec'] = [str(buildobj)]
 178             if str(buildobj).lower().endswith("dsc"):
 179                 try:
 180                     groupobj['dsc'].append(str(buildobj))
 181                 except:
 182                     groupobj['dsc'] = [str(buildobj)]
 183
 184             if str(buildobj).lower().endswith("inf"):
 185                 try:
 186                     groupobj['inf'].append(str(buildobj))
 187                 except:
 188                     groupobj['inf'] = [str(buildobj)]
 189
 190         print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
 191         print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
 192         print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))