BaseTools/Source/Python/AutoGen/AutoGenWorker.py

   1 ## @file
   2 # Create makefile for MS nmake and GNU make
   3 #
   4 # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7 from __future__ import absolute_import
   8 import multiprocessing as mp
   9 import threading
  10 from Common.Misc import PathClass
  11 from AutoGen.ModuleAutoGen import ModuleAutoGen
  12 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
  13 import Common.GlobalData as GlobalData
  14 import Common.EdkLogger as EdkLogger
  15 import os
  16 from Common.MultipleWorkspace import MultipleWorkspace as mws
  17 from AutoGen.AutoGen import AutoGen
  18 from Workspace.WorkspaceDatabase import BuildDB
  19 from queue import Empty
  20 import traceback
  21 import sys
  22 from AutoGen.DataPipe import MemoryDataPipe
  23 def clearQ(q):
  24     try:
  25         while True:
  26             q.get_nowait()
  27     except Empty:
  28         pass
  29 class AutoGenManager(threading.Thread):
  30     def __init__(self,autogen_workers, feedback_q,error_event):
  31         super(AutoGenManager,self).__init__()
  32         self.autogen_workers = autogen_workers
  33         self.feedback_q = feedback_q
  34         self.terminate = False
  35         self.Status = True
  36         self.error_event = error_event
  37     def run(self):
  38         try:
  39             fin_num = 0
  40             while True:
  41                 badnews = self.feedback_q.get()
  42                 if badnews is None:
  43                     break
  44                 if badnews == "Done":
  45                     fin_num += 1
  46                 else:
  47                     self.Status = False
  48                     self.TerminateWorkers()
  49                 if fin_num == len(self.autogen_workers):
  50                     self.clearQueue()
  51                     for w in self.autogen_workers:
  52                         w.join()
  53                     break
  54         except Exception:
  55             return
  56
  57     def clearQueue(self):
  58         taskq = self.autogen_workers[0].module_queue
  59         clearQ(taskq)
  60         clearQ(self.feedback_q)
  61
  62     def TerminateWorkers(self):
  63         self.error_event.set()
  64     def kill(self):
  65         self.feedback_q.put(None)
  66 class AutoGenWorkerInProcess(mp.Process):
  67     def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,error_event):
  68         mp.Process.__init__(self)
  69         self.module_queue = module_queue
  70         self.data_pipe_file_path =data_pipe_file_path
  71         self.data_pipe = None
  72         self.feedback_q = feedback_q
  73         self.PlatformMetaFileSet = {}
  74         self.file_lock = file_lock
  75         self.error_event = error_event
  76     def GetPlatformMetaFile(self,filepath,root):
  77         try:
  78             return self.PlatformMetaFileSet[(filepath,root)]
  79         except:
  80             self.PlatformMetaFileSet[(filepath,root)]  = filepath
  81             return self.PlatformMetaFileSet[(filepath,root)]
  82     def run(self):
  83         try:
  84             taskname = "Init"
  85             with self.file_lock:
  86                 if not os.path.exists(self.data_pipe_file_path):
  87                     self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
  88                 self.data_pipe = MemoryDataPipe()
  89                 self.data_pipe.load(self.data_pipe_file_path)
  90             EdkLogger.Initialize()
  91             loglevel = self.data_pipe.Get("LogLevel")
  92             if not loglevel:
  93                 loglevel = EdkLogger.INFO
  94             EdkLogger.SetLevel(loglevel)
  95             logfile = self.data_pipe.Get("LogFile")
  96             if logfile:
  97                 EdkLogger.SetLogFile(logfile)
  98             target = self.data_pipe.Get("P_Info").get("Target")
  99             toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 100             archlist = self.data_pipe.Get("P_Info").get("ArchList")
 101
 102             active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
 103             workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
 104             PackagesPath = os.getenv("PACKAGES_PATH")
 105             mws.setWs(workspacedir, PackagesPath)
 106             self.Wa = WorkSpaceInfo(
 107                 workspacedir,active_p,target,toolchain,archlist
 108                 )
 109             self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
 110             GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
 111             GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
 112             os.environ._data = self.data_pipe.Get("Env_Var")
 113             GlobalData.gWorkspace = workspacedir
 114             GlobalData.gDisableIncludePathCheck = False
 115             GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
 116             GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
 117             pcd_from_build_option = []
 118             for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
 119                 pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
 120                 if pcd_tuple[2].strip():
 121                     pcd_id = ".".join((pcd_id,pcd_tuple[2]))
 122                 pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
 123             GlobalData.BuildOptionPcd = pcd_from_build_option
 124             module_count = 0
 125             FfsCmd = self.data_pipe.Get("FfsCommand")
 126             if FfsCmd is None:
 127                 FfsCmd = {}
 128             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
 129                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))
 130             libConstPcd = self.data_pipe.Get("LibConstPcd")
 131             Refes = self.data_pipe.Get("REFS")
 132             while True:
 133                 if self.module_queue.empty():
 134                     break
 135                 if self.error_event.is_set():
 136                     break
 137                 module_count += 1
 138                 module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
 139                 modulefullpath = os.path.join(module_root,module_file)
 140                 taskname = " : ".join((modulefullpath,module_arch))
 141                 module_metafile = PathClass(module_file,module_root)
 142                 if module_path:
 143                     module_metafile.Path = module_path
 144                 if module_basename:
 145                     module_metafile.BaseName = module_basename
 146                 if module_originalpath:
 147                     module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
 148                 arch = module_arch
 149                 target = self.data_pipe.Get("P_Info").get("Target")
 150                 toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
 151                 Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
 152                 Ma.IsLibrary = IsLib
 153                 if IsLib:
 154                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
 155                         Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 156                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
 157                         Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 158                 Ma.CreateCodeFile(False)
 159                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
 160         except Empty:
 161             pass
 162         except:
 163             traceback.print_exc(file=sys.stdout)
 164             self.feedback_q.put(taskname)
 165         finally:
 166             self.feedback_q.put("Done")
 167     def printStatus(self):
 168         print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
 169         print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
 170         groupobj = {}
 171         for buildobj in BuildDB.BuildObject.GetCache().values():
 172             if str(buildobj).lower().endswith("dec"):
 173                 try:
 174                     groupobj['dec'].append(str(buildobj))
 175                 except:
 176                     groupobj['dec'] = [str(buildobj)]
 177             if str(buildobj).lower().endswith("dsc"):
 178                 try:
 179                     groupobj['dsc'].append(str(buildobj))
 180                 except:
 181                     groupobj['dsc'] = [str(buildobj)]
 182
 183             if str(buildobj).lower().endswith("inf"):
 184                 try:
 185                     groupobj['inf'].append(str(buildobj))
 186                 except:
 187                     groupobj['inf'] = [str(buildobj)]
 188
 189         print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
 190         print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
 191         print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))