]>
git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/AutoGen/AutoGenWorker.py
2 # Create makefile for MS nmake and GNU make
4 # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
5 # SPDX-License-Identifier: BSD-2-Clause-Patent
7 from __future__
import absolute_import
8 import multiprocessing
as mp
10 from Common
.Misc
import PathClass
11 from AutoGen
.ModuleAutoGen
import ModuleAutoGen
12 from AutoGen
.ModuleAutoGenHelper
import WorkSpaceInfo
,AutoGenInfo
13 import Common
.GlobalData
as GlobalData
14 import Common
.EdkLogger
as EdkLogger
16 from Common
.MultipleWorkspace
import MultipleWorkspace
as mws
17 from AutoGen
.AutoGen
import AutoGen
18 from Workspace
.WorkspaceDatabase
import BuildDB
19 from queue
import Empty
22 from AutoGen
.DataPipe
import MemoryDataPipe
31 class LogAgent(threading
.Thread
):
32 def __init__(self
,log_q
,log_level
,log_file
=None):
33 super(LogAgent
,self
).__init
__()
35 self
.log_level
= log_level
36 self
.log_file
= log_file
38 # For DEBUG level (All DEBUG_0~9 are applicable)
39 self
._DebugLogger
_agent
= logging
.getLogger("tool_debug_agent")
40 _DebugFormatter
= logging
.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt
="%H:%M:%S")
41 self
._DebugLogger
_agent
.setLevel(self
.log_level
)
42 _DebugChannel
= logging
.StreamHandler(sys
.stdout
)
43 _DebugChannel
.setFormatter(_DebugFormatter
)
44 self
._DebugLogger
_agent
.addHandler(_DebugChannel
)
46 # For VERBOSE, INFO, WARN level
47 self
._InfoLogger
_agent
= logging
.getLogger("tool_info_agent")
48 _InfoFormatter
= logging
.Formatter("%(message)s")
49 self
._InfoLogger
_agent
.setLevel(self
.log_level
)
50 _InfoChannel
= logging
.StreamHandler(sys
.stdout
)
51 _InfoChannel
.setFormatter(_InfoFormatter
)
52 self
._InfoLogger
_agent
.addHandler(_InfoChannel
)
55 self
._ErrorLogger
_agent
= logging
.getLogger("tool_error_agent")
56 _ErrorFormatter
= logging
.Formatter("%(message)s")
57 self
._ErrorLogger
_agent
.setLevel(self
.log_level
)
58 _ErrorCh
= logging
.StreamHandler(sys
.stderr
)
59 _ErrorCh
.setFormatter(_ErrorFormatter
)
60 self
._ErrorLogger
_agent
.addHandler(_ErrorCh
)
63 if os
.path
.exists(self
.log_file
):
64 os
.remove(self
.log_file
)
65 _Ch
= logging
.FileHandler(self
.log_file
)
66 _Ch
.setFormatter(_DebugFormatter
)
67 self
._DebugLogger
_agent
.addHandler(_Ch
)
69 _Ch
= logging
.FileHandler(self
.log_file
)
70 _Ch
.setFormatter(_InfoFormatter
)
71 self
._InfoLogger
_agent
.addHandler(_Ch
)
73 _Ch
= logging
.FileHandler(self
.log_file
)
74 _Ch
.setFormatter(_ErrorFormatter
)
75 self
._ErrorLogger
_agent
.addHandler(_Ch
)
80 log_message
= self
.log_q
.get()
81 if log_message
is None:
83 if log_message
.name
== "tool_error":
84 self
._ErrorLogger
_agent
.log(log_message
.levelno
,log_message
.getMessage())
85 elif log_message
.name
== "tool_info":
86 self
._InfoLogger
_agent
.log(log_message
.levelno
,log_message
.getMessage())
87 elif log_message
.name
== "tool_debug":
88 self
._DebugLogger
_agent
.log(log_message
.levelno
,log_message
.getMessage())
90 self
._InfoLogger
_agent
.log(log_message
.levelno
,log_message
.getMessage())
94 class AutoGenManager(threading
.Thread
):
95 def __init__(self
,autogen_workers
, feedback_q
,error_event
):
96 super(AutoGenManager
,self
).__init
__()
97 self
.autogen_workers
= autogen_workers
98 self
.feedback_q
= feedback_q
100 self
.error_event
= error_event
105 badnews
= self
.feedback_q
.get()
108 if badnews
== "Done":
112 self
.TerminateWorkers()
113 if fin_num
== len(self
.autogen_workers
):
115 for w
in self
.autogen_workers
:
121 def clearQueue(self
):
122 taskq
= self
.autogen_workers
[0].module_queue
124 clearQ(self
.feedback_q
)
126 def TerminateWorkers(self
):
127 self
.error_event
.set()
129 self
.feedback_q
.put(None)
130 class AutoGenWorkerInProcess(mp
.Process
):
131 def __init__(self
,module_queue
,data_pipe_file_path
,feedback_q
,file_lock
, share_data
,log_q
,error_event
):
132 mp
.Process
.__init
__(self
)
133 self
.module_queue
= module_queue
134 self
.data_pipe_file_path
=data_pipe_file_path
135 self
.data_pipe
= None
136 self
.feedback_q
= feedback_q
137 self
.PlatformMetaFileSet
= {}
138 self
.file_lock
= file_lock
139 self
.share_data
= share_data
141 self
.error_event
= error_event
142 def GetPlatformMetaFile(self
,filepath
,root
):
144 return self
.PlatformMetaFileSet
[(filepath
,root
)]
146 self
.PlatformMetaFileSet
[(filepath
,root
)] = filepath
147 return self
.PlatformMetaFileSet
[(filepath
,root
)]
152 if not os
.path
.exists(self
.data_pipe_file_path
):
153 self
.feedback_q
.put(taskname
+ ":" + "load data pipe %s failed." % self
.data_pipe_file_path
)
154 self
.data_pipe
= MemoryDataPipe()
155 self
.data_pipe
.load(self
.data_pipe_file_path
)
156 EdkLogger
.LogClientInitialize(self
.log_q
)
157 loglevel
= self
.data_pipe
.Get("LogLevel")
159 loglevel
= EdkLogger
.INFO
160 EdkLogger
.SetLevel(loglevel
)
161 target
= self
.data_pipe
.Get("P_Info").get("Target")
162 toolchain
= self
.data_pipe
.Get("P_Info").get("ToolChain")
163 archlist
= self
.data_pipe
.Get("P_Info").get("ArchList")
165 active_p
= self
.data_pipe
.Get("P_Info").get("ActivePlatform")
166 workspacedir
= self
.data_pipe
.Get("P_Info").get("WorkspaceDir")
167 PackagesPath
= os
.getenv("PACKAGES_PATH")
168 mws
.setWs(workspacedir
, PackagesPath
)
169 self
.Wa
= WorkSpaceInfo(
170 workspacedir
,active_p
,target
,toolchain
,archlist
172 self
.Wa
._SrcTimeStamp
= self
.data_pipe
.Get("Workspace_timestamp")
173 GlobalData
.gGlobalDefines
= self
.data_pipe
.Get("G_defines")
174 GlobalData
.gCommandLineDefines
= self
.data_pipe
.Get("CL_defines")
175 os
.environ
._data
= self
.data_pipe
.Get("Env_Var")
176 GlobalData
.gWorkspace
= workspacedir
177 GlobalData
.gDisableIncludePathCheck
= False
178 GlobalData
.gFdfParser
= self
.data_pipe
.Get("FdfParser")
179 GlobalData
.gDatabasePath
= self
.data_pipe
.Get("DatabasePath")
180 pcd_from_build_option
= []
181 for pcd_tuple
in self
.data_pipe
.Get("BuildOptPcd"):
182 pcd_id
= ".".join((pcd_tuple
[0],pcd_tuple
[1]))
183 if pcd_tuple
[2].strip():
184 pcd_id
= ".".join((pcd_id
,pcd_tuple
[2]))
185 pcd_from_build_option
.append("=".join((pcd_id
,pcd_tuple
[3])))
186 GlobalData
.BuildOptionPcd
= pcd_from_build_option
188 FfsCmd
= self
.data_pipe
.Get("FfsCommand")
191 PlatformMetaFile
= self
.GetPlatformMetaFile(self
.data_pipe
.Get("P_Info").get("ActivePlatform"),
192 self
.data_pipe
.Get("P_Info").get("WorkspaceDir"))
193 libConstPcd
= self
.data_pipe
.Get("LibConstPcd")
194 Refes
= self
.data_pipe
.Get("REFS")
196 if self
.module_queue
.empty():
198 if self
.error_event
.is_set():
201 module_file
,module_root
,module_path
,module_basename
,module_originalpath
,module_arch
,IsLib
= self
.module_queue
.get_nowait()
202 modulefullpath
= os
.path
.join(module_root
,module_file
)
203 taskname
= " : ".join((modulefullpath
,module_arch
))
204 module_metafile
= PathClass(module_file
,module_root
)
206 module_metafile
.Path
= module_path
208 module_metafile
.BaseName
= module_basename
209 if module_originalpath
:
210 module_metafile
.OriginalPath
= PathClass(module_originalpath
,module_root
)
212 target
= self
.data_pipe
.Get("P_Info").get("Target")
213 toolchain
= self
.data_pipe
.Get("P_Info").get("ToolChain")
214 Ma
= ModuleAutoGen(self
.Wa
,module_metafile
,target
,toolchain
,arch
,PlatformMetaFile
,self
.data_pipe
)
217 if (Ma
.MetaFile
.File
,Ma
.MetaFile
.Root
,Ma
.Arch
,Ma
.MetaFile
.Path
) in libConstPcd
:
218 Ma
.ConstPcd
= libConstPcd
[(Ma
.MetaFile
.File
,Ma
.MetaFile
.Root
,Ma
.Arch
,Ma
.MetaFile
.Path
)]
219 if (Ma
.MetaFile
.File
,Ma
.MetaFile
.Root
,Ma
.Arch
,Ma
.MetaFile
.Path
) in Refes
:
220 Ma
.ReferenceModules
= Refes
[(Ma
.MetaFile
.File
,Ma
.MetaFile
.Root
,Ma
.Arch
,Ma
.MetaFile
.Path
)]
221 Ma
.CreateCodeFile(False)
222 Ma
.CreateMakeFile(False,GenFfsList
=FfsCmd
.get((Ma
.MetaFile
.File
, Ma
.Arch
),[]))
226 traceback
.print_exc(file=sys
.stdout
)
227 self
.feedback_q
.put(taskname
)
229 self
.feedback_q
.put("Done")
230 def printStatus(self
):
231 print("Processs ID: %d Run %d modules in AutoGen " % (os
.getpid(),len(AutoGen
.Cache())))
232 print("Processs ID: %d Run %d modules in AutoGenInfo " % (os
.getpid(),len(AutoGenInfo
.GetCache())))
234 for buildobj
in BuildDB
.BuildObject
.GetCache().values():
235 if str(buildobj
).lower().endswith("dec"):
237 groupobj
['dec'].append(str(buildobj
))
239 groupobj
['dec'] = [str(buildobj
)]
240 if str(buildobj
).lower().endswith("dsc"):
242 groupobj
['dsc'].append(str(buildobj
))
244 groupobj
['dsc'] = [str(buildobj
)]
246 if str(buildobj
).lower().endswith("inf"):
248 groupobj
['inf'].append(str(buildobj
))
250 groupobj
['inf'] = [str(buildobj
)]
252 print("Processs ID: %d Run %d pkg in WDB " % (os
.getpid(),len(groupobj
.get("dec",[]))))
253 print("Processs ID: %d Run %d pla in WDB " % (os
.getpid(),len(groupobj
.get("dsc",[]))))
254 print("Processs ID: %d Run %d inf in WDB " % (os
.getpid(),len(groupobj
.get("inf",[]))))