]>
Commit | Line | Data |
---|---|---|
673d09a2 FB |
1 | ## @file\r |
2 | # Create makefile for MS nmake and GNU make\r | |
3 | #\r | |
4 | # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>\r | |
5 | # SPDX-License-Identifier: BSD-2-Clause-Patent\r | |
6 | #\r | |
7 | from __future__ import absolute_import\r | |
8 | import multiprocessing as mp\r | |
9 | import threading\r | |
10 | from Common.Misc import PathClass\r | |
11 | from AutoGen.ModuleAutoGen import ModuleAutoGen\r | |
12 | from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo\r | |
13 | import Common.GlobalData as GlobalData\r | |
14 | import Common.EdkLogger as EdkLogger\r | |
15 | import os\r | |
16 | from Common.MultipleWorkspace import MultipleWorkspace as mws\r | |
17 | from AutoGen.AutoGen import AutoGen\r | |
18 | from Workspace.WorkspaceDatabase import BuildDB\r | |
4acae2b3 FB |
19 | try:\r |
20 | from queue import Empty\r | |
21 | except:\r | |
22 | from Queue import Empty\r | |
673d09a2 FB |
23 | import traceback\r |
24 | import sys\r | |
25 | from AutoGen.DataPipe import MemoryDataPipe\r | |
26 | def clearQ(q):\r | |
27 | try:\r | |
28 | while True:\r | |
29 | q.get_nowait()\r | |
30 | except Empty:\r | |
31 | pass\r | |
636ed13a FB |
32 | import logging\r |
33 | \r | |
34 | class LogAgent(threading.Thread):\r | |
35 | def __init__(self,log_q,log_level,log_file=None):\r | |
36 | super(LogAgent,self).__init__()\r | |
37 | self.log_q = log_q\r | |
38 | self.log_level = log_level\r | |
39 | self.log_file = log_file\r | |
40 | def InitLogger(self):\r | |
41 | # For DEBUG level (All DEBUG_0~9 are applicable)\r | |
42 | self._DebugLogger_agent = logging.getLogger("tool_debug_agent")\r | |
43 | _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")\r | |
44 | self._DebugLogger_agent.setLevel(self.log_level)\r | |
45 | _DebugChannel = logging.StreamHandler(sys.stdout)\r | |
46 | _DebugChannel.setFormatter(_DebugFormatter)\r | |
47 | self._DebugLogger_agent.addHandler(_DebugChannel)\r | |
48 | \r | |
49 | # For VERBOSE, INFO, WARN level\r | |
50 | self._InfoLogger_agent = logging.getLogger("tool_info_agent")\r | |
51 | _InfoFormatter = logging.Formatter("%(message)s")\r | |
52 | self._InfoLogger_agent.setLevel(self.log_level)\r | |
53 | _InfoChannel = logging.StreamHandler(sys.stdout)\r | |
54 | _InfoChannel.setFormatter(_InfoFormatter)\r | |
55 | self._InfoLogger_agent.addHandler(_InfoChannel)\r | |
56 | \r | |
57 | # For ERROR level\r | |
58 | self._ErrorLogger_agent = logging.getLogger("tool_error_agent")\r | |
59 | _ErrorFormatter = logging.Formatter("%(message)s")\r | |
60 | self._ErrorLogger_agent.setLevel(self.log_level)\r | |
61 | _ErrorCh = logging.StreamHandler(sys.stderr)\r | |
62 | _ErrorCh.setFormatter(_ErrorFormatter)\r | |
63 | self._ErrorLogger_agent.addHandler(_ErrorCh)\r | |
64 | \r | |
65 | if self.log_file:\r | |
66 | if os.path.exists(self.log_file):\r | |
67 | os.remove(self.log_file)\r | |
68 | _Ch = logging.FileHandler(self.log_file)\r | |
69 | _Ch.setFormatter(_DebugFormatter)\r | |
70 | self._DebugLogger_agent.addHandler(_Ch)\r | |
71 | \r | |
72 | _Ch= logging.FileHandler(self.log_file)\r | |
73 | _Ch.setFormatter(_InfoFormatter)\r | |
74 | self._InfoLogger_agent.addHandler(_Ch)\r | |
75 | \r | |
76 | _Ch = logging.FileHandler(self.log_file)\r | |
77 | _Ch.setFormatter(_ErrorFormatter)\r | |
78 | self._ErrorLogger_agent.addHandler(_Ch)\r | |
79 | \r | |
80 | def run(self):\r | |
81 | self.InitLogger()\r | |
82 | while True:\r | |
83 | log_message = self.log_q.get()\r | |
84 | if log_message is None:\r | |
85 | break\r | |
86 | if log_message.name == "tool_error":\r | |
87 | self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
88 | elif log_message.name == "tool_info":\r | |
89 | self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
90 | elif log_message.name == "tool_debug":\r | |
91 | self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
92 | else:\r | |
93 | self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
94 | \r | |
95 | def kill(self):\r | |
96 | self.log_q.put(None)\r | |
673d09a2 FB |
97 | class AutoGenManager(threading.Thread):\r |
98 | def __init__(self,autogen_workers, feedback_q,error_event):\r | |
99 | super(AutoGenManager,self).__init__()\r | |
100 | self.autogen_workers = autogen_workers\r | |
101 | self.feedback_q = feedback_q\r | |
673d09a2 FB |
102 | self.Status = True\r |
103 | self.error_event = error_event\r | |
104 | def run(self):\r | |
105 | try:\r | |
106 | fin_num = 0\r | |
107 | while True:\r | |
108 | badnews = self.feedback_q.get()\r | |
109 | if badnews is None:\r | |
110 | break\r | |
111 | if badnews == "Done":\r | |
112 | fin_num += 1\r | |
113 | else:\r | |
114 | self.Status = False\r | |
115 | self.TerminateWorkers()\r | |
116 | if fin_num == len(self.autogen_workers):\r | |
117 | self.clearQueue()\r | |
118 | for w in self.autogen_workers:\r | |
119 | w.join()\r | |
120 | break\r | |
121 | except Exception:\r | |
122 | return\r | |
123 | \r | |
124 | def clearQueue(self):\r | |
125 | taskq = self.autogen_workers[0].module_queue\r | |
126 | clearQ(taskq)\r | |
127 | clearQ(self.feedback_q)\r | |
128 | \r | |
129 | def TerminateWorkers(self):\r | |
130 | self.error_event.set()\r | |
131 | def kill(self):\r | |
132 | self.feedback_q.put(None)\r | |
133 | class AutoGenWorkerInProcess(mp.Process):\r | |
636ed13a | 134 | def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,log_q,error_event):\r |
673d09a2 FB |
135 | mp.Process.__init__(self)\r |
136 | self.module_queue = module_queue\r | |
137 | self.data_pipe_file_path =data_pipe_file_path\r | |
138 | self.data_pipe = None\r | |
139 | self.feedback_q = feedback_q\r | |
140 | self.PlatformMetaFileSet = {}\r | |
141 | self.file_lock = file_lock\r | |
3285fbda | 142 | self.share_data = share_data\r |
636ed13a | 143 | self.log_q = log_q\r |
673d09a2 FB |
144 | self.error_event = error_event\r |
145 | def GetPlatformMetaFile(self,filepath,root):\r | |
146 | try:\r | |
147 | return self.PlatformMetaFileSet[(filepath,root)]\r | |
148 | except:\r | |
149 | self.PlatformMetaFileSet[(filepath,root)] = filepath\r | |
150 | return self.PlatformMetaFileSet[(filepath,root)]\r | |
151 | def run(self):\r | |
152 | try:\r | |
153 | taskname = "Init"\r | |
154 | with self.file_lock:\r | |
155 | if not os.path.exists(self.data_pipe_file_path):\r | |
156 | self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)\r | |
157 | self.data_pipe = MemoryDataPipe()\r | |
158 | self.data_pipe.load(self.data_pipe_file_path)\r | |
636ed13a | 159 | EdkLogger.LogClientInitialize(self.log_q)\r |
673d09a2 FB |
160 | loglevel = self.data_pipe.Get("LogLevel")\r |
161 | if not loglevel:\r | |
162 | loglevel = EdkLogger.INFO\r | |
163 | EdkLogger.SetLevel(loglevel)\r | |
673d09a2 FB |
164 | target = self.data_pipe.Get("P_Info").get("Target")\r |
165 | toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r | |
166 | archlist = self.data_pipe.Get("P_Info").get("ArchList")\r | |
167 | \r | |
168 | active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")\r | |
169 | workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")\r | |
170 | PackagesPath = os.getenv("PACKAGES_PATH")\r | |
171 | mws.setWs(workspacedir, PackagesPath)\r | |
172 | self.Wa = WorkSpaceInfo(\r | |
173 | workspacedir,active_p,target,toolchain,archlist\r | |
174 | )\r | |
175 | self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")\r | |
176 | GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")\r | |
177 | GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")\r | |
178 | os.environ._data = self.data_pipe.Get("Env_Var")\r | |
179 | GlobalData.gWorkspace = workspacedir\r | |
180 | GlobalData.gDisableIncludePathCheck = False\r | |
181 | GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")\r | |
182 | GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")\r | |
183 | pcd_from_build_option = []\r | |
184 | for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):\r | |
185 | pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))\r | |
186 | if pcd_tuple[2].strip():\r | |
187 | pcd_id = ".".join((pcd_id,pcd_tuple[2]))\r | |
188 | pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))\r | |
189 | GlobalData.BuildOptionPcd = pcd_from_build_option\r | |
190 | module_count = 0\r | |
191 | FfsCmd = self.data_pipe.Get("FfsCommand")\r | |
192 | if FfsCmd is None:\r | |
193 | FfsCmd = {}\r | |
194 | PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),\r | |
195 | self.data_pipe.Get("P_Info").get("WorkspaceDir"))\r | |
196 | libConstPcd = self.data_pipe.Get("LibConstPcd")\r | |
197 | Refes = self.data_pipe.Get("REFS")\r | |
198 | while True:\r | |
199 | if self.module_queue.empty():\r | |
200 | break\r | |
201 | if self.error_event.is_set():\r | |
202 | break\r | |
203 | module_count += 1\r | |
204 | module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()\r | |
205 | modulefullpath = os.path.join(module_root,module_file)\r | |
206 | taskname = " : ".join((modulefullpath,module_arch))\r | |
207 | module_metafile = PathClass(module_file,module_root)\r | |
208 | if module_path:\r | |
209 | module_metafile.Path = module_path\r | |
210 | if module_basename:\r | |
211 | module_metafile.BaseName = module_basename\r | |
212 | if module_originalpath:\r | |
213 | module_metafile.OriginalPath = PathClass(module_originalpath,module_root)\r | |
214 | arch = module_arch\r | |
215 | target = self.data_pipe.Get("P_Info").get("Target")\r | |
216 | toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r | |
217 | Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)\r | |
218 | Ma.IsLibrary = IsLib\r | |
219 | if IsLib:\r | |
220 | if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:\r | |
221 | Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r | |
222 | if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:\r | |
223 | Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r | |
224 | Ma.CreateCodeFile(False)\r | |
225 | Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))\r | |
226 | except Empty:\r | |
227 | pass\r | |
228 | except:\r | |
229 | traceback.print_exc(file=sys.stdout)\r | |
230 | self.feedback_q.put(taskname)\r | |
231 | finally:\r | |
232 | self.feedback_q.put("Done")\r | |
233 | def printStatus(self):\r | |
234 | print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))\r | |
235 | print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))\r | |
236 | groupobj = {}\r | |
237 | for buildobj in BuildDB.BuildObject.GetCache().values():\r | |
238 | if str(buildobj).lower().endswith("dec"):\r | |
239 | try:\r | |
240 | groupobj['dec'].append(str(buildobj))\r | |
241 | except:\r | |
242 | groupobj['dec'] = [str(buildobj)]\r | |
243 | if str(buildobj).lower().endswith("dsc"):\r | |
244 | try:\r | |
245 | groupobj['dsc'].append(str(buildobj))\r | |
246 | except:\r | |
247 | groupobj['dsc'] = [str(buildobj)]\r | |
248 | \r | |
249 | if str(buildobj).lower().endswith("inf"):\r | |
250 | try:\r | |
251 | groupobj['inf'].append(str(buildobj))\r | |
252 | except:\r | |
253 | groupobj['inf'] = [str(buildobj)]\r | |
254 | \r | |
255 | print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))\r | |
256 | print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))\r | |
257 | print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))\r |