]>
Commit | Line | Data |
---|---|---|
673d09a2 FB |
1 | ## @file\r |
2 | # Create makefile for MS nmake and GNU make\r | |
3 | #\r | |
4 | # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>\r | |
5 | # SPDX-License-Identifier: BSD-2-Clause-Patent\r | |
6 | #\r | |
7 | from __future__ import absolute_import\r | |
8 | import multiprocessing as mp\r | |
9 | import threading\r | |
10 | from Common.Misc import PathClass\r | |
11 | from AutoGen.ModuleAutoGen import ModuleAutoGen\r | |
12 | from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo\r | |
13 | import Common.GlobalData as GlobalData\r | |
14 | import Common.EdkLogger as EdkLogger\r | |
15 | import os\r | |
16 | from Common.MultipleWorkspace import MultipleWorkspace as mws\r | |
17 | from AutoGen.AutoGen import AutoGen\r | |
18 | from Workspace.WorkspaceDatabase import BuildDB\r | |
19 | from queue import Empty\r | |
20 | import traceback\r | |
21 | import sys\r | |
22 | from AutoGen.DataPipe import MemoryDataPipe\r | |
23 | def clearQ(q):\r | |
24 | try:\r | |
25 | while True:\r | |
26 | q.get_nowait()\r | |
27 | except Empty:\r | |
28 | pass\r | |
636ed13a FB |
29 | import logging\r |
30 | \r | |
31 | class LogAgent(threading.Thread):\r | |
32 | def __init__(self,log_q,log_level,log_file=None):\r | |
33 | super(LogAgent,self).__init__()\r | |
34 | self.log_q = log_q\r | |
35 | self.log_level = log_level\r | |
36 | self.log_file = log_file\r | |
37 | def InitLogger(self):\r | |
38 | # For DEBUG level (All DEBUG_0~9 are applicable)\r | |
39 | self._DebugLogger_agent = logging.getLogger("tool_debug_agent")\r | |
40 | _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")\r | |
41 | self._DebugLogger_agent.setLevel(self.log_level)\r | |
42 | _DebugChannel = logging.StreamHandler(sys.stdout)\r | |
43 | _DebugChannel.setFormatter(_DebugFormatter)\r | |
44 | self._DebugLogger_agent.addHandler(_DebugChannel)\r | |
45 | \r | |
46 | # For VERBOSE, INFO, WARN level\r | |
47 | self._InfoLogger_agent = logging.getLogger("tool_info_agent")\r | |
48 | _InfoFormatter = logging.Formatter("%(message)s")\r | |
49 | self._InfoLogger_agent.setLevel(self.log_level)\r | |
50 | _InfoChannel = logging.StreamHandler(sys.stdout)\r | |
51 | _InfoChannel.setFormatter(_InfoFormatter)\r | |
52 | self._InfoLogger_agent.addHandler(_InfoChannel)\r | |
53 | \r | |
54 | # For ERROR level\r | |
55 | self._ErrorLogger_agent = logging.getLogger("tool_error_agent")\r | |
56 | _ErrorFormatter = logging.Formatter("%(message)s")\r | |
57 | self._ErrorLogger_agent.setLevel(self.log_level)\r | |
58 | _ErrorCh = logging.StreamHandler(sys.stderr)\r | |
59 | _ErrorCh.setFormatter(_ErrorFormatter)\r | |
60 | self._ErrorLogger_agent.addHandler(_ErrorCh)\r | |
61 | \r | |
62 | if self.log_file:\r | |
63 | if os.path.exists(self.log_file):\r | |
64 | os.remove(self.log_file)\r | |
65 | _Ch = logging.FileHandler(self.log_file)\r | |
66 | _Ch.setFormatter(_DebugFormatter)\r | |
67 | self._DebugLogger_agent.addHandler(_Ch)\r | |
68 | \r | |
69 | _Ch= logging.FileHandler(self.log_file)\r | |
70 | _Ch.setFormatter(_InfoFormatter)\r | |
71 | self._InfoLogger_agent.addHandler(_Ch)\r | |
72 | \r | |
73 | _Ch = logging.FileHandler(self.log_file)\r | |
74 | _Ch.setFormatter(_ErrorFormatter)\r | |
75 | self._ErrorLogger_agent.addHandler(_Ch)\r | |
76 | \r | |
77 | def run(self):\r | |
78 | self.InitLogger()\r | |
79 | while True:\r | |
80 | log_message = self.log_q.get()\r | |
81 | if log_message is None:\r | |
82 | break\r | |
83 | if log_message.name == "tool_error":\r | |
84 | self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
85 | elif log_message.name == "tool_info":\r | |
86 | self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
87 | elif log_message.name == "tool_debug":\r | |
88 | self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
89 | else:\r | |
90 | self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())\r | |
91 | \r | |
92 | def kill(self):\r | |
93 | self.log_q.put(None)\r | |
673d09a2 FB |
94 | class AutoGenManager(threading.Thread):\r |
95 | def __init__(self,autogen_workers, feedback_q,error_event):\r | |
96 | super(AutoGenManager,self).__init__()\r | |
97 | self.autogen_workers = autogen_workers\r | |
98 | self.feedback_q = feedback_q\r | |
673d09a2 FB |
99 | self.Status = True\r |
100 | self.error_event = error_event\r | |
101 | def run(self):\r | |
102 | try:\r | |
103 | fin_num = 0\r | |
104 | while True:\r | |
105 | badnews = self.feedback_q.get()\r | |
106 | if badnews is None:\r | |
107 | break\r | |
108 | if badnews == "Done":\r | |
109 | fin_num += 1\r | |
110 | else:\r | |
111 | self.Status = False\r | |
112 | self.TerminateWorkers()\r | |
113 | if fin_num == len(self.autogen_workers):\r | |
114 | self.clearQueue()\r | |
115 | for w in self.autogen_workers:\r | |
116 | w.join()\r | |
117 | break\r | |
118 | except Exception:\r | |
119 | return\r | |
120 | \r | |
121 | def clearQueue(self):\r | |
122 | taskq = self.autogen_workers[0].module_queue\r | |
123 | clearQ(taskq)\r | |
124 | clearQ(self.feedback_q)\r | |
125 | \r | |
126 | def TerminateWorkers(self):\r | |
127 | self.error_event.set()\r | |
128 | def kill(self):\r | |
129 | self.feedback_q.put(None)\r | |
130 | class AutoGenWorkerInProcess(mp.Process):\r | |
636ed13a | 131 | def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,log_q,error_event):\r |
673d09a2 FB |
132 | mp.Process.__init__(self)\r |
133 | self.module_queue = module_queue\r | |
134 | self.data_pipe_file_path =data_pipe_file_path\r | |
135 | self.data_pipe = None\r | |
136 | self.feedback_q = feedback_q\r | |
137 | self.PlatformMetaFileSet = {}\r | |
138 | self.file_lock = file_lock\r | |
3285fbda | 139 | self.share_data = share_data\r |
636ed13a | 140 | self.log_q = log_q\r |
673d09a2 FB |
141 | self.error_event = error_event\r |
142 | def GetPlatformMetaFile(self,filepath,root):\r | |
143 | try:\r | |
144 | return self.PlatformMetaFileSet[(filepath,root)]\r | |
145 | except:\r | |
146 | self.PlatformMetaFileSet[(filepath,root)] = filepath\r | |
147 | return self.PlatformMetaFileSet[(filepath,root)]\r | |
148 | def run(self):\r | |
149 | try:\r | |
150 | taskname = "Init"\r | |
151 | with self.file_lock:\r | |
152 | if not os.path.exists(self.data_pipe_file_path):\r | |
153 | self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)\r | |
154 | self.data_pipe = MemoryDataPipe()\r | |
155 | self.data_pipe.load(self.data_pipe_file_path)\r | |
636ed13a | 156 | EdkLogger.LogClientInitialize(self.log_q)\r |
673d09a2 FB |
157 | loglevel = self.data_pipe.Get("LogLevel")\r |
158 | if not loglevel:\r | |
159 | loglevel = EdkLogger.INFO\r | |
160 | EdkLogger.SetLevel(loglevel)\r | |
673d09a2 FB |
161 | target = self.data_pipe.Get("P_Info").get("Target")\r |
162 | toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r | |
163 | archlist = self.data_pipe.Get("P_Info").get("ArchList")\r | |
164 | \r | |
165 | active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")\r | |
166 | workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")\r | |
167 | PackagesPath = os.getenv("PACKAGES_PATH")\r | |
168 | mws.setWs(workspacedir, PackagesPath)\r | |
169 | self.Wa = WorkSpaceInfo(\r | |
170 | workspacedir,active_p,target,toolchain,archlist\r | |
171 | )\r | |
172 | self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")\r | |
173 | GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")\r | |
174 | GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")\r | |
175 | os.environ._data = self.data_pipe.Get("Env_Var")\r | |
176 | GlobalData.gWorkspace = workspacedir\r | |
177 | GlobalData.gDisableIncludePathCheck = False\r | |
178 | GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")\r | |
179 | GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")\r | |
180 | pcd_from_build_option = []\r | |
181 | for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):\r | |
182 | pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))\r | |
183 | if pcd_tuple[2].strip():\r | |
184 | pcd_id = ".".join((pcd_id,pcd_tuple[2]))\r | |
185 | pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))\r | |
186 | GlobalData.BuildOptionPcd = pcd_from_build_option\r | |
187 | module_count = 0\r | |
188 | FfsCmd = self.data_pipe.Get("FfsCommand")\r | |
189 | if FfsCmd is None:\r | |
190 | FfsCmd = {}\r | |
191 | PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),\r | |
192 | self.data_pipe.Get("P_Info").get("WorkspaceDir"))\r | |
193 | libConstPcd = self.data_pipe.Get("LibConstPcd")\r | |
194 | Refes = self.data_pipe.Get("REFS")\r | |
195 | while True:\r | |
196 | if self.module_queue.empty():\r | |
197 | break\r | |
198 | if self.error_event.is_set():\r | |
199 | break\r | |
200 | module_count += 1\r | |
201 | module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()\r | |
202 | modulefullpath = os.path.join(module_root,module_file)\r | |
203 | taskname = " : ".join((modulefullpath,module_arch))\r | |
204 | module_metafile = PathClass(module_file,module_root)\r | |
205 | if module_path:\r | |
206 | module_metafile.Path = module_path\r | |
207 | if module_basename:\r | |
208 | module_metafile.BaseName = module_basename\r | |
209 | if module_originalpath:\r | |
210 | module_metafile.OriginalPath = PathClass(module_originalpath,module_root)\r | |
211 | arch = module_arch\r | |
212 | target = self.data_pipe.Get("P_Info").get("Target")\r | |
213 | toolchain = self.data_pipe.Get("P_Info").get("ToolChain")\r | |
214 | Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)\r | |
215 | Ma.IsLibrary = IsLib\r | |
216 | if IsLib:\r | |
217 | if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:\r | |
218 | Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r | |
219 | if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:\r | |
220 | Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]\r | |
221 | Ma.CreateCodeFile(False)\r | |
222 | Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))\r | |
223 | except Empty:\r | |
224 | pass\r | |
225 | except:\r | |
226 | traceback.print_exc(file=sys.stdout)\r | |
227 | self.feedback_q.put(taskname)\r | |
228 | finally:\r | |
229 | self.feedback_q.put("Done")\r | |
230 | def printStatus(self):\r | |
231 | print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))\r | |
232 | print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))\r | |
233 | groupobj = {}\r | |
234 | for buildobj in BuildDB.BuildObject.GetCache().values():\r | |
235 | if str(buildobj).lower().endswith("dec"):\r | |
236 | try:\r | |
237 | groupobj['dec'].append(str(buildobj))\r | |
238 | except:\r | |
239 | groupobj['dec'] = [str(buildobj)]\r | |
240 | if str(buildobj).lower().endswith("dsc"):\r | |
241 | try:\r | |
242 | groupobj['dsc'].append(str(buildobj))\r | |
243 | except:\r | |
244 | groupobj['dsc'] = [str(buildobj)]\r | |
245 | \r | |
246 | if str(buildobj).lower().endswith("inf"):\r | |
247 | try:\r | |
248 | groupobj['inf'].append(str(buildobj))\r | |
249 | except:\r | |
250 | groupobj['inf'] = [str(buildobj)]\r | |
251 | \r | |
252 | print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))\r | |
253 | print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))\r | |
254 | print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))\r |