# Import Modules\r
#\r
from __future__ import print_function\r
-import Common.LongFilePathOs as os\r
-import re\r
+from __future__ import absolute_import\r
+import os.path as path\r
import sys\r
+import os\r
+import re\r
import glob\r
import time\r
import platform\r
import traceback\r
-import encodings.ascii\r
import multiprocessing\r
-\r
-from struct import *\r
-from threading import *\r
+from threading import Thread,Event,BoundedSemaphore\r
import threading\r
+from subprocess import Popen,PIPE\r
+from collections import OrderedDict, defaultdict\r
from optparse import OptionParser\r
-from subprocess import *\r
+from AutoGen.PlatformAutoGen import PlatformAutoGen\r
+from AutoGen.ModuleAutoGen import ModuleAutoGen\r
+from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen\r
+from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager\r
+from AutoGen import GenMake\r
from Common import Misc as Utils\r
\r
-from Common.LongFilePathSupport import OpenLongFilePath as open\r
-from Common.TargetTxtClassObject import TargetTxtClassObject\r
-from Common.ToolDefClassObject import ToolDefClassObject\r
+from Common.TargetTxtClassObject import TargetTxt\r
+from Common.ToolDefClassObject import ToolDef\r
+from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory\r
+from Common.StringUtils import NormPath\r
+from Common.MultipleWorkspace import MultipleWorkspace as mws\r
+from Common.BuildToolError import *\r
from Common.DataType import *\r
+import Common.EdkLogger as EdkLogger\r
from Common.BuildVersion import gBUILD_VERSION\r
-from AutoGen.AutoGen import *\r
-from Common.BuildToolError import *\r
-from Workspace.WorkspaceDatabase import WorkspaceDatabase\r
-from Common.MultipleWorkspace import MultipleWorkspace as mws\r
+from Workspace.WorkspaceDatabase import BuildDB\r
\r
from BuildReport import BuildReport\r
-from GenPatchPcdTable.GenPatchPcdTable import *\r
-from PatchPcdValue.PatchPcdValue import *\r
+from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile\r
+from PatchPcdValue.PatchPcdValue import PatchBinaryFile\r
\r
-import Common.EdkLogger\r
import Common.GlobalData as GlobalData\r
from GenFds.GenFds import GenFds, GenFdsApi\r
-\r
-from collections import OrderedDict, defaultdict\r
+import multiprocessing as mp\r
+from multiprocessing import Manager\r
\r
# Version and Copyright\r
VersionNumber = "0.60" + ' ' + gBUILD_VERSION\r
# @param Obj The ModuleAutoGen object the build is working on\r
# @param Target The build target name, one of gSupportedTarget\r
#\r
- def __init__(self, Obj, Target):\r
- Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]\r
- BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)\r
+ def __init__(self, Obj, BuildCommand,Target):\r
+ Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]\r
+ BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)\r
if Target in [None, "", "all"]:\r
self.Target = "tbuild"\r
\r
# @param Obj The PlatformAutoGen object the build is working on\r
# @param Target The build target name, one of gSupportedTarget\r
#\r
- def __init__(self, Obj, Target):\r
- Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]\r
- Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])\r
- BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)\r
+ def __init__(self, Obj, BuildCommand, Target):\r
+ Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]\r
+ Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])\r
+ BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)\r
\r
## The class representing the task of a module build or platform build\r
#\r
self.ConfDirectory = BuildOptions.ConfDirectory\r
self.SpawnMode = True\r
self.BuildReport = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)\r
- self.TargetTxt = TargetTxtClassObject()\r
- self.ToolDef = ToolDefClassObject()\r
+ self.TargetTxt = TargetTxt\r
+ self.ToolDef = ToolDef\r
self.AutoGenTime = 0\r
self.MakeTime = 0\r
self.GenFdsTime = 0\r
GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))\r
if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):\r
os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))\r
- self.Db = WorkspaceDatabase()\r
+ self.Db = BuildDB\r
self.BuildDatabase = self.Db.BuildObject\r
self.Platform = None\r
self.ToolChainFamily = None\r
EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))\r
if self.Prebuild:\r
self.LaunchPrebuild()\r
- self.TargetTxt = TargetTxtClassObject()\r
- self.ToolDef = ToolDefClassObject()\r
+ self.TargetTxt = TargetTxt\r
+ self.ToolDef = ToolDef\r
if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):\r
self.InitBuild()\r
\r
+ self.AutoGenMgr = None\r
EdkLogger.info("")\r
os.chdir(self.WorkspaceDir)\r
+ self.share_data = Manager().dict()\r
+ def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):\r
+ try:\r
+ if SkipAutoGen:\r
+ return True,0\r
+ feedback_q = mp.Queue()\r
+ file_lock = mp.Lock()\r
+ error_event = mp.Event()\r
+ auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,error_event) for _ in range(self.ThreadNumber)]\r
+ self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)\r
+ self.AutoGenMgr.start()\r
+ for w in auto_workers:\r
+ w.start()\r
+ if PcdMaList is not None:\r
+ for PcdMa in PcdMaList:\r
+ PcdMa.CreateCodeFile(False)\r
+ PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))\r
+\r
+ self.AutoGenMgr.join()\r
+ rt = self.AutoGenMgr.Status\r
+ return rt, 0\r
+ except Exception as e:\r
+ return False,e.errcode\r
\r
## Load configuration\r
#\r
# This method will parse target.txt and get the build configurations.\r
#\r
def LoadConfiguration(self):\r
- #\r
- # Check target.txt and tools_def.txt and Init them\r
- #\r
- BuildConfigurationFile = os.path.normpath(os.path.join(GlobalData.gConfDirectory, gBuildConfiguration))\r
- if os.path.isfile(BuildConfigurationFile) == True:\r
- StatusCode = self.TargetTxt.LoadTargetTxtFile(BuildConfigurationFile)\r
-\r
- ToolDefinitionFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_CONF]\r
- if ToolDefinitionFile == '':\r
- ToolDefinitionFile = gToolsDefinition\r
- ToolDefinitionFile = os.path.normpath(mws.join(self.WorkspaceDir, 'Conf', ToolDefinitionFile))\r
- if os.path.isfile(ToolDefinitionFile) == True:\r
- StatusCode = self.ToolDef.LoadToolDefFile(ToolDefinitionFile)\r
- else:\r
- EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=ToolDefinitionFile)\r
- else:\r
- EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile)\r
\r
# if no ARCH given in command line, get it from target.txt\r
if not self.ArchList:\r
# @param CreateDepModuleMakeFile Flag used to indicate creating makefile\r
# for dependent modules/Libraries\r
#\r
- def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}):\r
+ def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):\r
if AutoGenObject is None:\r
return False\r
-\r
+ if FfsCommand is None:\r
+ FfsCommand = {}\r
# skip file generation for cleanxxx targets, run and fds target\r
if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:\r
# for target which must generate AutoGen code and makefile\r
- if not self.SkipAutoGen or Target == 'genc':\r
- self.Progress.Start("Generating code")\r
- AutoGenObject.CreateCodeFile(CreateDepsCodeFile)\r
- self.Progress.Stop("done!")\r
- if Target == "genc":\r
- return True\r
-\r
- if not self.SkipAutoGen or Target == 'genmake':\r
- self.Progress.Start("Generating makefile")\r
- AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand)\r
- self.Progress.Stop("done!")\r
- if Target == "genmake":\r
- return True\r
- else:\r
- # always recreate top/platform makefile when clean, just in case of inconsistency\r
+ mqueue = mp.Queue()\r
+ for m in AutoGenObject.GetAllModuleInfo:\r
+ mqueue.put(m)\r
+\r
+ AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}\r
+ self.Progress.Start("Generating makefile and code")\r
+ data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))\r
+ AutoGenObject.DataPipe.dump(data_pipe_file)\r
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)\r
+ self.Progress.Stop("done!")\r
+ if not autogen_rt:\r
+ self.AutoGenMgr.TerminateWorkers()\r
+ self.AutoGenMgr.join(0.1)\r
+ raise FatalError(errorcode)\r
AutoGenObject.CreateCodeFile(False)\r
AutoGenObject.CreateMakeFile(False)\r
+ else:\r
+ # always recreate top/platform makefile when clean, just in case of inconsistency\r
+ AutoGenObject.CreateCodeFile(True)\r
+ AutoGenObject.CreateMakeFile(True)\r
\r
if EdkLogger.GetLevel() == EdkLogger.QUIET:\r
EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))\r
return True\r
else:\r
# always recreate top/platform makefile when clean, just in case of inconsistency\r
- AutoGenObject.CreateCodeFile(False)\r
- AutoGenObject.CreateMakeFile(False)\r
+ AutoGenObject.CreateCodeFile(True)\r
+ AutoGenObject.CreateMakeFile(True)\r
\r
if EdkLogger.GetLevel() == EdkLogger.QUIET:\r
EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
for Arch in Wa.ArchList:\r
+ PcdMaList = []\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
for Module in Pa.Platform.Modules:\r
# Get ModuleAutoGen object to generate C code file and makefile\r
- Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)\r
+ Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
if Ma is None:\r
continue\r
+ if Ma.PcdIsDriver:\r
+ Ma.PlatformInfo = Pa\r
+ Ma.Workspace = Wa\r
+ PcdMaList.append(Ma)\r
self.BuildModules.append(Ma)\r
- self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict)\r
+ self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)\r
\r
# Create MAP file when Load Fix Address is enabled.\r
if self.Target in ["", "all", "fds"]:\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
for Module in Pa.Platform.Modules:\r
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:\r
- Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)\r
+ Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
if Ma is None:\r
continue\r
MaList.append(Ma)\r
MakeStart = time.time()\r
for Ma in self.BuildModules:\r
if not Ma.IsBinaryModule:\r
- Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))\r
+ Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))\r
# Break build if any build thread has error\r
if BuildTask.HasError():\r
# we need a full version of makefile for platform\r
Wa.CreateMakeFile(False)\r
\r
# Add ffs build to makefile\r
- CmdListDict = None\r
+ CmdListDict = {}\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
ExitFlag = threading.Event()\r
ExitFlag.clear()\r
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))\r
+ self.BuildModules = []\r
for Arch in Wa.ArchList:\r
+ PcdMaList = []\r
AutoGenStart = time.time()\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
if Inf in Pa.Platform.Modules:\r
continue\r
ModuleList.append(Inf)\r
+ Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}\r
+ Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}\r
for Module in ModuleList:\r
# Get ModuleAutoGen object to generate C code file and makefile\r
- Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)\r
+ Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
\r
if Ma is None:\r
continue\r
+ if Ma.PcdIsDriver:\r
+ Ma.PlatformInfo = Pa\r
+ Ma.Workspace = Wa\r
+ PcdMaList.append(Ma)\r
if Ma.CanSkipbyHash():\r
self.HashSkipModules.append(Ma)\r
if GlobalData.gBinCacheSource:\r
EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
\r
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
- if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:\r
# for target which must generate AutoGen code and makefile\r
- if not self.SkipAutoGen or self.Target == 'genc':\r
- Ma.CreateCodeFile(True)\r
- if self.Target == "genc":\r
- continue\r
\r
- if not self.SkipAutoGen or self.Target == 'genmake':\r
- if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict:\r
- Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch])\r
- del CmdListDict[Module.File, Arch]\r
- else:\r
- Ma.CreateMakeFile(True)\r
- if self.Target == "genmake":\r
- continue\r
self.BuildModules.append(Ma)\r
# Initialize all modules in tracking to 'FAIL'\r
if Ma.Arch not in GlobalData.gModuleBuildTracking:\r
GlobalData.gModuleBuildTracking[Ma.Arch] = dict()\r
if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:\r
GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'\r
+ mqueue = mp.Queue()\r
+ for m in Pa.GetAllModuleInfo:\r
+ mqueue.put(m)\r
+ data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))\r
+ Pa.DataPipe.dump(data_pipe_file)\r
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)\r
self.Progress.Stop("done!")\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
+ if not autogen_rt:\r
+ self.AutoGenMgr.TerminateWorkers()\r
+ self.AutoGenMgr.join(0.1)\r
+ raise FatalError(errorcode)\r
+ for Arch in Wa.ArchList:\r
MakeStart = time.time()\r
for Ma in self.BuildModules:\r
# Generate build task for the module\r
if not Ma.IsBinaryModule:\r
- Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))\r
+ Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))\r
# Break build if any build thread has error\r
if BuildTask.HasError():\r
# we need a full version of makefile for platform\r
EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)\r
ReturnCode = FORMAT_INVALID\r
except KeyboardInterrupt:\r
+ if MyBuild is not None:\r
+\r
+ # for multi-thread build exits safely\r
+ MyBuild.Relinquish()\r
ReturnCode = ABORT_ERROR\r
if Option is not None and Option.debug is not None:\r
EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())\r
return ReturnCode\r
\r
if __name__ == '__main__':\r
+ try:\r
+ mp.set_start_method('spawn')\r
+ except:\r
+ pass\r
r = Main()\r
## 0-127 is a safe return range, and 1 is a standard default error\r
if r < 0 or r > 127: r = 1\r
projects / mirror_edk2.git / blobdiff