# build a platform or a module\r
#\r
# Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>\r
-# Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>\r
+# Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR>\r
# Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>\r
+# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
#\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
#\r
import multiprocessing\r
from threading import Thread,Event,BoundedSemaphore\r
import threading\r
-from subprocess import Popen,PIPE\r
+from linecache import getlines\r
+from subprocess import Popen,PIPE, STDOUT\r
from collections import OrderedDict, defaultdict\r
-from Common.buildoptions import BuildOption,BuildTarget\r
+\r
from AutoGen.PlatformAutoGen import PlatformAutoGen\r
from AutoGen.ModuleAutoGen import ModuleAutoGen\r
from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen\r
from AutoGen import GenMake\r
from Common import Misc as Utils\r
\r
-from Common.TargetTxtClassObject import TargetTxt\r
-from Common.ToolDefClassObject import ToolDef\r
+from Common.TargetTxtClassObject import TargetTxtDict\r
+from Common.ToolDefClassObject import ToolDefDict\r
+from buildoptions import MyOptionParser\r
from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory\r
from Common.StringUtils import NormPath\r
from Common.MultipleWorkspace import MultipleWorkspace as mws\r
from AutoGen.DataPipe import MemoryDataPipe\r
from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo\r
from GenFds.FdfParser import FdfParser\r
-\r
+from AutoGen.IncludesAutoGen import IncludesAutoGen\r
+from GenFds.GenFds import resetFdsGlobalVariable\r
+from AutoGen.AutoGen import CalculatePriorityValue\r
\r
## standard targets of build command\r
gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']\r
\r
-## build configuration file\r
-gBuildConfiguration = "target.txt"\r
-gToolsDefinition = "tools_def.txt"\r
-\r
TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')\r
TmpTableDict = {}\r
\r
# @param To The stream message put on\r
# @param ExitFlag The flag used to indicate stopping reading\r
#\r
-def ReadMessage(From, To, ExitFlag):\r
+def ReadMessage(From, To, ExitFlag,MemTo=None):\r
while True:\r
# read one line a time\r
Line = From.readline()\r
# empty string means "end"\r
if Line is not None and Line != b"":\r
- To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))\r
+ LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')\r
+ if MemTo is not None:\r
+ if "Note: including file:" == LineStr.lstrip()[:21]:\r
+ MemTo.append(LineStr)\r
+ else:\r
+ To(LineStr)\r
+ MemTo.append(LineStr)\r
+ else:\r
+ To(LineStr)\r
else:\r
break\r
- if ExitFlag.isSet():\r
+ if ExitFlag.is_set():\r
break\r
\r
+class MakeSubProc(Popen):\r
+ def __init__(self,*args, **argv):\r
+ super(MakeSubProc,self).__init__(*args, **argv)\r
+ self.ProcOut = []\r
+\r
## Launch an external program\r
#\r
# This method will call subprocess.Popen to execute an external program with\r
# given options in specified directory. Because of the dead-lock issue during\r
-# redirecting output of the external program, threads are used to to do the\r
+# redirecting output of the external program, threads are used to do the\r
# redirection work.\r
#\r
# @param Command A list or string containing the call of the program\r
# @param WorkingDir The directory in which the program will be running\r
#\r
-def LaunchCommand(Command, WorkingDir):\r
+def LaunchCommand(Command, WorkingDir,ModuleAuto = None):\r
BeginTime = time.time()\r
# if working directory doesn't exist, Popen() will raise an exception\r
if not os.path.isdir(WorkingDir):\r
EndOfProcedure = None\r
try:\r
# launch the command\r
- Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)\r
+ Proc = MakeSubProc(Command, stdout=PIPE, stderr=STDOUT, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)\r
\r
# launch two threads to read the STDOUT and STDERR\r
EndOfProcedure = Event()\r
EndOfProcedure.clear()\r
if Proc.stdout:\r
- StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))\r
- StdOutThread.setName("STDOUT-Redirector")\r
- StdOutThread.setDaemon(False)\r
+ StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))\r
+ StdOutThread.name = "STDOUT-Redirector"\r
+ StdOutThread.daemon = False\r
StdOutThread.start()\r
\r
- if Proc.stderr:\r
- StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))\r
- StdErrThread.setName("STDERR-Redirector")\r
- StdErrThread.setDaemon(False)\r
- StdErrThread.start()\r
\r
# waiting for program exit\r
Proc.wait()\r
\r
if Proc.stdout:\r
StdOutThread.join()\r
- if Proc.stderr:\r
- StdErrThread.join()\r
\r
# check the return code of the program\r
if Proc.returncode != 0:\r
EdkLogger.info(RespContent)\r
\r
EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))\r
+ if ModuleAuto:\r
+ iau = IncludesAutoGen(WorkingDir,ModuleAuto)\r
+ if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:\r
+ iau.CreateDepsFileForMsvc(Proc.ProcOut)\r
+ else:\r
+ iau.UpdateDepsFileforNonMsvc()\r
+ iau.UpdateDepsFileforTrim()\r
+ iau.CreateModuleDeps()\r
+ iau.CreateDepsInclude()\r
+ iau.CreateDepsTarget()\r
return "%dms" % (int(round((time.time() - BeginTime) * 1000)))\r
\r
## The smallest unit that can be built in multi-thread build mode\r
@staticmethod\r
def StartScheduler(MaxThreadNumber, ExitFlag):\r
SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))\r
- SchedulerThread.setName("Build-Task-Scheduler")\r
- SchedulerThread.setDaemon(False)\r
+ SchedulerThread.name = "Build-Task-Scheduler"\r
+ SchedulerThread.daemon = False\r
SchedulerThread.start()\r
# wait for the scheduler to be started, especially useful in Linux\r
while not BuildTask.IsOnGoing():\r
# indicated to do so, or there's error in running thread\r
#\r
while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \\r
- or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():\r
+ or not ExitFlag.is_set()) and not BuildTask._ErrorFlag.is_set():\r
EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"\r
% (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))\r
\r
BuildTask._PendingQueueLock.release()\r
\r
# launch build thread until the maximum number of threads is reached\r
- while not BuildTask._ErrorFlag.isSet():\r
+ while not BuildTask._ErrorFlag.is_set():\r
# empty ready queue, do nothing further\r
if len(BuildTask._ReadyQueue) == 0:\r
break\r
time.sleep(0.01)\r
\r
# wait for all running threads exit\r
- if BuildTask._ErrorFlag.isSet():\r
+ if BuildTask._ErrorFlag.is_set():\r
EdkLogger.quiet("\nWaiting for all build threads exit...")\r
- # while not BuildTask._ErrorFlag.isSet() and \\r
+ # while not BuildTask._ErrorFlag.is_set() and \\r
while len(BuildTask._RunningQueue) > 0:\r
EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))\r
- EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))\r
+ EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.name for Th in threading.enumerate()))\r
# avoid tense loop\r
time.sleep(0.1)\r
except BaseException as X:\r
#\r
@staticmethod\r
def IsOnGoing():\r
- return not BuildTask._SchedulerStopped.isSet()\r
+ return not BuildTask._SchedulerStopped.is_set()\r
\r
## Abort the build\r
@staticmethod\r
#\r
@staticmethod\r
def HasError():\r
- return BuildTask._ErrorFlag.isSet()\r
+ return BuildTask._ErrorFlag.is_set()\r
\r
## Get error message in running thread\r
#\r
#\r
def AddDependency(self, Dependency):\r
for Dep in Dependency:\r
- if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR):\r
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):\r
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list\r
\r
## The thread wrapper of LaunchCommand function\r
#\r
def _CommandThread(self, Command, WorkingDir):\r
try:\r
- self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)\r
+ self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)\r
self.CompleteFlag = True\r
\r
- # Run hash operation post dependency, to account for libs\r
- if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:\r
- HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")\r
- SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True)\r
+ # Run hash operation post dependency to account for libs\r
+ # Run if --hash or --binary-destination\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ self.BuildItem.BuildObject.GenModuleHash()\r
+ if GlobalData.gBinCacheDest:\r
+ self.BuildItem.BuildObject.GenCMakeHash()\r
+\r
except:\r
#\r
# TRICK: hide the output of threads left running, so that the user can\r
# catch the error message easily\r
#\r
- if not BuildTask._ErrorFlag.isSet():\r
+ if not BuildTask._ErrorFlag.is_set():\r
GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),\r
self.BuildItem.BuildObject.Arch,\r
self.BuildItem.BuildObject.ToolChain,\r
EdkLogger.SetLevel(EdkLogger.ERROR)\r
BuildTask._ErrorFlag.set()\r
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \\r
- (threading.currentThread().getName(), Command, WorkingDir)\r
-\r
- # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'\r
- # If Module or Lib is being tracked, it did not fail header check test, and built successfully\r
- if (self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and\r
- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] != 'FAIL_METAFILE' and\r
- not BuildTask._ErrorFlag.isSet()\r
- ):\r
- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = 'SUCCESS'\r
+ (threading.current_thread().name, Command, WorkingDir)\r
\r
# indicate there's a thread is available for another build task\r
BuildTask._RunningQueueLock.acquire()\r
EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))\r
Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]\r
self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))\r
- self.BuildTread.setName("build thread")\r
- self.BuildTread.setDaemon(False)\r
+ self.BuildTread.name = "build thread"\r
+ self.BuildTread.daemon = False\r
self.BuildTread.start()\r
\r
## The class contains the information related to EFI image\r
self.ConfDirectory = BuildOptions.ConfDirectory\r
self.SpawnMode = True\r
self.BuildReport = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)\r
- self.TargetTxt = TargetTxt\r
- self.ToolDef = ToolDef\r
self.AutoGenTime = 0\r
self.MakeTime = 0\r
self.GenFdsTime = 0\r
+ self.MakeFileName = ""\r
+ TargetObj = TargetTxtDict()\r
+ ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"),"Conf")))\r
+ self.TargetTxt = TargetObj.Target\r
+ self.ToolDef = ToolDefObj.ToolDef\r
GlobalData.BuildOptionPcd = BuildOptions.OptionPcd if BuildOptions.OptionPcd else []\r
#Set global flag for build mode\r
GlobalData.gIgnoreSource = BuildOptions.IgnoreSources\r
EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))\r
if self.Prebuild:\r
self.LaunchPrebuild()\r
- self.TargetTxt = TargetTxt\r
- self.ToolDef = ToolDef\r
+ TargetObj = TargetTxtDict()\r
+ ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"), "Conf")))\r
+ self.TargetTxt = TargetObj.Target\r
+ self.ToolDef = ToolDefObj.ToolDef\r
if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):\r
self.InitBuild()\r
\r
self.AutoGenMgr = None\r
EdkLogger.info("")\r
os.chdir(self.WorkspaceDir)\r
- GlobalData.gCacheIR = Manager().dict()\r
self.log_q = log_q\r
GlobalData.file_lock = mp.Lock()\r
- GlobalData.cache_lock = mp.Lock()\r
- def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):\r
+ # Init cache data for local only\r
+ GlobalData.gPackageHashFile = dict()\r
+ GlobalData.gModulePreMakeCacheStatus = dict()\r
+ GlobalData.gModuleMakeCacheStatus = dict()\r
+ GlobalData.gHashChainStatus = dict()\r
+ GlobalData.gCMakeHashFile = dict()\r
+ GlobalData.gModuleHashFile = dict()\r
+ GlobalData.gFileHashDict = dict()\r
+ GlobalData.gModuleAllCacheStatus = set()\r
+ GlobalData.gModuleCacheHit = set()\r
+\r
+ def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):\r
try:\r
if SkipAutoGen:\r
return True,0\r
if FfsCmd is None:\r
FfsCmd = {}\r
GlobalData.FfsCmd = FfsCmd\r
- GlobalData.libConstPcd = DataPipe.Get("LibConstPcd")\r
- GlobalData.Refes = DataPipe.Get("REFS")\r
- auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,GlobalData.cache_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
+ auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]\r
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)\r
self.AutoGenMgr.start()\r
for w in auto_workers:\r
w.start()\r
if PcdMaList is not None:\r
for PcdMa in PcdMaList:\r
- if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- PcdMa.GenModuleFilesHash(share_data)\r
- PcdMa.GenPreMakefileHash(share_data)\r
- if PcdMa.CanSkipbyPreMakefileCache(share_data):\r
- continue\r
+ # SourceFileList calling sequence impact the makefile string sequence.\r
+ # Create cached SourceFileList here to unify its calling sequence for both\r
+ # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.\r
+ RetVal = PcdMa.SourceFileList\r
+ # Force cache miss for PCD driver\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:\r
+ cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))\r
\r
PcdMa.CreateCodeFile(False)\r
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))\r
-\r
+ PcdMa.CreateAsBuiltInf()\r
+ # Force cache miss for PCD driver\r
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- PcdMa.GenMakeHeaderFilesHash(share_data)\r
- PcdMa.GenMakeHash(share_data)\r
- if PcdMa.CanSkipbyMakeCache(share_data):\r
- continue\r
+ cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))\r
\r
self.AutoGenMgr.join()\r
rt = self.AutoGenMgr.Status\r
- return rt, 0\r
+ err = 0\r
+ if not rt:\r
+ err = UNKNOWN_ERROR\r
+ return rt, err\r
except FatalError as e:\r
return False, e.args[0]\r
except:\r
return False, UNKNOWN_ERROR\r
\r
+ ## Add TOOLCHAIN and FAMILY declared in DSC [BuildOptions] to ToolsDefTxtDatabase.\r
+ #\r
+ # Loop through the set of build targets, tool chains, and archs provided on either\r
+ # the command line or in target.txt to discover FAMILY and TOOLCHAIN delclarations\r
+ # in [BuildOptions] sections that may be within !if expressions that may use\r
+ # $(TARGET), $(TOOLCHAIN), $(TOOLCHAIN_TAG), or $(ARCH) operands.\r
+ #\r
+ def GetToolChainAndFamilyFromDsc (self, File):\r
+ SavedGlobalDefines = GlobalData.gGlobalDefines.copy()\r
+ for BuildTarget in self.BuildTargetList:\r
+ GlobalData.gGlobalDefines['TARGET'] = BuildTarget\r
+ for BuildToolChain in self.ToolChainList:\r
+ GlobalData.gGlobalDefines['TOOLCHAIN'] = BuildToolChain\r
+ GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = BuildToolChain\r
+ for BuildArch in self.ArchList:\r
+ GlobalData.gGlobalDefines['ARCH'] = BuildArch\r
+ dscobj = self.BuildDatabase[File, BuildArch]\r
+ for KeyFamily, Key, KeyCodeBase in dscobj.BuildOptions:\r
+ try:\r
+ Target, ToolChain, Arch, Tool, Attr = Key.split('_')\r
+ except:\r
+ continue\r
+ if ToolChain == TAB_STAR or Attr != TAB_TOD_DEFINES_FAMILY:\r
+ continue\r
+ try:\r
+ Family = dscobj.BuildOptions[(KeyFamily, Key, KeyCodeBase)]\r
+ Family = Family.strip().strip('=').strip()\r
+ except:\r
+ continue\r
+ if TAB_TOD_DEFINES_FAMILY not in self.ToolDef.ToolsDefTxtDatabase:\r
+ self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}\r
+ if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:\r
+ self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][ToolChain] = Family\r
+ if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolDef.ToolsDefTxtDatabase:\r
+ self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}\r
+ if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY]:\r
+ self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][ToolChain] = Family\r
+ if TAB_TOD_DEFINES_TOOL_CHAIN_TAG not in self.ToolDef.ToolsDefTxtDatabase:\r
+ self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = []\r
+ if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:\r
+ self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].append(ToolChain)\r
+ GlobalData.gGlobalDefines = SavedGlobalDefines\r
+\r
## Load configuration\r
#\r
# This method will parse target.txt and get the build configurations.\r
if self.ToolChainList is None or len(self.ToolChainList) == 0:\r
EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")\r
\r
+ if not self.PlatformFile:\r
+ PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]\r
+ if not PlatformFile:\r
+ # Try to find one in current directory\r
+ WorkingDirectory = os.getcwd()\r
+ FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))\r
+ FileNum = len(FileList)\r
+ if FileNum >= 2:\r
+ EdkLogger.error("build", OPTION_MISSING,\r
+ ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))\r
+ elif FileNum == 1:\r
+ PlatformFile = FileList[0]\r
+ else:\r
+ EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,\r
+ ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")\r
+\r
+ self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)\r
+\r
+ self.GetToolChainAndFamilyFromDsc (self.PlatformFile)\r
+\r
# check if the tool chains are defined or not\r
NewToolChainList = []\r
for ToolChain in self.ToolChainList:\r
ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])\r
self.ToolChainFamily = ToolChainFamily\r
\r
- if not self.PlatformFile:\r
- PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]\r
- if not PlatformFile:\r
- # Try to find one in current directory\r
- WorkingDirectory = os.getcwd()\r
- FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))\r
- FileNum = len(FileList)\r
- if FileNum >= 2:\r
- EdkLogger.error("build", OPTION_MISSING,\r
- ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))\r
- elif FileNum == 1:\r
- PlatformFile = FileList[0]\r
- else:\r
- EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,\r
- ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")\r
-\r
- self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)\r
self.ThreadNumber = ThreadNum()\r
## Initialize build configuration\r
#\r
EndOfProcedure.clear()\r
if Process.stdout:\r
StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))\r
- StdOutThread.setName("STDOUT-Redirector")\r
- StdOutThread.setDaemon(False)\r
+ StdOutThread.name = "STDOUT-Redirector"\r
+ StdOutThread.daemon = False\r
StdOutThread.start()\r
\r
if Process.stderr:\r
StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))\r
- StdErrThread.setName("STDERR-Redirector")\r
- StdErrThread.setDaemon(False)\r
+ StdErrThread.name = "STDERR-Redirector"\r
+ StdErrThread.daemon = False\r
StdErrThread.start()\r
# waiting for program exit\r
Process.wait()\r
EndOfProcedure.clear()\r
if Process.stdout:\r
StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))\r
- StdOutThread.setName("STDOUT-Redirector")\r
- StdOutThread.setDaemon(False)\r
+ StdOutThread.name = "STDOUT-Redirector"\r
+ StdOutThread.daemon = False\r
StdOutThread.start()\r
\r
if Process.stderr:\r
StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))\r
- StdErrThread.setName("STDERR-Redirector")\r
- StdErrThread.setDaemon(False)\r
+ StdErrThread.name = "STDERR-Redirector"\r
+ StdErrThread.daemon = False\r
StdErrThread.start()\r
# waiting for program exit\r
Process.wait()\r
EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')\r
EdkLogger.info("\n- Postbuild Done -\n")\r
\r
- ## Error handling for hash feature\r
- #\r
- # On BuildTask error, iterate through the Module Build tracking\r
- # dictionary to determine wheather a module failed to build. Invalidate\r
- # the hash associated with that module by removing it from storage.\r
- #\r
- #\r
- def invalidateHash(self):\r
- # Only for hashing feature\r
- if not GlobalData.gUseHashCache:\r
- return\r
-\r
- # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash\r
- for Ma in GlobalData.gModuleBuildTracking:\r
- # Skip invalidating for Successful Module/Lib builds\r
- if GlobalData.gModuleBuildTracking[Ma] == 'SUCCESS':\r
- continue\r
-\r
- # The module failed to build, failed to start building, or failed the header check test from this point on\r
-\r
- # Remove .hash from build\r
- ModuleHashFile = os.path.join(Ma.BuildDir, Ma.Name + ".hash")\r
- if os.path.exists(ModuleHashFile):\r
- os.remove(ModuleHashFile)\r
-\r
- # Remove .hash file from cache\r
- if GlobalData.gBinCacheDest:\r
- FileDir = os.path.join(GlobalData.gBinCacheDest, Ma.PlatformInfo.OutputDir, Ma.BuildTarget + "_" + Ma.ToolChain, Ma.Arch, Ma.SourceDir, Ma.MetaFile.BaseName)\r
- HashFile = os.path.join(FileDir, Ma.Name + '.hash')\r
- if os.path.exists(HashFile):\r
- os.remove(HashFile)\r
-\r
## Build a module or platform\r
#\r
# Create autogen code and makefile for a module or platform, and the launch\r
mqueue = mp.Queue()\r
for m in AutoGenObject.GetAllModuleInfo:\r
mqueue.put(m)\r
-\r
+ mqueue.put((None,None,None,None,None,None,None))\r
AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}\r
AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}\r
AutoGenObject.CreateLibModuelDirs()\r
self.Progress.Start("Generating makefile and code")\r
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))\r
AutoGenObject.DataPipe.dump(data_pipe_file)\r
- autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)\r
+ cqueue = mp.Queue()\r
+ autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)\r
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
with open(AutoGenIdFile,"w") as fw:\r
fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))\r
(AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),\r
ExtraData=str(AutoGenObject))\r
\r
- makefile = GenMake.BuildFile(AutoGenObject)._FILE_NAME_[GenMake.gMakeType]\r
-\r
# run\r
if Target == 'run':\r
return True\r
\r
+ # Fetch the MakeFileName.\r
+ self.MakeFileName = AutoGenObject.MakeFileName\r
+\r
# build modules\r
if BuildModule:\r
BuildCommand = BuildCommand + [Target]\r
LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)\r
- self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
return True\r
\r
# build library\r
if Target == 'libraries':\r
- for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
- NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']\r
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
+ DirList = []\r
+ for Lib in AutoGenObject.LibraryAutoGenList:\r
+ if not Lib.IsBinaryModule:\r
+ DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))\r
+ for Lib, LibAutoGen in DirList:\r
+ NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']\r
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)\r
return True\r
\r
# build module\r
if Target == 'modules':\r
- for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
- NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']\r
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
- for Mod in AutoGenObject.ModuleBuildDirectoryList:\r
- NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']\r
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
+ DirList = []\r
+ for Lib in AutoGenObject.LibraryAutoGenList:\r
+ if not Lib.IsBinaryModule:\r
+ DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))\r
+ for Lib, LibAutoGen in DirList:\r
+ NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']\r
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)\r
+\r
+ DirList = []\r
+ for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:\r
+ if not ModuleAutoGen.IsBinaryModule:\r
+ DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))\r
+ for Mod,ModAutoGen in DirList:\r
+ NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, self.MakeFileName)), 'pbuild']\r
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
return True\r
\r
# cleanlib\r
if Target == 'cleanlib':\r
for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
- LibMakefile = os.path.normpath(os.path.join(Lib, makefile))\r
+ LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))\r
if os.path.exists(LibMakefile):\r
NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']\r
LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
# clean\r
if Target == 'clean':\r
for Mod in AutoGenObject.ModuleBuildDirectoryList:\r
- ModMakefile = os.path.normpath(os.path.join(Mod, makefile))\r
+ ModMakefile = os.path.normpath(os.path.join(Mod, self.MakeFileName))\r
if os.path.exists(ModMakefile):\r
NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']\r
LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
- LibMakefile = os.path.normpath(os.path.join(Lib, makefile))\r
+ LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))\r
if os.path.exists(LibMakefile):\r
NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']\r
LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
return True\r
\r
if Target == 'fds':\r
if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db):\r
EdkLogger.error("build", COMMAND_FAILURE)\r
+ Threshold = self.GetFreeSizeThreshold()\r
+ if Threshold:\r
+ self.CheckFreeSizeThreshold(Threshold, AutoGenObject.FvDir)\r
return True\r
\r
# run\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
- # Add Platform and Package level hash in share_data for module hash calculation later\r
- if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
- GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
- for PkgName in GlobalData.gPackageHash.keys():\r
- GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
GlobalData.file_lock = mp.Lock()\r
- GlobalData.cache_lock = mp.Lock()\r
GlobalData.FfsCmd = CmdListDict\r
\r
self.Progress.Stop("done!")\r
AutoGenStart = time.time()\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)\r
- GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd")\r
- GlobalData.Refes = Pa.DataPipe.Get("REFS")\r
for Module in Pa.Platform.Modules:\r
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
Ma.Workspace = Wa\r
MaList.append(Ma)\r
\r
- if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- Ma.GenModuleFilesHash(GlobalData.gCacheIR)\r
- Ma.GenPreMakefileHash(GlobalData.gCacheIR)\r
- if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):\r
- self.HashSkipModules.append(Ma)\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:\r
+ if Ma.CanSkipbyPreMakeCache():\r
continue\r
+ else:\r
+ self.PreMakeCacheMiss.add(Ma)\r
\r
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'\r
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:\r
return True\r
\r
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:\r
- Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)\r
- Ma.GenMakeHash(GlobalData.gCacheIR)\r
- if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):\r
- self.HashSkipModules.append(Ma)\r
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
+ if Ma.CanSkipbyMakeCache():\r
continue\r
else:\r
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))\r
- Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)\r
+ self.MakeCacheMiss.add(Ma)\r
\r
self.BuildModules.append(Ma)\r
- # Initialize all modules in tracking to 'FAIL'\r
- GlobalData.gModuleBuildTracking[Ma] = 'FAIL'\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
MakeStart = time.time()\r
for Ma in self.BuildModules:\r
# we need a full version of makefile for platform\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
- self.invalidateHash()\r
Pa.CreateMakeFile(False)\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
# Start task scheduler\r
# in case there's an interruption. we need a full version of makefile for platform\r
Pa.CreateMakeFile(False)\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
BuildTask.WaitForComplete()\r
self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
self.BuildModules = []\r
self.MakeTime += int(round((time.time() - MakeContiue)))\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
\r
self.BuildReport.AddPlatformReport(Wa, MaList)\r
# Save MAP buffer into MAP file.\r
#\r
self._SaveMapFile (MapBuffer, Wa)\r
- self.invalidateHash()\r
\r
def _GenFfsCmd(self,ArchList):\r
# convert dictionary of Cmd:(Inf,Arch)\r
ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")\r
\r
for m_build_dir in LibraryBuildDirectoryList:\r
- if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):\r
+ if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):\r
return None\r
for m_build_dir in ModuleBuildDirectoryList:\r
- if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):\r
+ if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):\r
return None\r
Wa = WorkSpaceInfo(\r
workspacedir,active_p,target,toolchain,archlist\r
if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:\r
FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]\r
for FdRegion in FdDict.RegionList:\r
- if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):\r
+ if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):\r
if int(FdRegion.Offset) % 8 != 0:\r
EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))\r
Wa.FdfProfile = Fdf.Profile\r
self.BuildReport.AddPlatformReport(Wa)\r
Wa.CreateMakeFile(False)\r
\r
- # Add ffs build to makefile\r
+ # Add ffs build to makefile\r
CmdListDict = {}\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
- # Add Platform and Package level hash in share_data for module hash calculation later\r
- if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:\r
- GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash\r
- for PkgName in GlobalData.gPackageHash.keys():\r
- GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]\r
-\r
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))\r
BuildModules = []\r
- TotalModules = []\r
for Arch in Wa.ArchList:\r
PcdMaList = []\r
AutoGenStart = time.time()\r
ModuleList = []\r
for Inf in Pa.Platform.Modules:\r
ModuleList.append(Inf)\r
- # Add the INF only list in FDF\r
+ # Add the INF only list in FDF\r
if GlobalData.gFdfParser is not None:\r
for InfName in GlobalData.gFdfParser.Profile.InfList:\r
Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)\r
Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}\r
Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}\r
Pa.CreateLibModuelDirs()\r
+ # Fetch the MakeFileName.\r
+ self.MakeFileName = Pa.MakeFileName\r
+\r
Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}\r
Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}\r
Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}\r
+ # Prepare the cache share data for multiprocessing\r
+ Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}\r
ModuleCodaFile = {}\r
for ma in Pa.ModuleAutoGenList:\r
ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]\r
Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}\r
+ # ModuleList contains all driver modules only\r
for Module in ModuleList:\r
- # Get ModuleAutoGen object to generate C code file and makefile\r
+ # Get ModuleAutoGen object to generate C code file and makefile\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
-\r
if Ma is None:\r
continue\r
if Ma.PcdIsDriver:\r
Ma.PlatformInfo = Pa\r
Ma.Workspace = Wa\r
PcdMaList.append(Ma)\r
- TotalModules.append(Ma)\r
- # Initialize all modules in tracking to 'FAIL'\r
- GlobalData.gModuleBuildTracking[Ma] = 'FAIL'\r
-\r
+ self.AllDrivers.add(Ma)\r
+ self.AllModules.add(Ma)\r
\r
mqueue = mp.Queue()\r
+ cqueue = mp.Queue()\r
for m in Pa.GetAllModuleInfo:\r
mqueue.put(m)\r
+ module_file,module_root,module_path,module_basename,\\r
+ module_originalpath,module_arch,IsLib = m\r
+ Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\\r
+ ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
+ self.AllModules.add(Ma)\r
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))\r
Pa.DataPipe.dump(data_pipe_file)\r
\r
- autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)\r
-\r
- # Skip cache hit modules\r
- if GlobalData.gBinCacheSource:\r
- for Ma in TotalModules:\r
- if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
- GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:\r
- self.HashSkipModules.append(Ma)\r
- continue\r
- if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \\r
- GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:\r
- self.HashSkipModules.append(Ma)\r
- continue\r
- BuildModules.append(Ma)\r
- else:\r
- BuildModules.extend(TotalModules)\r
+ mqueue.put((None,None,None,None,None,None,None))\r
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)\r
\r
if not autogen_rt:\r
self.AutoGenMgr.TerminateWorkers()\r
self.AutoGenMgr.join(1)\r
raise FatalError(errorcode)\r
+\r
+ if GlobalData.gUseHashCache:\r
+ for item in GlobalData.gModuleAllCacheStatus:\r
+ (MetaFilePath, Arch, CacheStr, Status) = item\r
+ Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\\r
+ ToolChain, Arch, self.PlatformFile,Pa.DataPipe)\r
+ if CacheStr == "PreMakeCache" and Status == False:\r
+ self.PreMakeCacheMiss.add(Ma)\r
+ if CacheStr == "PreMakeCache" and Status == True:\r
+ self.PreMakeCacheHit.add(Ma)\r
+ GlobalData.gModuleCacheHit.add(Ma)\r
+ if CacheStr == "MakeCache" and Status == False:\r
+ self.MakeCacheMiss.add(Ma)\r
+ if CacheStr == "MakeCache" and Status == True:\r
+ self.MakeCacheHit.add(Ma)\r
+ GlobalData.gModuleCacheHit.add(Ma)\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")\r
with open(AutoGenIdFile,"w") as fw:\r
fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))\r
fw.write("BuildDir=%s\n" % Wa.BuildDir)\r
fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))\r
+\r
+ if GlobalData.gBinCacheSource:\r
+ BuildModules.extend(self.MakeCacheMiss)\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:\r
+ BuildModules.extend(self.PreMakeCacheMiss)\r
+ else:\r
+ BuildModules.extend(self.AllDrivers)\r
+\r
self.Progress.Stop("done!")\r
return Wa, BuildModules\r
\r
GlobalData.gGlobalDefines['TARGET'] = BuildTarget\r
index = 0\r
for ToolChain in self.ToolChainList:\r
+ resetFdsGlobalVariable()\r
GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain\r
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain\r
GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]\r
GlobalData.gAutoGenPhase = False\r
\r
if GlobalData.gBinCacheSource:\r
- EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))\r
- CacheHitMa = set()\r
- CacheNotHitMa = set()\r
- for IR in GlobalData.gCacheIR.keys():\r
- if 'PlatformHash' in IR or 'PackageHash' in IR:\r
- continue\r
- if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit:\r
- CacheHitMa.add(IR)\r
- else:\r
- # There might be binary module or module which has .inc files, not count for cache miss\r
- CacheNotHitMa.add(IR)\r
- EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa)))\r
+ EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))\r
+ EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))\r
+ EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))\r
\r
for Arch in Wa.ArchList:\r
MakeStart = time.time()\r
# we need a full version of makefile for platform\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
- self.invalidateHash()\r
Pa.CreateMakeFile(False)\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
# Start task scheduler\r
# in case there's an interruption. we need a full version of makefile for platform\r
\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
#\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
- self.CreateAsBuiltInf()\r
if GlobalData.gBinCacheDest:\r
- self.UpdateBuildCache()\r
+ self.GenDestCache()\r
+ elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:\r
+ # Only for --hash\r
+ # Update PreMakeCacheChain files\r
+ self.GenLocalPreMakeCache()\r
+ #\r
+ # Get Module List\r
+ #\r
+ ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}\r
self.BuildModules = []\r
self.MakeTime += int(round((time.time() - MakeContiue)))\r
#\r
# has been signaled.\r
#\r
if BuildTask.HasError():\r
- self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
\r
# Create MAP file when Load Fix Address is enabled.\r
#\r
if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:\r
EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")\r
- #\r
- # Get Module List\r
- #\r
- ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}\r
\r
#\r
# Rebase module to the preferred memory address before GenFds\r
GenFdsStart = time.time()\r
if GenFdsApi(Wa.GenFdsCommandDict, self.Db):\r
EdkLogger.error("build", COMMAND_FAILURE)\r
+ Threshold = self.GetFreeSizeThreshold()\r
+ if Threshold:\r
+ self.CheckFreeSizeThreshold(Threshold, Wa.FvDir)\r
\r
#\r
# Create MAP file for all platform FVs after GenFds.\r
#\r
self._SaveMapFile(MapBuffer, Wa)\r
self.CreateGuidedSectionToolsFile(Wa)\r
- self.invalidateHash()\r
+\r
+ ## GetFreeSizeThreshold()\r
+ #\r
+ # @retval int Threshold value\r
+ #\r
+ def GetFreeSizeThreshold(self):\r
+ Threshold = None\r
+ Threshold_Str = GlobalData.gCommandLineDefines.get('FV_SPARE_SPACE_THRESHOLD')\r
+ if Threshold_Str:\r
+ try:\r
+ if Threshold_Str.lower().startswith('0x'):\r
+ Threshold = int(Threshold_Str, 16)\r
+ else:\r
+ Threshold = int(Threshold_Str)\r
+ except:\r
+ EdkLogger.warn("build", 'incorrect value for FV_SPARE_SPACE_THRESHOLD %s.Only decimal or hex format is allowed.' % Threshold_Str)\r
+ return Threshold\r
+\r
+ def CheckFreeSizeThreshold(self, Threshold=None, FvDir=None):\r
+ if not isinstance(Threshold, int):\r
+ return\r
+ if not isinstance(FvDir, str) or not FvDir:\r
+ return\r
+ FdfParserObject = GlobalData.gFdfParser\r
+ FvRegionNameList = [FvName for FvName in FdfParserObject.Profile.FvDict if FdfParserObject.Profile.FvDict[FvName].FvRegionInFD]\r
+ for FvName in FdfParserObject.Profile.FvDict:\r
+ if FvName in FvRegionNameList:\r
+ FvSpaceInfoFileName = os.path.join(FvDir, FvName.upper() + '.Fv.map')\r
+ if os.path.exists(FvSpaceInfoFileName):\r
+ FileLinesList = getlines(FvSpaceInfoFileName)\r
+ for Line in FileLinesList:\r
+ NameValue = Line.split('=')\r
+ if len(NameValue) == 2 and NameValue[0].strip() == 'EFI_FV_SPACE_SIZE':\r
+ FreeSizeValue = int(NameValue[1].strip(), 0)\r
+ if FreeSizeValue < Threshold:\r
+ EdkLogger.error("build", FV_FREESIZE_ERROR,\r
+ '%s FV free space %d is not enough to meet with the required spare space %d set by -D FV_SPARE_SPACE_THRESHOLD option.' % (\r
+ FvName, FreeSizeValue, Threshold))\r
+ break\r
+\r
## Generate GuidedSectionTools.txt in the FV directories.\r
#\r
def CreateGuidedSectionToolsFile(self,Wa):\r
FvDir = Wa.FvDir\r
if not os.path.exists(FvDir):\r
continue\r
-\r
for Arch in self.ArchList:\r
- # Build up the list of supported architectures for this build\r
- prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)\r
-\r
- # Look through the tool definitions for GUIDed tools\r
+ guidList = []\r
+ tooldefguidList = []\r
guidAttribs = []\r
- for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():\r
- if attrib.upper().endswith('_GUID'):\r
- split = attrib.split('_')\r
- thisPrefix = '_'.join(split[0:3]) + '_'\r
- if thisPrefix == prefix:\r
- guid = self.ToolDef.ToolsDefTxtDictionary[attrib]\r
- guid = guid.lower()\r
- toolName = split[3]\r
- path = '_'.join(split[0:4]) + '_PATH'\r
- path = self.ToolDef.ToolsDefTxtDictionary[path]\r
- path = self.GetFullPathOfTool(path)\r
- guidAttribs.append((guid, toolName, path))\r
-\r
+ for Platform in Wa.AutoGenObjectList:\r
+ if Platform.BuildTarget != BuildTarget:\r
+ continue\r
+ if Platform.ToolChain != ToolChain:\r
+ continue\r
+ if Platform.Arch != Arch:\r
+ continue\r
+ if hasattr (Platform, 'BuildOption'):\r
+ for Tool in Platform.BuildOption:\r
+ if 'GUID' in Platform.BuildOption[Tool]:\r
+ if 'PATH' in Platform.BuildOption[Tool]:\r
+ value = Platform.BuildOption[Tool]['GUID']\r
+ if value in guidList:\r
+ EdkLogger.error("build", FORMAT_INVALID, "Duplicate GUID value %s used with Tool %s in DSC [BuildOptions]." % (value, Tool))\r
+ path = Platform.BuildOption[Tool]['PATH']\r
+ guidList.append(value)\r
+ guidAttribs.append((value, Tool, path))\r
+ for Tool in Platform.ToolDefinition:\r
+ if 'GUID' in Platform.ToolDefinition[Tool]:\r
+ if 'PATH' in Platform.ToolDefinition[Tool]:\r
+ value = Platform.ToolDefinition[Tool]['GUID']\r
+ if value in tooldefguidList:\r
+ EdkLogger.error("build", FORMAT_INVALID, "Duplicate GUID value %s used with Tool %s in tools_def.txt." % (value, Tool))\r
+ tooldefguidList.append(value)\r
+ if value in guidList:\r
+ # Already added by platform\r
+ continue\r
+ path = Platform.ToolDefinition[Tool]['PATH']\r
+ guidList.append(value)\r
+ guidAttribs.append((value, Tool, path))\r
+ # Sort by GuidTool name\r
+ guidAttribs = sorted (guidAttribs, key=lambda x: x[1])\r
# Write out GuidedSecTools.txt\r
toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')\r
toolsFile = open(toolsFile, 'wt')\r
print(' '.join(guidedSectionTool), file=toolsFile)\r
toolsFile.close()\r
\r
- ## Returns the full path of the tool.\r
+ ## Returns the real path of the tool.\r
#\r
- def GetFullPathOfTool (self, tool):\r
+ def GetRealPathOfTool (self, tool):\r
if os.path.exists(tool):\r
return os.path.realpath(tool)\r
- else:\r
- # We need to search for the tool using the\r
- # PATH environment variable.\r
- for dirInPath in os.environ['PATH'].split(os.pathsep):\r
- foundPath = os.path.join(dirInPath, tool)\r
- if os.path.exists(foundPath):\r
- return os.path.realpath(foundPath)\r
-\r
- # If the tool was not found in the path then we just return\r
- # the input tool.\r
return tool\r
\r
## Launch the module or platform build\r
#\r
def Launch(self):\r
+ self.AllDrivers = set()\r
+ self.AllModules = set()\r
+ self.PreMakeCacheMiss = set()\r
+ self.PreMakeCacheHit = set()\r
+ self.MakeCacheMiss = set()\r
+ self.MakeCacheHit = set()\r
if not self.ModuleFile:\r
if not self.SpawnMode or self.Target not in ["", "all"]:\r
self.SpawnMode = False\r
for Module in self.BuildModules:\r
Module.CreateAsBuiltInf()\r
\r
- def UpdateBuildCache(self):\r
- all_lib_set = set()\r
- all_mod_set = set()\r
- for Module in self.BuildModules:\r
+ def GenDestCache(self):\r
+ for Module in self.AllModules:\r
+ Module.GenPreMakefileHashList()\r
+ Module.GenMakefileHashList()\r
Module.CopyModuleToCache()\r
- all_mod_set.add(Module)\r
- for Module in self.HashSkipModules:\r
- Module.CopyModuleToCache()\r
- all_mod_set.add(Module)\r
- for Module in all_mod_set:\r
- for lib in Module.LibraryAutoGenList:\r
- all_lib_set.add(lib)\r
- for lib in all_lib_set:\r
- lib.CopyModuleToCache()\r
- all_lib_set.clear()\r
- all_mod_set.clear()\r
- self.HashSkipModules = []\r
+\r
+ def GenLocalPreMakeCache(self):\r
+ for Module in self.PreMakeCacheMiss:\r
+ Module.GenPreMakefileHashList()\r
+\r
## Do some clean-up works when error occurred\r
def Relinquish(self):\r
OldLogLevel = EdkLogger.GetLevel()\r
else:\r
return None\r
def ThreadNum():\r
+ OptionParser = MyOptionParser()\r
+ if not OptionParser.BuildOption and not OptionParser.BuildTarget:\r
+ OptionParser.GetOption()\r
+ BuildOption, BuildTarget = OptionParser.BuildOption, OptionParser.BuildTarget\r
ThreadNumber = BuildOption.ThreadNumber\r
+ GlobalData.gCmdConfDir = BuildOption.ConfDirectory\r
if ThreadNumber is None:\r
- ThreadNumber = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]\r
+ TargetObj = TargetTxtDict()\r
+ ThreadNumber = TargetObj.Target.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]\r
if ThreadNumber == '':\r
ThreadNumber = 0\r
else:\r
#\r
# Parse the options and args\r
#\r
- Option, Target = BuildOption, BuildTarget\r
+ OptionParser = MyOptionParser()\r
+ if not OptionParser.BuildOption and not OptionParser.BuildTarget:\r
+ OptionParser.GetOption()\r
+ Option, Target = OptionParser.BuildOption, OptionParser.BuildTarget\r
GlobalData.gOptions = Option\r
GlobalData.gCaseInsensitive = Option.CaseInsensitive\r
\r
Conclusion = "Done"\r
except:\r
Conclusion = "Failed"\r
+ ReturnCode = POSTBUILD_ERROR\r
elif ReturnCode == ABORT_ERROR:\r
Conclusion = "Aborted"\r
else:\r
## 0-127 is a safe return range, and 1 is a standard default error\r
if r < 0 or r > 127: r = 1\r
sys.exit(r)\r
-\r
projects / mirror_edk2.git / blobdiff