# Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>\r
# Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>\r
#\r
-# This program and the accompanying materials\r
-# are licensed and made available under the terms and conditions of the BSD License\r
-# which accompanies this distribution. The full text of the license may be found at\r
-# http://opensource.org/licenses/bsd-license.php\r
-#\r
-# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
-# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
+# SPDX-License-Identifier: BSD-2-Clause-Patent\r
#\r
\r
##\r
from __future__ import print_function\r
import Common.LongFilePathOs as os\r
import re\r
-from io import BytesIO\r
import sys\r
import glob\r
import time\r
# read one line a time\r
Line = From.readline()\r
# empty string means "end"\r
- if Line is not None and Line != "":\r
- To(Line.rstrip())\r
+ if Line is not None and Line != b"":\r
+ To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))\r
else:\r
break\r
if ExitFlag.isSet():\r
\r
# get all pending tasks\r
BuildTask._PendingQueueLock.acquire()\r
- BuildObjectList = BuildTask._PendingQueue.keys()\r
+ BuildObjectList = list(BuildTask._PendingQueue.keys())\r
#\r
# check if their dependency is resolved, and if true, move them\r
# into ready queue\r
time.sleep(0.1)\r
except BaseException as X:\r
#\r
- # TRICK: hide the output of threads left runing, so that the user can\r
+ # TRICK: hide the output of threads left running, so that the user can\r
# catch the error message easily\r
#\r
EdkLogger.SetLevel(EdkLogger.ERROR)\r
#\r
def AddDependency(self, Dependency):\r
for Dep in Dependency:\r
- if not Dep.BuildObject.IsBinaryModule:\r
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash():\r
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list\r
\r
## The thread wrapper of LaunchCommand function\r
try:\r
self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)\r
self.CompleteFlag = True\r
+\r
+ # Run hash operation post dependency, to account for libs\r
+ if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:\r
+ HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")\r
+ SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True)\r
except:\r
#\r
- # TRICK: hide the output of threads left runing, so that the user can\r
+ # TRICK: hide the output of threads left running, so that the user can\r
# catch the error message easily\r
#\r
if not BuildTask._ErrorFlag.isSet():\r
BuildTask._ErrorFlag.set()\r
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \\r
(threading.currentThread().getName(), Command, WorkingDir)\r
+ if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet():\r
+ GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True\r
# indicate there's a thread is available for another build task\r
BuildTask._RunningQueueLock.acquire()\r
BuildTask._RunningQueue.pop(self.BuildItem)\r
GlobalData.gBinCacheDest = BuildOptions.BinCacheDest\r
GlobalData.gBinCacheSource = BuildOptions.BinCacheSource\r
GlobalData.gEnableGenfdsMultiThread = BuildOptions.GenfdsMultiThread\r
+ GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck\r
\r
if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:\r
EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")\r
# Print the same path style with WORKSPACE env.\r
EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))\r
EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))\r
+ if "PYTHON3_ENABLE" in os.environ:\r
+ PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]\r
+ if PYTHON3_ENABLE != "TRUE":\r
+ PYTHON3_ENABLE = "FALSE"\r
+ EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))\r
+ if "PYTHON_COMMAND" in os.environ:\r
+ EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))\r
self.InitPreBuild()\r
self.InitPostBuild()\r
if self.Prebuild:\r
if Process.returncode != 0 :\r
EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')\r
EdkLogger.info("\n- Postbuild Done -\n")\r
+\r
+ ## Error handling for hash feature\r
+ #\r
+ # On BuildTask error, iterate through the Module Build tracking\r
+ # dictionary to determine wheather a module failed to build. Invalidate\r
+ # the hash associated with that module by removing it from storage.\r
+ #\r
+ #\r
+ def invalidateHash(self):\r
+ # GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash\r
+ for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys():\r
+ # False == FAIL : True == Success\r
+ # Skip invalidating for Successful module builds\r
+ if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True:\r
+ continue\r
+\r
+ # The module failed to build or failed to start building, from this point on\r
+\r
+ # Remove .hash from build\r
+ if GlobalData.gUseHashCache:\r
+ ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash")\r
+ if os.path.exists(ModuleHashFile):\r
+ os.remove(ModuleHashFile)\r
+\r
+ # Remove .hash file from cache\r
+ if GlobalData.gBinCacheDest:\r
+ FileDir = path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName)\r
+ HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash')\r
+ if os.path.exists(HashFile):\r
+ os.remove(HashFile)\r
+\r
## Build a module or platform\r
#\r
# Create autogen code and makefile for a module or platform, and the launch\r
LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)\r
LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)\r
#\r
- # Collect funtion address from Map file\r
+ # Collect function address from Map file\r
#\r
ImageMapTable = ModuleOutputImage.replace('.efi', '.map')\r
FunctionList = []\r
# Add general information.\r
#\r
if ModeIsSmm:\r
- MapBuffer.write('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))\r
+ MapBuffer.append('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))\r
elif AddrIsOffset:\r
- MapBuffer.write('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))\r
+ MapBuffer.append('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))\r
else:\r
- MapBuffer.write('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))\r
+ MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))\r
#\r
# Add guid and general seciton section.\r
#\r
elif SectionHeader[0] in ['.data', '.sdata']:\r
DataSectionAddress = SectionHeader[1]\r
if AddrIsOffset:\r
- MapBuffer.write('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))\r
+ MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))\r
else:\r
- MapBuffer.write('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))\r
+ MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))\r
#\r
# Add debug image full path.\r
#\r
- MapBuffer.write('(IMAGE=%s)\n\n' % (ModuleDebugImage))\r
+ MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))\r
#\r
- # Add funtion address\r
+ # Add function address\r
#\r
for Function in FunctionList:\r
if AddrIsOffset:\r
- MapBuffer.write(' -0x%010X %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))\r
+ MapBuffer.append(' -0x%010X %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))\r
else:\r
- MapBuffer.write(' 0x%010X %s\n' % (BaseAddress + Function[1], Function[0]))\r
+ MapBuffer.append(' 0x%010X %s\n' % (BaseAddress + Function[1], Function[0]))\r
ImageMap.close()\r
\r
#\r
GuidString = MatchGuid.group()\r
if GuidString.upper() in ModuleList:\r
Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)\r
- MapBuffer.write(Line)\r
+ MapBuffer.append(Line)\r
#\r
# Add the debug image full path.\r
#\r
if MatchGuid is not None:\r
GuidString = MatchGuid.group().split("=")[1]\r
if GuidString.upper() in ModuleList:\r
- MapBuffer.write('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))\r
+ MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))\r
\r
FvMap.close()\r
\r
if ReturnValue != 0:\r
EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)\r
\r
- MapBuffer.write('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize // 0x1000))\r
- MapBuffer.write('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize // 0x1000))\r
- MapBuffer.write('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize // 0x1000))\r
+ MapBuffer.append('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize // 0x1000))\r
+ MapBuffer.append('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize // 0x1000))\r
+ MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize // 0x1000))\r
if len (SmmModuleList) > 0:\r
- MapBuffer.write('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize // 0x1000))\r
+ MapBuffer.append('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize // 0x1000))\r
\r
PeiBaseAddr = TopMemoryAddress - RtSize - BtSize\r
BtBaseAddr = TopMemoryAddress - RtSize\r
self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)\r
self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)\r
self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)\r
- MapBuffer.write('\n\n')\r
+ MapBuffer.append('\n\n')\r
sys.stdout.write ("\n")\r
sys.stdout.flush()\r
\r
#\r
# Save address map into MAP file.\r
#\r
- SaveFileOnChange(MapFilePath, MapBuffer.getvalue(), False)\r
- MapBuffer.close()\r
+ SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)\r
if self.LoadFixAddress != 0:\r
sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))\r
sys.stdout.flush()\r
# Add ffs build to makefile\r
CmdListDict = {}\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
- CmdListDict = self._GenFfsCmd()\r
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
for Arch in Wa.ArchList:\r
GlobalData.gGlobalDefines['ARCH'] = Arch\r
if not Ma.IsLibrary:\r
ModuleList[Ma.Guid.upper()] = Ma\r
\r
- MapBuffer = BytesIO('')\r
+ MapBuffer = []\r
if self.LoadFixAddress != 0:\r
#\r
# Rebase module to the preferred memory address before GenFds\r
# Add ffs build to makefile\r
CmdListDict = None\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
- CmdListDict = self._GenFfsCmd()\r
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
self.Progress.Stop("done!")\r
MaList = []\r
ExitFlag = threading.Event()\r
for Module in Pa.Platform.Modules:\r
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:\r
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)\r
- if Ma is None: continue\r
+ if Ma is None:\r
+ continue\r
MaList.append(Ma)\r
if Ma.CanSkipbyHash():\r
self.HashSkipModules.append(Ma)\r
if self.Target == "genmake":\r
return True\r
self.BuildModules.append(Ma)\r
+ # Initialize all modules in tracking to False (FAIL)\r
+ if Ma not in GlobalData.gModuleBuildTracking:\r
+ GlobalData.gModuleBuildTracking[Ma] = False\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
MakeStart = time.time()\r
for Ma in self.BuildModules:\r
# we need a full version of makefile for platform\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
+ self.invalidateHash()\r
Pa.CreateMakeFile(False)\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
# Start task scheduler\r
# in case there's an interruption. we need a full version of makefile for platform\r
Pa.CreateMakeFile(False)\r
if BuildTask.HasError():\r
+ self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
self.CreateAsBuiltInf()\r
self.MakeTime += int(round((time.time() - MakeContiue)))\r
if BuildTask.HasError():\r
+ self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
\r
self.BuildReport.AddPlatformReport(Wa, MaList)\r
if not Ma.IsLibrary:\r
ModuleList[Ma.Guid.upper()] = Ma\r
\r
- MapBuffer = BytesIO('')\r
+ MapBuffer = []\r
if self.LoadFixAddress != 0:\r
#\r
# Rebase module to the preferred memory address before GenFds\r
#\r
self._SaveMapFile (MapBuffer, Wa)\r
\r
- def _GenFfsCmd(self):\r
+ def _GenFfsCmd(self,ArchList):\r
# convert dictionary of Cmd:(Inf,Arch)\r
# to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...\r
CmdSetDict = defaultdict(set)\r
- GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, self.ArchList, GlobalData)\r
+ GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)\r
for Cmd in GenFfsDict:\r
tmpInf, tmpArch = GenFfsDict[Cmd]\r
CmdSetDict[tmpInf, tmpArch].add(Cmd)\r
# Add ffs build to makefile\r
CmdListDict = None\r
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:\r
- CmdListDict = self._GenFfsCmd()\r
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)\r
\r
# multi-thread exit flag\r
ExitFlag = threading.Event()\r
if self.Target == "genmake":\r
continue\r
self.BuildModules.append(Ma)\r
+ # Initialize all modules in tracking to False (FAIL)\r
+ if Ma not in GlobalData.gModuleBuildTracking:\r
+ GlobalData.gModuleBuildTracking[Ma] = False\r
self.Progress.Stop("done!")\r
self.AutoGenTime += int(round((time.time() - AutoGenStart)))\r
MakeStart = time.time()\r
# we need a full version of makefile for platform\r
ExitFlag.set()\r
BuildTask.WaitForComplete()\r
+ self.invalidateHash()\r
Pa.CreateMakeFile(False)\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
# Start task scheduler\r
# in case there's an interruption. we need a full version of makefile for platform\r
Pa.CreateMakeFile(False)\r
if BuildTask.HasError():\r
+ self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
self.MakeTime += int(round((time.time() - MakeStart)))\r
\r
# has been signaled.\r
#\r
if BuildTask.HasError():\r
+ self.invalidateHash()\r
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)\r
\r
# Create MAP file when Load Fix Address is enabled.\r
#\r
# Rebase module to the preferred memory address before GenFds\r
#\r
- MapBuffer = BytesIO('')\r
+ MapBuffer = []\r
if self.LoadFixAddress != 0:\r
self._CollectModuleMapBuffer(MapBuffer, ModuleList)\r
\r
RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)\r
\r
def CreateAsBuiltInf(self):\r
+ all_lib_set = set()\r
+ all_mod_set = set()\r
for Module in self.BuildModules:\r
Module.CreateAsBuiltInf()\r
+ all_mod_set.add(Module)\r
for Module in self.HashSkipModules:\r
Module.CreateAsBuiltInf(True)\r
+ all_mod_set.add(Module)\r
+ for Module in all_mod_set:\r
+ for lib in Module.LibraryAutoGenList:\r
+ all_lib_set.add(lib)\r
+ for lib in all_lib_set:\r
+ lib.CreateAsBuiltInf(True)\r
+ all_lib_set.clear()\r
+ all_mod_set.clear()\r
self.BuildModules = []\r
self.HashSkipModules = []\r
## Do some clean-up works when error occurred\r
Parser.add_option("--binary-destination", action="store", type="string", dest="BinCacheDest", help="Generate a cache of binary files in the specified directory.")\r
Parser.add_option("--binary-source", action="store", type="string", dest="BinCacheSource", help="Consume a cache of binary files from the specified directory.")\r
Parser.add_option("--genfds-multi-thread", action="store_true", dest="GenfdsMultiThread", default=False, help="Enable GenFds multi thread to generate ffs file.")\r
+ Parser.add_option("--disable-include-path-check", action="store_true", dest="DisableIncludePathCheck", default=False, help="Disable the include path check for outside of package.")\r
(Opt, Args) = Parser.parse_args()\r
return (Opt, Args)\r
\r
projects / mirror_edk2.git / blobdiff