X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2Fbuild%2Fbuild.py;h=027061191c5eee3aa06867d83008f58a3b00ead8;hp=139a1dfe2983fa2ff037689e3279bab012a54dab;hb=a7ef158b07524f9afd0cefa3180aeac0fcb6e436;hpb=b3e94a06172113991f28a1eff096255c65702a0c
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 139a1dfe29..027061191c 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -5,13 +5,7 @@
# Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.
# Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.
#
-# This program and the accompanying materials
-# are licensed and made available under the terms and conditions of the BSD License
-# which accompanies this distribution. The full text of the license may be found at
-# http://opensource.org/licenses/bsd-license.php
-#
-# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
-# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
#
##
@@ -20,7 +14,6 @@
from __future__ import print_function
import Common.LongFilePathOs as os
import re
-from io import BytesIO
import sys
import glob
import time
@@ -182,8 +175,8 @@ def ReadMessage(From, To, ExitFlag):
# read one line a time
Line = From.readline()
# empty string means "end"
- if Line is not None and Line != "":
- To(Line.rstrip())
+ if Line is not None and Line != b"":
+ To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))
else:
break
if ExitFlag.isSet():
@@ -443,7 +436,7 @@ class BuildTask:
# get all pending tasks
BuildTask._PendingQueueLock.acquire()
- BuildObjectList = BuildTask._PendingQueue.keys()
+ BuildObjectList = list(BuildTask._PendingQueue.keys())
#
# check if their dependency is resolved, and if true, move them
# into ready queue
@@ -489,7 +482,7 @@ class BuildTask:
time.sleep(0.1)
except BaseException as X:
#
- # TRICK: hide the output of threads left runing, so that the user can
+ # TRICK: hide the output of threads left running, so that the user can
# catch the error message easily
#
EdkLogger.SetLevel(EdkLogger.ERROR)
@@ -600,7 +593,7 @@ class BuildTask:
#
def AddDependency(self, Dependency):
for Dep in Dependency:
- if not Dep.BuildObject.IsBinaryModule:
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash():
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list
## The thread wrapper of LaunchCommand function
@@ -612,9 +605,14 @@ class BuildTask:
try:
self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)
self.CompleteFlag = True
+
+ # Run hash operation post dependency, to account for libs
+ if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:
+ HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")
+ SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True)
except:
#
- # TRICK: hide the output of threads left runing, so that the user can
+ # TRICK: hide the output of threads left running, so that the user can
# catch the error message easily
#
if not BuildTask._ErrorFlag.isSet():
@@ -627,6 +625,8 @@ class BuildTask:
BuildTask._ErrorFlag.set()
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \
(threading.currentThread().getName(), Command, WorkingDir)
+ if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet():
+ GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True
# indicate there's a thread is available for another build task
BuildTask._RunningQueueLock.acquire()
BuildTask._RunningQueue.pop(self.BuildItem)
@@ -720,6 +720,7 @@ class Build():
GlobalData.gBinCacheDest = BuildOptions.BinCacheDest
GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
GlobalData.gEnableGenfdsMultiThread = BuildOptions.GenfdsMultiThread
+ GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
@@ -791,6 +792,13 @@ class Build():
# Print the same path style with WORKSPACE env.
EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
+ if "PYTHON3_ENABLE" in os.environ:
+ PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]
+ if PYTHON3_ENABLE != "TRUE":
+ PYTHON3_ENABLE = "FALSE"
+ EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))
+ if "PYTHON_COMMAND" in os.environ:
+ EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))
self.InitPreBuild()
self.InitPostBuild()
if self.Prebuild:
@@ -1137,6 +1145,37 @@ class Build():
if Process.returncode != 0 :
EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
EdkLogger.info("\n- Postbuild Done -\n")
+
+ ## Error handling for hash feature
+ #
+ # On BuildTask error, iterate through the Module Build tracking
+ # dictionary to determine wheather a module failed to build. Invalidate
+ # the hash associated with that module by removing it from storage.
+ #
+ #
+ def invalidateHash(self):
+ # GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash
+ for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys():
+ # False == FAIL : True == Success
+ # Skip invalidating for Successful module builds
+ if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True:
+ continue
+
+ # The module failed to build or failed to start building, from this point on
+
+ # Remove .hash from build
+ if GlobalData.gUseHashCache:
+ ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash")
+ if os.path.exists(ModuleHashFile):
+ os.remove(ModuleHashFile)
+
+ # Remove .hash file from cache
+ if GlobalData.gBinCacheDest:
+ FileDir = path.join(GlobalData.gBinCacheDest, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName)
+ HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash')
+ if os.path.exists(HashFile):
+ os.remove(HashFile)
+
## Build a module or platform
#
# Create autogen code and makefile for a module or platform, and the launch
@@ -1374,7 +1413,7 @@ class Build():
LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
#
- # Collect funtion address from Map file
+ # Collect function address from Map file
#
ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
FunctionList = []
@@ -1403,11 +1442,11 @@ class Build():
# Add general information.
#
if ModeIsSmm:
- MapBuffer.write('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
+ MapBuffer.append('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
elif AddrIsOffset:
- MapBuffer.write('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
+ MapBuffer.append('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
else:
- MapBuffer.write('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
+ MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
#
# Add guid and general seciton section.
#
@@ -1419,21 +1458,21 @@ class Build():
elif SectionHeader[0] in ['.data', '.sdata']:
DataSectionAddress = SectionHeader[1]
if AddrIsOffset:
- MapBuffer.write('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
+ MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
else:
- MapBuffer.write('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
+ MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
#
# Add debug image full path.
#
- MapBuffer.write('(IMAGE=%s)\n\n' % (ModuleDebugImage))
+ MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))
#
- # Add funtion address
+ # Add function address
#
for Function in FunctionList:
if AddrIsOffset:
- MapBuffer.write(' -0x%010X %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
+ MapBuffer.append(' -0x%010X %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
else:
- MapBuffer.write(' 0x%010X %s\n' % (BaseAddress + Function[1], Function[0]))
+ MapBuffer.append(' 0x%010X %s\n' % (BaseAddress + Function[1], Function[0]))
ImageMap.close()
#
@@ -1468,7 +1507,7 @@ class Build():
GuidString = MatchGuid.group()
if GuidString.upper() in ModuleList:
Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
- MapBuffer.write(Line)
+ MapBuffer.append(Line)
#
# Add the debug image full path.
#
@@ -1476,7 +1515,7 @@ class Build():
if MatchGuid is not None:
GuidString = MatchGuid.group().split("=")[1]
if GuidString.upper() in ModuleList:
- MapBuffer.write('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
+ MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
FvMap.close()
@@ -1592,11 +1631,11 @@ class Build():
if ReturnValue != 0:
EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
- MapBuffer.write('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize // 0x1000))
- MapBuffer.write('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize // 0x1000))
- MapBuffer.write('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize // 0x1000))
+ MapBuffer.append('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize // 0x1000))
+ MapBuffer.append('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize // 0x1000))
+ MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize // 0x1000))
if len (SmmModuleList) > 0:
- MapBuffer.write('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize // 0x1000))
+ MapBuffer.append('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize // 0x1000))
PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
BtBaseAddr = TopMemoryAddress - RtSize
@@ -1606,7 +1645,7 @@ class Build():
self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
- MapBuffer.write('\n\n')
+ MapBuffer.append('\n\n')
sys.stdout.write ("\n")
sys.stdout.flush()
@@ -1620,8 +1659,7 @@ class Build():
#
# Save address map into MAP file.
#
- SaveFileOnChange(MapFilePath, MapBuffer.getvalue(), False)
- MapBuffer.close()
+ SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)
if self.LoadFixAddress != 0:
sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
sys.stdout.flush()
@@ -1663,7 +1701,7 @@ class Build():
# Add ffs build to makefile
CmdListDict = {}
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
- CmdListDict = self._GenFfsCmd()
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)
for Arch in Wa.ArchList:
GlobalData.gGlobalDefines['ARCH'] = Arch
@@ -1696,7 +1734,7 @@ class Build():
if not Ma.IsLibrary:
ModuleList[Ma.Guid.upper()] = Ma
- MapBuffer = BytesIO('')
+ MapBuffer = []
if self.LoadFixAddress != 0:
#
# Rebase module to the preferred memory address before GenFds
@@ -1756,7 +1794,7 @@ class Build():
# Add ffs build to makefile
CmdListDict = None
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
- CmdListDict = self._GenFfsCmd()
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)
self.Progress.Stop("done!")
MaList = []
ExitFlag = threading.Event()
@@ -1769,7 +1807,8 @@ class Build():
for Module in Pa.Platform.Modules:
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
- if Ma is None: continue
+ if Ma is None:
+ continue
MaList.append(Ma)
if Ma.CanSkipbyHash():
self.HashSkipModules.append(Ma)
@@ -1794,6 +1833,9 @@ class Build():
if self.Target == "genmake":
return True
self.BuildModules.append(Ma)
+ # Initialize all modules in tracking to False (FAIL)
+ if Ma not in GlobalData.gModuleBuildTracking:
+ GlobalData.gModuleBuildTracking[Ma] = False
self.AutoGenTime += int(round((time.time() - AutoGenStart)))
MakeStart = time.time()
for Ma in self.BuildModules:
@@ -1804,6 +1846,7 @@ class Build():
# we need a full version of makefile for platform
ExitFlag.set()
BuildTask.WaitForComplete()
+ self.invalidateHash()
Pa.CreateMakeFile(False)
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
# Start task scheduler
@@ -1813,6 +1856,7 @@ class Build():
# in case there's an interruption. we need a full version of makefile for platform
Pa.CreateMakeFile(False)
if BuildTask.HasError():
+ self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart)))
@@ -1822,6 +1866,7 @@ class Build():
self.CreateAsBuiltInf()
self.MakeTime += int(round((time.time() - MakeContiue)))
if BuildTask.HasError():
+ self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.BuildReport.AddPlatformReport(Wa, MaList)
@@ -1854,7 +1899,7 @@ class Build():
if not Ma.IsLibrary:
ModuleList[Ma.Guid.upper()] = Ma
- MapBuffer = BytesIO('')
+ MapBuffer = []
if self.LoadFixAddress != 0:
#
# Rebase module to the preferred memory address before GenFds
@@ -1875,11 +1920,11 @@ class Build():
#
self._SaveMapFile (MapBuffer, Wa)
- def _GenFfsCmd(self):
+ def _GenFfsCmd(self,ArchList):
# convert dictionary of Cmd:(Inf,Arch)
# to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
CmdSetDict = defaultdict(set)
- GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, self.ArchList, GlobalData)
+ GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
for Cmd in GenFfsDict:
tmpInf, tmpArch = GenFfsDict[Cmd]
CmdSetDict[tmpInf, tmpArch].add(Cmd)
@@ -1923,7 +1968,7 @@ class Build():
# Add ffs build to makefile
CmdListDict = None
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
- CmdListDict = self._GenFfsCmd()
+ CmdListDict = self._GenFfsCmd(Wa.ArchList)
# multi-thread exit flag
ExitFlag = threading.Event()
@@ -1972,6 +2017,9 @@ class Build():
if self.Target == "genmake":
continue
self.BuildModules.append(Ma)
+ # Initialize all modules in tracking to False (FAIL)
+ if Ma not in GlobalData.gModuleBuildTracking:
+ GlobalData.gModuleBuildTracking[Ma] = False
self.Progress.Stop("done!")
self.AutoGenTime += int(round((time.time() - AutoGenStart)))
MakeStart = time.time()
@@ -1984,6 +2032,7 @@ class Build():
# we need a full version of makefile for platform
ExitFlag.set()
BuildTask.WaitForComplete()
+ self.invalidateHash()
Pa.CreateMakeFile(False)
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
# Start task scheduler
@@ -1993,6 +2042,7 @@ class Build():
# in case there's an interruption. we need a full version of makefile for platform
Pa.CreateMakeFile(False)
if BuildTask.HasError():
+ self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart)))
@@ -2012,6 +2062,7 @@ class Build():
# has been signaled.
#
if BuildTask.HasError():
+ self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
# Create MAP file when Load Fix Address is enabled.
@@ -2035,7 +2086,7 @@ class Build():
#
# Rebase module to the preferred memory address before GenFds
#
- MapBuffer = BytesIO('')
+ MapBuffer = []
if self.LoadFixAddress != 0:
self._CollectModuleMapBuffer(MapBuffer, ModuleList)
@@ -2143,10 +2194,21 @@ class Build():
RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
def CreateAsBuiltInf(self):
+ all_lib_set = set()
+ all_mod_set = set()
for Module in self.BuildModules:
Module.CreateAsBuiltInf()
+ all_mod_set.add(Module)
for Module in self.HashSkipModules:
Module.CreateAsBuiltInf(True)
+ all_mod_set.add(Module)
+ for Module in all_mod_set:
+ for lib in Module.LibraryAutoGenList:
+ all_lib_set.add(lib)
+ for lib in all_lib_set:
+ lib.CreateAsBuiltInf(True)
+ all_lib_set.clear()
+ all_mod_set.clear()
self.BuildModules = []
self.HashSkipModules = []
## Do some clean-up works when error occurred
@@ -2263,6 +2325,7 @@ def MyOptionParser():
Parser.add_option("--binary-destination", action="store", type="string", dest="BinCacheDest", help="Generate a cache of binary files in the specified directory.")
Parser.add_option("--binary-source", action="store", type="string", dest="BinCacheSource", help="Consume a cache of binary files from the specified directory.")
Parser.add_option("--genfds-multi-thread", action="store_true", dest="GenfdsMultiThread", default=False, help="Enable GenFds multi thread to generate ffs file.")
+ Parser.add_option("--disable-include-path-check", action="store_true", dest="DisableIncludePathCheck", default=False, help="Disable the include path check for outside of package.")
(Opt, Args) = Parser.parse_args()
return (Opt, Args)