BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
   5 #  Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.<BR>
   6 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
   7 #  Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
   8 #
   9 #  SPDX-License-Identifier: BSD-2-Clause-Patent
  10 #
  11
  12 ##
  13 # Import Modules
  14 #
  15 from __future__ import print_function
  16 from __future__ import absolute_import
  17 import os.path as path
  18 import sys
  19 import os
  20 import re
  21 import glob
  22 import time
  23 import platform
  24 import traceback
  25 import multiprocessing
  26 from threading import Thread,Event,BoundedSemaphore
  27 import threading
  28 from linecache import getlines
  29 from subprocess import Popen,PIPE, STDOUT
  30 from collections import OrderedDict, defaultdict
  31
  32 from AutoGen.PlatformAutoGen import PlatformAutoGen
  33 from AutoGen.ModuleAutoGen import ModuleAutoGen
  34 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
  35 from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\
  36     LogAgent
  37 from AutoGen import GenMake
  38 from Common import Misc as Utils
  39
  40 from Common.TargetTxtClassObject import TargetTxtDict
  41 from Common.ToolDefClassObject import ToolDefDict
  42 from buildoptions import MyOptionParser
  43 from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory
  44 from Common.StringUtils import NormPath
  45 from Common.MultipleWorkspace import MultipleWorkspace as mws
  46 from Common.BuildToolError import *
  47 from Common.DataType import *
  48 import Common.EdkLogger as EdkLogger
  49
  50 from Workspace.WorkspaceDatabase import BuildDB
  51
  52 from BuildReport import BuildReport
  53 from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile
  54 from PatchPcdValue.PatchPcdValue import PatchBinaryFile
  55
  56 import Common.GlobalData as GlobalData
  57 from GenFds.GenFds import GenFds, GenFdsApi
  58 import multiprocessing as mp
  59 from multiprocessing import Manager
  60 from AutoGen.DataPipe import MemoryDataPipe
  61 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
  62 from GenFds.FdfParser import FdfParser
  63 from AutoGen.IncludesAutoGen import IncludesAutoGen
  64 from GenFds.GenFds import resetFdsGlobalVariable
  65
  66 ## standard targets of build command
  67 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  68
  69 ## build configuration file
  70 gBuildConfiguration = "target.txt"
  71 gToolsDefinition = "tools_def.txt"
  72
  73 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
  74 TmpTableDict = {}
  75
  76 ## Check environment PATH variable to make sure the specified tool is found
  77 #
  78 #   If the tool is found in the PATH, then True is returned
  79 #   Otherwise, False is returned
  80 #
  81 def IsToolInPath(tool):
  82     if 'PATHEXT' in os.environ:
  83         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  84     else:
  85         extns = ('',)
  86     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  87         for ext in extns:
  88             if os.path.exists(os.path.join(pathDir, tool + ext)):
  89                 return True
  90     return False
  91
  92 ## Check environment variables
  93 #
  94 #  Check environment variables that must be set for build. Currently they are
  95 #
  96 #   WORKSPACE           The directory all packages/platforms start from
  97 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  98 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  99 #
 100 #   If any of above environment variable is not set or has error, the build
 101 #   will be broken.
 102 #
 103 def CheckEnvVariable():
 104     # check WORKSPACE
 105     if "WORKSPACE" not in os.environ:
 106         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 107                         ExtraData="WORKSPACE")
 108
 109     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
 110     if not os.path.exists(WorkspaceDir):
 111         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData=WorkspaceDir)
 112     elif ' ' in WorkspaceDir:
 113         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
 114                         ExtraData=WorkspaceDir)
 115     os.environ["WORKSPACE"] = WorkspaceDir
 116
 117     # set multiple workspace
 118     PackagesPath = os.getenv("PACKAGES_PATH")
 119     mws.setWs(WorkspaceDir, PackagesPath)
 120     if mws.PACKAGES_PATH:
 121         for Path in mws.PACKAGES_PATH:
 122             if not os.path.exists(Path):
 123                 EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData=Path)
 124             elif ' ' in Path:
 125                 EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path)
 126
 127
 128     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 129
 130     # check EDK_TOOLS_PATH
 131     if "EDK_TOOLS_PATH" not in os.environ:
 132         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 133                         ExtraData="EDK_TOOLS_PATH")
 134
 135     # check PATH
 136     if "PATH" not in os.environ:
 137         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 138                         ExtraData="PATH")
 139
 140     GlobalData.gWorkspace = WorkspaceDir
 141
 142     GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 143     GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"]
 144
 145 ## Get normalized file path
 146 #
 147 # Convert the path to be local format, and remove the WORKSPACE path at the
 148 # beginning if the file path is given in full path.
 149 #
 150 # @param  FilePath      File path to be normalized
 151 # @param  Workspace     Workspace path which the FilePath will be checked against
 152 #
 153 # @retval string        The normalized file path
 154 #
 155 def NormFile(FilePath, Workspace):
 156     # check if the path is absolute or relative
 157     if os.path.isabs(FilePath):
 158         FileFullPath = os.path.normpath(FilePath)
 159     else:
 160         FileFullPath = os.path.normpath(mws.join(Workspace, FilePath))
 161         Workspace = mws.getWs(Workspace, FilePath)
 162
 163     # check if the file path exists or not
 164     if not os.path.isfile(FileFullPath):
 165         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath)
 166
 167     # remove workspace directory from the beginning part of the file path
 168     if Workspace[-1] in ["\\", "/"]:
 169         return FileFullPath[len(Workspace):]
 170     else:
 171         return FileFullPath[(len(Workspace) + 1):]
 172
 173 ## Get the output of an external program
 174 #
 175 # This is the entrance method of thread reading output of an external program and
 176 # putting them in STDOUT/STDERR of current program.
 177 #
 178 # @param  From      The stream message read from
 179 # @param  To        The stream message put on
 180 # @param  ExitFlag  The flag used to indicate stopping reading
 181 #
 182 def ReadMessage(From, To, ExitFlag,MemTo=None):
 183     while True:
 184         # read one line a time
 185         Line = From.readline()
 186         # empty string means "end"
 187         if Line is not None and Line != b"":
 188             LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
 189             if MemTo is not None:
 190                 if "Note: including file:" ==  LineStr.lstrip()[:21]:
 191                     MemTo.append(LineStr)
 192                 else:
 193                     To(LineStr)
 194                     MemTo.append(LineStr)
 195             else:
 196                 To(LineStr)
 197         else:
 198             break
 199         if ExitFlag.isSet():
 200             break
 201
 202 class MakeSubProc(Popen):
 203     def __init__(self,*args, **argv):
 204         super(MakeSubProc,self).__init__(*args, **argv)
 205         self.ProcOut = []
 206
 207 ## Launch an external program
 208 #
 209 # This method will call subprocess.Popen to execute an external program with
 210 # given options in specified directory. Because of the dead-lock issue during
 211 # redirecting output of the external program, threads are used to to do the
 212 # redirection work.
 213 #
 214 # @param  Command               A list or string containing the call of the program
 215 # @param  WorkingDir            The directory in which the program will be running
 216 #
 217 def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
 218     BeginTime = time.time()
 219     # if working directory doesn't exist, Popen() will raise an exception
 220     if not os.path.isdir(WorkingDir):
 221         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 222
 223     # Command is used as the first Argument in following Popen().
 224     # It could be a string or sequence. We find that if command is a string in following Popen(),
 225     # ubuntu may fail with an error message that the command is not found.
 226     # So here we may need convert command from string to list instance.
 227     if platform.system() != 'Windows':
 228         if not isinstance(Command, list):
 229             Command = Command.split()
 230         Command = ' '.join(Command)
 231
 232     Proc = None
 233     EndOfProcedure = None
 234     try:
 235         # launch the command
 236         Proc = MakeSubProc(Command, stdout=PIPE, stderr=STDOUT, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 237
 238         # launch two threads to read the STDOUT and STDERR
 239         EndOfProcedure = Event()
 240         EndOfProcedure.clear()
 241         if Proc.stdout:
 242             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
 243             StdOutThread.setName("STDOUT-Redirector")
 244             StdOutThread.setDaemon(False)
 245             StdOutThread.start()
 246
 247
 248         # waiting for program exit
 249         Proc.wait()
 250     except: # in case of aborting
 251         # terminate the threads redirecting the program output
 252         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 253         if EndOfProcedure is not None:
 254             EndOfProcedure.set()
 255         if Proc is None:
 256             if not isinstance(Command, type("")):
 257                 Command = " ".join(Command)
 258             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 259
 260     if Proc.stdout:
 261         StdOutThread.join()
 262
 263     # check the return code of the program
 264     if Proc.returncode != 0:
 265         if not isinstance(Command, type("")):
 266             Command = " ".join(Command)
 267         # print out the Response file and its content when make failure
 268         RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
 269         if os.path.isfile(RespFile):
 270             f = open(RespFile)
 271             RespContent = f.read()
 272             f.close()
 273             EdkLogger.info(RespContent)
 274
 275         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 276     if ModuleAuto:
 277         iau = IncludesAutoGen(WorkingDir,ModuleAuto)
 278         if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
 279             iau.CreateDepsFileForMsvc(Proc.ProcOut)
 280         else:
 281             iau.UpdateDepsFileforNonMsvc()
 282         iau.UpdateDepsFileforTrim()
 283         iau.CreateModuleDeps()
 284         iau.CreateDepsInclude()
 285         iau.CreateDepsTarget()
 286     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
 287
 288 ## The smallest unit that can be built in multi-thread build mode
 289 #
 290 # This is the base class of build unit. The "Obj" parameter must provide
 291 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 292 # missing build.
 293 #
 294 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 295 #
 296 class BuildUnit:
 297     ## The constructor
 298     #
 299     #   @param  self        The object pointer
 300     #   @param  Obj         The object the build is working on
 301     #   @param  Target      The build target name, one of gSupportedTarget
 302     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 303     #   @param  WorkingDir  The directory build command starts in
 304     #
 305     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 306         self.BuildObject = Obj
 307         self.Dependency = Dependency
 308         self.WorkingDir = WorkingDir
 309         self.Target = Target
 310         self.BuildCommand = BuildCommand
 311         if not BuildCommand:
 312             EdkLogger.error("build", OPTION_MISSING,
 313                             "No build command found for this module. "
 314                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
 315                                 (Obj.BuildTarget, Obj.ToolChain, Obj.Arch),
 316                             ExtraData=str(Obj))
 317
 318
 319     ## str() method
 320     #
 321     #   It just returns the string representation of self.BuildObject
 322     #
 323     #   @param  self        The object pointer
 324     #
 325     def __str__(self):
 326         return str(self.BuildObject)
 327
 328     ## "==" operator method
 329     #
 330     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 331     #   provide its own __eq__() method.
 332     #
 333     #   @param  self        The object pointer
 334     #   @param  Other       The other BuildUnit object compared to
 335     #
 336     def __eq__(self, Other):
 337         return Other and self.BuildObject == Other.BuildObject \
 338                 and Other.BuildObject \
 339                 and self.BuildObject.Arch == Other.BuildObject.Arch
 340
 341     ## hash() method
 342     #
 343     #   It just returns the hash value of self.BuildObject which must be hashable.
 344     #
 345     #   @param  self        The object pointer
 346     #
 347     def __hash__(self):
 348         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 349
 350     def __repr__(self):
 351         return repr(self.BuildObject)
 352
 353 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 354 #
 355 # This class is for module build by nmake/make build system. The "Obj" parameter
 356 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 357 # be make units missing build.
 358 #
 359 # Currently the "Obj" should be only ModuleAutoGen object.
 360 #
 361 class ModuleMakeUnit(BuildUnit):
 362     ## The constructor
 363     #
 364     #   @param  self        The object pointer
 365     #   @param  Obj         The ModuleAutoGen object the build is working on
 366     #   @param  Target      The build target name, one of gSupportedTarget
 367     #
 368     def __init__(self, Obj, BuildCommand,Target):
 369         Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]
 370         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 371         if Target in [None, "", "all"]:
 372             self.Target = "tbuild"
 373
 374 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 375 #
 376 # This class is for platform build by nmake/make build system. The "Obj" parameter
 377 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 378 # be make units missing build.
 379 #
 380 # Currently the "Obj" should be only PlatformAutoGen object.
 381 #
 382 class PlatformMakeUnit(BuildUnit):
 383     ## The constructor
 384     #
 385     #   @param  self        The object pointer
 386     #   @param  Obj         The PlatformAutoGen object the build is working on
 387     #   @param  Target      The build target name, one of gSupportedTarget
 388     #
 389     def __init__(self, Obj, BuildCommand, Target):
 390         Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 391         Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])
 392         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 393
 394 ## The class representing the task of a module build or platform build
 395 #
 396 # This class manages the build tasks in multi-thread build mode. Its jobs include
 397 # scheduling thread running, catching thread error, monitor the thread status, etc.
 398 #
 399 class BuildTask:
 400     # queue for tasks waiting for schedule
 401     _PendingQueue = OrderedDict()
 402     _PendingQueueLock = threading.Lock()
 403
 404     # queue for tasks ready for running
 405     _ReadyQueue = OrderedDict()
 406     _ReadyQueueLock = threading.Lock()
 407
 408     # queue for run tasks
 409     _RunningQueue = OrderedDict()
 410     _RunningQueueLock = threading.Lock()
 411
 412     # queue containing all build tasks, in case duplicate build
 413     _TaskQueue = OrderedDict()
 414
 415     # flag indicating error occurs in a running thread
 416     _ErrorFlag = threading.Event()
 417     _ErrorFlag.clear()
 418     _ErrorMessage = ""
 419
 420     # BoundedSemaphore object used to control the number of running threads
 421     _Thread = None
 422
 423     # flag indicating if the scheduler is started or not
 424     _SchedulerStopped = threading.Event()
 425     _SchedulerStopped.set()
 426
 427     ## Start the task scheduler thread
 428     #
 429     #   @param  MaxThreadNumber     The maximum thread number
 430     #   @param  ExitFlag            Flag used to end the scheduler
 431     #
 432     @staticmethod
 433     def StartScheduler(MaxThreadNumber, ExitFlag):
 434         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 435         SchedulerThread.setName("Build-Task-Scheduler")
 436         SchedulerThread.setDaemon(False)
 437         SchedulerThread.start()
 438         # wait for the scheduler to be started, especially useful in Linux
 439         while not BuildTask.IsOnGoing():
 440             time.sleep(0.01)
 441
 442     ## Scheduler method
 443     #
 444     #   @param  MaxThreadNumber     The maximum thread number
 445     #   @param  ExitFlag            Flag used to end the scheduler
 446     #
 447     @staticmethod
 448     def Scheduler(MaxThreadNumber, ExitFlag):
 449         BuildTask._SchedulerStopped.clear()
 450         try:
 451             # use BoundedSemaphore to control the maximum running threads
 452             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 453             #
 454             # scheduling loop, which will exits when no pending/ready task and
 455             # indicated to do so, or there's error in running thread
 456             #
 457             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 458                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 459                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 460                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 461
 462                 # get all pending tasks
 463                 BuildTask._PendingQueueLock.acquire()
 464                 BuildObjectList = list(BuildTask._PendingQueue.keys())
 465                 #
 466                 # check if their dependency is resolved, and if true, move them
 467                 # into ready queue
 468                 #
 469                 for BuildObject in BuildObjectList:
 470                     Bt = BuildTask._PendingQueue[BuildObject]
 471                     if Bt.IsReady():
 472                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 473                 BuildTask._PendingQueueLock.release()
 474
 475                 # launch build thread until the maximum number of threads is reached
 476                 while not BuildTask._ErrorFlag.isSet():
 477                     # empty ready queue, do nothing further
 478                     if len(BuildTask._ReadyQueue) == 0:
 479                         break
 480
 481                     # wait for active thread(s) exit
 482                     BuildTask._Thread.acquire(True)
 483
 484                     # start a new build thread
 485                     Bo, Bt = BuildTask._ReadyQueue.popitem()
 486
 487                     # move into running queue
 488                     BuildTask._RunningQueueLock.acquire()
 489                     BuildTask._RunningQueue[Bo] = Bt
 490                     BuildTask._RunningQueueLock.release()
 491
 492                     Bt.Start()
 493                     # avoid tense loop
 494                     time.sleep(0.01)
 495
 496                 # avoid tense loop
 497                 time.sleep(0.01)
 498
 499             # wait for all running threads exit
 500             if BuildTask._ErrorFlag.isSet():
 501                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 502             # while not BuildTask._ErrorFlag.isSet() and \
 503             while len(BuildTask._RunningQueue) > 0:
 504                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 505                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))
 506                 # avoid tense loop
 507                 time.sleep(0.1)
 508         except BaseException as X:
 509             #
 510             # TRICK: hide the output of threads left running, so that the user can
 511             #        catch the error message easily
 512             #
 513             EdkLogger.SetLevel(EdkLogger.ERROR)
 514             BuildTask._ErrorFlag.set()
 515             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 516
 517         BuildTask._PendingQueue.clear()
 518         BuildTask._ReadyQueue.clear()
 519         BuildTask._RunningQueue.clear()
 520         BuildTask._TaskQueue.clear()
 521         BuildTask._SchedulerStopped.set()
 522
 523     ## Wait for all running method exit
 524     #
 525     @staticmethod
 526     def WaitForComplete():
 527         BuildTask._SchedulerStopped.wait()
 528
 529     ## Check if the scheduler is running or not
 530     #
 531     @staticmethod
 532     def IsOnGoing():
 533         return not BuildTask._SchedulerStopped.isSet()
 534
 535     ## Abort the build
 536     @staticmethod
 537     def Abort():
 538         if BuildTask.IsOnGoing():
 539             BuildTask._ErrorFlag.set()
 540             BuildTask.WaitForComplete()
 541
 542     ## Check if there's error in running thread
 543     #
 544     #   Since the main thread cannot catch exceptions in other thread, we have to
 545     #   use threading.Event to communicate this formation to main thread.
 546     #
 547     @staticmethod
 548     def HasError():
 549         return BuildTask._ErrorFlag.isSet()
 550
 551     ## Get error message in running thread
 552     #
 553     #   Since the main thread cannot catch exceptions in other thread, we have to
 554     #   use a static variable to communicate this message to main thread.
 555     #
 556     @staticmethod
 557     def GetErrorMessage():
 558         return BuildTask._ErrorMessage
 559
 560     ## Factory method to create a BuildTask object
 561     #
 562     #   This method will check if a module is building or has been built. And if
 563     #   true, just return the associated BuildTask object in the _TaskQueue. If
 564     #   not, create and return a new BuildTask object. The new BuildTask object
 565     #   will be appended to the _PendingQueue for scheduling later.
 566     #
 567     #   @param  BuildItem       A BuildUnit object representing a build object
 568     #   @param  Dependency      The dependent build object of BuildItem
 569     #
 570     @staticmethod
 571     def New(BuildItem, Dependency=None):
 572         if BuildItem in BuildTask._TaskQueue:
 573             Bt = BuildTask._TaskQueue[BuildItem]
 574             return Bt
 575
 576         Bt = BuildTask()
 577         Bt._Init(BuildItem, Dependency)
 578         BuildTask._TaskQueue[BuildItem] = Bt
 579
 580         BuildTask._PendingQueueLock.acquire()
 581         BuildTask._PendingQueue[BuildItem] = Bt
 582         BuildTask._PendingQueueLock.release()
 583
 584         return Bt
 585
 586     ## The real constructor of BuildTask
 587     #
 588     #   @param  BuildItem       A BuildUnit object representing a build object
 589     #   @param  Dependency      The dependent build object of BuildItem
 590     #
 591     def _Init(self, BuildItem, Dependency=None):
 592         self.BuildItem = BuildItem
 593
 594         self.DependencyList = []
 595         if Dependency is None:
 596             Dependency = BuildItem.Dependency
 597         else:
 598             Dependency.extend(BuildItem.Dependency)
 599         self.AddDependency(Dependency)
 600         # flag indicating build completes, used to avoid unnecessary re-build
 601         self.CompleteFlag = False
 602
 603     ## Check if all dependent build tasks are completed or not
 604     #
 605     def IsReady(self):
 606         ReadyFlag = True
 607         for Dep in self.DependencyList:
 608             if Dep.CompleteFlag == True:
 609                 continue
 610             ReadyFlag = False
 611             break
 612
 613         return ReadyFlag
 614
 615     ## Add dependent build task
 616     #
 617     #   @param  Dependency      The list of dependent build objects
 618     #
 619     def AddDependency(self, Dependency):
 620         for Dep in Dependency:
 621             if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):
 622                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 623
 624     ## The thread wrapper of LaunchCommand function
 625     #
 626     # @param  Command               A list or string contains the call of the command
 627     # @param  WorkingDir            The directory in which the program will be running
 628     #
 629     def _CommandThread(self, Command, WorkingDir):
 630         try:
 631             self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
 632             self.CompleteFlag = True
 633
 634             # Run hash operation post dependency to account for libs
 635             # Run if --hash or --binary-destination
 636             if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
 637                 self.BuildItem.BuildObject.GenModuleHash()
 638             if GlobalData.gBinCacheDest:
 639                 self.BuildItem.BuildObject.GenCMakeHash()
 640
 641         except:
 642             #
 643             # TRICK: hide the output of threads left running, so that the user can
 644             #        catch the error message easily
 645             #
 646             if not BuildTask._ErrorFlag.isSet():
 647                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 648                                                                   self.BuildItem.BuildObject.Arch,
 649                                                                   self.BuildItem.BuildObject.ToolChain,
 650                                                                   self.BuildItem.BuildObject.BuildTarget
 651                                                                  )
 652             EdkLogger.SetLevel(EdkLogger.ERROR)
 653             BuildTask._ErrorFlag.set()
 654             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 655                                       (threading.currentThread().getName(), Command, WorkingDir)
 656
 657         # indicate there's a thread is available for another build task
 658         BuildTask._RunningQueueLock.acquire()
 659         BuildTask._RunningQueue.pop(self.BuildItem)
 660         BuildTask._RunningQueueLock.release()
 661         BuildTask._Thread.release()
 662
 663     ## Start build task thread
 664     #
 665     def Start(self):
 666         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 667         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 668         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 669         self.BuildTread.setName("build thread")
 670         self.BuildTread.setDaemon(False)
 671         self.BuildTread.start()
 672
 673 ## The class contains the information related to EFI image
 674 #
 675 class PeImageInfo():
 676     ## Constructor
 677     #
 678     # Constructor will load all required image information.
 679     #
 680     #   @param  BaseName          The full file path of image.
 681     #   @param  Guid              The GUID for image.
 682     #   @param  Arch              Arch of this image.
 683     #   @param  OutputDir         The output directory for image.
 684     #   @param  DebugDir          The debug directory for image.
 685     #   @param  ImageClass        PeImage Information
 686     #
 687     def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass):
 688         self.BaseName         = BaseName
 689         self.Guid             = Guid
 690         self.Arch             = Arch
 691         self.OutputDir        = OutputDir
 692         self.DebugDir         = DebugDir
 693         self.Image            = ImageClass
 694         self.Image.Size       = (self.Image.Size // 0x1000 + 1) * 0x1000
 695
 696 ## The class implementing the EDK2 build process
 697 #
 698 #   The build process includes:
 699 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 700 #       2. Parse DSC file of active platform
 701 #       3. Parse FDF file if any
 702 #       4. Establish build database, including parse all other files (module, package)
 703 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 704 #       6. Call build command
 705 #
 706 class Build():
 707     ## Constructor
 708     #
 709     # Constructor will load all necessary configurations, parse platform, modules
 710     # and packages and the establish a database for AutoGen.
 711     #
 712     #   @param  Target              The build command target, one of gSupportedTarget
 713     #   @param  WorkspaceDir        The directory of workspace
 714     #   @param  BuildOptions        Build options passed from command line
 715     #
 716     def __init__(self, Target, WorkspaceDir, BuildOptions,log_q):
 717         self.WorkspaceDir   = WorkspaceDir
 718         self.Target         = Target
 719         self.PlatformFile   = BuildOptions.PlatformFile
 720         self.ModuleFile     = BuildOptions.ModuleFile
 721         self.ArchList       = BuildOptions.TargetArch
 722         self.ToolChainList  = BuildOptions.ToolChain
 723         self.BuildTargetList= BuildOptions.BuildTarget
 724         self.Fdf            = BuildOptions.FdfFile
 725         self.FdList         = BuildOptions.RomImage
 726         self.FvList         = BuildOptions.FvImage
 727         self.CapList        = BuildOptions.CapName
 728         self.SilentMode     = BuildOptions.SilentMode
 729         self.ThreadNumber   = 1
 730         self.SkipAutoGen    = BuildOptions.SkipAutoGen
 731         self.Reparse        = BuildOptions.Reparse
 732         self.SkuId          = BuildOptions.SkuId
 733         if self.SkuId:
 734             GlobalData.gSKUID_CMD = self.SkuId
 735         self.ConfDirectory = BuildOptions.ConfDirectory
 736         self.SpawnMode      = True
 737         self.BuildReport    = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)
 738         self.AutoGenTime    = 0
 739         self.MakeTime       = 0
 740         self.GenFdsTime     = 0
 741         self.MakeFileName   = ""
 742         TargetObj = TargetTxtDict()
 743         ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"),"Conf")))
 744         self.TargetTxt = TargetObj.Target
 745         self.ToolDef = ToolDefObj.ToolDef
 746         GlobalData.BuildOptionPcd     = BuildOptions.OptionPcd if BuildOptions.OptionPcd else []
 747         #Set global flag for build mode
 748         GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
 749         GlobalData.gUseHashCache = BuildOptions.UseHashCache
 750         GlobalData.gBinCacheDest   = BuildOptions.BinCacheDest
 751         GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
 752         GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread
 753         GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
 754
 755         if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
 756             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
 757
 758         if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
 759             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.")
 760
 761         if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
 762             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.")
 763
 764         if GlobalData.gBinCacheSource:
 765             BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
 766             if not os.path.isabs(BinCacheSource):
 767                 BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
 768             GlobalData.gBinCacheSource = BinCacheSource
 769         else:
 770             if GlobalData.gBinCacheSource is not None:
 771                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-source.")
 772
 773         if GlobalData.gBinCacheDest:
 774             BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
 775             if not os.path.isabs(BinCacheDest):
 776                 BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
 777             GlobalData.gBinCacheDest = BinCacheDest
 778         else:
 779             if GlobalData.gBinCacheDest is not None:
 780                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-destination.")
 781
 782         GlobalData.gDatabasePath = os.path.normpath(os.path.join(GlobalData.gConfDirectory, GlobalData.gDatabasePath))
 783         if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):
 784             os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))
 785         self.Db = BuildDB
 786         self.BuildDatabase = self.Db.BuildObject
 787         self.Platform = None
 788         self.ToolChainFamily = None
 789         self.LoadFixAddress = 0
 790         self.UniFlag        = BuildOptions.Flag
 791         self.BuildModules = []
 792         self.HashSkipModules = []
 793         self.Db_Flag = False
 794         self.LaunchPrebuildFlag = False
 795         self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory, '.cache', '.PlatformBuild')
 796         if BuildOptions.CommandLength:
 797             GlobalData.gCommandMaxLength = BuildOptions.CommandLength
 798
 799         # print dot character during doing some time-consuming work
 800         self.Progress = Utils.Progressor()
 801         # print current build environment and configuration
 802         EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 803         if "PACKAGES_PATH" in os.environ:
 804             # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
 805             EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"]))))
 806         EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 807         if "EDK_TOOLS_BIN" in os.environ:
 808             # Print the same path style with WORKSPACE env.
 809             EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
 810         EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
 811         if "PYTHON3_ENABLE" in os.environ:
 812             PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]
 813             if PYTHON3_ENABLE != "TRUE":
 814                 PYTHON3_ENABLE = "FALSE"
 815             EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))
 816         if "PYTHON_COMMAND" in os.environ:
 817             EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))
 818         self.InitPreBuild()
 819         self.InitPostBuild()
 820         if self.Prebuild:
 821             EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.Prebuild))
 822         if self.Postbuild:
 823             EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))
 824         if self.Prebuild:
 825             self.LaunchPrebuild()
 826             TargetObj = TargetTxtDict()
 827             ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"), "Conf")))
 828             self.TargetTxt = TargetObj.Target
 829             self.ToolDef = ToolDefObj.ToolDef
 830         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
 831             self.InitBuild()
 832
 833         self.AutoGenMgr = None
 834         EdkLogger.info("")
 835         os.chdir(self.WorkspaceDir)
 836         self.log_q = log_q
 837         GlobalData.file_lock =  mp.Lock()
 838         # Init cache data for local only
 839         GlobalData.gPackageHashFile = dict()
 840         GlobalData.gModulePreMakeCacheStatus = dict()
 841         GlobalData.gModuleMakeCacheStatus = dict()
 842         GlobalData.gHashChainStatus = dict()
 843         GlobalData.gCMakeHashFile = dict()
 844         GlobalData.gModuleHashFile = dict()
 845         GlobalData.gFileHashDict = dict()
 846         GlobalData.gModuleAllCacheStatus = set()
 847         GlobalData.gModuleCacheHit = set()
 848
 849     def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):
 850         try:
 851             if SkipAutoGen:
 852                 return True,0
 853             feedback_q = mp.Queue()
 854             error_event = mp.Event()
 855             FfsCmd = DataPipe.Get("FfsCommand")
 856             if FfsCmd is None:
 857                 FfsCmd = {}
 858             GlobalData.FfsCmd = FfsCmd
 859             auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]
 860             self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
 861             self.AutoGenMgr.start()
 862             for w in auto_workers:
 863                 w.start()
 864             if PcdMaList is not None:
 865                 for PcdMa in PcdMaList:
 866                     # SourceFileList calling sequence impact the makefile string sequence.
 867                     # Create cached SourceFileList here to unify its calling sequence for both
 868                     # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
 869                     RetVal = PcdMa.SourceFileList
 870                     # Force cache miss for PCD driver
 871                     if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
 872                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))
 873
 874                     PcdMa.CreateCodeFile(False)
 875                     PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
 876
 877                     # Force cache miss for PCD driver
 878                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
 879                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))
 880
 881             self.AutoGenMgr.join()
 882             rt = self.AutoGenMgr.Status
 883             err = 0
 884             if not rt:
 885                 err = UNKNOWN_ERROR
 886             return rt, err
 887         except FatalError as e:
 888             return False, e.args[0]
 889         except:
 890             return False, UNKNOWN_ERROR
 891
 892     ## Load configuration
 893     #
 894     #   This method will parse target.txt and get the build configurations.
 895     #
 896     def LoadConfiguration(self):
 897
 898         # if no ARCH given in command line, get it from target.txt
 899         if not self.ArchList:
 900             self.ArchList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET_ARCH]
 901         self.ArchList = tuple(self.ArchList)
 902
 903         # if no build target given in command line, get it from target.txt
 904         if not self.BuildTargetList:
 905             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
 906
 907         # if no tool chain given in command line, get it from target.txt
 908         if not self.ToolChainList:
 909             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 910             if self.ToolChainList is None or len(self.ToolChainList) == 0:
 911                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 912
 913         # check if the tool chains are defined or not
 914         NewToolChainList = []
 915         for ToolChain in self.ToolChainList:
 916             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 917                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 918             else:
 919                 NewToolChainList.append(ToolChain)
 920         # if no tool chain available, break the build
 921         if len(NewToolChainList) == 0:
 922             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 923                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 924         else:
 925             self.ToolChainList = NewToolChainList
 926
 927         ToolChainFamily = []
 928         ToolDefinition = self.ToolDef.ToolsDefTxtDatabase
 929         for Tool in self.ToolChainList:
 930             if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \
 931                or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]:
 932                 EdkLogger.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool)
 933                 ToolChainFamily.append(TAB_COMPILER_MSFT)
 934             else:
 935                 ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
 936         self.ToolChainFamily = ToolChainFamily
 937
 938         if not self.PlatformFile:
 939             PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 940             if not PlatformFile:
 941                 # Try to find one in current directory
 942                 WorkingDirectory = os.getcwd()
 943                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 944                 FileNum = len(FileList)
 945                 if FileNum >= 2:
 946                     EdkLogger.error("build", OPTION_MISSING,
 947                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 948                 elif FileNum == 1:
 949                     PlatformFile = FileList[0]
 950                 else:
 951                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 952                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 953
 954             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 955         self.ThreadNumber   = ThreadNum()
 956     ## Initialize build configuration
 957     #
 958     #   This method will parse DSC file and merge the configurations from
 959     #   command line and target.txt, then get the final build configurations.
 960     #
 961     def InitBuild(self):
 962         # parse target.txt, tools_def.txt, and platform file
 963         self.LoadConfiguration()
 964
 965         # Allow case-insensitive for those from command line or configuration file
 966         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 967         if ErrorCode != 0:
 968             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 969
 970
 971     def InitPreBuild(self):
 972         self.LoadConfiguration()
 973         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 974         if ErrorCode != 0:
 975             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 976         if self.BuildTargetList:
 977             GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0]
 978         if self.ArchList:
 979             GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0]
 980         if self.ToolChainList:
 981             GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0]
 982             GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0]
 983         if self.ToolChainFamily:
 984             GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
 985         if 'PREBUILD' in GlobalData.gCommandLineDefines:
 986             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
 987         else:
 988             self.Db_Flag = True
 989             Platform = self.Db.MapPlatform(str(self.PlatformFile))
 990             self.Prebuild = str(Platform.Prebuild)
 991         if self.Prebuild:
 992             PrebuildList = []
 993             #
 994             # Evaluate all arguments and convert arguments that are WORKSPACE
 995             # relative paths to absolute paths.  Filter arguments that look like
 996             # flags or do not follow the file/dir naming rules to avoid false
 997             # positives on this conversion.
 998             #
 999             for Arg in self.Prebuild.split():
1000                 #
1001                 # Do not modify Arg if it looks like a flag or an absolute file path
1002                 #
1003                 if Arg.startswith('-')  or os.path.isabs(Arg):
1004                     PrebuildList.append(Arg)
1005                     continue
1006                 #
1007                 # Do not modify Arg if it does not look like a Workspace relative
1008                 # path that starts with a valid package directory name
1009                 #
1010                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1011                     PrebuildList.append(Arg)
1012                     continue
1013                 #
1014                 # If Arg looks like a WORKSPACE relative path, then convert to an
1015                 # absolute path and check to see if the file exists.
1016                 #
1017                 Temp = mws.join(self.WorkspaceDir, Arg)
1018                 if os.path.isfile(Temp):
1019                     Arg = Temp
1020                 PrebuildList.append(Arg)
1021             self.Prebuild       = ' '.join(PrebuildList)
1022             self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1023
1024     def InitPostBuild(self):
1025         if 'POSTBUILD' in GlobalData.gCommandLineDefines:
1026             self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD')
1027         else:
1028             Platform = self.Db.MapPlatform(str(self.PlatformFile))
1029             self.Postbuild = str(Platform.Postbuild)
1030         if self.Postbuild:
1031             PostbuildList = []
1032             #
1033             # Evaluate all arguments and convert arguments that are WORKSPACE
1034             # relative paths to absolute paths.  Filter arguments that look like
1035             # flags or do not follow the file/dir naming rules to avoid false
1036             # positives on this conversion.
1037             #
1038             for Arg in self.Postbuild.split():
1039                 #
1040                 # Do not modify Arg if it looks like a flag or an absolute file path
1041                 #
1042                 if Arg.startswith('-')  or os.path.isabs(Arg):
1043                     PostbuildList.append(Arg)
1044                     continue
1045                 #
1046                 # Do not modify Arg if it does not look like a Workspace relative
1047                 # path that starts with a valid package directory name
1048                 #
1049                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1050                     PostbuildList.append(Arg)
1051                     continue
1052                 #
1053                 # If Arg looks like a WORKSPACE relative path, then convert to an
1054                 # absolute path and check to see if the file exists.
1055                 #
1056                 Temp = mws.join(self.WorkspaceDir, Arg)
1057                 if os.path.isfile(Temp):
1058                     Arg = Temp
1059                 PostbuildList.append(Arg)
1060             self.Postbuild       = ' '.join(PostbuildList)
1061             self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1062
1063     def PassCommandOption(self, BuildTarget, TargetArch, ToolChain, PlatformFile, Target):
1064         BuildStr = ''
1065         if GlobalData.gCommand and isinstance(GlobalData.gCommand, list):
1066             BuildStr += ' ' + ' '.join(GlobalData.gCommand)
1067         TargetFlag = False
1068         ArchFlag = False
1069         ToolChainFlag = False
1070         PlatformFileFlag = False
1071
1072         if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget:
1073             TargetFlag = True
1074         if GlobalData.gOptions and not GlobalData.gOptions.TargetArch:
1075             ArchFlag = True
1076         if GlobalData.gOptions and not GlobalData.gOptions.ToolChain:
1077             ToolChainFlag = True
1078         if GlobalData.gOptions and not GlobalData.gOptions.PlatformFile:
1079             PlatformFileFlag = True
1080
1081         if TargetFlag and BuildTarget:
1082             if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple):
1083                 BuildStr += ' -b ' + ' -b '.join(BuildTarget)
1084             elif isinstance(BuildTarget, str):
1085                 BuildStr += ' -b ' + BuildTarget
1086         if ArchFlag and TargetArch:
1087             if isinstance(TargetArch, list) or isinstance(TargetArch, tuple):
1088                 BuildStr += ' -a ' + ' -a '.join(TargetArch)
1089             elif isinstance(TargetArch, str):
1090                 BuildStr += ' -a ' + TargetArch
1091         if ToolChainFlag and ToolChain:
1092             if isinstance(ToolChain, list) or isinstance(ToolChain, tuple):
1093                 BuildStr += ' -t ' + ' -t '.join(ToolChain)
1094             elif isinstance(ToolChain, str):
1095                 BuildStr += ' -t ' + ToolChain
1096         if PlatformFileFlag and PlatformFile:
1097             if isinstance(PlatformFile, list) or isinstance(PlatformFile, tuple):
1098                 BuildStr += ' -p ' + ' -p '.join(PlatformFile)
1099             elif isinstance(PlatformFile, str):
1100                 BuildStr += ' -p' + PlatformFile
1101         BuildStr += ' --conf=' + GlobalData.gConfDirectory
1102         if Target:
1103             BuildStr += ' ' + Target
1104
1105         return BuildStr
1106
1107     def LaunchPrebuild(self):
1108         if self.Prebuild:
1109             EdkLogger.info("\n- Prebuild Start -\n")
1110             self.LaunchPrebuildFlag = True
1111             #
1112             # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1113             # and preserve them for the rest of the main build step, because the child process environment will
1114             # evaporate as soon as it exits, we cannot get it in build step.
1115             #
1116             PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory, '.cache', '.PrebuildEnv')
1117             if os.path.isfile(PrebuildEnvFile):
1118                 os.remove(PrebuildEnvFile)
1119             if os.path.isfile(self.PlatformBuildPath):
1120                 os.remove(self.PlatformBuildPath)
1121             if sys.platform == "win32":
1122                 args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile))
1123                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1124             else:
1125                 args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile))
1126                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1127
1128             # launch two threads to read the STDOUT and STDERR
1129             EndOfProcedure = Event()
1130             EndOfProcedure.clear()
1131             if Process.stdout:
1132                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1133                 StdOutThread.setName("STDOUT-Redirector")
1134                 StdOutThread.setDaemon(False)
1135                 StdOutThread.start()
1136
1137             if Process.stderr:
1138                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1139                 StdErrThread.setName("STDERR-Redirector")
1140                 StdErrThread.setDaemon(False)
1141                 StdErrThread.start()
1142             # waiting for program exit
1143             Process.wait()
1144
1145             if Process.stdout:
1146                 StdOutThread.join()
1147             if Process.stderr:
1148                 StdErrThread.join()
1149             if Process.returncode != 0 :
1150                 EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!')
1151
1152             if os.path.exists(PrebuildEnvFile):
1153                 f = open(PrebuildEnvFile)
1154                 envs = f.readlines()
1155                 f.close()
1156                 envs = [l.split("=", 1) for l in envs ]
1157                 envs = [[I.strip() for I in item] for item in envs if len(item) == 2]
1158                 os.environ.update(dict(envs))
1159             EdkLogger.info("\n- Prebuild Done -\n")
1160
1161     def LaunchPostbuild(self):
1162         if self.Postbuild:
1163             EdkLogger.info("\n- Postbuild Start -\n")
1164             if sys.platform == "win32":
1165                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1166             else:
1167                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1168             # launch two threads to read the STDOUT and STDERR
1169             EndOfProcedure = Event()
1170             EndOfProcedure.clear()
1171             if Process.stdout:
1172                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1173                 StdOutThread.setName("STDOUT-Redirector")
1174                 StdOutThread.setDaemon(False)
1175                 StdOutThread.start()
1176
1177             if Process.stderr:
1178                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1179                 StdErrThread.setName("STDERR-Redirector")
1180                 StdErrThread.setDaemon(False)
1181                 StdErrThread.start()
1182             # waiting for program exit
1183             Process.wait()
1184
1185             if Process.stdout:
1186                 StdOutThread.join()
1187             if Process.stderr:
1188                 StdErrThread.join()
1189             if Process.returncode != 0 :
1190                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
1191             EdkLogger.info("\n- Postbuild Done -\n")
1192
1193     ## Build a module or platform
1194     #
1195     # Create autogen code and makefile for a module or platform, and the launch
1196     # "make" command to build it
1197     #
1198     #   @param  Target                      The target of build command
1199     #   @param  Platform                    The platform file
1200     #   @param  Module                      The module file
1201     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1202     #   @param  ToolChain                   The name of toolchain to build
1203     #   @param  Arch                        The arch of the module/platform
1204     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1205     #                                       for dependent modules/Libraries
1206     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1207     #                                       for dependent modules/Libraries
1208     #
1209     def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):
1210         if AutoGenObject is None:
1211             return False
1212         if FfsCommand is None:
1213             FfsCommand = {}
1214         # skip file generation for cleanxxx targets, run and fds target
1215         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1216             # for target which must generate AutoGen code and makefile
1217             mqueue = mp.Queue()
1218             for m in AutoGenObject.GetAllModuleInfo:
1219                 mqueue.put(m)
1220
1221             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
1222             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
1223             AutoGenObject.CreateLibModuelDirs()
1224             AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
1225             AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
1226             AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
1227             self.Progress.Start("Generating makefile and code")
1228             data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
1229             AutoGenObject.DataPipe.dump(data_pipe_file)
1230             cqueue = mp.Queue()
1231             autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
1232             AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
1233             with open(AutoGenIdFile,"w") as fw:
1234                 fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
1235                 fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
1236                 fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
1237             self.Progress.Stop("done!")
1238             if not autogen_rt:
1239                 self.AutoGenMgr.TerminateWorkers()
1240                 self.AutoGenMgr.join(1)
1241                 raise FatalError(errorcode)
1242             AutoGenObject.CreateCodeFile(False)
1243             AutoGenObject.CreateMakeFile(False)
1244         else:
1245             # always recreate top/platform makefile when clean, just in case of inconsistency
1246             AutoGenObject.CreateCodeFile(True)
1247             AutoGenObject.CreateMakeFile(True)
1248
1249         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1250             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1251
1252         BuildCommand = AutoGenObject.BuildCommand
1253         if BuildCommand is None or len(BuildCommand) == 0:
1254             EdkLogger.error("build", OPTION_MISSING,
1255                             "No build command found for this module. "
1256                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1257                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1258                             ExtraData=str(AutoGenObject))
1259
1260         # run
1261         if Target == 'run':
1262             return True
1263
1264         # build modules
1265         if BuildModule:
1266             BuildCommand = BuildCommand + [Target]
1267             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1268             self.CreateAsBuiltInf()
1269             if GlobalData.gBinCacheDest:
1270                 self.GenDestCache()
1271             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1272                 # Only for --hash
1273                 # Update PreMakeCacheChain files
1274                 self.GenLocalPreMakeCache()
1275             self.BuildModules = []
1276             return True
1277
1278         # build library
1279         if Target == 'libraries':
1280             DirList = []
1281             for Lib in AutoGenObject.LibraryAutoGenList:
1282                 if not Lib.IsBinaryModule:
1283                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1284             for Lib, LibAutoGen in DirList:
1285                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']
1286                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1287             return True
1288
1289         # build module
1290         if Target == 'modules':
1291             DirList = []
1292             for Lib in AutoGenObject.LibraryAutoGenList:
1293                 if not Lib.IsBinaryModule:
1294                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1295             for Lib, LibAutoGen in DirList:
1296                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']
1297                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1298
1299             DirList = []
1300             for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
1301                 if not ModuleAutoGen.IsBinaryModule:
1302                     DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
1303             for Mod,ModAutoGen in DirList:
1304                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, self.MakeFileName)), 'pbuild']
1305                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
1306             self.CreateAsBuiltInf()
1307             if GlobalData.gBinCacheDest:
1308                 self.GenDestCache()
1309             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1310                 # Only for --hash
1311                 # Update PreMakeCacheChain files
1312                 self.GenLocalPreMakeCache()
1313             self.BuildModules = []
1314             return True
1315
1316         # cleanlib
1317         if Target == 'cleanlib':
1318             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1319                 LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))
1320                 if os.path.exists(LibMakefile):
1321                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1322                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1323             return True
1324
1325         # clean
1326         if Target == 'clean':
1327             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1328                 ModMakefile = os.path.normpath(os.path.join(Mod, self.MakeFileName))
1329                 if os.path.exists(ModMakefile):
1330                     NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']
1331                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1332             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1333                 LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))
1334                 if os.path.exists(LibMakefile):
1335                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1336                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1337             return True
1338
1339         # cleanall
1340         if Target == 'cleanall':
1341             try:
1342                 #os.rmdir(AutoGenObject.BuildDir)
1343                 RemoveDirectory(AutoGenObject.BuildDir, True)
1344             except WindowsError as X:
1345                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1346         return True
1347
1348     ## Build a module or platform
1349     #
1350     # Create autogen code and makefile for a module or platform, and the launch
1351     # "make" command to build it
1352     #
1353     #   @param  Target                      The target of build command
1354     #   @param  Platform                    The platform file
1355     #   @param  Module                      The module file
1356     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1357     #   @param  ToolChain                   The name of toolchain to build
1358     #   @param  Arch                        The arch of the module/platform
1359     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1360     #                                       for dependent modules/Libraries
1361     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1362     #                                       for dependent modules/Libraries
1363     #
1364     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1365         if AutoGenObject is None:
1366             return False
1367
1368         # skip file generation for cleanxxx targets, run and fds target
1369         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1370             # for target which must generate AutoGen code and makefile
1371             if not self.SkipAutoGen or Target == 'genc':
1372                 self.Progress.Start("Generating code")
1373                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1374                 self.Progress.Stop("done!")
1375             if Target == "genc":
1376                 return True
1377
1378             if not self.SkipAutoGen or Target == 'genmake':
1379                 self.Progress.Start("Generating makefile")
1380                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1381                 #AutoGenObject.CreateAsBuiltInf()
1382                 self.Progress.Stop("done!")
1383             if Target == "genmake":
1384                 return True
1385         else:
1386             # always recreate top/platform makefile when clean, just in case of inconsistency
1387             AutoGenObject.CreateCodeFile(True)
1388             AutoGenObject.CreateMakeFile(True)
1389
1390         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1391             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1392
1393         BuildCommand = AutoGenObject.BuildCommand
1394         if BuildCommand is None or len(BuildCommand) == 0:
1395             EdkLogger.error("build", OPTION_MISSING,
1396                             "No build command found for this module. "
1397                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1398                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1399                             ExtraData=str(AutoGenObject))
1400
1401         # build modules
1402         if BuildModule:
1403             if Target != 'fds':
1404                 BuildCommand = BuildCommand + [Target]
1405             AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1406             self.CreateAsBuiltInf()
1407             if GlobalData.gBinCacheDest:
1408                 self.GenDestCache()
1409             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1410                 # Only for --hash
1411                 # Update PreMakeCacheChain files
1412                 self.GenLocalPreMakeCache()
1413             self.BuildModules = []
1414             return True
1415
1416         # genfds
1417         if Target == 'fds':
1418             if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db):
1419                 EdkLogger.error("build", COMMAND_FAILURE)
1420             Threshold = self.GetFreeSizeThreshold()
1421             if Threshold:
1422                 self.CheckFreeSizeThreshold(Threshold, AutoGenObject.FvDir)
1423             return True
1424
1425         # run
1426         if Target == 'run':
1427             return True
1428
1429         # build library
1430         if Target == 'libraries':
1431             pass
1432
1433         # not build modules
1434
1435
1436         # cleanall
1437         if Target == 'cleanall':
1438             try:
1439                 #os.rmdir(AutoGenObject.BuildDir)
1440                 RemoveDirectory(AutoGenObject.BuildDir, True)
1441             except WindowsError as X:
1442                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1443         return True
1444
1445     ## Rebase module image and Get function address for the input module list.
1446     #
1447     def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False):
1448         if ModeIsSmm:
1449             AddrIsOffset = False
1450         for InfFile in ModuleList:
1451             sys.stdout.write (".")
1452             sys.stdout.flush()
1453             ModuleInfo = ModuleList[InfFile]
1454             ModuleName = ModuleInfo.BaseName
1455             ModuleOutputImage = ModuleInfo.Image.FileName
1456             ModuleDebugImage  = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi')
1457             ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1458             if not ModeIsSmm:
1459                 BaseAddress = BaseAddress - ModuleInfo.Image.Size
1460                 #
1461                 # Update Image to new BaseAddress by GenFw tool
1462                 #
1463                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1464                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1465             else:
1466                 #
1467                 # Set new address to the section header only for SMM driver.
1468                 #
1469                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1470                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1471             #
1472             # Collect function address from Map file
1473             #
1474             ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
1475             FunctionList = []
1476             if os.path.exists(ImageMapTable):
1477                 OrigImageBaseAddress = 0
1478                 ImageMap = open(ImageMapTable, 'r')
1479                 for LinStr in ImageMap:
1480                     if len (LinStr.strip()) == 0:
1481                         continue
1482                     #
1483                     # Get the preferred address set on link time.
1484                     #
1485                     if LinStr.find ('Preferred load address is') != -1:
1486                         StrList = LinStr.split()
1487                         OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16)
1488
1489                     StrList = LinStr.split()
1490                     if len (StrList) > 4:
1491                         if StrList[3] == 'f' or StrList[3] == 'F':
1492                             Name = StrList[1]
1493                             RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress
1494                             FunctionList.append ((Name, RelativeAddress))
1495
1496                 ImageMap.close()
1497             #
1498             # Add general information.
1499             #
1500             if ModeIsSmm:
1501                 MapBuffer.append('\n\n%s (Fixed SMRAM Offset,   BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1502             elif AddrIsOffset:
1503                 MapBuffer.append('\n\n%s (Fixed Memory Offset,  BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
1504             else:
1505                 MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1506             #
1507             # Add guid and general seciton section.
1508             #
1509             TextSectionAddress = 0
1510             DataSectionAddress = 0
1511             for SectionHeader in ModuleInfo.Image.SectionHeaderList:
1512                 if SectionHeader[0] == '.text':
1513                     TextSectionAddress = SectionHeader[1]
1514                 elif SectionHeader[0] in ['.data', '.sdata']:
1515                     DataSectionAddress = SectionHeader[1]
1516             if AddrIsOffset:
1517                 MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
1518             else:
1519                 MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
1520             #
1521             # Add debug image full path.
1522             #
1523             MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))
1524             #
1525             # Add function address
1526             #
1527             for Function in FunctionList:
1528                 if AddrIsOffset:
1529                     MapBuffer.append('  -0x%010X    %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
1530                 else:
1531                     MapBuffer.append('  0x%010X    %s\n' % (BaseAddress + Function[1], Function[0]))
1532             ImageMap.close()
1533
1534             #
1535             # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1536             #
1537             if ModeIsSmm:
1538                 BaseAddress = BaseAddress + ModuleInfo.Image.Size
1539
1540     ## Collect MAP information of all FVs
1541     #
1542     def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList):
1543         if self.Fdf:
1544             # First get the XIP base address for FV map file.
1545             GuidPattern = re.compile("[-a-fA-F0-9]+")
1546             GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+")
1547             for FvName in Wa.FdfProfile.FvDict:
1548                 FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map')
1549                 if not os.path.exists(FvMapBuffer):
1550                     continue
1551                 FvMap = open(FvMapBuffer, 'r')
1552                 #skip FV size information
1553                 FvMap.readline()
1554                 FvMap.readline()
1555                 FvMap.readline()
1556                 FvMap.readline()
1557                 for Line in FvMap:
1558                     MatchGuid = GuidPattern.match(Line)
1559                     if MatchGuid is not None:
1560                         #
1561                         # Replace GUID with module name
1562                         #
1563                         GuidString = MatchGuid.group()
1564                         if GuidString.upper() in ModuleList:
1565                             Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
1566                     MapBuffer.append(Line)
1567                     #
1568                     # Add the debug image full path.
1569                     #
1570                     MatchGuid = GuidName.match(Line)
1571                     if MatchGuid is not None:
1572                         GuidString = MatchGuid.group().split("=")[1]
1573                         if GuidString.upper() in ModuleList:
1574                             MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
1575
1576                 FvMap.close()
1577
1578     ## Collect MAP information of all modules
1579     #
1580     def _CollectModuleMapBuffer (self, MapBuffer, ModuleList):
1581         sys.stdout.write ("Generate Load Module At Fix Address Map")
1582         sys.stdout.flush()
1583         PatchEfiImageList = []
1584         PeiModuleList  = {}
1585         BtModuleList   = {}
1586         RtModuleList   = {}
1587         SmmModuleList  = {}
1588         PeiSize = 0
1589         BtSize  = 0
1590         RtSize  = 0
1591         # reserve 4K size in SMRAM to make SMM module address not from 0.
1592         SmmSize = 0x1000
1593         for ModuleGuid in ModuleList:
1594             Module = ModuleList[ModuleGuid]
1595             GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget)
1596
1597             OutputImageFile = ''
1598             for ResultFile in Module.CodaTargetList:
1599                 if str(ResultFile.Target).endswith('.efi'):
1600                     #
1601                     # module list for PEI, DXE, RUNTIME and SMM
1602                     #
1603                     OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi')
1604                     ImageClass = PeImageClass (OutputImageFile)
1605                     if not ImageClass.IsValid:
1606                         EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo)
1607                     ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass)
1608                     if Module.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_PEIM, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER, EDK_COMPONENT_TYPE_PIC_PEIM, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM, SUP_MODULE_DXE_CORE]:
1609                         PeiModuleList[Module.MetaFile] = ImageInfo
1610                         PeiSize += ImageInfo.Image.Size
1611                     elif Module.ModuleType in [EDK_COMPONENT_TYPE_BS_DRIVER, SUP_MODULE_DXE_DRIVER, SUP_MODULE_UEFI_DRIVER]:
1612                         BtModuleList[Module.MetaFile] = ImageInfo
1613                         BtSize += ImageInfo.Image.Size
1614                     elif Module.ModuleType in [SUP_MODULE_DXE_RUNTIME_DRIVER, EDK_COMPONENT_TYPE_RT_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, EDK_COMPONENT_TYPE_SAL_RT_DRIVER]:
1615                         RtModuleList[Module.MetaFile] = ImageInfo
1616                         RtSize += ImageInfo.Image.Size
1617                     elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]:
1618                         SmmModuleList[Module.MetaFile] = ImageInfo
1619                         SmmSize += ImageInfo.Image.Size
1620                         if Module.ModuleType == SUP_MODULE_DXE_SMM_DRIVER:
1621                             PiSpecVersion = Module.Module.Specification.get('PI_SPECIFICATION_VERSION', '0x00000000')
1622                             # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1623                             if int(PiSpecVersion, 16) < 0x0001000A:
1624                                 BtModuleList[Module.MetaFile] = ImageInfo
1625                                 BtSize += ImageInfo.Image.Size
1626                     break
1627             #
1628             # EFI image is final target.
1629             # Check EFI image contains patchable FixAddress related PCDs.
1630             #
1631             if OutputImageFile != '':
1632                 ModuleIsPatch = False
1633                 for Pcd in Module.ModulePcdList:
1634                     if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1635                         ModuleIsPatch = True
1636                         break
1637                 if not ModuleIsPatch:
1638                     for Pcd in Module.LibraryPcdList:
1639                         if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1640                             ModuleIsPatch = True
1641                             break
1642
1643                 if not ModuleIsPatch:
1644                     continue
1645                 #
1646                 # Module includes the patchable load fix address PCDs.
1647                 # It will be fixed up later.
1648                 #
1649                 PatchEfiImageList.append (OutputImageFile)
1650
1651         #
1652         # Get Top Memory address
1653         #
1654         ReservedRuntimeMemorySize = 0
1655         TopMemoryAddress = 0
1656         if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF:
1657             TopMemoryAddress = 0
1658         else:
1659             TopMemoryAddress = self.LoadFixAddress
1660             if TopMemoryAddress < RtSize + BtSize + PeiSize:
1661                 EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1662
1663         #
1664         # Patch FixAddress related PCDs into EFI image
1665         #
1666         for EfiImage in PatchEfiImageList:
1667             EfiImageMap = EfiImage.replace('.efi', '.map')
1668             if not os.path.exists(EfiImageMap):
1669                 continue
1670             #
1671             # Get PCD offset in EFI image by GenPatchPcdTable function
1672             #
1673             PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage)
1674             #
1675             # Patch real PCD value by PatchPcdValue tool
1676             #
1677             for PcdInfo in PcdTable:
1678                 ReturnValue = 0
1679                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:
1680                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000))
1681                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:
1682                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000))
1683                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:
1684                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000))
1685                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:
1686                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000))
1687                 if ReturnValue != 0:
1688                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
1689
1690         MapBuffer.append('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize // 0x1000))
1691         MapBuffer.append('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize // 0x1000))
1692         MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize // 0x1000))
1693         if len (SmmModuleList) > 0:
1694             MapBuffer.append('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize // 0x1000))
1695
1696         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
1697         BtBaseAddr  = TopMemoryAddress - RtSize
1698         RtBaseAddr  = TopMemoryAddress - ReservedRuntimeMemorySize
1699
1700         self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0)
1701         self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
1702         self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
1703         self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
1704         MapBuffer.append('\n\n')
1705         sys.stdout.write ("\n")
1706         sys.stdout.flush()
1707
1708     ## Save platform Map file
1709     #
1710     def _SaveMapFile (self, MapBuffer, Wa):
1711         #
1712         # Map file path is got.
1713         #
1714         MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map')
1715         #
1716         # Save address map into MAP file.
1717         #
1718         SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)
1719         if self.LoadFixAddress != 0:
1720             sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
1721         sys.stdout.flush()
1722
1723     ## Build active platform for different build targets and different tool chains
1724     #
1725     def _BuildPlatform(self):
1726         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1727         for BuildTarget in self.BuildTargetList:
1728             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1729             index = 0
1730             for ToolChain in self.ToolChainList:
1731                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1732                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1733                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1734                 index += 1
1735                 Wa = WorkspaceAutoGen(
1736                         self.WorkspaceDir,
1737                         self.PlatformFile,
1738                         BuildTarget,
1739                         ToolChain,
1740                         self.ArchList,
1741                         self.BuildDatabase,
1742                         self.TargetTxt,
1743                         self.ToolDef,
1744                         self.Fdf,
1745                         self.FdList,
1746                         self.FvList,
1747                         self.CapList,
1748                         self.SkuId,
1749                         self.UniFlag,
1750                         self.Progress
1751                         )
1752                 self.Fdf = Wa.FdfFile
1753                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1754                 self.BuildReport.AddPlatformReport(Wa)
1755                 self.Progress.Stop("done!")
1756
1757                 # Add ffs build to makefile
1758                 CmdListDict = {}
1759                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1760                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1761
1762                 for Arch in Wa.ArchList:
1763                     PcdMaList    = []
1764                     GlobalData.gGlobalDefines['ARCH'] = Arch
1765                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1766                     for Module in Pa.Platform.Modules:
1767                         # Get ModuleAutoGen object to generate C code file and makefile
1768                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1769                         if Ma is None:
1770                             continue
1771                         if Ma.PcdIsDriver:
1772                             Ma.PlatformInfo = Pa
1773                             Ma.Workspace = Wa
1774                             PcdMaList.append(Ma)
1775                         self.BuildModules.append(Ma)
1776                     Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
1777                     Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
1778                     self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
1779
1780                 # Create MAP file when Load Fix Address is enabled.
1781                 if self.Target in ["", "all", "fds"]:
1782                     for Arch in Wa.ArchList:
1783                         GlobalData.gGlobalDefines['ARCH'] = Arch
1784                         #
1785                         # Check whether the set fix address is above 4G for 32bit image.
1786                         #
1787                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1788                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1789                     #
1790                     # Get Module List
1791                     #
1792                     ModuleList = {}
1793                     for Pa in Wa.AutoGenObjectList:
1794                         for Ma in Pa.ModuleAutoGenList:
1795                             if Ma is None:
1796                                 continue
1797                             if not Ma.IsLibrary:
1798                                 ModuleList[Ma.Guid.upper()] = Ma
1799
1800                     MapBuffer = []
1801                     if self.LoadFixAddress != 0:
1802                         #
1803                         # Rebase module to the preferred memory address before GenFds
1804                         #
1805                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1806                     if self.Fdf:
1807                         #
1808                         # create FDS again for the updated EFI image
1809                         #
1810                         self._Build("fds", Wa)
1811                         #
1812                         # Create MAP file for all platform FVs after GenFds.
1813                         #
1814                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1815                     #
1816                     # Save MAP buffer into MAP file.
1817                     #
1818                     self._SaveMapFile (MapBuffer, Wa)
1819                 self.CreateGuidedSectionToolsFile(Wa)
1820
1821     ## Build active module for different build targets, different tool chains and different archs
1822     #
1823     def _BuildModule(self):
1824         for BuildTarget in self.BuildTargetList:
1825             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1826             index = 0
1827             for ToolChain in self.ToolChainList:
1828                 WorkspaceAutoGenTime = time.time()
1829                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1830                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1831                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1832                 index += 1
1833                 #
1834                 # module build needs platform build information, so get platform
1835                 # AutoGen first
1836                 #
1837                 Wa = WorkspaceAutoGen(
1838                         self.WorkspaceDir,
1839                         self.PlatformFile,
1840                         BuildTarget,
1841                         ToolChain,
1842                         self.ArchList,
1843                         self.BuildDatabase,
1844                         self.TargetTxt,
1845                         self.ToolDef,
1846                         self.Fdf,
1847                         self.FdList,
1848                         self.FvList,
1849                         self.CapList,
1850                         self.SkuId,
1851                         self.UniFlag,
1852                         self.Progress,
1853                         self.ModuleFile
1854                         )
1855                 self.Fdf = Wa.FdfFile
1856                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1857                 Wa.CreateMakeFile(False)
1858                 # Add ffs build to makefile
1859                 CmdListDict = None
1860                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1861                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1862
1863                 GlobalData.file_lock = mp.Lock()
1864                 GlobalData.FfsCmd = CmdListDict
1865
1866                 self.Progress.Stop("done!")
1867                 MaList = []
1868                 ExitFlag = threading.Event()
1869                 ExitFlag.clear()
1870                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
1871                 for Arch in Wa.ArchList:
1872                     AutoGenStart = time.time()
1873                     GlobalData.gGlobalDefines['ARCH'] = Arch
1874                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1875                     for Module in Pa.Platform.Modules:
1876                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
1877                             Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1878                             if Ma is None:
1879                                 continue
1880                             if Ma.PcdIsDriver:
1881                                 Ma.PlatformInfo = Pa
1882                                 Ma.Workspace = Wa
1883                             MaList.append(Ma)
1884
1885                             if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
1886                                 if Ma.CanSkipbyPreMakeCache():
1887                                     continue
1888                                 else:
1889                                     self.PreMakeCacheMiss.add(Ma)
1890
1891                             # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1892                             if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1893                                 # for target which must generate AutoGen code and makefile
1894                                 if not self.SkipAutoGen or self.Target == 'genc':
1895                                     self.Progress.Start("Generating code")
1896                                     Ma.CreateCodeFile(True)
1897                                     self.Progress.Stop("done!")
1898                                 if self.Target == "genc":
1899                                     return True
1900                                 if not self.SkipAutoGen or self.Target == 'genmake':
1901                                     self.Progress.Start("Generating makefile")
1902                                     if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:
1903                                         Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])
1904                                         del CmdListDict[Module.Path, Arch]
1905                                     else:
1906                                         Ma.CreateMakeFile(True)
1907                                     self.Progress.Stop("done!")
1908                                 if self.Target == "genmake":
1909                                     return True
1910
1911                                 if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
1912                                     if Ma.CanSkipbyMakeCache():
1913                                         continue
1914                                     else:
1915                                         self.MakeCacheMiss.add(Ma)
1916
1917                             self.BuildModules.append(Ma)
1918                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
1919                     MakeStart = time.time()
1920                     for Ma in self.BuildModules:
1921                         if not Ma.IsBinaryModule:
1922                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
1923                         # Break build if any build thread has error
1924                         if BuildTask.HasError():
1925                             # we need a full version of makefile for platform
1926                             ExitFlag.set()
1927                             BuildTask.WaitForComplete()
1928                             Pa.CreateMakeFile(False)
1929                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1930                         # Start task scheduler
1931                         if not BuildTask.IsOnGoing():
1932                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1933
1934                     # in case there's an interruption. we need a full version of makefile for platform
1935                     Pa.CreateMakeFile(False)
1936                     if BuildTask.HasError():
1937                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1938                     self.MakeTime += int(round((time.time() - MakeStart)))
1939
1940                 MakeContiue = time.time()
1941                 ExitFlag.set()
1942                 BuildTask.WaitForComplete()
1943                 self.CreateAsBuiltInf()
1944                 if GlobalData.gBinCacheDest:
1945                     self.GenDestCache()
1946                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1947                     # Only for --hash
1948                     # Update PreMakeCacheChain files
1949                     self.GenLocalPreMakeCache()
1950                 self.BuildModules = []
1951                 self.MakeTime += int(round((time.time() - MakeContiue)))
1952                 if BuildTask.HasError():
1953                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1954
1955                 self.BuildReport.AddPlatformReport(Wa, MaList)
1956                 if MaList == []:
1957                     EdkLogger.error(
1958                                 'build',
1959                                 BUILD_ERROR,
1960                                 "Module for [%s] is not a component of active platform."\
1961                                 " Please make sure that the ARCH and inf file path are"\
1962                                 " given in the same as in [%s]" % \
1963                                     (', '.join(Wa.ArchList), self.PlatformFile),
1964                                 ExtraData=self.ModuleFile
1965                                 )
1966                 # Create MAP file when Load Fix Address is enabled.
1967                 if self.Target == "fds" and self.Fdf:
1968                     for Arch in Wa.ArchList:
1969                         #
1970                         # Check whether the set fix address is above 4G for 32bit image.
1971                         #
1972                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1973                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1974                     #
1975                     # Get Module List
1976                     #
1977                     ModuleList = {}
1978                     for Pa in Wa.AutoGenObjectList:
1979                         for Ma in Pa.ModuleAutoGenList:
1980                             if Ma is None:
1981                                 continue
1982                             if not Ma.IsLibrary:
1983                                 ModuleList[Ma.Guid.upper()] = Ma
1984
1985                     MapBuffer = []
1986                     if self.LoadFixAddress != 0:
1987                         #
1988                         # Rebase module to the preferred memory address before GenFds
1989                         #
1990                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1991                     #
1992                     # create FDS again for the updated EFI image
1993                     #
1994                     GenFdsStart = time.time()
1995                     self._Build("fds", Wa)
1996                     self.GenFdsTime += int(round((time.time() - GenFdsStart)))
1997                     #
1998                     # Create MAP file for all platform FVs after GenFds.
1999                     #
2000                     self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2001                     #
2002                     # Save MAP buffer into MAP file.
2003                     #
2004                     self._SaveMapFile (MapBuffer, Wa)
2005
2006     def _GenFfsCmd(self,ArchList):
2007         # convert dictionary of Cmd:(Inf,Arch)
2008         # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
2009         CmdSetDict = defaultdict(set)
2010         GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
2011         for Cmd in GenFfsDict:
2012             tmpInf, tmpArch = GenFfsDict[Cmd]
2013             CmdSetDict[tmpInf, tmpArch].add(Cmd)
2014         return CmdSetDict
2015     def VerifyAutoGenFiles(self):
2016         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2017         try:
2018             with open(AutoGenIdFile) as fd:
2019                 lines = fd.readlines()
2020         except:
2021             return None
2022         for line in lines:
2023             if "Arch" in line:
2024                 ArchList = line.strip().split("=")[1].split("|")
2025             if "BuildDir" in line:
2026                 BuildDir = line.split("=")[1].strip()
2027             if "PlatformGuid" in line:
2028                 PlatformGuid = line.split("=")[1].strip()
2029         GlobalVarList = []
2030         for arch in ArchList:
2031             global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
2032             if not os.path.exists(global_var):
2033                 return None
2034             GlobalVarList.append(global_var)
2035         for global_var in GlobalVarList:
2036             data_pipe = MemoryDataPipe()
2037             data_pipe.load(global_var)
2038             target = data_pipe.Get("P_Info").get("Target")
2039             toolchain = data_pipe.Get("P_Info").get("ToolChain")
2040             archlist = data_pipe.Get("P_Info").get("ArchList")
2041             Arch = data_pipe.Get("P_Info").get("Arch")
2042             active_p = data_pipe.Get("P_Info").get("ActivePlatform")
2043             workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
2044             PackagesPath = os.getenv("PACKAGES_PATH")
2045             mws.setWs(workspacedir, PackagesPath)
2046             LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
2047             ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
2048
2049             for m_build_dir in LibraryBuildDirectoryList:
2050                 if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):
2051                     return None
2052             for m_build_dir in ModuleBuildDirectoryList:
2053                 if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):
2054                     return None
2055             Wa = WorkSpaceInfo(
2056                 workspacedir,active_p,target,toolchain,archlist
2057                 )
2058             Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
2059             Wa.AutoGenObjectList.append(Pa)
2060         return Wa
2061     def SetupMakeSetting(self,Wa):
2062         BuildModules = []
2063         for Pa in Wa.AutoGenObjectList:
2064             for m in Pa._MbList:
2065                 ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
2066                 BuildModules.append(ma)
2067         fdf_file = Wa.FlashDefinition
2068         if fdf_file:
2069             Fdf = FdfParser(fdf_file.Path)
2070             Fdf.ParseFile()
2071             GlobalData.gFdfParser = Fdf
2072             if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
2073                 FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
2074                 for FdRegion in FdDict.RegionList:
2075                     if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
2076                         if int(FdRegion.Offset) % 8 != 0:
2077                             EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
2078             Wa.FdfProfile = Fdf.Profile
2079             self.Fdf = Fdf
2080         else:
2081             self.Fdf = None
2082         return BuildModules
2083
2084     ## Build a platform in multi-thread mode
2085     #
2086     def PerformAutoGen(self,BuildTarget,ToolChain):
2087         WorkspaceAutoGenTime = time.time()
2088         Wa = WorkspaceAutoGen(
2089                 self.WorkspaceDir,
2090                 self.PlatformFile,
2091                 BuildTarget,
2092                 ToolChain,
2093                 self.ArchList,
2094                 self.BuildDatabase,
2095                 self.TargetTxt,
2096                 self.ToolDef,
2097                 self.Fdf,
2098                 self.FdList,
2099                 self.FvList,
2100                 self.CapList,
2101                 self.SkuId,
2102                 self.UniFlag,
2103                 self.Progress
2104                 )
2105         self.Fdf = Wa.FdfFile
2106         self.LoadFixAddress = Wa.Platform.LoadFixAddress
2107         self.BuildReport.AddPlatformReport(Wa)
2108         Wa.CreateMakeFile(False)
2109
2110         # Add ffs build to makefile
2111         CmdListDict = {}
2112         if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
2113             CmdListDict = self._GenFfsCmd(Wa.ArchList)
2114
2115         self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
2116         BuildModules = []
2117         for Arch in Wa.ArchList:
2118             PcdMaList    = []
2119             AutoGenStart = time.time()
2120             GlobalData.gGlobalDefines['ARCH'] = Arch
2121             Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
2122             if Pa is None:
2123                 continue
2124             ModuleList = []
2125             for Inf in Pa.Platform.Modules:
2126                 ModuleList.append(Inf)
2127             # Add the INF only list in FDF
2128             if GlobalData.gFdfParser is not None:
2129                 for InfName in GlobalData.gFdfParser.Profile.InfList:
2130                     Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
2131                     if Inf in Pa.Platform.Modules:
2132                         continue
2133                     ModuleList.append(Inf)
2134             Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
2135             Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
2136             Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
2137             Pa.CreateLibModuelDirs()
2138             # Fetch the MakeFileName.
2139             self.MakeFileName = Pa.MakeFileName
2140             if not self.MakeFileName:
2141                 self.MakeFileName = Pa.MakeFile
2142
2143             Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
2144             Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
2145             Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
2146             # Prepare the cache share data for multiprocessing
2147             Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}
2148             ModuleCodaFile = {}
2149             for ma in Pa.ModuleAutoGenList:
2150                 ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
2151             Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
2152             # ModuleList contains all driver modules only
2153             for Module in ModuleList:
2154                 # Get ModuleAutoGen object to generate C code file and makefile
2155                 Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2156                 if Ma is None:
2157                     continue
2158                 if Ma.PcdIsDriver:
2159                     Ma.PlatformInfo = Pa
2160                     Ma.Workspace = Wa
2161                     PcdMaList.append(Ma)
2162                 self.AllDrivers.add(Ma)
2163                 self.AllModules.add(Ma)
2164
2165             mqueue = mp.Queue()
2166             cqueue = mp.Queue()
2167             for m in Pa.GetAllModuleInfo:
2168                 mqueue.put(m)
2169                 module_file,module_root,module_path,module_basename,\
2170                     module_originalpath,module_arch,IsLib = m
2171                 Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\
2172                                   ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2173                 self.AllModules.add(Ma)
2174             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
2175             Pa.DataPipe.dump(data_pipe_file)
2176
2177             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
2178
2179             if not autogen_rt:
2180                 self.AutoGenMgr.TerminateWorkers()
2181                 self.AutoGenMgr.join(1)
2182                 raise FatalError(errorcode)
2183
2184             if GlobalData.gUseHashCache:
2185                 for item in GlobalData.gModuleAllCacheStatus:
2186                     (MetaFilePath, Arch, CacheStr, Status) = item
2187                     Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\
2188                                       ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2189                     if CacheStr == "PreMakeCache" and Status == False:
2190                         self.PreMakeCacheMiss.add(Ma)
2191                     if CacheStr == "PreMakeCache" and Status == True:
2192                         self.PreMakeCacheHit.add(Ma)
2193                         GlobalData.gModuleCacheHit.add(Ma)
2194                     if CacheStr == "MakeCache" and Status == False:
2195                         self.MakeCacheMiss.add(Ma)
2196                     if CacheStr == "MakeCache" and Status == True:
2197                         self.MakeCacheHit.add(Ma)
2198                         GlobalData.gModuleCacheHit.add(Ma)
2199             self.AutoGenTime += int(round((time.time() - AutoGenStart)))
2200         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2201         with open(AutoGenIdFile,"w") as fw:
2202             fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
2203             fw.write("BuildDir=%s\n" % Wa.BuildDir)
2204             fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
2205
2206         if GlobalData.gBinCacheSource:
2207             BuildModules.extend(self.MakeCacheMiss)
2208         elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:
2209             BuildModules.extend(self.PreMakeCacheMiss)
2210         else:
2211             BuildModules.extend(self.AllDrivers)
2212
2213         self.Progress.Stop("done!")
2214         return Wa, BuildModules
2215
2216     def _MultiThreadBuildPlatform(self):
2217         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
2218         for BuildTarget in self.BuildTargetList:
2219             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
2220             index = 0
2221             for ToolChain in self.ToolChainList:
2222                 resetFdsGlobalVariable()
2223                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
2224                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
2225                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
2226                 index += 1
2227                 ExitFlag = threading.Event()
2228                 ExitFlag.clear()
2229                 if self.SkipAutoGen:
2230                     Wa = self.VerifyAutoGenFiles()
2231                     if Wa is None:
2232                         self.SkipAutoGen = False
2233                         Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2234                     else:
2235                         GlobalData.gAutoGenPhase = True
2236                         self.BuildModules = self.SetupMakeSetting(Wa)
2237                 else:
2238                     Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2239                 Pa = Wa.AutoGenObjectList[0]
2240                 GlobalData.gAutoGenPhase = False
2241
2242                 if GlobalData.gBinCacheSource:
2243                     EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))
2244                     EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))
2245                     EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))
2246
2247                 for Arch in Wa.ArchList:
2248                     MakeStart = time.time()
2249                     for Ma in set(self.BuildModules):
2250                         # Generate build task for the module
2251                         if not Ma.IsBinaryModule:
2252                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
2253                         # Break build if any build thread has error
2254                         if BuildTask.HasError():
2255                             # we need a full version of makefile for platform
2256                             ExitFlag.set()
2257                             BuildTask.WaitForComplete()
2258                             Pa.CreateMakeFile(False)
2259                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2260                         # Start task scheduler
2261                         if not BuildTask.IsOnGoing():
2262                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
2263
2264                     # in case there's an interruption. we need a full version of makefile for platform
2265
2266                     if BuildTask.HasError():
2267                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2268                     self.MakeTime += int(round((time.time() - MakeStart)))
2269
2270                 MakeContiue = time.time()
2271                 #
2272                 #
2273                 # All modules have been put in build tasks queue. Tell task scheduler
2274                 # to exit if all tasks are completed
2275                 #
2276                 ExitFlag.set()
2277                 BuildTask.WaitForComplete()
2278                 self.CreateAsBuiltInf()
2279                 if GlobalData.gBinCacheDest:
2280                     self.GenDestCache()
2281                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
2282                     # Only for --hash
2283                     # Update PreMakeCacheChain files
2284                     self.GenLocalPreMakeCache()
2285                 #
2286                 # Get Module List
2287                 #
2288                 ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}
2289                 self.BuildModules = []
2290                 self.MakeTime += int(round((time.time() - MakeContiue)))
2291                 #
2292                 # Check for build error, and raise exception if one
2293                 # has been signaled.
2294                 #
2295                 if BuildTask.HasError():
2296                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2297
2298                 # Create MAP file when Load Fix Address is enabled.
2299                 if self.Target in ["", "all", "fds"]:
2300                     for Arch in Wa.ArchList:
2301                         #
2302                         # Check whether the set fix address is above 4G for 32bit image.
2303                         #
2304                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
2305                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2306
2307                     #
2308                     # Rebase module to the preferred memory address before GenFds
2309                     #
2310                     MapBuffer = []
2311                     if self.LoadFixAddress != 0:
2312                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
2313
2314                     if self.Fdf:
2315                         #
2316                         # Generate FD image if there's a FDF file found
2317                         #
2318                         GenFdsStart = time.time()
2319                         if GenFdsApi(Wa.GenFdsCommandDict, self.Db):
2320                             EdkLogger.error("build", COMMAND_FAILURE)
2321                         Threshold = self.GetFreeSizeThreshold()
2322                         if Threshold:
2323                             self.CheckFreeSizeThreshold(Threshold, Wa.FvDir)
2324
2325                         #
2326                         # Create MAP file for all platform FVs after GenFds.
2327                         #
2328                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2329                         self.GenFdsTime += int(round((time.time() - GenFdsStart)))
2330                     #
2331                     # Save MAP buffer into MAP file.
2332                     #
2333                     self._SaveMapFile(MapBuffer, Wa)
2334                 self.CreateGuidedSectionToolsFile(Wa)
2335
2336     ## GetFreeSizeThreshold()
2337     #
2338     #   @retval int             Threshold value
2339     #
2340     def GetFreeSizeThreshold(self):
2341         Threshold = None
2342         Threshold_Str = GlobalData.gCommandLineDefines.get('FV_SPARE_SPACE_THRESHOLD')
2343         if Threshold_Str:
2344             try:
2345                 if Threshold_Str.lower().startswith('0x'):
2346                     Threshold = int(Threshold_Str, 16)
2347                 else:
2348                     Threshold = int(Threshold_Str)
2349             except:
2350                 EdkLogger.warn("build", 'incorrect value for FV_SPARE_SPACE_THRESHOLD %s.Only decimal or hex format is allowed.' % Threshold_Str)
2351         return Threshold
2352
2353     def CheckFreeSizeThreshold(self, Threshold=None, FvDir=None):
2354         if not isinstance(Threshold, int):
2355             return
2356         if not isinstance(FvDir, str) or not FvDir:
2357             return
2358         FdfParserObject = GlobalData.gFdfParser
2359         FvRegionNameList = [FvName for FvName in FdfParserObject.Profile.FvDict if FdfParserObject.Profile.FvDict[FvName].FvRegionInFD]
2360         for FvName in FdfParserObject.Profile.FvDict:
2361             if FvName in FvRegionNameList:
2362                 FvSpaceInfoFileName = os.path.join(FvDir, FvName.upper() + '.Fv.map')
2363                 if os.path.exists(FvSpaceInfoFileName):
2364                     FileLinesList = getlines(FvSpaceInfoFileName)
2365                     for Line in FileLinesList:
2366                         NameValue = Line.split('=')
2367                         if len(NameValue) == 2 and NameValue[0].strip() == 'EFI_FV_SPACE_SIZE':
2368                             FreeSizeValue = int(NameValue[1].strip(), 0)
2369                             if FreeSizeValue < Threshold:
2370                                 EdkLogger.error("build", FV_FREESIZE_ERROR,
2371                                                 '%s FV free space %d is not enough to meet with the required spare space %d set by -D FV_SPARE_SPACE_THRESHOLD option.' % (
2372                                                     FvName, FreeSizeValue, Threshold))
2373                             break
2374
2375     ## Generate GuidedSectionTools.txt in the FV directories.
2376     #
2377     def CreateGuidedSectionToolsFile(self,Wa):
2378         for BuildTarget in self.BuildTargetList:
2379             for ToolChain in self.ToolChainList:
2380                 FvDir = Wa.FvDir
2381                 if not os.path.exists(FvDir):
2382                     continue
2383
2384                 for Arch in self.ArchList:
2385                     # Build up the list of supported architectures for this build
2386                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
2387
2388                     # Look through the tool definitions for GUIDed tools
2389                     guidAttribs = []
2390                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():
2391                         if attrib.upper().endswith('_GUID'):
2392                             split = attrib.split('_')
2393                             thisPrefix = '_'.join(split[0:3]) + '_'
2394                             if thisPrefix == prefix:
2395                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
2396                                 guid = guid.lower()
2397                                 toolName = split[3]
2398                                 path = '_'.join(split[0:4]) + '_PATH'
2399                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
2400                                 path = self.GetRealPathOfTool(path)
2401                                 guidAttribs.append((guid, toolName, path))
2402
2403                     # Write out GuidedSecTools.txt
2404                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
2405                     toolsFile = open(toolsFile, 'wt')
2406                     for guidedSectionTool in guidAttribs:
2407                         print(' '.join(guidedSectionTool), file=toolsFile)
2408                     toolsFile.close()
2409
2410     ## Returns the real path of the tool.
2411     #
2412     def GetRealPathOfTool (self, tool):
2413         if os.path.exists(tool):
2414             return os.path.realpath(tool)
2415         return tool
2416
2417     ## Launch the module or platform build
2418     #
2419     def Launch(self):
2420         self.AllDrivers = set()
2421         self.AllModules = set()
2422         self.PreMakeCacheMiss = set()
2423         self.PreMakeCacheHit = set()
2424         self.MakeCacheMiss = set()
2425         self.MakeCacheHit = set()
2426         if not self.ModuleFile:
2427             if not self.SpawnMode or self.Target not in ["", "all"]:
2428                 self.SpawnMode = False
2429                 self._BuildPlatform()
2430             else:
2431                 self._MultiThreadBuildPlatform()
2432         else:
2433             self.SpawnMode = False
2434             self._BuildModule()
2435
2436         if self.Target == 'cleanall':
2437             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
2438
2439     def CreateAsBuiltInf(self):
2440         for Module in self.BuildModules:
2441             Module.CreateAsBuiltInf()
2442
2443     def GenDestCache(self):
2444         for Module in self.AllModules:
2445             Module.GenPreMakefileHashList()
2446             Module.GenMakefileHashList()
2447             Module.CopyModuleToCache()
2448
2449     def GenLocalPreMakeCache(self):
2450         for Module in self.PreMakeCacheMiss:
2451             Module.GenPreMakefileHashList()
2452
2453     ## Do some clean-up works when error occurred
2454     def Relinquish(self):
2455         OldLogLevel = EdkLogger.GetLevel()
2456         EdkLogger.SetLevel(EdkLogger.ERROR)
2457         Utils.Progressor.Abort()
2458         if self.SpawnMode == True:
2459             BuildTask.Abort()
2460         EdkLogger.SetLevel(OldLogLevel)
2461
2462 def ParseDefines(DefineList=[]):
2463     DefineDict = {}
2464     if DefineList is not None:
2465         for Define in DefineList:
2466             DefineTokenList = Define.split("=", 1)
2467             if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]):
2468                 EdkLogger.error('build', FORMAT_INVALID,
2469                                 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2470                                 ExtraData=DefineTokenList[0])
2471
2472             if len(DefineTokenList) == 1:
2473                 DefineDict[DefineTokenList[0]] = "TRUE"
2474             else:
2475                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
2476     return DefineDict
2477
2478
2479
2480 def LogBuildTime(Time):
2481     if Time:
2482         TimeDurStr = ''
2483         TimeDur = time.gmtime(Time)
2484         if TimeDur.tm_yday > 1:
2485             TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + ", %d day(s)" % (TimeDur.tm_yday - 1)
2486         else:
2487             TimeDurStr = time.strftime("%H:%M:%S", TimeDur)
2488         return TimeDurStr
2489     else:
2490         return None
2491 def ThreadNum():
2492     OptionParser = MyOptionParser()
2493     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2494         OptionParser.GetOption()
2495     BuildOption, BuildTarget = OptionParser.BuildOption, OptionParser.BuildTarget
2496     ThreadNumber = BuildOption.ThreadNumber
2497     GlobalData.gCmdConfDir = BuildOption.ConfDirectory
2498     if ThreadNumber is None:
2499         TargetObj = TargetTxtDict()
2500         ThreadNumber = TargetObj.Target.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
2501         if ThreadNumber == '':
2502             ThreadNumber = 0
2503         else:
2504             ThreadNumber = int(ThreadNumber, 0)
2505
2506     if ThreadNumber == 0:
2507         try:
2508             ThreadNumber = multiprocessing.cpu_count()
2509         except (ImportError, NotImplementedError):
2510             ThreadNumber = 1
2511     return ThreadNumber
2512 ## Tool entrance method
2513 #
2514 # This method mainly dispatch specific methods per the command line options.
2515 # If no error found, return zero value so the caller of this tool can know
2516 # if it's executed successfully or not.
2517 #
2518 #   @retval 0     Tool was successful
2519 #   @retval 1     Tool failed
2520 #
2521 LogQMaxSize = ThreadNum() * 10
2522 def Main():
2523     StartTime = time.time()
2524
2525     #
2526     # Create a log Queue
2527     #
2528     LogQ = mp.Queue(LogQMaxSize)
2529     # Initialize log system
2530     EdkLogger.LogClientInitialize(LogQ)
2531     GlobalData.gCommand = sys.argv[1:]
2532     #
2533     # Parse the options and args
2534     #
2535     OptionParser = MyOptionParser()
2536     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2537         OptionParser.GetOption()
2538     Option, Target = OptionParser.BuildOption, OptionParser.BuildTarget
2539     GlobalData.gOptions = Option
2540     GlobalData.gCaseInsensitive = Option.CaseInsensitive
2541
2542     # Set log level
2543     LogLevel = EdkLogger.INFO
2544     if Option.verbose is not None:
2545         EdkLogger.SetLevel(EdkLogger.VERBOSE)
2546         LogLevel = EdkLogger.VERBOSE
2547     elif Option.quiet is not None:
2548         EdkLogger.SetLevel(EdkLogger.QUIET)
2549         LogLevel = EdkLogger.QUIET
2550     elif Option.debug is not None:
2551         EdkLogger.SetLevel(Option.debug + 1)
2552         LogLevel = Option.debug + 1
2553     else:
2554         EdkLogger.SetLevel(EdkLogger.INFO)
2555
2556     if Option.WarningAsError == True:
2557         EdkLogger.SetWarningAsError()
2558     Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile)
2559     Log_Agent.start()
2560
2561     if platform.platform().find("Windows") >= 0:
2562         GlobalData.gIsWindows = True
2563     else:
2564         GlobalData.gIsWindows = False
2565
2566     EdkLogger.quiet("Build environment: %s" % platform.platform())
2567     EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime()));
2568     ReturnCode = 0
2569     MyBuild = None
2570     BuildError = True
2571     try:
2572         if len(Target) == 0:
2573             Target = "all"
2574         elif len(Target) >= 2:
2575             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
2576                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2577         else:
2578             Target = Target[0].lower()
2579
2580         if Target not in gSupportedTarget:
2581             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
2582                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2583
2584         #
2585         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2586         #
2587         CheckEnvVariable()
2588         GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros))
2589
2590         Workspace = os.getenv("WORKSPACE")
2591         #
2592         # Get files real name in workspace dir
2593         #
2594         GlobalData.gAllFiles = Utils.DirCache(Workspace)
2595
2596         WorkingDirectory = os.getcwd()
2597         if not Option.ModuleFile:
2598             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
2599             FileNum = len(FileList)
2600             if FileNum >= 2:
2601                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
2602                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2603             elif FileNum == 1:
2604                 Option.ModuleFile = NormFile(FileList[0], Workspace)
2605
2606         if Option.ModuleFile:
2607             if os.path.isabs (Option.ModuleFile):
2608                 if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0:
2609                     Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace)
2610             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
2611             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
2612             if ErrorCode != 0:
2613                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2614
2615         if Option.PlatformFile is not None:
2616             if os.path.isabs (Option.PlatformFile):
2617                 if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0:
2618                     Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace)
2619             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
2620
2621         if Option.FdfFile is not None:
2622             if os.path.isabs (Option.FdfFile):
2623                 if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0:
2624                     Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace)
2625             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
2626             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
2627             if ErrorCode != 0:
2628                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2629
2630         if Option.Flag is not None and Option.Flag not in ['-c', '-s']:
2631             EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s")
2632
2633         MyBuild = Build(Target, Workspace, Option,LogQ)
2634         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
2635         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
2636             MyBuild.Launch()
2637
2638         #
2639         # All job done, no error found and no exception raised
2640         #
2641         BuildError = False
2642     except FatalError as X:
2643         if MyBuild is not None:
2644             # for multi-thread build exits safely
2645             MyBuild.Relinquish()
2646         if Option is not None and Option.debug is not None:
2647             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2648         ReturnCode = X.args[0]
2649     except Warning as X:
2650         # error from Fdf parser
2651         if MyBuild is not None:
2652             # for multi-thread build exits safely
2653             MyBuild.Relinquish()
2654         if Option is not None and Option.debug is not None:
2655             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2656         else:
2657             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
2658         ReturnCode = FORMAT_INVALID
2659     except KeyboardInterrupt:
2660         if MyBuild is not None:
2661
2662             # for multi-thread build exits safely
2663             MyBuild.Relinquish()
2664         ReturnCode = ABORT_ERROR
2665         if Option is not None and Option.debug is not None:
2666             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2667     except:
2668         if MyBuild is not None:
2669             # for multi-thread build exits safely
2670             MyBuild.Relinquish()
2671
2672         # try to get the meta-file from the object causing exception
2673         Tb = sys.exc_info()[-1]
2674         MetaFile = GlobalData.gProcessingFile
2675         while Tb is not None:
2676             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
2677                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
2678             Tb = Tb.tb_next
2679         EdkLogger.error(
2680                     "\nbuild",
2681                     CODE_ERROR,
2682                     "Unknown fatal error when processing [%s]" % MetaFile,
2683                     ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR,
2684                     RaiseError=False
2685                     )
2686         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2687         ReturnCode = CODE_ERROR
2688     finally:
2689         Utils.Progressor.Abort()
2690         Utils.ClearDuplicatedInf()
2691
2692     if ReturnCode == 0:
2693         try:
2694             MyBuild.LaunchPostbuild()
2695             Conclusion = "Done"
2696         except:
2697             Conclusion = "Failed"
2698     elif ReturnCode == ABORT_ERROR:
2699         Conclusion = "Aborted"
2700     else:
2701         Conclusion = "Failed"
2702     FinishTime = time.time()
2703     BuildDuration = time.gmtime(int(round(FinishTime - StartTime)))
2704     BuildDurationStr = ""
2705     if BuildDuration.tm_yday > 1:
2706         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1)
2707     else:
2708         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
2709     if MyBuild is not None:
2710         if not BuildError:
2711             MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
2712
2713     EdkLogger.SetLevel(EdkLogger.QUIET)
2714     EdkLogger.quiet("\n- %s -" % Conclusion)
2715     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
2716     EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
2717     Log_Agent.kill()
2718     Log_Agent.join()
2719     return ReturnCode
2720
2721 if __name__ == '__main__':
2722     try:
2723         mp.set_start_method('spawn')
2724     except:
2725         pass
2726     r = Main()
2727     ## 0-127 is a safe return range, and 1 is a standard default error
2728     if r < 0 or r > 127: r = 1
2729     sys.exit(r)