BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
   5 #  Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.<BR>
   6 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
   7 #  Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
   8 #
   9 #  SPDX-License-Identifier: BSD-2-Clause-Patent
  10 #
  11
  12 ##
  13 # Import Modules
  14 #
  15 from __future__ import print_function
  16 from __future__ import absolute_import
  17 import os.path as path
  18 import sys
  19 import os
  20 import re
  21 import glob
  22 import time
  23 import platform
  24 import traceback
  25 import multiprocessing
  26 from threading import Thread,Event,BoundedSemaphore
  27 import threading
  28 from subprocess import Popen,PIPE, STDOUT
  29 from collections import OrderedDict, defaultdict
  30
  31 from AutoGen.PlatformAutoGen import PlatformAutoGen
  32 from AutoGen.ModuleAutoGen import ModuleAutoGen
  33 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
  34 from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\
  35     LogAgent
  36 from AutoGen import GenMake
  37 from Common import Misc as Utils
  38
  39 from Common.TargetTxtClassObject import TargetTxtDict
  40 from Common.ToolDefClassObject import ToolDefDict
  41 from buildoptions import MyOptionParser
  42 from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory
  43 from Common.StringUtils import NormPath
  44 from Common.MultipleWorkspace import MultipleWorkspace as mws
  45 from Common.BuildToolError import *
  46 from Common.DataType import *
  47 import Common.EdkLogger as EdkLogger
  48
  49 from Workspace.WorkspaceDatabase import BuildDB
  50
  51 from BuildReport import BuildReport
  52 from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile
  53 from PatchPcdValue.PatchPcdValue import PatchBinaryFile
  54
  55 import Common.GlobalData as GlobalData
  56 from GenFds.GenFds import GenFds, GenFdsApi
  57 import multiprocessing as mp
  58 from multiprocessing import Manager
  59 from AutoGen.DataPipe import MemoryDataPipe
  60 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
  61 from GenFds.FdfParser import FdfParser
  62 from AutoGen.IncludesAutoGen import IncludesAutoGen
  63 from GenFds.GenFds import resetFdsGlobalVariable
  64
  65 ## standard targets of build command
  66 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  67
  68 ## build configuration file
  69 gBuildConfiguration = "target.txt"
  70 gToolsDefinition = "tools_def.txt"
  71
  72 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
  73 TmpTableDict = {}
  74
  75 ## Check environment PATH variable to make sure the specified tool is found
  76 #
  77 #   If the tool is found in the PATH, then True is returned
  78 #   Otherwise, False is returned
  79 #
  80 def IsToolInPath(tool):
  81     if 'PATHEXT' in os.environ:
  82         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  83     else:
  84         extns = ('',)
  85     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  86         for ext in extns:
  87             if os.path.exists(os.path.join(pathDir, tool + ext)):
  88                 return True
  89     return False
  90
  91 ## Check environment variables
  92 #
  93 #  Check environment variables that must be set for build. Currently they are
  94 #
  95 #   WORKSPACE           The directory all packages/platforms start from
  96 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  97 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  98 #
  99 #   If any of above environment variable is not set or has error, the build
 100 #   will be broken.
 101 #
 102 def CheckEnvVariable():
 103     # check WORKSPACE
 104     if "WORKSPACE" not in os.environ:
 105         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 106                         ExtraData="WORKSPACE")
 107
 108     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
 109     if not os.path.exists(WorkspaceDir):
 110         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData=WorkspaceDir)
 111     elif ' ' in WorkspaceDir:
 112         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
 113                         ExtraData=WorkspaceDir)
 114     os.environ["WORKSPACE"] = WorkspaceDir
 115
 116     # set multiple workspace
 117     PackagesPath = os.getenv("PACKAGES_PATH")
 118     mws.setWs(WorkspaceDir, PackagesPath)
 119     if mws.PACKAGES_PATH:
 120         for Path in mws.PACKAGES_PATH:
 121             if not os.path.exists(Path):
 122                 EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData=Path)
 123             elif ' ' in Path:
 124                 EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path)
 125
 126
 127     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 128
 129     # check EDK_TOOLS_PATH
 130     if "EDK_TOOLS_PATH" not in os.environ:
 131         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 132                         ExtraData="EDK_TOOLS_PATH")
 133
 134     # check PATH
 135     if "PATH" not in os.environ:
 136         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 137                         ExtraData="PATH")
 138
 139     GlobalData.gWorkspace = WorkspaceDir
 140
 141     GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 142     GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"]
 143
 144 ## Get normalized file path
 145 #
 146 # Convert the path to be local format, and remove the WORKSPACE path at the
 147 # beginning if the file path is given in full path.
 148 #
 149 # @param  FilePath      File path to be normalized
 150 # @param  Workspace     Workspace path which the FilePath will be checked against
 151 #
 152 # @retval string        The normalized file path
 153 #
 154 def NormFile(FilePath, Workspace):
 155     # check if the path is absolute or relative
 156     if os.path.isabs(FilePath):
 157         FileFullPath = os.path.normpath(FilePath)
 158     else:
 159         FileFullPath = os.path.normpath(mws.join(Workspace, FilePath))
 160         Workspace = mws.getWs(Workspace, FilePath)
 161
 162     # check if the file path exists or not
 163     if not os.path.isfile(FileFullPath):
 164         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath)
 165
 166     # remove workspace directory from the beginning part of the file path
 167     if Workspace[-1] in ["\\", "/"]:
 168         return FileFullPath[len(Workspace):]
 169     else:
 170         return FileFullPath[(len(Workspace) + 1):]
 171
 172 ## Get the output of an external program
 173 #
 174 # This is the entrance method of thread reading output of an external program and
 175 # putting them in STDOUT/STDERR of current program.
 176 #
 177 # @param  From      The stream message read from
 178 # @param  To        The stream message put on
 179 # @param  ExitFlag  The flag used to indicate stopping reading
 180 #
 181 def ReadMessage(From, To, ExitFlag,MemTo=None):
 182     while True:
 183         # read one line a time
 184         Line = From.readline()
 185         # empty string means "end"
 186         if Line is not None and Line != b"":
 187             LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
 188             if MemTo is not None:
 189                 if "Note: including file:" ==  LineStr.lstrip()[:21]:
 190                     MemTo.append(LineStr)
 191                 else:
 192                     To(LineStr)
 193                     MemTo.append(LineStr)
 194             else:
 195                 To(LineStr)
 196         else:
 197             break
 198         if ExitFlag.isSet():
 199             break
 200
 201 class MakeSubProc(Popen):
 202     def __init__(self,*args, **argv):
 203         super(MakeSubProc,self).__init__(*args, **argv)
 204         self.ProcOut = []
 205
 206 ## Launch an external program
 207 #
 208 # This method will call subprocess.Popen to execute an external program with
 209 # given options in specified directory. Because of the dead-lock issue during
 210 # redirecting output of the external program, threads are used to to do the
 211 # redirection work.
 212 #
 213 # @param  Command               A list or string containing the call of the program
 214 # @param  WorkingDir            The directory in which the program will be running
 215 #
 216 def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
 217     BeginTime = time.time()
 218     # if working directory doesn't exist, Popen() will raise an exception
 219     if not os.path.isdir(WorkingDir):
 220         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 221
 222     # Command is used as the first Argument in following Popen().
 223     # It could be a string or sequence. We find that if command is a string in following Popen(),
 224     # ubuntu may fail with an error message that the command is not found.
 225     # So here we may need convert command from string to list instance.
 226     if platform.system() != 'Windows':
 227         if not isinstance(Command, list):
 228             Command = Command.split()
 229         Command = ' '.join(Command)
 230
 231     Proc = None
 232     EndOfProcedure = None
 233     try:
 234         # launch the command
 235         Proc = MakeSubProc(Command, stdout=PIPE, stderr=STDOUT, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 236
 237         # launch two threads to read the STDOUT and STDERR
 238         EndOfProcedure = Event()
 239         EndOfProcedure.clear()
 240         if Proc.stdout:
 241             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
 242             StdOutThread.setName("STDOUT-Redirector")
 243             StdOutThread.setDaemon(False)
 244             StdOutThread.start()
 245
 246
 247         # waiting for program exit
 248         Proc.wait()
 249     except: # in case of aborting
 250         # terminate the threads redirecting the program output
 251         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 252         if EndOfProcedure is not None:
 253             EndOfProcedure.set()
 254         if Proc is None:
 255             if not isinstance(Command, type("")):
 256                 Command = " ".join(Command)
 257             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 258
 259     if Proc.stdout:
 260         StdOutThread.join()
 261
 262     # check the return code of the program
 263     if Proc.returncode != 0:
 264         if not isinstance(Command, type("")):
 265             Command = " ".join(Command)
 266         # print out the Response file and its content when make failure
 267         RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
 268         if os.path.isfile(RespFile):
 269             f = open(RespFile)
 270             RespContent = f.read()
 271             f.close()
 272             EdkLogger.info(RespContent)
 273
 274         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 275     if ModuleAuto:
 276         iau = IncludesAutoGen(WorkingDir,ModuleAuto)
 277         if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
 278             iau.CreateDepsFileForMsvc(Proc.ProcOut)
 279         else:
 280             iau.UpdateDepsFileforNonMsvc()
 281         iau.UpdateDepsFileforTrim()
 282         iau.CreateModuleDeps()
 283         iau.CreateDepsInclude()
 284         iau.CreateDepsTarget()
 285     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
 286
 287 ## The smallest unit that can be built in multi-thread build mode
 288 #
 289 # This is the base class of build unit. The "Obj" parameter must provide
 290 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 291 # missing build.
 292 #
 293 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 294 #
 295 class BuildUnit:
 296     ## The constructor
 297     #
 298     #   @param  self        The object pointer
 299     #   @param  Obj         The object the build is working on
 300     #   @param  Target      The build target name, one of gSupportedTarget
 301     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 302     #   @param  WorkingDir  The directory build command starts in
 303     #
 304     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 305         self.BuildObject = Obj
 306         self.Dependency = Dependency
 307         self.WorkingDir = WorkingDir
 308         self.Target = Target
 309         self.BuildCommand = BuildCommand
 310         if not BuildCommand:
 311             EdkLogger.error("build", OPTION_MISSING,
 312                             "No build command found for this module. "
 313                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
 314                                 (Obj.BuildTarget, Obj.ToolChain, Obj.Arch),
 315                             ExtraData=str(Obj))
 316
 317
 318     ## str() method
 319     #
 320     #   It just returns the string representation of self.BuildObject
 321     #
 322     #   @param  self        The object pointer
 323     #
 324     def __str__(self):
 325         return str(self.BuildObject)
 326
 327     ## "==" operator method
 328     #
 329     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 330     #   provide its own __eq__() method.
 331     #
 332     #   @param  self        The object pointer
 333     #   @param  Other       The other BuildUnit object compared to
 334     #
 335     def __eq__(self, Other):
 336         return Other and self.BuildObject == Other.BuildObject \
 337                 and Other.BuildObject \
 338                 and self.BuildObject.Arch == Other.BuildObject.Arch
 339
 340     ## hash() method
 341     #
 342     #   It just returns the hash value of self.BuildObject which must be hashable.
 343     #
 344     #   @param  self        The object pointer
 345     #
 346     def __hash__(self):
 347         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 348
 349     def __repr__(self):
 350         return repr(self.BuildObject)
 351
 352 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 353 #
 354 # This class is for module build by nmake/make build system. The "Obj" parameter
 355 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 356 # be make units missing build.
 357 #
 358 # Currently the "Obj" should be only ModuleAutoGen object.
 359 #
 360 class ModuleMakeUnit(BuildUnit):
 361     ## The constructor
 362     #
 363     #   @param  self        The object pointer
 364     #   @param  Obj         The ModuleAutoGen object the build is working on
 365     #   @param  Target      The build target name, one of gSupportedTarget
 366     #
 367     def __init__(self, Obj, BuildCommand,Target):
 368         Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]
 369         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 370         if Target in [None, "", "all"]:
 371             self.Target = "tbuild"
 372
 373 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 374 #
 375 # This class is for platform build by nmake/make build system. The "Obj" parameter
 376 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 377 # be make units missing build.
 378 #
 379 # Currently the "Obj" should be only PlatformAutoGen object.
 380 #
 381 class PlatformMakeUnit(BuildUnit):
 382     ## The constructor
 383     #
 384     #   @param  self        The object pointer
 385     #   @param  Obj         The PlatformAutoGen object the build is working on
 386     #   @param  Target      The build target name, one of gSupportedTarget
 387     #
 388     def __init__(self, Obj, BuildCommand, Target):
 389         Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 390         Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])
 391         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 392
 393 ## The class representing the task of a module build or platform build
 394 #
 395 # This class manages the build tasks in multi-thread build mode. Its jobs include
 396 # scheduling thread running, catching thread error, monitor the thread status, etc.
 397 #
 398 class BuildTask:
 399     # queue for tasks waiting for schedule
 400     _PendingQueue = OrderedDict()
 401     _PendingQueueLock = threading.Lock()
 402
 403     # queue for tasks ready for running
 404     _ReadyQueue = OrderedDict()
 405     _ReadyQueueLock = threading.Lock()
 406
 407     # queue for run tasks
 408     _RunningQueue = OrderedDict()
 409     _RunningQueueLock = threading.Lock()
 410
 411     # queue containing all build tasks, in case duplicate build
 412     _TaskQueue = OrderedDict()
 413
 414     # flag indicating error occurs in a running thread
 415     _ErrorFlag = threading.Event()
 416     _ErrorFlag.clear()
 417     _ErrorMessage = ""
 418
 419     # BoundedSemaphore object used to control the number of running threads
 420     _Thread = None
 421
 422     # flag indicating if the scheduler is started or not
 423     _SchedulerStopped = threading.Event()
 424     _SchedulerStopped.set()
 425
 426     ## Start the task scheduler thread
 427     #
 428     #   @param  MaxThreadNumber     The maximum thread number
 429     #   @param  ExitFlag            Flag used to end the scheduler
 430     #
 431     @staticmethod
 432     def StartScheduler(MaxThreadNumber, ExitFlag):
 433         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 434         SchedulerThread.setName("Build-Task-Scheduler")
 435         SchedulerThread.setDaemon(False)
 436         SchedulerThread.start()
 437         # wait for the scheduler to be started, especially useful in Linux
 438         while not BuildTask.IsOnGoing():
 439             time.sleep(0.01)
 440
 441     ## Scheduler method
 442     #
 443     #   @param  MaxThreadNumber     The maximum thread number
 444     #   @param  ExitFlag            Flag used to end the scheduler
 445     #
 446     @staticmethod
 447     def Scheduler(MaxThreadNumber, ExitFlag):
 448         BuildTask._SchedulerStopped.clear()
 449         try:
 450             # use BoundedSemaphore to control the maximum running threads
 451             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 452             #
 453             # scheduling loop, which will exits when no pending/ready task and
 454             # indicated to do so, or there's error in running thread
 455             #
 456             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 457                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 458                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 459                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 460
 461                 # get all pending tasks
 462                 BuildTask._PendingQueueLock.acquire()
 463                 BuildObjectList = list(BuildTask._PendingQueue.keys())
 464                 #
 465                 # check if their dependency is resolved, and if true, move them
 466                 # into ready queue
 467                 #
 468                 for BuildObject in BuildObjectList:
 469                     Bt = BuildTask._PendingQueue[BuildObject]
 470                     if Bt.IsReady():
 471                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 472                 BuildTask._PendingQueueLock.release()
 473
 474                 # launch build thread until the maximum number of threads is reached
 475                 while not BuildTask._ErrorFlag.isSet():
 476                     # empty ready queue, do nothing further
 477                     if len(BuildTask._ReadyQueue) == 0:
 478                         break
 479
 480                     # wait for active thread(s) exit
 481                     BuildTask._Thread.acquire(True)
 482
 483                     # start a new build thread
 484                     Bo, Bt = BuildTask._ReadyQueue.popitem()
 485
 486                     # move into running queue
 487                     BuildTask._RunningQueueLock.acquire()
 488                     BuildTask._RunningQueue[Bo] = Bt
 489                     BuildTask._RunningQueueLock.release()
 490
 491                     Bt.Start()
 492                     # avoid tense loop
 493                     time.sleep(0.01)
 494
 495                 # avoid tense loop
 496                 time.sleep(0.01)
 497
 498             # wait for all running threads exit
 499             if BuildTask._ErrorFlag.isSet():
 500                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 501             # while not BuildTask._ErrorFlag.isSet() and \
 502             while len(BuildTask._RunningQueue) > 0:
 503                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 504                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))
 505                 # avoid tense loop
 506                 time.sleep(0.1)
 507         except BaseException as X:
 508             #
 509             # TRICK: hide the output of threads left running, so that the user can
 510             #        catch the error message easily
 511             #
 512             EdkLogger.SetLevel(EdkLogger.ERROR)
 513             BuildTask._ErrorFlag.set()
 514             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 515
 516         BuildTask._PendingQueue.clear()
 517         BuildTask._ReadyQueue.clear()
 518         BuildTask._RunningQueue.clear()
 519         BuildTask._TaskQueue.clear()
 520         BuildTask._SchedulerStopped.set()
 521
 522     ## Wait for all running method exit
 523     #
 524     @staticmethod
 525     def WaitForComplete():
 526         BuildTask._SchedulerStopped.wait()
 527
 528     ## Check if the scheduler is running or not
 529     #
 530     @staticmethod
 531     def IsOnGoing():
 532         return not BuildTask._SchedulerStopped.isSet()
 533
 534     ## Abort the build
 535     @staticmethod
 536     def Abort():
 537         if BuildTask.IsOnGoing():
 538             BuildTask._ErrorFlag.set()
 539             BuildTask.WaitForComplete()
 540
 541     ## Check if there's error in running thread
 542     #
 543     #   Since the main thread cannot catch exceptions in other thread, we have to
 544     #   use threading.Event to communicate this formation to main thread.
 545     #
 546     @staticmethod
 547     def HasError():
 548         return BuildTask._ErrorFlag.isSet()
 549
 550     ## Get error message in running thread
 551     #
 552     #   Since the main thread cannot catch exceptions in other thread, we have to
 553     #   use a static variable to communicate this message to main thread.
 554     #
 555     @staticmethod
 556     def GetErrorMessage():
 557         return BuildTask._ErrorMessage
 558
 559     ## Factory method to create a BuildTask object
 560     #
 561     #   This method will check if a module is building or has been built. And if
 562     #   true, just return the associated BuildTask object in the _TaskQueue. If
 563     #   not, create and return a new BuildTask object. The new BuildTask object
 564     #   will be appended to the _PendingQueue for scheduling later.
 565     #
 566     #   @param  BuildItem       A BuildUnit object representing a build object
 567     #   @param  Dependency      The dependent build object of BuildItem
 568     #
 569     @staticmethod
 570     def New(BuildItem, Dependency=None):
 571         if BuildItem in BuildTask._TaskQueue:
 572             Bt = BuildTask._TaskQueue[BuildItem]
 573             return Bt
 574
 575         Bt = BuildTask()
 576         Bt._Init(BuildItem, Dependency)
 577         BuildTask._TaskQueue[BuildItem] = Bt
 578
 579         BuildTask._PendingQueueLock.acquire()
 580         BuildTask._PendingQueue[BuildItem] = Bt
 581         BuildTask._PendingQueueLock.release()
 582
 583         return Bt
 584
 585     ## The real constructor of BuildTask
 586     #
 587     #   @param  BuildItem       A BuildUnit object representing a build object
 588     #   @param  Dependency      The dependent build object of BuildItem
 589     #
 590     def _Init(self, BuildItem, Dependency=None):
 591         self.BuildItem = BuildItem
 592
 593         self.DependencyList = []
 594         if Dependency is None:
 595             Dependency = BuildItem.Dependency
 596         else:
 597             Dependency.extend(BuildItem.Dependency)
 598         self.AddDependency(Dependency)
 599         # flag indicating build completes, used to avoid unnecessary re-build
 600         self.CompleteFlag = False
 601
 602     ## Check if all dependent build tasks are completed or not
 603     #
 604     def IsReady(self):
 605         ReadyFlag = True
 606         for Dep in self.DependencyList:
 607             if Dep.CompleteFlag == True:
 608                 continue
 609             ReadyFlag = False
 610             break
 611
 612         return ReadyFlag
 613
 614     ## Add dependent build task
 615     #
 616     #   @param  Dependency      The list of dependent build objects
 617     #
 618     def AddDependency(self, Dependency):
 619         for Dep in Dependency:
 620             if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):
 621                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 622
 623     ## The thread wrapper of LaunchCommand function
 624     #
 625     # @param  Command               A list or string contains the call of the command
 626     # @param  WorkingDir            The directory in which the program will be running
 627     #
 628     def _CommandThread(self, Command, WorkingDir):
 629         try:
 630             self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
 631             self.CompleteFlag = True
 632
 633             # Run hash operation post dependency to account for libs
 634             # Run if --hash or --binary-destination
 635             if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
 636                 self.BuildItem.BuildObject.GenModuleHash()
 637             if GlobalData.gBinCacheDest:
 638                 self.BuildItem.BuildObject.GenCMakeHash()
 639
 640         except:
 641             #
 642             # TRICK: hide the output of threads left running, so that the user can
 643             #        catch the error message easily
 644             #
 645             if not BuildTask._ErrorFlag.isSet():
 646                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 647                                                                   self.BuildItem.BuildObject.Arch,
 648                                                                   self.BuildItem.BuildObject.ToolChain,
 649                                                                   self.BuildItem.BuildObject.BuildTarget
 650                                                                  )
 651             EdkLogger.SetLevel(EdkLogger.ERROR)
 652             BuildTask._ErrorFlag.set()
 653             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 654                                       (threading.currentThread().getName(), Command, WorkingDir)
 655
 656         # indicate there's a thread is available for another build task
 657         BuildTask._RunningQueueLock.acquire()
 658         BuildTask._RunningQueue.pop(self.BuildItem)
 659         BuildTask._RunningQueueLock.release()
 660         BuildTask._Thread.release()
 661
 662     ## Start build task thread
 663     #
 664     def Start(self):
 665         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 666         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 667         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 668         self.BuildTread.setName("build thread")
 669         self.BuildTread.setDaemon(False)
 670         self.BuildTread.start()
 671
 672 ## The class contains the information related to EFI image
 673 #
 674 class PeImageInfo():
 675     ## Constructor
 676     #
 677     # Constructor will load all required image information.
 678     #
 679     #   @param  BaseName          The full file path of image.
 680     #   @param  Guid              The GUID for image.
 681     #   @param  Arch              Arch of this image.
 682     #   @param  OutputDir         The output directory for image.
 683     #   @param  DebugDir          The debug directory for image.
 684     #   @param  ImageClass        PeImage Information
 685     #
 686     def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass):
 687         self.BaseName         = BaseName
 688         self.Guid             = Guid
 689         self.Arch             = Arch
 690         self.OutputDir        = OutputDir
 691         self.DebugDir         = DebugDir
 692         self.Image            = ImageClass
 693         self.Image.Size       = (self.Image.Size // 0x1000 + 1) * 0x1000
 694
 695 ## The class implementing the EDK2 build process
 696 #
 697 #   The build process includes:
 698 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 699 #       2. Parse DSC file of active platform
 700 #       3. Parse FDF file if any
 701 #       4. Establish build database, including parse all other files (module, package)
 702 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 703 #       6. Call build command
 704 #
 705 class Build():
 706     ## Constructor
 707     #
 708     # Constructor will load all necessary configurations, parse platform, modules
 709     # and packages and the establish a database for AutoGen.
 710     #
 711     #   @param  Target              The build command target, one of gSupportedTarget
 712     #   @param  WorkspaceDir        The directory of workspace
 713     #   @param  BuildOptions        Build options passed from command line
 714     #
 715     def __init__(self, Target, WorkspaceDir, BuildOptions,log_q):
 716         self.WorkspaceDir   = WorkspaceDir
 717         self.Target         = Target
 718         self.PlatformFile   = BuildOptions.PlatformFile
 719         self.ModuleFile     = BuildOptions.ModuleFile
 720         self.ArchList       = BuildOptions.TargetArch
 721         self.ToolChainList  = BuildOptions.ToolChain
 722         self.BuildTargetList= BuildOptions.BuildTarget
 723         self.Fdf            = BuildOptions.FdfFile
 724         self.FdList         = BuildOptions.RomImage
 725         self.FvList         = BuildOptions.FvImage
 726         self.CapList        = BuildOptions.CapName
 727         self.SilentMode     = BuildOptions.SilentMode
 728         self.ThreadNumber   = 1
 729         self.SkipAutoGen    = BuildOptions.SkipAutoGen
 730         self.Reparse        = BuildOptions.Reparse
 731         self.SkuId          = BuildOptions.SkuId
 732         if self.SkuId:
 733             GlobalData.gSKUID_CMD = self.SkuId
 734         self.ConfDirectory = BuildOptions.ConfDirectory
 735         self.SpawnMode      = True
 736         self.BuildReport    = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)
 737         self.AutoGenTime    = 0
 738         self.MakeTime       = 0
 739         self.GenFdsTime     = 0
 740         self.MakeFileName   = ""
 741         TargetObj = TargetTxtDict()
 742         ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"),"Conf")))
 743         self.TargetTxt = TargetObj.Target
 744         self.ToolDef = ToolDefObj.ToolDef
 745         GlobalData.BuildOptionPcd     = BuildOptions.OptionPcd if BuildOptions.OptionPcd else []
 746         #Set global flag for build mode
 747         GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
 748         GlobalData.gUseHashCache = BuildOptions.UseHashCache
 749         GlobalData.gBinCacheDest   = BuildOptions.BinCacheDest
 750         GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
 751         GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread
 752         GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
 753
 754         if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
 755             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
 756
 757         if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
 758             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.")
 759
 760         if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
 761             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.")
 762
 763         if GlobalData.gBinCacheSource:
 764             BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
 765             if not os.path.isabs(BinCacheSource):
 766                 BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
 767             GlobalData.gBinCacheSource = BinCacheSource
 768         else:
 769             if GlobalData.gBinCacheSource is not None:
 770                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-source.")
 771
 772         if GlobalData.gBinCacheDest:
 773             BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
 774             if not os.path.isabs(BinCacheDest):
 775                 BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
 776             GlobalData.gBinCacheDest = BinCacheDest
 777         else:
 778             if GlobalData.gBinCacheDest is not None:
 779                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-destination.")
 780
 781         GlobalData.gDatabasePath = os.path.normpath(os.path.join(GlobalData.gConfDirectory, GlobalData.gDatabasePath))
 782         if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):
 783             os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))
 784         self.Db = BuildDB
 785         self.BuildDatabase = self.Db.BuildObject
 786         self.Platform = None
 787         self.ToolChainFamily = None
 788         self.LoadFixAddress = 0
 789         self.UniFlag        = BuildOptions.Flag
 790         self.BuildModules = []
 791         self.HashSkipModules = []
 792         self.Db_Flag = False
 793         self.LaunchPrebuildFlag = False
 794         self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory, '.cache', '.PlatformBuild')
 795         if BuildOptions.CommandLength:
 796             GlobalData.gCommandMaxLength = BuildOptions.CommandLength
 797
 798         # print dot character during doing some time-consuming work
 799         self.Progress = Utils.Progressor()
 800         # print current build environment and configuration
 801         EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 802         if "PACKAGES_PATH" in os.environ:
 803             # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
 804             EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"]))))
 805         EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 806         if "EDK_TOOLS_BIN" in os.environ:
 807             # Print the same path style with WORKSPACE env.
 808             EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
 809         EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
 810         if "PYTHON3_ENABLE" in os.environ:
 811             PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]
 812             if PYTHON3_ENABLE != "TRUE":
 813                 PYTHON3_ENABLE = "FALSE"
 814             EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))
 815         if "PYTHON_COMMAND" in os.environ:
 816             EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))
 817         self.InitPreBuild()
 818         self.InitPostBuild()
 819         if self.Prebuild:
 820             EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.Prebuild))
 821         if self.Postbuild:
 822             EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))
 823         if self.Prebuild:
 824             self.LaunchPrebuild()
 825             TargetObj = TargetTxtDict()
 826             ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"), "Conf")))
 827             self.TargetTxt = TargetObj.Target
 828             self.ToolDef = ToolDefObj.ToolDef
 829         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
 830             self.InitBuild()
 831
 832         self.AutoGenMgr = None
 833         EdkLogger.info("")
 834         os.chdir(self.WorkspaceDir)
 835         self.log_q = log_q
 836         GlobalData.file_lock =  mp.Lock()
 837         # Init cache data for local only
 838         GlobalData.gPackageHashFile = dict()
 839         GlobalData.gModulePreMakeCacheStatus = dict()
 840         GlobalData.gModuleMakeCacheStatus = dict()
 841         GlobalData.gHashChainStatus = dict()
 842         GlobalData.gCMakeHashFile = dict()
 843         GlobalData.gModuleHashFile = dict()
 844         GlobalData.gFileHashDict = dict()
 845         GlobalData.gModuleAllCacheStatus = set()
 846         GlobalData.gModuleCacheHit = set()
 847
 848     def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):
 849         try:
 850             if SkipAutoGen:
 851                 return True,0
 852             feedback_q = mp.Queue()
 853             error_event = mp.Event()
 854             FfsCmd = DataPipe.Get("FfsCommand")
 855             if FfsCmd is None:
 856                 FfsCmd = {}
 857             GlobalData.FfsCmd = FfsCmd
 858             auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]
 859             self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
 860             self.AutoGenMgr.start()
 861             for w in auto_workers:
 862                 w.start()
 863             if PcdMaList is not None:
 864                 for PcdMa in PcdMaList:
 865                     # SourceFileList calling sequence impact the makefile string sequence.
 866                     # Create cached SourceFileList here to unify its calling sequence for both
 867                     # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
 868                     RetVal = PcdMa.SourceFileList
 869                     # Force cache miss for PCD driver
 870                     if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
 871                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))
 872
 873                     PcdMa.CreateCodeFile(False)
 874                     PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
 875
 876                     # Force cache miss for PCD driver
 877                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
 878                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))
 879
 880             self.AutoGenMgr.join()
 881             rt = self.AutoGenMgr.Status
 882             return rt, 0
 883         except FatalError as e:
 884             return False, e.args[0]
 885         except:
 886             return False, UNKNOWN_ERROR
 887
 888     ## Load configuration
 889     #
 890     #   This method will parse target.txt and get the build configurations.
 891     #
 892     def LoadConfiguration(self):
 893
 894         # if no ARCH given in command line, get it from target.txt
 895         if not self.ArchList:
 896             self.ArchList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET_ARCH]
 897         self.ArchList = tuple(self.ArchList)
 898
 899         # if no build target given in command line, get it from target.txt
 900         if not self.BuildTargetList:
 901             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
 902
 903         # if no tool chain given in command line, get it from target.txt
 904         if not self.ToolChainList:
 905             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 906             if self.ToolChainList is None or len(self.ToolChainList) == 0:
 907                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 908
 909         # check if the tool chains are defined or not
 910         NewToolChainList = []
 911         for ToolChain in self.ToolChainList:
 912             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 913                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 914             else:
 915                 NewToolChainList.append(ToolChain)
 916         # if no tool chain available, break the build
 917         if len(NewToolChainList) == 0:
 918             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 919                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 920         else:
 921             self.ToolChainList = NewToolChainList
 922
 923         ToolChainFamily = []
 924         ToolDefinition = self.ToolDef.ToolsDefTxtDatabase
 925         for Tool in self.ToolChainList:
 926             if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \
 927                or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]:
 928                 EdkLogger.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool)
 929                 ToolChainFamily.append(TAB_COMPILER_MSFT)
 930             else:
 931                 ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
 932         self.ToolChainFamily = ToolChainFamily
 933
 934         if not self.PlatformFile:
 935             PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 936             if not PlatformFile:
 937                 # Try to find one in current directory
 938                 WorkingDirectory = os.getcwd()
 939                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 940                 FileNum = len(FileList)
 941                 if FileNum >= 2:
 942                     EdkLogger.error("build", OPTION_MISSING,
 943                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 944                 elif FileNum == 1:
 945                     PlatformFile = FileList[0]
 946                 else:
 947                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 948                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 949
 950             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 951         self.ThreadNumber   = ThreadNum()
 952     ## Initialize build configuration
 953     #
 954     #   This method will parse DSC file and merge the configurations from
 955     #   command line and target.txt, then get the final build configurations.
 956     #
 957     def InitBuild(self):
 958         # parse target.txt, tools_def.txt, and platform file
 959         self.LoadConfiguration()
 960
 961         # Allow case-insensitive for those from command line or configuration file
 962         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 963         if ErrorCode != 0:
 964             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 965
 966
 967     def InitPreBuild(self):
 968         self.LoadConfiguration()
 969         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 970         if ErrorCode != 0:
 971             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 972         if self.BuildTargetList:
 973             GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0]
 974         if self.ArchList:
 975             GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0]
 976         if self.ToolChainList:
 977             GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0]
 978             GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0]
 979         if self.ToolChainFamily:
 980             GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
 981         if 'PREBUILD' in GlobalData.gCommandLineDefines:
 982             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
 983         else:
 984             self.Db_Flag = True
 985             Platform = self.Db.MapPlatform(str(self.PlatformFile))
 986             self.Prebuild = str(Platform.Prebuild)
 987         if self.Prebuild:
 988             PrebuildList = []
 989             #
 990             # Evaluate all arguments and convert arguments that are WORKSPACE
 991             # relative paths to absolute paths.  Filter arguments that look like
 992             # flags or do not follow the file/dir naming rules to avoid false
 993             # positives on this conversion.
 994             #
 995             for Arg in self.Prebuild.split():
 996                 #
 997                 # Do not modify Arg if it looks like a flag or an absolute file path
 998                 #
 999                 if Arg.startswith('-')  or os.path.isabs(Arg):
1000                     PrebuildList.append(Arg)
1001                     continue
1002                 #
1003                 # Do not modify Arg if it does not look like a Workspace relative
1004                 # path that starts with a valid package directory name
1005                 #
1006                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1007                     PrebuildList.append(Arg)
1008                     continue
1009                 #
1010                 # If Arg looks like a WORKSPACE relative path, then convert to an
1011                 # absolute path and check to see if the file exists.
1012                 #
1013                 Temp = mws.join(self.WorkspaceDir, Arg)
1014                 if os.path.isfile(Temp):
1015                     Arg = Temp
1016                 PrebuildList.append(Arg)
1017             self.Prebuild       = ' '.join(PrebuildList)
1018             self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1019
1020     def InitPostBuild(self):
1021         if 'POSTBUILD' in GlobalData.gCommandLineDefines:
1022             self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD')
1023         else:
1024             Platform = self.Db.MapPlatform(str(self.PlatformFile))
1025             self.Postbuild = str(Platform.Postbuild)
1026         if self.Postbuild:
1027             PostbuildList = []
1028             #
1029             # Evaluate all arguments and convert arguments that are WORKSPACE
1030             # relative paths to absolute paths.  Filter arguments that look like
1031             # flags or do not follow the file/dir naming rules to avoid false
1032             # positives on this conversion.
1033             #
1034             for Arg in self.Postbuild.split():
1035                 #
1036                 # Do not modify Arg if it looks like a flag or an absolute file path
1037                 #
1038                 if Arg.startswith('-')  or os.path.isabs(Arg):
1039                     PostbuildList.append(Arg)
1040                     continue
1041                 #
1042                 # Do not modify Arg if it does not look like a Workspace relative
1043                 # path that starts with a valid package directory name
1044                 #
1045                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1046                     PostbuildList.append(Arg)
1047                     continue
1048                 #
1049                 # If Arg looks like a WORKSPACE relative path, then convert to an
1050                 # absolute path and check to see if the file exists.
1051                 #
1052                 Temp = mws.join(self.WorkspaceDir, Arg)
1053                 if os.path.isfile(Temp):
1054                     Arg = Temp
1055                 PostbuildList.append(Arg)
1056             self.Postbuild       = ' '.join(PostbuildList)
1057             self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1058
1059     def PassCommandOption(self, BuildTarget, TargetArch, ToolChain, PlatformFile, Target):
1060         BuildStr = ''
1061         if GlobalData.gCommand and isinstance(GlobalData.gCommand, list):
1062             BuildStr += ' ' + ' '.join(GlobalData.gCommand)
1063         TargetFlag = False
1064         ArchFlag = False
1065         ToolChainFlag = False
1066         PlatformFileFlag = False
1067
1068         if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget:
1069             TargetFlag = True
1070         if GlobalData.gOptions and not GlobalData.gOptions.TargetArch:
1071             ArchFlag = True
1072         if GlobalData.gOptions and not GlobalData.gOptions.ToolChain:
1073             ToolChainFlag = True
1074         if GlobalData.gOptions and not GlobalData.gOptions.PlatformFile:
1075             PlatformFileFlag = True
1076
1077         if TargetFlag and BuildTarget:
1078             if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple):
1079                 BuildStr += ' -b ' + ' -b '.join(BuildTarget)
1080             elif isinstance(BuildTarget, str):
1081                 BuildStr += ' -b ' + BuildTarget
1082         if ArchFlag and TargetArch:
1083             if isinstance(TargetArch, list) or isinstance(TargetArch, tuple):
1084                 BuildStr += ' -a ' + ' -a '.join(TargetArch)
1085             elif isinstance(TargetArch, str):
1086                 BuildStr += ' -a ' + TargetArch
1087         if ToolChainFlag and ToolChain:
1088             if isinstance(ToolChain, list) or isinstance(ToolChain, tuple):
1089                 BuildStr += ' -t ' + ' -t '.join(ToolChain)
1090             elif isinstance(ToolChain, str):
1091                 BuildStr += ' -t ' + ToolChain
1092         if PlatformFileFlag and PlatformFile:
1093             if isinstance(PlatformFile, list) or isinstance(PlatformFile, tuple):
1094                 BuildStr += ' -p ' + ' -p '.join(PlatformFile)
1095             elif isinstance(PlatformFile, str):
1096                 BuildStr += ' -p' + PlatformFile
1097         BuildStr += ' --conf=' + GlobalData.gConfDirectory
1098         if Target:
1099             BuildStr += ' ' + Target
1100
1101         return BuildStr
1102
1103     def LaunchPrebuild(self):
1104         if self.Prebuild:
1105             EdkLogger.info("\n- Prebuild Start -\n")
1106             self.LaunchPrebuildFlag = True
1107             #
1108             # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1109             # and preserve them for the rest of the main build step, because the child process environment will
1110             # evaporate as soon as it exits, we cannot get it in build step.
1111             #
1112             PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory, '.cache', '.PrebuildEnv')
1113             if os.path.isfile(PrebuildEnvFile):
1114                 os.remove(PrebuildEnvFile)
1115             if os.path.isfile(self.PlatformBuildPath):
1116                 os.remove(self.PlatformBuildPath)
1117             if sys.platform == "win32":
1118                 args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile))
1119                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1120             else:
1121                 args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile))
1122                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1123
1124             # launch two threads to read the STDOUT and STDERR
1125             EndOfProcedure = Event()
1126             EndOfProcedure.clear()
1127             if Process.stdout:
1128                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1129                 StdOutThread.setName("STDOUT-Redirector")
1130                 StdOutThread.setDaemon(False)
1131                 StdOutThread.start()
1132
1133             if Process.stderr:
1134                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1135                 StdErrThread.setName("STDERR-Redirector")
1136                 StdErrThread.setDaemon(False)
1137                 StdErrThread.start()
1138             # waiting for program exit
1139             Process.wait()
1140
1141             if Process.stdout:
1142                 StdOutThread.join()
1143             if Process.stderr:
1144                 StdErrThread.join()
1145             if Process.returncode != 0 :
1146                 EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!')
1147
1148             if os.path.exists(PrebuildEnvFile):
1149                 f = open(PrebuildEnvFile)
1150                 envs = f.readlines()
1151                 f.close()
1152                 envs = [l.split("=", 1) for l in envs ]
1153                 envs = [[I.strip() for I in item] for item in envs if len(item) == 2]
1154                 os.environ.update(dict(envs))
1155             EdkLogger.info("\n- Prebuild Done -\n")
1156
1157     def LaunchPostbuild(self):
1158         if self.Postbuild:
1159             EdkLogger.info("\n- Postbuild Start -\n")
1160             if sys.platform == "win32":
1161                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1162             else:
1163                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1164             # launch two threads to read the STDOUT and STDERR
1165             EndOfProcedure = Event()
1166             EndOfProcedure.clear()
1167             if Process.stdout:
1168                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1169                 StdOutThread.setName("STDOUT-Redirector")
1170                 StdOutThread.setDaemon(False)
1171                 StdOutThread.start()
1172
1173             if Process.stderr:
1174                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1175                 StdErrThread.setName("STDERR-Redirector")
1176                 StdErrThread.setDaemon(False)
1177                 StdErrThread.start()
1178             # waiting for program exit
1179             Process.wait()
1180
1181             if Process.stdout:
1182                 StdOutThread.join()
1183             if Process.stderr:
1184                 StdErrThread.join()
1185             if Process.returncode != 0 :
1186                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
1187             EdkLogger.info("\n- Postbuild Done -\n")
1188
1189     ## Build a module or platform
1190     #
1191     # Create autogen code and makefile for a module or platform, and the launch
1192     # "make" command to build it
1193     #
1194     #   @param  Target                      The target of build command
1195     #   @param  Platform                    The platform file
1196     #   @param  Module                      The module file
1197     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1198     #   @param  ToolChain                   The name of toolchain to build
1199     #   @param  Arch                        The arch of the module/platform
1200     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1201     #                                       for dependent modules/Libraries
1202     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1203     #                                       for dependent modules/Libraries
1204     #
1205     def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):
1206         if AutoGenObject is None:
1207             return False
1208         if FfsCommand is None:
1209             FfsCommand = {}
1210         # skip file generation for cleanxxx targets, run and fds target
1211         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1212             # for target which must generate AutoGen code and makefile
1213             mqueue = mp.Queue()
1214             for m in AutoGenObject.GetAllModuleInfo:
1215                 mqueue.put(m)
1216
1217             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
1218             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
1219             AutoGenObject.CreateLibModuelDirs()
1220             AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
1221             AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
1222             AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
1223             self.Progress.Start("Generating makefile and code")
1224             data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
1225             AutoGenObject.DataPipe.dump(data_pipe_file)
1226             cqueue = mp.Queue()
1227             autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
1228             AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
1229             with open(AutoGenIdFile,"w") as fw:
1230                 fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
1231                 fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
1232                 fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
1233             self.Progress.Stop("done!")
1234             if not autogen_rt:
1235                 self.AutoGenMgr.TerminateWorkers()
1236                 self.AutoGenMgr.join(1)
1237                 raise FatalError(errorcode)
1238             AutoGenObject.CreateCodeFile(False)
1239             AutoGenObject.CreateMakeFile(False)
1240         else:
1241             # always recreate top/platform makefile when clean, just in case of inconsistency
1242             AutoGenObject.CreateCodeFile(True)
1243             AutoGenObject.CreateMakeFile(True)
1244
1245         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1246             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1247
1248         BuildCommand = AutoGenObject.BuildCommand
1249         if BuildCommand is None or len(BuildCommand) == 0:
1250             EdkLogger.error("build", OPTION_MISSING,
1251                             "No build command found for this module. "
1252                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1253                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1254                             ExtraData=str(AutoGenObject))
1255
1256         # run
1257         if Target == 'run':
1258             return True
1259
1260         # build modules
1261         if BuildModule:
1262             BuildCommand = BuildCommand + [Target]
1263             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1264             self.CreateAsBuiltInf()
1265             if GlobalData.gBinCacheDest:
1266                 self.GenDestCache()
1267             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1268                 # Only for --hash
1269                 # Update PreMakeCacheChain files
1270                 self.GenLocalPreMakeCache()
1271             self.BuildModules = []
1272             return True
1273
1274         # build library
1275         if Target == 'libraries':
1276             DirList = []
1277             for Lib in AutoGenObject.LibraryAutoGenList:
1278                 if not Lib.IsBinaryModule:
1279                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1280             for Lib, LibAutoGen in DirList:
1281                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']
1282                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1283             return True
1284
1285         # build module
1286         if Target == 'modules':
1287             DirList = []
1288             for Lib in AutoGenObject.LibraryAutoGenList:
1289                 if not Lib.IsBinaryModule:
1290                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1291             for Lib, LibAutoGen in DirList:
1292                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']
1293                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1294
1295             DirList = []
1296             for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
1297                 if not ModuleAutoGen.IsBinaryModule:
1298                     DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
1299             for Mod,ModAutoGen in DirList:
1300                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, self.MakeFileName)), 'pbuild']
1301                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
1302             self.CreateAsBuiltInf()
1303             if GlobalData.gBinCacheDest:
1304                 self.GenDestCache()
1305             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1306                 # Only for --hash
1307                 # Update PreMakeCacheChain files
1308                 self.GenLocalPreMakeCache()
1309             self.BuildModules = []
1310             return True
1311
1312         # cleanlib
1313         if Target == 'cleanlib':
1314             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1315                 LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))
1316                 if os.path.exists(LibMakefile):
1317                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1318                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1319             return True
1320
1321         # clean
1322         if Target == 'clean':
1323             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1324                 ModMakefile = os.path.normpath(os.path.join(Mod, self.MakeFileName))
1325                 if os.path.exists(ModMakefile):
1326                     NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']
1327                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1328             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1329                 LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))
1330                 if os.path.exists(LibMakefile):
1331                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1332                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1333             return True
1334
1335         # cleanall
1336         if Target == 'cleanall':
1337             try:
1338                 #os.rmdir(AutoGenObject.BuildDir)
1339                 RemoveDirectory(AutoGenObject.BuildDir, True)
1340             except WindowsError as X:
1341                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1342         return True
1343
1344     ## Build a module or platform
1345     #
1346     # Create autogen code and makefile for a module or platform, and the launch
1347     # "make" command to build it
1348     #
1349     #   @param  Target                      The target of build command
1350     #   @param  Platform                    The platform file
1351     #   @param  Module                      The module file
1352     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1353     #   @param  ToolChain                   The name of toolchain to build
1354     #   @param  Arch                        The arch of the module/platform
1355     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1356     #                                       for dependent modules/Libraries
1357     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1358     #                                       for dependent modules/Libraries
1359     #
1360     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1361         if AutoGenObject is None:
1362             return False
1363
1364         # skip file generation for cleanxxx targets, run and fds target
1365         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1366             # for target which must generate AutoGen code and makefile
1367             if not self.SkipAutoGen or Target == 'genc':
1368                 self.Progress.Start("Generating code")
1369                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1370                 self.Progress.Stop("done!")
1371             if Target == "genc":
1372                 return True
1373
1374             if not self.SkipAutoGen or Target == 'genmake':
1375                 self.Progress.Start("Generating makefile")
1376                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1377                 #AutoGenObject.CreateAsBuiltInf()
1378                 self.Progress.Stop("done!")
1379             if Target == "genmake":
1380                 return True
1381         else:
1382             # always recreate top/platform makefile when clean, just in case of inconsistency
1383             AutoGenObject.CreateCodeFile(True)
1384             AutoGenObject.CreateMakeFile(True)
1385
1386         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1387             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1388
1389         BuildCommand = AutoGenObject.BuildCommand
1390         if BuildCommand is None or len(BuildCommand) == 0:
1391             EdkLogger.error("build", OPTION_MISSING,
1392                             "No build command found for this module. "
1393                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1394                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1395                             ExtraData=str(AutoGenObject))
1396
1397         # build modules
1398         if BuildModule:
1399             if Target != 'fds':
1400                 BuildCommand = BuildCommand + [Target]
1401             AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1402             self.CreateAsBuiltInf()
1403             if GlobalData.gBinCacheDest:
1404                 self.GenDestCache()
1405             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1406                 # Only for --hash
1407                 # Update PreMakeCacheChain files
1408                 self.GenLocalPreMakeCache()
1409             self.BuildModules = []
1410             return True
1411
1412         # genfds
1413         if Target == 'fds':
1414             if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db):
1415                 EdkLogger.error("build", COMMAND_FAILURE)
1416             return True
1417
1418         # run
1419         if Target == 'run':
1420             return True
1421
1422         # build library
1423         if Target == 'libraries':
1424             pass
1425
1426         # not build modules
1427
1428
1429         # cleanall
1430         if Target == 'cleanall':
1431             try:
1432                 #os.rmdir(AutoGenObject.BuildDir)
1433                 RemoveDirectory(AutoGenObject.BuildDir, True)
1434             except WindowsError as X:
1435                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1436         return True
1437
1438     ## Rebase module image and Get function address for the input module list.
1439     #
1440     def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False):
1441         if ModeIsSmm:
1442             AddrIsOffset = False
1443         for InfFile in ModuleList:
1444             sys.stdout.write (".")
1445             sys.stdout.flush()
1446             ModuleInfo = ModuleList[InfFile]
1447             ModuleName = ModuleInfo.BaseName
1448             ModuleOutputImage = ModuleInfo.Image.FileName
1449             ModuleDebugImage  = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi')
1450             ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1451             if not ModeIsSmm:
1452                 BaseAddress = BaseAddress - ModuleInfo.Image.Size
1453                 #
1454                 # Update Image to new BaseAddress by GenFw tool
1455                 #
1456                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1457                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1458             else:
1459                 #
1460                 # Set new address to the section header only for SMM driver.
1461                 #
1462                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1463                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1464             #
1465             # Collect function address from Map file
1466             #
1467             ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
1468             FunctionList = []
1469             if os.path.exists(ImageMapTable):
1470                 OrigImageBaseAddress = 0
1471                 ImageMap = open(ImageMapTable, 'r')
1472                 for LinStr in ImageMap:
1473                     if len (LinStr.strip()) == 0:
1474                         continue
1475                     #
1476                     # Get the preferred address set on link time.
1477                     #
1478                     if LinStr.find ('Preferred load address is') != -1:
1479                         StrList = LinStr.split()
1480                         OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16)
1481
1482                     StrList = LinStr.split()
1483                     if len (StrList) > 4:
1484                         if StrList[3] == 'f' or StrList[3] == 'F':
1485                             Name = StrList[1]
1486                             RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress
1487                             FunctionList.append ((Name, RelativeAddress))
1488
1489                 ImageMap.close()
1490             #
1491             # Add general information.
1492             #
1493             if ModeIsSmm:
1494                 MapBuffer.append('\n\n%s (Fixed SMRAM Offset,   BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1495             elif AddrIsOffset:
1496                 MapBuffer.append('\n\n%s (Fixed Memory Offset,  BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
1497             else:
1498                 MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1499             #
1500             # Add guid and general seciton section.
1501             #
1502             TextSectionAddress = 0
1503             DataSectionAddress = 0
1504             for SectionHeader in ModuleInfo.Image.SectionHeaderList:
1505                 if SectionHeader[0] == '.text':
1506                     TextSectionAddress = SectionHeader[1]
1507                 elif SectionHeader[0] in ['.data', '.sdata']:
1508                     DataSectionAddress = SectionHeader[1]
1509             if AddrIsOffset:
1510                 MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
1511             else:
1512                 MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
1513             #
1514             # Add debug image full path.
1515             #
1516             MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))
1517             #
1518             # Add function address
1519             #
1520             for Function in FunctionList:
1521                 if AddrIsOffset:
1522                     MapBuffer.append('  -0x%010X    %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
1523                 else:
1524                     MapBuffer.append('  0x%010X    %s\n' % (BaseAddress + Function[1], Function[0]))
1525             ImageMap.close()
1526
1527             #
1528             # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1529             #
1530             if ModeIsSmm:
1531                 BaseAddress = BaseAddress + ModuleInfo.Image.Size
1532
1533     ## Collect MAP information of all FVs
1534     #
1535     def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList):
1536         if self.Fdf:
1537             # First get the XIP base address for FV map file.
1538             GuidPattern = re.compile("[-a-fA-F0-9]+")
1539             GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+")
1540             for FvName in Wa.FdfProfile.FvDict:
1541                 FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map')
1542                 if not os.path.exists(FvMapBuffer):
1543                     continue
1544                 FvMap = open(FvMapBuffer, 'r')
1545                 #skip FV size information
1546                 FvMap.readline()
1547                 FvMap.readline()
1548                 FvMap.readline()
1549                 FvMap.readline()
1550                 for Line in FvMap:
1551                     MatchGuid = GuidPattern.match(Line)
1552                     if MatchGuid is not None:
1553                         #
1554                         # Replace GUID with module name
1555                         #
1556                         GuidString = MatchGuid.group()
1557                         if GuidString.upper() in ModuleList:
1558                             Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
1559                     MapBuffer.append(Line)
1560                     #
1561                     # Add the debug image full path.
1562                     #
1563                     MatchGuid = GuidName.match(Line)
1564                     if MatchGuid is not None:
1565                         GuidString = MatchGuid.group().split("=")[1]
1566                         if GuidString.upper() in ModuleList:
1567                             MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
1568
1569                 FvMap.close()
1570
1571     ## Collect MAP information of all modules
1572     #
1573     def _CollectModuleMapBuffer (self, MapBuffer, ModuleList):
1574         sys.stdout.write ("Generate Load Module At Fix Address Map")
1575         sys.stdout.flush()
1576         PatchEfiImageList = []
1577         PeiModuleList  = {}
1578         BtModuleList   = {}
1579         RtModuleList   = {}
1580         SmmModuleList  = {}
1581         PeiSize = 0
1582         BtSize  = 0
1583         RtSize  = 0
1584         # reserve 4K size in SMRAM to make SMM module address not from 0.
1585         SmmSize = 0x1000
1586         for ModuleGuid in ModuleList:
1587             Module = ModuleList[ModuleGuid]
1588             GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget)
1589
1590             OutputImageFile = ''
1591             for ResultFile in Module.CodaTargetList:
1592                 if str(ResultFile.Target).endswith('.efi'):
1593                     #
1594                     # module list for PEI, DXE, RUNTIME and SMM
1595                     #
1596                     OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi')
1597                     ImageClass = PeImageClass (OutputImageFile)
1598                     if not ImageClass.IsValid:
1599                         EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo)
1600                     ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass)
1601                     if Module.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_PEIM, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER, EDK_COMPONENT_TYPE_PIC_PEIM, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM, SUP_MODULE_DXE_CORE]:
1602                         PeiModuleList[Module.MetaFile] = ImageInfo
1603                         PeiSize += ImageInfo.Image.Size
1604                     elif Module.ModuleType in [EDK_COMPONENT_TYPE_BS_DRIVER, SUP_MODULE_DXE_DRIVER, SUP_MODULE_UEFI_DRIVER]:
1605                         BtModuleList[Module.MetaFile] = ImageInfo
1606                         BtSize += ImageInfo.Image.Size
1607                     elif Module.ModuleType in [SUP_MODULE_DXE_RUNTIME_DRIVER, EDK_COMPONENT_TYPE_RT_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, EDK_COMPONENT_TYPE_SAL_RT_DRIVER]:
1608                         RtModuleList[Module.MetaFile] = ImageInfo
1609                         RtSize += ImageInfo.Image.Size
1610                     elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]:
1611                         SmmModuleList[Module.MetaFile] = ImageInfo
1612                         SmmSize += ImageInfo.Image.Size
1613                         if Module.ModuleType == SUP_MODULE_DXE_SMM_DRIVER:
1614                             PiSpecVersion = Module.Module.Specification.get('PI_SPECIFICATION_VERSION', '0x00000000')
1615                             # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1616                             if int(PiSpecVersion, 16) < 0x0001000A:
1617                                 BtModuleList[Module.MetaFile] = ImageInfo
1618                                 BtSize += ImageInfo.Image.Size
1619                     break
1620             #
1621             # EFI image is final target.
1622             # Check EFI image contains patchable FixAddress related PCDs.
1623             #
1624             if OutputImageFile != '':
1625                 ModuleIsPatch = False
1626                 for Pcd in Module.ModulePcdList:
1627                     if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1628                         ModuleIsPatch = True
1629                         break
1630                 if not ModuleIsPatch:
1631                     for Pcd in Module.LibraryPcdList:
1632                         if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1633                             ModuleIsPatch = True
1634                             break
1635
1636                 if not ModuleIsPatch:
1637                     continue
1638                 #
1639                 # Module includes the patchable load fix address PCDs.
1640                 # It will be fixed up later.
1641                 #
1642                 PatchEfiImageList.append (OutputImageFile)
1643
1644         #
1645         # Get Top Memory address
1646         #
1647         ReservedRuntimeMemorySize = 0
1648         TopMemoryAddress = 0
1649         if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF:
1650             TopMemoryAddress = 0
1651         else:
1652             TopMemoryAddress = self.LoadFixAddress
1653             if TopMemoryAddress < RtSize + BtSize + PeiSize:
1654                 EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1655
1656         #
1657         # Patch FixAddress related PCDs into EFI image
1658         #
1659         for EfiImage in PatchEfiImageList:
1660             EfiImageMap = EfiImage.replace('.efi', '.map')
1661             if not os.path.exists(EfiImageMap):
1662                 continue
1663             #
1664             # Get PCD offset in EFI image by GenPatchPcdTable function
1665             #
1666             PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage)
1667             #
1668             # Patch real PCD value by PatchPcdValue tool
1669             #
1670             for PcdInfo in PcdTable:
1671                 ReturnValue = 0
1672                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:
1673                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000))
1674                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:
1675                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000))
1676                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:
1677                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000))
1678                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:
1679                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000))
1680                 if ReturnValue != 0:
1681                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
1682
1683         MapBuffer.append('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize // 0x1000))
1684         MapBuffer.append('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize // 0x1000))
1685         MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize // 0x1000))
1686         if len (SmmModuleList) > 0:
1687             MapBuffer.append('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize // 0x1000))
1688
1689         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
1690         BtBaseAddr  = TopMemoryAddress - RtSize
1691         RtBaseAddr  = TopMemoryAddress - ReservedRuntimeMemorySize
1692
1693         self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0)
1694         self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
1695         self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
1696         self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
1697         MapBuffer.append('\n\n')
1698         sys.stdout.write ("\n")
1699         sys.stdout.flush()
1700
1701     ## Save platform Map file
1702     #
1703     def _SaveMapFile (self, MapBuffer, Wa):
1704         #
1705         # Map file path is got.
1706         #
1707         MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map')
1708         #
1709         # Save address map into MAP file.
1710         #
1711         SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)
1712         if self.LoadFixAddress != 0:
1713             sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
1714         sys.stdout.flush()
1715
1716     ## Build active platform for different build targets and different tool chains
1717     #
1718     def _BuildPlatform(self):
1719         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1720         for BuildTarget in self.BuildTargetList:
1721             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1722             index = 0
1723             for ToolChain in self.ToolChainList:
1724                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1725                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1726                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1727                 index += 1
1728                 Wa = WorkspaceAutoGen(
1729                         self.WorkspaceDir,
1730                         self.PlatformFile,
1731                         BuildTarget,
1732                         ToolChain,
1733                         self.ArchList,
1734                         self.BuildDatabase,
1735                         self.TargetTxt,
1736                         self.ToolDef,
1737                         self.Fdf,
1738                         self.FdList,
1739                         self.FvList,
1740                         self.CapList,
1741                         self.SkuId,
1742                         self.UniFlag,
1743                         self.Progress
1744                         )
1745                 self.Fdf = Wa.FdfFile
1746                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1747                 self.BuildReport.AddPlatformReport(Wa)
1748                 self.Progress.Stop("done!")
1749
1750                 # Add ffs build to makefile
1751                 CmdListDict = {}
1752                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1753                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1754
1755                 for Arch in Wa.ArchList:
1756                     PcdMaList    = []
1757                     GlobalData.gGlobalDefines['ARCH'] = Arch
1758                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1759                     for Module in Pa.Platform.Modules:
1760                         # Get ModuleAutoGen object to generate C code file and makefile
1761                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1762                         if Ma is None:
1763                             continue
1764                         if Ma.PcdIsDriver:
1765                             Ma.PlatformInfo = Pa
1766                             Ma.Workspace = Wa
1767                             PcdMaList.append(Ma)
1768                         self.BuildModules.append(Ma)
1769                     Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
1770                     Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
1771                     self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
1772
1773                 # Create MAP file when Load Fix Address is enabled.
1774                 if self.Target in ["", "all", "fds"]:
1775                     for Arch in Wa.ArchList:
1776                         GlobalData.gGlobalDefines['ARCH'] = Arch
1777                         #
1778                         # Check whether the set fix address is above 4G for 32bit image.
1779                         #
1780                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1781                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1782                     #
1783                     # Get Module List
1784                     #
1785                     ModuleList = {}
1786                     for Pa in Wa.AutoGenObjectList:
1787                         for Ma in Pa.ModuleAutoGenList:
1788                             if Ma is None:
1789                                 continue
1790                             if not Ma.IsLibrary:
1791                                 ModuleList[Ma.Guid.upper()] = Ma
1792
1793                     MapBuffer = []
1794                     if self.LoadFixAddress != 0:
1795                         #
1796                         # Rebase module to the preferred memory address before GenFds
1797                         #
1798                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1799                     if self.Fdf:
1800                         #
1801                         # create FDS again for the updated EFI image
1802                         #
1803                         self._Build("fds", Wa)
1804                         #
1805                         # Create MAP file for all platform FVs after GenFds.
1806                         #
1807                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1808                     #
1809                     # Save MAP buffer into MAP file.
1810                     #
1811                     self._SaveMapFile (MapBuffer, Wa)
1812                 self.CreateGuidedSectionToolsFile(Wa)
1813
1814     ## Build active module for different build targets, different tool chains and different archs
1815     #
1816     def _BuildModule(self):
1817         for BuildTarget in self.BuildTargetList:
1818             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1819             index = 0
1820             for ToolChain in self.ToolChainList:
1821                 WorkspaceAutoGenTime = time.time()
1822                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1823                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1824                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1825                 index += 1
1826                 #
1827                 # module build needs platform build information, so get platform
1828                 # AutoGen first
1829                 #
1830                 Wa = WorkspaceAutoGen(
1831                         self.WorkspaceDir,
1832                         self.PlatformFile,
1833                         BuildTarget,
1834                         ToolChain,
1835                         self.ArchList,
1836                         self.BuildDatabase,
1837                         self.TargetTxt,
1838                         self.ToolDef,
1839                         self.Fdf,
1840                         self.FdList,
1841                         self.FvList,
1842                         self.CapList,
1843                         self.SkuId,
1844                         self.UniFlag,
1845                         self.Progress,
1846                         self.ModuleFile
1847                         )
1848                 self.Fdf = Wa.FdfFile
1849                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1850                 Wa.CreateMakeFile(False)
1851                 # Add ffs build to makefile
1852                 CmdListDict = None
1853                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1854                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1855
1856                 GlobalData.file_lock = mp.Lock()
1857                 GlobalData.FfsCmd = CmdListDict
1858
1859                 self.Progress.Stop("done!")
1860                 MaList = []
1861                 ExitFlag = threading.Event()
1862                 ExitFlag.clear()
1863                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
1864                 for Arch in Wa.ArchList:
1865                     AutoGenStart = time.time()
1866                     GlobalData.gGlobalDefines['ARCH'] = Arch
1867                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1868                     for Module in Pa.Platform.Modules:
1869                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
1870                             Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1871                             if Ma is None:
1872                                 continue
1873                             if Ma.PcdIsDriver:
1874                                 Ma.PlatformInfo = Pa
1875                                 Ma.Workspace = Wa
1876                             MaList.append(Ma)
1877
1878                             if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
1879                                 if Ma.CanSkipbyPreMakeCache():
1880                                     continue
1881                                 else:
1882                                     self.PreMakeCacheMiss.add(Ma)
1883
1884                             # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1885                             if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1886                                 # for target which must generate AutoGen code and makefile
1887                                 if not self.SkipAutoGen or self.Target == 'genc':
1888                                     self.Progress.Start("Generating code")
1889                                     Ma.CreateCodeFile(True)
1890                                     self.Progress.Stop("done!")
1891                                 if self.Target == "genc":
1892                                     return True
1893                                 if not self.SkipAutoGen or self.Target == 'genmake':
1894                                     self.Progress.Start("Generating makefile")
1895                                     if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:
1896                                         Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])
1897                                         del CmdListDict[Module.Path, Arch]
1898                                     else:
1899                                         Ma.CreateMakeFile(True)
1900                                     self.Progress.Stop("done!")
1901                                 if self.Target == "genmake":
1902                                     return True
1903
1904                                 if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
1905                                     if Ma.CanSkipbyMakeCache():
1906                                         continue
1907                                     else:
1908                                         self.MakeCacheMiss.add(Ma)
1909
1910                             self.BuildModules.append(Ma)
1911                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
1912                     MakeStart = time.time()
1913                     for Ma in self.BuildModules:
1914                         if not Ma.IsBinaryModule:
1915                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
1916                         # Break build if any build thread has error
1917                         if BuildTask.HasError():
1918                             # we need a full version of makefile for platform
1919                             ExitFlag.set()
1920                             BuildTask.WaitForComplete()
1921                             Pa.CreateMakeFile(False)
1922                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1923                         # Start task scheduler
1924                         if not BuildTask.IsOnGoing():
1925                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1926
1927                     # in case there's an interruption. we need a full version of makefile for platform
1928                     Pa.CreateMakeFile(False)
1929                     if BuildTask.HasError():
1930                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1931                     self.MakeTime += int(round((time.time() - MakeStart)))
1932
1933                 MakeContiue = time.time()
1934                 ExitFlag.set()
1935                 BuildTask.WaitForComplete()
1936                 self.CreateAsBuiltInf()
1937                 if GlobalData.gBinCacheDest:
1938                     self.GenDestCache()
1939                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1940                     # Only for --hash
1941                     # Update PreMakeCacheChain files
1942                     self.GenLocalPreMakeCache()
1943                 self.BuildModules = []
1944                 self.MakeTime += int(round((time.time() - MakeContiue)))
1945                 if BuildTask.HasError():
1946                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1947
1948                 self.BuildReport.AddPlatformReport(Wa, MaList)
1949                 if MaList == []:
1950                     EdkLogger.error(
1951                                 'build',
1952                                 BUILD_ERROR,
1953                                 "Module for [%s] is not a component of active platform."\
1954                                 " Please make sure that the ARCH and inf file path are"\
1955                                 " given in the same as in [%s]" % \
1956                                     (', '.join(Wa.ArchList), self.PlatformFile),
1957                                 ExtraData=self.ModuleFile
1958                                 )
1959                 # Create MAP file when Load Fix Address is enabled.
1960                 if self.Target == "fds" and self.Fdf:
1961                     for Arch in Wa.ArchList:
1962                         #
1963                         # Check whether the set fix address is above 4G for 32bit image.
1964                         #
1965                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1966                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1967                     #
1968                     # Get Module List
1969                     #
1970                     ModuleList = {}
1971                     for Pa in Wa.AutoGenObjectList:
1972                         for Ma in Pa.ModuleAutoGenList:
1973                             if Ma is None:
1974                                 continue
1975                             if not Ma.IsLibrary:
1976                                 ModuleList[Ma.Guid.upper()] = Ma
1977
1978                     MapBuffer = []
1979                     if self.LoadFixAddress != 0:
1980                         #
1981                         # Rebase module to the preferred memory address before GenFds
1982                         #
1983                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1984                     #
1985                     # create FDS again for the updated EFI image
1986                     #
1987                     GenFdsStart = time.time()
1988                     self._Build("fds", Wa)
1989                     self.GenFdsTime += int(round((time.time() - GenFdsStart)))
1990                     #
1991                     # Create MAP file for all platform FVs after GenFds.
1992                     #
1993                     self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1994                     #
1995                     # Save MAP buffer into MAP file.
1996                     #
1997                     self._SaveMapFile (MapBuffer, Wa)
1998
1999     def _GenFfsCmd(self,ArchList):
2000         # convert dictionary of Cmd:(Inf,Arch)
2001         # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
2002         CmdSetDict = defaultdict(set)
2003         GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
2004         for Cmd in GenFfsDict:
2005             tmpInf, tmpArch = GenFfsDict[Cmd]
2006             CmdSetDict[tmpInf, tmpArch].add(Cmd)
2007         return CmdSetDict
2008     def VerifyAutoGenFiles(self):
2009         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2010         try:
2011             with open(AutoGenIdFile) as fd:
2012                 lines = fd.readlines()
2013         except:
2014             return None
2015         for line in lines:
2016             if "Arch" in line:
2017                 ArchList = line.strip().split("=")[1].split("|")
2018             if "BuildDir" in line:
2019                 BuildDir = line.split("=")[1].strip()
2020             if "PlatformGuid" in line:
2021                 PlatformGuid = line.split("=")[1].strip()
2022         GlobalVarList = []
2023         for arch in ArchList:
2024             global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
2025             if not os.path.exists(global_var):
2026                 return None
2027             GlobalVarList.append(global_var)
2028         for global_var in GlobalVarList:
2029             data_pipe = MemoryDataPipe()
2030             data_pipe.load(global_var)
2031             target = data_pipe.Get("P_Info").get("Target")
2032             toolchain = data_pipe.Get("P_Info").get("ToolChain")
2033             archlist = data_pipe.Get("P_Info").get("ArchList")
2034             Arch = data_pipe.Get("P_Info").get("Arch")
2035             active_p = data_pipe.Get("P_Info").get("ActivePlatform")
2036             workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
2037             PackagesPath = os.getenv("PACKAGES_PATH")
2038             mws.setWs(workspacedir, PackagesPath)
2039             LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
2040             ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
2041
2042             for m_build_dir in LibraryBuildDirectoryList:
2043                 if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):
2044                     return None
2045             for m_build_dir in ModuleBuildDirectoryList:
2046                 if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):
2047                     return None
2048             Wa = WorkSpaceInfo(
2049                 workspacedir,active_p,target,toolchain,archlist
2050                 )
2051             Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
2052             Wa.AutoGenObjectList.append(Pa)
2053         return Wa
2054     def SetupMakeSetting(self,Wa):
2055         BuildModules = []
2056         for Pa in Wa.AutoGenObjectList:
2057             for m in Pa._MbList:
2058                 ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
2059                 BuildModules.append(ma)
2060         fdf_file = Wa.FlashDefinition
2061         if fdf_file:
2062             Fdf = FdfParser(fdf_file.Path)
2063             Fdf.ParseFile()
2064             GlobalData.gFdfParser = Fdf
2065             if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
2066                 FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
2067                 for FdRegion in FdDict.RegionList:
2068                     if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
2069                         if int(FdRegion.Offset) % 8 != 0:
2070                             EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
2071             Wa.FdfProfile = Fdf.Profile
2072             self.Fdf = Fdf
2073         else:
2074             self.Fdf = None
2075         return BuildModules
2076
2077     ## Build a platform in multi-thread mode
2078     #
2079     def PerformAutoGen(self,BuildTarget,ToolChain):
2080         WorkspaceAutoGenTime = time.time()
2081         Wa = WorkspaceAutoGen(
2082                 self.WorkspaceDir,
2083                 self.PlatformFile,
2084                 BuildTarget,
2085                 ToolChain,
2086                 self.ArchList,
2087                 self.BuildDatabase,
2088                 self.TargetTxt,
2089                 self.ToolDef,
2090                 self.Fdf,
2091                 self.FdList,
2092                 self.FvList,
2093                 self.CapList,
2094                 self.SkuId,
2095                 self.UniFlag,
2096                 self.Progress
2097                 )
2098         self.Fdf = Wa.FdfFile
2099         self.LoadFixAddress = Wa.Platform.LoadFixAddress
2100         self.BuildReport.AddPlatformReport(Wa)
2101         Wa.CreateMakeFile(False)
2102
2103         # Add ffs build to makefile
2104         CmdListDict = {}
2105         if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
2106             CmdListDict = self._GenFfsCmd(Wa.ArchList)
2107
2108         self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
2109         BuildModules = []
2110         for Arch in Wa.ArchList:
2111             PcdMaList    = []
2112             AutoGenStart = time.time()
2113             GlobalData.gGlobalDefines['ARCH'] = Arch
2114             Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
2115             if Pa is None:
2116                 continue
2117             ModuleList = []
2118             for Inf in Pa.Platform.Modules:
2119                 ModuleList.append(Inf)
2120             # Add the INF only list in FDF
2121             if GlobalData.gFdfParser is not None:
2122                 for InfName in GlobalData.gFdfParser.Profile.InfList:
2123                     Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
2124                     if Inf in Pa.Platform.Modules:
2125                         continue
2126                     ModuleList.append(Inf)
2127             Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
2128             Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
2129             Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
2130             Pa.CreateLibModuelDirs()
2131             # Fetch the MakeFileName.
2132             self.MakeFileName = Pa.MakeFileName
2133             if not self.MakeFileName:
2134                 self.MakeFileName = Pa.MakeFile
2135
2136             Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
2137             Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
2138             Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
2139             # Prepare the cache share data for multiprocessing
2140             Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}
2141             ModuleCodaFile = {}
2142             for ma in Pa.ModuleAutoGenList:
2143                 ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
2144             Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
2145             # ModuleList contains all driver modules only
2146             for Module in ModuleList:
2147                 # Get ModuleAutoGen object to generate C code file and makefile
2148                 Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2149                 if Ma is None:
2150                     continue
2151                 if Ma.PcdIsDriver:
2152                     Ma.PlatformInfo = Pa
2153                     Ma.Workspace = Wa
2154                     PcdMaList.append(Ma)
2155                 self.AllDrivers.add(Ma)
2156                 self.AllModules.add(Ma)
2157
2158             mqueue = mp.Queue()
2159             cqueue = mp.Queue()
2160             for m in Pa.GetAllModuleInfo:
2161                 mqueue.put(m)
2162                 module_file,module_root,module_path,module_basename,\
2163                     module_originalpath,module_arch,IsLib = m
2164                 Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\
2165                                   ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2166                 self.AllModules.add(Ma)
2167             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
2168             Pa.DataPipe.dump(data_pipe_file)
2169
2170             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
2171
2172             if not autogen_rt:
2173                 self.AutoGenMgr.TerminateWorkers()
2174                 self.AutoGenMgr.join(1)
2175                 raise FatalError(errorcode)
2176
2177             if GlobalData.gUseHashCache:
2178                 for item in GlobalData.gModuleAllCacheStatus:
2179                     (MetaFilePath, Arch, CacheStr, Status) = item
2180                     Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\
2181                                       ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2182                     if CacheStr == "PreMakeCache" and Status == False:
2183                         self.PreMakeCacheMiss.add(Ma)
2184                     if CacheStr == "PreMakeCache" and Status == True:
2185                         self.PreMakeCacheHit.add(Ma)
2186                         GlobalData.gModuleCacheHit.add(Ma)
2187                     if CacheStr == "MakeCache" and Status == False:
2188                         self.MakeCacheMiss.add(Ma)
2189                     if CacheStr == "MakeCache" and Status == True:
2190                         self.MakeCacheHit.add(Ma)
2191                         GlobalData.gModuleCacheHit.add(Ma)
2192             self.AutoGenTime += int(round((time.time() - AutoGenStart)))
2193         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2194         with open(AutoGenIdFile,"w") as fw:
2195             fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
2196             fw.write("BuildDir=%s\n" % Wa.BuildDir)
2197             fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
2198
2199         if GlobalData.gBinCacheSource:
2200             BuildModules.extend(self.MakeCacheMiss)
2201         elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:
2202             BuildModules.extend(self.PreMakeCacheMiss)
2203         else:
2204             BuildModules.extend(self.AllDrivers)
2205
2206         self.Progress.Stop("done!")
2207         return Wa, BuildModules
2208
2209     def _MultiThreadBuildPlatform(self):
2210         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
2211         for BuildTarget in self.BuildTargetList:
2212             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
2213             index = 0
2214             for ToolChain in self.ToolChainList:
2215                 resetFdsGlobalVariable()
2216                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
2217                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
2218                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
2219                 index += 1
2220                 ExitFlag = threading.Event()
2221                 ExitFlag.clear()
2222                 if self.SkipAutoGen:
2223                     Wa = self.VerifyAutoGenFiles()
2224                     if Wa is None:
2225                         self.SkipAutoGen = False
2226                         Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2227                     else:
2228                         GlobalData.gAutoGenPhase = True
2229                         self.BuildModules = self.SetupMakeSetting(Wa)
2230                 else:
2231                     Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2232                 Pa = Wa.AutoGenObjectList[0]
2233                 GlobalData.gAutoGenPhase = False
2234
2235                 if GlobalData.gBinCacheSource:
2236                     EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))
2237                     EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))
2238                     EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))
2239
2240                 for Arch in Wa.ArchList:
2241                     MakeStart = time.time()
2242                     for Ma in set(self.BuildModules):
2243                         # Generate build task for the module
2244                         if not Ma.IsBinaryModule:
2245                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
2246                         # Break build if any build thread has error
2247                         if BuildTask.HasError():
2248                             # we need a full version of makefile for platform
2249                             ExitFlag.set()
2250                             BuildTask.WaitForComplete()
2251                             Pa.CreateMakeFile(False)
2252                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2253                         # Start task scheduler
2254                         if not BuildTask.IsOnGoing():
2255                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
2256
2257                     # in case there's an interruption. we need a full version of makefile for platform
2258
2259                     if BuildTask.HasError():
2260                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2261                     self.MakeTime += int(round((time.time() - MakeStart)))
2262
2263                 MakeContiue = time.time()
2264                 #
2265                 #
2266                 # All modules have been put in build tasks queue. Tell task scheduler
2267                 # to exit if all tasks are completed
2268                 #
2269                 ExitFlag.set()
2270                 BuildTask.WaitForComplete()
2271                 self.CreateAsBuiltInf()
2272                 if GlobalData.gBinCacheDest:
2273                     self.GenDestCache()
2274                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
2275                     # Only for --hash
2276                     # Update PreMakeCacheChain files
2277                     self.GenLocalPreMakeCache()
2278                 #
2279                 # Get Module List
2280                 #
2281                 ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}
2282                 self.BuildModules = []
2283                 self.MakeTime += int(round((time.time() - MakeContiue)))
2284                 #
2285                 # Check for build error, and raise exception if one
2286                 # has been signaled.
2287                 #
2288                 if BuildTask.HasError():
2289                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2290
2291                 # Create MAP file when Load Fix Address is enabled.
2292                 if self.Target in ["", "all", "fds"]:
2293                     for Arch in Wa.ArchList:
2294                         #
2295                         # Check whether the set fix address is above 4G for 32bit image.
2296                         #
2297                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
2298                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2299
2300                     #
2301                     # Rebase module to the preferred memory address before GenFds
2302                     #
2303                     MapBuffer = []
2304                     if self.LoadFixAddress != 0:
2305                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
2306
2307                     if self.Fdf:
2308                         #
2309                         # Generate FD image if there's a FDF file found
2310                         #
2311                         GenFdsStart = time.time()
2312                         if GenFdsApi(Wa.GenFdsCommandDict, self.Db):
2313                             EdkLogger.error("build", COMMAND_FAILURE)
2314
2315                         #
2316                         # Create MAP file for all platform FVs after GenFds.
2317                         #
2318                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2319                         self.GenFdsTime += int(round((time.time() - GenFdsStart)))
2320                     #
2321                     # Save MAP buffer into MAP file.
2322                     #
2323                     self._SaveMapFile(MapBuffer, Wa)
2324                 self.CreateGuidedSectionToolsFile(Wa)
2325     ## Generate GuidedSectionTools.txt in the FV directories.
2326     #
2327     def CreateGuidedSectionToolsFile(self,Wa):
2328         for BuildTarget in self.BuildTargetList:
2329             for ToolChain in self.ToolChainList:
2330                 FvDir = Wa.FvDir
2331                 if not os.path.exists(FvDir):
2332                     continue
2333
2334                 for Arch in self.ArchList:
2335                     # Build up the list of supported architectures for this build
2336                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
2337
2338                     # Look through the tool definitions for GUIDed tools
2339                     guidAttribs = []
2340                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():
2341                         if attrib.upper().endswith('_GUID'):
2342                             split = attrib.split('_')
2343                             thisPrefix = '_'.join(split[0:3]) + '_'
2344                             if thisPrefix == prefix:
2345                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
2346                                 guid = guid.lower()
2347                                 toolName = split[3]
2348                                 path = '_'.join(split[0:4]) + '_PATH'
2349                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
2350                                 path = self.GetRealPathOfTool(path)
2351                                 guidAttribs.append((guid, toolName, path))
2352
2353                     # Write out GuidedSecTools.txt
2354                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
2355                     toolsFile = open(toolsFile, 'wt')
2356                     for guidedSectionTool in guidAttribs:
2357                         print(' '.join(guidedSectionTool), file=toolsFile)
2358                     toolsFile.close()
2359
2360     ## Returns the real path of the tool.
2361     #
2362     def GetRealPathOfTool (self, tool):
2363         if os.path.exists(tool):
2364             return os.path.realpath(tool)
2365         return tool
2366
2367     ## Launch the module or platform build
2368     #
2369     def Launch(self):
2370         self.AllDrivers = set()
2371         self.AllModules = set()
2372         self.PreMakeCacheMiss = set()
2373         self.PreMakeCacheHit = set()
2374         self.MakeCacheMiss = set()
2375         self.MakeCacheHit = set()
2376         if not self.ModuleFile:
2377             if not self.SpawnMode or self.Target not in ["", "all"]:
2378                 self.SpawnMode = False
2379                 self._BuildPlatform()
2380             else:
2381                 self._MultiThreadBuildPlatform()
2382         else:
2383             self.SpawnMode = False
2384             self._BuildModule()
2385
2386         if self.Target == 'cleanall':
2387             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
2388
2389     def CreateAsBuiltInf(self):
2390         for Module in self.BuildModules:
2391             Module.CreateAsBuiltInf()
2392
2393     def GenDestCache(self):
2394         for Module in self.AllModules:
2395             Module.GenPreMakefileHashList()
2396             Module.GenMakefileHashList()
2397             Module.CopyModuleToCache()
2398
2399     def GenLocalPreMakeCache(self):
2400         for Module in self.PreMakeCacheMiss:
2401             Module.GenPreMakefileHashList()
2402
2403     ## Do some clean-up works when error occurred
2404     def Relinquish(self):
2405         OldLogLevel = EdkLogger.GetLevel()
2406         EdkLogger.SetLevel(EdkLogger.ERROR)
2407         Utils.Progressor.Abort()
2408         if self.SpawnMode == True:
2409             BuildTask.Abort()
2410         EdkLogger.SetLevel(OldLogLevel)
2411
2412 def ParseDefines(DefineList=[]):
2413     DefineDict = {}
2414     if DefineList is not None:
2415         for Define in DefineList:
2416             DefineTokenList = Define.split("=", 1)
2417             if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]):
2418                 EdkLogger.error('build', FORMAT_INVALID,
2419                                 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2420                                 ExtraData=DefineTokenList[0])
2421
2422             if len(DefineTokenList) == 1:
2423                 DefineDict[DefineTokenList[0]] = "TRUE"
2424             else:
2425                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
2426     return DefineDict
2427
2428
2429
2430 def LogBuildTime(Time):
2431     if Time:
2432         TimeDurStr = ''
2433         TimeDur = time.gmtime(Time)
2434         if TimeDur.tm_yday > 1:
2435             TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + ", %d day(s)" % (TimeDur.tm_yday - 1)
2436         else:
2437             TimeDurStr = time.strftime("%H:%M:%S", TimeDur)
2438         return TimeDurStr
2439     else:
2440         return None
2441 def ThreadNum():
2442     OptionParser = MyOptionParser()
2443     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2444         OptionParser.GetOption()
2445     BuildOption, BuildTarget = OptionParser.BuildOption, OptionParser.BuildTarget
2446     ThreadNumber = BuildOption.ThreadNumber
2447     GlobalData.gCmdConfDir = BuildOption.ConfDirectory
2448     if ThreadNumber is None:
2449         TargetObj = TargetTxtDict()
2450         ThreadNumber = TargetObj.Target.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
2451         if ThreadNumber == '':
2452             ThreadNumber = 0
2453         else:
2454             ThreadNumber = int(ThreadNumber, 0)
2455
2456     if ThreadNumber == 0:
2457         try:
2458             ThreadNumber = multiprocessing.cpu_count()
2459         except (ImportError, NotImplementedError):
2460             ThreadNumber = 1
2461     return ThreadNumber
2462 ## Tool entrance method
2463 #
2464 # This method mainly dispatch specific methods per the command line options.
2465 # If no error found, return zero value so the caller of this tool can know
2466 # if it's executed successfully or not.
2467 #
2468 #   @retval 0     Tool was successful
2469 #   @retval 1     Tool failed
2470 #
2471 LogQMaxSize = ThreadNum() * 10
2472 def Main():
2473     StartTime = time.time()
2474
2475     #
2476     # Create a log Queue
2477     #
2478     LogQ = mp.Queue(LogQMaxSize)
2479     # Initialize log system
2480     EdkLogger.LogClientInitialize(LogQ)
2481     GlobalData.gCommand = sys.argv[1:]
2482     #
2483     # Parse the options and args
2484     #
2485     OptionParser = MyOptionParser()
2486     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2487         OptionParser.GetOption()
2488     Option, Target = OptionParser.BuildOption, OptionParser.BuildTarget
2489     GlobalData.gOptions = Option
2490     GlobalData.gCaseInsensitive = Option.CaseInsensitive
2491
2492     # Set log level
2493     LogLevel = EdkLogger.INFO
2494     if Option.verbose is not None:
2495         EdkLogger.SetLevel(EdkLogger.VERBOSE)
2496         LogLevel = EdkLogger.VERBOSE
2497     elif Option.quiet is not None:
2498         EdkLogger.SetLevel(EdkLogger.QUIET)
2499         LogLevel = EdkLogger.QUIET
2500     elif Option.debug is not None:
2501         EdkLogger.SetLevel(Option.debug + 1)
2502         LogLevel = Option.debug + 1
2503     else:
2504         EdkLogger.SetLevel(EdkLogger.INFO)
2505
2506     if Option.WarningAsError == True:
2507         EdkLogger.SetWarningAsError()
2508     Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile)
2509     Log_Agent.start()
2510
2511     if platform.platform().find("Windows") >= 0:
2512         GlobalData.gIsWindows = True
2513     else:
2514         GlobalData.gIsWindows = False
2515
2516     EdkLogger.quiet("Build environment: %s" % platform.platform())
2517     EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime()));
2518     ReturnCode = 0
2519     MyBuild = None
2520     BuildError = True
2521     try:
2522         if len(Target) == 0:
2523             Target = "all"
2524         elif len(Target) >= 2:
2525             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
2526                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2527         else:
2528             Target = Target[0].lower()
2529
2530         if Target not in gSupportedTarget:
2531             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
2532                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2533
2534         #
2535         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2536         #
2537         CheckEnvVariable()
2538         GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros))
2539
2540         Workspace = os.getenv("WORKSPACE")
2541         #
2542         # Get files real name in workspace dir
2543         #
2544         GlobalData.gAllFiles = Utils.DirCache(Workspace)
2545
2546         WorkingDirectory = os.getcwd()
2547         if not Option.ModuleFile:
2548             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
2549             FileNum = len(FileList)
2550             if FileNum >= 2:
2551                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
2552                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2553             elif FileNum == 1:
2554                 Option.ModuleFile = NormFile(FileList[0], Workspace)
2555
2556         if Option.ModuleFile:
2557             if os.path.isabs (Option.ModuleFile):
2558                 if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0:
2559                     Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace)
2560             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
2561             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
2562             if ErrorCode != 0:
2563                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2564
2565         if Option.PlatformFile is not None:
2566             if os.path.isabs (Option.PlatformFile):
2567                 if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0:
2568                     Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace)
2569             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
2570
2571         if Option.FdfFile is not None:
2572             if os.path.isabs (Option.FdfFile):
2573                 if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0:
2574                     Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace)
2575             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
2576             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
2577             if ErrorCode != 0:
2578                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2579
2580         if Option.Flag is not None and Option.Flag not in ['-c', '-s']:
2581             EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s")
2582
2583         MyBuild = Build(Target, Workspace, Option,LogQ)
2584         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
2585         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
2586             MyBuild.Launch()
2587
2588         #
2589         # All job done, no error found and no exception raised
2590         #
2591         BuildError = False
2592     except FatalError as X:
2593         if MyBuild is not None:
2594             # for multi-thread build exits safely
2595             MyBuild.Relinquish()
2596         if Option is not None and Option.debug is not None:
2597             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2598         ReturnCode = X.args[0]
2599     except Warning as X:
2600         # error from Fdf parser
2601         if MyBuild is not None:
2602             # for multi-thread build exits safely
2603             MyBuild.Relinquish()
2604         if Option is not None and Option.debug is not None:
2605             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2606         else:
2607             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
2608         ReturnCode = FORMAT_INVALID
2609     except KeyboardInterrupt:
2610         if MyBuild is not None:
2611
2612             # for multi-thread build exits safely
2613             MyBuild.Relinquish()
2614         ReturnCode = ABORT_ERROR
2615         if Option is not None and Option.debug is not None:
2616             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2617     except:
2618         if MyBuild is not None:
2619             # for multi-thread build exits safely
2620             MyBuild.Relinquish()
2621
2622         # try to get the meta-file from the object causing exception
2623         Tb = sys.exc_info()[-1]
2624         MetaFile = GlobalData.gProcessingFile
2625         while Tb is not None:
2626             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
2627                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
2628             Tb = Tb.tb_next
2629         EdkLogger.error(
2630                     "\nbuild",
2631                     CODE_ERROR,
2632                     "Unknown fatal error when processing [%s]" % MetaFile,
2633                     ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR,
2634                     RaiseError=False
2635                     )
2636         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2637         ReturnCode = CODE_ERROR
2638     finally:
2639         Utils.Progressor.Abort()
2640         Utils.ClearDuplicatedInf()
2641
2642     if ReturnCode == 0:
2643         try:
2644             MyBuild.LaunchPostbuild()
2645             Conclusion = "Done"
2646         except:
2647             Conclusion = "Failed"
2648     elif ReturnCode == ABORT_ERROR:
2649         Conclusion = "Aborted"
2650     else:
2651         Conclusion = "Failed"
2652     FinishTime = time.time()
2653     BuildDuration = time.gmtime(int(round(FinishTime - StartTime)))
2654     BuildDurationStr = ""
2655     if BuildDuration.tm_yday > 1:
2656         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1)
2657     else:
2658         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
2659     if MyBuild is not None:
2660         if not BuildError:
2661             MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
2662
2663     EdkLogger.SetLevel(EdkLogger.QUIET)
2664     EdkLogger.quiet("\n- %s -" % Conclusion)
2665     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
2666     EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
2667     Log_Agent.kill()
2668     Log_Agent.join()
2669     return ReturnCode
2670
2671 if __name__ == '__main__':
2672     try:
2673         mp.set_start_method('spawn')
2674     except:
2675         pass
2676     r = Main()
2677     ## 0-127 is a safe return range, and 1 is a standard default error
2678     if r < 0 or r > 127: r = 1
2679     sys.exit(r)
2680