BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
   5 #  Copyright (c) 2007 - 2017, Intel Corporation. All rights reserved.<BR>
   6 #
   7 #  This program and the accompanying materials
   8 #  are licensed and made available under the terms and conditions of the BSD License
   9 #  which accompanies this distribution.  The full text of the license may be found at
  10 #  http://opensource.org/licenses/bsd-license.php
  11 #
  12 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14 #
  15
  16 ##
  17 # Import Modules
  18 #
  19 import Common.LongFilePathOs as os
  20 import re
  21 import StringIO
  22 import sys
  23 import glob
  24 import time
  25 import platform
  26 import traceback
  27 import encodings.ascii
  28 import itertools
  29
  30 from struct import *
  31 from threading import *
  32 from optparse import OptionParser
  33 from subprocess import *
  34 from Common import Misc as Utils
  35
  36 from Common.LongFilePathSupport import OpenLongFilePath as open
  37 from Common.LongFilePathSupport import LongFilePath
  38 from Common.TargetTxtClassObject import *
  39 from Common.ToolDefClassObject import *
  40 from Common.DataType import *
  41 from Common.BuildVersion import gBUILD_VERSION
  42 from AutoGen.AutoGen import *
  43 from Common.BuildToolError import *
  44 from Workspace.WorkspaceDatabase import *
  45 from Common.MultipleWorkspace import MultipleWorkspace as mws
  46
  47 from BuildReport import BuildReport
  48 from GenPatchPcdTable.GenPatchPcdTable import *
  49 from PatchPcdValue.PatchPcdValue import *
  50
  51 import Common.EdkLogger
  52 import Common.GlobalData as GlobalData
  53
  54 # Version and Copyright
  55 VersionNumber = "0.60" + ' ' + gBUILD_VERSION
  56 __version__ = "%prog Version " + VersionNumber
  57 __copyright__ = "Copyright (c) 2007 - 2017, Intel Corporation  All rights reserved."
  58
  59 ## standard targets of build command
  60 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  61
  62 ## build configuration file
  63 gBuildConfiguration = "target.txt"
  64 gToolsDefinition = "tools_def.txt"
  65
  66 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
  67 TmpTableDict = {}
  68
  69 ## Check environment PATH variable to make sure the specified tool is found
  70 #
  71 #   If the tool is found in the PATH, then True is returned
  72 #   Otherwise, False is returned
  73 #
  74 def IsToolInPath(tool):
  75     if os.environ.has_key('PATHEXT'):
  76         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  77     else:
  78         extns = ('',)
  79     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  80         for ext in extns:
  81             if os.path.exists(os.path.join(pathDir, tool + ext)):
  82                 return True
  83     return False
  84
  85 ## Check environment variables
  86 #
  87 #  Check environment variables that must be set for build. Currently they are
  88 #
  89 #   WORKSPACE           The directory all packages/platforms start from
  90 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  91 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  92 #
  93 #   If any of above environment variable is not set or has error, the build
  94 #   will be broken.
  95 #
  96 def CheckEnvVariable():
  97     # check WORKSPACE
  98     if "WORKSPACE" not in os.environ:
  99         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 100                         ExtraData="WORKSPACE")
 101
 102     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
 103     if not os.path.exists(WorkspaceDir):
 104         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData="%s" % WorkspaceDir)
 105     elif ' ' in WorkspaceDir:
 106         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
 107                         ExtraData=WorkspaceDir)
 108     os.environ["WORKSPACE"] = WorkspaceDir
 109
 110     # set multiple workspace
 111     PackagesPath = os.getenv("PACKAGES_PATH")
 112     mws.setWs(WorkspaceDir, PackagesPath)
 113     if mws.PACKAGES_PATH:
 114         for Path in mws.PACKAGES_PATH:
 115             if not os.path.exists(Path):
 116                 EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData="%s" % Path)
 117             elif ' ' in Path:
 118                 EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path)
 119
 120     #
 121     # Check EFI_SOURCE (Edk build convention). EDK_SOURCE will always point to ECP
 122     #
 123     if "ECP_SOURCE" not in os.environ:
 124         os.environ["ECP_SOURCE"] = mws.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
 125     if "EFI_SOURCE" not in os.environ:
 126         os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
 127     if "EDK_SOURCE" not in os.environ:
 128         os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
 129
 130     #
 131     # Unify case of characters on case-insensitive systems
 132     #
 133     EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
 134     EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
 135     EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
 136
 137     os.environ["EFI_SOURCE"] = EfiSourceDir
 138     os.environ["EDK_SOURCE"] = EdkSourceDir
 139     os.environ["ECP_SOURCE"] = EcpSourceDir
 140     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 141
 142     if not os.path.exists(EcpSourceDir):
 143         EdkLogger.verbose("ECP_SOURCE = %s doesn't exist. Edk modules could not be built." % EcpSourceDir)
 144     elif ' ' in EcpSourceDir:
 145         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in ECP_SOURCE path",
 146                         ExtraData=EcpSourceDir)
 147     if not os.path.exists(EdkSourceDir):
 148         if EdkSourceDir == EcpSourceDir:
 149             EdkLogger.verbose("EDK_SOURCE = %s doesn't exist. Edk modules could not be built." % EdkSourceDir)
 150         else:
 151             EdkLogger.error("build", PARAMETER_INVALID, "EDK_SOURCE does not exist",
 152                             ExtraData=EdkSourceDir)
 153     elif ' ' in EdkSourceDir:
 154         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in EDK_SOURCE path",
 155                         ExtraData=EdkSourceDir)
 156     if not os.path.exists(EfiSourceDir):
 157         if EfiSourceDir == EcpSourceDir:
 158             EdkLogger.verbose("EFI_SOURCE = %s doesn't exist. Edk modules could not be built." % EfiSourceDir)
 159         else:
 160             EdkLogger.error("build", PARAMETER_INVALID, "EFI_SOURCE does not exist",
 161                             ExtraData=EfiSourceDir)
 162     elif ' ' in EfiSourceDir:
 163         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in EFI_SOURCE path",
 164                         ExtraData=EfiSourceDir)
 165
 166     # check those variables on single workspace case
 167     if not PackagesPath:
 168         # change absolute path to relative path to WORKSPACE
 169         if EfiSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 170             EdkLogger.error("build", PARAMETER_INVALID, "EFI_SOURCE is not under WORKSPACE",
 171                             ExtraData="WORKSPACE = %s\n    EFI_SOURCE = %s" % (WorkspaceDir, EfiSourceDir))
 172         if EdkSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 173             EdkLogger.error("build", PARAMETER_INVALID, "EDK_SOURCE is not under WORKSPACE",
 174                             ExtraData="WORKSPACE = %s\n    EDK_SOURCE = %s" % (WorkspaceDir, EdkSourceDir))
 175         if EcpSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 176             EdkLogger.error("build", PARAMETER_INVALID, "ECP_SOURCE is not under WORKSPACE",
 177                             ExtraData="WORKSPACE = %s\n    ECP_SOURCE = %s" % (WorkspaceDir, EcpSourceDir))
 178
 179     # check EDK_TOOLS_PATH
 180     if "EDK_TOOLS_PATH" not in os.environ:
 181         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 182                         ExtraData="EDK_TOOLS_PATH")
 183
 184     # check PATH
 185     if "PATH" not in os.environ:
 186         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 187                         ExtraData="PATH")
 188
 189     GlobalData.gWorkspace = WorkspaceDir
 190     GlobalData.gEfiSource = EfiSourceDir
 191     GlobalData.gEdkSource = EdkSourceDir
 192     GlobalData.gEcpSource = EcpSourceDir
 193
 194     GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 195     GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
 196     GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
 197     GlobalData.gGlobalDefines["ECP_SOURCE"] = EcpSourceDir
 198     GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"]
 199
 200 ## Get normalized file path
 201 #
 202 # Convert the path to be local format, and remove the WORKSPACE path at the
 203 # beginning if the file path is given in full path.
 204 #
 205 # @param  FilePath      File path to be normalized
 206 # @param  Workspace     Workspace path which the FilePath will be checked against
 207 #
 208 # @retval string        The normalized file path
 209 #
 210 def NormFile(FilePath, Workspace):
 211     # check if the path is absolute or relative
 212     if os.path.isabs(FilePath):
 213         FileFullPath = os.path.normpath(FilePath)
 214     else:
 215         FileFullPath = os.path.normpath(mws.join(Workspace, FilePath))
 216         Workspace = mws.getWs(Workspace, FilePath)
 217
 218     # check if the file path exists or not
 219     if not os.path.isfile(FileFullPath):
 220         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath)
 221
 222     # remove workspace directory from the beginning part of the file path
 223     if Workspace[-1] in ["\\", "/"]:
 224         return FileFullPath[len(Workspace):]
 225     else:
 226         return FileFullPath[(len(Workspace) + 1):]
 227
 228 ## Get the output of an external program
 229 #
 230 # This is the entrance method of thread reading output of an external program and
 231 # putting them in STDOUT/STDERR of current program.
 232 #
 233 # @param  From      The stream message read from
 234 # @param  To        The stream message put on
 235 # @param  ExitFlag  The flag used to indicate stopping reading
 236 #
 237 def ReadMessage(From, To, ExitFlag):
 238     while True:
 239         # read one line a time
 240         Line = From.readline()
 241         # empty string means "end"
 242         if Line != None and Line != "":
 243             To(Line.rstrip())
 244         else:
 245             break
 246         if ExitFlag.isSet():
 247             break
 248
 249 ## Launch an external program
 250 #
 251 # This method will call subprocess.Popen to execute an external program with
 252 # given options in specified directory. Because of the dead-lock issue during
 253 # redirecting output of the external program, threads are used to to do the
 254 # redirection work.
 255 #
 256 # @param  Command               A list or string containing the call of the program
 257 # @param  WorkingDir            The directory in which the program will be running
 258 #
 259 def LaunchCommand(Command, WorkingDir):
 260     # if working directory doesn't exist, Popen() will raise an exception
 261     if not os.path.isdir(WorkingDir):
 262         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 263
 264     # Command is used as the first Argument in following Popen().
 265     # It could be a string or sequence. We find that if command is a string in following Popen(),
 266     # ubuntu may fail with an error message that the command is not found.
 267     # So here we may need convert command from string to list instance.
 268     if platform.system() != 'Windows':
 269         if not isinstance(Command, list):
 270             Command = Command.split()
 271         Command = ' '.join(Command)
 272
 273     Proc = None
 274     EndOfProcedure = None
 275     try:
 276         # launch the command
 277         Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 278
 279         # launch two threads to read the STDOUT and STDERR
 280         EndOfProcedure = Event()
 281         EndOfProcedure.clear()
 282         if Proc.stdout:
 283             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))
 284             StdOutThread.setName("STDOUT-Redirector")
 285             StdOutThread.setDaemon(False)
 286             StdOutThread.start()
 287
 288         if Proc.stderr:
 289             StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))
 290             StdErrThread.setName("STDERR-Redirector")
 291             StdErrThread.setDaemon(False)
 292             StdErrThread.start()
 293
 294         # waiting for program exit
 295         Proc.wait()
 296     except: # in case of aborting
 297         # terminate the threads redirecting the program output
 298         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 299         if EndOfProcedure != None:
 300             EndOfProcedure.set()
 301         if Proc == None:
 302             if type(Command) != type(""):
 303                 Command = " ".join(Command)
 304             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 305
 306     if Proc.stdout:
 307         StdOutThread.join()
 308     if Proc.stderr:
 309         StdErrThread.join()
 310
 311     # check the return code of the program
 312     if Proc.returncode != 0:
 313         if type(Command) != type(""):
 314             Command = " ".join(Command)
 315         # print out the Response file and its content when make failure
 316         RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
 317         if os.path.isfile(RespFile):
 318             f = open(RespFile)
 319             RespContent = f.read()
 320             f.close()
 321             EdkLogger.info(RespContent)
 322
 323         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 324
 325 ## The smallest unit that can be built in multi-thread build mode
 326 #
 327 # This is the base class of build unit. The "Obj" parameter must provide
 328 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 329 # missing build.
 330 #
 331 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 332 #
 333 class BuildUnit:
 334     ## The constructor
 335     #
 336     #   @param  self        The object pointer
 337     #   @param  Obj         The object the build is working on
 338     #   @param  Target      The build target name, one of gSupportedTarget
 339     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 340     #   @param  WorkingDir  The directory build command starts in
 341     #
 342     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 343         self.BuildObject = Obj
 344         self.Dependency = Dependency
 345         self.WorkingDir = WorkingDir
 346         self.Target = Target
 347         self.BuildCommand = BuildCommand
 348         if not BuildCommand:
 349             EdkLogger.error("build", OPTION_MISSING,
 350                             "No build command found for this module. "
 351                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
 352                                 (Obj.BuildTarget, Obj.ToolChain, Obj.Arch),
 353                             ExtraData=str(Obj))
 354
 355
 356     ## str() method
 357     #
 358     #   It just returns the string representation of self.BuildObject
 359     #
 360     #   @param  self        The object pointer
 361     #
 362     def __str__(self):
 363         return str(self.BuildObject)
 364
 365     ## "==" operator method
 366     #
 367     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 368     #   provide its own __eq__() method.
 369     #
 370     #   @param  self        The object pointer
 371     #   @param  Other       The other BuildUnit object compared to
 372     #
 373     def __eq__(self, Other):
 374         return Other != None and self.BuildObject == Other.BuildObject \
 375                 and self.BuildObject.Arch == Other.BuildObject.Arch
 376
 377     ## hash() method
 378     #
 379     #   It just returns the hash value of self.BuildObject which must be hashable.
 380     #
 381     #   @param  self        The object pointer
 382     #
 383     def __hash__(self):
 384         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 385
 386     def __repr__(self):
 387         return repr(self.BuildObject)
 388
 389 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 390 #
 391 # This class is for module build by nmake/make build system. The "Obj" parameter
 392 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 393 # be make units missing build.
 394 #
 395 # Currently the "Obj" should be only ModuleAutoGen object.
 396 #
 397 class ModuleMakeUnit(BuildUnit):
 398     ## The constructor
 399     #
 400     #   @param  self        The object pointer
 401     #   @param  Obj         The ModuleAutoGen object the build is working on
 402     #   @param  Target      The build target name, one of gSupportedTarget
 403     #
 404     def __init__(self, Obj, Target):
 405         Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]
 406         BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
 407         if Target in [None, "", "all"]:
 408             self.Target = "tbuild"
 409
 410 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 411 #
 412 # This class is for platform build by nmake/make build system. The "Obj" parameter
 413 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 414 # be make units missing build.
 415 #
 416 # Currently the "Obj" should be only PlatformAutoGen object.
 417 #
 418 class PlatformMakeUnit(BuildUnit):
 419     ## The constructor
 420     #
 421     #   @param  self        The object pointer
 422     #   @param  Obj         The PlatformAutoGen object the build is working on
 423     #   @param  Target      The build target name, one of gSupportedTarget
 424     #
 425     def __init__(self, Obj, Target):
 426         Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 427         Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])
 428         BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
 429
 430 ## The class representing the task of a module build or platform build
 431 #
 432 # This class manages the build tasks in multi-thread build mode. Its jobs include
 433 # scheduling thread running, catching thread error, monitor the thread status, etc.
 434 #
 435 class BuildTask:
 436     # queue for tasks waiting for schedule
 437     _PendingQueue = sdict()
 438     _PendingQueueLock = threading.Lock()
 439
 440     # queue for tasks ready for running
 441     _ReadyQueue = sdict()
 442     _ReadyQueueLock = threading.Lock()
 443
 444     # queue for run tasks
 445     _RunningQueue = sdict()
 446     _RunningQueueLock = threading.Lock()
 447
 448     # queue containing all build tasks, in case duplicate build
 449     _TaskQueue = sdict()
 450
 451     # flag indicating error occurs in a running thread
 452     _ErrorFlag = threading.Event()
 453     _ErrorFlag.clear()
 454     _ErrorMessage = ""
 455
 456     # BoundedSemaphore object used to control the number of running threads
 457     _Thread = None
 458
 459     # flag indicating if the scheduler is started or not
 460     _SchedulerStopped = threading.Event()
 461     _SchedulerStopped.set()
 462
 463     ## Start the task scheduler thread
 464     #
 465     #   @param  MaxThreadNumber     The maximum thread number
 466     #   @param  ExitFlag            Flag used to end the scheduler
 467     #
 468     @staticmethod
 469     def StartScheduler(MaxThreadNumber, ExitFlag):
 470         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 471         SchedulerThread.setName("Build-Task-Scheduler")
 472         SchedulerThread.setDaemon(False)
 473         SchedulerThread.start()
 474         # wait for the scheduler to be started, especially useful in Linux
 475         while not BuildTask.IsOnGoing():
 476             time.sleep(0.01)
 477
 478     ## Scheduler method
 479     #
 480     #   @param  MaxThreadNumber     The maximum thread number
 481     #   @param  ExitFlag            Flag used to end the scheduler
 482     #
 483     @staticmethod
 484     def Scheduler(MaxThreadNumber, ExitFlag):
 485         BuildTask._SchedulerStopped.clear()
 486         try:
 487             # use BoundedSemaphore to control the maximum running threads
 488             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 489             #
 490             # scheduling loop, which will exits when no pending/ready task and
 491             # indicated to do so, or there's error in running thread
 492             #
 493             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 494                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 495                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 496                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 497
 498                 # get all pending tasks
 499                 BuildTask._PendingQueueLock.acquire()
 500                 BuildObjectList = BuildTask._PendingQueue.keys()
 501                 #
 502                 # check if their dependency is resolved, and if true, move them
 503                 # into ready queue
 504                 #
 505                 for BuildObject in BuildObjectList:
 506                     Bt = BuildTask._PendingQueue[BuildObject]
 507                     if Bt.IsReady():
 508                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 509                 BuildTask._PendingQueueLock.release()
 510
 511                 # launch build thread until the maximum number of threads is reached
 512                 while not BuildTask._ErrorFlag.isSet():
 513                     # empty ready queue, do nothing further
 514                     if len(BuildTask._ReadyQueue) == 0:
 515                         break
 516
 517                     # wait for active thread(s) exit
 518                     BuildTask._Thread.acquire(True)
 519
 520                     # start a new build thread
 521                     Bo = BuildTask._ReadyQueue.keys()[0]
 522                     Bt = BuildTask._ReadyQueue.pop(Bo)
 523
 524                     # move into running queue
 525                     BuildTask._RunningQueueLock.acquire()
 526                     BuildTask._RunningQueue[Bo] = Bt
 527                     BuildTask._RunningQueueLock.release()
 528
 529                     Bt.Start()
 530                     # avoid tense loop
 531                     time.sleep(0.01)
 532
 533                 # avoid tense loop
 534                 time.sleep(0.01)
 535
 536             # wait for all running threads exit
 537             if BuildTask._ErrorFlag.isSet():
 538                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 539             # while not BuildTask._ErrorFlag.isSet() and \
 540             while len(BuildTask._RunningQueue) > 0:
 541                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 542                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join([Th.getName() for Th in threading.enumerate()]))
 543                 # avoid tense loop
 544                 time.sleep(0.1)
 545         except BaseException, X:
 546             #
 547             # TRICK: hide the output of threads left runing, so that the user can
 548             #        catch the error message easily
 549             #
 550             EdkLogger.SetLevel(EdkLogger.ERROR)
 551             BuildTask._ErrorFlag.set()
 552             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 553
 554         BuildTask._PendingQueue.clear()
 555         BuildTask._ReadyQueue.clear()
 556         BuildTask._RunningQueue.clear()
 557         BuildTask._TaskQueue.clear()
 558         BuildTask._SchedulerStopped.set()
 559
 560     ## Wait for all running method exit
 561     #
 562     @staticmethod
 563     def WaitForComplete():
 564         BuildTask._SchedulerStopped.wait()
 565
 566     ## Check if the scheduler is running or not
 567     #
 568     @staticmethod
 569     def IsOnGoing():
 570         return not BuildTask._SchedulerStopped.isSet()
 571
 572     ## Abort the build
 573     @staticmethod
 574     def Abort():
 575         if BuildTask.IsOnGoing():
 576             BuildTask._ErrorFlag.set()
 577             BuildTask.WaitForComplete()
 578
 579     ## Check if there's error in running thread
 580     #
 581     #   Since the main thread cannot catch exceptions in other thread, we have to
 582     #   use threading.Event to communicate this formation to main thread.
 583     #
 584     @staticmethod
 585     def HasError():
 586         return BuildTask._ErrorFlag.isSet()
 587
 588     ## Get error message in running thread
 589     #
 590     #   Since the main thread cannot catch exceptions in other thread, we have to
 591     #   use a static variable to communicate this message to main thread.
 592     #
 593     @staticmethod
 594     def GetErrorMessage():
 595         return BuildTask._ErrorMessage
 596
 597     ## Factory method to create a BuildTask object
 598     #
 599     #   This method will check if a module is building or has been built. And if
 600     #   true, just return the associated BuildTask object in the _TaskQueue. If
 601     #   not, create and return a new BuildTask object. The new BuildTask object
 602     #   will be appended to the _PendingQueue for scheduling later.
 603     #
 604     #   @param  BuildItem       A BuildUnit object representing a build object
 605     #   @param  Dependency      The dependent build object of BuildItem
 606     #
 607     @staticmethod
 608     def New(BuildItem, Dependency=None):
 609         if BuildItem in BuildTask._TaskQueue:
 610             Bt = BuildTask._TaskQueue[BuildItem]
 611             return Bt
 612
 613         Bt = BuildTask()
 614         Bt._Init(BuildItem, Dependency)
 615         BuildTask._TaskQueue[BuildItem] = Bt
 616
 617         BuildTask._PendingQueueLock.acquire()
 618         BuildTask._PendingQueue[BuildItem] = Bt
 619         BuildTask._PendingQueueLock.release()
 620
 621         return Bt
 622
 623     ## The real constructor of BuildTask
 624     #
 625     #   @param  BuildItem       A BuildUnit object representing a build object
 626     #   @param  Dependency      The dependent build object of BuildItem
 627     #
 628     def _Init(self, BuildItem, Dependency=None):
 629         self.BuildItem = BuildItem
 630
 631         self.DependencyList = []
 632         if Dependency == None:
 633             Dependency = BuildItem.Dependency
 634         else:
 635             Dependency.extend(BuildItem.Dependency)
 636         self.AddDependency(Dependency)
 637         # flag indicating build completes, used to avoid unnecessary re-build
 638         self.CompleteFlag = False
 639
 640     ## Check if all dependent build tasks are completed or not
 641     #
 642     def IsReady(self):
 643         ReadyFlag = True
 644         for Dep in self.DependencyList:
 645             if Dep.CompleteFlag == True:
 646                 continue
 647             ReadyFlag = False
 648             break
 649
 650         return ReadyFlag
 651
 652     ## Add dependent build task
 653     #
 654     #   @param  Dependency      The list of dependent build objects
 655     #
 656     def AddDependency(self, Dependency):
 657         for Dep in Dependency:
 658             if not Dep.BuildObject.IsBinaryModule:
 659                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 660
 661     ## The thread wrapper of LaunchCommand function
 662     #
 663     # @param  Command               A list or string contains the call of the command
 664     # @param  WorkingDir            The directory in which the program will be running
 665     #
 666     def _CommandThread(self, Command, WorkingDir):
 667         try:
 668             LaunchCommand(Command, WorkingDir)
 669             self.CompleteFlag = True
 670         except:
 671             #
 672             # TRICK: hide the output of threads left runing, so that the user can
 673             #        catch the error message easily
 674             #
 675             if not BuildTask._ErrorFlag.isSet():
 676                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 677                                                                   self.BuildItem.BuildObject.Arch,
 678                                                                   self.BuildItem.BuildObject.ToolChain,
 679                                                                   self.BuildItem.BuildObject.BuildTarget
 680                                                                  )
 681             EdkLogger.SetLevel(EdkLogger.ERROR)
 682             BuildTask._ErrorFlag.set()
 683             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 684                                       (threading.currentThread().getName(), Command, WorkingDir)
 685         # indicate there's a thread is available for another build task
 686         BuildTask._RunningQueueLock.acquire()
 687         BuildTask._RunningQueue.pop(self.BuildItem)
 688         BuildTask._RunningQueueLock.release()
 689         BuildTask._Thread.release()
 690
 691     ## Start build task thread
 692     #
 693     def Start(self):
 694         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 695         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 696         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 697         self.BuildTread.setName("build thread")
 698         self.BuildTread.setDaemon(False)
 699         self.BuildTread.start()
 700
 701 ## The class contains the information related to EFI image
 702 #
 703 class PeImageInfo():
 704     ## Constructor
 705     #
 706     # Constructor will load all required image information.
 707     #
 708     #   @param  BaseName          The full file path of image.
 709     #   @param  Guid              The GUID for image.
 710     #   @param  Arch              Arch of this image.
 711     #   @param  OutputDir         The output directory for image.
 712     #   @param  DebugDir          The debug directory for image.
 713     #   @param  ImageClass        PeImage Information
 714     #
 715     def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass):
 716         self.BaseName         = BaseName
 717         self.Guid             = Guid
 718         self.Arch             = Arch
 719         self.OutputDir        = OutputDir
 720         self.DebugDir         = DebugDir
 721         self.Image            = ImageClass
 722         self.Image.Size       = (self.Image.Size / 0x1000 + 1) * 0x1000
 723
 724 ## The class implementing the EDK2 build process
 725 #
 726 #   The build process includes:
 727 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 728 #       2. Parse DSC file of active platform
 729 #       3. Parse FDF file if any
 730 #       4. Establish build database, including parse all other files (module, package)
 731 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 732 #       6. Call build command
 733 #
 734 class Build():
 735     ## Constructor
 736     #
 737     # Constructor will load all necessary configurations, parse platform, modules
 738     # and packages and the establish a database for AutoGen.
 739     #
 740     #   @param  Target              The build command target, one of gSupportedTarget
 741     #   @param  WorkspaceDir        The directory of workspace
 742     #   @param  BuildOptions        Build options passed from command line
 743     #
 744     def __init__(self, Target, WorkspaceDir, BuildOptions):
 745         self.WorkspaceDir   = WorkspaceDir
 746         self.Target         = Target
 747         self.PlatformFile   = BuildOptions.PlatformFile
 748         self.ModuleFile     = BuildOptions.ModuleFile
 749         self.ArchList       = BuildOptions.TargetArch
 750         self.ToolChainList  = BuildOptions.ToolChain
 751         self.BuildTargetList= BuildOptions.BuildTarget
 752         self.Fdf            = BuildOptions.FdfFile
 753         self.FdList         = BuildOptions.RomImage
 754         self.FvList         = BuildOptions.FvImage
 755         self.CapList        = BuildOptions.CapName
 756         self.SilentMode     = BuildOptions.SilentMode
 757         self.ThreadNumber   = BuildOptions.ThreadNumber
 758         self.SkipAutoGen    = BuildOptions.SkipAutoGen
 759         self.Reparse        = BuildOptions.Reparse
 760         self.SkuId          = BuildOptions.SkuId
 761         self.ConfDirectory = BuildOptions.ConfDirectory
 762         self.SpawnMode      = True
 763         self.BuildReport    = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)
 764         self.TargetTxt      = TargetTxtClassObject()
 765         self.ToolDef        = ToolDefClassObject()
 766         GlobalData.BuildOptionPcd     = BuildOptions.OptionPcd
 767         #Set global flag for build mode
 768         GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
 769
 770         if self.ConfDirectory:
 771             # Get alternate Conf location, if it is absolute, then just use the absolute directory name
 772             ConfDirectoryPath = os.path.normpath(self.ConfDirectory)
 773
 774             if not os.path.isabs(ConfDirectoryPath):
 775                 # Since alternate directory name is not absolute, the alternate directory is located within the WORKSPACE
 776                 # This also handles someone specifying the Conf directory in the workspace. Using --conf=Conf
 777                 ConfDirectoryPath = mws.join(self.WorkspaceDir, ConfDirectoryPath)
 778         else:
 779             if "CONF_PATH" in os.environ:
 780                 ConfDirectoryPath = os.path.normcase(os.path.normpath(os.environ["CONF_PATH"]))
 781             else:
 782                 # Get standard WORKSPACE/Conf use the absolute path to the WORKSPACE/Conf
 783                 ConfDirectoryPath = mws.join(self.WorkspaceDir, 'Conf')
 784         GlobalData.gConfDirectory = ConfDirectoryPath
 785         GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))
 786
 787         if BuildOptions.DisableCache:
 788             self.Db         = WorkspaceDatabase(":memory:")
 789         else:
 790             self.Db = WorkspaceDatabase(GlobalData.gDatabasePath, self.Reparse)
 791         self.BuildDatabase = self.Db.BuildObject
 792         self.Platform = None
 793         self.ToolChainFamily = None
 794         self.LoadFixAddress = 0
 795         self.UniFlag        = BuildOptions.Flag
 796         self.BuildModules = []
 797         self.Db_Flag = False
 798         self.LaunchPrebuildFlag = False
 799         self.PrebuildScript = ''
 800         self.PostbuildScript = ''
 801         self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory,'.cache', '.PlatformBuild')
 802         if BuildOptions.CommandLength:
 803             GlobalData.gCommandMaxLength = BuildOptions.CommandLength
 804
 805         # print dot character during doing some time-consuming work
 806         self.Progress = Utils.Progressor()
 807         # print current build environment and configuration
 808         EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 809         if "PACKAGES_PATH" in os.environ:
 810             # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
 811             EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"]))))
 812         EdkLogger.quiet("%-16s = %s" % ("ECP_SOURCE", os.environ["ECP_SOURCE"]))
 813         EdkLogger.quiet("%-16s = %s" % ("EDK_SOURCE", os.environ["EDK_SOURCE"]))
 814         EdkLogger.quiet("%-16s = %s" % ("EFI_SOURCE", os.environ["EFI_SOURCE"]))
 815         EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 816         if "EDK_TOOLS_BIN" in os.environ:
 817             # Print the same path style with WORKSPACE env.
 818             EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
 819         EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
 820         self.InitPreBuild()
 821         self.InitPostBuild()
 822         if self.PrebuildScript:
 823             EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.PrebuildScript))
 824         if self.PostbuildScript:
 825             EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.PostbuildScript))
 826         if self.PrebuildScript:
 827             self.LaunchPrebuild()
 828             self.TargetTxt = TargetTxtClassObject()
 829             self.ToolDef   = ToolDefClassObject()
 830         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
 831             self.InitBuild()
 832
 833         EdkLogger.info("")
 834         os.chdir(self.WorkspaceDir)
 835
 836     ## Load configuration
 837     #
 838     #   This method will parse target.txt and get the build configurations.
 839     #
 840     def LoadConfiguration(self):
 841         #
 842         # Check target.txt and tools_def.txt and Init them
 843         #
 844         BuildConfigurationFile = os.path.normpath(os.path.join(GlobalData.gConfDirectory, gBuildConfiguration))
 845         if os.path.isfile(BuildConfigurationFile) == True:
 846             StatusCode = self.TargetTxt.LoadTargetTxtFile(BuildConfigurationFile)
 847
 848             ToolDefinitionFile = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 849             if ToolDefinitionFile == '':
 850                 ToolDefinitionFile = gToolsDefinition
 851                 ToolDefinitionFile = os.path.normpath(mws.join(self.WorkspaceDir, 'Conf', ToolDefinitionFile))
 852             if os.path.isfile(ToolDefinitionFile) == True:
 853                 StatusCode = self.ToolDef.LoadToolDefFile(ToolDefinitionFile)
 854             else:
 855                 EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=ToolDefinitionFile)
 856         else:
 857             EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile)
 858
 859         # if no ARCH given in command line, get it from target.txt
 860         if not self.ArchList:
 861             self.ArchList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TARGET_ARCH]
 862         self.ArchList = tuple(self.ArchList)
 863
 864         # if no build target given in command line, get it from target.txt
 865         if not self.BuildTargetList:
 866             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TARGET]
 867
 868         # if no tool chain given in command line, get it from target.txt
 869         if not self.ToolChainList:
 870             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 871             if self.ToolChainList == None or len(self.ToolChainList) == 0:
 872                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 873
 874         # check if the tool chains are defined or not
 875         NewToolChainList = []
 876         for ToolChain in self.ToolChainList:
 877             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 878                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 879             else:
 880                 NewToolChainList.append(ToolChain)
 881         # if no tool chain available, break the build
 882         if len(NewToolChainList) == 0:
 883             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 884                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 885         else:
 886             self.ToolChainList = NewToolChainList
 887
 888         ToolChainFamily = []
 889         ToolDefinition = self.ToolDef.ToolsDefTxtDatabase
 890         for Tool in self.ToolChainList:
 891             if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \
 892                or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]:
 893                 EdkLogger.warn("No tool chain family found in configuration for %s. Default to MSFT." % Tool)
 894                 ToolChainFamily.append("MSFT")
 895             else:
 896                 ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
 897         self.ToolChainFamily = ToolChainFamily
 898
 899         if self.ThreadNumber == None:
 900             self.ThreadNumber = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
 901             if self.ThreadNumber == '':
 902                 self.ThreadNumber = 0
 903             else:
 904                 self.ThreadNumber = int(self.ThreadNumber, 0)
 905
 906         if self.ThreadNumber == 0:
 907             self.ThreadNumber = 1
 908
 909         if not self.PlatformFile:
 910             PlatformFile = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 911             if not PlatformFile:
 912                 # Try to find one in current directory
 913                 WorkingDirectory = os.getcwd()
 914                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 915                 FileNum = len(FileList)
 916                 if FileNum >= 2:
 917                     EdkLogger.error("build", OPTION_MISSING,
 918                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 919                 elif FileNum == 1:
 920                     PlatformFile = FileList[0]
 921                 else:
 922                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 923                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 924
 925             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 926
 927     ## Initialize build configuration
 928     #
 929     #   This method will parse DSC file and merge the configurations from
 930     #   command line and target.txt, then get the final build configurations.
 931     #
 932     def InitBuild(self):
 933         # parse target.txt, tools_def.txt, and platform file
 934         self.LoadConfiguration()
 935
 936         # Allow case-insensitive for those from command line or configuration file
 937         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 938         if ErrorCode != 0:
 939             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 940
 941         # create metafile database
 942         if not self.Db_Flag:
 943             self.Db.InitDatabase()
 944
 945     def InitPreBuild(self):
 946         self.LoadConfiguration()
 947         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 948         if ErrorCode != 0:
 949             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 950         if self.BuildTargetList:
 951             GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0]
 952         if self.ArchList:
 953             GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0]
 954         if self.ToolChainList:
 955             GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0]
 956             GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0]
 957         if self.ToolChainFamily:
 958             GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
 959         if 'PREBUILD' in GlobalData.gCommandLineDefines.keys():
 960             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
 961         else:
 962             self.Db.InitDatabase()
 963             self.Db_Flag = True
 964             Platform = self.Db._MapPlatform(str(self.PlatformFile))
 965             self.Prebuild = str(Platform.Prebuild)
 966         if self.Prebuild:
 967             PrebuildList = self.Prebuild.split()
 968             if not os.path.isabs(PrebuildList[0]):
 969                 PrebuildList[0] = mws.join(self.WorkspaceDir, PrebuildList[0])
 970             if os.path.isfile(PrebuildList[0]):
 971                 self.PrebuildScript = PrebuildList[0]
 972                 self.Prebuild = ' '.join(PrebuildList)
 973                 self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList)
 974                 #self.LaunchPrebuild()
 975             else:
 976                 EdkLogger.error("Prebuild", PREBUILD_ERROR, "the prebuild script %s is not exist.\n If you'd like to disable the Prebuild process, please use the format: -D PREBUILD=\"\" " %(PrebuildList[0]))
 977
 978     def InitPostBuild(self):
 979         if 'POSTBUILD' in GlobalData.gCommandLineDefines.keys():
 980             self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD')
 981         else:
 982             Platform = self.Db._MapPlatform(str(self.PlatformFile))
 983             self.Postbuild = str(Platform.Postbuild)
 984         if self.Postbuild:
 985             PostbuildList = self.Postbuild.split()
 986             if not os.path.isabs(PostbuildList[0]):
 987                 PostbuildList[0] = mws.join(self.WorkspaceDir, PostbuildList[0])
 988             if os.path.isfile(PostbuildList[0]):
 989                 self.PostbuildScript = PostbuildList[0]
 990                 self.Postbuild = ' '.join(PostbuildList)
 991                 self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList)
 992             else:
 993                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, "the postbuild script %s is not exist.\n If you'd like to disable the Postbuild process, please use the format: -D POSTBUILD=\"\" " %(PostbuildList[0]))
 994
 995     def PassCommandOption(self, BuildTarget, TargetArch, ToolChain):
 996         BuildStr = ''
 997         if GlobalData.gCommand and isinstance(GlobalData.gCommand, list):
 998             BuildStr += ' ' + ' '.join(GlobalData.gCommand)
 999         TargetFlag = False
1000         ArchFlag = False
1001         ToolChainFlag = False
1002
1003         if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget:
1004             TargetFlag = True
1005         if GlobalData.gOptions and not GlobalData.gOptions.TargetArch:
1006             ArchFlag = True
1007         if GlobalData.gOptions and not GlobalData.gOptions.ToolChain:
1008             ToolChainFlag = True
1009
1010         if TargetFlag and BuildTarget:
1011             if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple):
1012                 BuildStr += ' -b ' + ' -b '.join(BuildTarget)
1013             elif isinstance(BuildTarget, str):
1014                 BuildStr += ' -b ' + BuildTarget
1015         if ArchFlag and TargetArch:
1016             if isinstance(TargetArch, list) or isinstance(TargetArch, tuple):
1017                 BuildStr += ' -a ' + ' -a '.join(TargetArch)
1018             elif isinstance(TargetArch, str):
1019                 BuildStr += ' -a ' + TargetArch
1020         if ToolChainFlag and ToolChain:
1021             if isinstance(ToolChain, list) or isinstance(ToolChain, tuple):
1022                 BuildStr += ' -t ' + ' -t '.join(ToolChain)
1023             elif isinstance(ToolChain, str):
1024                 BuildStr += ' -t ' + ToolChain
1025
1026         return BuildStr
1027
1028     def LaunchPrebuild(self):
1029         if self.Prebuild:
1030             EdkLogger.info("\n- Prebuild Start -\n")
1031             self.LaunchPrebuildFlag = True
1032             #
1033             # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1034             # and preserve them for the rest of the main build step, because the child process environment will
1035             # evaporate as soon as it exits, we cannot get it in build step.
1036             #
1037             PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory,'.cache','.PrebuildEnv')
1038             if os.path.isfile(PrebuildEnvFile):
1039                 os.remove(PrebuildEnvFile)
1040             if os.path.isfile(self.PlatformBuildPath):
1041                 os.remove(self.PlatformBuildPath)
1042             if sys.platform == "win32":
1043                 args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile))
1044                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1045             else:
1046                 args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile))
1047                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1048
1049             # launch two threads to read the STDOUT and STDERR
1050             EndOfProcedure = Event()
1051             EndOfProcedure.clear()
1052             if Process.stdout:
1053                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1054                 StdOutThread.setName("STDOUT-Redirector")
1055                 StdOutThread.setDaemon(False)
1056                 StdOutThread.start()
1057
1058             if Process.stderr:
1059                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1060                 StdErrThread.setName("STDERR-Redirector")
1061                 StdErrThread.setDaemon(False)
1062                 StdErrThread.start()
1063             # waiting for program exit
1064             Process.wait()
1065
1066             if Process.stdout:
1067                 StdOutThread.join()
1068             if Process.stderr:
1069                 StdErrThread.join()
1070             if Process.returncode != 0 :
1071                 EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!')
1072
1073             if os.path.exists(PrebuildEnvFile):
1074                 f = open(PrebuildEnvFile)
1075                 envs = f.readlines()
1076                 f.close()
1077                 envs = itertools.imap(lambda l: l.split('=',1), envs)
1078                 envs = itertools.ifilter(lambda l: len(l) == 2, envs)
1079                 envs = itertools.imap(lambda l: [i.strip() for i in l], envs)
1080                 os.environ.update(dict(envs))
1081             EdkLogger.info("\n- Prebuild Done -\n")
1082
1083     def LaunchPostbuild(self):
1084         if self.Postbuild:
1085             EdkLogger.info("\n- Postbuild Start -\n")
1086             if sys.platform == "win32":
1087                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1088             else:
1089                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1090             # launch two threads to read the STDOUT and STDERR
1091             EndOfProcedure = Event()
1092             EndOfProcedure.clear()
1093             if Process.stdout:
1094                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1095                 StdOutThread.setName("STDOUT-Redirector")
1096                 StdOutThread.setDaemon(False)
1097                 StdOutThread.start()
1098
1099             if Process.stderr:
1100                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1101                 StdErrThread.setName("STDERR-Redirector")
1102                 StdErrThread.setDaemon(False)
1103                 StdErrThread.start()
1104             # waiting for program exit
1105             Process.wait()
1106
1107             if Process.stdout:
1108                 StdOutThread.join()
1109             if Process.stderr:
1110                 StdErrThread.join()
1111             if Process.returncode != 0 :
1112                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
1113             EdkLogger.info("\n- Postbuild Done -\n")
1114     ## Build a module or platform
1115     #
1116     # Create autogen code and makefile for a module or platform, and the launch
1117     # "make" command to build it
1118     #
1119     #   @param  Target                      The target of build command
1120     #   @param  Platform                    The platform file
1121     #   @param  Module                      The module file
1122     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1123     #   @param  ToolChain                   The name of toolchain to build
1124     #   @param  Arch                        The arch of the module/platform
1125     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1126     #                                       for dependent modules/Libraries
1127     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1128     #                                       for dependent modules/Libraries
1129     #
1130     def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1131         if AutoGenObject == None:
1132             return False
1133
1134         # skip file generation for cleanxxx targets, run and fds target
1135         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1136             # for target which must generate AutoGen code and makefile
1137             if not self.SkipAutoGen or Target == 'genc':
1138                 self.Progress.Start("Generating code")
1139                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1140                 self.Progress.Stop("done!")
1141             if Target == "genc":
1142                 return True
1143
1144             if not self.SkipAutoGen or Target == 'genmake':
1145                 self.Progress.Start("Generating makefile")
1146                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1147                 self.Progress.Stop("done!")
1148             if Target == "genmake":
1149                 return True
1150         else:
1151             # always recreate top/platform makefile when clean, just in case of inconsistency
1152             AutoGenObject.CreateCodeFile(False)
1153             AutoGenObject.CreateMakeFile(False)
1154
1155         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1156             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1157
1158         BuildCommand = AutoGenObject.BuildCommand
1159         if BuildCommand == None or len(BuildCommand) == 0:
1160             EdkLogger.error("build", OPTION_MISSING,
1161                             "No build command found for this module. "
1162                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1163                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1164                             ExtraData=str(AutoGenObject))
1165
1166         makefile = GenMake.BuildFile(AutoGenObject)._FILE_NAME_[GenMake.gMakeType]
1167
1168         # run
1169         if Target == 'run':
1170             RunDir = os.path.normpath(os.path.join(AutoGenObject.BuildDir, GlobalData.gGlobalDefines['ARCH']))
1171             Command = '.\SecMain'
1172             os.chdir(RunDir)
1173             LaunchCommand(Command, RunDir)
1174             return True
1175
1176         # build modules
1177         if BuildModule:
1178             BuildCommand = BuildCommand + [Target]
1179             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1180             self.CreateAsBuiltInf()
1181             return True
1182
1183         # build library
1184         if Target == 'libraries':
1185             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1186                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
1187                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1188             return True
1189
1190         # build module
1191         if Target == 'modules':
1192             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1193                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
1194                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1195             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1196                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']
1197                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1198             self.CreateAsBuiltInf()
1199             return True
1200
1201         # cleanlib
1202         if Target == 'cleanlib':
1203             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1204                 LibMakefile = os.path.normpath(os.path.join(Lib, makefile))
1205                 if os.path.exists(LibMakefile):
1206                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1207                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1208             return True
1209
1210         # clean
1211         if Target == 'clean':
1212             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1213                 ModMakefile = os.path.normpath(os.path.join(Mod, makefile))
1214                 if os.path.exists(ModMakefile):
1215                     NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']
1216                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1217             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1218                 LibMakefile = os.path.normpath(os.path.join(Lib, makefile))
1219                 if os.path.exists(LibMakefile):
1220                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1221                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1222             return True
1223
1224         # cleanall
1225         if Target == 'cleanall':
1226             try:
1227                 #os.rmdir(AutoGenObject.BuildDir)
1228                 RemoveDirectory(AutoGenObject.BuildDir, True)
1229             except WindowsError, X:
1230                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1231         return True
1232
1233     ## Build a module or platform
1234     #
1235     # Create autogen code and makefile for a module or platform, and the launch
1236     # "make" command to build it
1237     #
1238     #   @param  Target                      The target of build command
1239     #   @param  Platform                    The platform file
1240     #   @param  Module                      The module file
1241     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1242     #   @param  ToolChain                   The name of toolchain to build
1243     #   @param  Arch                        The arch of the module/platform
1244     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1245     #                                       for dependent modules/Libraries
1246     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1247     #                                       for dependent modules/Libraries
1248     #
1249     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1250         if AutoGenObject == None:
1251             return False
1252
1253         # skip file generation for cleanxxx targets, run and fds target
1254         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1255             # for target which must generate AutoGen code and makefile
1256             if not self.SkipAutoGen or Target == 'genc':
1257                 self.Progress.Start("Generating code")
1258                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1259                 self.Progress.Stop("done!")
1260             if Target == "genc":
1261                 return True
1262
1263             if not self.SkipAutoGen or Target == 'genmake':
1264                 self.Progress.Start("Generating makefile")
1265                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1266                 #AutoGenObject.CreateAsBuiltInf()
1267                 self.Progress.Stop("done!")
1268             if Target == "genmake":
1269                 return True
1270         else:
1271             # always recreate top/platform makefile when clean, just in case of inconsistency
1272             AutoGenObject.CreateCodeFile(False)
1273             AutoGenObject.CreateMakeFile(False)
1274
1275         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1276             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1277
1278         BuildCommand = AutoGenObject.BuildCommand
1279         if BuildCommand == None or len(BuildCommand) == 0:
1280             EdkLogger.error("build", OPTION_MISSING,
1281                             "No build command found for this module. "
1282                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1283                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1284                             ExtraData=str(AutoGenObject))
1285
1286         # build modules
1287         if BuildModule:
1288             if Target != 'fds':
1289                 BuildCommand = BuildCommand + [Target]
1290             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1291             self.CreateAsBuiltInf()
1292             return True
1293
1294         # genfds
1295         if Target == 'fds':
1296             LaunchCommand(AutoGenObject.GenFdsCommand, AutoGenObject.MakeFileDir)
1297             return True
1298
1299         # run
1300         if Target == 'run':
1301             RunDir = os.path.normpath(os.path.join(AutoGenObject.BuildDir, GlobalData.gGlobalDefines['ARCH']))
1302             Command = '.\SecMain'
1303             os.chdir(RunDir)
1304             LaunchCommand(Command, RunDir)
1305             return True
1306
1307         # build library
1308         if Target == 'libraries':
1309             pass
1310
1311         # not build modules
1312
1313
1314         # cleanall
1315         if Target == 'cleanall':
1316             try:
1317                 #os.rmdir(AutoGenObject.BuildDir)
1318                 RemoveDirectory(AutoGenObject.BuildDir, True)
1319             except WindowsError, X:
1320                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1321         return True
1322
1323     ## Rebase module image and Get function address for the input module list.
1324     #
1325     def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False):
1326         if ModeIsSmm:
1327             AddrIsOffset = False
1328         InfFileNameList = ModuleList.keys()
1329         #InfFileNameList.sort()
1330         for InfFile in InfFileNameList:
1331             sys.stdout.write (".")
1332             sys.stdout.flush()
1333             ModuleInfo = ModuleList[InfFile]
1334             ModuleName = ModuleInfo.BaseName
1335             ModuleOutputImage = ModuleInfo.Image.FileName
1336             ModuleDebugImage  = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi')
1337             ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1338             if not ModeIsSmm:
1339                 BaseAddress = BaseAddress - ModuleInfo.Image.Size
1340                 #
1341                 # Update Image to new BaseAddress by GenFw tool
1342                 #
1343                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1344                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1345             else:
1346                 #
1347                 # Set new address to the section header only for SMM driver.
1348                 #
1349                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1350                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1351             #
1352             # Collect funtion address from Map file
1353             #
1354             ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
1355             FunctionList = []
1356             if os.path.exists(ImageMapTable):
1357                 OrigImageBaseAddress = 0
1358                 ImageMap = open(ImageMapTable, 'r')
1359                 for LinStr in ImageMap:
1360                     if len (LinStr.strip()) == 0:
1361                         continue
1362                     #
1363                     # Get the preferred address set on link time.
1364                     #
1365                     if LinStr.find ('Preferred load address is') != -1:
1366                         StrList = LinStr.split()
1367                         OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16)
1368
1369                     StrList = LinStr.split()
1370                     if len (StrList) > 4:
1371                         if StrList[3] == 'f' or StrList[3] == 'F':
1372                             Name = StrList[1]
1373                             RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress
1374                             FunctionList.append ((Name, RelativeAddress))
1375                             if ModuleInfo.Arch == 'IPF' and Name.endswith('_ModuleEntryPoint'):
1376                                 #
1377                                 # Get the real entry point address for IPF image.
1378                                 #
1379                                 ModuleInfo.Image.EntryPoint = RelativeAddress
1380                 ImageMap.close()
1381             #
1382             # Add general information.
1383             #
1384             if ModeIsSmm:
1385                 MapBuffer.write('\n\n%s (Fixed SMRAM Offset,   BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1386             elif AddrIsOffset:
1387                 MapBuffer.write('\n\n%s (Fixed Memory Offset,  BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
1388             else:
1389                 MapBuffer.write('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1390             #
1391             # Add guid and general seciton section.
1392             #
1393             TextSectionAddress = 0
1394             DataSectionAddress = 0
1395             for SectionHeader in ModuleInfo.Image.SectionHeaderList:
1396                 if SectionHeader[0] == '.text':
1397                     TextSectionAddress = SectionHeader[1]
1398                 elif SectionHeader[0] in ['.data', '.sdata']:
1399                     DataSectionAddress = SectionHeader[1]
1400             if AddrIsOffset:
1401                 MapBuffer.write('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
1402             else:
1403                 MapBuffer.write('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
1404             #
1405             # Add debug image full path.
1406             #
1407             MapBuffer.write('(IMAGE=%s)\n\n' % (ModuleDebugImage))
1408             #
1409             # Add funtion address
1410             #
1411             for Function in FunctionList:
1412                 if AddrIsOffset:
1413                     MapBuffer.write('  -0x%010X    %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
1414                 else:
1415                     MapBuffer.write('  0x%010X    %s\n' % (BaseAddress + Function[1], Function[0]))
1416             ImageMap.close()
1417
1418             #
1419             # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1420             #
1421             if ModeIsSmm:
1422                 BaseAddress = BaseAddress + ModuleInfo.Image.Size
1423
1424     ## Collect MAP information of all FVs
1425     #
1426     def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList):
1427         if self.Fdf:
1428             # First get the XIP base address for FV map file.
1429             GuidPattern = re.compile("[-a-fA-F0-9]+")
1430             GuidName = re.compile("\(GUID=[-a-fA-F0-9]+")
1431             for FvName in Wa.FdfProfile.FvDict.keys():
1432                 FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map')
1433                 if not os.path.exists(FvMapBuffer):
1434                     continue
1435                 FvMap = open(FvMapBuffer, 'r')
1436                 #skip FV size information
1437                 FvMap.readline()
1438                 FvMap.readline()
1439                 FvMap.readline()
1440                 FvMap.readline()
1441                 for Line in FvMap:
1442                     MatchGuid = GuidPattern.match(Line)
1443                     if MatchGuid != None:
1444                         #
1445                         # Replace GUID with module name
1446                         #
1447                         GuidString = MatchGuid.group()
1448                         if GuidString.upper() in ModuleList:
1449                             Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
1450                     MapBuffer.write('%s' % (Line))
1451                     #
1452                     # Add the debug image full path.
1453                     #
1454                     MatchGuid = GuidName.match(Line)
1455                     if MatchGuid != None:
1456                         GuidString = MatchGuid.group().split("=")[1]
1457                         if GuidString.upper() in ModuleList:
1458                             MapBuffer.write('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
1459
1460                 FvMap.close()
1461
1462     ## Collect MAP information of all modules
1463     #
1464     def _CollectModuleMapBuffer (self, MapBuffer, ModuleList):
1465         sys.stdout.write ("Generate Load Module At Fix Address Map")
1466         sys.stdout.flush()
1467         PatchEfiImageList = []
1468         PeiModuleList  = {}
1469         BtModuleList   = {}
1470         RtModuleList   = {}
1471         SmmModuleList  = {}
1472         PeiSize = 0
1473         BtSize  = 0
1474         RtSize  = 0
1475         # reserve 4K size in SMRAM to make SMM module address not from 0.
1476         SmmSize = 0x1000
1477         IsIpfPlatform = False
1478         if 'IPF' in self.ArchList:
1479             IsIpfPlatform = True
1480         for ModuleGuid in ModuleList:
1481             Module = ModuleList[ModuleGuid]
1482             GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget)
1483
1484             OutputImageFile = ''
1485             for ResultFile in Module.CodaTargetList:
1486                 if str(ResultFile.Target).endswith('.efi'):
1487                     #
1488                     # module list for PEI, DXE, RUNTIME and SMM
1489                     #
1490                     OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi')
1491                     ImageClass = PeImageClass (OutputImageFile)
1492                     if not ImageClass.IsValid:
1493                         EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo)
1494                     ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass)
1495                     if Module.ModuleType in ['PEI_CORE', 'PEIM', 'COMBINED_PEIM_DRIVER', 'PIC_PEIM', 'RELOCATABLE_PEIM', 'DXE_CORE']:
1496                         PeiModuleList[Module.MetaFile] = ImageInfo
1497                         PeiSize += ImageInfo.Image.Size
1498                     elif Module.ModuleType in ['BS_DRIVER', 'DXE_DRIVER', 'UEFI_DRIVER']:
1499                         BtModuleList[Module.MetaFile] = ImageInfo
1500                         BtSize += ImageInfo.Image.Size
1501                     elif Module.ModuleType in ['DXE_RUNTIME_DRIVER', 'RT_DRIVER', 'DXE_SAL_DRIVER', 'SAL_RT_DRIVER']:
1502                         RtModuleList[Module.MetaFile] = ImageInfo
1503                         #IPF runtime driver needs to be at 2 page alignment.
1504                         if IsIpfPlatform and ImageInfo.Image.Size % 0x2000 != 0:
1505                             ImageInfo.Image.Size = (ImageInfo.Image.Size / 0x2000 + 1) * 0x2000
1506                         RtSize += ImageInfo.Image.Size
1507                     elif Module.ModuleType in ['SMM_CORE', 'DXE_SMM_DRIVER', 'MM_STANDALONE', 'MM_CORE_STANDALONE']:
1508                         SmmModuleList[Module.MetaFile] = ImageInfo
1509                         SmmSize += ImageInfo.Image.Size
1510                         if Module.ModuleType == 'DXE_SMM_DRIVER':
1511                             PiSpecVersion = '0x00000000'
1512                             if 'PI_SPECIFICATION_VERSION' in Module.Module.Specification:
1513                                 PiSpecVersion = Module.Module.Specification['PI_SPECIFICATION_VERSION']
1514                             # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1515                             if int(PiSpecVersion, 16) < 0x0001000A:
1516                                 BtModuleList[Module.MetaFile] = ImageInfo
1517                                 BtSize += ImageInfo.Image.Size
1518                     break
1519             #
1520             # EFI image is final target.
1521             # Check EFI image contains patchable FixAddress related PCDs.
1522             #
1523             if OutputImageFile != '':
1524                 ModuleIsPatch = False
1525                 for Pcd in Module.ModulePcdList:
1526                     if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_LIST:
1527                         ModuleIsPatch = True
1528                         break
1529                 if not ModuleIsPatch:
1530                     for Pcd in Module.LibraryPcdList:
1531                         if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_LIST:
1532                             ModuleIsPatch = True
1533                             break
1534
1535                 if not ModuleIsPatch:
1536                     continue
1537                 #
1538                 # Module includes the patchable load fix address PCDs.
1539                 # It will be fixed up later.
1540                 #
1541                 PatchEfiImageList.append (OutputImageFile)
1542
1543         #
1544         # Get Top Memory address
1545         #
1546         ReservedRuntimeMemorySize = 0
1547         TopMemoryAddress = 0
1548         if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF:
1549             TopMemoryAddress = 0
1550         else:
1551             TopMemoryAddress = self.LoadFixAddress
1552             if TopMemoryAddress < RtSize + BtSize + PeiSize:
1553                 EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1554             # Make IPF runtime driver at 2 page alignment.
1555             if IsIpfPlatform:
1556                 ReservedRuntimeMemorySize = TopMemoryAddress % 0x2000
1557                 RtSize = RtSize + ReservedRuntimeMemorySize
1558
1559         #
1560         # Patch FixAddress related PCDs into EFI image
1561         #
1562         for EfiImage in PatchEfiImageList:
1563             EfiImageMap = EfiImage.replace('.efi', '.map')
1564             if not os.path.exists(EfiImageMap):
1565                 continue
1566             #
1567             # Get PCD offset in EFI image by GenPatchPcdTable function
1568             #
1569             PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage)
1570             #
1571             # Patch real PCD value by PatchPcdValue tool
1572             #
1573             for PcdInfo in PcdTable:
1574                 ReturnValue = 0
1575                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:
1576                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize / 0x1000))
1577                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:
1578                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize / 0x1000))
1579                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:
1580                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize / 0x1000))
1581                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:
1582                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize / 0x1000))
1583                 if ReturnValue != 0:
1584                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
1585
1586         MapBuffer.write('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize / 0x1000))
1587         MapBuffer.write('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize / 0x1000))
1588         MapBuffer.write('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize / 0x1000))
1589         if len (SmmModuleList) > 0:
1590             MapBuffer.write('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize / 0x1000))
1591
1592         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
1593         BtBaseAddr  = TopMemoryAddress - RtSize
1594         RtBaseAddr  = TopMemoryAddress - ReservedRuntimeMemorySize
1595
1596         self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0)
1597         self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
1598         self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
1599         self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
1600         MapBuffer.write('\n\n')
1601         sys.stdout.write ("\n")
1602         sys.stdout.flush()
1603
1604     ## Save platform Map file
1605     #
1606     def _SaveMapFile (self, MapBuffer, Wa):
1607         #
1608         # Map file path is got.
1609         #
1610         MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map')
1611         #
1612         # Save address map into MAP file.
1613         #
1614         SaveFileOnChange(MapFilePath, MapBuffer.getvalue(), False)
1615         MapBuffer.close()
1616         if self.LoadFixAddress != 0:
1617             sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
1618         sys.stdout.flush()
1619
1620     ## Build active platform for different build targets and different tool chains
1621     #
1622     def _BuildPlatform(self):
1623         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1624         for BuildTarget in self.BuildTargetList:
1625             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1626             index = 0
1627             for ToolChain in self.ToolChainList:
1628                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1629                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1630                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1631                 index += 1
1632                 Wa = WorkspaceAutoGen(
1633                         self.WorkspaceDir,
1634                         self.PlatformFile,
1635                         BuildTarget,
1636                         ToolChain,
1637                         self.ArchList,
1638                         self.BuildDatabase,
1639                         self.TargetTxt,
1640                         self.ToolDef,
1641                         self.Fdf,
1642                         self.FdList,
1643                         self.FvList,
1644                         self.CapList,
1645                         self.SkuId,
1646                         self.UniFlag,
1647                         self.Progress
1648                         )
1649                 self.Fdf = Wa.FdfFile
1650                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1651                 self.BuildReport.AddPlatformReport(Wa)
1652                 self.Progress.Stop("done!")
1653                 for Arch in Wa.ArchList:
1654                     GlobalData.gGlobalDefines['ARCH'] = Arch
1655                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1656                     for Module in Pa.Platform.Modules:
1657                         # Get ModuleAutoGen object to generate C code file and makefile
1658                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
1659                         if Ma == None:
1660                             continue
1661                         self.BuildModules.append(Ma)
1662                     self._BuildPa(self.Target, Pa)
1663
1664                 # Create MAP file when Load Fix Address is enabled.
1665                 if self.Target in ["", "all", "fds"]:
1666                     for Arch in Wa.ArchList:
1667                         GlobalData.gGlobalDefines['ARCH'] = Arch
1668                         #
1669                         # Check whether the set fix address is above 4G for 32bit image.
1670                         #
1671                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1672                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1673                     #
1674                     # Get Module List
1675                     #
1676                     ModuleList = {}
1677                     for Pa in Wa.AutoGenObjectList:
1678                         for Ma in Pa.ModuleAutoGenList:
1679                             if Ma == None:
1680                                 continue
1681                             if not Ma.IsLibrary:
1682                                 ModuleList[Ma.Guid.upper()] = Ma
1683
1684                     MapBuffer = StringIO('')
1685                     if self.LoadFixAddress != 0:
1686                         #
1687                         # Rebase module to the preferred memory address before GenFds
1688                         #
1689                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1690                     if self.Fdf:
1691                         #
1692                         # create FDS again for the updated EFI image
1693                         #
1694                         self._Build("fds", Wa)
1695                         #
1696                         # Create MAP file for all platform FVs after GenFds.
1697                         #
1698                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1699                     #
1700                     # Save MAP buffer into MAP file.
1701                     #
1702                     self._SaveMapFile (MapBuffer, Wa)
1703
1704     ## Build active module for different build targets, different tool chains and different archs
1705     #
1706     def _BuildModule(self):
1707         for BuildTarget in self.BuildTargetList:
1708             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1709             index = 0
1710             for ToolChain in self.ToolChainList:
1711                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1712                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1713                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1714                 index += 1
1715                 #
1716                 # module build needs platform build information, so get platform
1717                 # AutoGen first
1718                 #
1719                 Wa = WorkspaceAutoGen(
1720                         self.WorkspaceDir,
1721                         self.PlatformFile,
1722                         BuildTarget,
1723                         ToolChain,
1724                         self.ArchList,
1725                         self.BuildDatabase,
1726                         self.TargetTxt,
1727                         self.ToolDef,
1728                         self.Fdf,
1729                         self.FdList,
1730                         self.FvList,
1731                         self.CapList,
1732                         self.SkuId,
1733                         self.UniFlag,
1734                         self.Progress,
1735                         self.ModuleFile
1736                         )
1737                 self.Fdf = Wa.FdfFile
1738                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1739                 Wa.CreateMakeFile(False)
1740                 self.Progress.Stop("done!")
1741                 MaList = []
1742                 for Arch in Wa.ArchList:
1743                     GlobalData.gGlobalDefines['ARCH'] = Arch
1744                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1745                     for Module in Pa.Platform.Modules:
1746                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.File == Module.File:
1747                             Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
1748                             if Ma == None: continue
1749                             MaList.append(Ma)
1750                             self.BuildModules.append(Ma)
1751                             if not Ma.IsBinaryModule:
1752                                 self._Build(self.Target, Ma, BuildModule=True)
1753
1754                 self.BuildReport.AddPlatformReport(Wa, MaList)
1755                 if MaList == []:
1756                     EdkLogger.error(
1757                                 'build',
1758                                 BUILD_ERROR,
1759                                 "Module for [%s] is not a component of active platform."\
1760                                 " Please make sure that the ARCH and inf file path are"\
1761                                 " given in the same as in [%s]" % \
1762                                     (', '.join(Wa.ArchList), self.PlatformFile),
1763                                 ExtraData=self.ModuleFile
1764                                 )
1765                 # Create MAP file when Load Fix Address is enabled.
1766                 if self.Target == "fds" and self.Fdf:
1767                     for Arch in Wa.ArchList:
1768                         #
1769                         # Check whether the set fix address is above 4G for 32bit image.
1770                         #
1771                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1772                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1773                     #
1774                     # Get Module List
1775                     #
1776                     ModuleList = {}
1777                     for Pa in Wa.AutoGenObjectList:
1778                         for Ma in Pa.ModuleAutoGenList:
1779                             if Ma == None:
1780                                 continue
1781                             if not Ma.IsLibrary:
1782                                 ModuleList[Ma.Guid.upper()] = Ma
1783
1784                     MapBuffer = StringIO('')
1785                     if self.LoadFixAddress != 0:
1786                         #
1787                         # Rebase module to the preferred memory address before GenFds
1788                         #
1789                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1790                     #
1791                     # create FDS again for the updated EFI image
1792                     #
1793                     self._Build("fds", Wa)
1794                     #
1795                     # Create MAP file for all platform FVs after GenFds.
1796                     #
1797                     self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1798                     #
1799                     # Save MAP buffer into MAP file.
1800                     #
1801                     self._SaveMapFile (MapBuffer, Wa)
1802
1803     ## Build a platform in multi-thread mode
1804     #
1805     def _MultiThreadBuildPlatform(self):
1806         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1807         for BuildTarget in self.BuildTargetList:
1808             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1809             index = 0
1810             for ToolChain in self.ToolChainList:
1811                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1812                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1813                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1814                 index += 1
1815                 Wa = WorkspaceAutoGen(
1816                         self.WorkspaceDir,
1817                         self.PlatformFile,
1818                         BuildTarget,
1819                         ToolChain,
1820                         self.ArchList,
1821                         self.BuildDatabase,
1822                         self.TargetTxt,
1823                         self.ToolDef,
1824                         self.Fdf,
1825                         self.FdList,
1826                         self.FvList,
1827                         self.CapList,
1828                         self.SkuId,
1829                         self.UniFlag,
1830                         self.Progress
1831                         )
1832                 self.Fdf = Wa.FdfFile
1833                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1834                 self.BuildReport.AddPlatformReport(Wa)
1835                 Wa.CreateMakeFile(False)
1836
1837                 # multi-thread exit flag
1838                 ExitFlag = threading.Event()
1839                 ExitFlag.clear()
1840                 for Arch in Wa.ArchList:
1841                     GlobalData.gGlobalDefines['ARCH'] = Arch
1842                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1843                     if Pa == None:
1844                         continue
1845                     ModuleList = []
1846                     for Inf in Pa.Platform.Modules:
1847                         ModuleList.append(Inf)
1848                     # Add the INF only list in FDF
1849                     if GlobalData.gFdfParser != None:
1850                         for InfName in GlobalData.gFdfParser.Profile.InfList:
1851                             Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
1852                             if Inf in Pa.Platform.Modules:
1853                                 continue
1854                             ModuleList.append(Inf)
1855                     for Module in ModuleList:
1856                         # Get ModuleAutoGen object to generate C code file and makefile
1857                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
1858
1859                         if Ma == None:
1860                             continue
1861                         # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1862                         if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1863                             # for target which must generate AutoGen code and makefile
1864                             if not self.SkipAutoGen or self.Target == 'genc':
1865                                 Ma.CreateCodeFile(True)
1866                             if self.Target == "genc":
1867                                 continue
1868
1869                             if not self.SkipAutoGen or self.Target == 'genmake':
1870                                 Ma.CreateMakeFile(True)
1871                             if self.Target == "genmake":
1872                                 continue
1873                         self.BuildModules.append(Ma)
1874                     self.Progress.Stop("done!")
1875
1876                     for Ma in self.BuildModules:
1877                         # Generate build task for the module
1878                         if not Ma.IsBinaryModule:
1879                             Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))
1880                         # Break build if any build thread has error
1881                         if BuildTask.HasError():
1882                             # we need a full version of makefile for platform
1883                             ExitFlag.set()
1884                             BuildTask.WaitForComplete()
1885                             Pa.CreateMakeFile(False)
1886                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1887                         # Start task scheduler
1888                         if not BuildTask.IsOnGoing():
1889                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1890
1891                     # in case there's an interruption. we need a full version of makefile for platform
1892                     Pa.CreateMakeFile(False)
1893                     if BuildTask.HasError():
1894                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1895
1896                 #
1897                 # Save temp tables to a TmpTableDict.
1898                 #
1899                 for Key in Wa.BuildDatabase._CACHE_:
1900                     if Wa.BuildDatabase._CACHE_[Key]._RawData and Wa.BuildDatabase._CACHE_[Key]._RawData._Table and Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:
1901                         if TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table):
1902                             TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] = Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur
1903                 #
1904                 #
1905                 # All modules have been put in build tasks queue. Tell task scheduler
1906                 # to exit if all tasks are completed
1907                 #
1908                 ExitFlag.set()
1909                 BuildTask.WaitForComplete()
1910                 self.CreateAsBuiltInf()
1911
1912                 #
1913                 # Check for build error, and raise exception if one
1914                 # has been signaled.
1915                 #
1916                 if BuildTask.HasError():
1917                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1918
1919                 # Create MAP file when Load Fix Address is enabled.
1920                 if self.Target in ["", "all", "fds"]:
1921                     for Arch in Wa.ArchList:
1922                         #
1923                         # Check whether the set fix address is above 4G for 32bit image.
1924                         #
1925                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1926                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1927                     #
1928                     # Get Module List
1929                     #
1930                     ModuleList = {}
1931                     for Pa in Wa.AutoGenObjectList:
1932                         for Ma in Pa.ModuleAutoGenList:
1933                             if Ma == None:
1934                                 continue
1935                             if not Ma.IsLibrary:
1936                                 ModuleList[Ma.Guid.upper()] = Ma
1937                     #
1938                     # Rebase module to the preferred memory address before GenFds
1939                     #
1940                     MapBuffer = StringIO('')
1941                     if self.LoadFixAddress != 0:
1942                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1943
1944                     if self.Fdf:
1945                         #
1946                         # Generate FD image if there's a FDF file found
1947                         #
1948                         LaunchCommand(Wa.GenFdsCommand, os.getcwd())
1949
1950                         #
1951                         # Create MAP file for all platform FVs after GenFds.
1952                         #
1953                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1954                     #
1955                     # Save MAP buffer into MAP file.
1956                     #
1957                     self._SaveMapFile(MapBuffer, Wa)
1958
1959     ## Generate GuidedSectionTools.txt in the FV directories.
1960     #
1961     def CreateGuidedSectionToolsFile(self):
1962         for BuildTarget in self.BuildTargetList:
1963             for ToolChain in self.ToolChainList:
1964                 Wa = WorkspaceAutoGen(
1965                         self.WorkspaceDir,
1966                         self.PlatformFile,
1967                         BuildTarget,
1968                         ToolChain,
1969                         self.ArchList,
1970                         self.BuildDatabase,
1971                         self.TargetTxt,
1972                         self.ToolDef,
1973                         self.Fdf,
1974                         self.FdList,
1975                         self.FvList,
1976                         self.CapList,
1977                         self.SkuId,
1978                         self.UniFlag
1979                         )
1980                 FvDir = Wa.FvDir
1981                 if not os.path.exists(FvDir):
1982                     continue
1983
1984                 for Arch in self.ArchList:
1985                     # Build up the list of supported architectures for this build
1986                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
1987
1988                     # Look through the tool definitions for GUIDed tools
1989                     guidAttribs = []
1990                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.iteritems():
1991                         if attrib.upper().endswith('_GUID'):
1992                             split = attrib.split('_')
1993                             thisPrefix = '_'.join(split[0:3]) + '_'
1994                             if thisPrefix == prefix:
1995                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
1996                                 guid = guid.lower()
1997                                 toolName = split[3]
1998                                 path = '_'.join(split[0:4]) + '_PATH'
1999                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
2000                                 path = self.GetFullPathOfTool(path)
2001                                 guidAttribs.append((guid, toolName, path))
2002
2003                     # Write out GuidedSecTools.txt
2004                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
2005                     toolsFile = open(toolsFile, 'wt')
2006                     for guidedSectionTool in guidAttribs:
2007                         print >> toolsFile, ' '.join(guidedSectionTool)
2008                     toolsFile.close()
2009
2010     ## Returns the full path of the tool.
2011     #
2012     def GetFullPathOfTool (self, tool):
2013         if os.path.exists(tool):
2014             return os.path.realpath(tool)
2015         else:
2016             # We need to search for the tool using the
2017             # PATH environment variable.
2018             for dirInPath in os.environ['PATH'].split(os.pathsep):
2019                 foundPath = os.path.join(dirInPath, tool)
2020                 if os.path.exists(foundPath):
2021                     return os.path.realpath(foundPath)
2022
2023         # If the tool was not found in the path then we just return
2024         # the input tool.
2025         return tool
2026
2027     ## Launch the module or platform build
2028     #
2029     def Launch(self):
2030         if not self.ModuleFile:
2031             if not self.SpawnMode or self.Target not in ["", "all"]:
2032                 self.SpawnMode = False
2033                 self._BuildPlatform()
2034             else:
2035                 self._MultiThreadBuildPlatform()
2036             self.CreateGuidedSectionToolsFile()
2037         else:
2038             self.SpawnMode = False
2039             self._BuildModule()
2040
2041         if self.Target == 'cleanall':
2042             self.Db.Close()
2043             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
2044
2045     def CreateAsBuiltInf(self):
2046         for Module in self.BuildModules:
2047             Module.CreateAsBuiltInf()
2048         self.BuildModules = []
2049     ## Do some clean-up works when error occurred
2050     def Relinquish(self):
2051         OldLogLevel = EdkLogger.GetLevel()
2052         EdkLogger.SetLevel(EdkLogger.ERROR)
2053         #self.DumpBuildData()
2054         Utils.Progressor.Abort()
2055         if self.SpawnMode == True:
2056             BuildTask.Abort()
2057         EdkLogger.SetLevel(OldLogLevel)
2058
2059     def DumpBuildData(self):
2060         CacheDirectory = os.path.dirname(GlobalData.gDatabasePath)
2061         Utils.CreateDirectory(CacheDirectory)
2062         Utils.DataDump(Utils.gFileTimeStampCache, os.path.join(CacheDirectory, "gFileTimeStampCache"))
2063         Utils.DataDump(Utils.gDependencyDatabase, os.path.join(CacheDirectory, "gDependencyDatabase"))
2064
2065     def RestoreBuildData(self):
2066         FilePath = os.path.join(os.path.dirname(GlobalData.gDatabasePath), "gFileTimeStampCache")
2067         if Utils.gFileTimeStampCache == {} and os.path.isfile(FilePath):
2068             Utils.gFileTimeStampCache = Utils.DataRestore(FilePath)
2069             if Utils.gFileTimeStampCache == None:
2070                 Utils.gFileTimeStampCache = {}
2071
2072         FilePath = os.path.join(os.path.dirname(GlobalData.gDatabasePath), "gDependencyDatabase")
2073         if Utils.gDependencyDatabase == {} and os.path.isfile(FilePath):
2074             Utils.gDependencyDatabase = Utils.DataRestore(FilePath)
2075             if Utils.gDependencyDatabase == None:
2076                 Utils.gDependencyDatabase = {}
2077
2078 def ParseDefines(DefineList=[]):
2079     DefineDict = {}
2080     if DefineList != None:
2081         for Define in DefineList:
2082             DefineTokenList = Define.split("=", 1)
2083             if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]):
2084                 EdkLogger.error('build', FORMAT_INVALID,
2085                                 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2086                                 ExtraData=DefineTokenList[0])
2087
2088             if len(DefineTokenList) == 1:
2089                 DefineDict[DefineTokenList[0]] = "TRUE"
2090             else:
2091                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
2092     return DefineDict
2093
2094 gParamCheck = []
2095 def SingleCheckCallback(option, opt_str, value, parser):
2096     if option not in gParamCheck:
2097         setattr(parser.values, option.dest, value)
2098         gParamCheck.append(option)
2099     else:
2100         parser.error("Option %s only allows one instance in command line!" % option)
2101
2102 ## Parse command line options
2103 #
2104 # Using standard Python module optparse to parse command line option of this tool.
2105 #
2106 #   @retval Opt   A optparse.Values object containing the parsed options
2107 #   @retval Args  Target of build command
2108 #
2109 def MyOptionParser():
2110     Parser = OptionParser(description=__copyright__, version=__version__, prog="build.exe", usage="%prog [options] [all|fds|genc|genmake|clean|cleanall|cleanlib|modules|libraries|run]")
2111     Parser.add_option("-a", "--arch", action="append", type="choice", choices=['IA32', 'X64', 'IPF', 'EBC', 'ARM', 'AARCH64'], dest="TargetArch",
2112         help="ARCHS is one of list: IA32, X64, IPF, ARM, AARCH64 or EBC, which overrides target.txt's TARGET_ARCH definition. To specify more archs, please repeat this option.")
2113     Parser.add_option("-p", "--platform", action="callback", type="string", dest="PlatformFile", callback=SingleCheckCallback,
2114         help="Build the platform specified by the DSC file name argument, overriding target.txt's ACTIVE_PLATFORM definition.")
2115     Parser.add_option("-m", "--module", action="callback", type="string", dest="ModuleFile", callback=SingleCheckCallback,
2116         help="Build the module specified by the INF file name argument.")
2117     Parser.add_option("-b", "--buildtarget", type="string", dest="BuildTarget", help="Using the TARGET to build the platform, overriding target.txt's TARGET definition.",
2118                       action="append")
2119     Parser.add_option("-t", "--tagname", action="append", type="string", dest="ToolChain",
2120         help="Using the Tool Chain Tagname to build the platform, overriding target.txt's TOOL_CHAIN_TAG definition.")
2121     Parser.add_option("-x", "--sku-id", action="callback", type="string", dest="SkuId", callback=SingleCheckCallback,
2122         help="Using this name of SKU ID to build the platform, overriding SKUID_IDENTIFIER in DSC file.")
2123
2124     Parser.add_option("-n", action="callback", type="int", dest="ThreadNumber", callback=SingleCheckCallback,
2125         help="Build the platform using multi-threaded compiler. The value overrides target.txt's MAX_CONCURRENT_THREAD_NUMBER. Less than 2 will disable multi-thread builds.")
2126
2127     Parser.add_option("-f", "--fdf", action="callback", type="string", dest="FdfFile", callback=SingleCheckCallback,
2128         help="The name of the FDF file to use, which overrides the setting in the DSC file.")
2129     Parser.add_option("-r", "--rom-image", action="append", type="string", dest="RomImage", default=[],
2130         help="The name of FD to be generated. The name must be from [FD] section in FDF file.")
2131     Parser.add_option("-i", "--fv-image", action="append", type="string", dest="FvImage", default=[],
2132         help="The name of FV to be generated. The name must be from [FV] section in FDF file.")
2133     Parser.add_option("-C", "--capsule-image", action="append", type="string", dest="CapName", default=[],
2134         help="The name of Capsule to be generated. The name must be from [Capsule] section in FDF file.")
2135     Parser.add_option("-u", "--skip-autogen", action="store_true", dest="SkipAutoGen", help="Skip AutoGen step.")
2136     Parser.add_option("-e", "--re-parse", action="store_true", dest="Reparse", help="Re-parse all meta-data files.")
2137
2138     Parser.add_option("-c", "--case-insensitive", action="store_true", dest="CaseInsensitive", default=False, help="Don't check case of file name.")
2139
2140     Parser.add_option("-w", "--warning-as-error", action="store_true", dest="WarningAsError", help="Treat warning in tools as error.")
2141     Parser.add_option("-j", "--log", action="store", dest="LogFile", help="Put log in specified file as well as on console.")
2142
2143     Parser.add_option("-s", "--silent", action="store_true", type=None, dest="SilentMode",
2144         help="Make use of silent mode of (n)make.")
2145     Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
2146     Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
2147                                                                                "including library instances selected, final dependency expression, "\
2148                                                                                "and warning messages, etc.")
2149     Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
2150     Parser.add_option("-D", "--define", action="append", type="string", dest="Macros", help="Macro: \"Name [= Value]\".")
2151
2152     Parser.add_option("-y", "--report-file", action="store", dest="ReportFile", help="Create/overwrite the report to the specified filename.")
2153     Parser.add_option("-Y", "--report-type", action="append", type="choice", choices=['PCD','LIBRARY','FLASH','DEPEX','BUILD_FLAGS','FIXED_ADDRESS','HASH','EXECUTION_ORDER'], dest="ReportType", default=[],
2154         help="Flags that control the type of build report to generate.  Must be one of: [PCD, LIBRARY, FLASH, DEPEX, BUILD_FLAGS, FIXED_ADDRESS, HASH, EXECUTION_ORDER].  "\
2155              "To specify more than one flag, repeat this option on the command line and the default flag set is [PCD, LIBRARY, FLASH, DEPEX, BUILD_FLAGS, FIXED_ADDRESS]")
2156     Parser.add_option("-F", "--flag", action="store", type="string", dest="Flag",
2157         help="Specify the specific option to parse EDK UNI file. Must be one of: [-c, -s]. -c is for EDK framework UNI file, and -s is for EDK UEFI UNI file. "\
2158              "This option can also be specified by setting *_*_*_BUILD_FLAGS in [BuildOptions] section of platform DSC. If they are both specified, this value "\
2159              "will override the setting in [BuildOptions] section of platform DSC.")
2160     Parser.add_option("-N", "--no-cache", action="store_true", dest="DisableCache", default=False, help="Disable build cache mechanism")
2161     Parser.add_option("--conf", action="store", type="string", dest="ConfDirectory", help="Specify the customized Conf directory.")
2162     Parser.add_option("--check-usage", action="store_true", dest="CheckUsage", default=False, help="Check usage content of entries listed in INF file.")
2163     Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files")
2164     Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: \"PcdName=Value\" ")
2165     Parser.add_option("-l", "--cmd-len", action="store", type="int", dest="CommandLength", help="Specify the maximum line length of build command. Default is 4096.")
2166
2167     (Opt, Args) = Parser.parse_args()
2168     return (Opt, Args)
2169
2170 ## Tool entrance method
2171 #
2172 # This method mainly dispatch specific methods per the command line options.
2173 # If no error found, return zero value so the caller of this tool can know
2174 # if it's executed successfully or not.
2175 #
2176 #   @retval 0     Tool was successful
2177 #   @retval 1     Tool failed
2178 #
2179 def Main():
2180     StartTime = time.time()
2181
2182     # Initialize log system
2183     EdkLogger.Initialize()
2184     GlobalData.gCommand = sys.argv[1:]
2185     #
2186     # Parse the options and args
2187     #
2188     (Option, Target) = MyOptionParser()
2189     GlobalData.gOptions = Option
2190     GlobalData.gCaseInsensitive = Option.CaseInsensitive
2191
2192     # Set log level
2193     if Option.verbose != None:
2194         EdkLogger.SetLevel(EdkLogger.VERBOSE)
2195     elif Option.quiet != None:
2196         EdkLogger.SetLevel(EdkLogger.QUIET)
2197     elif Option.debug != None:
2198         EdkLogger.SetLevel(Option.debug + 1)
2199     else:
2200         EdkLogger.SetLevel(EdkLogger.INFO)
2201
2202     if Option.LogFile != None:
2203         EdkLogger.SetLogFile(Option.LogFile)
2204
2205     if Option.WarningAsError == True:
2206         EdkLogger.SetWarningAsError()
2207
2208     if platform.platform().find("Windows") >= 0:
2209         GlobalData.gIsWindows = True
2210     else:
2211         GlobalData.gIsWindows = False
2212
2213     EdkLogger.quiet("Build environment: %s" % platform.platform())
2214     EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime()));
2215     ReturnCode = 0
2216     MyBuild = None
2217     BuildError = True
2218     try:
2219         if len(Target) == 0:
2220             Target = "all"
2221         elif len(Target) >= 2:
2222             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
2223                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2224         else:
2225             Target = Target[0].lower()
2226
2227         if Target not in gSupportedTarget:
2228             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
2229                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2230
2231         #
2232         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2233         #
2234         CheckEnvVariable()
2235         GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros))
2236
2237         Workspace = os.getenv("WORKSPACE")
2238         #
2239         # Get files real name in workspace dir
2240         #
2241         GlobalData.gAllFiles = Utils.DirCache(Workspace)
2242
2243         WorkingDirectory = os.getcwd()
2244         if not Option.ModuleFile:
2245             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
2246             FileNum = len(FileList)
2247             if FileNum >= 2:
2248                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
2249                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2250             elif FileNum == 1:
2251                 Option.ModuleFile = NormFile(FileList[0], Workspace)
2252
2253         if Option.ModuleFile:
2254             if os.path.isabs (Option.ModuleFile):
2255                 if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0:
2256                     Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace)
2257             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
2258             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
2259             if ErrorCode != 0:
2260                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2261
2262         if Option.PlatformFile != None:
2263             if os.path.isabs (Option.PlatformFile):
2264                 if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0:
2265                     Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace)
2266             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
2267
2268         if Option.FdfFile != None:
2269             if os.path.isabs (Option.FdfFile):
2270                 if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0:
2271                     Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace)
2272             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
2273             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
2274             if ErrorCode != 0:
2275                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2276
2277         if Option.Flag != None and Option.Flag not in ['-c', '-s']:
2278             EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s")
2279
2280         MyBuild = Build(Target, Workspace, Option)
2281         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
2282         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
2283             MyBuild.Launch()
2284         # Drop temp tables to avoid database locked.
2285         for TmpTableName in TmpTableDict:
2286             SqlCommand = """drop table IF EXISTS %s""" % TmpTableName
2287             TmpTableDict[TmpTableName].execute(SqlCommand)
2288         #MyBuild.DumpBuildData()
2289         #
2290         # All job done, no error found and no exception raised
2291         #
2292         BuildError = False
2293     except FatalError, X:
2294         if MyBuild != None:
2295             # for multi-thread build exits safely
2296             MyBuild.Relinquish()
2297         if Option != None and Option.debug != None:
2298             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2299         ReturnCode = X.args[0]
2300     except Warning, X:
2301         # error from Fdf parser
2302         if MyBuild != None:
2303             # for multi-thread build exits safely
2304             MyBuild.Relinquish()
2305         if Option != None and Option.debug != None:
2306             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2307         else:
2308             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
2309         ReturnCode = FORMAT_INVALID
2310     except KeyboardInterrupt:
2311         ReturnCode = ABORT_ERROR
2312         if Option != None and Option.debug != None:
2313             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2314     except:
2315         if MyBuild != None:
2316             # for multi-thread build exits safely
2317             MyBuild.Relinquish()
2318
2319         # try to get the meta-file from the object causing exception
2320         Tb = sys.exc_info()[-1]
2321         MetaFile = GlobalData.gProcessingFile
2322         while Tb != None:
2323             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
2324                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
2325             Tb = Tb.tb_next
2326         EdkLogger.error(
2327                     "\nbuild",
2328                     CODE_ERROR,
2329                     "Unknown fatal error when processing [%s]" % MetaFile,
2330                     ExtraData="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n",
2331                     RaiseError=False
2332                     )
2333         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2334         ReturnCode = CODE_ERROR
2335     finally:
2336         Utils.Progressor.Abort()
2337         Utils.ClearDuplicatedInf()
2338
2339     if ReturnCode == 0:
2340         try:
2341             MyBuild.LaunchPostbuild()
2342             Conclusion = "Done"
2343         except:
2344             Conclusion = "Failed"
2345     elif ReturnCode == ABORT_ERROR:
2346         Conclusion = "Aborted"
2347     else:
2348         Conclusion = "Failed"
2349     FinishTime = time.time()
2350     BuildDuration = time.gmtime(int(round(FinishTime - StartTime)))
2351     BuildDurationStr = ""
2352     if BuildDuration.tm_yday > 1:
2353         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1)
2354     else:
2355         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
2356     if MyBuild != None:
2357         if not BuildError:
2358             MyBuild.BuildReport.GenerateReport(BuildDurationStr)
2359         MyBuild.Db.Close()
2360     EdkLogger.SetLevel(EdkLogger.QUIET)
2361     EdkLogger.quiet("\n- %s -" % Conclusion)
2362     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
2363     EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
2364     return ReturnCode
2365
2366 if __name__ == '__main__':
2367     r = Main()
2368     ## 0-127 is a safe return range, and 1 is a standard default error
2369     if r < 0 or r > 127: r = 1
2370     sys.exit(r)
2371