BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2007, Intel Corporation
   5 #
   6 #  All rights reserved. This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 import os
  19 import re
  20 import sys
  21 import glob
  22 import time
  23 import platform
  24 import traceback
  25
  26 from threading import *
  27 from optparse import OptionParser
  28 from subprocess import *
  29 from Common import Misc as Utils
  30
  31 from Common.TargetTxtClassObject import *
  32 from Common.ToolDefClassObject import *
  33 from Common.DataType import *
  34 from AutoGen.AutoGen import *
  35 from Common.BuildToolError import *
  36 from Workspace.WorkspaceDatabase import *
  37
  38 import Common.EdkLogger
  39 import Common.GlobalData as GlobalData
  40
  41 # Version and Copyright
  42 VersionNumber = "0.5"
  43 __version__ = "%prog Version " + VersionNumber
  44 __copyright__ = "Copyright (c) 2007, Intel Corporation  All rights reserved."
  45
  46 ## standard targets of build command
  47 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  48
  49 ## build configuration file
  50 gBuildConfiguration = "Conf/target.txt"
  51 gBuildCacheDir = "Conf/.cache"
  52 gToolsDefinition = "Conf/tools_def.txt"
  53
  54 ## Check environment PATH variable to make sure the specified tool is found
  55 #
  56 #   If the tool is found in the PATH, then True is returned
  57 #   Otherwise, False is returned
  58 #
  59 def IsToolInPath(tool):
  60     if os.environ.has_key('PATHEXT'):
  61         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  62     else:
  63         extns = ('',)
  64     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  65         for ext in extns:
  66             if os.path.exists(os.path.join(pathDir, tool + ext)):
  67                 return True
  68     return False
  69
  70 ## Check environment variables
  71 #
  72 #  Check environment variables that must be set for build. Currently they are
  73 #
  74 #   WORKSPACE           The directory all packages/platforms start from
  75 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  76 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  77 #
  78 #   If any of above environment variable is not set or has error, the build
  79 #   will be broken.
  80 #
  81 def CheckEnvVariable():
  82     # check WORKSPACE
  83     if "WORKSPACE" not in os.environ:
  84         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
  85                         ExtraData="WORKSPACE")
  86
  87     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  88     if not os.path.exists(WorkspaceDir):
  89         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData="%s" % WorkspaceDir)
  90     elif ' ' in WorkspaceDir:
  91         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
  92                         ExtraData=WorkspaceDir)
  93     os.environ["WORKSPACE"] = WorkspaceDir
  94
  95     #
  96     # Check EFI_SOURCE (R8 build convention). EDK_SOURCE will always point to ECP
  97     #
  98     os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  99     if "EFI_SOURCE" not in os.environ:
 100         os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
 101     if "EDK_SOURCE" not in os.environ:
 102         os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
 103
 104     #
 105     # Unify case of characters on case-insensitive systems
 106     #
 107     EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
 108     EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
 109     EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
 110
 111     os.environ["EFI_SOURCE"] = EfiSourceDir
 112     os.environ["EDK_SOURCE"] = EdkSourceDir
 113     os.environ["ECP_SOURCE"] = EcpSourceDir
 114     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 115
 116     if not os.path.exists(EcpSourceDir):
 117         EdkLogger.verbose("ECP_SOURCE = %s doesn't exist. R8 modules could not be built." % EcpSourceDir)
 118     elif ' ' in EcpSourceDir:
 119         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in ECP_SOURCE path",
 120                         ExtraData=EcpSourceDir)
 121     if not os.path.exists(EdkSourceDir):
 122         if EdkSourceDir == EcpSourceDir:
 123             EdkLogger.verbose("EDK_SOURCE = %s doesn't exist. R8 modules could not be built." % EdkSourceDir)
 124         else:
 125             EdkLogger.error("build", PARAMETER_INVALID, "EDK_SOURCE does not exist",
 126                             ExtraData=EdkSourceDir)
 127     elif ' ' in EdkSourceDir:
 128         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in EDK_SOURCE path",
 129                         ExtraData=EdkSourceDir)
 130     if not os.path.exists(EfiSourceDir):
 131         if EfiSourceDir == EcpSourceDir:
 132             EdkLogger.verbose("EFI_SOURCE = %s doesn't exist. R8 modules could not be built." % EfiSourceDir)
 133         else:
 134             EdkLogger.error("build", PARAMETER_INVALID, "EFI_SOURCE does not exist",
 135                             ExtraData=EfiSourceDir)
 136     elif ' ' in EfiSourceDir:
 137         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in EFI_SOURCE path",
 138                         ExtraData=EfiSourceDir)
 139
 140     # change absolute path to relative path to WORKSPACE
 141     if EfiSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 142         EdkLogger.error("build", PARAMETER_INVALID, "EFI_SOURCE is not under WORKSPACE",
 143                         ExtraData="WORKSPACE = %s\n    EFI_SOURCE = %s" % (WorkspaceDir, EfiSourceDir))
 144     if EdkSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 145         EdkLogger.error("build", PARAMETER_INVALID, "EDK_SOURCE is not under WORKSPACE",
 146                         ExtraData="WORKSPACE = %s\n    EDK_SOURCE = %s" % (WorkspaceDir, EdkSourceDir))
 147     if EcpSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 148         EdkLogger.error("build", PARAMETER_INVALID, "ECP_SOURCE is not under WORKSPACE",
 149                         ExtraData="WORKSPACE = %s\n    ECP_SOURCE = %s" % (WorkspaceDir, EcpSourceDir))
 150
 151     # check EDK_TOOLS_PATH
 152     if "EDK_TOOLS_PATH" not in os.environ:
 153         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 154                         ExtraData="EDK_TOOLS_PATH")
 155
 156     # check PATH
 157     if "PATH" not in os.environ:
 158         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 159                         ExtraData="PATH")
 160
 161     GlobalData.gWorkspace = WorkspaceDir
 162     GlobalData.gEfiSource = EfiSourceDir
 163     GlobalData.gEdkSource = EdkSourceDir
 164     GlobalData.gEcpSource = EcpSourceDir
 165
 166 ## Get normalized file path
 167 #
 168 # Convert the path to be local format, and remove the WORKSPACE path at the
 169 # beginning if the file path is given in full path.
 170 #
 171 # @param  FilePath      File path to be normalized
 172 # @param  Workspace     Workspace path which the FilePath will be checked against
 173 #
 174 # @retval string        The normalized file path
 175 #
 176 def NormFile(FilePath, Workspace):
 177     # check if the path is absolute or relative
 178     if os.path.isabs(FilePath):
 179         FileFullPath = os.path.normpath(FilePath)
 180     else:
 181         FileFullPath = os.path.normpath(os.path.join(Workspace, FilePath))
 182
 183     # check if the file path exists or not
 184     if not os.path.isfile(FileFullPath):
 185         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)"  % FileFullPath)
 186
 187     # remove workspace directory from the beginning part of the file path
 188     if Workspace[-1] in ["\\", "/"]:
 189         return FileFullPath[len(Workspace):]
 190     else:
 191         return FileFullPath[(len(Workspace) + 1):]
 192
 193 ## Get the output of an external program
 194 #
 195 # This is the entrance method of thread reading output of an external program and
 196 # putting them in STDOUT/STDERR of current program.
 197 #
 198 # @param  From      The stream message read from
 199 # @param  To        The stream message put on
 200 # @param  ExitFlag  The flag used to indicate stopping reading
 201 #
 202 def ReadMessage(From, To, ExitFlag):
 203     while True:
 204         # read one line a time
 205         Line = From.readline()
 206         # empty string means "end"
 207         if Line != None and Line != "":
 208             To(Line.rstrip())
 209         else:
 210             break
 211         if ExitFlag.isSet():
 212             break
 213
 214 ## Launch an external program
 215 #
 216 # This method will call subprocess.Popen to execute an external program with
 217 # given options in specified directory. Because of the dead-lock issue during
 218 # redirecting output of the external program, threads are used to to do the
 219 # redirection work.
 220 #
 221 # @param  Command               A list or string containing the call of the program
 222 # @param  WorkingDir            The directory in which the program will be running
 223 #
 224 def LaunchCommand(Command, WorkingDir):
 225     # if working directory doesn't exist, Popen() will raise an exception
 226     if not os.path.isdir(WorkingDir):
 227         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 228
 229     Proc = None
 230     EndOfProcedure = None
 231     try:
 232         # launch the command
 233         Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1)
 234
 235         # launch two threads to read the STDOUT and STDERR
 236         EndOfProcedure = Event()
 237         EndOfProcedure.clear()
 238         if Proc.stdout:
 239             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))
 240             StdOutThread.setName("STDOUT-Redirector")
 241             StdOutThread.setDaemon(False)
 242             StdOutThread.start()
 243
 244         if Proc.stderr:
 245             StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))
 246             StdErrThread.setName("STDERR-Redirector")
 247             StdErrThread.setDaemon(False)
 248             StdErrThread.start()
 249
 250         # waiting for program exit
 251         Proc.wait()
 252     except: # in case of aborting
 253         # terminate the threads redirecting the program output
 254         if EndOfProcedure != None:
 255             EndOfProcedure.set()
 256         if Proc == None:
 257             if type(Command) != type(""):
 258                 Command = " ".join(Command)
 259             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 260
 261     if Proc.stdout:
 262         StdOutThread.join()
 263     if Proc.stderr:
 264         StdErrThread.join()
 265
 266     # check the return code of the program
 267     if Proc.returncode != 0:
 268         if type(Command) != type(""):
 269             Command = " ".join(Command)
 270         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 271
 272 ## The smallest unit that can be built in multi-thread build mode
 273 #
 274 # This is the base class of build unit. The "Obj" parameter must provide
 275 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 276 # missing build.
 277 #
 278 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 279 #
 280 class BuildUnit:
 281     ## The constructor
 282     #
 283     #   @param  self        The object pointer
 284     #   @param  Obj         The object the build is working on
 285     #   @param  Target      The build target name, one of gSupportedTarget
 286     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 287     #   @param  WorkingDir  The directory build command starts in
 288     #
 289     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 290         self.BuildObject = Obj
 291         self.Dependency = Dependency
 292         self.WorkingDir = WorkingDir
 293         self.Target = Target
 294         self.BuildCommand = BuildCommand
 295         if BuildCommand == None or len(BuildCommand) == 0:
 296             EdkLogger.error("build", OPTION_MISSING, "No build command found for",
 297                             ExtraData=str(Obj))
 298
 299     ## str() method
 300     #
 301     #   It just returns the string representaion of self.BuildObject
 302     #
 303     #   @param  self        The object pointer
 304     #
 305     def __str__(self):
 306         return str(self.BuildObject)
 307
 308     ## "==" operator method
 309     #
 310     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 311     #   provide its own __eq__() method.
 312     #
 313     #   @param  self        The object pointer
 314     #   @param  Other       The other BuildUnit object compared to
 315     #
 316     def __eq__(self, Other):
 317         return Other != None and self.BuildObject == Other.BuildObject \
 318                 and self.BuildObject.Arch == Other.BuildObject.Arch
 319
 320     ## hash() method
 321     #
 322     #   It just returns the hash value of self.BuildObject which must be hashable.
 323     #
 324     #   @param  self        The object pointer
 325     #
 326     def __hash__(self):
 327         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 328
 329     def __repr__(self):
 330         return repr(self.BuildObject)
 331
 332 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 333 #
 334 # This class is for module build by nmake/make build system. The "Obj" parameter
 335 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 336 # be make units missing build.
 337 #
 338 # Currently the "Obj" should be only ModuleAutoGen object.
 339 #
 340 class ModuleMakeUnit(BuildUnit):
 341     ## The constructor
 342     #
 343     #   @param  self        The object pointer
 344     #   @param  Obj         The ModuleAutoGen object the build is working on
 345     #   @param  Target      The build target name, one of gSupportedTarget
 346     #
 347     def __init__(self, Obj, Target):
 348         Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]
 349         BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
 350         if Target in [None, "", "all"]:
 351             self.Target = "tbuild"
 352
 353 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 354 #
 355 # This class is for platform build by nmake/make build system. The "Obj" parameter
 356 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 357 # be make units missing build.
 358 #
 359 # Currently the "Obj" should be only PlatformAutoGen object.
 360 #
 361 class PlatformMakeUnit(BuildUnit):
 362     ## The constructor
 363     #
 364     #   @param  self        The object pointer
 365     #   @param  Obj         The PlatformAutoGen object the build is working on
 366     #   @param  Target      The build target name, one of gSupportedTarget
 367     #
 368     def __init__(self, Obj, Target):
 369         Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 370         Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])
 371         BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
 372
 373 ## The class representing the task of a module build or platform build
 374 #
 375 # This class manages the build tasks in multi-thread build mode. Its jobs include
 376 # scheduling thread running, catching thread error, monitor the thread status, etc.
 377 #
 378 class BuildTask:
 379     # queue for tasks waiting for schedule
 380     _PendingQueue = sdict()
 381     _PendingQueueLock = threading.Lock()
 382
 383     # queue for tasks ready for running
 384     _ReadyQueue = sdict()
 385     _ReadyQueueLock = threading.Lock()
 386
 387     # queue for run tasks
 388     _RunningQueue = sdict()
 389     _RunningQueueLock = threading.Lock()
 390
 391     # queue containing all build tasks, in case duplicate build
 392     _TaskQueue = sdict()
 393
 394     # flag indicating error occurs in a running thread
 395     _ErrorFlag = threading.Event()
 396     _ErrorFlag.clear()
 397     _ErrorMessage = ""
 398
 399     # BoundedSemaphore object used to control the number of running threads
 400     _Thread = None
 401
 402     # flag indicating if the scheduler is started or not
 403     _SchedulerStopped = threading.Event()
 404     _SchedulerStopped.set()
 405
 406     ## Start the task scheduler thread
 407     #
 408     #   @param  MaxThreadNumber     The maximum thread number
 409     #   @param  ExitFlag            Flag used to end the scheduler
 410     #
 411     @staticmethod
 412     def StartScheduler(MaxThreadNumber, ExitFlag):
 413         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 414         SchedulerThread.setName("Build-Task-Scheduler")
 415         SchedulerThread.setDaemon(False)
 416         SchedulerThread.start()
 417         # wait for the scheduler to be started, especially useful in Linux
 418         while not BuildTask.IsOnGoing():
 419             time.sleep(0.01)
 420
 421     ## Scheduler method
 422     #
 423     #   @param  MaxThreadNumber     The maximum thread number
 424     #   @param  ExitFlag            Flag used to end the scheduler
 425     #
 426     @staticmethod
 427     def Scheduler(MaxThreadNumber, ExitFlag):
 428         BuildTask._SchedulerStopped.clear()
 429         try:
 430             # use BoundedSemaphore to control the maximum running threads
 431             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 432             #
 433             # scheduling loop, which will exits when no pending/ready task and
 434             # indicated to do so, or there's error in running thread
 435             #
 436             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 437                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 438                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 439                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 440
 441                 # get all pending tasks
 442                 BuildTask._PendingQueueLock.acquire()
 443                 BuildObjectList = BuildTask._PendingQueue.keys()
 444                 #
 445                 # check if their dependency is resolved, and if true, move them
 446                 # into ready queue
 447                 #
 448                 for BuildObject in BuildObjectList:
 449                     Bt = BuildTask._PendingQueue[BuildObject]
 450                     if Bt.IsReady():
 451                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 452                 BuildTask._PendingQueueLock.release()
 453
 454                 # launch build thread until the maximum number of threads is reached
 455                 while not BuildTask._ErrorFlag.isSet():
 456                     # empty ready queue, do nothing further
 457                     if len(BuildTask._ReadyQueue) == 0:
 458                         break
 459
 460                     # wait for active thread(s) exit
 461                     BuildTask._Thread.acquire(True)
 462
 463                     # start a new build thread
 464                     Bo = BuildTask._ReadyQueue.keys()[0]
 465                     Bt = BuildTask._ReadyQueue.pop(Bo)
 466
 467                     # move into running queue
 468                     BuildTask._RunningQueueLock.acquire()
 469                     BuildTask._RunningQueue[Bo] = Bt
 470                     BuildTask._RunningQueueLock.release()
 471
 472                     Bt.Start()
 473                     # avoid tense loop
 474                     time.sleep(0.01)
 475
 476                 # avoid tense loop
 477                 time.sleep(0.01)
 478
 479             # wait for all running threads exit
 480             if BuildTask._ErrorFlag.isSet():
 481                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 482             # while not BuildTask._ErrorFlag.isSet() and \
 483             while len(BuildTask._RunningQueue) > 0:
 484                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 485                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join([Th.getName() for Th in threading.enumerate()]))
 486                 # avoid tense loop
 487                 time.sleep(0.1)
 488         except BaseException, X:
 489             #
 490             # TRICK: hide the output of threads left runing, so that the user can
 491             #        catch the error message easily
 492             #
 493             EdkLogger.SetLevel(EdkLogger.ERROR)
 494             BuildTask._ErrorFlag.set()
 495             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 496
 497         BuildTask._PendingQueue.clear()
 498         BuildTask._ReadyQueue.clear()
 499         BuildTask._RunningQueue.clear()
 500         BuildTask._TaskQueue.clear()
 501         BuildTask._SchedulerStopped.set()
 502
 503     ## Wait for all running method exit
 504     #
 505     @staticmethod
 506     def WaitForComplete():
 507         BuildTask._SchedulerStopped.wait()
 508
 509     ## Check if the scheduler is running or not
 510     #
 511     @staticmethod
 512     def IsOnGoing():
 513         return not BuildTask._SchedulerStopped.isSet()
 514
 515     ## Abort the build
 516     @staticmethod
 517     def Abort():
 518         if BuildTask.IsOnGoing():
 519             BuildTask._ErrorFlag.set()
 520             BuildTask.WaitForComplete()
 521
 522     ## Check if there's error in running thread
 523     #
 524     #   Since the main thread cannot catch exceptions in other thread, we have to
 525     #   use threading.Event to communicate this formation to main thread.
 526     #
 527     @staticmethod
 528     def HasError():
 529         return BuildTask._ErrorFlag.isSet()
 530
 531     ## Get error message in running thread
 532     #
 533     #   Since the main thread cannot catch exceptions in other thread, we have to
 534     #   use a static variable to communicate this message to main thread.
 535     #
 536     @staticmethod
 537     def GetErrorMessage():
 538         return BuildTask._ErrorMessage
 539
 540     ## Factory method to create a BuildTask object
 541     #
 542     #   This method will check if a module is building or has been built. And if
 543     #   true, just return the associated BuildTask object in the _TaskQueue. If
 544     #   not, create and return a new BuildTask object. The new BuildTask object
 545     #   will be appended to the _PendingQueue for scheduling later.
 546     #
 547     #   @param  BuildItem       A BuildUnit object representing a build object
 548     #   @param  Dependency      The dependent build object of BuildItem
 549     #
 550     @staticmethod
 551     def New(BuildItem, Dependency=None):
 552         if BuildItem in BuildTask._TaskQueue:
 553             Bt = BuildTask._TaskQueue[BuildItem]
 554             return Bt
 555
 556         Bt = BuildTask()
 557         Bt._Init(BuildItem, Dependency)
 558         BuildTask._TaskQueue[BuildItem] = Bt
 559
 560         BuildTask._PendingQueueLock.acquire()
 561         BuildTask._PendingQueue[BuildItem] = Bt
 562         BuildTask._PendingQueueLock.release()
 563
 564         return Bt
 565
 566     ## The real constructor of BuildTask
 567     #
 568     #   @param  BuildItem       A BuildUnit object representing a build object
 569     #   @param  Dependency      The dependent build object of BuildItem
 570     #
 571     def _Init(self, BuildItem, Dependency=None):
 572         self.BuildItem = BuildItem
 573
 574         self.DependencyList = []
 575         if Dependency == None:
 576             Dependency = BuildItem.Dependency
 577         else:
 578             Dependency.extend(BuildItem.Dependency)
 579         self.AddDependency(Dependency)
 580         # flag indicating build completes, used to avoid unnecessary re-build
 581         self.CompleteFlag = False
 582
 583     ## Check if all dependent build tasks are completed or not
 584     #
 585     def IsReady(self):
 586         ReadyFlag = True
 587         for Dep in self.DependencyList:
 588             if Dep.CompleteFlag == True:
 589                 continue
 590             ReadyFlag = False
 591             break
 592
 593         return ReadyFlag
 594
 595     ## Add dependent build task
 596     #
 597     #   @param  Dependency      The list of dependent build objects
 598     #
 599     def AddDependency(self, Dependency):
 600         for Dep in Dependency:
 601             self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 602
 603     ## The thread wrapper of LaunchCommand function
 604     #
 605     # @param  Command               A list or string contains the call of the command
 606     # @param  WorkingDir            The directory in which the program will be running
 607     #
 608     def _CommandThread(self, Command, WorkingDir):
 609         try:
 610             LaunchCommand(Command, WorkingDir)
 611             self.CompleteFlag = True
 612         except:
 613             #
 614             # TRICK: hide the output of threads left runing, so that the user can
 615             #        catch the error message easily
 616             #
 617             if not BuildTask._ErrorFlag.isSet():
 618                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 619                                                                   self.BuildItem.BuildObject.Arch,
 620                                                                   self.BuildItem.BuildObject.ToolChain,
 621                                                                   self.BuildItem.BuildObject.BuildTarget
 622                                                                  )
 623             EdkLogger.SetLevel(EdkLogger.ERROR)
 624             BuildTask._ErrorFlag.set()
 625             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 626                                       (threading.currentThread().getName(), Command, WorkingDir)
 627         # indicate there's a thread is available for another build task
 628         BuildTask._RunningQueueLock.acquire()
 629         BuildTask._RunningQueue.pop(self.BuildItem)
 630         BuildTask._RunningQueueLock.release()
 631         BuildTask._Thread.release()
 632
 633     ## Start build task thread
 634     #
 635     def Start(self):
 636         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 637         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 638         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 639         self.BuildTread.setName("build thread")
 640         self.BuildTread.setDaemon(False)
 641         self.BuildTread.start()
 642
 643 ## The class implementing the EDK2 build process
 644 #
 645 #   The build process includes:
 646 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 647 #       2. Parse DSC file of active platform
 648 #       3. Parse FDF file if any
 649 #       4. Establish build database, including parse all other files (module, package)
 650 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 651 #       6. Call build command
 652 #
 653 class Build():
 654     ## Constructor
 655     #
 656     # Constructor will load all necessary configurations, parse platform, modules
 657     # and packages and the establish a database for AutoGen.
 658     #
 659     #   @param  Target              The build command target, one of gSupportedTarget
 660     #   @param  WorkspaceDir        The directory of workspace
 661     #   @param  Platform            The DSC file of active platform
 662     #   @param  Module              The INF file of active module, if any
 663     #   @param  Arch                The Arch list of platform or module
 664     #   @param  ToolChain           The name list of toolchain
 665     #   @param  BuildTarget         The "DEBUG" or "RELEASE" build
 666     #   @param  FlashDefinition     The FDF file of active platform
 667     #   @param  FdList=[]           The FD names to be individually built
 668     #   @param  FvList=[]           The FV names to be individually built
 669     #   @param  MakefileType        The type of makefile (for MSFT make or GNU make)
 670     #   @param  SilentMode          Indicate multi-thread build mode
 671     #   @param  ThreadNumber        The maximum number of thread if in multi-thread build mode
 672     #   @param  SkipAutoGen         Skip AutoGen step
 673     #   @param  Reparse             Re-parse all meta files
 674     #   @param  SkuId               SKU id from command line
 675     #
 676     def __init__(self, Target, WorkspaceDir, Platform, Module, Arch, ToolChain,
 677                  BuildTarget, FlashDefinition, FdList=[], FvList=[],
 678                  MakefileType="nmake", SilentMode=False, ThreadNumber=2,
 679                  SkipAutoGen=False, Reparse=False, SkuId=None,
 680                  ReportFile=None, ReportType=None):
 681
 682         self.WorkspaceDir = WorkspaceDir
 683         self.Target         = Target
 684         self.PlatformFile   = Platform
 685         self.ModuleFile     = Module
 686         self.ArchList       = Arch
 687         self.ToolChainList  = ToolChain
 688         self.BuildTargetList= BuildTarget
 689         self.Fdf            = FlashDefinition
 690         self.FdList         = FdList
 691         self.FvList         = FvList
 692         self.MakefileType   = MakefileType
 693         self.SilentMode     = SilentMode
 694         self.ThreadNumber   = ThreadNumber
 695         self.SkipAutoGen    = SkipAutoGen
 696         self.Reparse        = Reparse
 697         self.SkuId          = SkuId
 698         self.SpawnMode      = True
 699         self.ReportFile     = ReportFile
 700         if ReportType == None:
 701           self.ReportType   = ['ALL']
 702         else:
 703           self.ReportType   = ReportType
 704
 705         self.TargetTxt      = TargetTxtClassObject()
 706         self.ToolDef        = ToolDefClassObject()
 707         self.Db             = WorkspaceDatabase(None, GlobalData.gGlobalDefines, self.Reparse)
 708         #self.Db             = WorkspaceDatabase(None, {}, self.Reparse)
 709         self.BuildDatabase  = self.Db.BuildObject
 710         self.Platform       = None
 711
 712         # print dot charater during doing some time-consuming work
 713         self.Progress = Utils.Progressor()
 714
 715         # parse target.txt, tools_def.txt, and platform file
 716         #self.RestoreBuildData()
 717         self.LoadConfiguration()
 718         self.InitBuild()
 719
 720         # print current build environment and configuration
 721         EdkLogger.quiet("%-24s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 722         EdkLogger.quiet("%-24s = %s" % ("ECP_SOURCE", os.environ["ECP_SOURCE"]))
 723         EdkLogger.quiet("%-24s = %s" % ("EDK_SOURCE", os.environ["EDK_SOURCE"]))
 724         EdkLogger.quiet("%-24s = %s" % ("EFI_SOURCE", os.environ["EFI_SOURCE"]))
 725         EdkLogger.quiet("%-24s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 726
 727         EdkLogger.info('\n%-24s = %s' % ("TARGET_ARCH", ' '.join(self.ArchList)))
 728         EdkLogger.info('%-24s = %s' % ("TARGET", ' '.join(self.BuildTargetList)))
 729         EdkLogger.info('%-24s = %s' % ("TOOL_CHAIN_TAG", ' '.join(self.ToolChainList)))
 730
 731         EdkLogger.info('\n%-24s = %s' % ("Active Platform", self.PlatformFile))
 732
 733         if self.Fdf != None and self.Fdf != "":
 734             EdkLogger.info('%-24s = %s' % ("Flash Image Definition", self.Fdf))
 735
 736         if self.ModuleFile != None and self.ModuleFile != "":
 737             EdkLogger.info('%-24s = %s' % ("Active Module", self.ModuleFile))
 738
 739         os.chdir(self.WorkspaceDir)
 740         self.Progress.Start("\nProcessing meta-data")
 741
 742     ## Load configuration
 743     #
 744     #   This method will parse target.txt and get the build configurations.
 745     #
 746     def LoadConfiguration(self):
 747         #
 748         # Check target.txt and tools_def.txt and Init them
 749         #
 750         BuildConfigurationFile = os.path.normpath(os.path.join(self.WorkspaceDir, gBuildConfiguration))
 751         if os.path.isfile(BuildConfigurationFile) == True:
 752             StatusCode = self.TargetTxt.LoadTargetTxtFile(BuildConfigurationFile)
 753
 754             ToolDefinitionFile = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 755             if ToolDefinitionFile == '':
 756                 ToolDefinitionFile = gToolsDefinition
 757             ToolDefinitionFile = os.path.normpath(os.path.join(self.WorkspaceDir, ToolDefinitionFile))
 758             if os.path.isfile(ToolDefinitionFile) == True:
 759                 StatusCode = self.ToolDef.LoadToolDefFile(ToolDefinitionFile)
 760             else:
 761                 EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=ToolDefinitionFile)
 762         else:
 763             EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile)
 764
 765         # if no ARCH given in command line, get it from target.txt
 766         if self.ArchList == None or len(self.ArchList) == 0:
 767             self.ArchList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TARGET_ARCH]
 768
 769         # if no build target given in command line, get it from target.txt
 770         if self.BuildTargetList == None or len(self.BuildTargetList) == 0:
 771             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TARGET]
 772
 773         # if no tool chain given in command line, get it from target.txt
 774         if self.ToolChainList == None or len(self.ToolChainList) == 0:
 775             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 776             if self.ToolChainList == None or len(self.ToolChainList) == 0:
 777                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 778
 779         # check if the tool chains are defined or not
 780         NewToolChainList = []
 781         for ToolChain in self.ToolChainList:
 782             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 783                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 784             else:
 785                 NewToolChainList.append(ToolChain)
 786         # if no tool chain available, break the build
 787         if len(NewToolChainList) == 0:
 788             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 789                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 790         else:
 791             self.ToolChainList = NewToolChainList
 792
 793         if self.ThreadNumber == None:
 794             self.ThreadNumber = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
 795             if self.ThreadNumber == '':
 796                 self.ThreadNumber = 0
 797             else:
 798                 self.ThreadNumber = int(self.ThreadNumber, 0)
 799
 800         if self.ThreadNumber == 0:
 801             self.ThreadNumber = 1
 802
 803         if not self.PlatformFile:
 804             PlatformFile = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 805             if not PlatformFile:
 806                 # Try to find one in current directory
 807                 WorkingDirectory = os.getcwd()
 808                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 809                 FileNum = len(FileList)
 810                 if FileNum >= 2:
 811                     EdkLogger.error("build", OPTION_MISSING,
 812                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 813                 elif FileNum == 1:
 814                     PlatformFile = FileList[0]
 815                 else:
 816                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 817                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 818
 819             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 820             ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 821             if ErrorCode != 0:
 822                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 823
 824     ## Initialize build configuration
 825     #
 826     #   This method will parse DSC file and merge the configurations from
 827     #   command line and target.txt, then get the final build configurations.
 828     #
 829     def InitBuild(self):
 830         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc")
 831         if ErrorCode != 0:
 832             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 833
 834         # create metafile database
 835         self.Db.InitDatabase()
 836
 837         # we need information in platform description file to determine how to build
 838         self.Platform = self.BuildDatabase[self.PlatformFile, 'COMMON']
 839         if not self.Fdf:
 840             self.Fdf = self.Platform.FlashDefinition
 841
 842         if self.SkuId == None or self.SkuId == '':
 843             self.SkuId = self.Platform.SkuName
 844
 845         # check FD/FV build target
 846         if self.Fdf == None or self.Fdf == "":
 847             if self.FdList != []:
 848                 EdkLogger.info("No flash definition file found. FD [%s] will be ignored." % " ".join(self.FdList))
 849                 self.FdList = []
 850             if self.FvList != []:
 851                 EdkLogger.info("No flash definition file found. FV [%s] will be ignored." % " ".join(self.FvList))
 852                 self.FvList = []
 853         else:
 854             FdfParserObj = FdfParser(str(self.Fdf))
 855             FdfParserObj.ParseFile()
 856             for fvname in self.FvList:
 857                 if fvname.upper() not in FdfParserObj.Profile.FvDict.keys():
 858                     EdkLogger.error("build", OPTION_VALUE_INVALID,
 859                                     "No such an FV in FDF file: %s" % fvname)
 860
 861         #
 862         # Merge Arch
 863         #
 864         if self.ArchList == None or len(self.ArchList) == 0:
 865             ArchList = set(self.Platform.SupArchList)
 866         else:
 867             ArchList = set(self.ArchList) & set(self.Platform.SupArchList)
 868         if len(ArchList) == 0:
 869             EdkLogger.error("build", PARAMETER_INVALID,
 870                             ExtraData = "Active platform supports [%s] only, but [%s] is given."
 871                                         % (" ".join(self.Platform.SupArchList), " ".join(self.ArchList)))
 872         elif len(ArchList) != len(self.ArchList):
 873             SkippedArchList = set(self.ArchList).symmetric_difference(set(self.Platform.SupArchList))
 874             EdkLogger.verbose("\nArch [%s] is ignored because active platform supports [%s] but [%s] is specified !"
 875                            % (" ".join(SkippedArchList), " ".join(self.Platform.SupArchList), " ".join(self.ArchList)))
 876         self.ArchList = tuple(ArchList)
 877
 878         # Merge build target
 879         if self.BuildTargetList == None or len(self.BuildTargetList) == 0:
 880             BuildTargetList = self.Platform.BuildTargets
 881         else:
 882             BuildTargetList = list(set(self.BuildTargetList) & set(self.Platform.BuildTargets))
 883         if BuildTargetList == []:
 884             EdkLogger.error("build", PARAMETER_INVALID, "Active platform only supports [%s], but [%s] is given"
 885                                 % (" ".join(self.Platform.BuildTargets), " ".join(self.BuildTargetList)))
 886         self.BuildTargetList = BuildTargetList
 887
 888     ## Build a module or platform
 889     #
 890     # Create autogen code and makfile for a module or platform, and the launch
 891     # "make" command to build it
 892     #
 893     #   @param  Target                      The target of build command
 894     #   @param  Platform                    The platform file
 895     #   @param  Module                      The module file
 896     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
 897     #   @param  ToolChain                   The name of toolchain to build
 898     #   @param  Arch                        The arch of the module/platform
 899     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
 900     #                                       for dependent modules/Libraries
 901     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
 902     #                                       for dependent modules/Libraries
 903     #
 904     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True):
 905         if AutoGenObject == None:
 906             return False
 907
 908         # skip file generation for cleanxxx targets, run and fds target
 909         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
 910             # for target which must generate AutoGen code and makefile
 911             if not self.SkipAutoGen or Target == 'genc':
 912                 self.Progress.Start("Generating code")
 913                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
 914                 self.Progress.Stop("done!")
 915             if Target == "genc":
 916                 return True
 917
 918             if not self.SkipAutoGen or Target == 'genmake':
 919                 self.Progress.Start("Generating makefile")
 920                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
 921                 self.Progress.Stop("done!")
 922             if Target == "genmake":
 923                 return True
 924         else:
 925             # always recreate top/platform makefile when clean, just in case of inconsistency
 926             AutoGenObject.CreateCodeFile(False)
 927             AutoGenObject.CreateMakeFile(False)
 928
 929         if EdkLogger.GetLevel() == EdkLogger.QUIET:
 930             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
 931
 932         BuildCommand = AutoGenObject.BuildCommand
 933         if BuildCommand == None or len(BuildCommand) == 0:
 934             EdkLogger.error("build", OPTION_MISSING, ExtraData="No MAKE command found for [%s, %s, %s]" % Key)
 935
 936         BuildCommand = BuildCommand + [Target]
 937         LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
 938         if Target == 'cleanall':
 939             try:
 940                 #os.rmdir(AutoGenObject.BuildDir)
 941                 RemoveDirectory(AutoGenObject.BuildDir, True)
 942             except WindowsError, X:
 943                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
 944         return True
 945
 946     ## Build active platform for different build targets and different tool chains
 947     #
 948     def _BuildPlatform(self):
 949         for BuildTarget in self.BuildTargetList:
 950             for ToolChain in self.ToolChainList:
 951                 Wa = WorkspaceAutoGen(
 952                         self.WorkspaceDir,
 953                         self.Platform,
 954                         BuildTarget,
 955                         ToolChain,
 956                         self.ArchList,
 957                         self.BuildDatabase,
 958                         self.TargetTxt,
 959                         self.ToolDef,
 960                         self.Fdf,
 961                         self.FdList,
 962                         self.FvList,
 963                         self.SkuId,
 964                         self.ReportFile,
 965                         self.ReportType
 966                         )
 967                 self.Progress.Stop("done!")
 968                 self._Build(self.Target, Wa)
 969
 970     ## Build active module for different build targets, different tool chains and different archs
 971     #
 972     def _BuildModule(self):
 973         for BuildTarget in self.BuildTargetList:
 974             for ToolChain in self.ToolChainList:
 975                 #
 976                 # module build needs platform build information, so get platform
 977                 # AutoGen first
 978                 #
 979                 Wa = WorkspaceAutoGen(
 980                         self.WorkspaceDir,
 981                         self.Platform,
 982                         BuildTarget,
 983                         ToolChain,
 984                         self.ArchList,
 985                         self.BuildDatabase,
 986                         self.TargetTxt,
 987                         self.ToolDef,
 988                         self.Fdf,
 989                         self.FdList,
 990                         self.FvList,
 991                         self.SkuId,
 992                         self.ReportFile,
 993                         self.ReportType
 994                         )
 995                 Wa.CreateMakeFile(False)
 996                 self.Progress.Stop("done!")
 997                 MaList = []
 998                 for Arch in self.ArchList:
 999                     Ma = ModuleAutoGen(Wa, self.ModuleFile, BuildTarget, ToolChain, Arch, self.PlatformFile)
1000                     if Ma == None: continue
1001                     MaList.append(Ma)
1002                     self._Build(self.Target, Ma)
1003                 if MaList == []:
1004                     EdkLogger.error(
1005                                 'build',
1006                                 BUILD_ERROR,
1007                                 "Module for [%s] is not a component of active platform."\
1008                                 " Please make sure that the ARCH and inf file path are"\
1009                                 " given in the same as in [%s]" %\
1010                                     (', '.join(self.ArchList), self.Platform),
1011                                 ExtraData=self.ModuleFile
1012                                 )
1013
1014     ## Build a platform in multi-thread mode
1015     #
1016     def _MultiThreadBuildPlatform(self):
1017         for BuildTarget in self.BuildTargetList:
1018             for ToolChain in self.ToolChainList:
1019                 Wa = WorkspaceAutoGen(
1020                         self.WorkspaceDir,
1021                         self.Platform,
1022                         BuildTarget,
1023                         ToolChain,
1024                         self.ArchList,
1025                         self.BuildDatabase,
1026                         self.TargetTxt,
1027                         self.ToolDef,
1028                         self.Fdf,
1029                         self.FdList,
1030                         self.FvList,
1031                         self.SkuId,
1032                         self.ReportFile,
1033                         self.ReportType
1034                         )
1035                 Wa.CreateMakeFile(False)
1036
1037                 # multi-thread exit flag
1038                 ExitFlag = threading.Event()
1039                 ExitFlag.clear()
1040                 for Arch in self.ArchList:
1041                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1042                     if Pa == None:
1043                         continue
1044                     for Module in Pa.Platform.Modules:
1045                         # Get ModuleAutoGen object to generate C code file and makefile
1046                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
1047                         if Ma == None:
1048                             continue
1049                         # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1050                         if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1051                             # for target which must generate AutoGen code and makefile
1052                             if not self.SkipAutoGen or self.Target == 'genc':
1053                                 Ma.CreateCodeFile(True)
1054                             if self.Target == "genc":
1055                                 continue
1056
1057                             if not self.SkipAutoGen or self.Target == 'genmake':
1058                                 Ma.CreateMakeFile(True)
1059                             if self.Target == "genmake":
1060                                 continue
1061                         self.Progress.Stop("done!")
1062                         # Generate build task for the module
1063                         Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))
1064                         # Break build if any build thread has error
1065                         if BuildTask.HasError():
1066                             # we need a full version of makefile for platform
1067                             ExitFlag.set()
1068                             BuildTask.WaitForComplete()
1069                             Pa.CreateMakeFile(False)
1070                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1071                         # Start task scheduler
1072                         if not BuildTask.IsOnGoing():
1073                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1074
1075                     # in case there's an interruption. we need a full version of makefile for platform
1076                     Pa.CreateMakeFile(False)
1077                     if BuildTask.HasError():
1078                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1079
1080                 #
1081                 # All modules have been put in build tasks queue. Tell task scheduler
1082                 # to exit if all tasks are completed
1083                 #
1084                 ExitFlag.set()
1085                 BuildTask.WaitForComplete()
1086
1087                 #
1088                 # Check for build error, and raise exception if one
1089                 # has been signaled.
1090                 #
1091                 if BuildTask.HasError():
1092                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1093
1094                 # Generate FD image if there's a FDF file found
1095                 if self.Fdf != '' and self.Target in ["", "all", "fds"]:
1096                     LaunchCommand(Wa.BuildCommand + ["fds"], Wa.MakeFileDir)
1097
1098     ## Generate GuidedSectionTools.txt in the FV directories.
1099     #
1100     def CreateGuidedSectionToolsFile(self):
1101         for Arch in self.ArchList:
1102             for BuildTarget in self.BuildTargetList:
1103                 for ToolChain in self.ToolChainList:
1104                     FvDir = os.path.join(
1105                                 self.WorkspaceDir,
1106                                 self.Platform.OutputDirectory,
1107                                 '_'.join((BuildTarget, ToolChain)),
1108                                 'FV'
1109                                 )
1110                     if not os.path.exists(FvDir):
1111                         continue
1112                     # Build up the list of supported architectures for this build
1113                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
1114
1115                     # Look through the tool definitions for GUIDed tools
1116                     guidAttribs = []
1117                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.iteritems():
1118                         if attrib.upper().endswith('_GUID'):
1119                             split = attrib.split('_')
1120                             thisPrefix = '_'.join(split[0:3]) + '_'
1121                             if thisPrefix == prefix:
1122                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
1123                                 guid = guid.lower()
1124                                 toolName = split[3]
1125                                 path = '_'.join(split[0:4]) + '_PATH'
1126                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
1127                                 path = self.GetFullPathOfTool(path)
1128                                 guidAttribs.append((guid, toolName, path))
1129
1130                     # Write out GuidedSecTools.txt
1131                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
1132                     toolsFile = open(toolsFile, 'wt')
1133                     for guidedSectionTool in guidAttribs:
1134                         print >> toolsFile, ' '.join(guidedSectionTool)
1135                     toolsFile.close()
1136
1137     ## Returns the full path of the tool.
1138     #
1139     def GetFullPathOfTool (self, tool):
1140         if os.path.exists(tool):
1141             return os.path.realpath(tool)
1142         else:
1143             # We need to search for the tool using the
1144             # PATH environment variable.
1145             for dirInPath in os.environ['PATH'].split(os.pathsep):
1146                 foundPath = os.path.join(dirInPath, tool)
1147                 if os.path.exists(foundPath):
1148                     return os.path.realpath(foundPath)
1149
1150         # If the tool was not found in the path then we just return
1151         # the input tool.
1152         return tool
1153
1154     ## Launch the module or platform build
1155     #
1156     def Launch(self):
1157         if self.ModuleFile == None or self.ModuleFile == "":
1158             if not self.SpawnMode or self.Target not in ["", "all"]:
1159                 self.SpawnMode = False
1160                 self._BuildPlatform()
1161             else:
1162                 self._MultiThreadBuildPlatform()
1163             self.CreateGuidedSectionToolsFile()
1164         else:
1165             self.SpawnMode = False
1166             self._BuildModule()
1167
1168     ## Do some clean-up works when error occurred
1169     def Relinquish(self):
1170         OldLogLevel = EdkLogger.GetLevel()
1171         EdkLogger.SetLevel(EdkLogger.ERROR)
1172         #self.DumpBuildData()
1173         Utils.Progressor.Abort()
1174         if self.SpawnMode == True:
1175             BuildTask.Abort()
1176         EdkLogger.SetLevel(OldLogLevel)
1177
1178     def DumpBuildData(self):
1179         CacheDirectory = os.path.join(self.WorkspaceDir, gBuildCacheDir)
1180         Utils.CreateDirectory(CacheDirectory)
1181         Utils.DataDump(Utils.gFileTimeStampCache, os.path.join(CacheDirectory, "gFileTimeStampCache"))
1182         Utils.DataDump(Utils.gDependencyDatabase, os.path.join(CacheDirectory, "gDependencyDatabase"))
1183
1184     def RestoreBuildData(self):
1185         FilePath = os.path.join(self.WorkspaceDir, gBuildCacheDir, "gFileTimeStampCache")
1186         if Utils.gFileTimeStampCache == {} and os.path.isfile(FilePath):
1187             Utils.gFileTimeStampCache = Utils.DataRestore(FilePath)
1188             if Utils.gFileTimeStampCache == None:
1189                 Utils.gFileTimeStampCache = {}
1190
1191         FilePath = os.path.join(self.WorkspaceDir, gBuildCacheDir, "gDependencyDatabase")
1192         if Utils.gDependencyDatabase == {} and os.path.isfile(FilePath):
1193             Utils.gDependencyDatabase = Utils.DataRestore(FilePath)
1194             if Utils.gDependencyDatabase == None:
1195                 Utils.gDependencyDatabase = {}
1196
1197 def ParseDefines(DefineList=[]):
1198     DefineDict = {}
1199     if DefineList != None:
1200         for Define in DefineList:
1201             DefineTokenList = Define.split("=", 1)
1202             if len(DefineTokenList) == 1:
1203                 DefineDict[DefineTokenList[0]] = ""
1204             else:
1205                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
1206     return DefineDict
1207
1208 gParamCheck = []
1209 def SingleCheckCallback(option, opt_str, value, parser):
1210     if option not in gParamCheck:
1211         setattr(parser.values, option.dest, value)
1212         gParamCheck.append(option)
1213     else:
1214         parser.error("Option %s only allows one instance in command line!" % option)
1215
1216 ## Parse command line options
1217 #
1218 # Using standard Python module optparse to parse command line option of this tool.
1219 #
1220 #   @retval Opt   A optparse.Values object containing the parsed options
1221 #   @retval Args  Target of build command
1222 #
1223 def MyOptionParser():
1224     Parser = OptionParser(description=__copyright__,version=__version__,prog="build.exe",usage="%prog [options] [all|fds|genc|genmake|clean|cleanall|cleanlib|modules|libraries|run]")
1225     Parser.add_option("-a", "--arch", action="append", type="choice", choices=['IA32','X64','IPF','EBC','ARM'], dest="TargetArch",
1226         help="ARCHS is one of list: IA32, X64, IPF, ARM or EBC, which overrides target.txt's TARGET_ARCH definition. To specify more archs, please repeat this option.")
1227     Parser.add_option("-p", "--platform", action="callback", type="string", dest="PlatformFile", callback=SingleCheckCallback,
1228         help="Build the platform specified by the DSC file name argument, overriding target.txt's ACTIVE_PLATFORM definition.")
1229     Parser.add_option("-m", "--module", action="callback", type="string", dest="ModuleFile", callback=SingleCheckCallback,
1230         help="Build the module specified by the INF file name argument.")
1231     Parser.add_option("-b", "--buildtarget", action="append", type="choice", choices=['DEBUG','RELEASE'], dest="BuildTarget",
1232         help="BuildTarget is one of list: DEBUG, RELEASE, which overrides target.txt's TARGET definition. To specify more TARGET, please repeat this option.")
1233     Parser.add_option("-t", "--tagname", action="append", type="string", dest="ToolChain",
1234         help="Using the Tool Chain Tagname to build the platform, overriding target.txt's TOOL_CHAIN_TAG definition.")
1235     Parser.add_option("-x", "--sku-id", action="callback", type="string", dest="SkuId", callback=SingleCheckCallback,
1236         help="Using this name of SKU ID to build the platform, overriding SKUID_IDENTIFIER in DSC file.")
1237
1238     Parser.add_option("-n", action="callback", type="int", dest="ThreadNumber", callback=SingleCheckCallback,
1239         help="Build the platform using multi-threaded compiler. The value overrides target.txt's MAX_CONCURRENT_THREAD_NUMBER. Less than 2 will disable multi-thread builds.")
1240
1241     Parser.add_option("-f", "--fdf", action="callback", type="string", dest="FdfFile", callback=SingleCheckCallback,
1242         help="The name of the FDF file to use, which overrides the setting in the DSC file.")
1243     Parser.add_option("-r", "--rom-image", action="append", type="string", dest="RomImage", default=[],
1244         help="The name of FD to be generated. The name must be from [FD] section in FDF file.")
1245     Parser.add_option("-i", "--fv-image", action="append", type="string", dest="FvImage", default=[],
1246         help="The name of FV to be generated. The name must be from [FV] section in FDF file.")
1247
1248     Parser.add_option("-u", "--skip-autogen", action="store_true", dest="SkipAutoGen", help="Skip AutoGen step.")
1249     Parser.add_option("-e", "--re-parse", action="store_true", dest="Reparse", help="Re-parse all meta-data files.")
1250
1251     Parser.add_option("-c", "--case-insensitive", action="store_true", dest="CaseInsensitive", help="Don't check case of file name.")
1252
1253     # Parser.add_option("-D", "--define", action="append", dest="Defines", metavar="NAME[=[VALUE]]",
1254     #     help="Define global macro which can be used in DSC/DEC/INF files.")
1255
1256     Parser.add_option("-w", "--warning-as-error", action="store_true", dest="WarningAsError", help="Treat warning in tools as error.")
1257     Parser.add_option("-j", "--log", action="store", dest="LogFile", help="Put log in specified file as well as on console.")
1258
1259     Parser.add_option("-s", "--silent", action="store_true", type=None, dest="SilentMode",
1260         help="Make use of silent mode of (n)make.")
1261     Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
1262     Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
1263                                                                                "including library instances selected, final dependency expression, "\
1264                                                                                "and warning messages, etc.")
1265     Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
1266     Parser.add_option("-D", "--define", action="append", type="string", dest="Macros", help="Macro: \"Name [= Value]\".")
1267
1268     Parser.add_option("-y", "--report-file", action="store", dest="ReportFile", help="Create/overwrite the report to the specified filename.")
1269     Parser.add_option("-Y", "--report-type", action="append", type="choice", choices=['ALL','PCD',], dest="ReportType",
1270         help="Flags that control the type of build report to generate.  Must be one of: [ALL, PCD].  To specify more than one flag, repeat this option on the command line.")
1271
1272     (Opt, Args)=Parser.parse_args()
1273     return (Opt, Args)
1274
1275 ## Tool entrance method
1276 #
1277 # This method mainly dispatch specific methods per the command line options.
1278 # If no error found, return zero value so the caller of this tool can know
1279 # if it's executed successfully or not.
1280 #
1281 #   @retval 0     Tool was successful
1282 #   @retval 1     Tool failed
1283 #
1284 def Main():
1285     StartTime = time.time()
1286
1287     # Initialize log system
1288     EdkLogger.Initialize()
1289
1290     #
1291     # Parse the options and args
1292     #
1293     (Option, Target) = MyOptionParser()
1294     GlobalData.gOptions = Option
1295     GlobalData.gCaseInsensitive = Option.CaseInsensitive
1296
1297     # Set log level
1298     if Option.verbose != None:
1299         EdkLogger.SetLevel(EdkLogger.VERBOSE)
1300     elif Option.quiet != None:
1301         EdkLogger.SetLevel(EdkLogger.QUIET)
1302     elif Option.debug != None:
1303         EdkLogger.SetLevel(Option.debug + 1)
1304     else:
1305         EdkLogger.SetLevel(EdkLogger.INFO)
1306
1307     if Option.LogFile != None:
1308         EdkLogger.SetLogFile(Option.LogFile)
1309
1310     if Option.WarningAsError == True:
1311         EdkLogger.SetWarningAsError()
1312
1313     if platform.platform().find("Windows") >= 0:
1314         GlobalData.gIsWindows = True
1315     else:
1316         GlobalData.gIsWindows = False
1317
1318     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[%s]\n" % platform.platform())
1319     ReturnCode = 0
1320     MyBuild = None
1321     try:
1322         if len(Target) == 0:
1323             Target = "all"
1324         elif len(Target) >= 2:
1325             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
1326                             ExtraData="Please select one of: %s" %(' '.join(gSupportedTarget)))
1327         else:
1328             Target = Target[0].lower()
1329
1330         if Target not in gSupportedTarget:
1331             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
1332                             ExtraData="Please select one of: %s" %(' '.join(gSupportedTarget)))
1333
1334         GlobalData.gGlobalDefines = ParseDefines(Option.Macros)
1335         #
1336         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
1337         #
1338         CheckEnvVariable()
1339         Workspace = os.getenv("WORKSPACE")
1340         #
1341         # Get files real name in workspace dir
1342         #
1343         GlobalData.gAllFiles = Utils.DirCache(Workspace)
1344
1345         WorkingDirectory = os.getcwd()
1346         if not Option.ModuleFile:
1347             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
1348             FileNum = len(FileList)
1349             if FileNum >= 2:
1350                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
1351                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
1352             elif FileNum == 1:
1353                 Option.ModuleFile = NormFile(FileList[0], Workspace)
1354
1355         if Option.ModuleFile:
1356             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
1357             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
1358             if ErrorCode != 0:
1359                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1360
1361         if Option.PlatformFile != None:
1362             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
1363             ErrorCode, ErrorInfo = Option.PlatformFile.Validate(".dsc", False)
1364             if ErrorCode != 0:
1365                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1366
1367         if Option.FdfFile != None:
1368             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
1369             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
1370             if ErrorCode != 0:
1371                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1372
1373         MyBuild = Build(Target, Workspace, Option.PlatformFile, Option.ModuleFile,
1374                         Option.TargetArch, Option.ToolChain, Option.BuildTarget,
1375                         Option.FdfFile, Option.RomImage, Option.FvImage,
1376                         None, Option.SilentMode, Option.ThreadNumber,
1377                         Option.SkipAutoGen, Option.Reparse, Option.SkuId,
1378                         Option.ReportFile, Option.ReportType)
1379         MyBuild.Launch()
1380         #MyBuild.DumpBuildData()
1381     except FatalError, X:
1382         if MyBuild != None:
1383             # for multi-thread build exits safely
1384             MyBuild.Relinquish()
1385         if Option != None and Option.debug != None:
1386             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1387         ReturnCode = X.args[0]
1388     except Warning, X:
1389         # error from Fdf parser
1390         if MyBuild != None:
1391             # for multi-thread build exits safely
1392             MyBuild.Relinquish()
1393         if Option != None and Option.debug != None:
1394             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1395         else:
1396             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError = False)
1397         ReturnCode = FORMAT_INVALID
1398     except KeyboardInterrupt:
1399         ReturnCode = ABORT_ERROR
1400         if Option != None and Option.debug != None:
1401             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1402     except:
1403         if MyBuild != None:
1404             # for multi-thread build exits safely
1405             MyBuild.Relinquish()
1406
1407         # try to get the meta-file from the object causing exception
1408         Tb = sys.exc_info()[-1]
1409         MetaFile = GlobalData.gProcessingFile
1410         while Tb != None:
1411             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
1412                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
1413             Tb = Tb.tb_next
1414         EdkLogger.error(
1415                     "\nbuild",
1416                     CODE_ERROR,
1417                     "Unknown fatal error when processing [%s]" % MetaFile,
1418                     ExtraData="\n(Please send email to dev@buildtools.tianocore.org for help, attaching following call stack trace!)\n",
1419                     RaiseError=False
1420                     )
1421         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1422         ReturnCode = CODE_ERROR
1423     finally:
1424         Utils.Progressor.Abort()
1425
1426     if MyBuild != None:
1427         MyBuild.Db.Close()
1428
1429     if ReturnCode == 0:
1430         Conclusion = "Done"
1431     elif ReturnCode == ABORT_ERROR:
1432         Conclusion = "Aborted"
1433     else:
1434         Conclusion = "Failed"
1435     FinishTime = time.time()
1436     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
1437     EdkLogger.SetLevel(EdkLogger.QUIET)
1438     EdkLogger.quiet("\n- %s -\n%s [%s]" % (Conclusion, time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))
1439
1440     return ReturnCode
1441
1442 if __name__ == '__main__':
1443     r = Main()
1444     ## 0-127 is a safe return range, and 1 is a standard default error
1445     if r < 0 or r > 127: r = 1
1446     sys.exit(r)
1447