BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2007, Intel Corporation
   5 #
   6 #  All rights reserved. This program and the accompanying materials
   7 #  are licensed and made available under the terms and conditions of the BSD License
   8 #  which accompanies this distribution.  The full text of the license may be found at
   9 #  http://opensource.org/licenses/bsd-license.php
  10 #
  11 #  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  12 #  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  13 #
  14
  15 ##
  16 # Import Modules
  17 #
  18 import os
  19 import re
  20 import sys
  21 import glob
  22 import time
  23 import platform
  24 import traceback
  25
  26 from threading import *
  27 from optparse import OptionParser
  28 from subprocess import *
  29 from Common import Misc as Utils
  30
  31 from Common.TargetTxtClassObject import *
  32 from Common.ToolDefClassObject import *
  33 from Common.DataType import *
  34 from AutoGen.AutoGen import *
  35 from Common.BuildToolError import *
  36 from Workspace.WorkspaceDatabase import *
  37
  38 import Common.EdkLogger
  39 import Common.GlobalData as GlobalData
  40
  41 # Version and Copyright
  42 VersionNumber = "0.5"
  43 __version__ = "%prog Version " + VersionNumber
  44 __copyright__ = "Copyright (c) 2007, Intel Corporation  All rights reserved."
  45
  46 ## standard targets of build command
  47 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  48
  49 ## build configuration file
  50 gBuildConfiguration = "Conf/target.txt"
  51 gBuildCacheDir = "Conf/.cache"
  52 gToolsDefinition = "Conf/tools_def.txt"
  53
  54 ## Check environment PATH variable to make sure the specified tool is found
  55 #
  56 #   If the tool is found in the PATH, then True is returned
  57 #   Otherwise, False is returned
  58 #
  59 def IsToolInPath(tool):
  60     if os.environ.has_key('PATHEXT'):
  61         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  62     else:
  63         extns = ('',)
  64     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  65         for ext in extns:
  66             if os.path.exists(os.path.join(pathDir, tool + ext)):
  67                 return True
  68     return False
  69
  70 ## Check environment variables
  71 #
  72 #  Check environment variables that must be set for build. Currently they are
  73 #
  74 #   WORKSPACE           The directory all packages/platforms start from
  75 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  76 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  77 #
  78 #   If any of above environment variable is not set or has error, the build
  79 #   will be broken.
  80 #
  81 def CheckEnvVariable():
  82     # check WORKSPACE
  83     if "WORKSPACE" not in os.environ:
  84         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
  85                         ExtraData="WORKSPACE")
  86
  87     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
  88     if not os.path.exists(WorkspaceDir):
  89         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData="%s" % WorkspaceDir)
  90     elif ' ' in WorkspaceDir:
  91         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
  92                         ExtraData=WorkspaceDir)
  93     os.environ["WORKSPACE"] = WorkspaceDir
  94
  95     #
  96     # Check EFI_SOURCE (R8 build convention). EDK_SOURCE will always point to ECP
  97     #
  98     os.environ["ECP_SOURCE"] = os.path.join(WorkspaceDir, GlobalData.gEdkCompatibilityPkg)
  99     if "EFI_SOURCE" not in os.environ:
 100         os.environ["EFI_SOURCE"] = os.environ["ECP_SOURCE"]
 101     if "EDK_SOURCE" not in os.environ:
 102         os.environ["EDK_SOURCE"] = os.environ["ECP_SOURCE"]
 103
 104     #
 105     # Unify case of characters on case-insensitive systems
 106     #
 107     EfiSourceDir = os.path.normcase(os.path.normpath(os.environ["EFI_SOURCE"]))
 108     EdkSourceDir = os.path.normcase(os.path.normpath(os.environ["EDK_SOURCE"]))
 109     EcpSourceDir = os.path.normcase(os.path.normpath(os.environ["ECP_SOURCE"]))
 110
 111     os.environ["EFI_SOURCE"] = EfiSourceDir
 112     os.environ["EDK_SOURCE"] = EdkSourceDir
 113     os.environ["ECP_SOURCE"] = EcpSourceDir
 114     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 115
 116     if not os.path.exists(EcpSourceDir):
 117         EdkLogger.verbose("ECP_SOURCE = %s doesn't exist. R8 modules could not be built." % EcpSourceDir)
 118     elif ' ' in EcpSourceDir:
 119         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in ECP_SOURCE path",
 120                         ExtraData=EcpSourceDir)
 121     if not os.path.exists(EdkSourceDir):
 122         if EdkSourceDir == EcpSourceDir:
 123             EdkLogger.verbose("EDK_SOURCE = %s doesn't exist. R8 modules could not be built." % EdkSourceDir)
 124         else:
 125             EdkLogger.error("build", PARAMETER_INVALID, "EDK_SOURCE does not exist",
 126                             ExtraData=EdkSourceDir)
 127     elif ' ' in EdkSourceDir:
 128         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in EDK_SOURCE path",
 129                         ExtraData=EdkSourceDir)
 130     if not os.path.exists(EfiSourceDir):
 131         if EfiSourceDir == EcpSourceDir:
 132             EdkLogger.verbose("EFI_SOURCE = %s doesn't exist. R8 modules could not be built." % EfiSourceDir)
 133         else:
 134             EdkLogger.error("build", PARAMETER_INVALID, "EFI_SOURCE does not exist",
 135                             ExtraData=EfiSourceDir)
 136     elif ' ' in EfiSourceDir:
 137         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in EFI_SOURCE path",
 138                         ExtraData=EfiSourceDir)
 139
 140     # change absolute path to relative path to WORKSPACE
 141     if EfiSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 142         EdkLogger.error("build", PARAMETER_INVALID, "EFI_SOURCE is not under WORKSPACE",
 143                         ExtraData="WORKSPACE = %s\n    EFI_SOURCE = %s" % (WorkspaceDir, EfiSourceDir))
 144     if EdkSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 145         EdkLogger.error("build", PARAMETER_INVALID, "EDK_SOURCE is not under WORKSPACE",
 146                         ExtraData="WORKSPACE = %s\n    EDK_SOURCE = %s" % (WorkspaceDir, EdkSourceDir))
 147     if EcpSourceDir.upper().find(WorkspaceDir.upper()) != 0:
 148         EdkLogger.error("build", PARAMETER_INVALID, "ECP_SOURCE is not under WORKSPACE",
 149                         ExtraData="WORKSPACE = %s\n    ECP_SOURCE = %s" % (WorkspaceDir, EcpSourceDir))
 150
 151     # check EDK_TOOLS_PATH
 152     if "EDK_TOOLS_PATH" not in os.environ:
 153         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 154                         ExtraData="EDK_TOOLS_PATH")
 155
 156     # check PATH
 157     if "PATH" not in os.environ:
 158         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 159                         ExtraData="PATH")
 160
 161     # for macro replacement in R9 DSC/DEC/INF file
 162     GlobalData.gGlobalDefines["WORKSPACE"] = ""
 163
 164     # for macro replacement in R8 INF file
 165     GlobalData.gGlobalDefines["EFI_SOURCE"] = EfiSourceDir
 166     GlobalData.gGlobalDefines["EDK_SOURCE"] = EdkSourceDir
 167
 168     GlobalData.gWorkspace = WorkspaceDir
 169     GlobalData.gEfiSource = EfiSourceDir
 170     GlobalData.gEdkSource = EdkSourceDir
 171     GlobalData.gEcpSource = EcpSourceDir
 172
 173 ## Get normalized file path
 174 #
 175 # Convert the path to be local format, and remove the WORKSPACE path at the
 176 # beginning if the file path is given in full path.
 177 #
 178 # @param  FilePath      File path to be normalized
 179 # @param  Workspace     Workspace path which the FilePath will be checked against
 180 #
 181 # @retval string        The normalized file path
 182 #
 183 def NormFile(FilePath, Workspace):
 184     # check if the path is absolute or relative
 185     if os.path.isabs(FilePath):
 186         FileFullPath = os.path.normpath(FilePath)
 187     else:
 188         FileFullPath = os.path.normpath(os.path.join(Workspace, FilePath))
 189
 190     # check if the file path exists or not
 191     if not os.path.isfile(FileFullPath):
 192         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)"  % FileFullPath)
 193
 194     # remove workspace directory from the beginning part of the file path
 195     if Workspace[-1] in ["\\", "/"]:
 196         return FileFullPath[len(Workspace):]
 197     else:
 198         return FileFullPath[(len(Workspace) + 1):]
 199
 200 ## Get the output of an external program
 201 #
 202 # This is the entrance method of thread reading output of an external program and
 203 # putting them in STDOUT/STDERR of current program.
 204 #
 205 # @param  From      The stream message read from
 206 # @param  To        The stream message put on
 207 # @param  ExitFlag  The flag used to indicate stopping reading
 208 #
 209 def ReadMessage(From, To, ExitFlag):
 210     while True:
 211         # read one line a time
 212         Line = From.readline()
 213         # empty string means "end"
 214         if Line != None and Line != "":
 215             To(Line.rstrip())
 216         else:
 217             break
 218         if ExitFlag.isSet():
 219             break
 220
 221 ## Launch an external program
 222 #
 223 # This method will call subprocess.Popen to execute an external program with
 224 # given options in specified directory. Because of the dead-lock issue during
 225 # redirecting output of the external program, threads are used to to do the
 226 # redirection work.
 227 #
 228 # @param  Command               A list or string containing the call of the program
 229 # @param  WorkingDir            The directory in which the program will be running
 230 #
 231 def LaunchCommand(Command, WorkingDir):
 232     # if working directory doesn't exist, Popen() will raise an exception
 233     if not os.path.isdir(WorkingDir):
 234         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 235
 236     Proc = None
 237     EndOfProcedure = None
 238     try:
 239         # launch the command
 240         Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1)
 241
 242         # launch two threads to read the STDOUT and STDERR
 243         EndOfProcedure = Event()
 244         EndOfProcedure.clear()
 245         if Proc.stdout:
 246             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))
 247             StdOutThread.setName("STDOUT-Redirector")
 248             StdOutThread.setDaemon(False)
 249             StdOutThread.start()
 250
 251         if Proc.stderr:
 252             StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))
 253             StdErrThread.setName("STDERR-Redirector")
 254             StdErrThread.setDaemon(False)
 255             StdErrThread.start()
 256
 257         # waiting for program exit
 258         Proc.wait()
 259     except: # in case of aborting
 260         # terminate the threads redirecting the program output
 261         if EndOfProcedure != None:
 262             EndOfProcedure.set()
 263         if Proc == None:
 264             if type(Command) != type(""):
 265                 Command = " ".join(Command)
 266             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 267
 268     if Proc.stdout:
 269         StdOutThread.join()
 270     if Proc.stderr:
 271         StdErrThread.join()
 272
 273     # check the return code of the program
 274     if Proc.returncode != 0:
 275         if type(Command) != type(""):
 276             Command = " ".join(Command)
 277         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 278
 279 ## The smallest unit that can be built in multi-thread build mode
 280 #
 281 # This is the base class of build unit. The "Obj" parameter must provide
 282 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 283 # missing build.
 284 #
 285 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 286 #
 287 class BuildUnit:
 288     ## The constructor
 289     #
 290     #   @param  self        The object pointer
 291     #   @param  Obj         The object the build is working on
 292     #   @param  Target      The build target name, one of gSupportedTarget
 293     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 294     #   @param  WorkingDir  The directory build command starts in
 295     #
 296     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 297         self.BuildObject = Obj
 298         self.Dependency = Dependency
 299         self.WorkingDir = WorkingDir
 300         self.Target = Target
 301         self.BuildCommand = BuildCommand
 302         if BuildCommand == None or len(BuildCommand) == 0:
 303             EdkLogger.error("build", OPTION_MISSING, "No build command found for",
 304                             ExtraData=str(Obj))
 305
 306     ## str() method
 307     #
 308     #   It just returns the string representaion of self.BuildObject
 309     #
 310     #   @param  self        The object pointer
 311     #
 312     def __str__(self):
 313         return str(self.BuildObject)
 314
 315     ## "==" operator method
 316     #
 317     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 318     #   provide its own __eq__() method.
 319     #
 320     #   @param  self        The object pointer
 321     #   @param  Other       The other BuildUnit object compared to
 322     #
 323     def __eq__(self, Other):
 324         return Other != None and self.BuildObject == Other.BuildObject \
 325                 and self.BuildObject.Arch == Other.BuildObject.Arch
 326
 327     ## hash() method
 328     #
 329     #   It just returns the hash value of self.BuildObject which must be hashable.
 330     #
 331     #   @param  self        The object pointer
 332     #
 333     def __hash__(self):
 334         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 335
 336     def __repr__(self):
 337         return repr(self.BuildObject)
 338
 339 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 340 #
 341 # This class is for module build by nmake/make build system. The "Obj" parameter
 342 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 343 # be make units missing build.
 344 #
 345 # Currently the "Obj" should be only ModuleAutoGen object.
 346 #
 347 class ModuleMakeUnit(BuildUnit):
 348     ## The constructor
 349     #
 350     #   @param  self        The object pointer
 351     #   @param  Obj         The ModuleAutoGen object the build is working on
 352     #   @param  Target      The build target name, one of gSupportedTarget
 353     #
 354     def __init__(self, Obj, Target):
 355         Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]
 356         BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
 357         if Target in [None, "", "all"]:
 358             self.Target = "tbuild"
 359
 360 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 361 #
 362 # This class is for platform build by nmake/make build system. The "Obj" parameter
 363 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 364 # be make units missing build.
 365 #
 366 # Currently the "Obj" should be only PlatformAutoGen object.
 367 #
 368 class PlatformMakeUnit(BuildUnit):
 369     ## The constructor
 370     #
 371     #   @param  self        The object pointer
 372     #   @param  Obj         The PlatformAutoGen object the build is working on
 373     #   @param  Target      The build target name, one of gSupportedTarget
 374     #
 375     def __init__(self, Obj, Target):
 376         Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 377         Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])
 378         BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)
 379
 380 ## The class representing the task of a module build or platform build
 381 #
 382 # This class manages the build tasks in multi-thread build mode. Its jobs include
 383 # scheduling thread running, catching thread error, monitor the thread status, etc.
 384 #
 385 class BuildTask:
 386     # queue for tasks waiting for schedule
 387     _PendingQueue = sdict()
 388     _PendingQueueLock = threading.Lock()
 389
 390     # queue for tasks ready for running
 391     _ReadyQueue = sdict()
 392     _ReadyQueueLock = threading.Lock()
 393
 394     # queue for run tasks
 395     _RunningQueue = sdict()
 396     _RunningQueueLock = threading.Lock()
 397
 398     # queue containing all build tasks, in case duplicate build
 399     _TaskQueue = sdict()
 400
 401     # flag indicating error occurs in a running thread
 402     _ErrorFlag = threading.Event()
 403     _ErrorFlag.clear()
 404     _ErrorMessage = ""
 405
 406     # BoundedSemaphore object used to control the number of running threads
 407     _Thread = None
 408
 409     # flag indicating if the scheduler is started or not
 410     _SchedulerStopped = threading.Event()
 411     _SchedulerStopped.set()
 412
 413     ## Start the task scheduler thread
 414     #
 415     #   @param  MaxThreadNumber     The maximum thread number
 416     #   @param  ExitFlag            Flag used to end the scheduler
 417     #
 418     @staticmethod
 419     def StartScheduler(MaxThreadNumber, ExitFlag):
 420         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 421         SchedulerThread.setName("Build-Task-Scheduler")
 422         SchedulerThread.setDaemon(False)
 423         SchedulerThread.start()
 424         # wait for the scheduler to be started, especially useful in Linux
 425         while not BuildTask.IsOnGoing():
 426             time.sleep(0.01)
 427
 428     ## Scheduler method
 429     #
 430     #   @param  MaxThreadNumber     The maximum thread number
 431     #   @param  ExitFlag            Flag used to end the scheduler
 432     #
 433     @staticmethod
 434     def Scheduler(MaxThreadNumber, ExitFlag):
 435         BuildTask._SchedulerStopped.clear()
 436         try:
 437             # use BoundedSemaphore to control the maximum running threads
 438             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 439             #
 440             # scheduling loop, which will exits when no pending/ready task and
 441             # indicated to do so, or there's error in running thread
 442             #
 443             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 444                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 445                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 446                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 447
 448                 # get all pending tasks
 449                 BuildTask._PendingQueueLock.acquire()
 450                 BuildObjectList = BuildTask._PendingQueue.keys()
 451                 #
 452                 # check if their dependency is resolved, and if true, move them
 453                 # into ready queue
 454                 #
 455                 for BuildObject in BuildObjectList:
 456                     Bt = BuildTask._PendingQueue[BuildObject]
 457                     if Bt.IsReady():
 458                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 459                 BuildTask._PendingQueueLock.release()
 460
 461                 # launch build thread until the maximum number of threads is reached
 462                 while not BuildTask._ErrorFlag.isSet():
 463                     # empty ready queue, do nothing further
 464                     if len(BuildTask._ReadyQueue) == 0:
 465                         break
 466
 467                     # wait for active thread(s) exit
 468                     BuildTask._Thread.acquire(True)
 469
 470                     # start a new build thread
 471                     Bo = BuildTask._ReadyQueue.keys()[0]
 472                     Bt = BuildTask._ReadyQueue.pop(Bo)
 473
 474                     # move into running queue
 475                     BuildTask._RunningQueueLock.acquire()
 476                     BuildTask._RunningQueue[Bo] = Bt
 477                     BuildTask._RunningQueueLock.release()
 478
 479                     Bt.Start()
 480                     # avoid tense loop
 481                     time.sleep(0.01)
 482
 483                 # avoid tense loop
 484                 time.sleep(0.01)
 485
 486             # wait for all running threads exit
 487             if BuildTask._ErrorFlag.isSet():
 488                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 489             # while not BuildTask._ErrorFlag.isSet() and \
 490             while len(BuildTask._RunningQueue) > 0:
 491                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 492                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join([Th.getName() for Th in threading.enumerate()]))
 493                 # avoid tense loop
 494                 time.sleep(0.1)
 495         except BaseException, X:
 496             #
 497             # TRICK: hide the output of threads left runing, so that the user can
 498             #        catch the error message easily
 499             #
 500             EdkLogger.SetLevel(EdkLogger.ERROR)
 501             BuildTask._ErrorFlag.set()
 502             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 503
 504         BuildTask._PendingQueue.clear()
 505         BuildTask._ReadyQueue.clear()
 506         BuildTask._RunningQueue.clear()
 507         BuildTask._TaskQueue.clear()
 508         BuildTask._SchedulerStopped.set()
 509
 510     ## Wait for all running method exit
 511     #
 512     @staticmethod
 513     def WaitForComplete():
 514         BuildTask._SchedulerStopped.wait()
 515
 516     ## Check if the scheduler is running or not
 517     #
 518     @staticmethod
 519     def IsOnGoing():
 520         return not BuildTask._SchedulerStopped.isSet()
 521
 522     ## Abort the build
 523     @staticmethod
 524     def Abort():
 525         if BuildTask.IsOnGoing():
 526             BuildTask._ErrorFlag.set()
 527             BuildTask.WaitForComplete()
 528
 529     ## Check if there's error in running thread
 530     #
 531     #   Since the main thread cannot catch exceptions in other thread, we have to
 532     #   use threading.Event to communicate this formation to main thread.
 533     #
 534     @staticmethod
 535     def HasError():
 536         return BuildTask._ErrorFlag.isSet()
 537
 538     ## Get error message in running thread
 539     #
 540     #   Since the main thread cannot catch exceptions in other thread, we have to
 541     #   use a static variable to communicate this message to main thread.
 542     #
 543     @staticmethod
 544     def GetErrorMessage():
 545         return BuildTask._ErrorMessage
 546
 547     ## Factory method to create a BuildTask object
 548     #
 549     #   This method will check if a module is building or has been built. And if
 550     #   true, just return the associated BuildTask object in the _TaskQueue. If
 551     #   not, create and return a new BuildTask object. The new BuildTask object
 552     #   will be appended to the _PendingQueue for scheduling later.
 553     #
 554     #   @param  BuildItem       A BuildUnit object representing a build object
 555     #   @param  Dependency      The dependent build object of BuildItem
 556     #
 557     @staticmethod
 558     def New(BuildItem, Dependency=None):
 559         if BuildItem in BuildTask._TaskQueue:
 560             Bt = BuildTask._TaskQueue[BuildItem]
 561             return Bt
 562
 563         Bt = BuildTask()
 564         Bt._Init(BuildItem, Dependency)
 565         BuildTask._TaskQueue[BuildItem] = Bt
 566
 567         BuildTask._PendingQueueLock.acquire()
 568         BuildTask._PendingQueue[BuildItem] = Bt
 569         BuildTask._PendingQueueLock.release()
 570
 571         return Bt
 572
 573     ## The real constructor of BuildTask
 574     #
 575     #   @param  BuildItem       A BuildUnit object representing a build object
 576     #   @param  Dependency      The dependent build object of BuildItem
 577     #
 578     def _Init(self, BuildItem, Dependency=None):
 579         self.BuildItem = BuildItem
 580
 581         self.DependencyList = []
 582         if Dependency == None:
 583             Dependency = BuildItem.Dependency
 584         else:
 585             Dependency.extend(BuildItem.Dependency)
 586         self.AddDependency(Dependency)
 587         # flag indicating build completes, used to avoid unnecessary re-build
 588         self.CompleteFlag = False
 589
 590     ## Check if all dependent build tasks are completed or not
 591     #
 592     def IsReady(self):
 593         ReadyFlag = True
 594         for Dep in self.DependencyList:
 595             if Dep.CompleteFlag == True:
 596                 continue
 597             ReadyFlag = False
 598             break
 599
 600         return ReadyFlag
 601
 602     ## Add dependent build task
 603     #
 604     #   @param  Dependency      The list of dependent build objects
 605     #
 606     def AddDependency(self, Dependency):
 607         for Dep in Dependency:
 608             self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 609
 610     ## The thread wrapper of LaunchCommand function
 611     #
 612     # @param  Command               A list or string contains the call of the command
 613     # @param  WorkingDir            The directory in which the program will be running
 614     #
 615     def _CommandThread(self, Command, WorkingDir):
 616         try:
 617             LaunchCommand(Command, WorkingDir)
 618             self.CompleteFlag = True
 619         except:
 620             #
 621             # TRICK: hide the output of threads left runing, so that the user can
 622             #        catch the error message easily
 623             #
 624             if not BuildTask._ErrorFlag.isSet():
 625                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 626                                                                   self.BuildItem.BuildObject.Arch,
 627                                                                   self.BuildItem.BuildObject.ToolChain,
 628                                                                   self.BuildItem.BuildObject.BuildTarget
 629                                                                  )
 630             EdkLogger.SetLevel(EdkLogger.ERROR)
 631             BuildTask._ErrorFlag.set()
 632             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 633                                       (threading.currentThread().getName(), Command, WorkingDir)
 634         # indicate there's a thread is available for another build task
 635         BuildTask._RunningQueueLock.acquire()
 636         BuildTask._RunningQueue.pop(self.BuildItem)
 637         BuildTask._RunningQueueLock.release()
 638         BuildTask._Thread.release()
 639
 640     ## Start build task thread
 641     #
 642     def Start(self):
 643         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 644         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 645         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 646         self.BuildTread.setName("build thread")
 647         self.BuildTread.setDaemon(False)
 648         self.BuildTread.start()
 649
 650 ## The class implementing the EDK2 build process
 651 #
 652 #   The build process includes:
 653 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 654 #       2. Parse DSC file of active platform
 655 #       3. Parse FDF file if any
 656 #       4. Establish build database, including parse all other files (module, package)
 657 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 658 #       6. Call build command
 659 #
 660 class Build():
 661     ## Constructor
 662     #
 663     # Constructor will load all necessary configurations, parse platform, modules
 664     # and packages and the establish a database for AutoGen.
 665     #
 666     #   @param  Target              The build command target, one of gSupportedTarget
 667     #   @param  WorkspaceDir        The directory of workspace
 668     #   @param  Platform            The DSC file of active platform
 669     #   @param  Module              The INF file of active module, if any
 670     #   @param  Arch                The Arch list of platform or module
 671     #   @param  ToolChain           The name list of toolchain
 672     #   @param  BuildTarget         The "DEBUG" or "RELEASE" build
 673     #   @param  FlashDefinition     The FDF file of active platform
 674     #   @param  FdList=[]           The FD names to be individually built
 675     #   @param  FvList=[]           The FV names to be individually built
 676     #   @param  MakefileType        The type of makefile (for MSFT make or GNU make)
 677     #   @param  SilentMode          Indicate multi-thread build mode
 678     #   @param  ThreadNumber        The maximum number of thread if in multi-thread build mode
 679     #   @param  SkipAutoGen         Skip AutoGen step
 680     #   @param  Reparse             Re-parse all meta files
 681     #   @param  SkuId               SKU id from command line
 682     #
 683     def __init__(self, Target, WorkspaceDir, Platform, Module, Arch, ToolChain,
 684                  BuildTarget, FlashDefinition, FdList=[], FvList=[],
 685                  MakefileType="nmake", SilentMode=False, ThreadNumber=2,
 686                  SkipAutoGen=False, Reparse=False, SkuId=None):
 687
 688         self.WorkspaceDir = WorkspaceDir
 689         self.Target         = Target
 690         self.PlatformFile   = Platform
 691         self.ModuleFile     = Module
 692         self.ArchList       = Arch
 693         self.ToolChainList  = ToolChain
 694         self.BuildTargetList= BuildTarget
 695         self.Fdf            = FlashDefinition
 696         self.FdList         = FdList
 697         self.FvList         = FvList
 698         self.MakefileType   = MakefileType
 699         self.SilentMode     = SilentMode
 700         self.ThreadNumber   = ThreadNumber
 701         self.SkipAutoGen    = SkipAutoGen
 702         self.Reparse        = Reparse
 703         self.SkuId          = SkuId
 704         self.SpawnMode      = True
 705
 706         self.TargetTxt      = TargetTxtClassObject()
 707         self.ToolDef        = ToolDefClassObject()
 708         #self.Db             = WorkspaceDatabase(None, GlobalData.gGlobalDefines, self.Reparse)
 709         self.Db             = WorkspaceDatabase(None, {}, self.Reparse)
 710         self.BuildDatabase  = self.Db.BuildObject
 711         self.Platform       = None
 712
 713         # print dot charater during doing some time-consuming work
 714         self.Progress = Utils.Progressor()
 715
 716         # parse target.txt, tools_def.txt, and platform file
 717         #self.RestoreBuildData()
 718         self.LoadConfiguration()
 719         self.InitBuild()
 720
 721         # print current build environment and configuration
 722         EdkLogger.quiet("%-24s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 723         EdkLogger.quiet("%-24s = %s" % ("ECP_SOURCE", os.environ["ECP_SOURCE"]))
 724         EdkLogger.quiet("%-24s = %s" % ("EDK_SOURCE", os.environ["EDK_SOURCE"]))
 725         EdkLogger.quiet("%-24s = %s" % ("EFI_SOURCE", os.environ["EFI_SOURCE"]))
 726         EdkLogger.quiet("%-24s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 727
 728         EdkLogger.info('\n%-24s = %s' % ("TARGET_ARCH", ' '.join(self.ArchList)))
 729         EdkLogger.info('%-24s = %s' % ("TARGET", ' '.join(self.BuildTargetList)))
 730         EdkLogger.info('%-24s = %s' % ("TOOL_CHAIN_TAG", ' '.join(self.ToolChainList)))
 731
 732         EdkLogger.info('\n%-24s = %s' % ("Active Platform", self.PlatformFile))
 733
 734         if self.Fdf != None and self.Fdf != "":
 735             EdkLogger.info('%-24s = %s' % ("Flash Image Definition", self.Fdf))
 736
 737         if self.ModuleFile != None and self.ModuleFile != "":
 738             EdkLogger.info('%-24s = %s' % ("Active Module", self.ModuleFile))
 739
 740         os.chdir(self.WorkspaceDir)
 741         self.Progress.Start("\nProcessing meta-data")
 742
 743     ## Load configuration
 744     #
 745     #   This method will parse target.txt and get the build configurations.
 746     #
 747     def LoadConfiguration(self):
 748         #
 749         # Check target.txt and tools_def.txt and Init them
 750         #
 751         BuildConfigurationFile = os.path.normpath(os.path.join(self.WorkspaceDir, gBuildConfiguration))
 752         if os.path.isfile(BuildConfigurationFile) == True:
 753             StatusCode = self.TargetTxt.LoadTargetTxtFile(BuildConfigurationFile)
 754
 755             ToolDefinitionFile = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 756             if ToolDefinitionFile == '':
 757                 ToolDefinitionFile = gToolsDefinition
 758             ToolDefinitionFile = os.path.normpath(os.path.join(self.WorkspaceDir, ToolDefinitionFile))
 759             if os.path.isfile(ToolDefinitionFile) == True:
 760                 StatusCode = self.ToolDef.LoadToolDefFile(ToolDefinitionFile)
 761             else:
 762                 EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=ToolDefinitionFile)
 763         else:
 764             EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=BuildConfigurationFile)
 765
 766         # if no ARCH given in command line, get it from target.txt
 767         if self.ArchList == None or len(self.ArchList) == 0:
 768             self.ArchList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TARGET_ARCH]
 769
 770         # if no build target given in command line, get it from target.txt
 771         if self.BuildTargetList == None or len(self.BuildTargetList) == 0:
 772             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TARGET]
 773
 774         # if no tool chain given in command line, get it from target.txt
 775         if self.ToolChainList == None or len(self.ToolChainList) == 0:
 776             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 777             if self.ToolChainList == None or len(self.ToolChainList) == 0:
 778                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 779
 780         # check if the tool chains are defined or not
 781         NewToolChainList = []
 782         for ToolChain in self.ToolChainList:
 783             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 784                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 785             else:
 786                 NewToolChainList.append(ToolChain)
 787         # if no tool chain available, break the build
 788         if len(NewToolChainList) == 0:
 789             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 790                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 791         else:
 792             self.ToolChainList = NewToolChainList
 793
 794         if self.ThreadNumber == None:
 795             self.ThreadNumber = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
 796             if self.ThreadNumber == '':
 797                 self.ThreadNumber = 0
 798             else:
 799                 self.ThreadNumber = int(self.ThreadNumber, 0)
 800
 801         if self.ThreadNumber == 0:
 802             self.ThreadNumber = 1
 803
 804         if not self.PlatformFile:
 805             PlatformFile = self.TargetTxt.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 806             if not PlatformFile:
 807                 # Try to find one in current directory
 808                 WorkingDirectory = os.getcwd()
 809                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 810                 FileNum = len(FileList)
 811                 if FileNum >= 2:
 812                     EdkLogger.error("build", OPTION_MISSING,
 813                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 814                 elif FileNum == 1:
 815                     PlatformFile = FileList[0]
 816                 else:
 817                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 818                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 819
 820             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 821             ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 822             if ErrorCode != 0:
 823                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 824
 825     ## Initialize build configuration
 826     #
 827     #   This method will parse DSC file and merge the configurations from
 828     #   command line and target.txt, then get the final build configurations.
 829     #
 830     def InitBuild(self):
 831         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc")
 832         if ErrorCode != 0:
 833             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 834
 835         # create metafile database
 836         self.Db.InitDatabase()
 837
 838         # we need information in platform description file to determine how to build
 839         self.Platform = self.BuildDatabase[self.PlatformFile, 'COMMON']
 840         if not self.Fdf:
 841             self.Fdf = self.Platform.FlashDefinition
 842
 843         if self.SkuId == None or self.SkuId == '':
 844             self.SkuId = self.Platform.SkuName
 845
 846         # check FD/FV build target
 847         if self.Fdf == None or self.Fdf == "":
 848             if self.FdList != []:
 849                 EdkLogger.info("No flash definition file found. FD [%s] will be ignored." % " ".join(self.FdList))
 850                 self.FdList = []
 851             if self.FvList != []:
 852                 EdkLogger.info("No flash definition file found. FV [%s] will be ignored." % " ".join(self.FvList))
 853                 self.FvList = []
 854         else:
 855             FdfParserObj = FdfParser(str(self.Fdf))
 856             FdfParserObj.ParseFile()
 857             for fvname in self.FvList:
 858                 if fvname.upper() not in FdfParserObj.Profile.FvDict.keys():
 859                     EdkLogger.error("build", OPTION_VALUE_INVALID,
 860                                     "No such an FV in FDF file: %s" % fvname)
 861
 862         #
 863         # Merge Arch
 864         #
 865         if self.ArchList == None or len(self.ArchList) == 0:
 866             ArchList = set(self.Platform.SupArchList)
 867         else:
 868             ArchList = set(self.ArchList) & set(self.Platform.SupArchList)
 869         if len(ArchList) == 0:
 870             EdkLogger.error("build", PARAMETER_INVALID,
 871                             ExtraData = "Active platform supports [%s] only, but [%s] is given."
 872                                         % (" ".join(self.Platform.SupArchList), " ".join(self.ArchList)))
 873         elif len(ArchList) != len(self.ArchList):
 874             SkippedArchList = set(self.ArchList).symmetric_difference(set(self.Platform.SupArchList))
 875             EdkLogger.verbose("\nArch [%s] is ignored because active platform supports [%s] but [%s] is specified !"
 876                            % (" ".join(SkippedArchList), " ".join(self.Platform.SupArchList), " ".join(self.ArchList)))
 877         self.ArchList = tuple(ArchList)
 878
 879         # Merge build target
 880         if self.BuildTargetList == None or len(self.BuildTargetList) == 0:
 881             BuildTargetList = self.Platform.BuildTargets
 882         else:
 883             BuildTargetList = list(set(self.BuildTargetList) & set(self.Platform.BuildTargets))
 884         if BuildTargetList == []:
 885             EdkLogger.error("build", PARAMETER_INVALID, "Active platform only supports [%s], but [%s] is given"
 886                                 % (" ".join(self.Platform.BuildTargets), " ".join(self.BuildTargetList)))
 887         self.BuildTargetList = BuildTargetList
 888
 889     ## Build a module or platform
 890     #
 891     # Create autogen code and makfile for a module or platform, and the launch
 892     # "make" command to build it
 893     #
 894     #   @param  Target                      The target of build command
 895     #   @param  Platform                    The platform file
 896     #   @param  Module                      The module file
 897     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
 898     #   @param  ToolChain                   The name of toolchain to build
 899     #   @param  Arch                        The arch of the module/platform
 900     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
 901     #                                       for dependent modules/Libraries
 902     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
 903     #                                       for dependent modules/Libraries
 904     #
 905     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True):
 906         if AutoGenObject == None:
 907             return False
 908
 909         # skip file generation for cleanxxx targets, run and fds target
 910         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
 911             # for target which must generate AutoGen code and makefile
 912             if not self.SkipAutoGen or Target == 'genc':
 913                 self.Progress.Start("Generating code")
 914                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
 915                 self.Progress.Stop("done!")
 916             if Target == "genc":
 917                 return True
 918
 919             if not self.SkipAutoGen or Target == 'genmake':
 920                 self.Progress.Start("Generating makefile")
 921                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
 922                 self.Progress.Stop("done!")
 923             if Target == "genmake":
 924                 return True
 925         else:
 926             # always recreate top/platform makefile when clean, just in case of inconsistency
 927             AutoGenObject.CreateCodeFile(False)
 928             AutoGenObject.CreateMakeFile(False)
 929
 930         if EdkLogger.GetLevel() == EdkLogger.QUIET:
 931             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
 932
 933         BuildCommand = AutoGenObject.BuildCommand
 934         if BuildCommand == None or len(BuildCommand) == 0:
 935             EdkLogger.error("build", OPTION_MISSING, ExtraData="No MAKE command found for [%s, %s, %s]" % Key)
 936
 937         BuildCommand = BuildCommand + [Target]
 938         LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
 939         if Target == 'cleanall':
 940             try:
 941                 #os.rmdir(AutoGenObject.BuildDir)
 942                 RemoveDirectory(AutoGenObject.BuildDir, True)
 943             except WindowsError, X:
 944                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
 945         return True
 946
 947     ## Build active platform for different build targets and different tool chains
 948     #
 949     def _BuildPlatform(self):
 950         for BuildTarget in self.BuildTargetList:
 951             for ToolChain in self.ToolChainList:
 952                 Wa = WorkspaceAutoGen(
 953                         self.WorkspaceDir,
 954                         self.Platform,
 955                         BuildTarget,
 956                         ToolChain,
 957                         self.ArchList,
 958                         self.BuildDatabase,
 959                         self.TargetTxt,
 960                         self.ToolDef,
 961                         self.Fdf,
 962                         self.FdList,
 963                         self.FvList,
 964                         self.SkuId
 965                         )
 966                 self.Progress.Stop("done!")
 967                 self._Build(self.Target, Wa)
 968
 969     ## Build active module for different build targets, different tool chains and different archs
 970     #
 971     def _BuildModule(self):
 972         for BuildTarget in self.BuildTargetList:
 973             for ToolChain in self.ToolChainList:
 974                 #
 975                 # module build needs platform build information, so get platform
 976                 # AutoGen first
 977                 #
 978                 Wa = WorkspaceAutoGen(
 979                         self.WorkspaceDir,
 980                         self.Platform,
 981                         BuildTarget,
 982                         ToolChain,
 983                         self.ArchList,
 984                         self.BuildDatabase,
 985                         self.TargetTxt,
 986                         self.ToolDef,
 987                         self.Fdf,
 988                         self.FdList,
 989                         self.FvList,
 990                         self.SkuId
 991                         )
 992                 Wa.CreateMakeFile(False)
 993                 self.Progress.Stop("done!")
 994                 MaList = []
 995                 for Arch in self.ArchList:
 996                     Ma = ModuleAutoGen(Wa, self.ModuleFile, BuildTarget, ToolChain, Arch, self.PlatformFile)
 997                     if Ma == None: continue
 998                     MaList.append(Ma)
 999                     self._Build(self.Target, Ma)
1000                 if MaList == []:
1001                     EdkLogger.error(
1002                                 'build',
1003                                 BUILD_ERROR,
1004                                 "Module for [%s] is not a component of active platform."\
1005                                 " Please make sure that the ARCH and inf file path are"\
1006                                 " given in the same as in [%s]" %\
1007                                     (', '.join(self.ArchList), self.Platform),
1008                                 ExtraData=self.ModuleFile
1009                                 )
1010
1011     ## Build a platform in multi-thread mode
1012     #
1013     def _MultiThreadBuildPlatform(self):
1014         for BuildTarget in self.BuildTargetList:
1015             for ToolChain in self.ToolChainList:
1016                 Wa = WorkspaceAutoGen(
1017                         self.WorkspaceDir,
1018                         self.Platform,
1019                         BuildTarget,
1020                         ToolChain,
1021                         self.ArchList,
1022                         self.BuildDatabase,
1023                         self.TargetTxt,
1024                         self.ToolDef,
1025                         self.Fdf,
1026                         self.FdList,
1027                         self.FvList,
1028                         self.SkuId
1029                         )
1030                 Wa.CreateMakeFile(False)
1031
1032                 # multi-thread exit flag
1033                 ExitFlag = threading.Event()
1034                 ExitFlag.clear()
1035                 for Arch in self.ArchList:
1036                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1037                     if Pa == None:
1038                         continue
1039                     for Module in Pa.Platform.Modules:
1040                         # Get ModuleAutoGen object to generate C code file and makefile
1041                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
1042                         if Ma == None:
1043                             continue
1044                         # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1045                         if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1046                             # for target which must generate AutoGen code and makefile
1047                             if not self.SkipAutoGen or self.Target == 'genc':
1048                                 Ma.CreateCodeFile(True)
1049                             if self.Target == "genc":
1050                                 continue
1051
1052                             if not self.SkipAutoGen or self.Target == 'genmake':
1053                                 Ma.CreateMakeFile(True)
1054                             if self.Target == "genmake":
1055                                 continue
1056                         self.Progress.Stop("done!")
1057                         # Generate build task for the module
1058                         Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))
1059                         # Break build if any build thread has error
1060                         if BuildTask.HasError():
1061                             # we need a full version of makefile for platform
1062                             ExitFlag.set()
1063                             BuildTask.WaitForComplete()
1064                             Pa.CreateMakeFile(False)
1065                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1066                         # Start task scheduler
1067                         if not BuildTask.IsOnGoing():
1068                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1069
1070                     # in case there's an interruption. we need a full version of makefile for platform
1071                     Pa.CreateMakeFile(False)
1072                     if BuildTask.HasError():
1073                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1074
1075                 #
1076                 # All modules have been put in build tasks queue. Tell task scheduler
1077                 # to exit if all tasks are completed
1078                 #
1079                 ExitFlag.set()
1080                 BuildTask.WaitForComplete()
1081
1082                 #
1083                 # Check for build error, and raise exception if one
1084                 # has been signaled.
1085                 #
1086                 if BuildTask.HasError():
1087                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1088
1089                 # Generate FD image if there's a FDF file found
1090                 if self.Fdf != '' and self.Target in ["", "all", "fds"]:
1091                     LaunchCommand(Wa.BuildCommand + ["fds"], Wa.MakeFileDir)
1092
1093     ## Generate GuidedSectionTools.txt in the FV directories.
1094     #
1095     def CreateGuidedSectionToolsFile(self):
1096         for Arch in self.ArchList:
1097             for BuildTarget in self.BuildTargetList:
1098                 for ToolChain in self.ToolChainList:
1099                     FvDir = os.path.join(
1100                                 self.WorkspaceDir,
1101                                 self.Platform.OutputDirectory,
1102                                 '_'.join((BuildTarget, ToolChain)),
1103                                 'FV'
1104                                 )
1105                     if not os.path.exists(FvDir):
1106                         continue
1107                     # Build up the list of supported architectures for this build
1108                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
1109
1110                     # Look through the tool definitions for GUIDed tools
1111                     guidAttribs = []
1112                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.iteritems():
1113                         if attrib.upper().endswith('_GUID'):
1114                             split = attrib.split('_')
1115                             thisPrefix = '_'.join(split[0:3]) + '_'
1116                             if thisPrefix == prefix:
1117                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
1118                                 guid = guid.lower()
1119                                 toolName = split[3]
1120                                 path = '_'.join(split[0:4]) + '_PATH'
1121                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
1122                                 path = self.GetFullPathOfTool(path)
1123                                 guidAttribs.append((guid, toolName, path))
1124
1125                     # Write out GuidedSecTools.txt
1126                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
1127                     toolsFile = open(toolsFile, 'wt')
1128                     for guidedSectionTool in guidAttribs:
1129                         print >> toolsFile, ' '.join(guidedSectionTool)
1130                     toolsFile.close()
1131
1132     ## Returns the full path of the tool.
1133     #
1134     def GetFullPathOfTool (self, tool):
1135         if os.path.exists(tool):
1136             return os.path.realpath(tool)
1137         else:
1138             # We need to search for the tool using the
1139             # PATH environment variable.
1140             for dirInPath in os.environ['PATH'].split(os.pathsep):
1141                 foundPath = os.path.join(dirInPath, tool)
1142                 if os.path.exists(foundPath):
1143                     return os.path.realpath(foundPath)
1144
1145         # If the tool was not found in the path then we just return
1146         # the input tool.
1147         return tool
1148
1149     ## Launch the module or platform build
1150     #
1151     def Launch(self):
1152         if self.ModuleFile == None or self.ModuleFile == "":
1153             if not self.SpawnMode or self.Target not in ["", "all"]:
1154                 self.SpawnMode = False
1155                 self._BuildPlatform()
1156             else:
1157                 self._MultiThreadBuildPlatform()
1158             self.CreateGuidedSectionToolsFile()
1159         else:
1160             self.SpawnMode = False
1161             self._BuildModule()
1162
1163     ## Do some clean-up works when error occurred
1164     def Relinquish(self):
1165         OldLogLevel = EdkLogger.GetLevel()
1166         EdkLogger.SetLevel(EdkLogger.ERROR)
1167         #self.DumpBuildData()
1168         Utils.Progressor.Abort()
1169         if self.SpawnMode == True:
1170             BuildTask.Abort()
1171         EdkLogger.SetLevel(OldLogLevel)
1172
1173     def DumpBuildData(self):
1174         CacheDirectory = os.path.join(self.WorkspaceDir, gBuildCacheDir)
1175         Utils.CreateDirectory(CacheDirectory)
1176         Utils.DataDump(Utils.gFileTimeStampCache, os.path.join(CacheDirectory, "gFileTimeStampCache"))
1177         Utils.DataDump(Utils.gDependencyDatabase, os.path.join(CacheDirectory, "gDependencyDatabase"))
1178
1179     def RestoreBuildData(self):
1180         FilePath = os.path.join(self.WorkspaceDir, gBuildCacheDir, "gFileTimeStampCache")
1181         if Utils.gFileTimeStampCache == {} and os.path.isfile(FilePath):
1182             Utils.gFileTimeStampCache = Utils.DataRestore(FilePath)
1183             if Utils.gFileTimeStampCache == None:
1184                 Utils.gFileTimeStampCache = {}
1185
1186         FilePath = os.path.join(self.WorkspaceDir, gBuildCacheDir, "gDependencyDatabase")
1187         if Utils.gDependencyDatabase == {} and os.path.isfile(FilePath):
1188             Utils.gDependencyDatabase = Utils.DataRestore(FilePath)
1189             if Utils.gDependencyDatabase == None:
1190                 Utils.gDependencyDatabase = {}
1191
1192 def ParseDefines(DefineList=[]):
1193     DefineDict = {}
1194     if DefineList != None:
1195         for Define in DefineList:
1196             DefineTokenList = Define.split("=", 1)
1197             if len(DefineTokenList) == 1:
1198                 DefineDict[DefineTokenList[0]] = ""
1199             else:
1200                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
1201     return DefineDict
1202
1203 gParamCheck = []
1204 def SingleCheckCallback(option, opt_str, value, parser):
1205     if option not in gParamCheck:
1206         setattr(parser.values, option.dest, value)
1207         gParamCheck.append(option)
1208     else:
1209         parser.error("Option %s only allows one instance in command line!" % option)
1210
1211 ## Parse command line options
1212 #
1213 # Using standard Python module optparse to parse command line option of this tool.
1214 #
1215 #   @retval Opt   A optparse.Values object containing the parsed options
1216 #   @retval Args  Target of build command
1217 #
1218 def MyOptionParser():
1219     Parser = OptionParser(description=__copyright__,version=__version__,prog="build.exe",usage="%prog [options] [all|fds|genc|genmake|clean|cleanall|cleanlib|modules|libraries|run]")
1220     Parser.add_option("-a", "--arch", action="append", type="choice", choices=['IA32','X64','IPF','EBC','ARM'], dest="TargetArch",
1221         help="ARCHS is one of list: IA32, X64, IPF, ARM or EBC, which overrides target.txt's TARGET_ARCH definition. To specify more archs, please repeat this option.")
1222     Parser.add_option("-p", "--platform", action="callback", type="string", dest="PlatformFile", callback=SingleCheckCallback,
1223         help="Build the platform specified by the DSC file name argument, overriding target.txt's ACTIVE_PLATFORM definition.")
1224     Parser.add_option("-m", "--module", action="callback", type="string", dest="ModuleFile", callback=SingleCheckCallback,
1225         help="Build the module specified by the INF file name argument.")
1226     Parser.add_option("-b", "--buildtarget", action="append", type="choice", choices=['DEBUG','RELEASE'], dest="BuildTarget",
1227         help="BuildTarget is one of list: DEBUG, RELEASE, which overrides target.txt's TARGET definition. To specify more TARGET, please repeat this option.")
1228     Parser.add_option("-t", "--tagname", action="append", type="string", dest="ToolChain",
1229         help="Using the Tool Chain Tagname to build the platform, overriding target.txt's TOOL_CHAIN_TAG definition.")
1230     Parser.add_option("-x", "--sku-id", action="callback", type="string", dest="SkuId", callback=SingleCheckCallback,
1231         help="Using this name of SKU ID to build the platform, overriding SKUID_IDENTIFIER in DSC file.")
1232
1233     Parser.add_option("-n", action="callback", type="int", dest="ThreadNumber", callback=SingleCheckCallback,
1234         help="Build the platform using multi-threaded compiler. The value overrides target.txt's MAX_CONCURRENT_THREAD_NUMBER. Less than 2 will disable multi-thread builds.")
1235
1236     Parser.add_option("-f", "--fdf", action="callback", type="string", dest="FdfFile", callback=SingleCheckCallback,
1237         help="The name of the FDF file to use, which overrides the setting in the DSC file.")
1238     Parser.add_option("-r", "--rom-image", action="append", type="string", dest="RomImage", default=[],
1239         help="The name of FD to be generated. The name must be from [FD] section in FDF file.")
1240     Parser.add_option("-i", "--fv-image", action="append", type="string", dest="FvImage", default=[],
1241         help="The name of FV to be generated. The name must be from [FV] section in FDF file.")
1242
1243     Parser.add_option("-u", "--skip-autogen", action="store_true", dest="SkipAutoGen", help="Skip AutoGen step.")
1244     Parser.add_option("-e", "--re-parse", action="store_true", dest="Reparse", help="Re-parse all meta-data files.")
1245
1246     Parser.add_option("-c", "--case-insensitive", action="store_true", dest="CaseInsensitive", help="Don't check case of file name.")
1247
1248     # Parser.add_option("-D", "--define", action="append", dest="Defines", metavar="NAME[=[VALUE]]",
1249     #     help="Define global macro which can be used in DSC/DEC/INF files.")
1250
1251     Parser.add_option("-w", "--warning-as-error", action="store_true", dest="WarningAsError", help="Treat warning in tools as error.")
1252     Parser.add_option("-j", "--log", action="store", dest="LogFile", help="Put log in specified file as well as on console.")
1253
1254     Parser.add_option("-s", "--silent", action="store_true", type=None, dest="SilentMode",
1255         help="Make use of silent mode of (n)make.")
1256     Parser.add_option("-q", "--quiet", action="store_true", type=None, help="Disable all messages except FATAL ERRORS.")
1257     Parser.add_option("-v", "--verbose", action="store_true", type=None, help="Turn on verbose output with informational messages printed, "\
1258                                                                                "including library instances selected, final dependency expression, "\
1259                                                                                "and warning messages, etc.")
1260     Parser.add_option("-d", "--debug", action="store", type="int", help="Enable debug messages at specified level.")
1261
1262     (Opt, Args)=Parser.parse_args()
1263     return (Opt, Args)
1264
1265 ## Tool entrance method
1266 #
1267 # This method mainly dispatch specific methods per the command line options.
1268 # If no error found, return zero value so the caller of this tool can know
1269 # if it's executed successfully or not.
1270 #
1271 #   @retval 0     Tool was successful
1272 #   @retval 1     Tool failed
1273 #
1274 def Main():
1275     StartTime = time.time()
1276
1277     # Initialize log system
1278     EdkLogger.Initialize()
1279
1280     #
1281     # Parse the options and args
1282     #
1283     (Option, Target) = MyOptionParser()
1284     GlobalData.gOptions = Option
1285     GlobalData.gCaseInsensitive = Option.CaseInsensitive
1286
1287     # Set log level
1288     if Option.verbose != None:
1289         EdkLogger.SetLevel(EdkLogger.VERBOSE)
1290     elif Option.quiet != None:
1291         EdkLogger.SetLevel(EdkLogger.QUIET)
1292     elif Option.debug != None:
1293         EdkLogger.SetLevel(Option.debug + 1)
1294     else:
1295         EdkLogger.SetLevel(EdkLogger.INFO)
1296
1297     if Option.LogFile != None:
1298         EdkLogger.SetLogFile(Option.LogFile)
1299
1300     if Option.WarningAsError == True:
1301         EdkLogger.SetWarningAsError()
1302
1303     if platform.platform().find("Windows") >= 0:
1304         GlobalData.gIsWindows = True
1305     else:
1306         GlobalData.gIsWindows = False
1307
1308     EdkLogger.quiet(time.strftime("%H:%M:%S, %b.%d %Y ", time.localtime()) + "[%s]\n" % platform.platform())
1309     ReturnCode = 0
1310     MyBuild = None
1311     try:
1312         if len(Target) == 0:
1313             Target = "all"
1314         elif len(Target) >= 2:
1315             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
1316                             ExtraData="Please select one of: %s" %(' '.join(gSupportedTarget)))
1317         else:
1318             Target = Target[0].lower()
1319
1320         if Target not in gSupportedTarget:
1321             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
1322                             ExtraData="Please select one of: %s" %(' '.join(gSupportedTarget)))
1323
1324         # GlobalData.gGlobalDefines = ParseDefines(Option.Defines)
1325         #
1326         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
1327         #
1328         CheckEnvVariable()
1329         Workspace = os.getenv("WORKSPACE")
1330         #
1331         # Get files real name in workspace dir
1332         #
1333         GlobalData.gAllFiles = Utils.DirCache(Workspace)
1334
1335         WorkingDirectory = os.getcwd()
1336         if not Option.ModuleFile:
1337             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
1338             FileNum = len(FileList)
1339             if FileNum >= 2:
1340                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
1341                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
1342             elif FileNum == 1:
1343                 Option.ModuleFile = NormFile(FileList[0], Workspace)
1344
1345         if Option.ModuleFile:
1346             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
1347             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
1348             if ErrorCode != 0:
1349                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1350
1351         if Option.PlatformFile != None:
1352             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
1353             ErrorCode, ErrorInfo = Option.PlatformFile.Validate(".dsc", False)
1354             if ErrorCode != 0:
1355                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1356
1357         if Option.FdfFile != None:
1358             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
1359             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
1360             if ErrorCode != 0:
1361                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1362
1363         MyBuild = Build(Target, Workspace, Option.PlatformFile, Option.ModuleFile,
1364                         Option.TargetArch, Option.ToolChain, Option.BuildTarget,
1365                         Option.FdfFile, Option.RomImage, Option.FvImage,
1366                         None, Option.SilentMode, Option.ThreadNumber,
1367                         Option.SkipAutoGen, Option.Reparse, Option.SkuId)
1368         MyBuild.Launch()
1369         #MyBuild.DumpBuildData()
1370     except FatalError, X:
1371         if MyBuild != None:
1372             # for multi-thread build exits safely
1373             MyBuild.Relinquish()
1374         if Option != None and Option.debug != None:
1375             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1376         ReturnCode = X.args[0]
1377     except Warning, X:
1378         # error from Fdf parser
1379         if MyBuild != None:
1380             # for multi-thread build exits safely
1381             MyBuild.Relinquish()
1382         if Option != None and Option.debug != None:
1383             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1384         else:
1385             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError = False)
1386         ReturnCode = FORMAT_INVALID
1387     except KeyboardInterrupt:
1388         ReturnCode = ABORT_ERROR
1389         if Option != None and Option.debug != None:
1390             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1391     except:
1392         if MyBuild != None:
1393             # for multi-thread build exits safely
1394             MyBuild.Relinquish()
1395
1396         # try to get the meta-file from the object causing exception
1397         Tb = sys.exc_info()[-1]
1398         MetaFile = GlobalData.gProcessingFile
1399         while Tb != None:
1400             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
1401                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
1402             Tb = Tb.tb_next
1403         EdkLogger.error(
1404                     "\nbuild",
1405                     CODE_ERROR,
1406                     "Unknown fatal error when processing [%s]" % MetaFile,
1407                     ExtraData="\n(Please send email to dev@buildtools.tianocore.org for help, attaching following call stack trace!)\n",
1408                     RaiseError=False
1409                     )
1410         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
1411         ReturnCode = CODE_ERROR
1412     finally:
1413         Utils.Progressor.Abort()
1414
1415     if MyBuild != None:
1416         MyBuild.Db.Close()
1417
1418     if ReturnCode == 0:
1419         Conclusion = "Done"
1420     elif ReturnCode == ABORT_ERROR:
1421         Conclusion = "Aborted"
1422     else:
1423         Conclusion = "Failed"
1424     FinishTime = time.time()
1425     BuildDuration = time.strftime("%M:%S", time.gmtime(int(round(FinishTime - StartTime))))
1426     EdkLogger.SetLevel(EdkLogger.QUIET)
1427     EdkLogger.quiet("\n- %s -\n%s [%s]" % (Conclusion, time.strftime("%H:%M:%S, %b.%d %Y", time.localtime()), BuildDuration))
1428
1429     return ReturnCode
1430
1431 if __name__ == '__main__':
1432     r = Main()
1433     ## 0-127 is a safe return range, and 1 is a standard default error
1434     if r < 0 or r > 127: r = 1
1435     sys.exit(r)
1436