BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
   5 #  Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.<BR>
   6 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
   7 #
   8 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   9 #
  10
  11 ##
  12 # Import Modules
  13 #
  14 from __future__ import print_function
  15 from __future__ import absolute_import
  16 import os.path as path
  17 import sys
  18 import os
  19 import re
  20 import glob
  21 import time
  22 import platform
  23 import traceback
  24 import multiprocessing
  25 from threading import Thread,Event,BoundedSemaphore
  26 import threading
  27 from subprocess import Popen,PIPE, STDOUT
  28 from collections import OrderedDict, defaultdict
  29
  30 from AutoGen.PlatformAutoGen import PlatformAutoGen
  31 from AutoGen.ModuleAutoGen import ModuleAutoGen
  32 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
  33 from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\
  34     LogAgent
  35 from AutoGen import GenMake
  36 from Common import Misc as Utils
  37
  38 from Common.TargetTxtClassObject import TargetTxtDict
  39 from Common.ToolDefClassObject import ToolDefDict
  40 from buildoptions import MyOptionParser
  41 from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory
  42 from Common.StringUtils import NormPath
  43 from Common.MultipleWorkspace import MultipleWorkspace as mws
  44 from Common.BuildToolError import *
  45 from Common.DataType import *
  46 import Common.EdkLogger as EdkLogger
  47
  48 from Workspace.WorkspaceDatabase import BuildDB
  49
  50 from BuildReport import BuildReport
  51 from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile
  52 from PatchPcdValue.PatchPcdValue import PatchBinaryFile
  53
  54 import Common.GlobalData as GlobalData
  55 from GenFds.GenFds import GenFds, GenFdsApi
  56 import multiprocessing as mp
  57 from multiprocessing import Manager
  58 from AutoGen.DataPipe import MemoryDataPipe
  59 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
  60 from GenFds.FdfParser import FdfParser
  61 from AutoGen.IncludesAutoGen import IncludesAutoGen
  62 from GenFds.GenFds import resetFdsGlobalVariable
  63
  64 ## standard targets of build command
  65 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  66
  67 ## build configuration file
  68 gBuildConfiguration = "target.txt"
  69 gToolsDefinition = "tools_def.txt"
  70
  71 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
  72 TmpTableDict = {}
  73
  74 ## Check environment PATH variable to make sure the specified tool is found
  75 #
  76 #   If the tool is found in the PATH, then True is returned
  77 #   Otherwise, False is returned
  78 #
  79 def IsToolInPath(tool):
  80     if 'PATHEXT' in os.environ:
  81         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  82     else:
  83         extns = ('',)
  84     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  85         for ext in extns:
  86             if os.path.exists(os.path.join(pathDir, tool + ext)):
  87                 return True
  88     return False
  89
  90 ## Check environment variables
  91 #
  92 #  Check environment variables that must be set for build. Currently they are
  93 #
  94 #   WORKSPACE           The directory all packages/platforms start from
  95 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  96 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  97 #
  98 #   If any of above environment variable is not set or has error, the build
  99 #   will be broken.
 100 #
 101 def CheckEnvVariable():
 102     # check WORKSPACE
 103     if "WORKSPACE" not in os.environ:
 104         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 105                         ExtraData="WORKSPACE")
 106
 107     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
 108     if not os.path.exists(WorkspaceDir):
 109         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData=WorkspaceDir)
 110     elif ' ' in WorkspaceDir:
 111         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
 112                         ExtraData=WorkspaceDir)
 113     os.environ["WORKSPACE"] = WorkspaceDir
 114
 115     # set multiple workspace
 116     PackagesPath = os.getenv("PACKAGES_PATH")
 117     mws.setWs(WorkspaceDir, PackagesPath)
 118     if mws.PACKAGES_PATH:
 119         for Path in mws.PACKAGES_PATH:
 120             if not os.path.exists(Path):
 121                 EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData=Path)
 122             elif ' ' in Path:
 123                 EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path)
 124
 125
 126     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 127
 128     # check EDK_TOOLS_PATH
 129     if "EDK_TOOLS_PATH" not in os.environ:
 130         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 131                         ExtraData="EDK_TOOLS_PATH")
 132
 133     # check PATH
 134     if "PATH" not in os.environ:
 135         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 136                         ExtraData="PATH")
 137
 138     GlobalData.gWorkspace = WorkspaceDir
 139
 140     GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 141     GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"]
 142
 143 ## Get normalized file path
 144 #
 145 # Convert the path to be local format, and remove the WORKSPACE path at the
 146 # beginning if the file path is given in full path.
 147 #
 148 # @param  FilePath      File path to be normalized
 149 # @param  Workspace     Workspace path which the FilePath will be checked against
 150 #
 151 # @retval string        The normalized file path
 152 #
 153 def NormFile(FilePath, Workspace):
 154     # check if the path is absolute or relative
 155     if os.path.isabs(FilePath):
 156         FileFullPath = os.path.normpath(FilePath)
 157     else:
 158         FileFullPath = os.path.normpath(mws.join(Workspace, FilePath))
 159         Workspace = mws.getWs(Workspace, FilePath)
 160
 161     # check if the file path exists or not
 162     if not os.path.isfile(FileFullPath):
 163         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath)
 164
 165     # remove workspace directory from the beginning part of the file path
 166     if Workspace[-1] in ["\\", "/"]:
 167         return FileFullPath[len(Workspace):]
 168     else:
 169         return FileFullPath[(len(Workspace) + 1):]
 170
 171 ## Get the output of an external program
 172 #
 173 # This is the entrance method of thread reading output of an external program and
 174 # putting them in STDOUT/STDERR of current program.
 175 #
 176 # @param  From      The stream message read from
 177 # @param  To        The stream message put on
 178 # @param  ExitFlag  The flag used to indicate stopping reading
 179 #
 180 def ReadMessage(From, To, ExitFlag,MemTo=None):
 181     while True:
 182         # read one line a time
 183         Line = From.readline()
 184         # empty string means "end"
 185         if Line is not None and Line != b"":
 186             LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
 187             if MemTo is not None:
 188                 if "Note: including file:" ==  LineStr.lstrip()[:21]:
 189                     MemTo.append(LineStr)
 190                 else:
 191                     To(LineStr)
 192                     MemTo.append(LineStr)
 193             else:
 194                 To(LineStr)
 195         else:
 196             break
 197         if ExitFlag.isSet():
 198             break
 199
 200 class MakeSubProc(Popen):
 201     def __init__(self,*args, **argv):
 202         super(MakeSubProc,self).__init__(*args, **argv)
 203         self.ProcOut = []
 204
 205 ## Launch an external program
 206 #
 207 # This method will call subprocess.Popen to execute an external program with
 208 # given options in specified directory. Because of the dead-lock issue during
 209 # redirecting output of the external program, threads are used to to do the
 210 # redirection work.
 211 #
 212 # @param  Command               A list or string containing the call of the program
 213 # @param  WorkingDir            The directory in which the program will be running
 214 #
 215 def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
 216     BeginTime = time.time()
 217     # if working directory doesn't exist, Popen() will raise an exception
 218     if not os.path.isdir(WorkingDir):
 219         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 220
 221     # Command is used as the first Argument in following Popen().
 222     # It could be a string or sequence. We find that if command is a string in following Popen(),
 223     # ubuntu may fail with an error message that the command is not found.
 224     # So here we may need convert command from string to list instance.
 225     if platform.system() != 'Windows':
 226         if not isinstance(Command, list):
 227             Command = Command.split()
 228         Command = ' '.join(Command)
 229
 230     Proc = None
 231     EndOfProcedure = None
 232     try:
 233         # launch the command
 234         Proc = MakeSubProc(Command, stdout=PIPE, stderr=STDOUT, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 235
 236         # launch two threads to read the STDOUT and STDERR
 237         EndOfProcedure = Event()
 238         EndOfProcedure.clear()
 239         if Proc.stdout:
 240             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
 241             StdOutThread.setName("STDOUT-Redirector")
 242             StdOutThread.setDaemon(False)
 243             StdOutThread.start()
 244
 245
 246         # waiting for program exit
 247         Proc.wait()
 248     except: # in case of aborting
 249         # terminate the threads redirecting the program output
 250         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 251         if EndOfProcedure is not None:
 252             EndOfProcedure.set()
 253         if Proc is None:
 254             if not isinstance(Command, type("")):
 255                 Command = " ".join(Command)
 256             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 257
 258     if Proc.stdout:
 259         StdOutThread.join()
 260
 261     # check the return code of the program
 262     if Proc.returncode != 0:
 263         if not isinstance(Command, type("")):
 264             Command = " ".join(Command)
 265         # print out the Response file and its content when make failure
 266         RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
 267         if os.path.isfile(RespFile):
 268             f = open(RespFile)
 269             RespContent = f.read()
 270             f.close()
 271             EdkLogger.info(RespContent)
 272
 273         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 274     if ModuleAuto:
 275         iau = IncludesAutoGen(WorkingDir,ModuleAuto)
 276         if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
 277             iau.CreateDepsFileForMsvc(Proc.ProcOut)
 278         else:
 279             iau.UpdateDepsFileforNonMsvc()
 280         iau.UpdateDepsFileforTrim()
 281         iau.CreateModuleDeps()
 282         iau.CreateDepsInclude()
 283         iau.CreateDepsTarget()
 284     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
 285
 286 ## The smallest unit that can be built in multi-thread build mode
 287 #
 288 # This is the base class of build unit. The "Obj" parameter must provide
 289 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 290 # missing build.
 291 #
 292 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 293 #
 294 class BuildUnit:
 295     ## The constructor
 296     #
 297     #   @param  self        The object pointer
 298     #   @param  Obj         The object the build is working on
 299     #   @param  Target      The build target name, one of gSupportedTarget
 300     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 301     #   @param  WorkingDir  The directory build command starts in
 302     #
 303     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 304         self.BuildObject = Obj
 305         self.Dependency = Dependency
 306         self.WorkingDir = WorkingDir
 307         self.Target = Target
 308         self.BuildCommand = BuildCommand
 309         if not BuildCommand:
 310             EdkLogger.error("build", OPTION_MISSING,
 311                             "No build command found for this module. "
 312                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
 313                                 (Obj.BuildTarget, Obj.ToolChain, Obj.Arch),
 314                             ExtraData=str(Obj))
 315
 316
 317     ## str() method
 318     #
 319     #   It just returns the string representation of self.BuildObject
 320     #
 321     #   @param  self        The object pointer
 322     #
 323     def __str__(self):
 324         return str(self.BuildObject)
 325
 326     ## "==" operator method
 327     #
 328     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 329     #   provide its own __eq__() method.
 330     #
 331     #   @param  self        The object pointer
 332     #   @param  Other       The other BuildUnit object compared to
 333     #
 334     def __eq__(self, Other):
 335         return Other and self.BuildObject == Other.BuildObject \
 336                 and Other.BuildObject \
 337                 and self.BuildObject.Arch == Other.BuildObject.Arch
 338
 339     ## hash() method
 340     #
 341     #   It just returns the hash value of self.BuildObject which must be hashable.
 342     #
 343     #   @param  self        The object pointer
 344     #
 345     def __hash__(self):
 346         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 347
 348     def __repr__(self):
 349         return repr(self.BuildObject)
 350
 351 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 352 #
 353 # This class is for module build by nmake/make build system. The "Obj" parameter
 354 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 355 # be make units missing build.
 356 #
 357 # Currently the "Obj" should be only ModuleAutoGen object.
 358 #
 359 class ModuleMakeUnit(BuildUnit):
 360     ## The constructor
 361     #
 362     #   @param  self        The object pointer
 363     #   @param  Obj         The ModuleAutoGen object the build is working on
 364     #   @param  Target      The build target name, one of gSupportedTarget
 365     #
 366     def __init__(self, Obj, BuildCommand,Target):
 367         Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]
 368         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 369         if Target in [None, "", "all"]:
 370             self.Target = "tbuild"
 371
 372 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 373 #
 374 # This class is for platform build by nmake/make build system. The "Obj" parameter
 375 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 376 # be make units missing build.
 377 #
 378 # Currently the "Obj" should be only PlatformAutoGen object.
 379 #
 380 class PlatformMakeUnit(BuildUnit):
 381     ## The constructor
 382     #
 383     #   @param  self        The object pointer
 384     #   @param  Obj         The PlatformAutoGen object the build is working on
 385     #   @param  Target      The build target name, one of gSupportedTarget
 386     #
 387     def __init__(self, Obj, BuildCommand, Target):
 388         Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 389         Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])
 390         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 391
 392 ## The class representing the task of a module build or platform build
 393 #
 394 # This class manages the build tasks in multi-thread build mode. Its jobs include
 395 # scheduling thread running, catching thread error, monitor the thread status, etc.
 396 #
 397 class BuildTask:
 398     # queue for tasks waiting for schedule
 399     _PendingQueue = OrderedDict()
 400     _PendingQueueLock = threading.Lock()
 401
 402     # queue for tasks ready for running
 403     _ReadyQueue = OrderedDict()
 404     _ReadyQueueLock = threading.Lock()
 405
 406     # queue for run tasks
 407     _RunningQueue = OrderedDict()
 408     _RunningQueueLock = threading.Lock()
 409
 410     # queue containing all build tasks, in case duplicate build
 411     _TaskQueue = OrderedDict()
 412
 413     # flag indicating error occurs in a running thread
 414     _ErrorFlag = threading.Event()
 415     _ErrorFlag.clear()
 416     _ErrorMessage = ""
 417
 418     # BoundedSemaphore object used to control the number of running threads
 419     _Thread = None
 420
 421     # flag indicating if the scheduler is started or not
 422     _SchedulerStopped = threading.Event()
 423     _SchedulerStopped.set()
 424
 425     ## Start the task scheduler thread
 426     #
 427     #   @param  MaxThreadNumber     The maximum thread number
 428     #   @param  ExitFlag            Flag used to end the scheduler
 429     #
 430     @staticmethod
 431     def StartScheduler(MaxThreadNumber, ExitFlag):
 432         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 433         SchedulerThread.setName("Build-Task-Scheduler")
 434         SchedulerThread.setDaemon(False)
 435         SchedulerThread.start()
 436         # wait for the scheduler to be started, especially useful in Linux
 437         while not BuildTask.IsOnGoing():
 438             time.sleep(0.01)
 439
 440     ## Scheduler method
 441     #
 442     #   @param  MaxThreadNumber     The maximum thread number
 443     #   @param  ExitFlag            Flag used to end the scheduler
 444     #
 445     @staticmethod
 446     def Scheduler(MaxThreadNumber, ExitFlag):
 447         BuildTask._SchedulerStopped.clear()
 448         try:
 449             # use BoundedSemaphore to control the maximum running threads
 450             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 451             #
 452             # scheduling loop, which will exits when no pending/ready task and
 453             # indicated to do so, or there's error in running thread
 454             #
 455             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 456                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 457                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 458                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 459
 460                 # get all pending tasks
 461                 BuildTask._PendingQueueLock.acquire()
 462                 BuildObjectList = list(BuildTask._PendingQueue.keys())
 463                 #
 464                 # check if their dependency is resolved, and if true, move them
 465                 # into ready queue
 466                 #
 467                 for BuildObject in BuildObjectList:
 468                     Bt = BuildTask._PendingQueue[BuildObject]
 469                     if Bt.IsReady():
 470                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 471                 BuildTask._PendingQueueLock.release()
 472
 473                 # launch build thread until the maximum number of threads is reached
 474                 while not BuildTask._ErrorFlag.isSet():
 475                     # empty ready queue, do nothing further
 476                     if len(BuildTask._ReadyQueue) == 0:
 477                         break
 478
 479                     # wait for active thread(s) exit
 480                     BuildTask._Thread.acquire(True)
 481
 482                     # start a new build thread
 483                     Bo, Bt = BuildTask._ReadyQueue.popitem()
 484
 485                     # move into running queue
 486                     BuildTask._RunningQueueLock.acquire()
 487                     BuildTask._RunningQueue[Bo] = Bt
 488                     BuildTask._RunningQueueLock.release()
 489
 490                     Bt.Start()
 491                     # avoid tense loop
 492                     time.sleep(0.01)
 493
 494                 # avoid tense loop
 495                 time.sleep(0.01)
 496
 497             # wait for all running threads exit
 498             if BuildTask._ErrorFlag.isSet():
 499                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 500             # while not BuildTask._ErrorFlag.isSet() and \
 501             while len(BuildTask._RunningQueue) > 0:
 502                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 503                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))
 504                 # avoid tense loop
 505                 time.sleep(0.1)
 506         except BaseException as X:
 507             #
 508             # TRICK: hide the output of threads left running, so that the user can
 509             #        catch the error message easily
 510             #
 511             EdkLogger.SetLevel(EdkLogger.ERROR)
 512             BuildTask._ErrorFlag.set()
 513             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 514
 515         BuildTask._PendingQueue.clear()
 516         BuildTask._ReadyQueue.clear()
 517         BuildTask._RunningQueue.clear()
 518         BuildTask._TaskQueue.clear()
 519         BuildTask._SchedulerStopped.set()
 520
 521     ## Wait for all running method exit
 522     #
 523     @staticmethod
 524     def WaitForComplete():
 525         BuildTask._SchedulerStopped.wait()
 526
 527     ## Check if the scheduler is running or not
 528     #
 529     @staticmethod
 530     def IsOnGoing():
 531         return not BuildTask._SchedulerStopped.isSet()
 532
 533     ## Abort the build
 534     @staticmethod
 535     def Abort():
 536         if BuildTask.IsOnGoing():
 537             BuildTask._ErrorFlag.set()
 538             BuildTask.WaitForComplete()
 539
 540     ## Check if there's error in running thread
 541     #
 542     #   Since the main thread cannot catch exceptions in other thread, we have to
 543     #   use threading.Event to communicate this formation to main thread.
 544     #
 545     @staticmethod
 546     def HasError():
 547         return BuildTask._ErrorFlag.isSet()
 548
 549     ## Get error message in running thread
 550     #
 551     #   Since the main thread cannot catch exceptions in other thread, we have to
 552     #   use a static variable to communicate this message to main thread.
 553     #
 554     @staticmethod
 555     def GetErrorMessage():
 556         return BuildTask._ErrorMessage
 557
 558     ## Factory method to create a BuildTask object
 559     #
 560     #   This method will check if a module is building or has been built. And if
 561     #   true, just return the associated BuildTask object in the _TaskQueue. If
 562     #   not, create and return a new BuildTask object. The new BuildTask object
 563     #   will be appended to the _PendingQueue for scheduling later.
 564     #
 565     #   @param  BuildItem       A BuildUnit object representing a build object
 566     #   @param  Dependency      The dependent build object of BuildItem
 567     #
 568     @staticmethod
 569     def New(BuildItem, Dependency=None):
 570         if BuildItem in BuildTask._TaskQueue:
 571             Bt = BuildTask._TaskQueue[BuildItem]
 572             return Bt
 573
 574         Bt = BuildTask()
 575         Bt._Init(BuildItem, Dependency)
 576         BuildTask._TaskQueue[BuildItem] = Bt
 577
 578         BuildTask._PendingQueueLock.acquire()
 579         BuildTask._PendingQueue[BuildItem] = Bt
 580         BuildTask._PendingQueueLock.release()
 581
 582         return Bt
 583
 584     ## The real constructor of BuildTask
 585     #
 586     #   @param  BuildItem       A BuildUnit object representing a build object
 587     #   @param  Dependency      The dependent build object of BuildItem
 588     #
 589     def _Init(self, BuildItem, Dependency=None):
 590         self.BuildItem = BuildItem
 591
 592         self.DependencyList = []
 593         if Dependency is None:
 594             Dependency = BuildItem.Dependency
 595         else:
 596             Dependency.extend(BuildItem.Dependency)
 597         self.AddDependency(Dependency)
 598         # flag indicating build completes, used to avoid unnecessary re-build
 599         self.CompleteFlag = False
 600
 601     ## Check if all dependent build tasks are completed or not
 602     #
 603     def IsReady(self):
 604         ReadyFlag = True
 605         for Dep in self.DependencyList:
 606             if Dep.CompleteFlag == True:
 607                 continue
 608             ReadyFlag = False
 609             break
 610
 611         return ReadyFlag
 612
 613     ## Add dependent build task
 614     #
 615     #   @param  Dependency      The list of dependent build objects
 616     #
 617     def AddDependency(self, Dependency):
 618         for Dep in Dependency:
 619             if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):
 620                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 621
 622     ## The thread wrapper of LaunchCommand function
 623     #
 624     # @param  Command               A list or string contains the call of the command
 625     # @param  WorkingDir            The directory in which the program will be running
 626     #
 627     def _CommandThread(self, Command, WorkingDir):
 628         try:
 629             self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
 630             self.CompleteFlag = True
 631
 632             # Run hash operation post dependency to account for libs
 633             # Run if --hash or --binary-destination
 634             if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
 635                 self.BuildItem.BuildObject.GenModuleHash()
 636             if GlobalData.gBinCacheDest:
 637                 self.BuildItem.BuildObject.GenCMakeHash()
 638
 639         except:
 640             #
 641             # TRICK: hide the output of threads left running, so that the user can
 642             #        catch the error message easily
 643             #
 644             if not BuildTask._ErrorFlag.isSet():
 645                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 646                                                                   self.BuildItem.BuildObject.Arch,
 647                                                                   self.BuildItem.BuildObject.ToolChain,
 648                                                                   self.BuildItem.BuildObject.BuildTarget
 649                                                                  )
 650             EdkLogger.SetLevel(EdkLogger.ERROR)
 651             BuildTask._ErrorFlag.set()
 652             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 653                                       (threading.currentThread().getName(), Command, WorkingDir)
 654
 655         # indicate there's a thread is available for another build task
 656         BuildTask._RunningQueueLock.acquire()
 657         BuildTask._RunningQueue.pop(self.BuildItem)
 658         BuildTask._RunningQueueLock.release()
 659         BuildTask._Thread.release()
 660
 661     ## Start build task thread
 662     #
 663     def Start(self):
 664         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 665         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 666         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 667         self.BuildTread.setName("build thread")
 668         self.BuildTread.setDaemon(False)
 669         self.BuildTread.start()
 670
 671 ## The class contains the information related to EFI image
 672 #
 673 class PeImageInfo():
 674     ## Constructor
 675     #
 676     # Constructor will load all required image information.
 677     #
 678     #   @param  BaseName          The full file path of image.
 679     #   @param  Guid              The GUID for image.
 680     #   @param  Arch              Arch of this image.
 681     #   @param  OutputDir         The output directory for image.
 682     #   @param  DebugDir          The debug directory for image.
 683     #   @param  ImageClass        PeImage Information
 684     #
 685     def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass):
 686         self.BaseName         = BaseName
 687         self.Guid             = Guid
 688         self.Arch             = Arch
 689         self.OutputDir        = OutputDir
 690         self.DebugDir         = DebugDir
 691         self.Image            = ImageClass
 692         self.Image.Size       = (self.Image.Size // 0x1000 + 1) * 0x1000
 693
 694 ## The class implementing the EDK2 build process
 695 #
 696 #   The build process includes:
 697 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 698 #       2. Parse DSC file of active platform
 699 #       3. Parse FDF file if any
 700 #       4. Establish build database, including parse all other files (module, package)
 701 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 702 #       6. Call build command
 703 #
 704 class Build():
 705     ## Constructor
 706     #
 707     # Constructor will load all necessary configurations, parse platform, modules
 708     # and packages and the establish a database for AutoGen.
 709     #
 710     #   @param  Target              The build command target, one of gSupportedTarget
 711     #   @param  WorkspaceDir        The directory of workspace
 712     #   @param  BuildOptions        Build options passed from command line
 713     #
 714     def __init__(self, Target, WorkspaceDir, BuildOptions,log_q):
 715         self.WorkspaceDir   = WorkspaceDir
 716         self.Target         = Target
 717         self.PlatformFile   = BuildOptions.PlatformFile
 718         self.ModuleFile     = BuildOptions.ModuleFile
 719         self.ArchList       = BuildOptions.TargetArch
 720         self.ToolChainList  = BuildOptions.ToolChain
 721         self.BuildTargetList= BuildOptions.BuildTarget
 722         self.Fdf            = BuildOptions.FdfFile
 723         self.FdList         = BuildOptions.RomImage
 724         self.FvList         = BuildOptions.FvImage
 725         self.CapList        = BuildOptions.CapName
 726         self.SilentMode     = BuildOptions.SilentMode
 727         self.ThreadNumber   = 1
 728         self.SkipAutoGen    = BuildOptions.SkipAutoGen
 729         self.Reparse        = BuildOptions.Reparse
 730         self.SkuId          = BuildOptions.SkuId
 731         if self.SkuId:
 732             GlobalData.gSKUID_CMD = self.SkuId
 733         self.ConfDirectory = BuildOptions.ConfDirectory
 734         self.SpawnMode      = True
 735         self.BuildReport    = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)
 736         self.AutoGenTime    = 0
 737         self.MakeTime       = 0
 738         self.GenFdsTime     = 0
 739         TargetObj = TargetTxtDict()
 740         ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"),"Conf")))
 741         self.TargetTxt = TargetObj.Target
 742         self.ToolDef = ToolDefObj.ToolDef
 743         GlobalData.BuildOptionPcd     = BuildOptions.OptionPcd if BuildOptions.OptionPcd else []
 744         #Set global flag for build mode
 745         GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
 746         GlobalData.gUseHashCache = BuildOptions.UseHashCache
 747         GlobalData.gBinCacheDest   = BuildOptions.BinCacheDest
 748         GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
 749         GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread
 750         GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
 751
 752         if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
 753             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
 754
 755         if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
 756             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.")
 757
 758         if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
 759             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.")
 760
 761         if GlobalData.gBinCacheSource:
 762             BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
 763             if not os.path.isabs(BinCacheSource):
 764                 BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
 765             GlobalData.gBinCacheSource = BinCacheSource
 766         else:
 767             if GlobalData.gBinCacheSource is not None:
 768                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-source.")
 769
 770         if GlobalData.gBinCacheDest:
 771             BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
 772             if not os.path.isabs(BinCacheDest):
 773                 BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
 774             GlobalData.gBinCacheDest = BinCacheDest
 775         else:
 776             if GlobalData.gBinCacheDest is not None:
 777                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-destination.")
 778
 779         GlobalData.gDatabasePath = os.path.normpath(os.path.join(GlobalData.gConfDirectory, GlobalData.gDatabasePath))
 780         if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):
 781             os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))
 782         self.Db = BuildDB
 783         self.BuildDatabase = self.Db.BuildObject
 784         self.Platform = None
 785         self.ToolChainFamily = None
 786         self.LoadFixAddress = 0
 787         self.UniFlag        = BuildOptions.Flag
 788         self.BuildModules = []
 789         self.HashSkipModules = []
 790         self.Db_Flag = False
 791         self.LaunchPrebuildFlag = False
 792         self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory, '.cache', '.PlatformBuild')
 793         if BuildOptions.CommandLength:
 794             GlobalData.gCommandMaxLength = BuildOptions.CommandLength
 795
 796         # print dot character during doing some time-consuming work
 797         self.Progress = Utils.Progressor()
 798         # print current build environment and configuration
 799         EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 800         if "PACKAGES_PATH" in os.environ:
 801             # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
 802             EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"]))))
 803         EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 804         if "EDK_TOOLS_BIN" in os.environ:
 805             # Print the same path style with WORKSPACE env.
 806             EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
 807         EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
 808         if "PYTHON3_ENABLE" in os.environ:
 809             PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]
 810             if PYTHON3_ENABLE != "TRUE":
 811                 PYTHON3_ENABLE = "FALSE"
 812             EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))
 813         if "PYTHON_COMMAND" in os.environ:
 814             EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))
 815         self.InitPreBuild()
 816         self.InitPostBuild()
 817         if self.Prebuild:
 818             EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.Prebuild))
 819         if self.Postbuild:
 820             EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))
 821         if self.Prebuild:
 822             self.LaunchPrebuild()
 823             TargetObj = TargetTxtDict()
 824             ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"), "Conf")))
 825             self.TargetTxt = TargetObj.Target
 826             self.ToolDef = ToolDefObj.ToolDef
 827         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
 828             self.InitBuild()
 829
 830         self.AutoGenMgr = None
 831         EdkLogger.info("")
 832         os.chdir(self.WorkspaceDir)
 833         self.log_q = log_q
 834         GlobalData.file_lock =  mp.Lock()
 835         # Init cache data for local only
 836         GlobalData.gPackageHashFile = dict()
 837         GlobalData.gModulePreMakeCacheStatus = dict()
 838         GlobalData.gModuleMakeCacheStatus = dict()
 839         GlobalData.gHashChainStatus = dict()
 840         GlobalData.gCMakeHashFile = dict()
 841         GlobalData.gModuleHashFile = dict()
 842         GlobalData.gFileHashDict = dict()
 843         GlobalData.gModuleAllCacheStatus = set()
 844         GlobalData.gModuleCacheHit = set()
 845
 846     def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):
 847         try:
 848             if SkipAutoGen:
 849                 return True,0
 850             feedback_q = mp.Queue()
 851             error_event = mp.Event()
 852             FfsCmd = DataPipe.Get("FfsCommand")
 853             if FfsCmd is None:
 854                 FfsCmd = {}
 855             GlobalData.FfsCmd = FfsCmd
 856             auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]
 857             self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
 858             self.AutoGenMgr.start()
 859             for w in auto_workers:
 860                 w.start()
 861             if PcdMaList is not None:
 862                 for PcdMa in PcdMaList:
 863                     # SourceFileList calling sequence impact the makefile string sequence.
 864                     # Create cached SourceFileList here to unify its calling sequence for both
 865                     # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
 866                     RetVal = PcdMa.SourceFileList
 867                     # Force cache miss for PCD driver
 868                     if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
 869                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))
 870
 871                     PcdMa.CreateCodeFile(False)
 872                     PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
 873
 874                     # Force cache miss for PCD driver
 875                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
 876                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))
 877
 878             self.AutoGenMgr.join()
 879             rt = self.AutoGenMgr.Status
 880             return rt, 0
 881         except FatalError as e:
 882             return False, e.args[0]
 883         except:
 884             return False, UNKNOWN_ERROR
 885
 886     ## Load configuration
 887     #
 888     #   This method will parse target.txt and get the build configurations.
 889     #
 890     def LoadConfiguration(self):
 891
 892         # if no ARCH given in command line, get it from target.txt
 893         if not self.ArchList:
 894             self.ArchList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET_ARCH]
 895         self.ArchList = tuple(self.ArchList)
 896
 897         # if no build target given in command line, get it from target.txt
 898         if not self.BuildTargetList:
 899             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
 900
 901         # if no tool chain given in command line, get it from target.txt
 902         if not self.ToolChainList:
 903             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 904             if self.ToolChainList is None or len(self.ToolChainList) == 0:
 905                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 906
 907         # check if the tool chains are defined or not
 908         NewToolChainList = []
 909         for ToolChain in self.ToolChainList:
 910             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 911                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 912             else:
 913                 NewToolChainList.append(ToolChain)
 914         # if no tool chain available, break the build
 915         if len(NewToolChainList) == 0:
 916             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 917                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 918         else:
 919             self.ToolChainList = NewToolChainList
 920
 921         ToolChainFamily = []
 922         ToolDefinition = self.ToolDef.ToolsDefTxtDatabase
 923         for Tool in self.ToolChainList:
 924             if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \
 925                or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]:
 926                 EdkLogger.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool)
 927                 ToolChainFamily.append(TAB_COMPILER_MSFT)
 928             else:
 929                 ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
 930         self.ToolChainFamily = ToolChainFamily
 931
 932         if not self.PlatformFile:
 933             PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 934             if not PlatformFile:
 935                 # Try to find one in current directory
 936                 WorkingDirectory = os.getcwd()
 937                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 938                 FileNum = len(FileList)
 939                 if FileNum >= 2:
 940                     EdkLogger.error("build", OPTION_MISSING,
 941                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 942                 elif FileNum == 1:
 943                     PlatformFile = FileList[0]
 944                 else:
 945                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 946                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 947
 948             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 949         self.ThreadNumber   = ThreadNum()
 950     ## Initialize build configuration
 951     #
 952     #   This method will parse DSC file and merge the configurations from
 953     #   command line and target.txt, then get the final build configurations.
 954     #
 955     def InitBuild(self):
 956         # parse target.txt, tools_def.txt, and platform file
 957         self.LoadConfiguration()
 958
 959         # Allow case-insensitive for those from command line or configuration file
 960         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 961         if ErrorCode != 0:
 962             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 963
 964
 965     def InitPreBuild(self):
 966         self.LoadConfiguration()
 967         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 968         if ErrorCode != 0:
 969             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 970         if self.BuildTargetList:
 971             GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0]
 972         if self.ArchList:
 973             GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0]
 974         if self.ToolChainList:
 975             GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0]
 976             GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0]
 977         if self.ToolChainFamily:
 978             GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
 979         if 'PREBUILD' in GlobalData.gCommandLineDefines:
 980             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
 981         else:
 982             self.Db_Flag = True
 983             Platform = self.Db.MapPlatform(str(self.PlatformFile))
 984             self.Prebuild = str(Platform.Prebuild)
 985         if self.Prebuild:
 986             PrebuildList = []
 987             #
 988             # Evaluate all arguments and convert arguments that are WORKSPACE
 989             # relative paths to absolute paths.  Filter arguments that look like
 990             # flags or do not follow the file/dir naming rules to avoid false
 991             # positives on this conversion.
 992             #
 993             for Arg in self.Prebuild.split():
 994                 #
 995                 # Do not modify Arg if it looks like a flag or an absolute file path
 996                 #
 997                 if Arg.startswith('-')  or os.path.isabs(Arg):
 998                     PrebuildList.append(Arg)
 999                     continue
1000                 #
1001                 # Do not modify Arg if it does not look like a Workspace relative
1002                 # path that starts with a valid package directory name
1003                 #
1004                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1005                     PrebuildList.append(Arg)
1006                     continue
1007                 #
1008                 # If Arg looks like a WORKSPACE relative path, then convert to an
1009                 # absolute path and check to see if the file exists.
1010                 #
1011                 Temp = mws.join(self.WorkspaceDir, Arg)
1012                 if os.path.isfile(Temp):
1013                     Arg = Temp
1014                 PrebuildList.append(Arg)
1015             self.Prebuild       = ' '.join(PrebuildList)
1016             self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1017
1018     def InitPostBuild(self):
1019         if 'POSTBUILD' in GlobalData.gCommandLineDefines:
1020             self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD')
1021         else:
1022             Platform = self.Db.MapPlatform(str(self.PlatformFile))
1023             self.Postbuild = str(Platform.Postbuild)
1024         if self.Postbuild:
1025             PostbuildList = []
1026             #
1027             # Evaluate all arguments and convert arguments that are WORKSPACE
1028             # relative paths to absolute paths.  Filter arguments that look like
1029             # flags or do not follow the file/dir naming rules to avoid false
1030             # positives on this conversion.
1031             #
1032             for Arg in self.Postbuild.split():
1033                 #
1034                 # Do not modify Arg if it looks like a flag or an absolute file path
1035                 #
1036                 if Arg.startswith('-')  or os.path.isabs(Arg):
1037                     PostbuildList.append(Arg)
1038                     continue
1039                 #
1040                 # Do not modify Arg if it does not look like a Workspace relative
1041                 # path that starts with a valid package directory name
1042                 #
1043                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1044                     PostbuildList.append(Arg)
1045                     continue
1046                 #
1047                 # If Arg looks like a WORKSPACE relative path, then convert to an
1048                 # absolute path and check to see if the file exists.
1049                 #
1050                 Temp = mws.join(self.WorkspaceDir, Arg)
1051                 if os.path.isfile(Temp):
1052                     Arg = Temp
1053                 PostbuildList.append(Arg)
1054             self.Postbuild       = ' '.join(PostbuildList)
1055             self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1056
1057     def PassCommandOption(self, BuildTarget, TargetArch, ToolChain, PlatformFile, Target):
1058         BuildStr = ''
1059         if GlobalData.gCommand and isinstance(GlobalData.gCommand, list):
1060             BuildStr += ' ' + ' '.join(GlobalData.gCommand)
1061         TargetFlag = False
1062         ArchFlag = False
1063         ToolChainFlag = False
1064         PlatformFileFlag = False
1065
1066         if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget:
1067             TargetFlag = True
1068         if GlobalData.gOptions and not GlobalData.gOptions.TargetArch:
1069             ArchFlag = True
1070         if GlobalData.gOptions and not GlobalData.gOptions.ToolChain:
1071             ToolChainFlag = True
1072         if GlobalData.gOptions and not GlobalData.gOptions.PlatformFile:
1073             PlatformFileFlag = True
1074
1075         if TargetFlag and BuildTarget:
1076             if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple):
1077                 BuildStr += ' -b ' + ' -b '.join(BuildTarget)
1078             elif isinstance(BuildTarget, str):
1079                 BuildStr += ' -b ' + BuildTarget
1080         if ArchFlag and TargetArch:
1081             if isinstance(TargetArch, list) or isinstance(TargetArch, tuple):
1082                 BuildStr += ' -a ' + ' -a '.join(TargetArch)
1083             elif isinstance(TargetArch, str):
1084                 BuildStr += ' -a ' + TargetArch
1085         if ToolChainFlag and ToolChain:
1086             if isinstance(ToolChain, list) or isinstance(ToolChain, tuple):
1087                 BuildStr += ' -t ' + ' -t '.join(ToolChain)
1088             elif isinstance(ToolChain, str):
1089                 BuildStr += ' -t ' + ToolChain
1090         if PlatformFileFlag and PlatformFile:
1091             if isinstance(PlatformFile, list) or isinstance(PlatformFile, tuple):
1092                 BuildStr += ' -p ' + ' -p '.join(PlatformFile)
1093             elif isinstance(PlatformFile, str):
1094                 BuildStr += ' -p' + PlatformFile
1095         BuildStr += ' --conf=' + GlobalData.gConfDirectory
1096         if Target:
1097             BuildStr += ' ' + Target
1098
1099         return BuildStr
1100
1101     def LaunchPrebuild(self):
1102         if self.Prebuild:
1103             EdkLogger.info("\n- Prebuild Start -\n")
1104             self.LaunchPrebuildFlag = True
1105             #
1106             # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1107             # and preserve them for the rest of the main build step, because the child process environment will
1108             # evaporate as soon as it exits, we cannot get it in build step.
1109             #
1110             PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory, '.cache', '.PrebuildEnv')
1111             if os.path.isfile(PrebuildEnvFile):
1112                 os.remove(PrebuildEnvFile)
1113             if os.path.isfile(self.PlatformBuildPath):
1114                 os.remove(self.PlatformBuildPath)
1115             if sys.platform == "win32":
1116                 args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile))
1117                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1118             else:
1119                 args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile))
1120                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1121
1122             # launch two threads to read the STDOUT and STDERR
1123             EndOfProcedure = Event()
1124             EndOfProcedure.clear()
1125             if Process.stdout:
1126                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1127                 StdOutThread.setName("STDOUT-Redirector")
1128                 StdOutThread.setDaemon(False)
1129                 StdOutThread.start()
1130
1131             if Process.stderr:
1132                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1133                 StdErrThread.setName("STDERR-Redirector")
1134                 StdErrThread.setDaemon(False)
1135                 StdErrThread.start()
1136             # waiting for program exit
1137             Process.wait()
1138
1139             if Process.stdout:
1140                 StdOutThread.join()
1141             if Process.stderr:
1142                 StdErrThread.join()
1143             if Process.returncode != 0 :
1144                 EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!')
1145
1146             if os.path.exists(PrebuildEnvFile):
1147                 f = open(PrebuildEnvFile)
1148                 envs = f.readlines()
1149                 f.close()
1150                 envs = [l.split("=", 1) for l in envs ]
1151                 envs = [[I.strip() for I in item] for item in envs if len(item) == 2]
1152                 os.environ.update(dict(envs))
1153             EdkLogger.info("\n- Prebuild Done -\n")
1154
1155     def LaunchPostbuild(self):
1156         if self.Postbuild:
1157             EdkLogger.info("\n- Postbuild Start -\n")
1158             if sys.platform == "win32":
1159                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1160             else:
1161                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1162             # launch two threads to read the STDOUT and STDERR
1163             EndOfProcedure = Event()
1164             EndOfProcedure.clear()
1165             if Process.stdout:
1166                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1167                 StdOutThread.setName("STDOUT-Redirector")
1168                 StdOutThread.setDaemon(False)
1169                 StdOutThread.start()
1170
1171             if Process.stderr:
1172                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1173                 StdErrThread.setName("STDERR-Redirector")
1174                 StdErrThread.setDaemon(False)
1175                 StdErrThread.start()
1176             # waiting for program exit
1177             Process.wait()
1178
1179             if Process.stdout:
1180                 StdOutThread.join()
1181             if Process.stderr:
1182                 StdErrThread.join()
1183             if Process.returncode != 0 :
1184                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
1185             EdkLogger.info("\n- Postbuild Done -\n")
1186
1187     ## Build a module or platform
1188     #
1189     # Create autogen code and makefile for a module or platform, and the launch
1190     # "make" command to build it
1191     #
1192     #   @param  Target                      The target of build command
1193     #   @param  Platform                    The platform file
1194     #   @param  Module                      The module file
1195     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1196     #   @param  ToolChain                   The name of toolchain to build
1197     #   @param  Arch                        The arch of the module/platform
1198     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1199     #                                       for dependent modules/Libraries
1200     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1201     #                                       for dependent modules/Libraries
1202     #
1203     def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):
1204         if AutoGenObject is None:
1205             return False
1206         if FfsCommand is None:
1207             FfsCommand = {}
1208         # skip file generation for cleanxxx targets, run and fds target
1209         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1210             # for target which must generate AutoGen code and makefile
1211             mqueue = mp.Queue()
1212             for m in AutoGenObject.GetAllModuleInfo:
1213                 mqueue.put(m)
1214
1215             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
1216             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
1217             AutoGenObject.CreateLibModuelDirs()
1218             AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
1219             AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
1220             AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
1221             self.Progress.Start("Generating makefile and code")
1222             data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
1223             AutoGenObject.DataPipe.dump(data_pipe_file)
1224             cqueue = mp.Queue()
1225             autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
1226             AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
1227             with open(AutoGenIdFile,"w") as fw:
1228                 fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
1229                 fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
1230                 fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
1231             self.Progress.Stop("done!")
1232             if not autogen_rt:
1233                 self.AutoGenMgr.TerminateWorkers()
1234                 self.AutoGenMgr.join(1)
1235                 raise FatalError(errorcode)
1236             AutoGenObject.CreateCodeFile(False)
1237             AutoGenObject.CreateMakeFile(False)
1238         else:
1239             # always recreate top/platform makefile when clean, just in case of inconsistency
1240             AutoGenObject.CreateCodeFile(True)
1241             AutoGenObject.CreateMakeFile(True)
1242
1243         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1244             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1245
1246         BuildCommand = AutoGenObject.BuildCommand
1247         if BuildCommand is None or len(BuildCommand) == 0:
1248             EdkLogger.error("build", OPTION_MISSING,
1249                             "No build command found for this module. "
1250                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1251                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1252                             ExtraData=str(AutoGenObject))
1253
1254         makefile = GenMake.BuildFile(AutoGenObject)._FILE_NAME_[GenMake.gMakeType]
1255
1256         # run
1257         if Target == 'run':
1258             return True
1259
1260         # build modules
1261         if BuildModule:
1262             BuildCommand = BuildCommand + [Target]
1263             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1264             self.CreateAsBuiltInf()
1265             if GlobalData.gBinCacheDest:
1266                 self.GenDestCache()
1267             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1268                 # Only for --hash
1269                 # Update PreMakeCacheChain files
1270                 self.GenLocalPreMakeCache()
1271             self.BuildModules = []
1272             return True
1273
1274         # build library
1275         if Target == 'libraries':
1276             DirList = []
1277             for Lib in AutoGenObject.LibraryAutoGenList:
1278                 if not Lib.IsBinaryModule:
1279                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1280             for Lib, LibAutoGen in DirList:
1281                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
1282                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1283             return True
1284
1285         # build module
1286         if Target == 'modules':
1287             DirList = []
1288             for Lib in AutoGenObject.LibraryAutoGenList:
1289                 if not Lib.IsBinaryModule:
1290                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1291             for Lib, LibAutoGen in DirList:
1292                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
1293                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1294
1295             DirList = []
1296             for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
1297                 if not ModuleAutoGen.IsBinaryModule:
1298                     DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
1299             for Mod,ModAutoGen in DirList:
1300                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']
1301                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
1302             self.CreateAsBuiltInf()
1303             if GlobalData.gBinCacheDest:
1304                 self.GenDestCache()
1305             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1306                 # Only for --hash
1307                 # Update PreMakeCacheChain files
1308                 self.GenLocalPreMakeCache()
1309             self.BuildModules = []
1310             return True
1311
1312         # cleanlib
1313         if Target == 'cleanlib':
1314             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1315                 LibMakefile = os.path.normpath(os.path.join(Lib, makefile))
1316                 if os.path.exists(LibMakefile):
1317                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1318                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1319             return True
1320
1321         # clean
1322         if Target == 'clean':
1323             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1324                 ModMakefile = os.path.normpath(os.path.join(Mod, makefile))
1325                 if os.path.exists(ModMakefile):
1326                     NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']
1327                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1328             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1329                 LibMakefile = os.path.normpath(os.path.join(Lib, makefile))
1330                 if os.path.exists(LibMakefile):
1331                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1332                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1333             return True
1334
1335         # cleanall
1336         if Target == 'cleanall':
1337             try:
1338                 #os.rmdir(AutoGenObject.BuildDir)
1339                 RemoveDirectory(AutoGenObject.BuildDir, True)
1340             except WindowsError as X:
1341                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1342         return True
1343
1344     ## Build a module or platform
1345     #
1346     # Create autogen code and makefile for a module or platform, and the launch
1347     # "make" command to build it
1348     #
1349     #   @param  Target                      The target of build command
1350     #   @param  Platform                    The platform file
1351     #   @param  Module                      The module file
1352     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1353     #   @param  ToolChain                   The name of toolchain to build
1354     #   @param  Arch                        The arch of the module/platform
1355     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1356     #                                       for dependent modules/Libraries
1357     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1358     #                                       for dependent modules/Libraries
1359     #
1360     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1361         if AutoGenObject is None:
1362             return False
1363
1364         # skip file generation for cleanxxx targets, run and fds target
1365         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1366             # for target which must generate AutoGen code and makefile
1367             if not self.SkipAutoGen or Target == 'genc':
1368                 self.Progress.Start("Generating code")
1369                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1370                 self.Progress.Stop("done!")
1371             if Target == "genc":
1372                 return True
1373
1374             if not self.SkipAutoGen or Target == 'genmake':
1375                 self.Progress.Start("Generating makefile")
1376                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1377                 #AutoGenObject.CreateAsBuiltInf()
1378                 self.Progress.Stop("done!")
1379             if Target == "genmake":
1380                 return True
1381         else:
1382             # always recreate top/platform makefile when clean, just in case of inconsistency
1383             AutoGenObject.CreateCodeFile(True)
1384             AutoGenObject.CreateMakeFile(True)
1385
1386         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1387             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1388
1389         BuildCommand = AutoGenObject.BuildCommand
1390         if BuildCommand is None or len(BuildCommand) == 0:
1391             EdkLogger.error("build", OPTION_MISSING,
1392                             "No build command found for this module. "
1393                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1394                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1395                             ExtraData=str(AutoGenObject))
1396
1397         # build modules
1398         if BuildModule:
1399             if Target != 'fds':
1400                 BuildCommand = BuildCommand + [Target]
1401             AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1402             self.CreateAsBuiltInf()
1403             if GlobalData.gBinCacheDest:
1404                 self.GenDestCache()
1405             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1406                 # Only for --hash
1407                 # Update PreMakeCacheChain files
1408                 self.GenLocalPreMakeCache()
1409             self.BuildModules = []
1410             return True
1411
1412         # genfds
1413         if Target == 'fds':
1414             if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db):
1415                 EdkLogger.error("build", COMMAND_FAILURE)
1416             return True
1417
1418         # run
1419         if Target == 'run':
1420             return True
1421
1422         # build library
1423         if Target == 'libraries':
1424             pass
1425
1426         # not build modules
1427
1428
1429         # cleanall
1430         if Target == 'cleanall':
1431             try:
1432                 #os.rmdir(AutoGenObject.BuildDir)
1433                 RemoveDirectory(AutoGenObject.BuildDir, True)
1434             except WindowsError as X:
1435                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1436         return True
1437
1438     ## Rebase module image and Get function address for the input module list.
1439     #
1440     def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False):
1441         if ModeIsSmm:
1442             AddrIsOffset = False
1443         for InfFile in ModuleList:
1444             sys.stdout.write (".")
1445             sys.stdout.flush()
1446             ModuleInfo = ModuleList[InfFile]
1447             ModuleName = ModuleInfo.BaseName
1448             ModuleOutputImage = ModuleInfo.Image.FileName
1449             ModuleDebugImage  = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi')
1450             ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1451             if not ModeIsSmm:
1452                 BaseAddress = BaseAddress - ModuleInfo.Image.Size
1453                 #
1454                 # Update Image to new BaseAddress by GenFw tool
1455                 #
1456                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1457                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1458             else:
1459                 #
1460                 # Set new address to the section header only for SMM driver.
1461                 #
1462                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1463                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1464             #
1465             # Collect function address from Map file
1466             #
1467             ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
1468             FunctionList = []
1469             if os.path.exists(ImageMapTable):
1470                 OrigImageBaseAddress = 0
1471                 ImageMap = open(ImageMapTable, 'r')
1472                 for LinStr in ImageMap:
1473                     if len (LinStr.strip()) == 0:
1474                         continue
1475                     #
1476                     # Get the preferred address set on link time.
1477                     #
1478                     if LinStr.find ('Preferred load address is') != -1:
1479                         StrList = LinStr.split()
1480                         OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16)
1481
1482                     StrList = LinStr.split()
1483                     if len (StrList) > 4:
1484                         if StrList[3] == 'f' or StrList[3] == 'F':
1485                             Name = StrList[1]
1486                             RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress
1487                             FunctionList.append ((Name, RelativeAddress))
1488
1489                 ImageMap.close()
1490             #
1491             # Add general information.
1492             #
1493             if ModeIsSmm:
1494                 MapBuffer.append('\n\n%s (Fixed SMRAM Offset,   BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1495             elif AddrIsOffset:
1496                 MapBuffer.append('\n\n%s (Fixed Memory Offset,  BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
1497             else:
1498                 MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1499             #
1500             # Add guid and general seciton section.
1501             #
1502             TextSectionAddress = 0
1503             DataSectionAddress = 0
1504             for SectionHeader in ModuleInfo.Image.SectionHeaderList:
1505                 if SectionHeader[0] == '.text':
1506                     TextSectionAddress = SectionHeader[1]
1507                 elif SectionHeader[0] in ['.data', '.sdata']:
1508                     DataSectionAddress = SectionHeader[1]
1509             if AddrIsOffset:
1510                 MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
1511             else:
1512                 MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
1513             #
1514             # Add debug image full path.
1515             #
1516             MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))
1517             #
1518             # Add function address
1519             #
1520             for Function in FunctionList:
1521                 if AddrIsOffset:
1522                     MapBuffer.append('  -0x%010X    %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
1523                 else:
1524                     MapBuffer.append('  0x%010X    %s\n' % (BaseAddress + Function[1], Function[0]))
1525             ImageMap.close()
1526
1527             #
1528             # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1529             #
1530             if ModeIsSmm:
1531                 BaseAddress = BaseAddress + ModuleInfo.Image.Size
1532
1533     ## Collect MAP information of all FVs
1534     #
1535     def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList):
1536         if self.Fdf:
1537             # First get the XIP base address for FV map file.
1538             GuidPattern = re.compile("[-a-fA-F0-9]+")
1539             GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+")
1540             for FvName in Wa.FdfProfile.FvDict:
1541                 FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map')
1542                 if not os.path.exists(FvMapBuffer):
1543                     continue
1544                 FvMap = open(FvMapBuffer, 'r')
1545                 #skip FV size information
1546                 FvMap.readline()
1547                 FvMap.readline()
1548                 FvMap.readline()
1549                 FvMap.readline()
1550                 for Line in FvMap:
1551                     MatchGuid = GuidPattern.match(Line)
1552                     if MatchGuid is not None:
1553                         #
1554                         # Replace GUID with module name
1555                         #
1556                         GuidString = MatchGuid.group()
1557                         if GuidString.upper() in ModuleList:
1558                             Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
1559                     MapBuffer.append(Line)
1560                     #
1561                     # Add the debug image full path.
1562                     #
1563                     MatchGuid = GuidName.match(Line)
1564                     if MatchGuid is not None:
1565                         GuidString = MatchGuid.group().split("=")[1]
1566                         if GuidString.upper() in ModuleList:
1567                             MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
1568
1569                 FvMap.close()
1570
1571     ## Collect MAP information of all modules
1572     #
1573     def _CollectModuleMapBuffer (self, MapBuffer, ModuleList):
1574         sys.stdout.write ("Generate Load Module At Fix Address Map")
1575         sys.stdout.flush()
1576         PatchEfiImageList = []
1577         PeiModuleList  = {}
1578         BtModuleList   = {}
1579         RtModuleList   = {}
1580         SmmModuleList  = {}
1581         PeiSize = 0
1582         BtSize  = 0
1583         RtSize  = 0
1584         # reserve 4K size in SMRAM to make SMM module address not from 0.
1585         SmmSize = 0x1000
1586         for ModuleGuid in ModuleList:
1587             Module = ModuleList[ModuleGuid]
1588             GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget)
1589
1590             OutputImageFile = ''
1591             for ResultFile in Module.CodaTargetList:
1592                 if str(ResultFile.Target).endswith('.efi'):
1593                     #
1594                     # module list for PEI, DXE, RUNTIME and SMM
1595                     #
1596                     OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi')
1597                     ImageClass = PeImageClass (OutputImageFile)
1598                     if not ImageClass.IsValid:
1599                         EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo)
1600                     ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass)
1601                     if Module.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_PEIM, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER, EDK_COMPONENT_TYPE_PIC_PEIM, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM, SUP_MODULE_DXE_CORE]:
1602                         PeiModuleList[Module.MetaFile] = ImageInfo
1603                         PeiSize += ImageInfo.Image.Size
1604                     elif Module.ModuleType in [EDK_COMPONENT_TYPE_BS_DRIVER, SUP_MODULE_DXE_DRIVER, SUP_MODULE_UEFI_DRIVER]:
1605                         BtModuleList[Module.MetaFile] = ImageInfo
1606                         BtSize += ImageInfo.Image.Size
1607                     elif Module.ModuleType in [SUP_MODULE_DXE_RUNTIME_DRIVER, EDK_COMPONENT_TYPE_RT_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, EDK_COMPONENT_TYPE_SAL_RT_DRIVER]:
1608                         RtModuleList[Module.MetaFile] = ImageInfo
1609                         RtSize += ImageInfo.Image.Size
1610                     elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]:
1611                         SmmModuleList[Module.MetaFile] = ImageInfo
1612                         SmmSize += ImageInfo.Image.Size
1613                         if Module.ModuleType == SUP_MODULE_DXE_SMM_DRIVER:
1614                             PiSpecVersion = Module.Module.Specification.get('PI_SPECIFICATION_VERSION', '0x00000000')
1615                             # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1616                             if int(PiSpecVersion, 16) < 0x0001000A:
1617                                 BtModuleList[Module.MetaFile] = ImageInfo
1618                                 BtSize += ImageInfo.Image.Size
1619                     break
1620             #
1621             # EFI image is final target.
1622             # Check EFI image contains patchable FixAddress related PCDs.
1623             #
1624             if OutputImageFile != '':
1625                 ModuleIsPatch = False
1626                 for Pcd in Module.ModulePcdList:
1627                     if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1628                         ModuleIsPatch = True
1629                         break
1630                 if not ModuleIsPatch:
1631                     for Pcd in Module.LibraryPcdList:
1632                         if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1633                             ModuleIsPatch = True
1634                             break
1635
1636                 if not ModuleIsPatch:
1637                     continue
1638                 #
1639                 # Module includes the patchable load fix address PCDs.
1640                 # It will be fixed up later.
1641                 #
1642                 PatchEfiImageList.append (OutputImageFile)
1643
1644         #
1645         # Get Top Memory address
1646         #
1647         ReservedRuntimeMemorySize = 0
1648         TopMemoryAddress = 0
1649         if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF:
1650             TopMemoryAddress = 0
1651         else:
1652             TopMemoryAddress = self.LoadFixAddress
1653             if TopMemoryAddress < RtSize + BtSize + PeiSize:
1654                 EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1655
1656         #
1657         # Patch FixAddress related PCDs into EFI image
1658         #
1659         for EfiImage in PatchEfiImageList:
1660             EfiImageMap = EfiImage.replace('.efi', '.map')
1661             if not os.path.exists(EfiImageMap):
1662                 continue
1663             #
1664             # Get PCD offset in EFI image by GenPatchPcdTable function
1665             #
1666             PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage)
1667             #
1668             # Patch real PCD value by PatchPcdValue tool
1669             #
1670             for PcdInfo in PcdTable:
1671                 ReturnValue = 0
1672                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:
1673                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000))
1674                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:
1675                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000))
1676                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:
1677                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000))
1678                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:
1679                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000))
1680                 if ReturnValue != 0:
1681                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
1682
1683         MapBuffer.append('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize // 0x1000))
1684         MapBuffer.append('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize // 0x1000))
1685         MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize // 0x1000))
1686         if len (SmmModuleList) > 0:
1687             MapBuffer.append('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize // 0x1000))
1688
1689         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
1690         BtBaseAddr  = TopMemoryAddress - RtSize
1691         RtBaseAddr  = TopMemoryAddress - ReservedRuntimeMemorySize
1692
1693         self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0)
1694         self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
1695         self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
1696         self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
1697         MapBuffer.append('\n\n')
1698         sys.stdout.write ("\n")
1699         sys.stdout.flush()
1700
1701     ## Save platform Map file
1702     #
1703     def _SaveMapFile (self, MapBuffer, Wa):
1704         #
1705         # Map file path is got.
1706         #
1707         MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map')
1708         #
1709         # Save address map into MAP file.
1710         #
1711         SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)
1712         if self.LoadFixAddress != 0:
1713             sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
1714         sys.stdout.flush()
1715
1716     ## Build active platform for different build targets and different tool chains
1717     #
1718     def _BuildPlatform(self):
1719         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1720         for BuildTarget in self.BuildTargetList:
1721             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1722             index = 0
1723             for ToolChain in self.ToolChainList:
1724                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1725                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1726                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1727                 index += 1
1728                 Wa = WorkspaceAutoGen(
1729                         self.WorkspaceDir,
1730                         self.PlatformFile,
1731                         BuildTarget,
1732                         ToolChain,
1733                         self.ArchList,
1734                         self.BuildDatabase,
1735                         self.TargetTxt,
1736                         self.ToolDef,
1737                         self.Fdf,
1738                         self.FdList,
1739                         self.FvList,
1740                         self.CapList,
1741                         self.SkuId,
1742                         self.UniFlag,
1743                         self.Progress
1744                         )
1745                 self.Fdf = Wa.FdfFile
1746                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1747                 self.BuildReport.AddPlatformReport(Wa)
1748                 self.Progress.Stop("done!")
1749
1750                 # Add ffs build to makefile
1751                 CmdListDict = {}
1752                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1753                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1754
1755                 for Arch in Wa.ArchList:
1756                     PcdMaList    = []
1757                     GlobalData.gGlobalDefines['ARCH'] = Arch
1758                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1759                     for Module in Pa.Platform.Modules:
1760                         # Get ModuleAutoGen object to generate C code file and makefile
1761                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1762                         if Ma is None:
1763                             continue
1764                         if Ma.PcdIsDriver:
1765                             Ma.PlatformInfo = Pa
1766                             Ma.Workspace = Wa
1767                             PcdMaList.append(Ma)
1768                         self.BuildModules.append(Ma)
1769                     Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
1770                     Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
1771                     self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
1772
1773                 # Create MAP file when Load Fix Address is enabled.
1774                 if self.Target in ["", "all", "fds"]:
1775                     for Arch in Wa.ArchList:
1776                         GlobalData.gGlobalDefines['ARCH'] = Arch
1777                         #
1778                         # Check whether the set fix address is above 4G for 32bit image.
1779                         #
1780                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1781                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1782                     #
1783                     # Get Module List
1784                     #
1785                     ModuleList = {}
1786                     for Pa in Wa.AutoGenObjectList:
1787                         for Ma in Pa.ModuleAutoGenList:
1788                             if Ma is None:
1789                                 continue
1790                             if not Ma.IsLibrary:
1791                                 ModuleList[Ma.Guid.upper()] = Ma
1792
1793                     MapBuffer = []
1794                     if self.LoadFixAddress != 0:
1795                         #
1796                         # Rebase module to the preferred memory address before GenFds
1797                         #
1798                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1799                     if self.Fdf:
1800                         #
1801                         # create FDS again for the updated EFI image
1802                         #
1803                         self._Build("fds", Wa)
1804                         #
1805                         # Create MAP file for all platform FVs after GenFds.
1806                         #
1807                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1808                     #
1809                     # Save MAP buffer into MAP file.
1810                     #
1811                     self._SaveMapFile (MapBuffer, Wa)
1812                 self.CreateGuidedSectionToolsFile(Wa)
1813
1814     ## Build active module for different build targets, different tool chains and different archs
1815     #
1816     def _BuildModule(self):
1817         for BuildTarget in self.BuildTargetList:
1818             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1819             index = 0
1820             for ToolChain in self.ToolChainList:
1821                 WorkspaceAutoGenTime = time.time()
1822                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1823                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1824                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1825                 index += 1
1826                 #
1827                 # module build needs platform build information, so get platform
1828                 # AutoGen first
1829                 #
1830                 Wa = WorkspaceAutoGen(
1831                         self.WorkspaceDir,
1832                         self.PlatformFile,
1833                         BuildTarget,
1834                         ToolChain,
1835                         self.ArchList,
1836                         self.BuildDatabase,
1837                         self.TargetTxt,
1838                         self.ToolDef,
1839                         self.Fdf,
1840                         self.FdList,
1841                         self.FvList,
1842                         self.CapList,
1843                         self.SkuId,
1844                         self.UniFlag,
1845                         self.Progress,
1846                         self.ModuleFile
1847                         )
1848                 self.Fdf = Wa.FdfFile
1849                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1850                 Wa.CreateMakeFile(False)
1851                 # Add ffs build to makefile
1852                 CmdListDict = None
1853                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1854                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1855
1856                 GlobalData.file_lock = mp.Lock()
1857                 GlobalData.FfsCmd = CmdListDict
1858
1859                 self.Progress.Stop("done!")
1860                 MaList = []
1861                 ExitFlag = threading.Event()
1862                 ExitFlag.clear()
1863                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
1864                 for Arch in Wa.ArchList:
1865                     AutoGenStart = time.time()
1866                     GlobalData.gGlobalDefines['ARCH'] = Arch
1867                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1868                     for Module in Pa.Platform.Modules:
1869                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
1870                             Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1871                             if Ma is None:
1872                                 continue
1873                             if Ma.PcdIsDriver:
1874                                 Ma.PlatformInfo = Pa
1875                                 Ma.Workspace = Wa
1876                             MaList.append(Ma)
1877
1878                             if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
1879                                 if Ma.CanSkipbyPreMakeCache():
1880                                     continue
1881                                 else:
1882                                     self.PreMakeCacheMiss.add(Ma)
1883
1884                             # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1885                             if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1886                                 # for target which must generate AutoGen code and makefile
1887                                 if not self.SkipAutoGen or self.Target == 'genc':
1888                                     self.Progress.Start("Generating code")
1889                                     Ma.CreateCodeFile(True)
1890                                     self.Progress.Stop("done!")
1891                                 if self.Target == "genc":
1892                                     return True
1893                                 if not self.SkipAutoGen or self.Target == 'genmake':
1894                                     self.Progress.Start("Generating makefile")
1895                                     if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:
1896                                         Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])
1897                                         del CmdListDict[Module.Path, Arch]
1898                                     else:
1899                                         Ma.CreateMakeFile(True)
1900                                     self.Progress.Stop("done!")
1901                                 if self.Target == "genmake":
1902                                     return True
1903
1904                                 if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
1905                                     if Ma.CanSkipbyMakeCache():
1906                                         continue
1907                                     else:
1908                                         self.MakeCacheMiss.add(Ma)
1909
1910                             self.BuildModules.append(Ma)
1911                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
1912                     MakeStart = time.time()
1913                     for Ma in self.BuildModules:
1914                         if not Ma.IsBinaryModule:
1915                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
1916                         # Break build if any build thread has error
1917                         if BuildTask.HasError():
1918                             # we need a full version of makefile for platform
1919                             ExitFlag.set()
1920                             BuildTask.WaitForComplete()
1921                             Pa.CreateMakeFile(False)
1922                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1923                         # Start task scheduler
1924                         if not BuildTask.IsOnGoing():
1925                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1926
1927                     # in case there's an interruption. we need a full version of makefile for platform
1928                     Pa.CreateMakeFile(False)
1929                     if BuildTask.HasError():
1930                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1931                     self.MakeTime += int(round((time.time() - MakeStart)))
1932
1933                 MakeContiue = time.time()
1934                 ExitFlag.set()
1935                 BuildTask.WaitForComplete()
1936                 self.CreateAsBuiltInf()
1937                 if GlobalData.gBinCacheDest:
1938                     self.GenDestCache()
1939                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1940                     # Only for --hash
1941                     # Update PreMakeCacheChain files
1942                     self.GenLocalPreMakeCache()
1943                 self.BuildModules = []
1944                 self.MakeTime += int(round((time.time() - MakeContiue)))
1945                 if BuildTask.HasError():
1946                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1947
1948                 self.BuildReport.AddPlatformReport(Wa, MaList)
1949                 if MaList == []:
1950                     EdkLogger.error(
1951                                 'build',
1952                                 BUILD_ERROR,
1953                                 "Module for [%s] is not a component of active platform."\
1954                                 " Please make sure that the ARCH and inf file path are"\
1955                                 " given in the same as in [%s]" % \
1956                                     (', '.join(Wa.ArchList), self.PlatformFile),
1957                                 ExtraData=self.ModuleFile
1958                                 )
1959                 # Create MAP file when Load Fix Address is enabled.
1960                 if self.Target == "fds" and self.Fdf:
1961                     for Arch in Wa.ArchList:
1962                         #
1963                         # Check whether the set fix address is above 4G for 32bit image.
1964                         #
1965                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1966                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1967                     #
1968                     # Get Module List
1969                     #
1970                     ModuleList = {}
1971                     for Pa in Wa.AutoGenObjectList:
1972                         for Ma in Pa.ModuleAutoGenList:
1973                             if Ma is None:
1974                                 continue
1975                             if not Ma.IsLibrary:
1976                                 ModuleList[Ma.Guid.upper()] = Ma
1977
1978                     MapBuffer = []
1979                     if self.LoadFixAddress != 0:
1980                         #
1981                         # Rebase module to the preferred memory address before GenFds
1982                         #
1983                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1984                     #
1985                     # create FDS again for the updated EFI image
1986                     #
1987                     GenFdsStart = time.time()
1988                     self._Build("fds", Wa)
1989                     self.GenFdsTime += int(round((time.time() - GenFdsStart)))
1990                     #
1991                     # Create MAP file for all platform FVs after GenFds.
1992                     #
1993                     self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1994                     #
1995                     # Save MAP buffer into MAP file.
1996                     #
1997                     self._SaveMapFile (MapBuffer, Wa)
1998
1999     def _GenFfsCmd(self,ArchList):
2000         # convert dictionary of Cmd:(Inf,Arch)
2001         # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
2002         CmdSetDict = defaultdict(set)
2003         GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
2004         for Cmd in GenFfsDict:
2005             tmpInf, tmpArch = GenFfsDict[Cmd]
2006             CmdSetDict[tmpInf, tmpArch].add(Cmd)
2007         return CmdSetDict
2008     def VerifyAutoGenFiles(self):
2009         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2010         try:
2011             with open(AutoGenIdFile) as fd:
2012                 lines = fd.readlines()
2013         except:
2014             return None
2015         for line in lines:
2016             if "Arch" in line:
2017                 ArchList = line.strip().split("=")[1].split("|")
2018             if "BuildDir" in line:
2019                 BuildDir = line.split("=")[1].strip()
2020             if "PlatformGuid" in line:
2021                 PlatformGuid = line.split("=")[1].strip()
2022         GlobalVarList = []
2023         for arch in ArchList:
2024             global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
2025             if not os.path.exists(global_var):
2026                 return None
2027             GlobalVarList.append(global_var)
2028         for global_var in GlobalVarList:
2029             data_pipe = MemoryDataPipe()
2030             data_pipe.load(global_var)
2031             target = data_pipe.Get("P_Info").get("Target")
2032             toolchain = data_pipe.Get("P_Info").get("ToolChain")
2033             archlist = data_pipe.Get("P_Info").get("ArchList")
2034             Arch = data_pipe.Get("P_Info").get("Arch")
2035             active_p = data_pipe.Get("P_Info").get("ActivePlatform")
2036             workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
2037             PackagesPath = os.getenv("PACKAGES_PATH")
2038             mws.setWs(workspacedir, PackagesPath)
2039             LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
2040             ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
2041
2042             for m_build_dir in LibraryBuildDirectoryList:
2043                 if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
2044                     return None
2045             for m_build_dir in ModuleBuildDirectoryList:
2046                 if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
2047                     return None
2048             Wa = WorkSpaceInfo(
2049                 workspacedir,active_p,target,toolchain,archlist
2050                 )
2051             Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
2052             Wa.AutoGenObjectList.append(Pa)
2053         return Wa
2054     def SetupMakeSetting(self,Wa):
2055         BuildModules = []
2056         for Pa in Wa.AutoGenObjectList:
2057             for m in Pa._MbList:
2058                 ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
2059                 BuildModules.append(ma)
2060         fdf_file = Wa.FlashDefinition
2061         if fdf_file:
2062             Fdf = FdfParser(fdf_file.Path)
2063             Fdf.ParseFile()
2064             GlobalData.gFdfParser = Fdf
2065             if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
2066                 FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
2067                 for FdRegion in FdDict.RegionList:
2068                     if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
2069                         if int(FdRegion.Offset) % 8 != 0:
2070                             EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
2071             Wa.FdfProfile = Fdf.Profile
2072             self.Fdf = Fdf
2073         else:
2074             self.Fdf = None
2075         return BuildModules
2076
2077     ## Build a platform in multi-thread mode
2078     #
2079     def PerformAutoGen(self,BuildTarget,ToolChain):
2080         WorkspaceAutoGenTime = time.time()
2081         Wa = WorkspaceAutoGen(
2082                 self.WorkspaceDir,
2083                 self.PlatformFile,
2084                 BuildTarget,
2085                 ToolChain,
2086                 self.ArchList,
2087                 self.BuildDatabase,
2088                 self.TargetTxt,
2089                 self.ToolDef,
2090                 self.Fdf,
2091                 self.FdList,
2092                 self.FvList,
2093                 self.CapList,
2094                 self.SkuId,
2095                 self.UniFlag,
2096                 self.Progress
2097                 )
2098         self.Fdf = Wa.FdfFile
2099         self.LoadFixAddress = Wa.Platform.LoadFixAddress
2100         self.BuildReport.AddPlatformReport(Wa)
2101         Wa.CreateMakeFile(False)
2102
2103         # Add ffs build to makefile
2104         CmdListDict = {}
2105         if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
2106             CmdListDict = self._GenFfsCmd(Wa.ArchList)
2107
2108         self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
2109         BuildModules = []
2110         for Arch in Wa.ArchList:
2111             PcdMaList    = []
2112             AutoGenStart = time.time()
2113             GlobalData.gGlobalDefines['ARCH'] = Arch
2114             Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
2115             if Pa is None:
2116                 continue
2117             ModuleList = []
2118             for Inf in Pa.Platform.Modules:
2119                 ModuleList.append(Inf)
2120             # Add the INF only list in FDF
2121             if GlobalData.gFdfParser is not None:
2122                 for InfName in GlobalData.gFdfParser.Profile.InfList:
2123                     Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
2124                     if Inf in Pa.Platform.Modules:
2125                         continue
2126                     ModuleList.append(Inf)
2127             Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
2128             Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
2129             Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
2130             Pa.CreateLibModuelDirs()
2131             Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
2132             Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
2133             Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
2134             # Prepare the cache share data for multiprocessing
2135             Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}
2136             ModuleCodaFile = {}
2137             for ma in Pa.ModuleAutoGenList:
2138                 ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
2139             Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
2140             # ModuleList contains all driver modules only
2141             for Module in ModuleList:
2142                 # Get ModuleAutoGen object to generate C code file and makefile
2143                 Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2144                 if Ma is None:
2145                     continue
2146                 if Ma.PcdIsDriver:
2147                     Ma.PlatformInfo = Pa
2148                     Ma.Workspace = Wa
2149                     PcdMaList.append(Ma)
2150                 self.AllDrivers.add(Ma)
2151                 self.AllModules.add(Ma)
2152
2153             mqueue = mp.Queue()
2154             cqueue = mp.Queue()
2155             for m in Pa.GetAllModuleInfo:
2156                 mqueue.put(m)
2157                 module_file,module_root,module_path,module_basename,\
2158                     module_originalpath,module_arch,IsLib = m
2159                 Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\
2160                                   ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2161                 self.AllModules.add(Ma)
2162             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
2163             Pa.DataPipe.dump(data_pipe_file)
2164
2165             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
2166
2167             if not autogen_rt:
2168                 self.AutoGenMgr.TerminateWorkers()
2169                 self.AutoGenMgr.join(1)
2170                 raise FatalError(errorcode)
2171
2172             if GlobalData.gUseHashCache:
2173                 for item in GlobalData.gModuleAllCacheStatus:
2174                     (MetaFilePath, Arch, CacheStr, Status) = item
2175                     Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\
2176                                       ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2177                     if CacheStr == "PreMakeCache" and Status == False:
2178                         self.PreMakeCacheMiss.add(Ma)
2179                     if CacheStr == "PreMakeCache" and Status == True:
2180                         self.PreMakeCacheHit.add(Ma)
2181                         GlobalData.gModuleCacheHit.add(Ma)
2182                     if CacheStr == "MakeCache" and Status == False:
2183                         self.MakeCacheMiss.add(Ma)
2184                     if CacheStr == "MakeCache" and Status == True:
2185                         self.MakeCacheHit.add(Ma)
2186                         GlobalData.gModuleCacheHit.add(Ma)
2187             self.AutoGenTime += int(round((time.time() - AutoGenStart)))
2188         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2189         with open(AutoGenIdFile,"w") as fw:
2190             fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
2191             fw.write("BuildDir=%s\n" % Wa.BuildDir)
2192             fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
2193
2194         if GlobalData.gBinCacheSource:
2195             BuildModules.extend(self.MakeCacheMiss)
2196         elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:
2197             BuildModules.extend(self.PreMakeCacheMiss)
2198         else:
2199             BuildModules.extend(self.AllDrivers)
2200
2201         self.Progress.Stop("done!")
2202         return Wa, BuildModules
2203
2204     def _MultiThreadBuildPlatform(self):
2205         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
2206         for BuildTarget in self.BuildTargetList:
2207             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
2208             index = 0
2209             for ToolChain in self.ToolChainList:
2210                 resetFdsGlobalVariable()
2211                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
2212                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
2213                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
2214                 index += 1
2215                 ExitFlag = threading.Event()
2216                 ExitFlag.clear()
2217                 if self.SkipAutoGen:
2218                     Wa = self.VerifyAutoGenFiles()
2219                     if Wa is None:
2220                         self.SkipAutoGen = False
2221                         Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2222                     else:
2223                         GlobalData.gAutoGenPhase = True
2224                         self.BuildModules = self.SetupMakeSetting(Wa)
2225                 else:
2226                     Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2227                 Pa = Wa.AutoGenObjectList[0]
2228                 GlobalData.gAutoGenPhase = False
2229
2230                 if GlobalData.gBinCacheSource:
2231                     EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))
2232                     EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))
2233                     EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))
2234
2235                 for Arch in Wa.ArchList:
2236                     MakeStart = time.time()
2237                     for Ma in set(self.BuildModules):
2238                         # Generate build task for the module
2239                         if not Ma.IsBinaryModule:
2240                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
2241                         # Break build if any build thread has error
2242                         if BuildTask.HasError():
2243                             # we need a full version of makefile for platform
2244                             ExitFlag.set()
2245                             BuildTask.WaitForComplete()
2246                             Pa.CreateMakeFile(False)
2247                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2248                         # Start task scheduler
2249                         if not BuildTask.IsOnGoing():
2250                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
2251
2252                     # in case there's an interruption. we need a full version of makefile for platform
2253
2254                     if BuildTask.HasError():
2255                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2256                     self.MakeTime += int(round((time.time() - MakeStart)))
2257
2258                 MakeContiue = time.time()
2259                 #
2260                 #
2261                 # All modules have been put in build tasks queue. Tell task scheduler
2262                 # to exit if all tasks are completed
2263                 #
2264                 ExitFlag.set()
2265                 BuildTask.WaitForComplete()
2266                 self.CreateAsBuiltInf()
2267                 if GlobalData.gBinCacheDest:
2268                     self.GenDestCache()
2269                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
2270                     # Only for --hash
2271                     # Update PreMakeCacheChain files
2272                     self.GenLocalPreMakeCache()
2273                 #
2274                 # Get Module List
2275                 #
2276                 ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}
2277                 self.BuildModules = []
2278                 self.MakeTime += int(round((time.time() - MakeContiue)))
2279                 #
2280                 # Check for build error, and raise exception if one
2281                 # has been signaled.
2282                 #
2283                 if BuildTask.HasError():
2284                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2285
2286                 # Create MAP file when Load Fix Address is enabled.
2287                 if self.Target in ["", "all", "fds"]:
2288                     for Arch in Wa.ArchList:
2289                         #
2290                         # Check whether the set fix address is above 4G for 32bit image.
2291                         #
2292                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
2293                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2294
2295                     #
2296                     # Rebase module to the preferred memory address before GenFds
2297                     #
2298                     MapBuffer = []
2299                     if self.LoadFixAddress != 0:
2300                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
2301
2302                     if self.Fdf:
2303                         #
2304                         # Generate FD image if there's a FDF file found
2305                         #
2306                         GenFdsStart = time.time()
2307                         if GenFdsApi(Wa.GenFdsCommandDict, self.Db):
2308                             EdkLogger.error("build", COMMAND_FAILURE)
2309
2310                         #
2311                         # Create MAP file for all platform FVs after GenFds.
2312                         #
2313                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2314                         self.GenFdsTime += int(round((time.time() - GenFdsStart)))
2315                     #
2316                     # Save MAP buffer into MAP file.
2317                     #
2318                     self._SaveMapFile(MapBuffer, Wa)
2319                 self.CreateGuidedSectionToolsFile(Wa)
2320     ## Generate GuidedSectionTools.txt in the FV directories.
2321     #
2322     def CreateGuidedSectionToolsFile(self,Wa):
2323         for BuildTarget in self.BuildTargetList:
2324             for ToolChain in self.ToolChainList:
2325                 FvDir = Wa.FvDir
2326                 if not os.path.exists(FvDir):
2327                     continue
2328
2329                 for Arch in self.ArchList:
2330                     # Build up the list of supported architectures for this build
2331                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
2332
2333                     # Look through the tool definitions for GUIDed tools
2334                     guidAttribs = []
2335                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():
2336                         if attrib.upper().endswith('_GUID'):
2337                             split = attrib.split('_')
2338                             thisPrefix = '_'.join(split[0:3]) + '_'
2339                             if thisPrefix == prefix:
2340                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
2341                                 guid = guid.lower()
2342                                 toolName = split[3]
2343                                 path = '_'.join(split[0:4]) + '_PATH'
2344                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
2345                                 path = self.GetFullPathOfTool(path)
2346                                 guidAttribs.append((guid, toolName, path))
2347
2348                     # Write out GuidedSecTools.txt
2349                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
2350                     toolsFile = open(toolsFile, 'wt')
2351                     for guidedSectionTool in guidAttribs:
2352                         print(' '.join(guidedSectionTool), file=toolsFile)
2353                     toolsFile.close()
2354
2355     ## Returns the full path of the tool.
2356     #
2357     def GetFullPathOfTool (self, tool):
2358         if os.path.exists(tool):
2359             return os.path.realpath(tool)
2360         else:
2361             # We need to search for the tool using the
2362             # PATH environment variable.
2363             for dirInPath in os.environ['PATH'].split(os.pathsep):
2364                 foundPath = os.path.join(dirInPath, tool)
2365                 if os.path.exists(foundPath):
2366                     return os.path.realpath(foundPath)
2367
2368         # If the tool was not found in the path then we just return
2369         # the input tool.
2370         return tool
2371
2372     ## Launch the module or platform build
2373     #
2374     def Launch(self):
2375         self.AllDrivers = set()
2376         self.AllModules = set()
2377         self.PreMakeCacheMiss = set()
2378         self.PreMakeCacheHit = set()
2379         self.MakeCacheMiss = set()
2380         self.MakeCacheHit = set()
2381         if not self.ModuleFile:
2382             if not self.SpawnMode or self.Target not in ["", "all"]:
2383                 self.SpawnMode = False
2384                 self._BuildPlatform()
2385             else:
2386                 self._MultiThreadBuildPlatform()
2387         else:
2388             self.SpawnMode = False
2389             self._BuildModule()
2390
2391         if self.Target == 'cleanall':
2392             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
2393
2394     def CreateAsBuiltInf(self):
2395         for Module in self.BuildModules:
2396             Module.CreateAsBuiltInf()
2397
2398     def GenDestCache(self):
2399         for Module in self.AllModules:
2400             Module.GenPreMakefileHashList()
2401             Module.GenMakefileHashList()
2402             Module.CopyModuleToCache()
2403
2404     def GenLocalPreMakeCache(self):
2405         for Module in self.PreMakeCacheMiss:
2406             Module.GenPreMakefileHashList()
2407
2408     ## Do some clean-up works when error occurred
2409     def Relinquish(self):
2410         OldLogLevel = EdkLogger.GetLevel()
2411         EdkLogger.SetLevel(EdkLogger.ERROR)
2412         Utils.Progressor.Abort()
2413         if self.SpawnMode == True:
2414             BuildTask.Abort()
2415         EdkLogger.SetLevel(OldLogLevel)
2416
2417 def ParseDefines(DefineList=[]):
2418     DefineDict = {}
2419     if DefineList is not None:
2420         for Define in DefineList:
2421             DefineTokenList = Define.split("=", 1)
2422             if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]):
2423                 EdkLogger.error('build', FORMAT_INVALID,
2424                                 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2425                                 ExtraData=DefineTokenList[0])
2426
2427             if len(DefineTokenList) == 1:
2428                 DefineDict[DefineTokenList[0]] = "TRUE"
2429             else:
2430                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
2431     return DefineDict
2432
2433
2434
2435 def LogBuildTime(Time):
2436     if Time:
2437         TimeDurStr = ''
2438         TimeDur = time.gmtime(Time)
2439         if TimeDur.tm_yday > 1:
2440             TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + ", %d day(s)" % (TimeDur.tm_yday - 1)
2441         else:
2442             TimeDurStr = time.strftime("%H:%M:%S", TimeDur)
2443         return TimeDurStr
2444     else:
2445         return None
2446 def ThreadNum():
2447     OptionParser = MyOptionParser()
2448     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2449         OptionParser.GetOption()
2450     BuildOption, BuildTarget = OptionParser.BuildOption, OptionParser.BuildTarget
2451     ThreadNumber = BuildOption.ThreadNumber
2452     GlobalData.gCmdConfDir = BuildOption.ConfDirectory
2453     if ThreadNumber is None:
2454         TargetObj = TargetTxtDict()
2455         ThreadNumber = TargetObj.Target.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
2456         if ThreadNumber == '':
2457             ThreadNumber = 0
2458         else:
2459             ThreadNumber = int(ThreadNumber, 0)
2460
2461     if ThreadNumber == 0:
2462         try:
2463             ThreadNumber = multiprocessing.cpu_count()
2464         except (ImportError, NotImplementedError):
2465             ThreadNumber = 1
2466     return ThreadNumber
2467 ## Tool entrance method
2468 #
2469 # This method mainly dispatch specific methods per the command line options.
2470 # If no error found, return zero value so the caller of this tool can know
2471 # if it's executed successfully or not.
2472 #
2473 #   @retval 0     Tool was successful
2474 #   @retval 1     Tool failed
2475 #
2476 LogQMaxSize = ThreadNum() * 10
2477 def Main():
2478     StartTime = time.time()
2479
2480     #
2481     # Create a log Queue
2482     #
2483     LogQ = mp.Queue(LogQMaxSize)
2484     # Initialize log system
2485     EdkLogger.LogClientInitialize(LogQ)
2486     GlobalData.gCommand = sys.argv[1:]
2487     #
2488     # Parse the options and args
2489     #
2490     OptionParser = MyOptionParser()
2491     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2492         OptionParser.GetOption()
2493     Option, Target = OptionParser.BuildOption, OptionParser.BuildTarget
2494     GlobalData.gOptions = Option
2495     GlobalData.gCaseInsensitive = Option.CaseInsensitive
2496
2497     # Set log level
2498     LogLevel = EdkLogger.INFO
2499     if Option.verbose is not None:
2500         EdkLogger.SetLevel(EdkLogger.VERBOSE)
2501         LogLevel = EdkLogger.VERBOSE
2502     elif Option.quiet is not None:
2503         EdkLogger.SetLevel(EdkLogger.QUIET)
2504         LogLevel = EdkLogger.QUIET
2505     elif Option.debug is not None:
2506         EdkLogger.SetLevel(Option.debug + 1)
2507         LogLevel = Option.debug + 1
2508     else:
2509         EdkLogger.SetLevel(EdkLogger.INFO)
2510
2511     if Option.WarningAsError == True:
2512         EdkLogger.SetWarningAsError()
2513     Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile)
2514     Log_Agent.start()
2515
2516     if platform.platform().find("Windows") >= 0:
2517         GlobalData.gIsWindows = True
2518     else:
2519         GlobalData.gIsWindows = False
2520
2521     EdkLogger.quiet("Build environment: %s" % platform.platform())
2522     EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime()));
2523     ReturnCode = 0
2524     MyBuild = None
2525     BuildError = True
2526     try:
2527         if len(Target) == 0:
2528             Target = "all"
2529         elif len(Target) >= 2:
2530             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
2531                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2532         else:
2533             Target = Target[0].lower()
2534
2535         if Target not in gSupportedTarget:
2536             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
2537                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2538
2539         #
2540         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2541         #
2542         CheckEnvVariable()
2543         GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros))
2544
2545         Workspace = os.getenv("WORKSPACE")
2546         #
2547         # Get files real name in workspace dir
2548         #
2549         GlobalData.gAllFiles = Utils.DirCache(Workspace)
2550
2551         WorkingDirectory = os.getcwd()
2552         if not Option.ModuleFile:
2553             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
2554             FileNum = len(FileList)
2555             if FileNum >= 2:
2556                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
2557                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2558             elif FileNum == 1:
2559                 Option.ModuleFile = NormFile(FileList[0], Workspace)
2560
2561         if Option.ModuleFile:
2562             if os.path.isabs (Option.ModuleFile):
2563                 if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0:
2564                     Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace)
2565             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
2566             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
2567             if ErrorCode != 0:
2568                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2569
2570         if Option.PlatformFile is not None:
2571             if os.path.isabs (Option.PlatformFile):
2572                 if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0:
2573                     Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace)
2574             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
2575
2576         if Option.FdfFile is not None:
2577             if os.path.isabs (Option.FdfFile):
2578                 if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0:
2579                     Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace)
2580             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
2581             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
2582             if ErrorCode != 0:
2583                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2584
2585         if Option.Flag is not None and Option.Flag not in ['-c', '-s']:
2586             EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s")
2587
2588         MyBuild = Build(Target, Workspace, Option,LogQ)
2589         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
2590         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
2591             MyBuild.Launch()
2592
2593         #
2594         # All job done, no error found and no exception raised
2595         #
2596         BuildError = False
2597     except FatalError as X:
2598         if MyBuild is not None:
2599             # for multi-thread build exits safely
2600             MyBuild.Relinquish()
2601         if Option is not None and Option.debug is not None:
2602             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2603         ReturnCode = X.args[0]
2604     except Warning as X:
2605         # error from Fdf parser
2606         if MyBuild is not None:
2607             # for multi-thread build exits safely
2608             MyBuild.Relinquish()
2609         if Option is not None and Option.debug is not None:
2610             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2611         else:
2612             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
2613         ReturnCode = FORMAT_INVALID
2614     except KeyboardInterrupt:
2615         if MyBuild is not None:
2616
2617             # for multi-thread build exits safely
2618             MyBuild.Relinquish()
2619         ReturnCode = ABORT_ERROR
2620         if Option is not None and Option.debug is not None:
2621             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2622     except:
2623         if MyBuild is not None:
2624             # for multi-thread build exits safely
2625             MyBuild.Relinquish()
2626
2627         # try to get the meta-file from the object causing exception
2628         Tb = sys.exc_info()[-1]
2629         MetaFile = GlobalData.gProcessingFile
2630         while Tb is not None:
2631             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
2632                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
2633             Tb = Tb.tb_next
2634         EdkLogger.error(
2635                     "\nbuild",
2636                     CODE_ERROR,
2637                     "Unknown fatal error when processing [%s]" % MetaFile,
2638                     ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR,
2639                     RaiseError=False
2640                     )
2641         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2642         ReturnCode = CODE_ERROR
2643     finally:
2644         Utils.Progressor.Abort()
2645         Utils.ClearDuplicatedInf()
2646
2647     if ReturnCode == 0:
2648         try:
2649             MyBuild.LaunchPostbuild()
2650             Conclusion = "Done"
2651         except:
2652             Conclusion = "Failed"
2653     elif ReturnCode == ABORT_ERROR:
2654         Conclusion = "Aborted"
2655     else:
2656         Conclusion = "Failed"
2657     FinishTime = time.time()
2658     BuildDuration = time.gmtime(int(round(FinishTime - StartTime)))
2659     BuildDurationStr = ""
2660     if BuildDuration.tm_yday > 1:
2661         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1)
2662     else:
2663         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
2664     if MyBuild is not None:
2665         if not BuildError:
2666             MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
2667
2668     EdkLogger.SetLevel(EdkLogger.QUIET)
2669     EdkLogger.quiet("\n- %s -" % Conclusion)
2670     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
2671     EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
2672     Log_Agent.kill()
2673     Log_Agent.join()
2674     return ReturnCode
2675
2676 if __name__ == '__main__':
2677     try:
2678         mp.set_start_method('spawn')
2679     except:
2680         pass
2681     r = Main()
2682     ## 0-127 is a safe return range, and 1 is a standard default error
2683     if r < 0 or r > 127: r = 1
2684     sys.exit(r)
2685