BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
   5 #  Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   6 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
   7 #
   8 #  SPDX-License-Identifier: BSD-2-Clause-Patent
   9 #
  10
  11 ##
  12 # Import Modules
  13 #
  14 from __future__ import print_function
  15 from __future__ import absolute_import
  16 import os.path as path
  17 import sys
  18 import os
  19 import re
  20 import glob
  21 import time
  22 import platform
  23 import traceback
  24 import multiprocessing
  25 from threading import Thread,Event,BoundedSemaphore
  26 import threading
  27 from subprocess import Popen,PIPE
  28 from collections import OrderedDict, defaultdict
  29 from Common.buildoptions import BuildOption,BuildTarget
  30 from AutoGen.PlatformAutoGen import PlatformAutoGen
  31 from AutoGen.ModuleAutoGen import ModuleAutoGen
  32 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
  33 from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\
  34     LogAgent
  35 from AutoGen import GenMake
  36 from Common import Misc as Utils
  37
  38 from Common.TargetTxtClassObject import TargetTxt
  39 from Common.ToolDefClassObject import ToolDef
  40 from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory
  41 from Common.StringUtils import NormPath
  42 from Common.MultipleWorkspace import MultipleWorkspace as mws
  43 from Common.BuildToolError import *
  44 from Common.DataType import *
  45 import Common.EdkLogger as EdkLogger
  46
  47 from Workspace.WorkspaceDatabase import BuildDB
  48
  49 from BuildReport import BuildReport
  50 from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile
  51 from PatchPcdValue.PatchPcdValue import PatchBinaryFile
  52
  53 import Common.GlobalData as GlobalData
  54 from GenFds.GenFds import GenFds, GenFdsApi
  55 import multiprocessing as mp
  56 from multiprocessing import Manager
  57 from AutoGen.DataPipe import MemoryDataPipe
  58 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
  59 from GenFds.FdfParser import FdfParser
  60 from AutoGen.IncludesAutoGen import IncludesAutoGen
  61
  62 ## standard targets of build command
  63 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  64
  65 ## build configuration file
  66 gBuildConfiguration = "target.txt"
  67 gToolsDefinition = "tools_def.txt"
  68
  69 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
  70 TmpTableDict = {}
  71
  72 ## Check environment PATH variable to make sure the specified tool is found
  73 #
  74 #   If the tool is found in the PATH, then True is returned
  75 #   Otherwise, False is returned
  76 #
  77 def IsToolInPath(tool):
  78     if 'PATHEXT' in os.environ:
  79         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  80     else:
  81         extns = ('',)
  82     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  83         for ext in extns:
  84             if os.path.exists(os.path.join(pathDir, tool + ext)):
  85                 return True
  86     return False
  87
  88 ## Check environment variables
  89 #
  90 #  Check environment variables that must be set for build. Currently they are
  91 #
  92 #   WORKSPACE           The directory all packages/platforms start from
  93 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  94 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
  95 #
  96 #   If any of above environment variable is not set or has error, the build
  97 #   will be broken.
  98 #
  99 def CheckEnvVariable():
 100     # check WORKSPACE
 101     if "WORKSPACE" not in os.environ:
 102         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 103                         ExtraData="WORKSPACE")
 104
 105     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
 106     if not os.path.exists(WorkspaceDir):
 107         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData=WorkspaceDir)
 108     elif ' ' in WorkspaceDir:
 109         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
 110                         ExtraData=WorkspaceDir)
 111     os.environ["WORKSPACE"] = WorkspaceDir
 112
 113     # set multiple workspace
 114     PackagesPath = os.getenv("PACKAGES_PATH")
 115     mws.setWs(WorkspaceDir, PackagesPath)
 116     if mws.PACKAGES_PATH:
 117         for Path in mws.PACKAGES_PATH:
 118             if not os.path.exists(Path):
 119                 EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData=Path)
 120             elif ' ' in Path:
 121                 EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path)
 122
 123
 124     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 125
 126     # check EDK_TOOLS_PATH
 127     if "EDK_TOOLS_PATH" not in os.environ:
 128         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 129                         ExtraData="EDK_TOOLS_PATH")
 130
 131     # check PATH
 132     if "PATH" not in os.environ:
 133         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 134                         ExtraData="PATH")
 135
 136     GlobalData.gWorkspace = WorkspaceDir
 137
 138     GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 139     GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"]
 140
 141 ## Get normalized file path
 142 #
 143 # Convert the path to be local format, and remove the WORKSPACE path at the
 144 # beginning if the file path is given in full path.
 145 #
 146 # @param  FilePath      File path to be normalized
 147 # @param  Workspace     Workspace path which the FilePath will be checked against
 148 #
 149 # @retval string        The normalized file path
 150 #
 151 def NormFile(FilePath, Workspace):
 152     # check if the path is absolute or relative
 153     if os.path.isabs(FilePath):
 154         FileFullPath = os.path.normpath(FilePath)
 155     else:
 156         FileFullPath = os.path.normpath(mws.join(Workspace, FilePath))
 157         Workspace = mws.getWs(Workspace, FilePath)
 158
 159     # check if the file path exists or not
 160     if not os.path.isfile(FileFullPath):
 161         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath)
 162
 163     # remove workspace directory from the beginning part of the file path
 164     if Workspace[-1] in ["\\", "/"]:
 165         return FileFullPath[len(Workspace):]
 166     else:
 167         return FileFullPath[(len(Workspace) + 1):]
 168
 169 ## Get the output of an external program
 170 #
 171 # This is the entrance method of thread reading output of an external program and
 172 # putting them in STDOUT/STDERR of current program.
 173 #
 174 # @param  From      The stream message read from
 175 # @param  To        The stream message put on
 176 # @param  ExitFlag  The flag used to indicate stopping reading
 177 #
 178 def ReadMessage(From, To, ExitFlag,MemTo=None):
 179     while True:
 180         # read one line a time
 181         Line = From.readline()
 182         # empty string means "end"
 183         if Line is not None and Line != b"":
 184             LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
 185             if MemTo is not None:
 186                 if "Note: including file:" ==  LineStr.lstrip()[:21]:
 187                     MemTo.append(LineStr)
 188                 else:
 189                     To(LineStr)
 190                     MemTo.append(LineStr)
 191             else:
 192                 To(LineStr)
 193         else:
 194             break
 195         if ExitFlag.isSet():
 196             break
 197
 198 class MakeSubProc(Popen):
 199     def __init__(self,*args, **argv):
 200         super(MakeSubProc,self).__init__(*args, **argv)
 201         self.ProcOut = []
 202
 203 ## Launch an external program
 204 #
 205 # This method will call subprocess.Popen to execute an external program with
 206 # given options in specified directory. Because of the dead-lock issue during
 207 # redirecting output of the external program, threads are used to to do the
 208 # redirection work.
 209 #
 210 # @param  Command               A list or string containing the call of the program
 211 # @param  WorkingDir            The directory in which the program will be running
 212 #
 213 def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
 214     BeginTime = time.time()
 215     # if working directory doesn't exist, Popen() will raise an exception
 216     if not os.path.isdir(WorkingDir):
 217         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 218
 219     # Command is used as the first Argument in following Popen().
 220     # It could be a string or sequence. We find that if command is a string in following Popen(),
 221     # ubuntu may fail with an error message that the command is not found.
 222     # So here we may need convert command from string to list instance.
 223     if platform.system() != 'Windows':
 224         if not isinstance(Command, list):
 225             Command = Command.split()
 226         Command = ' '.join(Command)
 227
 228     Proc = None
 229     EndOfProcedure = None
 230     try:
 231         # launch the command
 232         Proc = MakeSubProc(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 233
 234         # launch two threads to read the STDOUT and STDERR
 235         EndOfProcedure = Event()
 236         EndOfProcedure.clear()
 237         if Proc.stdout:
 238             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
 239             StdOutThread.setName("STDOUT-Redirector")
 240             StdOutThread.setDaemon(False)
 241             StdOutThread.start()
 242
 243         if Proc.stderr:
 244             StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure,Proc.ProcOut))
 245             StdErrThread.setName("STDERR-Redirector")
 246             StdErrThread.setDaemon(False)
 247             StdErrThread.start()
 248
 249         # waiting for program exit
 250         Proc.wait()
 251     except: # in case of aborting
 252         # terminate the threads redirecting the program output
 253         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 254         if EndOfProcedure is not None:
 255             EndOfProcedure.set()
 256         if Proc is None:
 257             if not isinstance(Command, type("")):
 258                 Command = " ".join(Command)
 259             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 260
 261     if Proc.stdout:
 262         StdOutThread.join()
 263     if Proc.stderr:
 264         StdErrThread.join()
 265
 266     # check the return code of the program
 267     if Proc.returncode != 0:
 268         if not isinstance(Command, type("")):
 269             Command = " ".join(Command)
 270         # print out the Response file and its content when make failure
 271         RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
 272         if os.path.isfile(RespFile):
 273             f = open(RespFile)
 274             RespContent = f.read()
 275             f.close()
 276             EdkLogger.info(RespContent)
 277
 278         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 279     if ModuleAuto:
 280         iau = IncludesAutoGen(WorkingDir,ModuleAuto)
 281         if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
 282             iau.CreateDepsFileForMsvc(Proc.ProcOut)
 283         else:
 284             iau.UpdateDepsFileforNonMsvc()
 285         iau.UpdateDepsFileforTrim()
 286         iau.CreateModuleDeps()
 287         iau.CreateDepsInclude()
 288     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
 289
 290 ## The smallest unit that can be built in multi-thread build mode
 291 #
 292 # This is the base class of build unit. The "Obj" parameter must provide
 293 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 294 # missing build.
 295 #
 296 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 297 #
 298 class BuildUnit:
 299     ## The constructor
 300     #
 301     #   @param  self        The object pointer
 302     #   @param  Obj         The object the build is working on
 303     #   @param  Target      The build target name, one of gSupportedTarget
 304     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 305     #   @param  WorkingDir  The directory build command starts in
 306     #
 307     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 308         self.BuildObject = Obj
 309         self.Dependency = Dependency
 310         self.WorkingDir = WorkingDir
 311         self.Target = Target
 312         self.BuildCommand = BuildCommand
 313         if not BuildCommand:
 314             EdkLogger.error("build", OPTION_MISSING,
 315                             "No build command found for this module. "
 316                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
 317                                 (Obj.BuildTarget, Obj.ToolChain, Obj.Arch),
 318                             ExtraData=str(Obj))
 319
 320
 321     ## str() method
 322     #
 323     #   It just returns the string representation of self.BuildObject
 324     #
 325     #   @param  self        The object pointer
 326     #
 327     def __str__(self):
 328         return str(self.BuildObject)
 329
 330     ## "==" operator method
 331     #
 332     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 333     #   provide its own __eq__() method.
 334     #
 335     #   @param  self        The object pointer
 336     #   @param  Other       The other BuildUnit object compared to
 337     #
 338     def __eq__(self, Other):
 339         return Other and self.BuildObject == Other.BuildObject \
 340                 and Other.BuildObject \
 341                 and self.BuildObject.Arch == Other.BuildObject.Arch
 342
 343     ## hash() method
 344     #
 345     #   It just returns the hash value of self.BuildObject which must be hashable.
 346     #
 347     #   @param  self        The object pointer
 348     #
 349     def __hash__(self):
 350         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 351
 352     def __repr__(self):
 353         return repr(self.BuildObject)
 354
 355 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 356 #
 357 # This class is for module build by nmake/make build system. The "Obj" parameter
 358 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 359 # be make units missing build.
 360 #
 361 # Currently the "Obj" should be only ModuleAutoGen object.
 362 #
 363 class ModuleMakeUnit(BuildUnit):
 364     ## The constructor
 365     #
 366     #   @param  self        The object pointer
 367     #   @param  Obj         The ModuleAutoGen object the build is working on
 368     #   @param  Target      The build target name, one of gSupportedTarget
 369     #
 370     def __init__(self, Obj, BuildCommand,Target):
 371         Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]
 372         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 373         if Target in [None, "", "all"]:
 374             self.Target = "tbuild"
 375
 376 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 377 #
 378 # This class is for platform build by nmake/make build system. The "Obj" parameter
 379 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 380 # be make units missing build.
 381 #
 382 # Currently the "Obj" should be only PlatformAutoGen object.
 383 #
 384 class PlatformMakeUnit(BuildUnit):
 385     ## The constructor
 386     #
 387     #   @param  self        The object pointer
 388     #   @param  Obj         The PlatformAutoGen object the build is working on
 389     #   @param  Target      The build target name, one of gSupportedTarget
 390     #
 391     def __init__(self, Obj, BuildCommand, Target):
 392         Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 393         Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])
 394         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 395
 396 ## The class representing the task of a module build or platform build
 397 #
 398 # This class manages the build tasks in multi-thread build mode. Its jobs include
 399 # scheduling thread running, catching thread error, monitor the thread status, etc.
 400 #
 401 class BuildTask:
 402     # queue for tasks waiting for schedule
 403     _PendingQueue = OrderedDict()
 404     _PendingQueueLock = threading.Lock()
 405
 406     # queue for tasks ready for running
 407     _ReadyQueue = OrderedDict()
 408     _ReadyQueueLock = threading.Lock()
 409
 410     # queue for run tasks
 411     _RunningQueue = OrderedDict()
 412     _RunningQueueLock = threading.Lock()
 413
 414     # queue containing all build tasks, in case duplicate build
 415     _TaskQueue = OrderedDict()
 416
 417     # flag indicating error occurs in a running thread
 418     _ErrorFlag = threading.Event()
 419     _ErrorFlag.clear()
 420     _ErrorMessage = ""
 421
 422     # BoundedSemaphore object used to control the number of running threads
 423     _Thread = None
 424
 425     # flag indicating if the scheduler is started or not
 426     _SchedulerStopped = threading.Event()
 427     _SchedulerStopped.set()
 428
 429     ## Start the task scheduler thread
 430     #
 431     #   @param  MaxThreadNumber     The maximum thread number
 432     #   @param  ExitFlag            Flag used to end the scheduler
 433     #
 434     @staticmethod
 435     def StartScheduler(MaxThreadNumber, ExitFlag):
 436         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 437         SchedulerThread.setName("Build-Task-Scheduler")
 438         SchedulerThread.setDaemon(False)
 439         SchedulerThread.start()
 440         # wait for the scheduler to be started, especially useful in Linux
 441         while not BuildTask.IsOnGoing():
 442             time.sleep(0.01)
 443
 444     ## Scheduler method
 445     #
 446     #   @param  MaxThreadNumber     The maximum thread number
 447     #   @param  ExitFlag            Flag used to end the scheduler
 448     #
 449     @staticmethod
 450     def Scheduler(MaxThreadNumber, ExitFlag):
 451         BuildTask._SchedulerStopped.clear()
 452         try:
 453             # use BoundedSemaphore to control the maximum running threads
 454             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 455             #
 456             # scheduling loop, which will exits when no pending/ready task and
 457             # indicated to do so, or there's error in running thread
 458             #
 459             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 460                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 461                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 462                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 463
 464                 # get all pending tasks
 465                 BuildTask._PendingQueueLock.acquire()
 466                 BuildObjectList = list(BuildTask._PendingQueue.keys())
 467                 #
 468                 # check if their dependency is resolved, and if true, move them
 469                 # into ready queue
 470                 #
 471                 for BuildObject in BuildObjectList:
 472                     Bt = BuildTask._PendingQueue[BuildObject]
 473                     if Bt.IsReady():
 474                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 475                 BuildTask._PendingQueueLock.release()
 476
 477                 # launch build thread until the maximum number of threads is reached
 478                 while not BuildTask._ErrorFlag.isSet():
 479                     # empty ready queue, do nothing further
 480                     if len(BuildTask._ReadyQueue) == 0:
 481                         break
 482
 483                     # wait for active thread(s) exit
 484                     BuildTask._Thread.acquire(True)
 485
 486                     # start a new build thread
 487                     Bo, Bt = BuildTask._ReadyQueue.popitem()
 488
 489                     # move into running queue
 490                     BuildTask._RunningQueueLock.acquire()
 491                     BuildTask._RunningQueue[Bo] = Bt
 492                     BuildTask._RunningQueueLock.release()
 493
 494                     Bt.Start()
 495                     # avoid tense loop
 496                     time.sleep(0.01)
 497
 498                 # avoid tense loop
 499                 time.sleep(0.01)
 500
 501             # wait for all running threads exit
 502             if BuildTask._ErrorFlag.isSet():
 503                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 504             # while not BuildTask._ErrorFlag.isSet() and \
 505             while len(BuildTask._RunningQueue) > 0:
 506                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 507                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))
 508                 # avoid tense loop
 509                 time.sleep(0.1)
 510         except BaseException as X:
 511             #
 512             # TRICK: hide the output of threads left running, so that the user can
 513             #        catch the error message easily
 514             #
 515             EdkLogger.SetLevel(EdkLogger.ERROR)
 516             BuildTask._ErrorFlag.set()
 517             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 518
 519         BuildTask._PendingQueue.clear()
 520         BuildTask._ReadyQueue.clear()
 521         BuildTask._RunningQueue.clear()
 522         BuildTask._TaskQueue.clear()
 523         BuildTask._SchedulerStopped.set()
 524
 525     ## Wait for all running method exit
 526     #
 527     @staticmethod
 528     def WaitForComplete():
 529         BuildTask._SchedulerStopped.wait()
 530
 531     ## Check if the scheduler is running or not
 532     #
 533     @staticmethod
 534     def IsOnGoing():
 535         return not BuildTask._SchedulerStopped.isSet()
 536
 537     ## Abort the build
 538     @staticmethod
 539     def Abort():
 540         if BuildTask.IsOnGoing():
 541             BuildTask._ErrorFlag.set()
 542             BuildTask.WaitForComplete()
 543
 544     ## Check if there's error in running thread
 545     #
 546     #   Since the main thread cannot catch exceptions in other thread, we have to
 547     #   use threading.Event to communicate this formation to main thread.
 548     #
 549     @staticmethod
 550     def HasError():
 551         return BuildTask._ErrorFlag.isSet()
 552
 553     ## Get error message in running thread
 554     #
 555     #   Since the main thread cannot catch exceptions in other thread, we have to
 556     #   use a static variable to communicate this message to main thread.
 557     #
 558     @staticmethod
 559     def GetErrorMessage():
 560         return BuildTask._ErrorMessage
 561
 562     ## Factory method to create a BuildTask object
 563     #
 564     #   This method will check if a module is building or has been built. And if
 565     #   true, just return the associated BuildTask object in the _TaskQueue. If
 566     #   not, create and return a new BuildTask object. The new BuildTask object
 567     #   will be appended to the _PendingQueue for scheduling later.
 568     #
 569     #   @param  BuildItem       A BuildUnit object representing a build object
 570     #   @param  Dependency      The dependent build object of BuildItem
 571     #
 572     @staticmethod
 573     def New(BuildItem, Dependency=None):
 574         if BuildItem in BuildTask._TaskQueue:
 575             Bt = BuildTask._TaskQueue[BuildItem]
 576             return Bt
 577
 578         Bt = BuildTask()
 579         Bt._Init(BuildItem, Dependency)
 580         BuildTask._TaskQueue[BuildItem] = Bt
 581
 582         BuildTask._PendingQueueLock.acquire()
 583         BuildTask._PendingQueue[BuildItem] = Bt
 584         BuildTask._PendingQueueLock.release()
 585
 586         return Bt
 587
 588     ## The real constructor of BuildTask
 589     #
 590     #   @param  BuildItem       A BuildUnit object representing a build object
 591     #   @param  Dependency      The dependent build object of BuildItem
 592     #
 593     def _Init(self, BuildItem, Dependency=None):
 594         self.BuildItem = BuildItem
 595
 596         self.DependencyList = []
 597         if Dependency is None:
 598             Dependency = BuildItem.Dependency
 599         else:
 600             Dependency.extend(BuildItem.Dependency)
 601         self.AddDependency(Dependency)
 602         # flag indicating build completes, used to avoid unnecessary re-build
 603         self.CompleteFlag = False
 604
 605     ## Check if all dependent build tasks are completed or not
 606     #
 607     def IsReady(self):
 608         ReadyFlag = True
 609         for Dep in self.DependencyList:
 610             if Dep.CompleteFlag == True:
 611                 continue
 612             ReadyFlag = False
 613             break
 614
 615         return ReadyFlag
 616
 617     ## Add dependent build task
 618     #
 619     #   @param  Dependency      The list of dependent build objects
 620     #
 621     def AddDependency(self, Dependency):
 622         for Dep in Dependency:
 623             if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):
 624                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 625
 626     ## The thread wrapper of LaunchCommand function
 627     #
 628     # @param  Command               A list or string contains the call of the command
 629     # @param  WorkingDir            The directory in which the program will be running
 630     #
 631     def _CommandThread(self, Command, WorkingDir):
 632         try:
 633             self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
 634             self.CompleteFlag = True
 635
 636             # Run hash operation post dependency to account for libs
 637             # Run if --hash or --binary-destination
 638             if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
 639                 self.BuildItem.BuildObject.GenModuleHash()
 640             if GlobalData.gBinCacheDest:
 641                 self.BuildItem.BuildObject.GenCMakeHash()
 642
 643         except:
 644             #
 645             # TRICK: hide the output of threads left running, so that the user can
 646             #        catch the error message easily
 647             #
 648             if not BuildTask._ErrorFlag.isSet():
 649                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 650                                                                   self.BuildItem.BuildObject.Arch,
 651                                                                   self.BuildItem.BuildObject.ToolChain,
 652                                                                   self.BuildItem.BuildObject.BuildTarget
 653                                                                  )
 654             EdkLogger.SetLevel(EdkLogger.ERROR)
 655             BuildTask._ErrorFlag.set()
 656             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 657                                       (threading.currentThread().getName(), Command, WorkingDir)
 658
 659         # indicate there's a thread is available for another build task
 660         BuildTask._RunningQueueLock.acquire()
 661         BuildTask._RunningQueue.pop(self.BuildItem)
 662         BuildTask._RunningQueueLock.release()
 663         BuildTask._Thread.release()
 664
 665     ## Start build task thread
 666     #
 667     def Start(self):
 668         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 669         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 670         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 671         self.BuildTread.setName("build thread")
 672         self.BuildTread.setDaemon(False)
 673         self.BuildTread.start()
 674
 675 ## The class contains the information related to EFI image
 676 #
 677 class PeImageInfo():
 678     ## Constructor
 679     #
 680     # Constructor will load all required image information.
 681     #
 682     #   @param  BaseName          The full file path of image.
 683     #   @param  Guid              The GUID for image.
 684     #   @param  Arch              Arch of this image.
 685     #   @param  OutputDir         The output directory for image.
 686     #   @param  DebugDir          The debug directory for image.
 687     #   @param  ImageClass        PeImage Information
 688     #
 689     def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass):
 690         self.BaseName         = BaseName
 691         self.Guid             = Guid
 692         self.Arch             = Arch
 693         self.OutputDir        = OutputDir
 694         self.DebugDir         = DebugDir
 695         self.Image            = ImageClass
 696         self.Image.Size       = (self.Image.Size // 0x1000 + 1) * 0x1000
 697
 698 ## The class implementing the EDK2 build process
 699 #
 700 #   The build process includes:
 701 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 702 #       2. Parse DSC file of active platform
 703 #       3. Parse FDF file if any
 704 #       4. Establish build database, including parse all other files (module, package)
 705 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 706 #       6. Call build command
 707 #
 708 class Build():
 709     ## Constructor
 710     #
 711     # Constructor will load all necessary configurations, parse platform, modules
 712     # and packages and the establish a database for AutoGen.
 713     #
 714     #   @param  Target              The build command target, one of gSupportedTarget
 715     #   @param  WorkspaceDir        The directory of workspace
 716     #   @param  BuildOptions        Build options passed from command line
 717     #
 718     def __init__(self, Target, WorkspaceDir, BuildOptions,log_q):
 719         self.WorkspaceDir   = WorkspaceDir
 720         self.Target         = Target
 721         self.PlatformFile   = BuildOptions.PlatformFile
 722         self.ModuleFile     = BuildOptions.ModuleFile
 723         self.ArchList       = BuildOptions.TargetArch
 724         self.ToolChainList  = BuildOptions.ToolChain
 725         self.BuildTargetList= BuildOptions.BuildTarget
 726         self.Fdf            = BuildOptions.FdfFile
 727         self.FdList         = BuildOptions.RomImage
 728         self.FvList         = BuildOptions.FvImage
 729         self.CapList        = BuildOptions.CapName
 730         self.SilentMode     = BuildOptions.SilentMode
 731         self.ThreadNumber   = 1
 732         self.SkipAutoGen    = BuildOptions.SkipAutoGen
 733         self.Reparse        = BuildOptions.Reparse
 734         self.SkuId          = BuildOptions.SkuId
 735         if self.SkuId:
 736             GlobalData.gSKUID_CMD = self.SkuId
 737         self.ConfDirectory = BuildOptions.ConfDirectory
 738         self.SpawnMode      = True
 739         self.BuildReport    = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)
 740         self.TargetTxt      = TargetTxt
 741         self.ToolDef        = ToolDef
 742         self.AutoGenTime    = 0
 743         self.MakeTime       = 0
 744         self.GenFdsTime     = 0
 745         GlobalData.BuildOptionPcd     = BuildOptions.OptionPcd if BuildOptions.OptionPcd else []
 746         #Set global flag for build mode
 747         GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
 748         GlobalData.gUseHashCache = BuildOptions.UseHashCache
 749         GlobalData.gBinCacheDest   = BuildOptions.BinCacheDest
 750         GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
 751         GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread
 752         GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
 753
 754         if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
 755             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
 756
 757         if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
 758             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.")
 759
 760         if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
 761             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.")
 762
 763         if GlobalData.gBinCacheSource:
 764             BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
 765             if not os.path.isabs(BinCacheSource):
 766                 BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
 767             GlobalData.gBinCacheSource = BinCacheSource
 768         else:
 769             if GlobalData.gBinCacheSource is not None:
 770                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-source.")
 771
 772         if GlobalData.gBinCacheDest:
 773             BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
 774             if not os.path.isabs(BinCacheDest):
 775                 BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
 776             GlobalData.gBinCacheDest = BinCacheDest
 777         else:
 778             if GlobalData.gBinCacheDest is not None:
 779                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-destination.")
 780
 781         GlobalData.gDatabasePath = os.path.normpath(os.path.join(GlobalData.gConfDirectory, GlobalData.gDatabasePath))
 782         if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):
 783             os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))
 784         self.Db = BuildDB
 785         self.BuildDatabase = self.Db.BuildObject
 786         self.Platform = None
 787         self.ToolChainFamily = None
 788         self.LoadFixAddress = 0
 789         self.UniFlag        = BuildOptions.Flag
 790         self.BuildModules = []
 791         self.HashSkipModules = []
 792         self.Db_Flag = False
 793         self.LaunchPrebuildFlag = False
 794         self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory, '.cache', '.PlatformBuild')
 795         if BuildOptions.CommandLength:
 796             GlobalData.gCommandMaxLength = BuildOptions.CommandLength
 797
 798         # print dot character during doing some time-consuming work
 799         self.Progress = Utils.Progressor()
 800         # print current build environment and configuration
 801         EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 802         if "PACKAGES_PATH" in os.environ:
 803             # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
 804             EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"]))))
 805         EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 806         if "EDK_TOOLS_BIN" in os.environ:
 807             # Print the same path style with WORKSPACE env.
 808             EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
 809         EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
 810         if "PYTHON3_ENABLE" in os.environ:
 811             PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]
 812             if PYTHON3_ENABLE != "TRUE":
 813                 PYTHON3_ENABLE = "FALSE"
 814             EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))
 815         if "PYTHON_COMMAND" in os.environ:
 816             EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))
 817         self.InitPreBuild()
 818         self.InitPostBuild()
 819         if self.Prebuild:
 820             EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.Prebuild))
 821         if self.Postbuild:
 822             EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))
 823         if self.Prebuild:
 824             self.LaunchPrebuild()
 825             self.TargetTxt = TargetTxt
 826             self.ToolDef   = ToolDef
 827         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
 828             self.InitBuild()
 829
 830         self.AutoGenMgr = None
 831         EdkLogger.info("")
 832         os.chdir(self.WorkspaceDir)
 833         self.log_q = log_q
 834         GlobalData.file_lock =  mp.Lock()
 835         # Init cache data for local only
 836         GlobalData.gPackageHashFile = dict()
 837         GlobalData.gModulePreMakeCacheStatus = dict()
 838         GlobalData.gModuleMakeCacheStatus = dict()
 839         GlobalData.gHashChainStatus = dict()
 840         GlobalData.gCMakeHashFile = dict()
 841         GlobalData.gModuleHashFile = dict()
 842         GlobalData.gFileHashDict = dict()
 843         GlobalData.gModuleAllCacheStatus = set()
 844         GlobalData.gModuleCacheHit = set()
 845
 846     def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):
 847         try:
 848             if SkipAutoGen:
 849                 return True,0
 850             feedback_q = mp.Queue()
 851             error_event = mp.Event()
 852             FfsCmd = DataPipe.Get("FfsCommand")
 853             if FfsCmd is None:
 854                 FfsCmd = {}
 855             GlobalData.FfsCmd = FfsCmd
 856             auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]
 857             self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
 858             self.AutoGenMgr.start()
 859             for w in auto_workers:
 860                 w.start()
 861             if PcdMaList is not None:
 862                 for PcdMa in PcdMaList:
 863                     # SourceFileList calling sequence impact the makefile string sequence.
 864                     # Create cached SourceFileList here to unify its calling sequence for both
 865                     # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
 866                     RetVal = PcdMa.SourceFileList
 867                     # Force cache miss for PCD driver
 868                     if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
 869                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))
 870
 871                     PcdMa.CreateCodeFile(False)
 872                     PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
 873
 874                     # Force cache miss for PCD driver
 875                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
 876                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))
 877
 878             self.AutoGenMgr.join()
 879             rt = self.AutoGenMgr.Status
 880             return rt, 0
 881         except FatalError as e:
 882             return False, e.args[0]
 883         except:
 884             return False, UNKNOWN_ERROR
 885
 886     ## Load configuration
 887     #
 888     #   This method will parse target.txt and get the build configurations.
 889     #
 890     def LoadConfiguration(self):
 891
 892         # if no ARCH given in command line, get it from target.txt
 893         if not self.ArchList:
 894             self.ArchList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET_ARCH]
 895         self.ArchList = tuple(self.ArchList)
 896
 897         # if no build target given in command line, get it from target.txt
 898         if not self.BuildTargetList:
 899             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
 900
 901         # if no tool chain given in command line, get it from target.txt
 902         if not self.ToolChainList:
 903             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 904             if self.ToolChainList is None or len(self.ToolChainList) == 0:
 905                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 906
 907         # check if the tool chains are defined or not
 908         NewToolChainList = []
 909         for ToolChain in self.ToolChainList:
 910             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 911                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 912             else:
 913                 NewToolChainList.append(ToolChain)
 914         # if no tool chain available, break the build
 915         if len(NewToolChainList) == 0:
 916             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 917                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 918         else:
 919             self.ToolChainList = NewToolChainList
 920
 921         ToolChainFamily = []
 922         ToolDefinition = self.ToolDef.ToolsDefTxtDatabase
 923         for Tool in self.ToolChainList:
 924             if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \
 925                or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]:
 926                 EdkLogger.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool)
 927                 ToolChainFamily.append(TAB_COMPILER_MSFT)
 928             else:
 929                 ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
 930         self.ToolChainFamily = ToolChainFamily
 931
 932         if not self.PlatformFile:
 933             PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 934             if not PlatformFile:
 935                 # Try to find one in current directory
 936                 WorkingDirectory = os.getcwd()
 937                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 938                 FileNum = len(FileList)
 939                 if FileNum >= 2:
 940                     EdkLogger.error("build", OPTION_MISSING,
 941                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 942                 elif FileNum == 1:
 943                     PlatformFile = FileList[0]
 944                 else:
 945                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 946                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 947
 948             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 949         self.ThreadNumber   = ThreadNum()
 950     ## Initialize build configuration
 951     #
 952     #   This method will parse DSC file and merge the configurations from
 953     #   command line and target.txt, then get the final build configurations.
 954     #
 955     def InitBuild(self):
 956         # parse target.txt, tools_def.txt, and platform file
 957         self.LoadConfiguration()
 958
 959         # Allow case-insensitive for those from command line or configuration file
 960         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 961         if ErrorCode != 0:
 962             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 963
 964
 965     def InitPreBuild(self):
 966         self.LoadConfiguration()
 967         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
 968         if ErrorCode != 0:
 969             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
 970         if self.BuildTargetList:
 971             GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0]
 972         if self.ArchList:
 973             GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0]
 974         if self.ToolChainList:
 975             GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0]
 976             GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0]
 977         if self.ToolChainFamily:
 978             GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
 979         if 'PREBUILD' in GlobalData.gCommandLineDefines:
 980             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
 981         else:
 982             self.Db_Flag = True
 983             Platform = self.Db.MapPlatform(str(self.PlatformFile))
 984             self.Prebuild = str(Platform.Prebuild)
 985         if self.Prebuild:
 986             PrebuildList = []
 987             #
 988             # Evaluate all arguments and convert arguments that are WORKSPACE
 989             # relative paths to absolute paths.  Filter arguments that look like
 990             # flags or do not follow the file/dir naming rules to avoid false
 991             # positives on this conversion.
 992             #
 993             for Arg in self.Prebuild.split():
 994                 #
 995                 # Do not modify Arg if it looks like a flag or an absolute file path
 996                 #
 997                 if Arg.startswith('-')  or os.path.isabs(Arg):
 998                     PrebuildList.append(Arg)
 999                     continue
1000                 #
1001                 # Do not modify Arg if it does not look like a Workspace relative
1002                 # path that starts with a valid package directory name
1003                 #
1004                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1005                     PrebuildList.append(Arg)
1006                     continue
1007                 #
1008                 # If Arg looks like a WORKSPACE relative path, then convert to an
1009                 # absolute path and check to see if the file exists.
1010                 #
1011                 Temp = mws.join(self.WorkspaceDir, Arg)
1012                 if os.path.isfile(Temp):
1013                     Arg = Temp
1014                 PrebuildList.append(Arg)
1015             self.Prebuild       = ' '.join(PrebuildList)
1016             self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1017
1018     def InitPostBuild(self):
1019         if 'POSTBUILD' in GlobalData.gCommandLineDefines:
1020             self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD')
1021         else:
1022             Platform = self.Db.MapPlatform(str(self.PlatformFile))
1023             self.Postbuild = str(Platform.Postbuild)
1024         if self.Postbuild:
1025             PostbuildList = []
1026             #
1027             # Evaluate all arguments and convert arguments that are WORKSPACE
1028             # relative paths to absolute paths.  Filter arguments that look like
1029             # flags or do not follow the file/dir naming rules to avoid false
1030             # positives on this conversion.
1031             #
1032             for Arg in self.Postbuild.split():
1033                 #
1034                 # Do not modify Arg if it looks like a flag or an absolute file path
1035                 #
1036                 if Arg.startswith('-')  or os.path.isabs(Arg):
1037                     PostbuildList.append(Arg)
1038                     continue
1039                 #
1040                 # Do not modify Arg if it does not look like a Workspace relative
1041                 # path that starts with a valid package directory name
1042                 #
1043                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1044                     PostbuildList.append(Arg)
1045                     continue
1046                 #
1047                 # If Arg looks like a WORKSPACE relative path, then convert to an
1048                 # absolute path and check to see if the file exists.
1049                 #
1050                 Temp = mws.join(self.WorkspaceDir, Arg)
1051                 if os.path.isfile(Temp):
1052                     Arg = Temp
1053                 PostbuildList.append(Arg)
1054             self.Postbuild       = ' '.join(PostbuildList)
1055             self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1056
1057     def PassCommandOption(self, BuildTarget, TargetArch, ToolChain, PlatformFile, Target):
1058         BuildStr = ''
1059         if GlobalData.gCommand and isinstance(GlobalData.gCommand, list):
1060             BuildStr += ' ' + ' '.join(GlobalData.gCommand)
1061         TargetFlag = False
1062         ArchFlag = False
1063         ToolChainFlag = False
1064         PlatformFileFlag = False
1065
1066         if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget:
1067             TargetFlag = True
1068         if GlobalData.gOptions and not GlobalData.gOptions.TargetArch:
1069             ArchFlag = True
1070         if GlobalData.gOptions and not GlobalData.gOptions.ToolChain:
1071             ToolChainFlag = True
1072         if GlobalData.gOptions and not GlobalData.gOptions.PlatformFile:
1073             PlatformFileFlag = True
1074
1075         if TargetFlag and BuildTarget:
1076             if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple):
1077                 BuildStr += ' -b ' + ' -b '.join(BuildTarget)
1078             elif isinstance(BuildTarget, str):
1079                 BuildStr += ' -b ' + BuildTarget
1080         if ArchFlag and TargetArch:
1081             if isinstance(TargetArch, list) or isinstance(TargetArch, tuple):
1082                 BuildStr += ' -a ' + ' -a '.join(TargetArch)
1083             elif isinstance(TargetArch, str):
1084                 BuildStr += ' -a ' + TargetArch
1085         if ToolChainFlag and ToolChain:
1086             if isinstance(ToolChain, list) or isinstance(ToolChain, tuple):
1087                 BuildStr += ' -t ' + ' -t '.join(ToolChain)
1088             elif isinstance(ToolChain, str):
1089                 BuildStr += ' -t ' + ToolChain
1090         if PlatformFileFlag and PlatformFile:
1091             if isinstance(PlatformFile, list) or isinstance(PlatformFile, tuple):
1092                 BuildStr += ' -p ' + ' -p '.join(PlatformFile)
1093             elif isinstance(PlatformFile, str):
1094                 BuildStr += ' -p' + PlatformFile
1095         BuildStr += ' --conf=' + GlobalData.gConfDirectory
1096         if Target:
1097             BuildStr += ' ' + Target
1098
1099         return BuildStr
1100
1101     def LaunchPrebuild(self):
1102         if self.Prebuild:
1103             EdkLogger.info("\n- Prebuild Start -\n")
1104             self.LaunchPrebuildFlag = True
1105             #
1106             # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1107             # and preserve them for the rest of the main build step, because the child process environment will
1108             # evaporate as soon as it exits, we cannot get it in build step.
1109             #
1110             PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory, '.cache', '.PrebuildEnv')
1111             if os.path.isfile(PrebuildEnvFile):
1112                 os.remove(PrebuildEnvFile)
1113             if os.path.isfile(self.PlatformBuildPath):
1114                 os.remove(self.PlatformBuildPath)
1115             if sys.platform == "win32":
1116                 args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile))
1117                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1118             else:
1119                 args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile))
1120                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1121
1122             # launch two threads to read the STDOUT and STDERR
1123             EndOfProcedure = Event()
1124             EndOfProcedure.clear()
1125             if Process.stdout:
1126                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1127                 StdOutThread.setName("STDOUT-Redirector")
1128                 StdOutThread.setDaemon(False)
1129                 StdOutThread.start()
1130
1131             if Process.stderr:
1132                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1133                 StdErrThread.setName("STDERR-Redirector")
1134                 StdErrThread.setDaemon(False)
1135                 StdErrThread.start()
1136             # waiting for program exit
1137             Process.wait()
1138
1139             if Process.stdout:
1140                 StdOutThread.join()
1141             if Process.stderr:
1142                 StdErrThread.join()
1143             if Process.returncode != 0 :
1144                 EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!')
1145
1146             if os.path.exists(PrebuildEnvFile):
1147                 f = open(PrebuildEnvFile)
1148                 envs = f.readlines()
1149                 f.close()
1150                 envs = [l.split("=", 1) for l in envs ]
1151                 envs = [[I.strip() for I in item] for item in envs if len(item) == 2]
1152                 os.environ.update(dict(envs))
1153             EdkLogger.info("\n- Prebuild Done -\n")
1154
1155     def LaunchPostbuild(self):
1156         if self.Postbuild:
1157             EdkLogger.info("\n- Postbuild Start -\n")
1158             if sys.platform == "win32":
1159                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1160             else:
1161                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1162             # launch two threads to read the STDOUT and STDERR
1163             EndOfProcedure = Event()
1164             EndOfProcedure.clear()
1165             if Process.stdout:
1166                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1167                 StdOutThread.setName("STDOUT-Redirector")
1168                 StdOutThread.setDaemon(False)
1169                 StdOutThread.start()
1170
1171             if Process.stderr:
1172                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1173                 StdErrThread.setName("STDERR-Redirector")
1174                 StdErrThread.setDaemon(False)
1175                 StdErrThread.start()
1176             # waiting for program exit
1177             Process.wait()
1178
1179             if Process.stdout:
1180                 StdOutThread.join()
1181             if Process.stderr:
1182                 StdErrThread.join()
1183             if Process.returncode != 0 :
1184                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
1185             EdkLogger.info("\n- Postbuild Done -\n")
1186
1187     ## Build a module or platform
1188     #
1189     # Create autogen code and makefile for a module or platform, and the launch
1190     # "make" command to build it
1191     #
1192     #   @param  Target                      The target of build command
1193     #   @param  Platform                    The platform file
1194     #   @param  Module                      The module file
1195     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1196     #   @param  ToolChain                   The name of toolchain to build
1197     #   @param  Arch                        The arch of the module/platform
1198     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1199     #                                       for dependent modules/Libraries
1200     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1201     #                                       for dependent modules/Libraries
1202     #
1203     def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):
1204         if AutoGenObject is None:
1205             return False
1206         if FfsCommand is None:
1207             FfsCommand = {}
1208         # skip file generation for cleanxxx targets, run and fds target
1209         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1210             # for target which must generate AutoGen code and makefile
1211             mqueue = mp.Queue()
1212             for m in AutoGenObject.GetAllModuleInfo:
1213                 mqueue.put(m)
1214
1215             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
1216             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
1217             AutoGenObject.CreateLibModuelDirs()
1218             AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
1219             AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
1220             AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
1221             self.Progress.Start("Generating makefile and code")
1222             data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
1223             AutoGenObject.DataPipe.dump(data_pipe_file)
1224             cqueue = mp.Queue()
1225             autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
1226             AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
1227             with open(AutoGenIdFile,"w") as fw:
1228                 fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
1229                 fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
1230                 fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
1231             self.Progress.Stop("done!")
1232             if not autogen_rt:
1233                 self.AutoGenMgr.TerminateWorkers()
1234                 self.AutoGenMgr.join(1)
1235                 raise FatalError(errorcode)
1236             AutoGenObject.CreateCodeFile(False)
1237             AutoGenObject.CreateMakeFile(False)
1238         else:
1239             # always recreate top/platform makefile when clean, just in case of inconsistency
1240             AutoGenObject.CreateCodeFile(True)
1241             AutoGenObject.CreateMakeFile(True)
1242
1243         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1244             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1245
1246         BuildCommand = AutoGenObject.BuildCommand
1247         if BuildCommand is None or len(BuildCommand) == 0:
1248             EdkLogger.error("build", OPTION_MISSING,
1249                             "No build command found for this module. "
1250                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1251                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1252                             ExtraData=str(AutoGenObject))
1253
1254         makefile = GenMake.BuildFile(AutoGenObject)._FILE_NAME_[GenMake.gMakeType]
1255
1256         # run
1257         if Target == 'run':
1258             return True
1259
1260         # build modules
1261         if BuildModule:
1262             BuildCommand = BuildCommand + [Target]
1263             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1264             self.CreateAsBuiltInf()
1265             if GlobalData.gBinCacheDest:
1266                 self.GenDestCache()
1267             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1268                 # Only for --hash
1269                 # Update PreMakeCacheChain files
1270                 self.GenLocalPreMakeCache()
1271             self.BuildModules = []
1272             return True
1273
1274         # build library
1275         if Target == 'libraries':
1276             DirList = []
1277             for Lib in AutoGenObject.LibraryAutoGenList:
1278                 if not Lib.IsBinaryModule:
1279                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1280             for Lib, LibAutoGen in DirList:
1281                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
1282                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1283             return True
1284
1285         # build module
1286         if Target == 'modules':
1287             DirList = []
1288             for Lib in AutoGenObject.LibraryAutoGenList:
1289                 if not Lib.IsBinaryModule:
1290                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1291             for Lib, LibAutoGen in DirList:
1292                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
1293                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1294
1295             DirList = []
1296             for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
1297                 if not ModuleAutoGen.IsBinaryModule:
1298                     DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
1299             for Mod,ModAutoGen in DirList:
1300                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']
1301                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
1302             self.CreateAsBuiltInf()
1303             if GlobalData.gBinCacheDest:
1304                 self.GenDestCache()
1305             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1306                 # Only for --hash
1307                 # Update PreMakeCacheChain files
1308                 self.GenLocalPreMakeCache()
1309             self.BuildModules = []
1310             return True
1311
1312         # cleanlib
1313         if Target == 'cleanlib':
1314             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1315                 LibMakefile = os.path.normpath(os.path.join(Lib, makefile))
1316                 if os.path.exists(LibMakefile):
1317                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1318                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1319             return True
1320
1321         # clean
1322         if Target == 'clean':
1323             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1324                 ModMakefile = os.path.normpath(os.path.join(Mod, makefile))
1325                 if os.path.exists(ModMakefile):
1326                     NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']
1327                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1328             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1329                 LibMakefile = os.path.normpath(os.path.join(Lib, makefile))
1330                 if os.path.exists(LibMakefile):
1331                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1332                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1333             return True
1334
1335         # cleanall
1336         if Target == 'cleanall':
1337             try:
1338                 #os.rmdir(AutoGenObject.BuildDir)
1339                 RemoveDirectory(AutoGenObject.BuildDir, True)
1340             except WindowsError as X:
1341                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1342         return True
1343
1344     ## Build a module or platform
1345     #
1346     # Create autogen code and makefile for a module or platform, and the launch
1347     # "make" command to build it
1348     #
1349     #   @param  Target                      The target of build command
1350     #   @param  Platform                    The platform file
1351     #   @param  Module                      The module file
1352     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1353     #   @param  ToolChain                   The name of toolchain to build
1354     #   @param  Arch                        The arch of the module/platform
1355     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1356     #                                       for dependent modules/Libraries
1357     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1358     #                                       for dependent modules/Libraries
1359     #
1360     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1361         if AutoGenObject is None:
1362             return False
1363
1364         # skip file generation for cleanxxx targets, run and fds target
1365         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1366             # for target which must generate AutoGen code and makefile
1367             if not self.SkipAutoGen or Target == 'genc':
1368                 self.Progress.Start("Generating code")
1369                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1370                 self.Progress.Stop("done!")
1371             if Target == "genc":
1372                 return True
1373
1374             if not self.SkipAutoGen or Target == 'genmake':
1375                 self.Progress.Start("Generating makefile")
1376                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1377                 #AutoGenObject.CreateAsBuiltInf()
1378                 self.Progress.Stop("done!")
1379             if Target == "genmake":
1380                 return True
1381         else:
1382             # always recreate top/platform makefile when clean, just in case of inconsistency
1383             AutoGenObject.CreateCodeFile(True)
1384             AutoGenObject.CreateMakeFile(True)
1385
1386         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1387             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1388
1389         BuildCommand = AutoGenObject.BuildCommand
1390         if BuildCommand is None or len(BuildCommand) == 0:
1391             EdkLogger.error("build", OPTION_MISSING,
1392                             "No build command found for this module. "
1393                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1394                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1395                             ExtraData=str(AutoGenObject))
1396
1397         # build modules
1398         if BuildModule:
1399             if Target != 'fds':
1400                 BuildCommand = BuildCommand + [Target]
1401             AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1402             self.CreateAsBuiltInf()
1403             if GlobalData.gBinCacheDest:
1404                 self.GenDestCache()
1405             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1406                 # Only for --hash
1407                 # Update PreMakeCacheChain files
1408                 self.GenLocalPreMakeCache()
1409             self.BuildModules = []
1410             return True
1411
1412         # genfds
1413         if Target == 'fds':
1414             if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db):
1415                 EdkLogger.error("build", COMMAND_FAILURE)
1416             return True
1417
1418         # run
1419         if Target == 'run':
1420             return True
1421
1422         # build library
1423         if Target == 'libraries':
1424             pass
1425
1426         # not build modules
1427
1428
1429         # cleanall
1430         if Target == 'cleanall':
1431             try:
1432                 #os.rmdir(AutoGenObject.BuildDir)
1433                 RemoveDirectory(AutoGenObject.BuildDir, True)
1434             except WindowsError as X:
1435                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1436         return True
1437
1438     ## Rebase module image and Get function address for the input module list.
1439     #
1440     def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False):
1441         if ModeIsSmm:
1442             AddrIsOffset = False
1443         for InfFile in ModuleList:
1444             sys.stdout.write (".")
1445             sys.stdout.flush()
1446             ModuleInfo = ModuleList[InfFile]
1447             ModuleName = ModuleInfo.BaseName
1448             ModuleOutputImage = ModuleInfo.Image.FileName
1449             ModuleDebugImage  = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi')
1450             ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1451             if not ModeIsSmm:
1452                 BaseAddress = BaseAddress - ModuleInfo.Image.Size
1453                 #
1454                 # Update Image to new BaseAddress by GenFw tool
1455                 #
1456                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1457                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1458             else:
1459                 #
1460                 # Set new address to the section header only for SMM driver.
1461                 #
1462                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1463                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1464             #
1465             # Collect function address from Map file
1466             #
1467             ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
1468             FunctionList = []
1469             if os.path.exists(ImageMapTable):
1470                 OrigImageBaseAddress = 0
1471                 ImageMap = open(ImageMapTable, 'r')
1472                 for LinStr in ImageMap:
1473                     if len (LinStr.strip()) == 0:
1474                         continue
1475                     #
1476                     # Get the preferred address set on link time.
1477                     #
1478                     if LinStr.find ('Preferred load address is') != -1:
1479                         StrList = LinStr.split()
1480                         OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16)
1481
1482                     StrList = LinStr.split()
1483                     if len (StrList) > 4:
1484                         if StrList[3] == 'f' or StrList[3] == 'F':
1485                             Name = StrList[1]
1486                             RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress
1487                             FunctionList.append ((Name, RelativeAddress))
1488
1489                 ImageMap.close()
1490             #
1491             # Add general information.
1492             #
1493             if ModeIsSmm:
1494                 MapBuffer.append('\n\n%s (Fixed SMRAM Offset,   BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1495             elif AddrIsOffset:
1496                 MapBuffer.append('\n\n%s (Fixed Memory Offset,  BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
1497             else:
1498                 MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1499             #
1500             # Add guid and general seciton section.
1501             #
1502             TextSectionAddress = 0
1503             DataSectionAddress = 0
1504             for SectionHeader in ModuleInfo.Image.SectionHeaderList:
1505                 if SectionHeader[0] == '.text':
1506                     TextSectionAddress = SectionHeader[1]
1507                 elif SectionHeader[0] in ['.data', '.sdata']:
1508                     DataSectionAddress = SectionHeader[1]
1509             if AddrIsOffset:
1510                 MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
1511             else:
1512                 MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
1513             #
1514             # Add debug image full path.
1515             #
1516             MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))
1517             #
1518             # Add function address
1519             #
1520             for Function in FunctionList:
1521                 if AddrIsOffset:
1522                     MapBuffer.append('  -0x%010X    %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
1523                 else:
1524                     MapBuffer.append('  0x%010X    %s\n' % (BaseAddress + Function[1], Function[0]))
1525             ImageMap.close()
1526
1527             #
1528             # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1529             #
1530             if ModeIsSmm:
1531                 BaseAddress = BaseAddress + ModuleInfo.Image.Size
1532
1533     ## Collect MAP information of all FVs
1534     #
1535     def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList):
1536         if self.Fdf:
1537             # First get the XIP base address for FV map file.
1538             GuidPattern = re.compile("[-a-fA-F0-9]+")
1539             GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+")
1540             for FvName in Wa.FdfProfile.FvDict:
1541                 FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map')
1542                 if not os.path.exists(FvMapBuffer):
1543                     continue
1544                 FvMap = open(FvMapBuffer, 'r')
1545                 #skip FV size information
1546                 FvMap.readline()
1547                 FvMap.readline()
1548                 FvMap.readline()
1549                 FvMap.readline()
1550                 for Line in FvMap:
1551                     MatchGuid = GuidPattern.match(Line)
1552                     if MatchGuid is not None:
1553                         #
1554                         # Replace GUID with module name
1555                         #
1556                         GuidString = MatchGuid.group()
1557                         if GuidString.upper() in ModuleList:
1558                             Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
1559                     MapBuffer.append(Line)
1560                     #
1561                     # Add the debug image full path.
1562                     #
1563                     MatchGuid = GuidName.match(Line)
1564                     if MatchGuid is not None:
1565                         GuidString = MatchGuid.group().split("=")[1]
1566                         if GuidString.upper() in ModuleList:
1567                             MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
1568
1569                 FvMap.close()
1570
1571     ## Collect MAP information of all modules
1572     #
1573     def _CollectModuleMapBuffer (self, MapBuffer, ModuleList):
1574         sys.stdout.write ("Generate Load Module At Fix Address Map")
1575         sys.stdout.flush()
1576         PatchEfiImageList = []
1577         PeiModuleList  = {}
1578         BtModuleList   = {}
1579         RtModuleList   = {}
1580         SmmModuleList  = {}
1581         PeiSize = 0
1582         BtSize  = 0
1583         RtSize  = 0
1584         # reserve 4K size in SMRAM to make SMM module address not from 0.
1585         SmmSize = 0x1000
1586         for ModuleGuid in ModuleList:
1587             Module = ModuleList[ModuleGuid]
1588             GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget)
1589
1590             OutputImageFile = ''
1591             for ResultFile in Module.CodaTargetList:
1592                 if str(ResultFile.Target).endswith('.efi'):
1593                     #
1594                     # module list for PEI, DXE, RUNTIME and SMM
1595                     #
1596                     OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi')
1597                     ImageClass = PeImageClass (OutputImageFile)
1598                     if not ImageClass.IsValid:
1599                         EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo)
1600                     ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass)
1601                     if Module.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_PEIM, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER, EDK_COMPONENT_TYPE_PIC_PEIM, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM, SUP_MODULE_DXE_CORE]:
1602                         PeiModuleList[Module.MetaFile] = ImageInfo
1603                         PeiSize += ImageInfo.Image.Size
1604                     elif Module.ModuleType in [EDK_COMPONENT_TYPE_BS_DRIVER, SUP_MODULE_DXE_DRIVER, SUP_MODULE_UEFI_DRIVER]:
1605                         BtModuleList[Module.MetaFile] = ImageInfo
1606                         BtSize += ImageInfo.Image.Size
1607                     elif Module.ModuleType in [SUP_MODULE_DXE_RUNTIME_DRIVER, EDK_COMPONENT_TYPE_RT_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, EDK_COMPONENT_TYPE_SAL_RT_DRIVER]:
1608                         RtModuleList[Module.MetaFile] = ImageInfo
1609                         RtSize += ImageInfo.Image.Size
1610                     elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]:
1611                         SmmModuleList[Module.MetaFile] = ImageInfo
1612                         SmmSize += ImageInfo.Image.Size
1613                         if Module.ModuleType == SUP_MODULE_DXE_SMM_DRIVER:
1614                             PiSpecVersion = Module.Module.Specification.get('PI_SPECIFICATION_VERSION', '0x00000000')
1615                             # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1616                             if int(PiSpecVersion, 16) < 0x0001000A:
1617                                 BtModuleList[Module.MetaFile] = ImageInfo
1618                                 BtSize += ImageInfo.Image.Size
1619                     break
1620             #
1621             # EFI image is final target.
1622             # Check EFI image contains patchable FixAddress related PCDs.
1623             #
1624             if OutputImageFile != '':
1625                 ModuleIsPatch = False
1626                 for Pcd in Module.ModulePcdList:
1627                     if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1628                         ModuleIsPatch = True
1629                         break
1630                 if not ModuleIsPatch:
1631                     for Pcd in Module.LibraryPcdList:
1632                         if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1633                             ModuleIsPatch = True
1634                             break
1635
1636                 if not ModuleIsPatch:
1637                     continue
1638                 #
1639                 # Module includes the patchable load fix address PCDs.
1640                 # It will be fixed up later.
1641                 #
1642                 PatchEfiImageList.append (OutputImageFile)
1643
1644         #
1645         # Get Top Memory address
1646         #
1647         ReservedRuntimeMemorySize = 0
1648         TopMemoryAddress = 0
1649         if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF:
1650             TopMemoryAddress = 0
1651         else:
1652             TopMemoryAddress = self.LoadFixAddress
1653             if TopMemoryAddress < RtSize + BtSize + PeiSize:
1654                 EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1655
1656         #
1657         # Patch FixAddress related PCDs into EFI image
1658         #
1659         for EfiImage in PatchEfiImageList:
1660             EfiImageMap = EfiImage.replace('.efi', '.map')
1661             if not os.path.exists(EfiImageMap):
1662                 continue
1663             #
1664             # Get PCD offset in EFI image by GenPatchPcdTable function
1665             #
1666             PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage)
1667             #
1668             # Patch real PCD value by PatchPcdValue tool
1669             #
1670             for PcdInfo in PcdTable:
1671                 ReturnValue = 0
1672                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:
1673                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000))
1674                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:
1675                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000))
1676                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:
1677                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000))
1678                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:
1679                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000))
1680                 if ReturnValue != 0:
1681                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
1682
1683         MapBuffer.append('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize // 0x1000))
1684         MapBuffer.append('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize // 0x1000))
1685         MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize // 0x1000))
1686         if len (SmmModuleList) > 0:
1687             MapBuffer.append('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize // 0x1000))
1688
1689         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
1690         BtBaseAddr  = TopMemoryAddress - RtSize
1691         RtBaseAddr  = TopMemoryAddress - ReservedRuntimeMemorySize
1692
1693         self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0)
1694         self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
1695         self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
1696         self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
1697         MapBuffer.append('\n\n')
1698         sys.stdout.write ("\n")
1699         sys.stdout.flush()
1700
1701     ## Save platform Map file
1702     #
1703     def _SaveMapFile (self, MapBuffer, Wa):
1704         #
1705         # Map file path is got.
1706         #
1707         MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map')
1708         #
1709         # Save address map into MAP file.
1710         #
1711         SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)
1712         if self.LoadFixAddress != 0:
1713             sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
1714         sys.stdout.flush()
1715
1716     ## Build active platform for different build targets and different tool chains
1717     #
1718     def _BuildPlatform(self):
1719         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1720         for BuildTarget in self.BuildTargetList:
1721             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1722             index = 0
1723             for ToolChain in self.ToolChainList:
1724                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1725                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1726                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1727                 index += 1
1728                 Wa = WorkspaceAutoGen(
1729                         self.WorkspaceDir,
1730                         self.PlatformFile,
1731                         BuildTarget,
1732                         ToolChain,
1733                         self.ArchList,
1734                         self.BuildDatabase,
1735                         self.TargetTxt,
1736                         self.ToolDef,
1737                         self.Fdf,
1738                         self.FdList,
1739                         self.FvList,
1740                         self.CapList,
1741                         self.SkuId,
1742                         self.UniFlag,
1743                         self.Progress
1744                         )
1745                 self.Fdf = Wa.FdfFile
1746                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1747                 self.BuildReport.AddPlatformReport(Wa)
1748                 self.Progress.Stop("done!")
1749
1750                 # Add ffs build to makefile
1751                 CmdListDict = {}
1752                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1753                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1754
1755                 for Arch in Wa.ArchList:
1756                     PcdMaList    = []
1757                     GlobalData.gGlobalDefines['ARCH'] = Arch
1758                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1759                     for Module in Pa.Platform.Modules:
1760                         # Get ModuleAutoGen object to generate C code file and makefile
1761                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1762                         if Ma is None:
1763                             continue
1764                         if Ma.PcdIsDriver:
1765                             Ma.PlatformInfo = Pa
1766                             Ma.Workspace = Wa
1767                             PcdMaList.append(Ma)
1768                         self.BuildModules.append(Ma)
1769                     Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
1770                     Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
1771                     self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
1772
1773                 # Create MAP file when Load Fix Address is enabled.
1774                 if self.Target in ["", "all", "fds"]:
1775                     for Arch in Wa.ArchList:
1776                         GlobalData.gGlobalDefines['ARCH'] = Arch
1777                         #
1778                         # Check whether the set fix address is above 4G for 32bit image.
1779                         #
1780                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1781                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1782                     #
1783                     # Get Module List
1784                     #
1785                     ModuleList = {}
1786                     for Pa in Wa.AutoGenObjectList:
1787                         for Ma in Pa.ModuleAutoGenList:
1788                             if Ma is None:
1789                                 continue
1790                             if not Ma.IsLibrary:
1791                                 ModuleList[Ma.Guid.upper()] = Ma
1792
1793                     MapBuffer = []
1794                     if self.LoadFixAddress != 0:
1795                         #
1796                         # Rebase module to the preferred memory address before GenFds
1797                         #
1798                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1799                     if self.Fdf:
1800                         #
1801                         # create FDS again for the updated EFI image
1802                         #
1803                         self._Build("fds", Wa)
1804                         #
1805                         # Create MAP file for all platform FVs after GenFds.
1806                         #
1807                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1808                     #
1809                     # Save MAP buffer into MAP file.
1810                     #
1811                     self._SaveMapFile (MapBuffer, Wa)
1812                 self.CreateGuidedSectionToolsFile(Wa)
1813
1814     ## Build active module for different build targets, different tool chains and different archs
1815     #
1816     def _BuildModule(self):
1817         for BuildTarget in self.BuildTargetList:
1818             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1819             index = 0
1820             for ToolChain in self.ToolChainList:
1821                 WorkspaceAutoGenTime = time.time()
1822                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1823                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1824                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1825                 index += 1
1826                 #
1827                 # module build needs platform build information, so get platform
1828                 # AutoGen first
1829                 #
1830                 Wa = WorkspaceAutoGen(
1831                         self.WorkspaceDir,
1832                         self.PlatformFile,
1833                         BuildTarget,
1834                         ToolChain,
1835                         self.ArchList,
1836                         self.BuildDatabase,
1837                         self.TargetTxt,
1838                         self.ToolDef,
1839                         self.Fdf,
1840                         self.FdList,
1841                         self.FvList,
1842                         self.CapList,
1843                         self.SkuId,
1844                         self.UniFlag,
1845                         self.Progress,
1846                         self.ModuleFile
1847                         )
1848                 self.Fdf = Wa.FdfFile
1849                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1850                 Wa.CreateMakeFile(False)
1851                 # Add ffs build to makefile
1852                 CmdListDict = None
1853                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1854                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1855
1856                 GlobalData.file_lock = mp.Lock()
1857                 GlobalData.FfsCmd = CmdListDict
1858
1859                 self.Progress.Stop("done!")
1860                 MaList = []
1861                 ExitFlag = threading.Event()
1862                 ExitFlag.clear()
1863                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
1864                 for Arch in Wa.ArchList:
1865                     AutoGenStart = time.time()
1866                     GlobalData.gGlobalDefines['ARCH'] = Arch
1867                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1868                     for Module in Pa.Platform.Modules:
1869                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
1870                             Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1871                             if Ma is None:
1872                                 continue
1873                             if Ma.PcdIsDriver:
1874                                 Ma.PlatformInfo = Pa
1875                                 Ma.Workspace = Wa
1876                             MaList.append(Ma)
1877
1878                             if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
1879                                 if Ma.CanSkipbyPreMakeCache():
1880                                     continue
1881                                 else:
1882                                     self.PreMakeCacheMiss.add(Ma)
1883
1884                             # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1885                             if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1886                                 # for target which must generate AutoGen code and makefile
1887                                 if not self.SkipAutoGen or self.Target == 'genc':
1888                                     self.Progress.Start("Generating code")
1889                                     Ma.CreateCodeFile(True)
1890                                     self.Progress.Stop("done!")
1891                                 if self.Target == "genc":
1892                                     return True
1893                                 if not self.SkipAutoGen or self.Target == 'genmake':
1894                                     self.Progress.Start("Generating makefile")
1895                                     if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:
1896                                         Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])
1897                                         del CmdListDict[Module.Path, Arch]
1898                                     else:
1899                                         Ma.CreateMakeFile(True)
1900                                     self.Progress.Stop("done!")
1901                                 if self.Target == "genmake":
1902                                     return True
1903
1904                                 if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
1905                                     if Ma.CanSkipbyMakeCache():
1906                                         continue
1907                                     else:
1908                                         self.MakeCacheMiss.add(Ma)
1909
1910                             self.BuildModules.append(Ma)
1911                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
1912                     MakeStart = time.time()
1913                     for Ma in self.BuildModules:
1914                         if not Ma.IsBinaryModule:
1915                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
1916                         # Break build if any build thread has error
1917                         if BuildTask.HasError():
1918                             # we need a full version of makefile for platform
1919                             ExitFlag.set()
1920                             BuildTask.WaitForComplete()
1921                             Pa.CreateMakeFile(False)
1922                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1923                         # Start task scheduler
1924                         if not BuildTask.IsOnGoing():
1925                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1926
1927                     # in case there's an interruption. we need a full version of makefile for platform
1928                     Pa.CreateMakeFile(False)
1929                     if BuildTask.HasError():
1930                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1931                     self.MakeTime += int(round((time.time() - MakeStart)))
1932
1933                 MakeContiue = time.time()
1934                 ExitFlag.set()
1935                 BuildTask.WaitForComplete()
1936                 self.CreateAsBuiltInf()
1937                 if GlobalData.gBinCacheDest:
1938                     self.GenDestCache()
1939                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1940                     # Only for --hash
1941                     # Update PreMakeCacheChain files
1942                     self.GenLocalPreMakeCache()
1943                 self.BuildModules = []
1944                 self.MakeTime += int(round((time.time() - MakeContiue)))
1945                 if BuildTask.HasError():
1946                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1947
1948                 self.BuildReport.AddPlatformReport(Wa, MaList)
1949                 if MaList == []:
1950                     EdkLogger.error(
1951                                 'build',
1952                                 BUILD_ERROR,
1953                                 "Module for [%s] is not a component of active platform."\
1954                                 " Please make sure that the ARCH and inf file path are"\
1955                                 " given in the same as in [%s]" % \
1956                                     (', '.join(Wa.ArchList), self.PlatformFile),
1957                                 ExtraData=self.ModuleFile
1958                                 )
1959                 # Create MAP file when Load Fix Address is enabled.
1960                 if self.Target == "fds" and self.Fdf:
1961                     for Arch in Wa.ArchList:
1962                         #
1963                         # Check whether the set fix address is above 4G for 32bit image.
1964                         #
1965                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1966                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1967                     #
1968                     # Get Module List
1969                     #
1970                     ModuleList = {}
1971                     for Pa in Wa.AutoGenObjectList:
1972                         for Ma in Pa.ModuleAutoGenList:
1973                             if Ma is None:
1974                                 continue
1975                             if not Ma.IsLibrary:
1976                                 ModuleList[Ma.Guid.upper()] = Ma
1977
1978                     MapBuffer = []
1979                     if self.LoadFixAddress != 0:
1980                         #
1981                         # Rebase module to the preferred memory address before GenFds
1982                         #
1983                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1984                     #
1985                     # create FDS again for the updated EFI image
1986                     #
1987                     GenFdsStart = time.time()
1988                     self._Build("fds", Wa)
1989                     self.GenFdsTime += int(round((time.time() - GenFdsStart)))
1990                     #
1991                     # Create MAP file for all platform FVs after GenFds.
1992                     #
1993                     self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1994                     #
1995                     # Save MAP buffer into MAP file.
1996                     #
1997                     self._SaveMapFile (MapBuffer, Wa)
1998
1999     def _GenFfsCmd(self,ArchList):
2000         # convert dictionary of Cmd:(Inf,Arch)
2001         # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
2002         CmdSetDict = defaultdict(set)
2003         GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
2004         for Cmd in GenFfsDict:
2005             tmpInf, tmpArch = GenFfsDict[Cmd]
2006             CmdSetDict[tmpInf, tmpArch].add(Cmd)
2007         return CmdSetDict
2008     def VerifyAutoGenFiles(self):
2009         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2010         try:
2011             with open(AutoGenIdFile) as fd:
2012                 lines = fd.readlines()
2013         except:
2014             return None
2015         for line in lines:
2016             if "Arch" in line:
2017                 ArchList = line.strip().split("=")[1].split("|")
2018             if "BuildDir" in line:
2019                 BuildDir = line.split("=")[1].strip()
2020             if "PlatformGuid" in line:
2021                 PlatformGuid = line.split("=")[1].strip()
2022         GlobalVarList = []
2023         for arch in ArchList:
2024             global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
2025             if not os.path.exists(global_var):
2026                 return None
2027             GlobalVarList.append(global_var)
2028         for global_var in GlobalVarList:
2029             data_pipe = MemoryDataPipe()
2030             data_pipe.load(global_var)
2031             target = data_pipe.Get("P_Info").get("Target")
2032             toolchain = data_pipe.Get("P_Info").get("ToolChain")
2033             archlist = data_pipe.Get("P_Info").get("ArchList")
2034             Arch = data_pipe.Get("P_Info").get("Arch")
2035             active_p = data_pipe.Get("P_Info").get("ActivePlatform")
2036             workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
2037             PackagesPath = os.getenv("PACKAGES_PATH")
2038             mws.setWs(workspacedir, PackagesPath)
2039             LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
2040             ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
2041
2042             for m_build_dir in LibraryBuildDirectoryList:
2043                 if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
2044                     return None
2045             for m_build_dir in ModuleBuildDirectoryList:
2046                 if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
2047                     return None
2048             Wa = WorkSpaceInfo(
2049                 workspacedir,active_p,target,toolchain,archlist
2050                 )
2051             Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
2052             Wa.AutoGenObjectList.append(Pa)
2053         return Wa
2054     def SetupMakeSetting(self,Wa):
2055         BuildModules = []
2056         for Pa in Wa.AutoGenObjectList:
2057             for m in Pa._MbList:
2058                 ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
2059                 BuildModules.append(ma)
2060         fdf_file = Wa.FlashDefinition
2061         if fdf_file:
2062             Fdf = FdfParser(fdf_file.Path)
2063             Fdf.ParseFile()
2064             GlobalData.gFdfParser = Fdf
2065             if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
2066                 FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
2067                 for FdRegion in FdDict.RegionList:
2068                     if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
2069                         if int(FdRegion.Offset) % 8 != 0:
2070                             EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
2071             Wa.FdfProfile = Fdf.Profile
2072             self.Fdf = Fdf
2073         else:
2074             self.Fdf = None
2075         return BuildModules
2076
2077     ## Build a platform in multi-thread mode
2078     #
2079     def PerformAutoGen(self,BuildTarget,ToolChain):
2080         WorkspaceAutoGenTime = time.time()
2081         Wa = WorkspaceAutoGen(
2082                 self.WorkspaceDir,
2083                 self.PlatformFile,
2084                 BuildTarget,
2085                 ToolChain,
2086                 self.ArchList,
2087                 self.BuildDatabase,
2088                 self.TargetTxt,
2089                 self.ToolDef,
2090                 self.Fdf,
2091                 self.FdList,
2092                 self.FvList,
2093                 self.CapList,
2094                 self.SkuId,
2095                 self.UniFlag,
2096                 self.Progress
2097                 )
2098         self.Fdf = Wa.FdfFile
2099         self.LoadFixAddress = Wa.Platform.LoadFixAddress
2100         self.BuildReport.AddPlatformReport(Wa)
2101         Wa.CreateMakeFile(False)
2102
2103         # Add ffs build to makefile
2104         CmdListDict = {}
2105         if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
2106             CmdListDict = self._GenFfsCmd(Wa.ArchList)
2107
2108         self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
2109         BuildModules = []
2110         for Arch in Wa.ArchList:
2111             PcdMaList    = []
2112             AutoGenStart = time.time()
2113             GlobalData.gGlobalDefines['ARCH'] = Arch
2114             Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
2115             if Pa is None:
2116                 continue
2117             ModuleList = []
2118             for Inf in Pa.Platform.Modules:
2119                 ModuleList.append(Inf)
2120             # Add the INF only list in FDF
2121             if GlobalData.gFdfParser is not None:
2122                 for InfName in GlobalData.gFdfParser.Profile.InfList:
2123                     Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
2124                     if Inf in Pa.Platform.Modules:
2125                         continue
2126                     ModuleList.append(Inf)
2127             Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
2128             Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
2129             Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
2130             Pa.CreateLibModuelDirs()
2131             Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
2132             Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
2133             Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
2134             # Prepare the cache share data for multiprocessing
2135             Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}
2136             ModuleCodaFile = {}
2137             for ma in Pa.ModuleAutoGenList:
2138                 ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
2139             Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
2140             # ModuleList contains all driver modules only
2141             for Module in ModuleList:
2142                 # Get ModuleAutoGen object to generate C code file and makefile
2143                 Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2144                 if Ma is None:
2145                     continue
2146                 if Ma.PcdIsDriver:
2147                     Ma.PlatformInfo = Pa
2148                     Ma.Workspace = Wa
2149                     PcdMaList.append(Ma)
2150                 self.AllDrivers.add(Ma)
2151                 self.AllModules.add(Ma)
2152
2153             mqueue = mp.Queue()
2154             cqueue = mp.Queue()
2155             for m in Pa.GetAllModuleInfo:
2156                 mqueue.put(m)
2157                 module_file,module_root,module_path,module_basename,\
2158                     module_originalpath,module_arch,IsLib = m
2159                 Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\
2160                                   ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2161                 self.AllModules.add(Ma)
2162             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
2163             Pa.DataPipe.dump(data_pipe_file)
2164
2165             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
2166
2167             if not autogen_rt:
2168                 self.AutoGenMgr.TerminateWorkers()
2169                 self.AutoGenMgr.join(1)
2170                 raise FatalError(errorcode)
2171
2172             if GlobalData.gUseHashCache:
2173                 for item in GlobalData.gModuleAllCacheStatus:
2174                     (MetaFilePath, Arch, CacheStr, Status) = item
2175                     Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\
2176                                       ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2177                     if CacheStr == "PreMakeCache" and Status == False:
2178                         self.PreMakeCacheMiss.add(Ma)
2179                     if CacheStr == "PreMakeCache" and Status == True:
2180                         self.PreMakeCacheHit.add(Ma)
2181                         GlobalData.gModuleCacheHit.add(Ma)
2182                     if CacheStr == "MakeCache" and Status == False:
2183                         self.MakeCacheMiss.add(Ma)
2184                     if CacheStr == "MakeCache" and Status == True:
2185                         self.MakeCacheHit.add(Ma)
2186                         GlobalData.gModuleCacheHit.add(Ma)
2187             self.AutoGenTime += int(round((time.time() - AutoGenStart)))
2188         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2189         with open(AutoGenIdFile,"w") as fw:
2190             fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
2191             fw.write("BuildDir=%s\n" % Wa.BuildDir)
2192             fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
2193
2194         if GlobalData.gBinCacheSource:
2195             BuildModules.extend(self.MakeCacheMiss)
2196         elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:
2197             BuildModules.extend(self.PreMakeCacheMiss)
2198         else:
2199             BuildModules.extend(self.AllDrivers)
2200
2201         self.Progress.Stop("done!")
2202         return Wa, BuildModules
2203
2204     def _MultiThreadBuildPlatform(self):
2205         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
2206         for BuildTarget in self.BuildTargetList:
2207             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
2208             index = 0
2209             for ToolChain in self.ToolChainList:
2210                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
2211                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
2212                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
2213                 index += 1
2214                 ExitFlag = threading.Event()
2215                 ExitFlag.clear()
2216                 if self.SkipAutoGen:
2217                     Wa = self.VerifyAutoGenFiles()
2218                     if Wa is None:
2219                         self.SkipAutoGen = False
2220                         Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2221                     else:
2222                         GlobalData.gAutoGenPhase = True
2223                         self.BuildModules = self.SetupMakeSetting(Wa)
2224                 else:
2225                     Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2226                 Pa = Wa.AutoGenObjectList[0]
2227                 GlobalData.gAutoGenPhase = False
2228
2229                 if GlobalData.gBinCacheSource:
2230                     EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))
2231                     EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))
2232                     EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))
2233
2234                 for Arch in Wa.ArchList:
2235                     MakeStart = time.time()
2236                     for Ma in set(self.BuildModules):
2237                         # Generate build task for the module
2238                         if not Ma.IsBinaryModule:
2239                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
2240                         # Break build if any build thread has error
2241                         if BuildTask.HasError():
2242                             # we need a full version of makefile for platform
2243                             ExitFlag.set()
2244                             BuildTask.WaitForComplete()
2245                             Pa.CreateMakeFile(False)
2246                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2247                         # Start task scheduler
2248                         if not BuildTask.IsOnGoing():
2249                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
2250
2251                     # in case there's an interruption. we need a full version of makefile for platform
2252
2253                     if BuildTask.HasError():
2254                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2255                     self.MakeTime += int(round((time.time() - MakeStart)))
2256
2257                 MakeContiue = time.time()
2258                 #
2259                 #
2260                 # All modules have been put in build tasks queue. Tell task scheduler
2261                 # to exit if all tasks are completed
2262                 #
2263                 ExitFlag.set()
2264                 BuildTask.WaitForComplete()
2265                 self.CreateAsBuiltInf()
2266                 if GlobalData.gBinCacheDest:
2267                     self.GenDestCache()
2268                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
2269                     # Only for --hash
2270                     # Update PreMakeCacheChain files
2271                     self.GenLocalPreMakeCache()
2272                 #
2273                 # Get Module List
2274                 #
2275                 ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}
2276                 self.BuildModules = []
2277                 self.MakeTime += int(round((time.time() - MakeContiue)))
2278                 #
2279                 # Check for build error, and raise exception if one
2280                 # has been signaled.
2281                 #
2282                 if BuildTask.HasError():
2283                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2284
2285                 # Create MAP file when Load Fix Address is enabled.
2286                 if self.Target in ["", "all", "fds"]:
2287                     for Arch in Wa.ArchList:
2288                         #
2289                         # Check whether the set fix address is above 4G for 32bit image.
2290                         #
2291                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
2292                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2293
2294                     #
2295                     # Rebase module to the preferred memory address before GenFds
2296                     #
2297                     MapBuffer = []
2298                     if self.LoadFixAddress != 0:
2299                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
2300
2301                     if self.Fdf:
2302                         #
2303                         # Generate FD image if there's a FDF file found
2304                         #
2305                         GenFdsStart = time.time()
2306                         if GenFdsApi(Wa.GenFdsCommandDict, self.Db):
2307                             EdkLogger.error("build", COMMAND_FAILURE)
2308
2309                         #
2310                         # Create MAP file for all platform FVs after GenFds.
2311                         #
2312                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2313                         self.GenFdsTime += int(round((time.time() - GenFdsStart)))
2314                     #
2315                     # Save MAP buffer into MAP file.
2316                     #
2317                     self._SaveMapFile(MapBuffer, Wa)
2318                 self.CreateGuidedSectionToolsFile(Wa)
2319     ## Generate GuidedSectionTools.txt in the FV directories.
2320     #
2321     def CreateGuidedSectionToolsFile(self,Wa):
2322         for BuildTarget in self.BuildTargetList:
2323             for ToolChain in self.ToolChainList:
2324                 FvDir = Wa.FvDir
2325                 if not os.path.exists(FvDir):
2326                     continue
2327
2328                 for Arch in self.ArchList:
2329                     # Build up the list of supported architectures for this build
2330                     prefix = '%s_%s_%s_' % (BuildTarget, ToolChain, Arch)
2331
2332                     # Look through the tool definitions for GUIDed tools
2333                     guidAttribs = []
2334                     for (attrib, value) in self.ToolDef.ToolsDefTxtDictionary.items():
2335                         if attrib.upper().endswith('_GUID'):
2336                             split = attrib.split('_')
2337                             thisPrefix = '_'.join(split[0:3]) + '_'
2338                             if thisPrefix == prefix:
2339                                 guid = self.ToolDef.ToolsDefTxtDictionary[attrib]
2340                                 guid = guid.lower()
2341                                 toolName = split[3]
2342                                 path = '_'.join(split[0:4]) + '_PATH'
2343                                 path = self.ToolDef.ToolsDefTxtDictionary[path]
2344                                 path = self.GetFullPathOfTool(path)
2345                                 guidAttribs.append((guid, toolName, path))
2346
2347                     # Write out GuidedSecTools.txt
2348                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
2349                     toolsFile = open(toolsFile, 'wt')
2350                     for guidedSectionTool in guidAttribs:
2351                         print(' '.join(guidedSectionTool), file=toolsFile)
2352                     toolsFile.close()
2353
2354     ## Returns the full path of the tool.
2355     #
2356     def GetFullPathOfTool (self, tool):
2357         if os.path.exists(tool):
2358             return os.path.realpath(tool)
2359         else:
2360             # We need to search for the tool using the
2361             # PATH environment variable.
2362             for dirInPath in os.environ['PATH'].split(os.pathsep):
2363                 foundPath = os.path.join(dirInPath, tool)
2364                 if os.path.exists(foundPath):
2365                     return os.path.realpath(foundPath)
2366
2367         # If the tool was not found in the path then we just return
2368         # the input tool.
2369         return tool
2370
2371     ## Launch the module or platform build
2372     #
2373     def Launch(self):
2374         self.AllDrivers = set()
2375         self.AllModules = set()
2376         self.PreMakeCacheMiss = set()
2377         self.PreMakeCacheHit = set()
2378         self.MakeCacheMiss = set()
2379         self.MakeCacheHit = set()
2380         if not self.ModuleFile:
2381             if not self.SpawnMode or self.Target not in ["", "all"]:
2382                 self.SpawnMode = False
2383                 self._BuildPlatform()
2384             else:
2385                 self._MultiThreadBuildPlatform()
2386         else:
2387             self.SpawnMode = False
2388             self._BuildModule()
2389
2390         if self.Target == 'cleanall':
2391             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
2392
2393     def CreateAsBuiltInf(self):
2394         for Module in self.BuildModules:
2395             Module.CreateAsBuiltInf()
2396
2397     def GenDestCache(self):
2398         for Module in self.AllModules:
2399             Module.GenPreMakefileHashList()
2400             Module.GenMakefileHashList()
2401             Module.CopyModuleToCache()
2402
2403     def GenLocalPreMakeCache(self):
2404         for Module in self.PreMakeCacheMiss:
2405             Module.GenPreMakefileHashList()
2406
2407     ## Do some clean-up works when error occurred
2408     def Relinquish(self):
2409         OldLogLevel = EdkLogger.GetLevel()
2410         EdkLogger.SetLevel(EdkLogger.ERROR)
2411         Utils.Progressor.Abort()
2412         if self.SpawnMode == True:
2413             BuildTask.Abort()
2414         EdkLogger.SetLevel(OldLogLevel)
2415
2416 def ParseDefines(DefineList=[]):
2417     DefineDict = {}
2418     if DefineList is not None:
2419         for Define in DefineList:
2420             DefineTokenList = Define.split("=", 1)
2421             if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]):
2422                 EdkLogger.error('build', FORMAT_INVALID,
2423                                 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2424                                 ExtraData=DefineTokenList[0])
2425
2426             if len(DefineTokenList) == 1:
2427                 DefineDict[DefineTokenList[0]] = "TRUE"
2428             else:
2429                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
2430     return DefineDict
2431
2432
2433
2434 def LogBuildTime(Time):
2435     if Time:
2436         TimeDurStr = ''
2437         TimeDur = time.gmtime(Time)
2438         if TimeDur.tm_yday > 1:
2439             TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + ", %d day(s)" % (TimeDur.tm_yday - 1)
2440         else:
2441             TimeDurStr = time.strftime("%H:%M:%S", TimeDur)
2442         return TimeDurStr
2443     else:
2444         return None
2445 def ThreadNum():
2446     ThreadNumber = BuildOption.ThreadNumber
2447     if ThreadNumber is None:
2448         ThreadNumber = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
2449         if ThreadNumber == '':
2450             ThreadNumber = 0
2451         else:
2452             ThreadNumber = int(ThreadNumber, 0)
2453
2454     if ThreadNumber == 0:
2455         try:
2456             ThreadNumber = multiprocessing.cpu_count()
2457         except (ImportError, NotImplementedError):
2458             ThreadNumber = 1
2459     return ThreadNumber
2460 ## Tool entrance method
2461 #
2462 # This method mainly dispatch specific methods per the command line options.
2463 # If no error found, return zero value so the caller of this tool can know
2464 # if it's executed successfully or not.
2465 #
2466 #   @retval 0     Tool was successful
2467 #   @retval 1     Tool failed
2468 #
2469 LogQMaxSize = ThreadNum() * 10
2470 def Main():
2471     StartTime = time.time()
2472
2473     #
2474     # Create a log Queue
2475     #
2476     LogQ = mp.Queue(LogQMaxSize)
2477     # Initialize log system
2478     EdkLogger.LogClientInitialize(LogQ)
2479     GlobalData.gCommand = sys.argv[1:]
2480     #
2481     # Parse the options and args
2482     #
2483     Option, Target = BuildOption, BuildTarget
2484     GlobalData.gOptions = Option
2485     GlobalData.gCaseInsensitive = Option.CaseInsensitive
2486
2487     # Set log level
2488     LogLevel = EdkLogger.INFO
2489     if Option.verbose is not None:
2490         EdkLogger.SetLevel(EdkLogger.VERBOSE)
2491         LogLevel = EdkLogger.VERBOSE
2492     elif Option.quiet is not None:
2493         EdkLogger.SetLevel(EdkLogger.QUIET)
2494         LogLevel = EdkLogger.QUIET
2495     elif Option.debug is not None:
2496         EdkLogger.SetLevel(Option.debug + 1)
2497         LogLevel = Option.debug + 1
2498     else:
2499         EdkLogger.SetLevel(EdkLogger.INFO)
2500
2501     if Option.WarningAsError == True:
2502         EdkLogger.SetWarningAsError()
2503     Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile)
2504     Log_Agent.start()
2505
2506     if platform.platform().find("Windows") >= 0:
2507         GlobalData.gIsWindows = True
2508     else:
2509         GlobalData.gIsWindows = False
2510
2511     EdkLogger.quiet("Build environment: %s" % platform.platform())
2512     EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime()));
2513     ReturnCode = 0
2514     MyBuild = None
2515     BuildError = True
2516     try:
2517         if len(Target) == 0:
2518             Target = "all"
2519         elif len(Target) >= 2:
2520             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
2521                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2522         else:
2523             Target = Target[0].lower()
2524
2525         if Target not in gSupportedTarget:
2526             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
2527                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2528
2529         #
2530         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2531         #
2532         CheckEnvVariable()
2533         GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros))
2534
2535         Workspace = os.getenv("WORKSPACE")
2536         #
2537         # Get files real name in workspace dir
2538         #
2539         GlobalData.gAllFiles = Utils.DirCache(Workspace)
2540
2541         WorkingDirectory = os.getcwd()
2542         if not Option.ModuleFile:
2543             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
2544             FileNum = len(FileList)
2545             if FileNum >= 2:
2546                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
2547                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2548             elif FileNum == 1:
2549                 Option.ModuleFile = NormFile(FileList[0], Workspace)
2550
2551         if Option.ModuleFile:
2552             if os.path.isabs (Option.ModuleFile):
2553                 if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0:
2554                     Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace)
2555             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
2556             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
2557             if ErrorCode != 0:
2558                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2559
2560         if Option.PlatformFile is not None:
2561             if os.path.isabs (Option.PlatformFile):
2562                 if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0:
2563                     Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace)
2564             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
2565
2566         if Option.FdfFile is not None:
2567             if os.path.isabs (Option.FdfFile):
2568                 if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0:
2569                     Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace)
2570             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
2571             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
2572             if ErrorCode != 0:
2573                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2574
2575         if Option.Flag is not None and Option.Flag not in ['-c', '-s']:
2576             EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s")
2577
2578         MyBuild = Build(Target, Workspace, Option,LogQ)
2579         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
2580         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
2581             MyBuild.Launch()
2582
2583         #
2584         # All job done, no error found and no exception raised
2585         #
2586         BuildError = False
2587     except FatalError as X:
2588         if MyBuild is not None:
2589             # for multi-thread build exits safely
2590             MyBuild.Relinquish()
2591         if Option is not None and Option.debug is not None:
2592             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2593         ReturnCode = X.args[0]
2594     except Warning as X:
2595         # error from Fdf parser
2596         if MyBuild is not None:
2597             # for multi-thread build exits safely
2598             MyBuild.Relinquish()
2599         if Option is not None and Option.debug is not None:
2600             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2601         else:
2602             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
2603         ReturnCode = FORMAT_INVALID
2604     except KeyboardInterrupt:
2605         if MyBuild is not None:
2606
2607             # for multi-thread build exits safely
2608             MyBuild.Relinquish()
2609         ReturnCode = ABORT_ERROR
2610         if Option is not None and Option.debug is not None:
2611             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2612     except:
2613         if MyBuild is not None:
2614             # for multi-thread build exits safely
2615             MyBuild.Relinquish()
2616
2617         # try to get the meta-file from the object causing exception
2618         Tb = sys.exc_info()[-1]
2619         MetaFile = GlobalData.gProcessingFile
2620         while Tb is not None:
2621             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
2622                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
2623             Tb = Tb.tb_next
2624         EdkLogger.error(
2625                     "\nbuild",
2626                     CODE_ERROR,
2627                     "Unknown fatal error when processing [%s]" % MetaFile,
2628                     ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR,
2629                     RaiseError=False
2630                     )
2631         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2632         ReturnCode = CODE_ERROR
2633     finally:
2634         Utils.Progressor.Abort()
2635         Utils.ClearDuplicatedInf()
2636
2637     if ReturnCode == 0:
2638         try:
2639             MyBuild.LaunchPostbuild()
2640             Conclusion = "Done"
2641         except:
2642             Conclusion = "Failed"
2643     elif ReturnCode == ABORT_ERROR:
2644         Conclusion = "Aborted"
2645     else:
2646         Conclusion = "Failed"
2647     FinishTime = time.time()
2648     BuildDuration = time.gmtime(int(round(FinishTime - StartTime)))
2649     BuildDurationStr = ""
2650     if BuildDuration.tm_yday > 1:
2651         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1)
2652     else:
2653         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
2654     if MyBuild is not None:
2655         if not BuildError:
2656             MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
2657
2658     EdkLogger.SetLevel(EdkLogger.QUIET)
2659     EdkLogger.quiet("\n- %s -" % Conclusion)
2660     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
2661     EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
2662     Log_Agent.kill()
2663     Log_Agent.join()
2664     return ReturnCode
2665
2666 if __name__ == '__main__':
2667     try:
2668         mp.set_start_method('spawn')
2669     except:
2670         pass
2671     r = Main()
2672     ## 0-127 is a safe return range, and 1 is a standard default error
2673     if r < 0 or r > 127: r = 1
2674     sys.exit(r)
2675