BaseTools/Source/Python/build/build.py

   1 ## @file
   2 # build a platform or a module
   3 #
   4 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
   5 #  Copyright (c) 2007 - 2021, Intel Corporation. All rights reserved.<BR>
   6 #  Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
   7 #  Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
   8 #
   9 #  SPDX-License-Identifier: BSD-2-Clause-Patent
  10 #
  11
  12 ##
  13 # Import Modules
  14 #
  15 from __future__ import print_function
  16 from __future__ import absolute_import
  17 import os.path as path
  18 import sys
  19 import os
  20 import re
  21 import glob
  22 import time
  23 import platform
  24 import traceback
  25 import multiprocessing
  26 from threading import Thread,Event,BoundedSemaphore
  27 import threading
  28 from linecache import getlines
  29 from subprocess import Popen,PIPE, STDOUT
  30 from collections import OrderedDict, defaultdict
  31
  32 from AutoGen.PlatformAutoGen import PlatformAutoGen
  33 from AutoGen.ModuleAutoGen import ModuleAutoGen
  34 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen
  35 from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager,\
  36     LogAgent
  37 from AutoGen import GenMake
  38 from Common import Misc as Utils
  39
  40 from Common.TargetTxtClassObject import TargetTxtDict
  41 from Common.ToolDefClassObject import ToolDefDict
  42 from buildoptions import MyOptionParser
  43 from Common.Misc import PathClass,SaveFileOnChange,RemoveDirectory
  44 from Common.StringUtils import NormPath
  45 from Common.MultipleWorkspace import MultipleWorkspace as mws
  46 from Common.BuildToolError import *
  47 from Common.DataType import *
  48 import Common.EdkLogger as EdkLogger
  49
  50 from Workspace.WorkspaceDatabase import BuildDB
  51
  52 from BuildReport import BuildReport
  53 from GenPatchPcdTable.GenPatchPcdTable import PeImageClass,parsePcdInfoFromMapFile
  54 from PatchPcdValue.PatchPcdValue import PatchBinaryFile
  55
  56 import Common.GlobalData as GlobalData
  57 from GenFds.GenFds import GenFds, GenFdsApi
  58 import multiprocessing as mp
  59 from multiprocessing import Manager
  60 from AutoGen.DataPipe import MemoryDataPipe
  61 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
  62 from GenFds.FdfParser import FdfParser
  63 from AutoGen.IncludesAutoGen import IncludesAutoGen
  64 from GenFds.GenFds import resetFdsGlobalVariable
  65 from AutoGen.AutoGen import CalculatePriorityValue
  66
  67 ## standard targets of build command
  68 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
  69
  70 ## build configuration file
  71 gBuildConfiguration = "target.txt"
  72 gToolsDefinition = "tools_def.txt"
  73
  74 TemporaryTablePattern = re.compile(r'^_\d+_\d+_[a-fA-F0-9]+$')
  75 TmpTableDict = {}
  76
  77 ## Check environment PATH variable to make sure the specified tool is found
  78 #
  79 #   If the tool is found in the PATH, then True is returned
  80 #   Otherwise, False is returned
  81 #
  82 def IsToolInPath(tool):
  83     if 'PATHEXT' in os.environ:
  84         extns = os.environ['PATHEXT'].split(os.path.pathsep)
  85     else:
  86         extns = ('',)
  87     for pathDir in os.environ['PATH'].split(os.path.pathsep):
  88         for ext in extns:
  89             if os.path.exists(os.path.join(pathDir, tool + ext)):
  90                 return True
  91     return False
  92
  93 ## Check environment variables
  94 #
  95 #  Check environment variables that must be set for build. Currently they are
  96 #
  97 #   WORKSPACE           The directory all packages/platforms start from
  98 #   EDK_TOOLS_PATH      The directory contains all tools needed by the build
  99 #   PATH                $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
 100 #
 101 #   If any of above environment variable is not set or has error, the build
 102 #   will be broken.
 103 #
 104 def CheckEnvVariable():
 105     # check WORKSPACE
 106     if "WORKSPACE" not in os.environ:
 107         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 108                         ExtraData="WORKSPACE")
 109
 110     WorkspaceDir = os.path.normcase(os.path.normpath(os.environ["WORKSPACE"]))
 111     if not os.path.exists(WorkspaceDir):
 112         EdkLogger.error("build", FILE_NOT_FOUND, "WORKSPACE doesn't exist", ExtraData=WorkspaceDir)
 113     elif ' ' in WorkspaceDir:
 114         EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in WORKSPACE path",
 115                         ExtraData=WorkspaceDir)
 116     os.environ["WORKSPACE"] = WorkspaceDir
 117
 118     # set multiple workspace
 119     PackagesPath = os.getenv("PACKAGES_PATH")
 120     mws.setWs(WorkspaceDir, PackagesPath)
 121     if mws.PACKAGES_PATH:
 122         for Path in mws.PACKAGES_PATH:
 123             if not os.path.exists(Path):
 124                 EdkLogger.error("build", FILE_NOT_FOUND, "One Path in PACKAGES_PATH doesn't exist", ExtraData=Path)
 125             elif ' ' in Path:
 126                 EdkLogger.error("build", FORMAT_NOT_SUPPORTED, "No space is allowed in PACKAGES_PATH", ExtraData=Path)
 127
 128
 129     os.environ["EDK_TOOLS_PATH"] = os.path.normcase(os.environ["EDK_TOOLS_PATH"])
 130
 131     # check EDK_TOOLS_PATH
 132     if "EDK_TOOLS_PATH" not in os.environ:
 133         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 134                         ExtraData="EDK_TOOLS_PATH")
 135
 136     # check PATH
 137     if "PATH" not in os.environ:
 138         EdkLogger.error("build", ATTRIBUTE_NOT_AVAILABLE, "Environment variable not found",
 139                         ExtraData="PATH")
 140
 141     GlobalData.gWorkspace = WorkspaceDir
 142
 143     GlobalData.gGlobalDefines["WORKSPACE"]  = WorkspaceDir
 144     GlobalData.gGlobalDefines["EDK_TOOLS_PATH"] = os.environ["EDK_TOOLS_PATH"]
 145
 146 ## Get normalized file path
 147 #
 148 # Convert the path to be local format, and remove the WORKSPACE path at the
 149 # beginning if the file path is given in full path.
 150 #
 151 # @param  FilePath      File path to be normalized
 152 # @param  Workspace     Workspace path which the FilePath will be checked against
 153 #
 154 # @retval string        The normalized file path
 155 #
 156 def NormFile(FilePath, Workspace):
 157     # check if the path is absolute or relative
 158     if os.path.isabs(FilePath):
 159         FileFullPath = os.path.normpath(FilePath)
 160     else:
 161         FileFullPath = os.path.normpath(mws.join(Workspace, FilePath))
 162         Workspace = mws.getWs(Workspace, FilePath)
 163
 164     # check if the file path exists or not
 165     if not os.path.isfile(FileFullPath):
 166         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath)
 167
 168     # remove workspace directory from the beginning part of the file path
 169     if Workspace[-1] in ["\\", "/"]:
 170         return FileFullPath[len(Workspace):]
 171     else:
 172         return FileFullPath[(len(Workspace) + 1):]
 173
 174 ## Get the output of an external program
 175 #
 176 # This is the entrance method of thread reading output of an external program and
 177 # putting them in STDOUT/STDERR of current program.
 178 #
 179 # @param  From      The stream message read from
 180 # @param  To        The stream message put on
 181 # @param  ExitFlag  The flag used to indicate stopping reading
 182 #
 183 def ReadMessage(From, To, ExitFlag,MemTo=None):
 184     while True:
 185         # read one line a time
 186         Line = From.readline()
 187         # empty string means "end"
 188         if Line is not None and Line != b"":
 189             LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
 190             if MemTo is not None:
 191                 if "Note: including file:" ==  LineStr.lstrip()[:21]:
 192                     MemTo.append(LineStr)
 193                 else:
 194                     To(LineStr)
 195                     MemTo.append(LineStr)
 196             else:
 197                 To(LineStr)
 198         else:
 199             break
 200         if ExitFlag.isSet():
 201             break
 202
 203 class MakeSubProc(Popen):
 204     def __init__(self,*args, **argv):
 205         super(MakeSubProc,self).__init__(*args, **argv)
 206         self.ProcOut = []
 207
 208 ## Launch an external program
 209 #
 210 # This method will call subprocess.Popen to execute an external program with
 211 # given options in specified directory. Because of the dead-lock issue during
 212 # redirecting output of the external program, threads are used to to do the
 213 # redirection work.
 214 #
 215 # @param  Command               A list or string containing the call of the program
 216 # @param  WorkingDir            The directory in which the program will be running
 217 #
 218 def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
 219     BeginTime = time.time()
 220     # if working directory doesn't exist, Popen() will raise an exception
 221     if not os.path.isdir(WorkingDir):
 222         EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
 223
 224     # Command is used as the first Argument in following Popen().
 225     # It could be a string or sequence. We find that if command is a string in following Popen(),
 226     # ubuntu may fail with an error message that the command is not found.
 227     # So here we may need convert command from string to list instance.
 228     if platform.system() != 'Windows':
 229         if not isinstance(Command, list):
 230             Command = Command.split()
 231         Command = ' '.join(Command)
 232
 233     Proc = None
 234     EndOfProcedure = None
 235     try:
 236         # launch the command
 237         Proc = MakeSubProc(Command, stdout=PIPE, stderr=STDOUT, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
 238
 239         # launch two threads to read the STDOUT and STDERR
 240         EndOfProcedure = Event()
 241         EndOfProcedure.clear()
 242         if Proc.stdout:
 243             StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
 244             StdOutThread.setName("STDOUT-Redirector")
 245             StdOutThread.setDaemon(False)
 246             StdOutThread.start()
 247
 248
 249         # waiting for program exit
 250         Proc.wait()
 251     except: # in case of aborting
 252         # terminate the threads redirecting the program output
 253         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 254         if EndOfProcedure is not None:
 255             EndOfProcedure.set()
 256         if Proc is None:
 257             if not isinstance(Command, type("")):
 258                 Command = " ".join(Command)
 259             EdkLogger.error("build", COMMAND_FAILURE, "Failed to start command", ExtraData="%s [%s]" % (Command, WorkingDir))
 260
 261     if Proc.stdout:
 262         StdOutThread.join()
 263
 264     # check the return code of the program
 265     if Proc.returncode != 0:
 266         if not isinstance(Command, type("")):
 267             Command = " ".join(Command)
 268         # print out the Response file and its content when make failure
 269         RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')
 270         if os.path.isfile(RespFile):
 271             f = open(RespFile)
 272             RespContent = f.read()
 273             f.close()
 274             EdkLogger.info(RespContent)
 275
 276         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
 277     if ModuleAuto:
 278         iau = IncludesAutoGen(WorkingDir,ModuleAuto)
 279         if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
 280             iau.CreateDepsFileForMsvc(Proc.ProcOut)
 281         else:
 282             iau.UpdateDepsFileforNonMsvc()
 283         iau.UpdateDepsFileforTrim()
 284         iau.CreateModuleDeps()
 285         iau.CreateDepsInclude()
 286         iau.CreateDepsTarget()
 287     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
 288
 289 ## The smallest unit that can be built in multi-thread build mode
 290 #
 291 # This is the base class of build unit. The "Obj" parameter must provide
 292 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
 293 # missing build.
 294 #
 295 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
 296 #
 297 class BuildUnit:
 298     ## The constructor
 299     #
 300     #   @param  self        The object pointer
 301     #   @param  Obj         The object the build is working on
 302     #   @param  Target      The build target name, one of gSupportedTarget
 303     #   @param  Dependency  The BuildUnit(s) which must be completed in advance
 304     #   @param  WorkingDir  The directory build command starts in
 305     #
 306     def __init__(self, Obj, BuildCommand, Target, Dependency, WorkingDir="."):
 307         self.BuildObject = Obj
 308         self.Dependency = Dependency
 309         self.WorkingDir = WorkingDir
 310         self.Target = Target
 311         self.BuildCommand = BuildCommand
 312         if not BuildCommand:
 313             EdkLogger.error("build", OPTION_MISSING,
 314                             "No build command found for this module. "
 315                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
 316                                 (Obj.BuildTarget, Obj.ToolChain, Obj.Arch),
 317                             ExtraData=str(Obj))
 318
 319
 320     ## str() method
 321     #
 322     #   It just returns the string representation of self.BuildObject
 323     #
 324     #   @param  self        The object pointer
 325     #
 326     def __str__(self):
 327         return str(self.BuildObject)
 328
 329     ## "==" operator method
 330     #
 331     #   It just compares self.BuildObject with "Other". So self.BuildObject must
 332     #   provide its own __eq__() method.
 333     #
 334     #   @param  self        The object pointer
 335     #   @param  Other       The other BuildUnit object compared to
 336     #
 337     def __eq__(self, Other):
 338         return Other and self.BuildObject == Other.BuildObject \
 339                 and Other.BuildObject \
 340                 and self.BuildObject.Arch == Other.BuildObject.Arch
 341
 342     ## hash() method
 343     #
 344     #   It just returns the hash value of self.BuildObject which must be hashable.
 345     #
 346     #   @param  self        The object pointer
 347     #
 348     def __hash__(self):
 349         return hash(self.BuildObject) + hash(self.BuildObject.Arch)
 350
 351     def __repr__(self):
 352         return repr(self.BuildObject)
 353
 354 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
 355 #
 356 # This class is for module build by nmake/make build system. The "Obj" parameter
 357 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 358 # be make units missing build.
 359 #
 360 # Currently the "Obj" should be only ModuleAutoGen object.
 361 #
 362 class ModuleMakeUnit(BuildUnit):
 363     ## The constructor
 364     #
 365     #   @param  self        The object pointer
 366     #   @param  Obj         The ModuleAutoGen object the build is working on
 367     #   @param  Target      The build target name, one of gSupportedTarget
 368     #
 369     def __init__(self, Obj, BuildCommand,Target):
 370         Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]
 371         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 372         if Target in [None, "", "all"]:
 373             self.Target = "tbuild"
 374
 375 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
 376 #
 377 # This class is for platform build by nmake/make build system. The "Obj" parameter
 378 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
 379 # be make units missing build.
 380 #
 381 # Currently the "Obj" should be only PlatformAutoGen object.
 382 #
 383 class PlatformMakeUnit(BuildUnit):
 384     ## The constructor
 385     #
 386     #   @param  self        The object pointer
 387     #   @param  Obj         The PlatformAutoGen object the build is working on
 388     #   @param  Target      The build target name, one of gSupportedTarget
 389     #
 390     def __init__(self, Obj, BuildCommand, Target):
 391         Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]
 392         Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])
 393         BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)
 394
 395 ## The class representing the task of a module build or platform build
 396 #
 397 # This class manages the build tasks in multi-thread build mode. Its jobs include
 398 # scheduling thread running, catching thread error, monitor the thread status, etc.
 399 #
 400 class BuildTask:
 401     # queue for tasks waiting for schedule
 402     _PendingQueue = OrderedDict()
 403     _PendingQueueLock = threading.Lock()
 404
 405     # queue for tasks ready for running
 406     _ReadyQueue = OrderedDict()
 407     _ReadyQueueLock = threading.Lock()
 408
 409     # queue for run tasks
 410     _RunningQueue = OrderedDict()
 411     _RunningQueueLock = threading.Lock()
 412
 413     # queue containing all build tasks, in case duplicate build
 414     _TaskQueue = OrderedDict()
 415
 416     # flag indicating error occurs in a running thread
 417     _ErrorFlag = threading.Event()
 418     _ErrorFlag.clear()
 419     _ErrorMessage = ""
 420
 421     # BoundedSemaphore object used to control the number of running threads
 422     _Thread = None
 423
 424     # flag indicating if the scheduler is started or not
 425     _SchedulerStopped = threading.Event()
 426     _SchedulerStopped.set()
 427
 428     ## Start the task scheduler thread
 429     #
 430     #   @param  MaxThreadNumber     The maximum thread number
 431     #   @param  ExitFlag            Flag used to end the scheduler
 432     #
 433     @staticmethod
 434     def StartScheduler(MaxThreadNumber, ExitFlag):
 435         SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
 436         SchedulerThread.setName("Build-Task-Scheduler")
 437         SchedulerThread.setDaemon(False)
 438         SchedulerThread.start()
 439         # wait for the scheduler to be started, especially useful in Linux
 440         while not BuildTask.IsOnGoing():
 441             time.sleep(0.01)
 442
 443     ## Scheduler method
 444     #
 445     #   @param  MaxThreadNumber     The maximum thread number
 446     #   @param  ExitFlag            Flag used to end the scheduler
 447     #
 448     @staticmethod
 449     def Scheduler(MaxThreadNumber, ExitFlag):
 450         BuildTask._SchedulerStopped.clear()
 451         try:
 452             # use BoundedSemaphore to control the maximum running threads
 453             BuildTask._Thread = BoundedSemaphore(MaxThreadNumber)
 454             #
 455             # scheduling loop, which will exits when no pending/ready task and
 456             # indicated to do so, or there's error in running thread
 457             #
 458             while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
 459                    or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
 460                 EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
 461                                 % (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
 462
 463                 # get all pending tasks
 464                 BuildTask._PendingQueueLock.acquire()
 465                 BuildObjectList = list(BuildTask._PendingQueue.keys())
 466                 #
 467                 # check if their dependency is resolved, and if true, move them
 468                 # into ready queue
 469                 #
 470                 for BuildObject in BuildObjectList:
 471                     Bt = BuildTask._PendingQueue[BuildObject]
 472                     if Bt.IsReady():
 473                         BuildTask._ReadyQueue[BuildObject] = BuildTask._PendingQueue.pop(BuildObject)
 474                 BuildTask._PendingQueueLock.release()
 475
 476                 # launch build thread until the maximum number of threads is reached
 477                 while not BuildTask._ErrorFlag.isSet():
 478                     # empty ready queue, do nothing further
 479                     if len(BuildTask._ReadyQueue) == 0:
 480                         break
 481
 482                     # wait for active thread(s) exit
 483                     BuildTask._Thread.acquire(True)
 484
 485                     # start a new build thread
 486                     Bo, Bt = BuildTask._ReadyQueue.popitem()
 487
 488                     # move into running queue
 489                     BuildTask._RunningQueueLock.acquire()
 490                     BuildTask._RunningQueue[Bo] = Bt
 491                     BuildTask._RunningQueueLock.release()
 492
 493                     Bt.Start()
 494                     # avoid tense loop
 495                     time.sleep(0.01)
 496
 497                 # avoid tense loop
 498                 time.sleep(0.01)
 499
 500             # wait for all running threads exit
 501             if BuildTask._ErrorFlag.isSet():
 502                 EdkLogger.quiet("\nWaiting for all build threads exit...")
 503             # while not BuildTask._ErrorFlag.isSet() and \
 504             while len(BuildTask._RunningQueue) > 0:
 505                 EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
 506                 EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))
 507                 # avoid tense loop
 508                 time.sleep(0.1)
 509         except BaseException as X:
 510             #
 511             # TRICK: hide the output of threads left running, so that the user can
 512             #        catch the error message easily
 513             #
 514             EdkLogger.SetLevel(EdkLogger.ERROR)
 515             BuildTask._ErrorFlag.set()
 516             BuildTask._ErrorMessage = "build thread scheduler error\n\t%s" % str(X)
 517
 518         BuildTask._PendingQueue.clear()
 519         BuildTask._ReadyQueue.clear()
 520         BuildTask._RunningQueue.clear()
 521         BuildTask._TaskQueue.clear()
 522         BuildTask._SchedulerStopped.set()
 523
 524     ## Wait for all running method exit
 525     #
 526     @staticmethod
 527     def WaitForComplete():
 528         BuildTask._SchedulerStopped.wait()
 529
 530     ## Check if the scheduler is running or not
 531     #
 532     @staticmethod
 533     def IsOnGoing():
 534         return not BuildTask._SchedulerStopped.isSet()
 535
 536     ## Abort the build
 537     @staticmethod
 538     def Abort():
 539         if BuildTask.IsOnGoing():
 540             BuildTask._ErrorFlag.set()
 541             BuildTask.WaitForComplete()
 542
 543     ## Check if there's error in running thread
 544     #
 545     #   Since the main thread cannot catch exceptions in other thread, we have to
 546     #   use threading.Event to communicate this formation to main thread.
 547     #
 548     @staticmethod
 549     def HasError():
 550         return BuildTask._ErrorFlag.isSet()
 551
 552     ## Get error message in running thread
 553     #
 554     #   Since the main thread cannot catch exceptions in other thread, we have to
 555     #   use a static variable to communicate this message to main thread.
 556     #
 557     @staticmethod
 558     def GetErrorMessage():
 559         return BuildTask._ErrorMessage
 560
 561     ## Factory method to create a BuildTask object
 562     #
 563     #   This method will check if a module is building or has been built. And if
 564     #   true, just return the associated BuildTask object in the _TaskQueue. If
 565     #   not, create and return a new BuildTask object. The new BuildTask object
 566     #   will be appended to the _PendingQueue for scheduling later.
 567     #
 568     #   @param  BuildItem       A BuildUnit object representing a build object
 569     #   @param  Dependency      The dependent build object of BuildItem
 570     #
 571     @staticmethod
 572     def New(BuildItem, Dependency=None):
 573         if BuildItem in BuildTask._TaskQueue:
 574             Bt = BuildTask._TaskQueue[BuildItem]
 575             return Bt
 576
 577         Bt = BuildTask()
 578         Bt._Init(BuildItem, Dependency)
 579         BuildTask._TaskQueue[BuildItem] = Bt
 580
 581         BuildTask._PendingQueueLock.acquire()
 582         BuildTask._PendingQueue[BuildItem] = Bt
 583         BuildTask._PendingQueueLock.release()
 584
 585         return Bt
 586
 587     ## The real constructor of BuildTask
 588     #
 589     #   @param  BuildItem       A BuildUnit object representing a build object
 590     #   @param  Dependency      The dependent build object of BuildItem
 591     #
 592     def _Init(self, BuildItem, Dependency=None):
 593         self.BuildItem = BuildItem
 594
 595         self.DependencyList = []
 596         if Dependency is None:
 597             Dependency = BuildItem.Dependency
 598         else:
 599             Dependency.extend(BuildItem.Dependency)
 600         self.AddDependency(Dependency)
 601         # flag indicating build completes, used to avoid unnecessary re-build
 602         self.CompleteFlag = False
 603
 604     ## Check if all dependent build tasks are completed or not
 605     #
 606     def IsReady(self):
 607         ReadyFlag = True
 608         for Dep in self.DependencyList:
 609             if Dep.CompleteFlag == True:
 610                 continue
 611             ReadyFlag = False
 612             break
 613
 614         return ReadyFlag
 615
 616     ## Add dependent build task
 617     #
 618     #   @param  Dependency      The list of dependent build objects
 619     #
 620     def AddDependency(self, Dependency):
 621         for Dep in Dependency:
 622             if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):
 623                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list
 624
 625     ## The thread wrapper of LaunchCommand function
 626     #
 627     # @param  Command               A list or string contains the call of the command
 628     # @param  WorkingDir            The directory in which the program will be running
 629     #
 630     def _CommandThread(self, Command, WorkingDir):
 631         try:
 632             self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
 633             self.CompleteFlag = True
 634
 635             # Run hash operation post dependency to account for libs
 636             # Run if --hash or --binary-destination
 637             if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
 638                 self.BuildItem.BuildObject.GenModuleHash()
 639             if GlobalData.gBinCacheDest:
 640                 self.BuildItem.BuildObject.GenCMakeHash()
 641
 642         except:
 643             #
 644             # TRICK: hide the output of threads left running, so that the user can
 645             #        catch the error message easily
 646             #
 647             if not BuildTask._ErrorFlag.isSet():
 648                 GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
 649                                                                   self.BuildItem.BuildObject.Arch,
 650                                                                   self.BuildItem.BuildObject.ToolChain,
 651                                                                   self.BuildItem.BuildObject.BuildTarget
 652                                                                  )
 653             EdkLogger.SetLevel(EdkLogger.ERROR)
 654             BuildTask._ErrorFlag.set()
 655             BuildTask._ErrorMessage = "%s broken\n    %s [%s]" % \
 656                                       (threading.currentThread().getName(), Command, WorkingDir)
 657
 658         # indicate there's a thread is available for another build task
 659         BuildTask._RunningQueueLock.acquire()
 660         BuildTask._RunningQueue.pop(self.BuildItem)
 661         BuildTask._RunningQueueLock.release()
 662         BuildTask._Thread.release()
 663
 664     ## Start build task thread
 665     #
 666     def Start(self):
 667         EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
 668         Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
 669         self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
 670         self.BuildTread.setName("build thread")
 671         self.BuildTread.setDaemon(False)
 672         self.BuildTread.start()
 673
 674 ## The class contains the information related to EFI image
 675 #
 676 class PeImageInfo():
 677     ## Constructor
 678     #
 679     # Constructor will load all required image information.
 680     #
 681     #   @param  BaseName          The full file path of image.
 682     #   @param  Guid              The GUID for image.
 683     #   @param  Arch              Arch of this image.
 684     #   @param  OutputDir         The output directory for image.
 685     #   @param  DebugDir          The debug directory for image.
 686     #   @param  ImageClass        PeImage Information
 687     #
 688     def __init__(self, BaseName, Guid, Arch, OutputDir, DebugDir, ImageClass):
 689         self.BaseName         = BaseName
 690         self.Guid             = Guid
 691         self.Arch             = Arch
 692         self.OutputDir        = OutputDir
 693         self.DebugDir         = DebugDir
 694         self.Image            = ImageClass
 695         self.Image.Size       = (self.Image.Size // 0x1000 + 1) * 0x1000
 696
 697 ## The class implementing the EDK2 build process
 698 #
 699 #   The build process includes:
 700 #       1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
 701 #       2. Parse DSC file of active platform
 702 #       3. Parse FDF file if any
 703 #       4. Establish build database, including parse all other files (module, package)
 704 #       5. Create AutoGen files (C code file, depex file, makefile) if necessary
 705 #       6. Call build command
 706 #
 707 class Build():
 708     ## Constructor
 709     #
 710     # Constructor will load all necessary configurations, parse platform, modules
 711     # and packages and the establish a database for AutoGen.
 712     #
 713     #   @param  Target              The build command target, one of gSupportedTarget
 714     #   @param  WorkspaceDir        The directory of workspace
 715     #   @param  BuildOptions        Build options passed from command line
 716     #
 717     def __init__(self, Target, WorkspaceDir, BuildOptions,log_q):
 718         self.WorkspaceDir   = WorkspaceDir
 719         self.Target         = Target
 720         self.PlatformFile   = BuildOptions.PlatformFile
 721         self.ModuleFile     = BuildOptions.ModuleFile
 722         self.ArchList       = BuildOptions.TargetArch
 723         self.ToolChainList  = BuildOptions.ToolChain
 724         self.BuildTargetList= BuildOptions.BuildTarget
 725         self.Fdf            = BuildOptions.FdfFile
 726         self.FdList         = BuildOptions.RomImage
 727         self.FvList         = BuildOptions.FvImage
 728         self.CapList        = BuildOptions.CapName
 729         self.SilentMode     = BuildOptions.SilentMode
 730         self.ThreadNumber   = 1
 731         self.SkipAutoGen    = BuildOptions.SkipAutoGen
 732         self.Reparse        = BuildOptions.Reparse
 733         self.SkuId          = BuildOptions.SkuId
 734         if self.SkuId:
 735             GlobalData.gSKUID_CMD = self.SkuId
 736         self.ConfDirectory = BuildOptions.ConfDirectory
 737         self.SpawnMode      = True
 738         self.BuildReport    = BuildReport(BuildOptions.ReportFile, BuildOptions.ReportType)
 739         self.AutoGenTime    = 0
 740         self.MakeTime       = 0
 741         self.GenFdsTime     = 0
 742         self.MakeFileName   = ""
 743         TargetObj = TargetTxtDict()
 744         ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"),"Conf")))
 745         self.TargetTxt = TargetObj.Target
 746         self.ToolDef = ToolDefObj.ToolDef
 747         GlobalData.BuildOptionPcd     = BuildOptions.OptionPcd if BuildOptions.OptionPcd else []
 748         #Set global flag for build mode
 749         GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
 750         GlobalData.gUseHashCache = BuildOptions.UseHashCache
 751         GlobalData.gBinCacheDest   = BuildOptions.BinCacheDest
 752         GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
 753         GlobalData.gEnableGenfdsMultiThread = not BuildOptions.NoGenfdsMultiThread
 754         GlobalData.gDisableIncludePathCheck = BuildOptions.DisableIncludePathCheck
 755
 756         if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
 757             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination must be used together with --hash.")
 758
 759         if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
 760             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-source must be used together with --hash.")
 761
 762         if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
 763             EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binary-destination can not be used together with --binary-source.")
 764
 765         if GlobalData.gBinCacheSource:
 766             BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
 767             if not os.path.isabs(BinCacheSource):
 768                 BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
 769             GlobalData.gBinCacheSource = BinCacheSource
 770         else:
 771             if GlobalData.gBinCacheSource is not None:
 772                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-source.")
 773
 774         if GlobalData.gBinCacheDest:
 775             BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
 776             if not os.path.isabs(BinCacheDest):
 777                 BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
 778             GlobalData.gBinCacheDest = BinCacheDest
 779         else:
 780             if GlobalData.gBinCacheDest is not None:
 781                 EdkLogger.error("build", OPTION_VALUE_INVALID, ExtraData="Invalid value of option --binary-destination.")
 782
 783         GlobalData.gDatabasePath = os.path.normpath(os.path.join(GlobalData.gConfDirectory, GlobalData.gDatabasePath))
 784         if not os.path.exists(os.path.join(GlobalData.gConfDirectory, '.cache')):
 785             os.makedirs(os.path.join(GlobalData.gConfDirectory, '.cache'))
 786         self.Db = BuildDB
 787         self.BuildDatabase = self.Db.BuildObject
 788         self.Platform = None
 789         self.ToolChainFamily = None
 790         self.LoadFixAddress = 0
 791         self.UniFlag        = BuildOptions.Flag
 792         self.BuildModules = []
 793         self.HashSkipModules = []
 794         self.Db_Flag = False
 795         self.LaunchPrebuildFlag = False
 796         self.PlatformBuildPath = os.path.join(GlobalData.gConfDirectory, '.cache', '.PlatformBuild')
 797         if BuildOptions.CommandLength:
 798             GlobalData.gCommandMaxLength = BuildOptions.CommandLength
 799
 800         # print dot character during doing some time-consuming work
 801         self.Progress = Utils.Progressor()
 802         # print current build environment and configuration
 803         EdkLogger.quiet("%-16s = %s" % ("WORKSPACE", os.environ["WORKSPACE"]))
 804         if "PACKAGES_PATH" in os.environ:
 805             # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
 806             EdkLogger.quiet("%-16s = %s" % ("PACKAGES_PATH", os.path.normcase(os.path.normpath(os.environ["PACKAGES_PATH"]))))
 807         EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os.environ["EDK_TOOLS_PATH"]))
 808         if "EDK_TOOLS_BIN" in os.environ:
 809             # Print the same path style with WORKSPACE env.
 810             EdkLogger.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os.path.normcase(os.path.normpath(os.environ["EDK_TOOLS_BIN"]))))
 811         EdkLogger.quiet("%-16s = %s" % ("CONF_PATH", GlobalData.gConfDirectory))
 812         if "PYTHON3_ENABLE" in os.environ:
 813             PYTHON3_ENABLE = os.environ["PYTHON3_ENABLE"]
 814             if PYTHON3_ENABLE != "TRUE":
 815                 PYTHON3_ENABLE = "FALSE"
 816             EdkLogger.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE))
 817         if "PYTHON_COMMAND" in os.environ:
 818             EdkLogger.quiet("%-16s = %s" % ("PYTHON_COMMAND", os.environ["PYTHON_COMMAND"]))
 819         self.InitPreBuild()
 820         self.InitPostBuild()
 821         if self.Prebuild:
 822             EdkLogger.quiet("%-16s = %s" % ("PREBUILD", self.Prebuild))
 823         if self.Postbuild:
 824             EdkLogger.quiet("%-16s = %s" % ("POSTBUILD", self.Postbuild))
 825         if self.Prebuild:
 826             self.LaunchPrebuild()
 827             TargetObj = TargetTxtDict()
 828             ToolDefObj = ToolDefDict((os.path.join(os.getenv("WORKSPACE"), "Conf")))
 829             self.TargetTxt = TargetObj.Target
 830             self.ToolDef = ToolDefObj.ToolDef
 831         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):
 832             self.InitBuild()
 833
 834         self.AutoGenMgr = None
 835         EdkLogger.info("")
 836         os.chdir(self.WorkspaceDir)
 837         self.log_q = log_q
 838         GlobalData.file_lock =  mp.Lock()
 839         # Init cache data for local only
 840         GlobalData.gPackageHashFile = dict()
 841         GlobalData.gModulePreMakeCacheStatus = dict()
 842         GlobalData.gModuleMakeCacheStatus = dict()
 843         GlobalData.gHashChainStatus = dict()
 844         GlobalData.gCMakeHashFile = dict()
 845         GlobalData.gModuleHashFile = dict()
 846         GlobalData.gFileHashDict = dict()
 847         GlobalData.gModuleAllCacheStatus = set()
 848         GlobalData.gModuleCacheHit = set()
 849
 850     def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):
 851         try:
 852             if SkipAutoGen:
 853                 return True,0
 854             feedback_q = mp.Queue()
 855             error_event = mp.Event()
 856             FfsCmd = DataPipe.Get("FfsCommand")
 857             if FfsCmd is None:
 858                 FfsCmd = {}
 859             GlobalData.FfsCmd = FfsCmd
 860             auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]
 861             self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
 862             self.AutoGenMgr.start()
 863             for w in auto_workers:
 864                 w.start()
 865             if PcdMaList is not None:
 866                 for PcdMa in PcdMaList:
 867                     # SourceFileList calling sequence impact the makefile string sequence.
 868                     # Create cached SourceFileList here to unify its calling sequence for both
 869                     # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
 870                     RetVal = PcdMa.SourceFileList
 871                     # Force cache miss for PCD driver
 872                     if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
 873                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))
 874
 875                     PcdMa.CreateCodeFile(False)
 876                     PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
 877                     PcdMa.CreateAsBuiltInf()
 878                     # Force cache miss for PCD driver
 879                     if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
 880                         cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))
 881
 882             self.AutoGenMgr.join()
 883             rt = self.AutoGenMgr.Status
 884             err = 0
 885             if not rt:
 886                 err = UNKNOWN_ERROR
 887             return rt, err
 888         except FatalError as e:
 889             return False, e.args[0]
 890         except:
 891             return False, UNKNOWN_ERROR
 892
 893     ## Add TOOLCHAIN and FAMILY declared in DSC [BuildOptions] to ToolsDefTxtDatabase.
 894     #
 895     # Loop through the set of build targets, tool chains, and archs provided on either
 896     # the command line or in target.txt to discover FAMILY and TOOLCHAIN delclarations
 897     # in [BuildOptions] sections that may be within !if expressions that may use
 898     # $(TARGET), $(TOOLCHAIN), $(TOOLCHAIN_TAG), or $(ARCH) operands.
 899     #
 900     def GetToolChainAndFamilyFromDsc (self, File):
 901         SavedGlobalDefines = GlobalData.gGlobalDefines.copy()
 902         for BuildTarget in self.BuildTargetList:
 903             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
 904             for BuildToolChain in self.ToolChainList:
 905                 GlobalData.gGlobalDefines['TOOLCHAIN']      = BuildToolChain
 906                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = BuildToolChain
 907                 for BuildArch in self.ArchList:
 908                     GlobalData.gGlobalDefines['ARCH'] = BuildArch
 909                     dscobj = self.BuildDatabase[File, BuildArch]
 910                     for KeyFamily, Key, KeyCodeBase in dscobj.BuildOptions:
 911                         try:
 912                             Target, ToolChain, Arch, Tool, Attr = Key.split('_')
 913                         except:
 914                             continue
 915                         if ToolChain == TAB_STAR or Attr != TAB_TOD_DEFINES_FAMILY:
 916                             continue
 917                         try:
 918                             Family = dscobj.BuildOptions[(KeyFamily, Key, KeyCodeBase)]
 919                             Family = Family.strip().strip('=').strip()
 920                         except:
 921                             continue
 922                         if TAB_TOD_DEFINES_FAMILY not in self.ToolDef.ToolsDefTxtDatabase:
 923                             self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
 924                         if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 925                             self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][ToolChain] = Family
 926                         if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolDef.ToolsDefTxtDatabase:
 927                             self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
 928                         if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY]:
 929                             self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][ToolChain] = Family
 930                         if TAB_TOD_DEFINES_TOOL_CHAIN_TAG not in self.ToolDef.ToolsDefTxtDatabase:
 931                             self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = []
 932                         if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 933                             self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].append(ToolChain)
 934         GlobalData.gGlobalDefines = SavedGlobalDefines
 935
 936     ## Load configuration
 937     #
 938     #   This method will parse target.txt and get the build configurations.
 939     #
 940     def LoadConfiguration(self):
 941
 942         # if no ARCH given in command line, get it from target.txt
 943         if not self.ArchList:
 944             self.ArchList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET_ARCH]
 945         self.ArchList = tuple(self.ArchList)
 946
 947         # if no build target given in command line, get it from target.txt
 948         if not self.BuildTargetList:
 949             self.BuildTargetList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
 950
 951         # if no tool chain given in command line, get it from target.txt
 952         if not self.ToolChainList:
 953             self.ToolChainList = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
 954             if self.ToolChainList is None or len(self.ToolChainList) == 0:
 955                 EdkLogger.error("build", RESOURCE_NOT_AVAILABLE, ExtraData="No toolchain given. Don't know how to build.\n")
 956
 957         if not self.PlatformFile:
 958             PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
 959             if not PlatformFile:
 960                 # Try to find one in current directory
 961                 WorkingDirectory = os.getcwd()
 962                 FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.dsc')))
 963                 FileNum = len(FileList)
 964                 if FileNum >= 2:
 965                     EdkLogger.error("build", OPTION_MISSING,
 966                                     ExtraData="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum, WorkingDirectory))
 967                 elif FileNum == 1:
 968                     PlatformFile = FileList[0]
 969                 else:
 970                     EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 971                                     ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
 972
 973             self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
 974
 975         self.GetToolChainAndFamilyFromDsc (self.PlatformFile)
 976
 977         # check if the tool chains are defined or not
 978         NewToolChainList = []
 979         for ToolChain in self.ToolChainList:
 980             if ToolChain not in self.ToolDef.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]:
 981                 EdkLogger.warn("build", "Tool chain [%s] is not defined" % ToolChain)
 982             else:
 983                 NewToolChainList.append(ToolChain)
 984         # if no tool chain available, break the build
 985         if len(NewToolChainList) == 0:
 986             EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
 987                             ExtraData="[%s] not defined. No toolchain available for build!\n" % ", ".join(self.ToolChainList))
 988         else:
 989             self.ToolChainList = NewToolChainList
 990
 991         ToolChainFamily = []
 992         ToolDefinition = self.ToolDef.ToolsDefTxtDatabase
 993         for Tool in self.ToolChainList:
 994             if TAB_TOD_DEFINES_FAMILY not in ToolDefinition or Tool not in ToolDefinition[TAB_TOD_DEFINES_FAMILY] \
 995                or not ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool]:
 996                 EdkLogger.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool)
 997                 ToolChainFamily.append(TAB_COMPILER_MSFT)
 998             else:
 999                 ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
1000         self.ToolChainFamily = ToolChainFamily
1001
1002         self.ThreadNumber   = ThreadNum()
1003     ## Initialize build configuration
1004     #
1005     #   This method will parse DSC file and merge the configurations from
1006     #   command line and target.txt, then get the final build configurations.
1007     #
1008     def InitBuild(self):
1009         # parse target.txt, tools_def.txt, and platform file
1010         self.LoadConfiguration()
1011
1012         # Allow case-insensitive for those from command line or configuration file
1013         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
1014         if ErrorCode != 0:
1015             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1016
1017
1018     def InitPreBuild(self):
1019         self.LoadConfiguration()
1020         ErrorCode, ErrorInfo = self.PlatformFile.Validate(".dsc", False)
1021         if ErrorCode != 0:
1022             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
1023         if self.BuildTargetList:
1024             GlobalData.gGlobalDefines['TARGET'] = self.BuildTargetList[0]
1025         if self.ArchList:
1026             GlobalData.gGlobalDefines['ARCH'] = self.ArchList[0]
1027         if self.ToolChainList:
1028             GlobalData.gGlobalDefines['TOOLCHAIN'] = self.ToolChainList[0]
1029             GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = self.ToolChainList[0]
1030         if self.ToolChainFamily:
1031             GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[0]
1032         if 'PREBUILD' in GlobalData.gCommandLineDefines:
1033             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')
1034         else:
1035             self.Db_Flag = True
1036             Platform = self.Db.MapPlatform(str(self.PlatformFile))
1037             self.Prebuild = str(Platform.Prebuild)
1038         if self.Prebuild:
1039             PrebuildList = []
1040             #
1041             # Evaluate all arguments and convert arguments that are WORKSPACE
1042             # relative paths to absolute paths.  Filter arguments that look like
1043             # flags or do not follow the file/dir naming rules to avoid false
1044             # positives on this conversion.
1045             #
1046             for Arg in self.Prebuild.split():
1047                 #
1048                 # Do not modify Arg if it looks like a flag or an absolute file path
1049                 #
1050                 if Arg.startswith('-')  or os.path.isabs(Arg):
1051                     PrebuildList.append(Arg)
1052                     continue
1053                 #
1054                 # Do not modify Arg if it does not look like a Workspace relative
1055                 # path that starts with a valid package directory name
1056                 #
1057                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1058                     PrebuildList.append(Arg)
1059                     continue
1060                 #
1061                 # If Arg looks like a WORKSPACE relative path, then convert to an
1062                 # absolute path and check to see if the file exists.
1063                 #
1064                 Temp = mws.join(self.WorkspaceDir, Arg)
1065                 if os.path.isfile(Temp):
1066                     Arg = Temp
1067                 PrebuildList.append(Arg)
1068             self.Prebuild       = ' '.join(PrebuildList)
1069             self.Prebuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1070
1071     def InitPostBuild(self):
1072         if 'POSTBUILD' in GlobalData.gCommandLineDefines:
1073             self.Postbuild = GlobalData.gCommandLineDefines.get('POSTBUILD')
1074         else:
1075             Platform = self.Db.MapPlatform(str(self.PlatformFile))
1076             self.Postbuild = str(Platform.Postbuild)
1077         if self.Postbuild:
1078             PostbuildList = []
1079             #
1080             # Evaluate all arguments and convert arguments that are WORKSPACE
1081             # relative paths to absolute paths.  Filter arguments that look like
1082             # flags or do not follow the file/dir naming rules to avoid false
1083             # positives on this conversion.
1084             #
1085             for Arg in self.Postbuild.split():
1086                 #
1087                 # Do not modify Arg if it looks like a flag or an absolute file path
1088                 #
1089                 if Arg.startswith('-')  or os.path.isabs(Arg):
1090                     PostbuildList.append(Arg)
1091                     continue
1092                 #
1093                 # Do not modify Arg if it does not look like a Workspace relative
1094                 # path that starts with a valid package directory name
1095                 #
1096                 if not Arg[0].isalpha() or os.path.dirname(Arg) == '':
1097                     PostbuildList.append(Arg)
1098                     continue
1099                 #
1100                 # If Arg looks like a WORKSPACE relative path, then convert to an
1101                 # absolute path and check to see if the file exists.
1102                 #
1103                 Temp = mws.join(self.WorkspaceDir, Arg)
1104                 if os.path.isfile(Temp):
1105                     Arg = Temp
1106                 PostbuildList.append(Arg)
1107             self.Postbuild       = ' '.join(PostbuildList)
1108             self.Postbuild += self.PassCommandOption(self.BuildTargetList, self.ArchList, self.ToolChainList, self.PlatformFile, self.Target)
1109
1110     def PassCommandOption(self, BuildTarget, TargetArch, ToolChain, PlatformFile, Target):
1111         BuildStr = ''
1112         if GlobalData.gCommand and isinstance(GlobalData.gCommand, list):
1113             BuildStr += ' ' + ' '.join(GlobalData.gCommand)
1114         TargetFlag = False
1115         ArchFlag = False
1116         ToolChainFlag = False
1117         PlatformFileFlag = False
1118
1119         if GlobalData.gOptions and not GlobalData.gOptions.BuildTarget:
1120             TargetFlag = True
1121         if GlobalData.gOptions and not GlobalData.gOptions.TargetArch:
1122             ArchFlag = True
1123         if GlobalData.gOptions and not GlobalData.gOptions.ToolChain:
1124             ToolChainFlag = True
1125         if GlobalData.gOptions and not GlobalData.gOptions.PlatformFile:
1126             PlatformFileFlag = True
1127
1128         if TargetFlag and BuildTarget:
1129             if isinstance(BuildTarget, list) or isinstance(BuildTarget, tuple):
1130                 BuildStr += ' -b ' + ' -b '.join(BuildTarget)
1131             elif isinstance(BuildTarget, str):
1132                 BuildStr += ' -b ' + BuildTarget
1133         if ArchFlag and TargetArch:
1134             if isinstance(TargetArch, list) or isinstance(TargetArch, tuple):
1135                 BuildStr += ' -a ' + ' -a '.join(TargetArch)
1136             elif isinstance(TargetArch, str):
1137                 BuildStr += ' -a ' + TargetArch
1138         if ToolChainFlag and ToolChain:
1139             if isinstance(ToolChain, list) or isinstance(ToolChain, tuple):
1140                 BuildStr += ' -t ' + ' -t '.join(ToolChain)
1141             elif isinstance(ToolChain, str):
1142                 BuildStr += ' -t ' + ToolChain
1143         if PlatformFileFlag and PlatformFile:
1144             if isinstance(PlatformFile, list) or isinstance(PlatformFile, tuple):
1145                 BuildStr += ' -p ' + ' -p '.join(PlatformFile)
1146             elif isinstance(PlatformFile, str):
1147                 BuildStr += ' -p' + PlatformFile
1148         BuildStr += ' --conf=' + GlobalData.gConfDirectory
1149         if Target:
1150             BuildStr += ' ' + Target
1151
1152         return BuildStr
1153
1154     def LaunchPrebuild(self):
1155         if self.Prebuild:
1156             EdkLogger.info("\n- Prebuild Start -\n")
1157             self.LaunchPrebuildFlag = True
1158             #
1159             # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1160             # and preserve them for the rest of the main build step, because the child process environment will
1161             # evaporate as soon as it exits, we cannot get it in build step.
1162             #
1163             PrebuildEnvFile = os.path.join(GlobalData.gConfDirectory, '.cache', '.PrebuildEnv')
1164             if os.path.isfile(PrebuildEnvFile):
1165                 os.remove(PrebuildEnvFile)
1166             if os.path.isfile(self.PlatformBuildPath):
1167                 os.remove(self.PlatformBuildPath)
1168             if sys.platform == "win32":
1169                 args = ' && '.join((self.Prebuild, 'set > ' + PrebuildEnvFile))
1170                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1171             else:
1172                 args = ' && '.join((self.Prebuild, 'env > ' + PrebuildEnvFile))
1173                 Process = Popen(args, stdout=PIPE, stderr=PIPE, shell=True)
1174
1175             # launch two threads to read the STDOUT and STDERR
1176             EndOfProcedure = Event()
1177             EndOfProcedure.clear()
1178             if Process.stdout:
1179                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1180                 StdOutThread.setName("STDOUT-Redirector")
1181                 StdOutThread.setDaemon(False)
1182                 StdOutThread.start()
1183
1184             if Process.stderr:
1185                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1186                 StdErrThread.setName("STDERR-Redirector")
1187                 StdErrThread.setDaemon(False)
1188                 StdErrThread.start()
1189             # waiting for program exit
1190             Process.wait()
1191
1192             if Process.stdout:
1193                 StdOutThread.join()
1194             if Process.stderr:
1195                 StdErrThread.join()
1196             if Process.returncode != 0 :
1197                 EdkLogger.error("Prebuild", PREBUILD_ERROR, 'Prebuild process is not success!')
1198
1199             if os.path.exists(PrebuildEnvFile):
1200                 f = open(PrebuildEnvFile)
1201                 envs = f.readlines()
1202                 f.close()
1203                 envs = [l.split("=", 1) for l in envs ]
1204                 envs = [[I.strip() for I in item] for item in envs if len(item) == 2]
1205                 os.environ.update(dict(envs))
1206             EdkLogger.info("\n- Prebuild Done -\n")
1207
1208     def LaunchPostbuild(self):
1209         if self.Postbuild:
1210             EdkLogger.info("\n- Postbuild Start -\n")
1211             if sys.platform == "win32":
1212                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1213             else:
1214                 Process = Popen(self.Postbuild, stdout=PIPE, stderr=PIPE, shell=True)
1215             # launch two threads to read the STDOUT and STDERR
1216             EndOfProcedure = Event()
1217             EndOfProcedure.clear()
1218             if Process.stdout:
1219                 StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
1220                 StdOutThread.setName("STDOUT-Redirector")
1221                 StdOutThread.setDaemon(False)
1222                 StdOutThread.start()
1223
1224             if Process.stderr:
1225                 StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
1226                 StdErrThread.setName("STDERR-Redirector")
1227                 StdErrThread.setDaemon(False)
1228                 StdErrThread.start()
1229             # waiting for program exit
1230             Process.wait()
1231
1232             if Process.stdout:
1233                 StdOutThread.join()
1234             if Process.stderr:
1235                 StdErrThread.join()
1236             if Process.returncode != 0 :
1237                 EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
1238             EdkLogger.info("\n- Postbuild Done -\n")
1239
1240     ## Build a module or platform
1241     #
1242     # Create autogen code and makefile for a module or platform, and the launch
1243     # "make" command to build it
1244     #
1245     #   @param  Target                      The target of build command
1246     #   @param  Platform                    The platform file
1247     #   @param  Module                      The module file
1248     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1249     #   @param  ToolChain                   The name of toolchain to build
1250     #   @param  Arch                        The arch of the module/platform
1251     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1252     #                                       for dependent modules/Libraries
1253     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1254     #                                       for dependent modules/Libraries
1255     #
1256     def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):
1257         if AutoGenObject is None:
1258             return False
1259         if FfsCommand is None:
1260             FfsCommand = {}
1261         # skip file generation for cleanxxx targets, run and fds target
1262         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1263             # for target which must generate AutoGen code and makefile
1264             mqueue = mp.Queue()
1265             for m in AutoGenObject.GetAllModuleInfo:
1266                 mqueue.put(m)
1267             mqueue.put((None,None,None,None,None,None,None))
1268             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
1269             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
1270             AutoGenObject.CreateLibModuelDirs()
1271             AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
1272             AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
1273             AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
1274             self.Progress.Start("Generating makefile and code")
1275             data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
1276             AutoGenObject.DataPipe.dump(data_pipe_file)
1277             cqueue = mp.Queue()
1278             autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
1279             AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
1280             with open(AutoGenIdFile,"w") as fw:
1281                 fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
1282                 fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
1283                 fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
1284             self.Progress.Stop("done!")
1285             if not autogen_rt:
1286                 self.AutoGenMgr.TerminateWorkers()
1287                 self.AutoGenMgr.join(1)
1288                 raise FatalError(errorcode)
1289             AutoGenObject.CreateCodeFile(False)
1290             AutoGenObject.CreateMakeFile(False)
1291         else:
1292             # always recreate top/platform makefile when clean, just in case of inconsistency
1293             AutoGenObject.CreateCodeFile(True)
1294             AutoGenObject.CreateMakeFile(True)
1295
1296         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1297             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1298
1299         BuildCommand = AutoGenObject.BuildCommand
1300         if BuildCommand is None or len(BuildCommand) == 0:
1301             EdkLogger.error("build", OPTION_MISSING,
1302                             "No build command found for this module. "
1303                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1304                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1305                             ExtraData=str(AutoGenObject))
1306
1307         # run
1308         if Target == 'run':
1309             return True
1310
1311         # build modules
1312         if BuildModule:
1313             BuildCommand = BuildCommand + [Target]
1314             LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1315             if GlobalData.gBinCacheDest:
1316                 self.GenDestCache()
1317             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1318                 # Only for --hash
1319                 # Update PreMakeCacheChain files
1320                 self.GenLocalPreMakeCache()
1321             self.BuildModules = []
1322             return True
1323
1324         # build library
1325         if Target == 'libraries':
1326             DirList = []
1327             for Lib in AutoGenObject.LibraryAutoGenList:
1328                 if not Lib.IsBinaryModule:
1329                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1330             for Lib, LibAutoGen in DirList:
1331                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']
1332                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1333             return True
1334
1335         # build module
1336         if Target == 'modules':
1337             DirList = []
1338             for Lib in AutoGenObject.LibraryAutoGenList:
1339                 if not Lib.IsBinaryModule:
1340                     DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
1341             for Lib, LibAutoGen in DirList:
1342                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, self.MakeFileName)), 'pbuild']
1343                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
1344
1345             DirList = []
1346             for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
1347                 if not ModuleAutoGen.IsBinaryModule:
1348                     DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
1349             for Mod,ModAutoGen in DirList:
1350                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, self.MakeFileName)), 'pbuild']
1351                 LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
1352             self.CreateAsBuiltInf()
1353             if GlobalData.gBinCacheDest:
1354                 self.GenDestCache()
1355             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1356                 # Only for --hash
1357                 # Update PreMakeCacheChain files
1358                 self.GenLocalPreMakeCache()
1359             self.BuildModules = []
1360             return True
1361
1362         # cleanlib
1363         if Target == 'cleanlib':
1364             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1365                 LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))
1366                 if os.path.exists(LibMakefile):
1367                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1368                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1369             return True
1370
1371         # clean
1372         if Target == 'clean':
1373             for Mod in AutoGenObject.ModuleBuildDirectoryList:
1374                 ModMakefile = os.path.normpath(os.path.join(Mod, self.MakeFileName))
1375                 if os.path.exists(ModMakefile):
1376                     NewBuildCommand = BuildCommand + ['-f', ModMakefile, 'cleanall']
1377                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1378             for Lib in AutoGenObject.LibraryBuildDirectoryList:
1379                 LibMakefile = os.path.normpath(os.path.join(Lib, self.MakeFileName))
1380                 if os.path.exists(LibMakefile):
1381                     NewBuildCommand = BuildCommand + ['-f', LibMakefile, 'cleanall']
1382                     LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
1383             return True
1384
1385         # cleanall
1386         if Target == 'cleanall':
1387             try:
1388                 #os.rmdir(AutoGenObject.BuildDir)
1389                 RemoveDirectory(AutoGenObject.BuildDir, True)
1390             except WindowsError as X:
1391                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1392         return True
1393
1394     ## Build a module or platform
1395     #
1396     # Create autogen code and makefile for a module or platform, and the launch
1397     # "make" command to build it
1398     #
1399     #   @param  Target                      The target of build command
1400     #   @param  Platform                    The platform file
1401     #   @param  Module                      The module file
1402     #   @param  BuildTarget                 The name of build target, one of "DEBUG", "RELEASE"
1403     #   @param  ToolChain                   The name of toolchain to build
1404     #   @param  Arch                        The arch of the module/platform
1405     #   @param  CreateDepModuleCodeFile     Flag used to indicate creating code
1406     #                                       for dependent modules/Libraries
1407     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile
1408     #                                       for dependent modules/Libraries
1409     #
1410     def _Build(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False):
1411         if AutoGenObject is None:
1412             return False
1413
1414         # skip file generation for cleanxxx targets, run and fds target
1415         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1416             # for target which must generate AutoGen code and makefile
1417             if not self.SkipAutoGen or Target == 'genc':
1418                 self.Progress.Start("Generating code")
1419                 AutoGenObject.CreateCodeFile(CreateDepsCodeFile)
1420                 self.Progress.Stop("done!")
1421             if Target == "genc":
1422                 return True
1423
1424             if not self.SkipAutoGen or Target == 'genmake':
1425                 self.Progress.Start("Generating makefile")
1426                 AutoGenObject.CreateMakeFile(CreateDepsMakeFile)
1427                 #AutoGenObject.CreateAsBuiltInf()
1428                 self.Progress.Stop("done!")
1429             if Target == "genmake":
1430                 return True
1431         else:
1432             # always recreate top/platform makefile when clean, just in case of inconsistency
1433             AutoGenObject.CreateCodeFile(True)
1434             AutoGenObject.CreateMakeFile(True)
1435
1436         if EdkLogger.GetLevel() == EdkLogger.QUIET:
1437             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))
1438
1439         BuildCommand = AutoGenObject.BuildCommand
1440         if BuildCommand is None or len(BuildCommand) == 0:
1441             EdkLogger.error("build", OPTION_MISSING,
1442                             "No build command found for this module. "
1443                             "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1444                                 (AutoGenObject.BuildTarget, AutoGenObject.ToolChain, AutoGenObject.Arch),
1445                             ExtraData=str(AutoGenObject))
1446
1447         # build modules
1448         if BuildModule:
1449             if Target != 'fds':
1450                 BuildCommand = BuildCommand + [Target]
1451             AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
1452             self.CreateAsBuiltInf()
1453             if GlobalData.gBinCacheDest:
1454                 self.GenDestCache()
1455             elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1456                 # Only for --hash
1457                 # Update PreMakeCacheChain files
1458                 self.GenLocalPreMakeCache()
1459             self.BuildModules = []
1460             return True
1461
1462         # genfds
1463         if Target == 'fds':
1464             if GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db):
1465                 EdkLogger.error("build", COMMAND_FAILURE)
1466             Threshold = self.GetFreeSizeThreshold()
1467             if Threshold:
1468                 self.CheckFreeSizeThreshold(Threshold, AutoGenObject.FvDir)
1469             return True
1470
1471         # run
1472         if Target == 'run':
1473             return True
1474
1475         # build library
1476         if Target == 'libraries':
1477             pass
1478
1479         # not build modules
1480
1481
1482         # cleanall
1483         if Target == 'cleanall':
1484             try:
1485                 #os.rmdir(AutoGenObject.BuildDir)
1486                 RemoveDirectory(AutoGenObject.BuildDir, True)
1487             except WindowsError as X:
1488                 EdkLogger.error("build", FILE_DELETE_FAILURE, ExtraData=str(X))
1489         return True
1490
1491     ## Rebase module image and Get function address for the input module list.
1492     #
1493     def _RebaseModule (self, MapBuffer, BaseAddress, ModuleList, AddrIsOffset = True, ModeIsSmm = False):
1494         if ModeIsSmm:
1495             AddrIsOffset = False
1496         for InfFile in ModuleList:
1497             sys.stdout.write (".")
1498             sys.stdout.flush()
1499             ModuleInfo = ModuleList[InfFile]
1500             ModuleName = ModuleInfo.BaseName
1501             ModuleOutputImage = ModuleInfo.Image.FileName
1502             ModuleDebugImage  = os.path.join(ModuleInfo.DebugDir, ModuleInfo.BaseName + '.efi')
1503             ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1504             if not ModeIsSmm:
1505                 BaseAddress = BaseAddress - ModuleInfo.Image.Size
1506                 #
1507                 # Update Image to new BaseAddress by GenFw tool
1508                 #
1509                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1510                 LaunchCommand(["GenFw", "--rebase", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1511             else:
1512                 #
1513                 # Set new address to the section header only for SMM driver.
1514                 #
1515                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleOutputImage], ModuleInfo.OutputDir)
1516                 LaunchCommand(["GenFw", "--address", str(BaseAddress), "-r", ModuleDebugImage], ModuleInfo.DebugDir)
1517             #
1518             # Collect function address from Map file
1519             #
1520             ImageMapTable = ModuleOutputImage.replace('.efi', '.map')
1521             FunctionList = []
1522             if os.path.exists(ImageMapTable):
1523                 OrigImageBaseAddress = 0
1524                 ImageMap = open(ImageMapTable, 'r')
1525                 for LinStr in ImageMap:
1526                     if len (LinStr.strip()) == 0:
1527                         continue
1528                     #
1529                     # Get the preferred address set on link time.
1530                     #
1531                     if LinStr.find ('Preferred load address is') != -1:
1532                         StrList = LinStr.split()
1533                         OrigImageBaseAddress = int (StrList[len(StrList) - 1], 16)
1534
1535                     StrList = LinStr.split()
1536                     if len (StrList) > 4:
1537                         if StrList[3] == 'f' or StrList[3] == 'F':
1538                             Name = StrList[1]
1539                             RelativeAddress = int (StrList[2], 16) - OrigImageBaseAddress
1540                             FunctionList.append ((Name, RelativeAddress))
1541
1542                 ImageMap.close()
1543             #
1544             # Add general information.
1545             #
1546             if ModeIsSmm:
1547                 MapBuffer.append('\n\n%s (Fixed SMRAM Offset,   BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1548             elif AddrIsOffset:
1549                 MapBuffer.append('\n\n%s (Fixed Memory Offset,  BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName, 0 - BaseAddress, 0 - (BaseAddress + ModuleInfo.Image.EntryPoint)))
1550             else:
1551                 MapBuffer.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X,  EntryPoint=0x%010X)\n' % (ModuleName, BaseAddress, BaseAddress + ModuleInfo.Image.EntryPoint))
1552             #
1553             # Add guid and general seciton section.
1554             #
1555             TextSectionAddress = 0
1556             DataSectionAddress = 0
1557             for SectionHeader in ModuleInfo.Image.SectionHeaderList:
1558                 if SectionHeader[0] == '.text':
1559                     TextSectionAddress = SectionHeader[1]
1560                 elif SectionHeader[0] in ['.data', '.sdata']:
1561                     DataSectionAddress = SectionHeader[1]
1562             if AddrIsOffset:
1563                 MapBuffer.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo.Guid, 0 - (BaseAddress + TextSectionAddress), 0 - (BaseAddress + DataSectionAddress)))
1564             else:
1565                 MapBuffer.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo.Guid, BaseAddress + TextSectionAddress, BaseAddress + DataSectionAddress))
1566             #
1567             # Add debug image full path.
1568             #
1569             MapBuffer.append('(IMAGE=%s)\n\n' % (ModuleDebugImage))
1570             #
1571             # Add function address
1572             #
1573             for Function in FunctionList:
1574                 if AddrIsOffset:
1575                     MapBuffer.append('  -0x%010X    %s\n' % (0 - (BaseAddress + Function[1]), Function[0]))
1576                 else:
1577                     MapBuffer.append('  0x%010X    %s\n' % (BaseAddress + Function[1], Function[0]))
1578             ImageMap.close()
1579
1580             #
1581             # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1582             #
1583             if ModeIsSmm:
1584                 BaseAddress = BaseAddress + ModuleInfo.Image.Size
1585
1586     ## Collect MAP information of all FVs
1587     #
1588     def _CollectFvMapBuffer (self, MapBuffer, Wa, ModuleList):
1589         if self.Fdf:
1590             # First get the XIP base address for FV map file.
1591             GuidPattern = re.compile("[-a-fA-F0-9]+")
1592             GuidName = re.compile(r"\(GUID=[-a-fA-F0-9]+")
1593             for FvName in Wa.FdfProfile.FvDict:
1594                 FvMapBuffer = os.path.join(Wa.FvDir, FvName + '.Fv.map')
1595                 if not os.path.exists(FvMapBuffer):
1596                     continue
1597                 FvMap = open(FvMapBuffer, 'r')
1598                 #skip FV size information
1599                 FvMap.readline()
1600                 FvMap.readline()
1601                 FvMap.readline()
1602                 FvMap.readline()
1603                 for Line in FvMap:
1604                     MatchGuid = GuidPattern.match(Line)
1605                     if MatchGuid is not None:
1606                         #
1607                         # Replace GUID with module name
1608                         #
1609                         GuidString = MatchGuid.group()
1610                         if GuidString.upper() in ModuleList:
1611                             Line = Line.replace(GuidString, ModuleList[GuidString.upper()].Name)
1612                     MapBuffer.append(Line)
1613                     #
1614                     # Add the debug image full path.
1615                     #
1616                     MatchGuid = GuidName.match(Line)
1617                     if MatchGuid is not None:
1618                         GuidString = MatchGuid.group().split("=")[1]
1619                         if GuidString.upper() in ModuleList:
1620                             MapBuffer.append('(IMAGE=%s)\n' % (os.path.join(ModuleList[GuidString.upper()].DebugDir, ModuleList[GuidString.upper()].Name + '.efi')))
1621
1622                 FvMap.close()
1623
1624     ## Collect MAP information of all modules
1625     #
1626     def _CollectModuleMapBuffer (self, MapBuffer, ModuleList):
1627         sys.stdout.write ("Generate Load Module At Fix Address Map")
1628         sys.stdout.flush()
1629         PatchEfiImageList = []
1630         PeiModuleList  = {}
1631         BtModuleList   = {}
1632         RtModuleList   = {}
1633         SmmModuleList  = {}
1634         PeiSize = 0
1635         BtSize  = 0
1636         RtSize  = 0
1637         # reserve 4K size in SMRAM to make SMM module address not from 0.
1638         SmmSize = 0x1000
1639         for ModuleGuid in ModuleList:
1640             Module = ModuleList[ModuleGuid]
1641             GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (Module.MetaFile, Module.Arch, Module.ToolChain, Module.BuildTarget)
1642
1643             OutputImageFile = ''
1644             for ResultFile in Module.CodaTargetList:
1645                 if str(ResultFile.Target).endswith('.efi'):
1646                     #
1647                     # module list for PEI, DXE, RUNTIME and SMM
1648                     #
1649                     OutputImageFile = os.path.join(Module.OutputDir, Module.Name + '.efi')
1650                     ImageClass = PeImageClass (OutputImageFile)
1651                     if not ImageClass.IsValid:
1652                         EdkLogger.error("build", FILE_PARSE_FAILURE, ExtraData=ImageClass.ErrorInfo)
1653                     ImageInfo = PeImageInfo(Module.Name, Module.Guid, Module.Arch, Module.OutputDir, Module.DebugDir, ImageClass)
1654                     if Module.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_PEIM, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER, EDK_COMPONENT_TYPE_PIC_PEIM, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM, SUP_MODULE_DXE_CORE]:
1655                         PeiModuleList[Module.MetaFile] = ImageInfo
1656                         PeiSize += ImageInfo.Image.Size
1657                     elif Module.ModuleType in [EDK_COMPONENT_TYPE_BS_DRIVER, SUP_MODULE_DXE_DRIVER, SUP_MODULE_UEFI_DRIVER]:
1658                         BtModuleList[Module.MetaFile] = ImageInfo
1659                         BtSize += ImageInfo.Image.Size
1660                     elif Module.ModuleType in [SUP_MODULE_DXE_RUNTIME_DRIVER, EDK_COMPONENT_TYPE_RT_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, EDK_COMPONENT_TYPE_SAL_RT_DRIVER]:
1661                         RtModuleList[Module.MetaFile] = ImageInfo
1662                         RtSize += ImageInfo.Image.Size
1663                     elif Module.ModuleType in [SUP_MODULE_SMM_CORE, SUP_MODULE_DXE_SMM_DRIVER, SUP_MODULE_MM_STANDALONE, SUP_MODULE_MM_CORE_STANDALONE]:
1664                         SmmModuleList[Module.MetaFile] = ImageInfo
1665                         SmmSize += ImageInfo.Image.Size
1666                         if Module.ModuleType == SUP_MODULE_DXE_SMM_DRIVER:
1667                             PiSpecVersion = Module.Module.Specification.get('PI_SPECIFICATION_VERSION', '0x00000000')
1668                             # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1669                             if int(PiSpecVersion, 16) < 0x0001000A:
1670                                 BtModuleList[Module.MetaFile] = ImageInfo
1671                                 BtSize += ImageInfo.Image.Size
1672                     break
1673             #
1674             # EFI image is final target.
1675             # Check EFI image contains patchable FixAddress related PCDs.
1676             #
1677             if OutputImageFile != '':
1678                 ModuleIsPatch = False
1679                 for Pcd in Module.ModulePcdList:
1680                     if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1681                         ModuleIsPatch = True
1682                         break
1683                 if not ModuleIsPatch:
1684                     for Pcd in Module.LibraryPcdList:
1685                         if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE and Pcd.TokenCName in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET:
1686                             ModuleIsPatch = True
1687                             break
1688
1689                 if not ModuleIsPatch:
1690                     continue
1691                 #
1692                 # Module includes the patchable load fix address PCDs.
1693                 # It will be fixed up later.
1694                 #
1695                 PatchEfiImageList.append (OutputImageFile)
1696
1697         #
1698         # Get Top Memory address
1699         #
1700         ReservedRuntimeMemorySize = 0
1701         TopMemoryAddress = 0
1702         if self.LoadFixAddress == 0xFFFFFFFFFFFFFFFF:
1703             TopMemoryAddress = 0
1704         else:
1705             TopMemoryAddress = self.LoadFixAddress
1706             if TopMemoryAddress < RtSize + BtSize + PeiSize:
1707                 EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1708
1709         #
1710         # Patch FixAddress related PCDs into EFI image
1711         #
1712         for EfiImage in PatchEfiImageList:
1713             EfiImageMap = EfiImage.replace('.efi', '.map')
1714             if not os.path.exists(EfiImageMap):
1715                 continue
1716             #
1717             # Get PCD offset in EFI image by GenPatchPcdTable function
1718             #
1719             PcdTable = parsePcdInfoFromMapFile(EfiImageMap, EfiImage)
1720             #
1721             # Patch real PCD value by PatchPcdValue tool
1722             #
1723             for PcdInfo in PcdTable:
1724                 ReturnValue = 0
1725                 if PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE:
1726                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE, str (PeiSize // 0x1000))
1727                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE:
1728                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE, str (BtSize // 0x1000))
1729                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE:
1730                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE, str (RtSize // 0x1000))
1731                 elif PcdInfo[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE and len (SmmModuleList) > 0:
1732                     ReturnValue, ErrorInfo = PatchBinaryFile (EfiImage, PcdInfo[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE, str (SmmSize // 0x1000))
1733                 if ReturnValue != 0:
1734                     EdkLogger.error("build", PARAMETER_INVALID, "Patch PCD value failed", ExtraData=ErrorInfo)
1735
1736         MapBuffer.append('PEI_CODE_PAGE_NUMBER      = 0x%x\n' % (PeiSize // 0x1000))
1737         MapBuffer.append('BOOT_CODE_PAGE_NUMBER     = 0x%x\n' % (BtSize // 0x1000))
1738         MapBuffer.append('RUNTIME_CODE_PAGE_NUMBER  = 0x%x\n' % (RtSize // 0x1000))
1739         if len (SmmModuleList) > 0:
1740             MapBuffer.append('SMM_CODE_PAGE_NUMBER      = 0x%x\n' % (SmmSize // 0x1000))
1741
1742         PeiBaseAddr = TopMemoryAddress - RtSize - BtSize
1743         BtBaseAddr  = TopMemoryAddress - RtSize
1744         RtBaseAddr  = TopMemoryAddress - ReservedRuntimeMemorySize
1745
1746         self._RebaseModule (MapBuffer, PeiBaseAddr, PeiModuleList, TopMemoryAddress == 0)
1747         self._RebaseModule (MapBuffer, BtBaseAddr, BtModuleList, TopMemoryAddress == 0)
1748         self._RebaseModule (MapBuffer, RtBaseAddr, RtModuleList, TopMemoryAddress == 0)
1749         self._RebaseModule (MapBuffer, 0x1000, SmmModuleList, AddrIsOffset=False, ModeIsSmm=True)
1750         MapBuffer.append('\n\n')
1751         sys.stdout.write ("\n")
1752         sys.stdout.flush()
1753
1754     ## Save platform Map file
1755     #
1756     def _SaveMapFile (self, MapBuffer, Wa):
1757         #
1758         # Map file path is got.
1759         #
1760         MapFilePath = os.path.join(Wa.BuildDir, Wa.Name + '.map')
1761         #
1762         # Save address map into MAP file.
1763         #
1764         SaveFileOnChange(MapFilePath, ''.join(MapBuffer), False)
1765         if self.LoadFixAddress != 0:
1766             sys.stdout.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath))
1767         sys.stdout.flush()
1768
1769     ## Build active platform for different build targets and different tool chains
1770     #
1771     def _BuildPlatform(self):
1772         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1773         for BuildTarget in self.BuildTargetList:
1774             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1775             index = 0
1776             for ToolChain in self.ToolChainList:
1777                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1778                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1779                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1780                 index += 1
1781                 Wa = WorkspaceAutoGen(
1782                         self.WorkspaceDir,
1783                         self.PlatformFile,
1784                         BuildTarget,
1785                         ToolChain,
1786                         self.ArchList,
1787                         self.BuildDatabase,
1788                         self.TargetTxt,
1789                         self.ToolDef,
1790                         self.Fdf,
1791                         self.FdList,
1792                         self.FvList,
1793                         self.CapList,
1794                         self.SkuId,
1795                         self.UniFlag,
1796                         self.Progress
1797                         )
1798                 self.Fdf = Wa.FdfFile
1799                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1800                 self.BuildReport.AddPlatformReport(Wa)
1801                 self.Progress.Stop("done!")
1802
1803                 # Add ffs build to makefile
1804                 CmdListDict = {}
1805                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1806                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1807
1808                 for Arch in Wa.ArchList:
1809                     PcdMaList    = []
1810                     GlobalData.gGlobalDefines['ARCH'] = Arch
1811                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1812                     for Module in Pa.Platform.Modules:
1813                         # Get ModuleAutoGen object to generate C code file and makefile
1814                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1815                         if Ma is None:
1816                             continue
1817                         if Ma.PcdIsDriver:
1818                             Ma.PlatformInfo = Pa
1819                             Ma.Workspace = Wa
1820                             PcdMaList.append(Ma)
1821                         self.BuildModules.append(Ma)
1822                     Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
1823                     Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
1824                     self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
1825
1826                 # Create MAP file when Load Fix Address is enabled.
1827                 if self.Target in ["", "all", "fds"]:
1828                     for Arch in Wa.ArchList:
1829                         GlobalData.gGlobalDefines['ARCH'] = Arch
1830                         #
1831                         # Check whether the set fix address is above 4G for 32bit image.
1832                         #
1833                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
1834                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1835                     #
1836                     # Get Module List
1837                     #
1838                     ModuleList = {}
1839                     for Pa in Wa.AutoGenObjectList:
1840                         for Ma in Pa.ModuleAutoGenList:
1841                             if Ma is None:
1842                                 continue
1843                             if not Ma.IsLibrary:
1844                                 ModuleList[Ma.Guid.upper()] = Ma
1845
1846                     MapBuffer = []
1847                     if self.LoadFixAddress != 0:
1848                         #
1849                         # Rebase module to the preferred memory address before GenFds
1850                         #
1851                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
1852                     if self.Fdf:
1853                         #
1854                         # create FDS again for the updated EFI image
1855                         #
1856                         self._Build("fds", Wa)
1857                         #
1858                         # Create MAP file for all platform FVs after GenFds.
1859                         #
1860                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
1861                     #
1862                     # Save MAP buffer into MAP file.
1863                     #
1864                     self._SaveMapFile (MapBuffer, Wa)
1865                 self.CreateGuidedSectionToolsFile(Wa)
1866
1867     ## Build active module for different build targets, different tool chains and different archs
1868     #
1869     def _BuildModule(self):
1870         for BuildTarget in self.BuildTargetList:
1871             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
1872             index = 0
1873             for ToolChain in self.ToolChainList:
1874                 WorkspaceAutoGenTime = time.time()
1875                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
1876                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
1877                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
1878                 index += 1
1879                 #
1880                 # module build needs platform build information, so get platform
1881                 # AutoGen first
1882                 #
1883                 Wa = WorkspaceAutoGen(
1884                         self.WorkspaceDir,
1885                         self.PlatformFile,
1886                         BuildTarget,
1887                         ToolChain,
1888                         self.ArchList,
1889                         self.BuildDatabase,
1890                         self.TargetTxt,
1891                         self.ToolDef,
1892                         self.Fdf,
1893                         self.FdList,
1894                         self.FvList,
1895                         self.CapList,
1896                         self.SkuId,
1897                         self.UniFlag,
1898                         self.Progress,
1899                         self.ModuleFile
1900                         )
1901                 self.Fdf = Wa.FdfFile
1902                 self.LoadFixAddress = Wa.Platform.LoadFixAddress
1903                 Wa.CreateMakeFile(False)
1904                 # Add ffs build to makefile
1905                 CmdListDict = None
1906                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
1907                     CmdListDict = self._GenFfsCmd(Wa.ArchList)
1908
1909                 GlobalData.file_lock = mp.Lock()
1910                 GlobalData.FfsCmd = CmdListDict
1911
1912                 self.Progress.Stop("done!")
1913                 MaList = []
1914                 ExitFlag = threading.Event()
1915                 ExitFlag.clear()
1916                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
1917                 for Arch in Wa.ArchList:
1918                     AutoGenStart = time.time()
1919                     GlobalData.gGlobalDefines['ARCH'] = Arch
1920                     Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
1921                     for Module in Pa.Platform.Modules:
1922                         if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
1923                             Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
1924                             if Ma is None:
1925                                 continue
1926                             if Ma.PcdIsDriver:
1927                                 Ma.PlatformInfo = Pa
1928                                 Ma.Workspace = Wa
1929                             MaList.append(Ma)
1930
1931                             if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
1932                                 if Ma.CanSkipbyPreMakeCache():
1933                                     continue
1934                                 else:
1935                                     self.PreMakeCacheMiss.add(Ma)
1936
1937                             # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1938                             if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1939                                 # for target which must generate AutoGen code and makefile
1940                                 if not self.SkipAutoGen or self.Target == 'genc':
1941                                     self.Progress.Start("Generating code")
1942                                     Ma.CreateCodeFile(True)
1943                                     self.Progress.Stop("done!")
1944                                 if self.Target == "genc":
1945                                     return True
1946                                 if not self.SkipAutoGen or self.Target == 'genmake':
1947                                     self.Progress.Start("Generating makefile")
1948                                     if CmdListDict and self.Fdf and (Module.Path, Arch) in CmdListDict:
1949                                         Ma.CreateMakeFile(True, CmdListDict[Module.Path, Arch])
1950                                         del CmdListDict[Module.Path, Arch]
1951                                     else:
1952                                         Ma.CreateMakeFile(True)
1953                                     self.Progress.Stop("done!")
1954                                 if self.Target == "genmake":
1955                                     return True
1956
1957                                 if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
1958                                     if Ma.CanSkipbyMakeCache():
1959                                         continue
1960                                     else:
1961                                         self.MakeCacheMiss.add(Ma)
1962
1963                             self.BuildModules.append(Ma)
1964                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))
1965                     MakeStart = time.time()
1966                     for Ma in self.BuildModules:
1967                         if not Ma.IsBinaryModule:
1968                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
1969                         # Break build if any build thread has error
1970                         if BuildTask.HasError():
1971                             # we need a full version of makefile for platform
1972                             ExitFlag.set()
1973                             BuildTask.WaitForComplete()
1974                             Pa.CreateMakeFile(False)
1975                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1976                         # Start task scheduler
1977                         if not BuildTask.IsOnGoing():
1978                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
1979
1980                     # in case there's an interruption. we need a full version of makefile for platform
1981                     Pa.CreateMakeFile(False)
1982                     if BuildTask.HasError():
1983                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
1984                     self.MakeTime += int(round((time.time() - MakeStart)))
1985
1986                 MakeContiue = time.time()
1987                 ExitFlag.set()
1988                 BuildTask.WaitForComplete()
1989                 self.CreateAsBuiltInf()
1990                 if GlobalData.gBinCacheDest:
1991                     self.GenDestCache()
1992                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
1993                     # Only for --hash
1994                     # Update PreMakeCacheChain files
1995                     self.GenLocalPreMakeCache()
1996                 self.BuildModules = []
1997                 self.MakeTime += int(round((time.time() - MakeContiue)))
1998                 if BuildTask.HasError():
1999                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2000
2001                 self.BuildReport.AddPlatformReport(Wa, MaList)
2002                 if MaList == []:
2003                     EdkLogger.error(
2004                                 'build',
2005                                 BUILD_ERROR,
2006                                 "Module for [%s] is not a component of active platform."\
2007                                 " Please make sure that the ARCH and inf file path are"\
2008                                 " given in the same as in [%s]" % \
2009                                     (', '.join(Wa.ArchList), self.PlatformFile),
2010                                 ExtraData=self.ModuleFile
2011                                 )
2012                 # Create MAP file when Load Fix Address is enabled.
2013                 if self.Target == "fds" and self.Fdf:
2014                     for Arch in Wa.ArchList:
2015                         #
2016                         # Check whether the set fix address is above 4G for 32bit image.
2017                         #
2018                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
2019                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2020                     #
2021                     # Get Module List
2022                     #
2023                     ModuleList = {}
2024                     for Pa in Wa.AutoGenObjectList:
2025                         for Ma in Pa.ModuleAutoGenList:
2026                             if Ma is None:
2027                                 continue
2028                             if not Ma.IsLibrary:
2029                                 ModuleList[Ma.Guid.upper()] = Ma
2030
2031                     MapBuffer = []
2032                     if self.LoadFixAddress != 0:
2033                         #
2034                         # Rebase module to the preferred memory address before GenFds
2035                         #
2036                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
2037                     #
2038                     # create FDS again for the updated EFI image
2039                     #
2040                     GenFdsStart = time.time()
2041                     self._Build("fds", Wa)
2042                     self.GenFdsTime += int(round((time.time() - GenFdsStart)))
2043                     #
2044                     # Create MAP file for all platform FVs after GenFds.
2045                     #
2046                     self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2047                     #
2048                     # Save MAP buffer into MAP file.
2049                     #
2050                     self._SaveMapFile (MapBuffer, Wa)
2051
2052     def _GenFfsCmd(self,ArchList):
2053         # convert dictionary of Cmd:(Inf,Arch)
2054         # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
2055         CmdSetDict = defaultdict(set)
2056         GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
2057         for Cmd in GenFfsDict:
2058             tmpInf, tmpArch = GenFfsDict[Cmd]
2059             CmdSetDict[tmpInf, tmpArch].add(Cmd)
2060         return CmdSetDict
2061     def VerifyAutoGenFiles(self):
2062         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2063         try:
2064             with open(AutoGenIdFile) as fd:
2065                 lines = fd.readlines()
2066         except:
2067             return None
2068         for line in lines:
2069             if "Arch" in line:
2070                 ArchList = line.strip().split("=")[1].split("|")
2071             if "BuildDir" in line:
2072                 BuildDir = line.split("=")[1].strip()
2073             if "PlatformGuid" in line:
2074                 PlatformGuid = line.split("=")[1].strip()
2075         GlobalVarList = []
2076         for arch in ArchList:
2077             global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
2078             if not os.path.exists(global_var):
2079                 return None
2080             GlobalVarList.append(global_var)
2081         for global_var in GlobalVarList:
2082             data_pipe = MemoryDataPipe()
2083             data_pipe.load(global_var)
2084             target = data_pipe.Get("P_Info").get("Target")
2085             toolchain = data_pipe.Get("P_Info").get("ToolChain")
2086             archlist = data_pipe.Get("P_Info").get("ArchList")
2087             Arch = data_pipe.Get("P_Info").get("Arch")
2088             active_p = data_pipe.Get("P_Info").get("ActivePlatform")
2089             workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
2090             PackagesPath = os.getenv("PACKAGES_PATH")
2091             mws.setWs(workspacedir, PackagesPath)
2092             LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
2093             ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
2094
2095             for m_build_dir in LibraryBuildDirectoryList:
2096                 if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):
2097                     return None
2098             for m_build_dir in ModuleBuildDirectoryList:
2099                 if not os.path.exists(os.path.join(m_build_dir,self.MakeFileName)):
2100                     return None
2101             Wa = WorkSpaceInfo(
2102                 workspacedir,active_p,target,toolchain,archlist
2103                 )
2104             Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
2105             Wa.AutoGenObjectList.append(Pa)
2106         return Wa
2107     def SetupMakeSetting(self,Wa):
2108         BuildModules = []
2109         for Pa in Wa.AutoGenObjectList:
2110             for m in Pa._MbList:
2111                 ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
2112                 BuildModules.append(ma)
2113         fdf_file = Wa.FlashDefinition
2114         if fdf_file:
2115             Fdf = FdfParser(fdf_file.Path)
2116             Fdf.ParseFile()
2117             GlobalData.gFdfParser = Fdf
2118             if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
2119                 FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
2120                 for FdRegion in FdDict.RegionList:
2121                     if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
2122                         if int(FdRegion.Offset) % 8 != 0:
2123                             EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
2124             Wa.FdfProfile = Fdf.Profile
2125             self.Fdf = Fdf
2126         else:
2127             self.Fdf = None
2128         return BuildModules
2129
2130     ## Build a platform in multi-thread mode
2131     #
2132     def PerformAutoGen(self,BuildTarget,ToolChain):
2133         WorkspaceAutoGenTime = time.time()
2134         Wa = WorkspaceAutoGen(
2135                 self.WorkspaceDir,
2136                 self.PlatformFile,
2137                 BuildTarget,
2138                 ToolChain,
2139                 self.ArchList,
2140                 self.BuildDatabase,
2141                 self.TargetTxt,
2142                 self.ToolDef,
2143                 self.Fdf,
2144                 self.FdList,
2145                 self.FvList,
2146                 self.CapList,
2147                 self.SkuId,
2148                 self.UniFlag,
2149                 self.Progress
2150                 )
2151         self.Fdf = Wa.FdfFile
2152         self.LoadFixAddress = Wa.Platform.LoadFixAddress
2153         self.BuildReport.AddPlatformReport(Wa)
2154         Wa.CreateMakeFile(False)
2155
2156         # Add ffs build to makefile
2157         CmdListDict = {}
2158         if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
2159             CmdListDict = self._GenFfsCmd(Wa.ArchList)
2160
2161         self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
2162         BuildModules = []
2163         for Arch in Wa.ArchList:
2164             PcdMaList    = []
2165             AutoGenStart = time.time()
2166             GlobalData.gGlobalDefines['ARCH'] = Arch
2167             Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
2168             if Pa is None:
2169                 continue
2170             ModuleList = []
2171             for Inf in Pa.Platform.Modules:
2172                 ModuleList.append(Inf)
2173             # Add the INF only list in FDF
2174             if GlobalData.gFdfParser is not None:
2175                 for InfName in GlobalData.gFdfParser.Profile.InfList:
2176                     Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
2177                     if Inf in Pa.Platform.Modules:
2178                         continue
2179                     ModuleList.append(Inf)
2180             Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
2181             Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
2182             Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
2183             Pa.CreateLibModuelDirs()
2184             # Fetch the MakeFileName.
2185             self.MakeFileName = Pa.MakeFileName
2186             if not self.MakeFileName:
2187                 self.MakeFileName = Pa.MakeFile
2188
2189             Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
2190             Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
2191             Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
2192             # Prepare the cache share data for multiprocessing
2193             Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}
2194             ModuleCodaFile = {}
2195             for ma in Pa.ModuleAutoGenList:
2196                 ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
2197             Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
2198             # ModuleList contains all driver modules only
2199             for Module in ModuleList:
2200                 # Get ModuleAutoGen object to generate C code file and makefile
2201                 Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2202                 if Ma is None:
2203                     continue
2204                 if Ma.PcdIsDriver:
2205                     Ma.PlatformInfo = Pa
2206                     Ma.Workspace = Wa
2207                     PcdMaList.append(Ma)
2208                 self.AllDrivers.add(Ma)
2209                 self.AllModules.add(Ma)
2210
2211             mqueue = mp.Queue()
2212             cqueue = mp.Queue()
2213             for m in Pa.GetAllModuleInfo:
2214                 mqueue.put(m)
2215                 module_file,module_root,module_path,module_basename,\
2216                     module_originalpath,module_arch,IsLib = m
2217                 Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\
2218                                   ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2219                 self.AllModules.add(Ma)
2220             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
2221             Pa.DataPipe.dump(data_pipe_file)
2222
2223             mqueue.put((None,None,None,None,None,None,None))
2224             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
2225
2226             if not autogen_rt:
2227                 self.AutoGenMgr.TerminateWorkers()
2228                 self.AutoGenMgr.join(1)
2229                 raise FatalError(errorcode)
2230
2231             if GlobalData.gUseHashCache:
2232                 for item in GlobalData.gModuleAllCacheStatus:
2233                     (MetaFilePath, Arch, CacheStr, Status) = item
2234                     Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\
2235                                       ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
2236                     if CacheStr == "PreMakeCache" and Status == False:
2237                         self.PreMakeCacheMiss.add(Ma)
2238                     if CacheStr == "PreMakeCache" and Status == True:
2239                         self.PreMakeCacheHit.add(Ma)
2240                         GlobalData.gModuleCacheHit.add(Ma)
2241                     if CacheStr == "MakeCache" and Status == False:
2242                         self.MakeCacheMiss.add(Ma)
2243                     if CacheStr == "MakeCache" and Status == True:
2244                         self.MakeCacheHit.add(Ma)
2245                         GlobalData.gModuleCacheHit.add(Ma)
2246             self.AutoGenTime += int(round((time.time() - AutoGenStart)))
2247         AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
2248         with open(AutoGenIdFile,"w") as fw:
2249             fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
2250             fw.write("BuildDir=%s\n" % Wa.BuildDir)
2251             fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
2252
2253         if GlobalData.gBinCacheSource:
2254             BuildModules.extend(self.MakeCacheMiss)
2255         elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:
2256             BuildModules.extend(self.PreMakeCacheMiss)
2257         else:
2258             BuildModules.extend(self.AllDrivers)
2259
2260         self.Progress.Stop("done!")
2261         return Wa, BuildModules
2262
2263     def _MultiThreadBuildPlatform(self):
2264         SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
2265         for BuildTarget in self.BuildTargetList:
2266             GlobalData.gGlobalDefines['TARGET'] = BuildTarget
2267             index = 0
2268             for ToolChain in self.ToolChainList:
2269                 resetFdsGlobalVariable()
2270                 GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
2271                 GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
2272                 GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
2273                 index += 1
2274                 ExitFlag = threading.Event()
2275                 ExitFlag.clear()
2276                 if self.SkipAutoGen:
2277                     Wa = self.VerifyAutoGenFiles()
2278                     if Wa is None:
2279                         self.SkipAutoGen = False
2280                         Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2281                     else:
2282                         GlobalData.gAutoGenPhase = True
2283                         self.BuildModules = self.SetupMakeSetting(Wa)
2284                 else:
2285                     Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
2286                 Pa = Wa.AutoGenObjectList[0]
2287                 GlobalData.gAutoGenPhase = False
2288
2289                 if GlobalData.gBinCacheSource:
2290                     EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))
2291                     EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))
2292                     EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))
2293
2294                 for Arch in Wa.ArchList:
2295                     MakeStart = time.time()
2296                     for Ma in set(self.BuildModules):
2297                         # Generate build task for the module
2298                         if not Ma.IsBinaryModule:
2299                             Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
2300                         # Break build if any build thread has error
2301                         if BuildTask.HasError():
2302                             # we need a full version of makefile for platform
2303                             ExitFlag.set()
2304                             BuildTask.WaitForComplete()
2305                             Pa.CreateMakeFile(False)
2306                             EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2307                         # Start task scheduler
2308                         if not BuildTask.IsOnGoing():
2309                             BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
2310
2311                     # in case there's an interruption. we need a full version of makefile for platform
2312
2313                     if BuildTask.HasError():
2314                         EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2315                     self.MakeTime += int(round((time.time() - MakeStart)))
2316
2317                 MakeContiue = time.time()
2318                 #
2319                 #
2320                 # All modules have been put in build tasks queue. Tell task scheduler
2321                 # to exit if all tasks are completed
2322                 #
2323                 ExitFlag.set()
2324                 BuildTask.WaitForComplete()
2325                 if GlobalData.gBinCacheDest:
2326                     self.GenDestCache()
2327                 elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
2328                     # Only for --hash
2329                     # Update PreMakeCacheChain files
2330                     self.GenLocalPreMakeCache()
2331                 #
2332                 # Get Module List
2333                 #
2334                 ModuleList = {ma.Guid.upper(): ma for ma in self.BuildModules}
2335                 self.BuildModules = []
2336                 self.MakeTime += int(round((time.time() - MakeContiue)))
2337                 #
2338                 # Check for build error, and raise exception if one
2339                 # has been signaled.
2340                 #
2341                 if BuildTask.HasError():
2342                     EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
2343
2344                 # Create MAP file when Load Fix Address is enabled.
2345                 if self.Target in ["", "all", "fds"]:
2346                     for Arch in Wa.ArchList:
2347                         #
2348                         # Check whether the set fix address is above 4G for 32bit image.
2349                         #
2350                         if (Arch == 'IA32' or Arch == 'ARM') and self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
2351                             EdkLogger.error("build", PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2352
2353                     #
2354                     # Rebase module to the preferred memory address before GenFds
2355                     #
2356                     MapBuffer = []
2357                     if self.LoadFixAddress != 0:
2358                         self._CollectModuleMapBuffer(MapBuffer, ModuleList)
2359
2360                     if self.Fdf:
2361                         #
2362                         # Generate FD image if there's a FDF file found
2363                         #
2364                         GenFdsStart = time.time()
2365                         if GenFdsApi(Wa.GenFdsCommandDict, self.Db):
2366                             EdkLogger.error("build", COMMAND_FAILURE)
2367                         Threshold = self.GetFreeSizeThreshold()
2368                         if Threshold:
2369                             self.CheckFreeSizeThreshold(Threshold, Wa.FvDir)
2370
2371                         #
2372                         # Create MAP file for all platform FVs after GenFds.
2373                         #
2374                         self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
2375                         self.GenFdsTime += int(round((time.time() - GenFdsStart)))
2376                     #
2377                     # Save MAP buffer into MAP file.
2378                     #
2379                     self._SaveMapFile(MapBuffer, Wa)
2380                 self.CreateGuidedSectionToolsFile(Wa)
2381
2382     ## GetFreeSizeThreshold()
2383     #
2384     #   @retval int             Threshold value
2385     #
2386     def GetFreeSizeThreshold(self):
2387         Threshold = None
2388         Threshold_Str = GlobalData.gCommandLineDefines.get('FV_SPARE_SPACE_THRESHOLD')
2389         if Threshold_Str:
2390             try:
2391                 if Threshold_Str.lower().startswith('0x'):
2392                     Threshold = int(Threshold_Str, 16)
2393                 else:
2394                     Threshold = int(Threshold_Str)
2395             except:
2396                 EdkLogger.warn("build", 'incorrect value for FV_SPARE_SPACE_THRESHOLD %s.Only decimal or hex format is allowed.' % Threshold_Str)
2397         return Threshold
2398
2399     def CheckFreeSizeThreshold(self, Threshold=None, FvDir=None):
2400         if not isinstance(Threshold, int):
2401             return
2402         if not isinstance(FvDir, str) or not FvDir:
2403             return
2404         FdfParserObject = GlobalData.gFdfParser
2405         FvRegionNameList = [FvName for FvName in FdfParserObject.Profile.FvDict if FdfParserObject.Profile.FvDict[FvName].FvRegionInFD]
2406         for FvName in FdfParserObject.Profile.FvDict:
2407             if FvName in FvRegionNameList:
2408                 FvSpaceInfoFileName = os.path.join(FvDir, FvName.upper() + '.Fv.map')
2409                 if os.path.exists(FvSpaceInfoFileName):
2410                     FileLinesList = getlines(FvSpaceInfoFileName)
2411                     for Line in FileLinesList:
2412                         NameValue = Line.split('=')
2413                         if len(NameValue) == 2 and NameValue[0].strip() == 'EFI_FV_SPACE_SIZE':
2414                             FreeSizeValue = int(NameValue[1].strip(), 0)
2415                             if FreeSizeValue < Threshold:
2416                                 EdkLogger.error("build", FV_FREESIZE_ERROR,
2417                                                 '%s FV free space %d is not enough to meet with the required spare space %d set by -D FV_SPARE_SPACE_THRESHOLD option.' % (
2418                                                     FvName, FreeSizeValue, Threshold))
2419                             break
2420
2421     ## Generate GuidedSectionTools.txt in the FV directories.
2422     #
2423     def CreateGuidedSectionToolsFile(self,Wa):
2424         for BuildTarget in self.BuildTargetList:
2425             for ToolChain in self.ToolChainList:
2426                 FvDir = Wa.FvDir
2427                 if not os.path.exists(FvDir):
2428                     continue
2429                 for Arch in self.ArchList:
2430                     guidList = []
2431                     tooldefguidList = []
2432                     guidAttribs = []
2433                     for Platform in Wa.AutoGenObjectList:
2434                         if Platform.BuildTarget != BuildTarget:
2435                             continue
2436                         if Platform.ToolChain != ToolChain:
2437                             continue
2438                         if Platform.Arch != Arch:
2439                             continue
2440                         if hasattr (Platform, 'BuildOption'):
2441                             for Tool in Platform.BuildOption:
2442                                 if 'GUID' in Platform.BuildOption[Tool]:
2443                                     if 'PATH' in Platform.BuildOption[Tool]:
2444                                         value = Platform.BuildOption[Tool]['GUID']
2445                                         if value in guidList:
2446                                             EdkLogger.error("build", FORMAT_INVALID, "Duplicate GUID value %s used with Tool %s in DSC [BuildOptions]." % (value, Tool))
2447                                         path = Platform.BuildOption[Tool]['PATH']
2448                                         guidList.append(value)
2449                                         guidAttribs.append((value, Tool, path))
2450                         for Tool in Platform.ToolDefinition:
2451                             if 'GUID' in Platform.ToolDefinition[Tool]:
2452                                 if 'PATH' in Platform.ToolDefinition[Tool]:
2453                                     value = Platform.ToolDefinition[Tool]['GUID']
2454                                     if value in tooldefguidList:
2455                                         EdkLogger.error("build", FORMAT_INVALID, "Duplicate GUID value %s used with Tool %s in tools_def.txt." % (value, Tool))
2456                                     tooldefguidList.append(value)
2457                                     if value in guidList:
2458                                         # Already added by platform
2459                                         continue
2460                                     path = Platform.ToolDefinition[Tool]['PATH']
2461                                     guidList.append(value)
2462                                     guidAttribs.append((value, Tool, path))
2463                     # Sort by GuidTool name
2464                     guidAttribs = sorted (guidAttribs, key=lambda x: x[1])
2465                     # Write out GuidedSecTools.txt
2466                     toolsFile = os.path.join(FvDir, 'GuidedSectionTools.txt')
2467                     toolsFile = open(toolsFile, 'wt')
2468                     for guidedSectionTool in guidAttribs:
2469                         print(' '.join(guidedSectionTool), file=toolsFile)
2470                     toolsFile.close()
2471
2472     ## Returns the real path of the tool.
2473     #
2474     def GetRealPathOfTool (self, tool):
2475         if os.path.exists(tool):
2476             return os.path.realpath(tool)
2477         return tool
2478
2479     ## Launch the module or platform build
2480     #
2481     def Launch(self):
2482         self.AllDrivers = set()
2483         self.AllModules = set()
2484         self.PreMakeCacheMiss = set()
2485         self.PreMakeCacheHit = set()
2486         self.MakeCacheMiss = set()
2487         self.MakeCacheHit = set()
2488         if not self.ModuleFile:
2489             if not self.SpawnMode or self.Target not in ["", "all"]:
2490                 self.SpawnMode = False
2491                 self._BuildPlatform()
2492             else:
2493                 self._MultiThreadBuildPlatform()
2494         else:
2495             self.SpawnMode = False
2496             self._BuildModule()
2497
2498         if self.Target == 'cleanall':
2499             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
2500
2501     def CreateAsBuiltInf(self):
2502         for Module in self.BuildModules:
2503             Module.CreateAsBuiltInf()
2504
2505     def GenDestCache(self):
2506         for Module in self.AllModules:
2507             Module.GenPreMakefileHashList()
2508             Module.GenMakefileHashList()
2509             Module.CopyModuleToCache()
2510
2511     def GenLocalPreMakeCache(self):
2512         for Module in self.PreMakeCacheMiss:
2513             Module.GenPreMakefileHashList()
2514
2515     ## Do some clean-up works when error occurred
2516     def Relinquish(self):
2517         OldLogLevel = EdkLogger.GetLevel()
2518         EdkLogger.SetLevel(EdkLogger.ERROR)
2519         Utils.Progressor.Abort()
2520         if self.SpawnMode == True:
2521             BuildTask.Abort()
2522         EdkLogger.SetLevel(OldLogLevel)
2523
2524 def ParseDefines(DefineList=[]):
2525     DefineDict = {}
2526     if DefineList is not None:
2527         for Define in DefineList:
2528             DefineTokenList = Define.split("=", 1)
2529             if not GlobalData.gMacroNamePattern.match(DefineTokenList[0]):
2530                 EdkLogger.error('build', FORMAT_INVALID,
2531                                 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2532                                 ExtraData=DefineTokenList[0])
2533
2534             if len(DefineTokenList) == 1:
2535                 DefineDict[DefineTokenList[0]] = "TRUE"
2536             else:
2537                 DefineDict[DefineTokenList[0]] = DefineTokenList[1].strip()
2538     return DefineDict
2539
2540
2541
2542 def LogBuildTime(Time):
2543     if Time:
2544         TimeDurStr = ''
2545         TimeDur = time.gmtime(Time)
2546         if TimeDur.tm_yday > 1:
2547             TimeDurStr = time.strftime("%H:%M:%S", TimeDur) + ", %d day(s)" % (TimeDur.tm_yday - 1)
2548         else:
2549             TimeDurStr = time.strftime("%H:%M:%S", TimeDur)
2550         return TimeDurStr
2551     else:
2552         return None
2553 def ThreadNum():
2554     OptionParser = MyOptionParser()
2555     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2556         OptionParser.GetOption()
2557     BuildOption, BuildTarget = OptionParser.BuildOption, OptionParser.BuildTarget
2558     ThreadNumber = BuildOption.ThreadNumber
2559     GlobalData.gCmdConfDir = BuildOption.ConfDirectory
2560     if ThreadNumber is None:
2561         TargetObj = TargetTxtDict()
2562         ThreadNumber = TargetObj.Target.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
2563         if ThreadNumber == '':
2564             ThreadNumber = 0
2565         else:
2566             ThreadNumber = int(ThreadNumber, 0)
2567
2568     if ThreadNumber == 0:
2569         try:
2570             ThreadNumber = multiprocessing.cpu_count()
2571         except (ImportError, NotImplementedError):
2572             ThreadNumber = 1
2573     return ThreadNumber
2574 ## Tool entrance method
2575 #
2576 # This method mainly dispatch specific methods per the command line options.
2577 # If no error found, return zero value so the caller of this tool can know
2578 # if it's executed successfully or not.
2579 #
2580 #   @retval 0     Tool was successful
2581 #   @retval 1     Tool failed
2582 #
2583 LogQMaxSize = ThreadNum() * 10
2584 def Main():
2585     StartTime = time.time()
2586
2587     #
2588     # Create a log Queue
2589     #
2590     LogQ = mp.Queue(LogQMaxSize)
2591     # Initialize log system
2592     EdkLogger.LogClientInitialize(LogQ)
2593     GlobalData.gCommand = sys.argv[1:]
2594     #
2595     # Parse the options and args
2596     #
2597     OptionParser = MyOptionParser()
2598     if not OptionParser.BuildOption and not OptionParser.BuildTarget:
2599         OptionParser.GetOption()
2600     Option, Target = OptionParser.BuildOption, OptionParser.BuildTarget
2601     GlobalData.gOptions = Option
2602     GlobalData.gCaseInsensitive = Option.CaseInsensitive
2603
2604     # Set log level
2605     LogLevel = EdkLogger.INFO
2606     if Option.verbose is not None:
2607         EdkLogger.SetLevel(EdkLogger.VERBOSE)
2608         LogLevel = EdkLogger.VERBOSE
2609     elif Option.quiet is not None:
2610         EdkLogger.SetLevel(EdkLogger.QUIET)
2611         LogLevel = EdkLogger.QUIET
2612     elif Option.debug is not None:
2613         EdkLogger.SetLevel(Option.debug + 1)
2614         LogLevel = Option.debug + 1
2615     else:
2616         EdkLogger.SetLevel(EdkLogger.INFO)
2617
2618     if Option.WarningAsError == True:
2619         EdkLogger.SetWarningAsError()
2620     Log_Agent = LogAgent(LogQ,LogLevel,Option.LogFile)
2621     Log_Agent.start()
2622
2623     if platform.platform().find("Windows") >= 0:
2624         GlobalData.gIsWindows = True
2625     else:
2626         GlobalData.gIsWindows = False
2627
2628     EdkLogger.quiet("Build environment: %s" % platform.platform())
2629     EdkLogger.quiet(time.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time.localtime()));
2630     ReturnCode = 0
2631     MyBuild = None
2632     BuildError = True
2633     try:
2634         if len(Target) == 0:
2635             Target = "all"
2636         elif len(Target) >= 2:
2637             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "More than one targets are not supported.",
2638                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2639         else:
2640             Target = Target[0].lower()
2641
2642         if Target not in gSupportedTarget:
2643             EdkLogger.error("build", OPTION_NOT_SUPPORTED, "Not supported target [%s]." % Target,
2644                             ExtraData="Please select one of: %s" % (' '.join(gSupportedTarget)))
2645
2646         #
2647         # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2648         #
2649         CheckEnvVariable()
2650         GlobalData.gCommandLineDefines.update(ParseDefines(Option.Macros))
2651
2652         Workspace = os.getenv("WORKSPACE")
2653         #
2654         # Get files real name in workspace dir
2655         #
2656         GlobalData.gAllFiles = Utils.DirCache(Workspace)
2657
2658         WorkingDirectory = os.getcwd()
2659         if not Option.ModuleFile:
2660             FileList = glob.glob(os.path.normpath(os.path.join(WorkingDirectory, '*.inf')))
2661             FileNum = len(FileList)
2662             if FileNum >= 2:
2663                 EdkLogger.error("build", OPTION_NOT_SUPPORTED, "There are %d INF files in %s." % (FileNum, WorkingDirectory),
2664                                 ExtraData="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2665             elif FileNum == 1:
2666                 Option.ModuleFile = NormFile(FileList[0], Workspace)
2667
2668         if Option.ModuleFile:
2669             if os.path.isabs (Option.ModuleFile):
2670                 if os.path.normcase (os.path.normpath(Option.ModuleFile)).find (Workspace) == 0:
2671                     Option.ModuleFile = NormFile(os.path.normpath(Option.ModuleFile), Workspace)
2672             Option.ModuleFile = PathClass(Option.ModuleFile, Workspace)
2673             ErrorCode, ErrorInfo = Option.ModuleFile.Validate(".inf", False)
2674             if ErrorCode != 0:
2675                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2676
2677         if Option.PlatformFile is not None:
2678             if os.path.isabs (Option.PlatformFile):
2679                 if os.path.normcase (os.path.normpath(Option.PlatformFile)).find (Workspace) == 0:
2680                     Option.PlatformFile = NormFile(os.path.normpath(Option.PlatformFile), Workspace)
2681             Option.PlatformFile = PathClass(Option.PlatformFile, Workspace)
2682
2683         if Option.FdfFile is not None:
2684             if os.path.isabs (Option.FdfFile):
2685                 if os.path.normcase (os.path.normpath(Option.FdfFile)).find (Workspace) == 0:
2686                     Option.FdfFile = NormFile(os.path.normpath(Option.FdfFile), Workspace)
2687             Option.FdfFile = PathClass(Option.FdfFile, Workspace)
2688             ErrorCode, ErrorInfo = Option.FdfFile.Validate(".fdf", False)
2689             if ErrorCode != 0:
2690                 EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)
2691
2692         if Option.Flag is not None and Option.Flag not in ['-c', '-s']:
2693             EdkLogger.error("build", OPTION_VALUE_INVALID, "UNI flag must be one of -c or -s")
2694
2695         MyBuild = Build(Target, Workspace, Option,LogQ)
2696         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)
2697         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):
2698             MyBuild.Launch()
2699
2700         #
2701         # All job done, no error found and no exception raised
2702         #
2703         BuildError = False
2704     except FatalError as X:
2705         if MyBuild is not None:
2706             # for multi-thread build exits safely
2707             MyBuild.Relinquish()
2708         if Option is not None and Option.debug is not None:
2709             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2710         ReturnCode = X.args[0]
2711     except Warning as X:
2712         # error from Fdf parser
2713         if MyBuild is not None:
2714             # for multi-thread build exits safely
2715             MyBuild.Relinquish()
2716         if Option is not None and Option.debug is not None:
2717             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2718         else:
2719             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
2720         ReturnCode = FORMAT_INVALID
2721     except KeyboardInterrupt:
2722         if MyBuild is not None:
2723
2724             # for multi-thread build exits safely
2725             MyBuild.Relinquish()
2726         ReturnCode = ABORT_ERROR
2727         if Option is not None and Option.debug is not None:
2728             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2729     except:
2730         if MyBuild is not None:
2731             # for multi-thread build exits safely
2732             MyBuild.Relinquish()
2733
2734         # try to get the meta-file from the object causing exception
2735         Tb = sys.exc_info()[-1]
2736         MetaFile = GlobalData.gProcessingFile
2737         while Tb is not None:
2738             if 'self' in Tb.tb_frame.f_locals and hasattr(Tb.tb_frame.f_locals['self'], 'MetaFile'):
2739                 MetaFile = Tb.tb_frame.f_locals['self'].MetaFile
2740             Tb = Tb.tb_next
2741         EdkLogger.error(
2742                     "\nbuild",
2743                     CODE_ERROR,
2744                     "Unknown fatal error when processing [%s]" % MetaFile,
2745                     ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR,
2746                     RaiseError=False
2747                     )
2748         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
2749         ReturnCode = CODE_ERROR
2750     finally:
2751         Utils.Progressor.Abort()
2752         Utils.ClearDuplicatedInf()
2753
2754     if ReturnCode == 0:
2755         try:
2756             MyBuild.LaunchPostbuild()
2757             Conclusion = "Done"
2758         except:
2759             Conclusion = "Failed"
2760     elif ReturnCode == ABORT_ERROR:
2761         Conclusion = "Aborted"
2762     else:
2763         Conclusion = "Failed"
2764     FinishTime = time.time()
2765     BuildDuration = time.gmtime(int(round(FinishTime - StartTime)))
2766     BuildDurationStr = ""
2767     if BuildDuration.tm_yday > 1:
2768         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration) + ", %d day(s)" % (BuildDuration.tm_yday - 1)
2769     else:
2770         BuildDurationStr = time.strftime("%H:%M:%S", BuildDuration)
2771     if MyBuild is not None:
2772         if not BuildError:
2773             MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))
2774
2775     EdkLogger.SetLevel(EdkLogger.QUIET)
2776     EdkLogger.quiet("\n- %s -" % Conclusion)
2777     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))
2778     EdkLogger.quiet("Build total time: %s\n" % BuildDurationStr)
2779     Log_Agent.kill()
2780     Log_Agent.join()
2781     return ReturnCode
2782
2783 if __name__ == '__main__':
2784     try:
2785         mp.set_start_method('spawn')
2786     except:
2787         pass
2788     r = Main()
2789     ## 0-127 is a safe return range, and 1 is a standard default error
2790     if r < 0 or r > 127: r = 1
2791     sys.exit(r)