2 # build a platform or a module
4 # Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
5 # Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.<BR>
6 # Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
7 # Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
9 # SPDX-License-Identifier: BSD-2-Clause-Patent
15 from __future__
import print_function
16 from __future__
import absolute_import
17 import os
.path
as path
25 import multiprocessing
26 from threading
import Thread
,Event
,BoundedSemaphore
28 from subprocess
import Popen
,PIPE
, STDOUT
29 from collections
import OrderedDict
, defaultdict
31 from AutoGen
.PlatformAutoGen
import PlatformAutoGen
32 from AutoGen
.ModuleAutoGen
import ModuleAutoGen
33 from AutoGen
.WorkspaceAutoGen
import WorkspaceAutoGen
34 from AutoGen
.AutoGenWorker
import AutoGenWorkerInProcess
,AutoGenManager
,\
36 from AutoGen
import GenMake
37 from Common
import Misc
as Utils
39 from Common
.TargetTxtClassObject
import TargetTxtDict
40 from Common
.ToolDefClassObject
import ToolDefDict
41 from buildoptions
import MyOptionParser
42 from Common
.Misc
import PathClass
,SaveFileOnChange
,RemoveDirectory
43 from Common
.StringUtils
import NormPath
44 from Common
.MultipleWorkspace
import MultipleWorkspace
as mws
45 from Common
.BuildToolError
import *
46 from Common
.DataType
import *
47 import Common
.EdkLogger
as EdkLogger
49 from Workspace
.WorkspaceDatabase
import BuildDB
51 from BuildReport
import BuildReport
52 from GenPatchPcdTable
.GenPatchPcdTable
import PeImageClass
,parsePcdInfoFromMapFile
53 from PatchPcdValue
.PatchPcdValue
import PatchBinaryFile
55 import Common
.GlobalData
as GlobalData
56 from GenFds
.GenFds
import GenFds
, GenFdsApi
57 import multiprocessing
as mp
58 from multiprocessing
import Manager
59 from AutoGen
.DataPipe
import MemoryDataPipe
60 from AutoGen
.ModuleAutoGenHelper
import WorkSpaceInfo
, PlatformInfo
61 from GenFds
.FdfParser
import FdfParser
62 from AutoGen
.IncludesAutoGen
import IncludesAutoGen
63 from GenFds
.GenFds
import resetFdsGlobalVariable
65 ## standard targets of build command
66 gSupportedTarget
= ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
68 ## build configuration file
69 gBuildConfiguration
= "target.txt"
70 gToolsDefinition
= "tools_def.txt"
72 TemporaryTablePattern
= re
.compile(r
'^_\d+_\d+_[a-fA-F0-9]+$')
75 ## Check environment PATH variable to make sure the specified tool is found
77 # If the tool is found in the PATH, then True is returned
78 # Otherwise, False is returned
80 def IsToolInPath(tool
):
81 if 'PATHEXT' in os
.environ
:
82 extns
= os
.environ
['PATHEXT'].split(os
.path
.pathsep
)
85 for pathDir
in os
.environ
['PATH'].split(os
.path
.pathsep
):
87 if os
.path
.exists(os
.path
.join(pathDir
, tool
+ ext
)):
91 ## Check environment variables
93 # Check environment variables that must be set for build. Currently they are
95 # WORKSPACE The directory all packages/platforms start from
96 # EDK_TOOLS_PATH The directory contains all tools needed by the build
97 # PATH $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
99 # If any of above environment variable is not set or has error, the build
102 def CheckEnvVariable():
104 if "WORKSPACE" not in os
.environ
:
105 EdkLogger
.error("build", ATTRIBUTE_NOT_AVAILABLE
, "Environment variable not found",
106 ExtraData
="WORKSPACE")
108 WorkspaceDir
= os
.path
.normcase(os
.path
.normpath(os
.environ
["WORKSPACE"]))
109 if not os
.path
.exists(WorkspaceDir
):
110 EdkLogger
.error("build", FILE_NOT_FOUND
, "WORKSPACE doesn't exist", ExtraData
=WorkspaceDir
)
111 elif ' ' in WorkspaceDir
:
112 EdkLogger
.error("build", FORMAT_NOT_SUPPORTED
, "No space is allowed in WORKSPACE path",
113 ExtraData
=WorkspaceDir
)
114 os
.environ
["WORKSPACE"] = WorkspaceDir
116 # set multiple workspace
117 PackagesPath
= os
.getenv("PACKAGES_PATH")
118 mws
.setWs(WorkspaceDir
, PackagesPath
)
119 if mws
.PACKAGES_PATH
:
120 for Path
in mws
.PACKAGES_PATH
:
121 if not os
.path
.exists(Path
):
122 EdkLogger
.error("build", FILE_NOT_FOUND
, "One Path in PACKAGES_PATH doesn't exist", ExtraData
=Path
)
124 EdkLogger
.error("build", FORMAT_NOT_SUPPORTED
, "No space is allowed in PACKAGES_PATH", ExtraData
=Path
)
127 os
.environ
["EDK_TOOLS_PATH"] = os
.path
.normcase(os
.environ
["EDK_TOOLS_PATH"])
129 # check EDK_TOOLS_PATH
130 if "EDK_TOOLS_PATH" not in os
.environ
:
131 EdkLogger
.error("build", ATTRIBUTE_NOT_AVAILABLE
, "Environment variable not found",
132 ExtraData
="EDK_TOOLS_PATH")
135 if "PATH" not in os
.environ
:
136 EdkLogger
.error("build", ATTRIBUTE_NOT_AVAILABLE
, "Environment variable not found",
139 GlobalData
.gWorkspace
= WorkspaceDir
141 GlobalData
.gGlobalDefines
["WORKSPACE"] = WorkspaceDir
142 GlobalData
.gGlobalDefines
["EDK_TOOLS_PATH"] = os
.environ
["EDK_TOOLS_PATH"]
144 ## Get normalized file path
146 # Convert the path to be local format, and remove the WORKSPACE path at the
147 # beginning if the file path is given in full path.
149 # @param FilePath File path to be normalized
150 # @param Workspace Workspace path which the FilePath will be checked against
152 # @retval string The normalized file path
154 def NormFile(FilePath
, Workspace
):
155 # check if the path is absolute or relative
156 if os
.path
.isabs(FilePath
):
157 FileFullPath
= os
.path
.normpath(FilePath
)
159 FileFullPath
= os
.path
.normpath(mws
.join(Workspace
, FilePath
))
160 Workspace
= mws
.getWs(Workspace
, FilePath
)
162 # check if the file path exists or not
163 if not os
.path
.isfile(FileFullPath
):
164 EdkLogger
.error("build", FILE_NOT_FOUND
, ExtraData
="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath
)
166 # remove workspace directory from the beginning part of the file path
167 if Workspace
[-1] in ["\\", "/"]:
168 return FileFullPath
[len(Workspace
):]
170 return FileFullPath
[(len(Workspace
) + 1):]
172 ## Get the output of an external program
174 # This is the entrance method of thread reading output of an external program and
175 # putting them in STDOUT/STDERR of current program.
177 # @param From The stream message read from
178 # @param To The stream message put on
179 # @param ExitFlag The flag used to indicate stopping reading
181 def ReadMessage(From
, To
, ExitFlag
,MemTo
=None):
183 # read one line a time
184 Line
= From
.readline()
185 # empty string means "end"
186 if Line
is not None and Line
!= b
"":
187 LineStr
= Line
.rstrip().decode(encoding
='utf-8', errors
='ignore')
188 if MemTo
is not None:
189 if "Note: including file:" == LineStr
.lstrip()[:21]:
190 MemTo
.append(LineStr
)
193 MemTo
.append(LineStr
)
201 class MakeSubProc(Popen
):
202 def __init__(self
,*args
, **argv
):
203 super(MakeSubProc
,self
).__init
__(*args
, **argv
)
206 ## Launch an external program
208 # This method will call subprocess.Popen to execute an external program with
209 # given options in specified directory. Because of the dead-lock issue during
210 # redirecting output of the external program, threads are used to to do the
213 # @param Command A list or string containing the call of the program
214 # @param WorkingDir The directory in which the program will be running
216 def LaunchCommand(Command
, WorkingDir
,ModuleAuto
= None):
217 BeginTime
= time
.time()
218 # if working directory doesn't exist, Popen() will raise an exception
219 if not os
.path
.isdir(WorkingDir
):
220 EdkLogger
.error("build", FILE_NOT_FOUND
, ExtraData
=WorkingDir
)
222 # Command is used as the first Argument in following Popen().
223 # It could be a string or sequence. We find that if command is a string in following Popen(),
224 # ubuntu may fail with an error message that the command is not found.
225 # So here we may need convert command from string to list instance.
226 if platform
.system() != 'Windows':
227 if not isinstance(Command
, list):
228 Command
= Command
.split()
229 Command
= ' '.join(Command
)
232 EndOfProcedure
= None
235 Proc
= MakeSubProc(Command
, stdout
=PIPE
, stderr
=STDOUT
, env
=os
.environ
, cwd
=WorkingDir
, bufsize
=-1, shell
=True)
237 # launch two threads to read the STDOUT and STDERR
238 EndOfProcedure
= Event()
239 EndOfProcedure
.clear()
241 StdOutThread
= Thread(target
=ReadMessage
, args
=(Proc
.stdout
, EdkLogger
.info
, EndOfProcedure
,Proc
.ProcOut
))
242 StdOutThread
.setName("STDOUT-Redirector")
243 StdOutThread
.setDaemon(False)
247 # waiting for program exit
249 except: # in case of aborting
250 # terminate the threads redirecting the program output
251 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
252 if EndOfProcedure
is not None:
255 if not isinstance(Command
, type("")):
256 Command
= " ".join(Command
)
257 EdkLogger
.error("build", COMMAND_FAILURE
, "Failed to start command", ExtraData
="%s [%s]" % (Command
, WorkingDir
))
262 # check the return code of the program
263 if Proc
.returncode
!= 0:
264 if not isinstance(Command
, type("")):
265 Command
= " ".join(Command
)
266 # print out the Response file and its content when make failure
267 RespFile
= os
.path
.join(WorkingDir
, 'OUTPUT', 'respfilelist.txt')
268 if os
.path
.isfile(RespFile
):
270 RespContent
= f
.read()
272 EdkLogger
.info(RespContent
)
274 EdkLogger
.error("build", COMMAND_FAILURE
, ExtraData
="%s [%s]" % (Command
, WorkingDir
))
276 iau
= IncludesAutoGen(WorkingDir
,ModuleAuto
)
277 if ModuleAuto
.ToolChainFamily
== TAB_COMPILER_MSFT
:
278 iau
.CreateDepsFileForMsvc(Proc
.ProcOut
)
280 iau
.UpdateDepsFileforNonMsvc()
281 iau
.UpdateDepsFileforTrim()
282 iau
.CreateModuleDeps()
283 iau
.CreateDepsInclude()
284 iau
.CreateDepsTarget()
285 return "%dms" % (int(round((time
.time() - BeginTime
) * 1000)))
287 ## The smallest unit that can be built in multi-thread build mode
289 # This is the base class of build unit. The "Obj" parameter must provide
290 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
293 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
298 # @param self The object pointer
299 # @param Obj The object the build is working on
300 # @param Target The build target name, one of gSupportedTarget
301 # @param Dependency The BuildUnit(s) which must be completed in advance
302 # @param WorkingDir The directory build command starts in
304 def __init__(self
, Obj
, BuildCommand
, Target
, Dependency
, WorkingDir
="."):
305 self
.BuildObject
= Obj
306 self
.Dependency
= Dependency
307 self
.WorkingDir
= WorkingDir
309 self
.BuildCommand
= BuildCommand
311 EdkLogger
.error("build", OPTION_MISSING
,
312 "No build command found for this module. "
313 "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
314 (Obj
.BuildTarget
, Obj
.ToolChain
, Obj
.Arch
),
320 # It just returns the string representation of self.BuildObject
322 # @param self The object pointer
325 return str(self
.BuildObject
)
327 ## "==" operator method
329 # It just compares self.BuildObject with "Other". So self.BuildObject must
330 # provide its own __eq__() method.
332 # @param self The object pointer
333 # @param Other The other BuildUnit object compared to
335 def __eq__(self
, Other
):
336 return Other
and self
.BuildObject
== Other
.BuildObject \
337 and Other
.BuildObject \
338 and self
.BuildObject
.Arch
== Other
.BuildObject
.Arch
342 # It just returns the hash value of self.BuildObject which must be hashable.
344 # @param self The object pointer
347 return hash(self
.BuildObject
) + hash(self
.BuildObject
.Arch
)
350 return repr(self
.BuildObject
)
352 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
354 # This class is for module build by nmake/make build system. The "Obj" parameter
355 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
356 # be make units missing build.
358 # Currently the "Obj" should be only ModuleAutoGen object.
360 class ModuleMakeUnit(BuildUnit
):
363 # @param self The object pointer
364 # @param Obj The ModuleAutoGen object the build is working on
365 # @param Target The build target name, one of gSupportedTarget
367 def __init__(self
, Obj
, BuildCommand
,Target
):
368 Dependency
= [ModuleMakeUnit(La
, BuildCommand
,Target
) for La
in Obj
.LibraryAutoGenList
]
369 BuildUnit
.__init
__(self
, Obj
, BuildCommand
, Target
, Dependency
, Obj
.MakeFileDir
)
370 if Target
in [None, "", "all"]:
371 self
.Target
= "tbuild"
373 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
375 # This class is for platform build by nmake/make build system. The "Obj" parameter
376 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
377 # be make units missing build.
379 # Currently the "Obj" should be only PlatformAutoGen object.
381 class PlatformMakeUnit(BuildUnit
):
384 # @param self The object pointer
385 # @param Obj The PlatformAutoGen object the build is working on
386 # @param Target The build target name, one of gSupportedTarget
388 def __init__(self
, Obj
, BuildCommand
, Target
):
389 Dependency
= [ModuleMakeUnit(Lib
, BuildCommand
, Target
) for Lib
in self
.BuildObject
.LibraryAutoGenList
]
390 Dependency
.extend([ModuleMakeUnit(Mod
, BuildCommand
,Target
) for Mod
in self
.BuildObject
.ModuleAutoGenList
])
391 BuildUnit
.__init
__(self
, Obj
, BuildCommand
, Target
, Dependency
, Obj
.MakeFileDir
)
393 ## The class representing the task of a module build or platform build
395 # This class manages the build tasks in multi-thread build mode. Its jobs include
396 # scheduling thread running, catching thread error, monitor the thread status, etc.
399 # queue for tasks waiting for schedule
400 _PendingQueue
= OrderedDict()
401 _PendingQueueLock
= threading
.Lock()
403 # queue for tasks ready for running
404 _ReadyQueue
= OrderedDict()
405 _ReadyQueueLock
= threading
.Lock()
407 # queue for run tasks
408 _RunningQueue
= OrderedDict()
409 _RunningQueueLock
= threading
.Lock()
411 # queue containing all build tasks, in case duplicate build
412 _TaskQueue
= OrderedDict()
414 # flag indicating error occurs in a running thread
415 _ErrorFlag
= threading
.Event()
419 # BoundedSemaphore object used to control the number of running threads
422 # flag indicating if the scheduler is started or not
423 _SchedulerStopped
= threading
.Event()
424 _SchedulerStopped
.set()
426 ## Start the task scheduler thread
428 # @param MaxThreadNumber The maximum thread number
429 # @param ExitFlag Flag used to end the scheduler
432 def StartScheduler(MaxThreadNumber
, ExitFlag
):
433 SchedulerThread
= Thread(target
=BuildTask
.Scheduler
, args
=(MaxThreadNumber
, ExitFlag
))
434 SchedulerThread
.setName("Build-Task-Scheduler")
435 SchedulerThread
.setDaemon(False)
436 SchedulerThread
.start()
437 # wait for the scheduler to be started, especially useful in Linux
438 while not BuildTask
.IsOnGoing():
443 # @param MaxThreadNumber The maximum thread number
444 # @param ExitFlag Flag used to end the scheduler
447 def Scheduler(MaxThreadNumber
, ExitFlag
):
448 BuildTask
._SchedulerStopped
.clear()
450 # use BoundedSemaphore to control the maximum running threads
451 BuildTask
._Thread
= BoundedSemaphore(MaxThreadNumber
)
453 # scheduling loop, which will exits when no pending/ready task and
454 # indicated to do so, or there's error in running thread
456 while (len(BuildTask
._PendingQueue
) > 0 or len(BuildTask
._ReadyQueue
) > 0 \
457 or not ExitFlag
.isSet()) and not BuildTask
._ErrorFlag
.isSet():
458 EdkLogger
.debug(EdkLogger
.DEBUG_8
, "Pending Queue (%d), Ready Queue (%d)"
459 % (len(BuildTask
._PendingQueue
), len(BuildTask
._ReadyQueue
)))
461 # get all pending tasks
462 BuildTask
._PendingQueueLock
.acquire()
463 BuildObjectList
= list(BuildTask
._PendingQueue
.keys())
465 # check if their dependency is resolved, and if true, move them
468 for BuildObject
in BuildObjectList
:
469 Bt
= BuildTask
._PendingQueue
[BuildObject
]
471 BuildTask
._ReadyQueue
[BuildObject
] = BuildTask
._PendingQueue
.pop(BuildObject
)
472 BuildTask
._PendingQueueLock
.release()
474 # launch build thread until the maximum number of threads is reached
475 while not BuildTask
._ErrorFlag
.isSet():
476 # empty ready queue, do nothing further
477 if len(BuildTask
._ReadyQueue
) == 0:
480 # wait for active thread(s) exit
481 BuildTask
._Thread
.acquire(True)
483 # start a new build thread
484 Bo
, Bt
= BuildTask
._ReadyQueue
.popitem()
486 # move into running queue
487 BuildTask
._RunningQueueLock
.acquire()
488 BuildTask
._RunningQueue
[Bo
] = Bt
489 BuildTask
._RunningQueueLock
.release()
498 # wait for all running threads exit
499 if BuildTask
._ErrorFlag
.isSet():
500 EdkLogger
.quiet("\nWaiting for all build threads exit...")
501 # while not BuildTask._ErrorFlag.isSet() and \
502 while len(BuildTask
._RunningQueue
) > 0:
503 EdkLogger
.verbose("Waiting for thread ending...(%d)" % len(BuildTask
._RunningQueue
))
504 EdkLogger
.debug(EdkLogger
.DEBUG_8
, "Threads [%s]" % ", ".join(Th
.getName() for Th
in threading
.enumerate()))
507 except BaseException
as X
:
509 # TRICK: hide the output of threads left running, so that the user can
510 # catch the error message easily
512 EdkLogger
.SetLevel(EdkLogger
.ERROR
)
513 BuildTask
._ErrorFlag
.set()
514 BuildTask
._ErrorMessage
= "build thread scheduler error\n\t%s" % str(X
)
516 BuildTask
._PendingQueue
.clear()
517 BuildTask
._ReadyQueue
.clear()
518 BuildTask
._RunningQueue
.clear()
519 BuildTask
._TaskQueue
.clear()
520 BuildTask
._SchedulerStopped
.set()
522 ## Wait for all running method exit
525 def WaitForComplete():
526 BuildTask
._SchedulerStopped
.wait()
528 ## Check if the scheduler is running or not
532 return not BuildTask
._SchedulerStopped
.isSet()
537 if BuildTask
.IsOnGoing():
538 BuildTask
._ErrorFlag
.set()
539 BuildTask
.WaitForComplete()
541 ## Check if there's error in running thread
543 # Since the main thread cannot catch exceptions in other thread, we have to
544 # use threading.Event to communicate this formation to main thread.
548 return BuildTask
._ErrorFlag
.isSet()
550 ## Get error message in running thread
552 # Since the main thread cannot catch exceptions in other thread, we have to
553 # use a static variable to communicate this message to main thread.
556 def GetErrorMessage():
557 return BuildTask
._ErrorMessage
559 ## Factory method to create a BuildTask object
561 # This method will check if a module is building or has been built. And if
562 # true, just return the associated BuildTask object in the _TaskQueue. If
563 # not, create and return a new BuildTask object. The new BuildTask object
564 # will be appended to the _PendingQueue for scheduling later.
566 # @param BuildItem A BuildUnit object representing a build object
567 # @param Dependency The dependent build object of BuildItem
570 def New(BuildItem
, Dependency
=None):
571 if BuildItem
in BuildTask
._TaskQueue
:
572 Bt
= BuildTask
._TaskQueue
[BuildItem
]
576 Bt
._Init
(BuildItem
, Dependency
)
577 BuildTask
._TaskQueue
[BuildItem
] = Bt
579 BuildTask
._PendingQueueLock
.acquire()
580 BuildTask
._PendingQueue
[BuildItem
] = Bt
581 BuildTask
._PendingQueueLock
.release()
585 ## The real constructor of BuildTask
587 # @param BuildItem A BuildUnit object representing a build object
588 # @param Dependency The dependent build object of BuildItem
590 def _Init(self
, BuildItem
, Dependency
=None):
591 self
.BuildItem
= BuildItem
593 self
.DependencyList
= []
594 if Dependency
is None:
595 Dependency
= BuildItem
.Dependency
597 Dependency
.extend(BuildItem
.Dependency
)
598 self
.AddDependency(Dependency
)
599 # flag indicating build completes, used to avoid unnecessary re-build
600 self
.CompleteFlag
= False
602 ## Check if all dependent build tasks are completed or not
606 for Dep
in self
.DependencyList
:
607 if Dep
.CompleteFlag
== True:
614 ## Add dependent build task
616 # @param Dependency The list of dependent build objects
618 def AddDependency(self
, Dependency
):
619 for Dep
in Dependency
:
620 if not Dep
.BuildObject
.IsBinaryModule
and not Dep
.BuildObject
.CanSkipbyCache(GlobalData
.gModuleCacheHit
):
621 self
.DependencyList
.append(BuildTask
.New(Dep
)) # BuildTask list
623 ## The thread wrapper of LaunchCommand function
625 # @param Command A list or string contains the call of the command
626 # @param WorkingDir The directory in which the program will be running
628 def _CommandThread(self
, Command
, WorkingDir
):
630 self
.BuildItem
.BuildObject
.BuildTime
= LaunchCommand(Command
, WorkingDir
,self
.BuildItem
.BuildObject
)
631 self
.CompleteFlag
= True
633 # Run hash operation post dependency to account for libs
634 # Run if --hash or --binary-destination
635 if GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheSource
:
636 self
.BuildItem
.BuildObject
.GenModuleHash()
637 if GlobalData
.gBinCacheDest
:
638 self
.BuildItem
.BuildObject
.GenCMakeHash()
642 # TRICK: hide the output of threads left running, so that the user can
643 # catch the error message easily
645 if not BuildTask
._ErrorFlag
.isSet():
646 GlobalData
.gBuildingModule
= "%s [%s, %s, %s]" % (str(self
.BuildItem
.BuildObject
),
647 self
.BuildItem
.BuildObject
.Arch
,
648 self
.BuildItem
.BuildObject
.ToolChain
,
649 self
.BuildItem
.BuildObject
.BuildTarget
651 EdkLogger
.SetLevel(EdkLogger
.ERROR
)
652 BuildTask
._ErrorFlag
.set()
653 BuildTask
._ErrorMessage
= "%s broken\n %s [%s]" % \
654 (threading
.currentThread().getName(), Command
, WorkingDir
)
656 # indicate there's a thread is available for another build task
657 BuildTask
._RunningQueueLock
.acquire()
658 BuildTask
._RunningQueue
.pop(self
.BuildItem
)
659 BuildTask
._RunningQueueLock
.release()
660 BuildTask
._Thread
.release()
662 ## Start build task thread
665 EdkLogger
.quiet("Building ... %s" % repr(self
.BuildItem
))
666 Command
= self
.BuildItem
.BuildCommand
+ [self
.BuildItem
.Target
]
667 self
.BuildTread
= Thread(target
=self
._CommandThread
, args
=(Command
, self
.BuildItem
.WorkingDir
))
668 self
.BuildTread
.setName("build thread")
669 self
.BuildTread
.setDaemon(False)
670 self
.BuildTread
.start()
672 ## The class contains the information related to EFI image
677 # Constructor will load all required image information.
679 # @param BaseName The full file path of image.
680 # @param Guid The GUID for image.
681 # @param Arch Arch of this image.
682 # @param OutputDir The output directory for image.
683 # @param DebugDir The debug directory for image.
684 # @param ImageClass PeImage Information
686 def __init__(self
, BaseName
, Guid
, Arch
, OutputDir
, DebugDir
, ImageClass
):
687 self
.BaseName
= BaseName
690 self
.OutputDir
= OutputDir
691 self
.DebugDir
= DebugDir
692 self
.Image
= ImageClass
693 self
.Image
.Size
= (self
.Image
.Size
// 0x1000 + 1) * 0x1000
695 ## The class implementing the EDK2 build process
697 # The build process includes:
698 # 1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
699 # 2. Parse DSC file of active platform
700 # 3. Parse FDF file if any
701 # 4. Establish build database, including parse all other files (module, package)
702 # 5. Create AutoGen files (C code file, depex file, makefile) if necessary
703 # 6. Call build command
708 # Constructor will load all necessary configurations, parse platform, modules
709 # and packages and the establish a database for AutoGen.
711 # @param Target The build command target, one of gSupportedTarget
712 # @param WorkspaceDir The directory of workspace
713 # @param BuildOptions Build options passed from command line
715 def __init__(self
, Target
, WorkspaceDir
, BuildOptions
,log_q
):
716 self
.WorkspaceDir
= WorkspaceDir
718 self
.PlatformFile
= BuildOptions
.PlatformFile
719 self
.ModuleFile
= BuildOptions
.ModuleFile
720 self
.ArchList
= BuildOptions
.TargetArch
721 self
.ToolChainList
= BuildOptions
.ToolChain
722 self
.BuildTargetList
= BuildOptions
.BuildTarget
723 self
.Fdf
= BuildOptions
.FdfFile
724 self
.FdList
= BuildOptions
.RomImage
725 self
.FvList
= BuildOptions
.FvImage
726 self
.CapList
= BuildOptions
.CapName
727 self
.SilentMode
= BuildOptions
.SilentMode
728 self
.ThreadNumber
= 1
729 self
.SkipAutoGen
= BuildOptions
.SkipAutoGen
730 self
.Reparse
= BuildOptions
.Reparse
731 self
.SkuId
= BuildOptions
.SkuId
733 GlobalData
.gSKUID_CMD
= self
.SkuId
734 self
.ConfDirectory
= BuildOptions
.ConfDirectory
735 self
.SpawnMode
= True
736 self
.BuildReport
= BuildReport(BuildOptions
.ReportFile
, BuildOptions
.ReportType
)
740 self
.MakeFileName
= ""
741 TargetObj
= TargetTxtDict()
742 ToolDefObj
= ToolDefDict((os
.path
.join(os
.getenv("WORKSPACE"),"Conf")))
743 self
.TargetTxt
= TargetObj
.Target
744 self
.ToolDef
= ToolDefObj
.ToolDef
745 GlobalData
.BuildOptionPcd
= BuildOptions
.OptionPcd
if BuildOptions
.OptionPcd
else []
746 #Set global flag for build mode
747 GlobalData
.gIgnoreSource
= BuildOptions
.IgnoreSources
748 GlobalData
.gUseHashCache
= BuildOptions
.UseHashCache
749 GlobalData
.gBinCacheDest
= BuildOptions
.BinCacheDest
750 GlobalData
.gBinCacheSource
= BuildOptions
.BinCacheSource
751 GlobalData
.gEnableGenfdsMultiThread
= not BuildOptions
.NoGenfdsMultiThread
752 GlobalData
.gDisableIncludePathCheck
= BuildOptions
.DisableIncludePathCheck
754 if GlobalData
.gBinCacheDest
and not GlobalData
.gUseHashCache
:
755 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, ExtraData
="--binary-destination must be used together with --hash.")
757 if GlobalData
.gBinCacheSource
and not GlobalData
.gUseHashCache
:
758 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, ExtraData
="--binary-source must be used together with --hash.")
760 if GlobalData
.gBinCacheDest
and GlobalData
.gBinCacheSource
:
761 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, ExtraData
="--binary-destination can not be used together with --binary-source.")
763 if GlobalData
.gBinCacheSource
:
764 BinCacheSource
= os
.path
.normpath(GlobalData
.gBinCacheSource
)
765 if not os
.path
.isabs(BinCacheSource
):
766 BinCacheSource
= mws
.join(self
.WorkspaceDir
, BinCacheSource
)
767 GlobalData
.gBinCacheSource
= BinCacheSource
769 if GlobalData
.gBinCacheSource
is not None:
770 EdkLogger
.error("build", OPTION_VALUE_INVALID
, ExtraData
="Invalid value of option --binary-source.")
772 if GlobalData
.gBinCacheDest
:
773 BinCacheDest
= os
.path
.normpath(GlobalData
.gBinCacheDest
)
774 if not os
.path
.isabs(BinCacheDest
):
775 BinCacheDest
= mws
.join(self
.WorkspaceDir
, BinCacheDest
)
776 GlobalData
.gBinCacheDest
= BinCacheDest
778 if GlobalData
.gBinCacheDest
is not None:
779 EdkLogger
.error("build", OPTION_VALUE_INVALID
, ExtraData
="Invalid value of option --binary-destination.")
781 GlobalData
.gDatabasePath
= os
.path
.normpath(os
.path
.join(GlobalData
.gConfDirectory
, GlobalData
.gDatabasePath
))
782 if not os
.path
.exists(os
.path
.join(GlobalData
.gConfDirectory
, '.cache')):
783 os
.makedirs(os
.path
.join(GlobalData
.gConfDirectory
, '.cache'))
785 self
.BuildDatabase
= self
.Db
.BuildObject
787 self
.ToolChainFamily
= None
788 self
.LoadFixAddress
= 0
789 self
.UniFlag
= BuildOptions
.Flag
790 self
.BuildModules
= []
791 self
.HashSkipModules
= []
793 self
.LaunchPrebuildFlag
= False
794 self
.PlatformBuildPath
= os
.path
.join(GlobalData
.gConfDirectory
, '.cache', '.PlatformBuild')
795 if BuildOptions
.CommandLength
:
796 GlobalData
.gCommandMaxLength
= BuildOptions
.CommandLength
798 # print dot character during doing some time-consuming work
799 self
.Progress
= Utils
.Progressor()
800 # print current build environment and configuration
801 EdkLogger
.quiet("%-16s = %s" % ("WORKSPACE", os
.environ
["WORKSPACE"]))
802 if "PACKAGES_PATH" in os
.environ
:
803 # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
804 EdkLogger
.quiet("%-16s = %s" % ("PACKAGES_PATH", os
.path
.normcase(os
.path
.normpath(os
.environ
["PACKAGES_PATH"]))))
805 EdkLogger
.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os
.environ
["EDK_TOOLS_PATH"]))
806 if "EDK_TOOLS_BIN" in os
.environ
:
807 # Print the same path style with WORKSPACE env.
808 EdkLogger
.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os
.path
.normcase(os
.path
.normpath(os
.environ
["EDK_TOOLS_BIN"]))))
809 EdkLogger
.quiet("%-16s = %s" % ("CONF_PATH", GlobalData
.gConfDirectory
))
810 if "PYTHON3_ENABLE" in os
.environ
:
811 PYTHON3_ENABLE
= os
.environ
["PYTHON3_ENABLE"]
812 if PYTHON3_ENABLE
!= "TRUE":
813 PYTHON3_ENABLE
= "FALSE"
814 EdkLogger
.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE
))
815 if "PYTHON_COMMAND" in os
.environ
:
816 EdkLogger
.quiet("%-16s = %s" % ("PYTHON_COMMAND", os
.environ
["PYTHON_COMMAND"]))
820 EdkLogger
.quiet("%-16s = %s" % ("PREBUILD", self
.Prebuild
))
822 EdkLogger
.quiet("%-16s = %s" % ("POSTBUILD", self
.Postbuild
))
824 self
.LaunchPrebuild()
825 TargetObj
= TargetTxtDict()
826 ToolDefObj
= ToolDefDict((os
.path
.join(os
.getenv("WORKSPACE"), "Conf")))
827 self
.TargetTxt
= TargetObj
.Target
828 self
.ToolDef
= ToolDefObj
.ToolDef
829 if not (self
.LaunchPrebuildFlag
and os
.path
.exists(self
.PlatformBuildPath
)):
832 self
.AutoGenMgr
= None
834 os
.chdir(self
.WorkspaceDir
)
836 GlobalData
.file_lock
= mp
.Lock()
837 # Init cache data for local only
838 GlobalData
.gPackageHashFile
= dict()
839 GlobalData
.gModulePreMakeCacheStatus
= dict()
840 GlobalData
.gModuleMakeCacheStatus
= dict()
841 GlobalData
.gHashChainStatus
= dict()
842 GlobalData
.gCMakeHashFile
= dict()
843 GlobalData
.gModuleHashFile
= dict()
844 GlobalData
.gFileHashDict
= dict()
845 GlobalData
.gModuleAllCacheStatus
= set()
846 GlobalData
.gModuleCacheHit
= set()
848 def StartAutoGen(self
,mqueue
, DataPipe
,SkipAutoGen
,PcdMaList
,cqueue
):
852 feedback_q
= mp
.Queue()
853 error_event
= mp
.Event()
854 FfsCmd
= DataPipe
.Get("FfsCommand")
857 GlobalData
.FfsCmd
= FfsCmd
858 auto_workers
= [AutoGenWorkerInProcess(mqueue
,DataPipe
.dump_file
,feedback_q
,GlobalData
.file_lock
,cqueue
,self
.log_q
,error_event
) for _
in range(self
.ThreadNumber
)]
859 self
.AutoGenMgr
= AutoGenManager(auto_workers
,feedback_q
,error_event
)
860 self
.AutoGenMgr
.start()
861 for w
in auto_workers
:
863 if PcdMaList
is not None:
864 for PcdMa
in PcdMaList
:
865 # SourceFileList calling sequence impact the makefile string sequence.
866 # Create cached SourceFileList here to unify its calling sequence for both
867 # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
868 RetVal
= PcdMa
.SourceFileList
869 # Force cache miss for PCD driver
870 if GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheDest
and self
.Target
in [None, "", "all"]:
871 cqueue
.put((PcdMa
.MetaFile
.Path
, PcdMa
.Arch
, "PreMakeCache", False))
873 PcdMa
.CreateCodeFile(False)
874 PcdMa
.CreateMakeFile(False,GenFfsList
= DataPipe
.Get("FfsCommand").get((PcdMa
.MetaFile
.Path
, PcdMa
.Arch
),[]))
876 # Force cache miss for PCD driver
877 if GlobalData
.gBinCacheSource
and self
.Target
in [None, "", "all"]:
878 cqueue
.put((PcdMa
.MetaFile
.Path
, PcdMa
.Arch
, "MakeCache", False))
880 self
.AutoGenMgr
.join()
881 rt
= self
.AutoGenMgr
.Status
883 except FatalError
as e
:
884 return False, e
.args
[0]
886 return False, UNKNOWN_ERROR
888 ## Load configuration
890 # This method will parse target.txt and get the build configurations.
892 def LoadConfiguration(self
):
894 # if no ARCH given in command line, get it from target.txt
895 if not self
.ArchList
:
896 self
.ArchList
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_TARGET_ARCH
]
897 self
.ArchList
= tuple(self
.ArchList
)
899 # if no build target given in command line, get it from target.txt
900 if not self
.BuildTargetList
:
901 self
.BuildTargetList
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_TARGET
]
903 # if no tool chain given in command line, get it from target.txt
904 if not self
.ToolChainList
:
905 self
.ToolChainList
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_TOOL_CHAIN_TAG
]
906 if self
.ToolChainList
is None or len(self
.ToolChainList
) == 0:
907 EdkLogger
.error("build", RESOURCE_NOT_AVAILABLE
, ExtraData
="No toolchain given. Don't know how to build.\n")
909 # check if the tool chains are defined or not
910 NewToolChainList
= []
911 for ToolChain
in self
.ToolChainList
:
912 if ToolChain
not in self
.ToolDef
.ToolsDefTxtDatabase
[TAB_TOD_DEFINES_TOOL_CHAIN_TAG
]:
913 EdkLogger
.warn("build", "Tool chain [%s] is not defined" % ToolChain
)
915 NewToolChainList
.append(ToolChain
)
916 # if no tool chain available, break the build
917 if len(NewToolChainList
) == 0:
918 EdkLogger
.error("build", RESOURCE_NOT_AVAILABLE
,
919 ExtraData
="[%s] not defined. No toolchain available for build!\n" % ", ".join(self
.ToolChainList
))
921 self
.ToolChainList
= NewToolChainList
924 ToolDefinition
= self
.ToolDef
.ToolsDefTxtDatabase
925 for Tool
in self
.ToolChainList
:
926 if TAB_TOD_DEFINES_FAMILY
not in ToolDefinition
or Tool
not in ToolDefinition
[TAB_TOD_DEFINES_FAMILY
] \
927 or not ToolDefinition
[TAB_TOD_DEFINES_FAMILY
][Tool
]:
928 EdkLogger
.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool
)
929 ToolChainFamily
.append(TAB_COMPILER_MSFT
)
931 ToolChainFamily
.append(ToolDefinition
[TAB_TOD_DEFINES_FAMILY
][Tool
])
932 self
.ToolChainFamily
= ToolChainFamily
934 if not self
.PlatformFile
:
935 PlatformFile
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_ACTIVE_PLATFORM
]
937 # Try to find one in current directory
938 WorkingDirectory
= os
.getcwd()
939 FileList
= glob
.glob(os
.path
.normpath(os
.path
.join(WorkingDirectory
, '*.dsc')))
940 FileNum
= len(FileList
)
942 EdkLogger
.error("build", OPTION_MISSING
,
943 ExtraData
="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum
, WorkingDirectory
))
945 PlatformFile
= FileList
[0]
947 EdkLogger
.error("build", RESOURCE_NOT_AVAILABLE
,
948 ExtraData
="No active platform specified in target.txt or command line! Nothing can be built.\n")
950 self
.PlatformFile
= PathClass(NormFile(PlatformFile
, self
.WorkspaceDir
), self
.WorkspaceDir
)
951 self
.ThreadNumber
= ThreadNum()
952 ## Initialize build configuration
954 # This method will parse DSC file and merge the configurations from
955 # command line and target.txt, then get the final build configurations.
958 # parse target.txt, tools_def.txt, and platform file
959 self
.LoadConfiguration()
961 # Allow case-insensitive for those from command line or configuration file
962 ErrorCode
, ErrorInfo
= self
.PlatformFile
.Validate(".dsc", False)
964 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
967 def InitPreBuild(self
):
968 self
.LoadConfiguration()
969 ErrorCode
, ErrorInfo
= self
.PlatformFile
.Validate(".dsc", False)
971 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
972 if self
.BuildTargetList
:
973 GlobalData
.gGlobalDefines
['TARGET'] = self
.BuildTargetList
[0]
975 GlobalData
.gGlobalDefines
['ARCH'] = self
.ArchList
[0]
976 if self
.ToolChainList
:
977 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = self
.ToolChainList
[0]
978 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = self
.ToolChainList
[0]
979 if self
.ToolChainFamily
:
980 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[0]
981 if 'PREBUILD' in GlobalData
.gCommandLineDefines
:
982 self
.Prebuild
= GlobalData
.gCommandLineDefines
.get('PREBUILD')
985 Platform
= self
.Db
.MapPlatform(str(self
.PlatformFile
))
986 self
.Prebuild
= str(Platform
.Prebuild
)
990 # Evaluate all arguments and convert arguments that are WORKSPACE
991 # relative paths to absolute paths. Filter arguments that look like
992 # flags or do not follow the file/dir naming rules to avoid false
993 # positives on this conversion.
995 for Arg
in self
.Prebuild
.split():
997 # Do not modify Arg if it looks like a flag or an absolute file path
999 if Arg
.startswith('-') or os
.path
.isabs(Arg
):
1000 PrebuildList
.append(Arg
)
1003 # Do not modify Arg if it does not look like a Workspace relative
1004 # path that starts with a valid package directory name
1006 if not Arg
[0].isalpha() or os
.path
.dirname(Arg
) == '':
1007 PrebuildList
.append(Arg
)
1010 # If Arg looks like a WORKSPACE relative path, then convert to an
1011 # absolute path and check to see if the file exists.
1013 Temp
= mws
.join(self
.WorkspaceDir
, Arg
)
1014 if os
.path
.isfile(Temp
):
1016 PrebuildList
.append(Arg
)
1017 self
.Prebuild
= ' '.join(PrebuildList
)
1018 self
.Prebuild
+= self
.PassCommandOption(self
.BuildTargetList
, self
.ArchList
, self
.ToolChainList
, self
.PlatformFile
, self
.Target
)
1020 def InitPostBuild(self
):
1021 if 'POSTBUILD' in GlobalData
.gCommandLineDefines
:
1022 self
.Postbuild
= GlobalData
.gCommandLineDefines
.get('POSTBUILD')
1024 Platform
= self
.Db
.MapPlatform(str(self
.PlatformFile
))
1025 self
.Postbuild
= str(Platform
.Postbuild
)
1029 # Evaluate all arguments and convert arguments that are WORKSPACE
1030 # relative paths to absolute paths. Filter arguments that look like
1031 # flags or do not follow the file/dir naming rules to avoid false
1032 # positives on this conversion.
1034 for Arg
in self
.Postbuild
.split():
1036 # Do not modify Arg if it looks like a flag or an absolute file path
1038 if Arg
.startswith('-') or os
.path
.isabs(Arg
):
1039 PostbuildList
.append(Arg
)
1042 # Do not modify Arg if it does not look like a Workspace relative
1043 # path that starts with a valid package directory name
1045 if not Arg
[0].isalpha() or os
.path
.dirname(Arg
) == '':
1046 PostbuildList
.append(Arg
)
1049 # If Arg looks like a WORKSPACE relative path, then convert to an
1050 # absolute path and check to see if the file exists.
1052 Temp
= mws
.join(self
.WorkspaceDir
, Arg
)
1053 if os
.path
.isfile(Temp
):
1055 PostbuildList
.append(Arg
)
1056 self
.Postbuild
= ' '.join(PostbuildList
)
1057 self
.Postbuild
+= self
.PassCommandOption(self
.BuildTargetList
, self
.ArchList
, self
.ToolChainList
, self
.PlatformFile
, self
.Target
)
1059 def PassCommandOption(self
, BuildTarget
, TargetArch
, ToolChain
, PlatformFile
, Target
):
1061 if GlobalData
.gCommand
and isinstance(GlobalData
.gCommand
, list):
1062 BuildStr
+= ' ' + ' '.join(GlobalData
.gCommand
)
1065 ToolChainFlag
= False
1066 PlatformFileFlag
= False
1068 if GlobalData
.gOptions
and not GlobalData
.gOptions
.BuildTarget
:
1070 if GlobalData
.gOptions
and not GlobalData
.gOptions
.TargetArch
:
1072 if GlobalData
.gOptions
and not GlobalData
.gOptions
.ToolChain
:
1073 ToolChainFlag
= True
1074 if GlobalData
.gOptions
and not GlobalData
.gOptions
.PlatformFile
:
1075 PlatformFileFlag
= True
1077 if TargetFlag
and BuildTarget
:
1078 if isinstance(BuildTarget
, list) or isinstance(BuildTarget
, tuple):
1079 BuildStr
+= ' -b ' + ' -b '.join(BuildTarget
)
1080 elif isinstance(BuildTarget
, str):
1081 BuildStr
+= ' -b ' + BuildTarget
1082 if ArchFlag
and TargetArch
:
1083 if isinstance(TargetArch
, list) or isinstance(TargetArch
, tuple):
1084 BuildStr
+= ' -a ' + ' -a '.join(TargetArch
)
1085 elif isinstance(TargetArch
, str):
1086 BuildStr
+= ' -a ' + TargetArch
1087 if ToolChainFlag
and ToolChain
:
1088 if isinstance(ToolChain
, list) or isinstance(ToolChain
, tuple):
1089 BuildStr
+= ' -t ' + ' -t '.join(ToolChain
)
1090 elif isinstance(ToolChain
, str):
1091 BuildStr
+= ' -t ' + ToolChain
1092 if PlatformFileFlag
and PlatformFile
:
1093 if isinstance(PlatformFile
, list) or isinstance(PlatformFile
, tuple):
1094 BuildStr
+= ' -p ' + ' -p '.join(PlatformFile
)
1095 elif isinstance(PlatformFile
, str):
1096 BuildStr
+= ' -p' + PlatformFile
1097 BuildStr
+= ' --conf=' + GlobalData
.gConfDirectory
1099 BuildStr
+= ' ' + Target
1103 def LaunchPrebuild(self
):
1105 EdkLogger
.info("\n- Prebuild Start -\n")
1106 self
.LaunchPrebuildFlag
= True
1108 # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1109 # and preserve them for the rest of the main build step, because the child process environment will
1110 # evaporate as soon as it exits, we cannot get it in build step.
1112 PrebuildEnvFile
= os
.path
.join(GlobalData
.gConfDirectory
, '.cache', '.PrebuildEnv')
1113 if os
.path
.isfile(PrebuildEnvFile
):
1114 os
.remove(PrebuildEnvFile
)
1115 if os
.path
.isfile(self
.PlatformBuildPath
):
1116 os
.remove(self
.PlatformBuildPath
)
1117 if sys
.platform
== "win32":
1118 args
= ' && '.join((self
.Prebuild
, 'set > ' + PrebuildEnvFile
))
1119 Process
= Popen(args
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1121 args
= ' && '.join((self
.Prebuild
, 'env > ' + PrebuildEnvFile
))
1122 Process
= Popen(args
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1124 # launch two threads to read the STDOUT and STDERR
1125 EndOfProcedure
= Event()
1126 EndOfProcedure
.clear()
1128 StdOutThread
= Thread(target
=ReadMessage
, args
=(Process
.stdout
, EdkLogger
.info
, EndOfProcedure
))
1129 StdOutThread
.setName("STDOUT-Redirector")
1130 StdOutThread
.setDaemon(False)
1131 StdOutThread
.start()
1134 StdErrThread
= Thread(target
=ReadMessage
, args
=(Process
.stderr
, EdkLogger
.quiet
, EndOfProcedure
))
1135 StdErrThread
.setName("STDERR-Redirector")
1136 StdErrThread
.setDaemon(False)
1137 StdErrThread
.start()
1138 # waiting for program exit
1145 if Process
.returncode
!= 0 :
1146 EdkLogger
.error("Prebuild", PREBUILD_ERROR
, 'Prebuild process is not success!')
1148 if os
.path
.exists(PrebuildEnvFile
):
1149 f
= open(PrebuildEnvFile
)
1150 envs
= f
.readlines()
1152 envs
= [l
.split("=", 1) for l
in envs
]
1153 envs
= [[I
.strip() for I
in item
] for item
in envs
if len(item
) == 2]
1154 os
.environ
.update(dict(envs
))
1155 EdkLogger
.info("\n- Prebuild Done -\n")
1157 def LaunchPostbuild(self
):
1159 EdkLogger
.info("\n- Postbuild Start -\n")
1160 if sys
.platform
== "win32":
1161 Process
= Popen(self
.Postbuild
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1163 Process
= Popen(self
.Postbuild
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1164 # launch two threads to read the STDOUT and STDERR
1165 EndOfProcedure
= Event()
1166 EndOfProcedure
.clear()
1168 StdOutThread
= Thread(target
=ReadMessage
, args
=(Process
.stdout
, EdkLogger
.info
, EndOfProcedure
))
1169 StdOutThread
.setName("STDOUT-Redirector")
1170 StdOutThread
.setDaemon(False)
1171 StdOutThread
.start()
1174 StdErrThread
= Thread(target
=ReadMessage
, args
=(Process
.stderr
, EdkLogger
.quiet
, EndOfProcedure
))
1175 StdErrThread
.setName("STDERR-Redirector")
1176 StdErrThread
.setDaemon(False)
1177 StdErrThread
.start()
1178 # waiting for program exit
1185 if Process
.returncode
!= 0 :
1186 EdkLogger
.error("Postbuild", POSTBUILD_ERROR
, 'Postbuild process is not success!')
1187 EdkLogger
.info("\n- Postbuild Done -\n")
1189 ## Build a module or platform
1191 # Create autogen code and makefile for a module or platform, and the launch
1192 # "make" command to build it
1194 # @param Target The target of build command
1195 # @param Platform The platform file
1196 # @param Module The module file
1197 # @param BuildTarget The name of build target, one of "DEBUG", "RELEASE"
1198 # @param ToolChain The name of toolchain to build
1199 # @param Arch The arch of the module/platform
1200 # @param CreateDepModuleCodeFile Flag used to indicate creating code
1201 # for dependent modules/Libraries
1202 # @param CreateDepModuleMakeFile Flag used to indicate creating makefile
1203 # for dependent modules/Libraries
1205 def _BuildPa(self
, Target
, AutoGenObject
, CreateDepsCodeFile
=True, CreateDepsMakeFile
=True, BuildModule
=False, FfsCommand
=None, PcdMaList
=None):
1206 if AutoGenObject
is None:
1208 if FfsCommand
is None:
1210 # skip file generation for cleanxxx targets, run and fds target
1211 if Target
not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1212 # for target which must generate AutoGen code and makefile
1214 for m
in AutoGenObject
.GetAllModuleInfo
:
1217 AutoGenObject
.DataPipe
.DataContainer
= {"CommandTarget": self
.Target
}
1218 AutoGenObject
.DataPipe
.DataContainer
= {"Workspace_timestamp": AutoGenObject
.Workspace
._SrcTimeStamp
}
1219 AutoGenObject
.CreateLibModuelDirs()
1220 AutoGenObject
.DataPipe
.DataContainer
= {"LibraryBuildDirectoryList":AutoGenObject
.LibraryBuildDirectoryList
}
1221 AutoGenObject
.DataPipe
.DataContainer
= {"ModuleBuildDirectoryList":AutoGenObject
.ModuleBuildDirectoryList
}
1222 AutoGenObject
.DataPipe
.DataContainer
= {"FdsCommandDict": AutoGenObject
.Workspace
.GenFdsCommandDict
}
1223 self
.Progress
.Start("Generating makefile and code")
1224 data_pipe_file
= os
.path
.join(AutoGenObject
.BuildDir
, "GlobalVar_%s_%s.bin" % (str(AutoGenObject
.Guid
),AutoGenObject
.Arch
))
1225 AutoGenObject
.DataPipe
.dump(data_pipe_file
)
1227 autogen_rt
,errorcode
= self
.StartAutoGen(mqueue
, AutoGenObject
.DataPipe
, self
.SkipAutoGen
, PcdMaList
, cqueue
)
1228 AutoGenIdFile
= os
.path
.join(GlobalData
.gConfDirectory
,".AutoGenIdFile.txt")
1229 with
open(AutoGenIdFile
,"w") as fw
:
1230 fw
.write("Arch=%s\n" % "|".join((AutoGenObject
.Workspace
.ArchList
)))
1231 fw
.write("BuildDir=%s\n" % AutoGenObject
.Workspace
.BuildDir
)
1232 fw
.write("PlatformGuid=%s\n" % str(AutoGenObject
.Guid
))
1233 self
.Progress
.Stop("done!")
1235 self
.AutoGenMgr
.TerminateWorkers()
1236 self
.AutoGenMgr
.join(1)
1237 raise FatalError(errorcode
)
1238 AutoGenObject
.CreateCodeFile(False)
1239 AutoGenObject
.CreateMakeFile(False)
1241 # always recreate top/platform makefile when clean, just in case of inconsistency
1242 AutoGenObject
.CreateCodeFile(True)
1243 AutoGenObject
.CreateMakeFile(True)
1245 if EdkLogger
.GetLevel() == EdkLogger
.QUIET
:
1246 EdkLogger
.quiet("Building ... %s" % repr(AutoGenObject
))
1248 BuildCommand
= AutoGenObject
.BuildCommand
1249 if BuildCommand
is None or len(BuildCommand
) == 0:
1250 EdkLogger
.error("build", OPTION_MISSING
,
1251 "No build command found for this module. "
1252 "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1253 (AutoGenObject
.BuildTarget
, AutoGenObject
.ToolChain
, AutoGenObject
.Arch
),
1254 ExtraData
=str(AutoGenObject
))
1262 BuildCommand
= BuildCommand
+ [Target
]
1263 LaunchCommand(BuildCommand
, AutoGenObject
.MakeFileDir
)
1264 self
.CreateAsBuiltInf()
1265 if GlobalData
.gBinCacheDest
:
1267 elif GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheSource
:
1269 # Update PreMakeCacheChain files
1270 self
.GenLocalPreMakeCache()
1271 self
.BuildModules
= []
1275 if Target
== 'libraries':
1277 for Lib
in AutoGenObject
.LibraryAutoGenList
:
1278 if not Lib
.IsBinaryModule
:
1279 DirList
.append((os
.path
.join(AutoGenObject
.BuildDir
, Lib
.BuildDir
),Lib
))
1280 for Lib
, LibAutoGen
in DirList
:
1281 NewBuildCommand
= BuildCommand
+ ['-f', os
.path
.normpath(os
.path
.join(Lib
, self
.MakeFileName
)), 'pbuild']
1282 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
,LibAutoGen
)
1286 if Target
== 'modules':
1288 for Lib
in AutoGenObject
.LibraryAutoGenList
:
1289 if not Lib
.IsBinaryModule
:
1290 DirList
.append((os
.path
.join(AutoGenObject
.BuildDir
, Lib
.BuildDir
),Lib
))
1291 for Lib
, LibAutoGen
in DirList
:
1292 NewBuildCommand
= BuildCommand
+ ['-f', os
.path
.normpath(os
.path
.join(Lib
, self
.MakeFileName
)), 'pbuild']
1293 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
,LibAutoGen
)
1296 for ModuleAutoGen
in AutoGenObject
.ModuleAutoGenList
:
1297 if not ModuleAutoGen
.IsBinaryModule
:
1298 DirList
.append((os
.path
.join(AutoGenObject
.BuildDir
, ModuleAutoGen
.BuildDir
),ModuleAutoGen
))
1299 for Mod
,ModAutoGen
in DirList
:
1300 NewBuildCommand
= BuildCommand
+ ['-f', os
.path
.normpath(os
.path
.join(Mod
, self
.MakeFileName
)), 'pbuild']
1301 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
,ModAutoGen
)
1302 self
.CreateAsBuiltInf()
1303 if GlobalData
.gBinCacheDest
:
1305 elif GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheSource
:
1307 # Update PreMakeCacheChain files
1308 self
.GenLocalPreMakeCache()
1309 self
.BuildModules
= []
1313 if Target
== 'cleanlib':
1314 for Lib
in AutoGenObject
.LibraryBuildDirectoryList
:
1315 LibMakefile
= os
.path
.normpath(os
.path
.join(Lib
, self
.MakeFileName
))
1316 if os
.path
.exists(LibMakefile
):
1317 NewBuildCommand
= BuildCommand
+ ['-f', LibMakefile
, 'cleanall']
1318 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1322 if Target
== 'clean':
1323 for Mod
in AutoGenObject
.ModuleBuildDirectoryList
:
1324 ModMakefile
= os
.path
.normpath(os
.path
.join(Mod
, self
.MakeFileName
))
1325 if os
.path
.exists(ModMakefile
):
1326 NewBuildCommand
= BuildCommand
+ ['-f', ModMakefile
, 'cleanall']
1327 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1328 for Lib
in AutoGenObject
.LibraryBuildDirectoryList
:
1329 LibMakefile
= os
.path
.normpath(os
.path
.join(Lib
, self
.MakeFileName
))
1330 if os
.path
.exists(LibMakefile
):
1331 NewBuildCommand
= BuildCommand
+ ['-f', LibMakefile
, 'cleanall']
1332 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1336 if Target
== 'cleanall':
1338 #os.rmdir(AutoGenObject.BuildDir)
1339 RemoveDirectory(AutoGenObject
.BuildDir
, True)
1340 except WindowsError as X
:
1341 EdkLogger
.error("build", FILE_DELETE_FAILURE
, ExtraData
=str(X
))
1344 ## Build a module or platform
1346 # Create autogen code and makefile for a module or platform, and the launch
1347 # "make" command to build it
1349 # @param Target The target of build command
1350 # @param Platform The platform file
1351 # @param Module The module file
1352 # @param BuildTarget The name of build target, one of "DEBUG", "RELEASE"
1353 # @param ToolChain The name of toolchain to build
1354 # @param Arch The arch of the module/platform
1355 # @param CreateDepModuleCodeFile Flag used to indicate creating code
1356 # for dependent modules/Libraries
1357 # @param CreateDepModuleMakeFile Flag used to indicate creating makefile
1358 # for dependent modules/Libraries
1360 def _Build(self
, Target
, AutoGenObject
, CreateDepsCodeFile
=True, CreateDepsMakeFile
=True, BuildModule
=False):
1361 if AutoGenObject
is None:
1364 # skip file generation for cleanxxx targets, run and fds target
1365 if Target
not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1366 # for target which must generate AutoGen code and makefile
1367 if not self
.SkipAutoGen
or Target
== 'genc':
1368 self
.Progress
.Start("Generating code")
1369 AutoGenObject
.CreateCodeFile(CreateDepsCodeFile
)
1370 self
.Progress
.Stop("done!")
1371 if Target
== "genc":
1374 if not self
.SkipAutoGen
or Target
== 'genmake':
1375 self
.Progress
.Start("Generating makefile")
1376 AutoGenObject
.CreateMakeFile(CreateDepsMakeFile
)
1377 #AutoGenObject.CreateAsBuiltInf()
1378 self
.Progress
.Stop("done!")
1379 if Target
== "genmake":
1382 # always recreate top/platform makefile when clean, just in case of inconsistency
1383 AutoGenObject
.CreateCodeFile(True)
1384 AutoGenObject
.CreateMakeFile(True)
1386 if EdkLogger
.GetLevel() == EdkLogger
.QUIET
:
1387 EdkLogger
.quiet("Building ... %s" % repr(AutoGenObject
))
1389 BuildCommand
= AutoGenObject
.BuildCommand
1390 if BuildCommand
is None or len(BuildCommand
) == 0:
1391 EdkLogger
.error("build", OPTION_MISSING
,
1392 "No build command found for this module. "
1393 "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1394 (AutoGenObject
.BuildTarget
, AutoGenObject
.ToolChain
, AutoGenObject
.Arch
),
1395 ExtraData
=str(AutoGenObject
))
1400 BuildCommand
= BuildCommand
+ [Target
]
1401 AutoGenObject
.BuildTime
= LaunchCommand(BuildCommand
, AutoGenObject
.MakeFileDir
)
1402 self
.CreateAsBuiltInf()
1403 if GlobalData
.gBinCacheDest
:
1405 elif GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheSource
:
1407 # Update PreMakeCacheChain files
1408 self
.GenLocalPreMakeCache()
1409 self
.BuildModules
= []
1414 if GenFdsApi(AutoGenObject
.GenFdsCommandDict
, self
.Db
):
1415 EdkLogger
.error("build", COMMAND_FAILURE
)
1423 if Target
== 'libraries':
1430 if Target
== 'cleanall':
1432 #os.rmdir(AutoGenObject.BuildDir)
1433 RemoveDirectory(AutoGenObject
.BuildDir
, True)
1434 except WindowsError as X
:
1435 EdkLogger
.error("build", FILE_DELETE_FAILURE
, ExtraData
=str(X
))
1438 ## Rebase module image and Get function address for the input module list.
1440 def _RebaseModule (self
, MapBuffer
, BaseAddress
, ModuleList
, AddrIsOffset
= True, ModeIsSmm
= False):
1442 AddrIsOffset
= False
1443 for InfFile
in ModuleList
:
1444 sys
.stdout
.write (".")
1446 ModuleInfo
= ModuleList
[InfFile
]
1447 ModuleName
= ModuleInfo
.BaseName
1448 ModuleOutputImage
= ModuleInfo
.Image
.FileName
1449 ModuleDebugImage
= os
.path
.join(ModuleInfo
.DebugDir
, ModuleInfo
.BaseName
+ '.efi')
1450 ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1452 BaseAddress
= BaseAddress
- ModuleInfo
.Image
.Size
1454 # Update Image to new BaseAddress by GenFw tool
1456 LaunchCommand(["GenFw", "--rebase", str(BaseAddress
), "-r", ModuleOutputImage
], ModuleInfo
.OutputDir
)
1457 LaunchCommand(["GenFw", "--rebase", str(BaseAddress
), "-r", ModuleDebugImage
], ModuleInfo
.DebugDir
)
1460 # Set new address to the section header only for SMM driver.
1462 LaunchCommand(["GenFw", "--address", str(BaseAddress
), "-r", ModuleOutputImage
], ModuleInfo
.OutputDir
)
1463 LaunchCommand(["GenFw", "--address", str(BaseAddress
), "-r", ModuleDebugImage
], ModuleInfo
.DebugDir
)
1465 # Collect function address from Map file
1467 ImageMapTable
= ModuleOutputImage
.replace('.efi', '.map')
1469 if os
.path
.exists(ImageMapTable
):
1470 OrigImageBaseAddress
= 0
1471 ImageMap
= open(ImageMapTable
, 'r')
1472 for LinStr
in ImageMap
:
1473 if len (LinStr
.strip()) == 0:
1476 # Get the preferred address set on link time.
1478 if LinStr
.find ('Preferred load address is') != -1:
1479 StrList
= LinStr
.split()
1480 OrigImageBaseAddress
= int (StrList
[len(StrList
) - 1], 16)
1482 StrList
= LinStr
.split()
1483 if len (StrList
) > 4:
1484 if StrList
[3] == 'f' or StrList
[3] == 'F':
1486 RelativeAddress
= int (StrList
[2], 16) - OrigImageBaseAddress
1487 FunctionList
.append ((Name
, RelativeAddress
))
1491 # Add general information.
1494 MapBuffer
.append('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName
, BaseAddress
, BaseAddress
+ ModuleInfo
.Image
.EntryPoint
))
1496 MapBuffer
.append('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName
, 0 - BaseAddress
, 0 - (BaseAddress
+ ModuleInfo
.Image
.EntryPoint
)))
1498 MapBuffer
.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName
, BaseAddress
, BaseAddress
+ ModuleInfo
.Image
.EntryPoint
))
1500 # Add guid and general seciton section.
1502 TextSectionAddress
= 0
1503 DataSectionAddress
= 0
1504 for SectionHeader
in ModuleInfo
.Image
.SectionHeaderList
:
1505 if SectionHeader
[0] == '.text':
1506 TextSectionAddress
= SectionHeader
[1]
1507 elif SectionHeader
[0] in ['.data', '.sdata']:
1508 DataSectionAddress
= SectionHeader
[1]
1510 MapBuffer
.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo
.Guid
, 0 - (BaseAddress
+ TextSectionAddress
), 0 - (BaseAddress
+ DataSectionAddress
)))
1512 MapBuffer
.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo
.Guid
, BaseAddress
+ TextSectionAddress
, BaseAddress
+ DataSectionAddress
))
1514 # Add debug image full path.
1516 MapBuffer
.append('(IMAGE=%s)\n\n' % (ModuleDebugImage
))
1518 # Add function address
1520 for Function
in FunctionList
:
1522 MapBuffer
.append(' -0x%010X %s\n' % (0 - (BaseAddress
+ Function
[1]), Function
[0]))
1524 MapBuffer
.append(' 0x%010X %s\n' % (BaseAddress
+ Function
[1], Function
[0]))
1528 # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1531 BaseAddress
= BaseAddress
+ ModuleInfo
.Image
.Size
1533 ## Collect MAP information of all FVs
1535 def _CollectFvMapBuffer (self
, MapBuffer
, Wa
, ModuleList
):
1537 # First get the XIP base address for FV map file.
1538 GuidPattern
= re
.compile("[-a-fA-F0-9]+")
1539 GuidName
= re
.compile(r
"\(GUID=[-a-fA-F0-9]+")
1540 for FvName
in Wa
.FdfProfile
.FvDict
:
1541 FvMapBuffer
= os
.path
.join(Wa
.FvDir
, FvName
+ '.Fv.map')
1542 if not os
.path
.exists(FvMapBuffer
):
1544 FvMap
= open(FvMapBuffer
, 'r')
1545 #skip FV size information
1551 MatchGuid
= GuidPattern
.match(Line
)
1552 if MatchGuid
is not None:
1554 # Replace GUID with module name
1556 GuidString
= MatchGuid
.group()
1557 if GuidString
.upper() in ModuleList
:
1558 Line
= Line
.replace(GuidString
, ModuleList
[GuidString
.upper()].Name
)
1559 MapBuffer
.append(Line
)
1561 # Add the debug image full path.
1563 MatchGuid
= GuidName
.match(Line
)
1564 if MatchGuid
is not None:
1565 GuidString
= MatchGuid
.group().split("=")[1]
1566 if GuidString
.upper() in ModuleList
:
1567 MapBuffer
.append('(IMAGE=%s)\n' % (os
.path
.join(ModuleList
[GuidString
.upper()].DebugDir
, ModuleList
[GuidString
.upper()].Name
+ '.efi')))
1571 ## Collect MAP information of all modules
1573 def _CollectModuleMapBuffer (self
, MapBuffer
, ModuleList
):
1574 sys
.stdout
.write ("Generate Load Module At Fix Address Map")
1576 PatchEfiImageList
= []
1584 # reserve 4K size in SMRAM to make SMM module address not from 0.
1586 for ModuleGuid
in ModuleList
:
1587 Module
= ModuleList
[ModuleGuid
]
1588 GlobalData
.gProcessingFile
= "%s [%s, %s, %s]" % (Module
.MetaFile
, Module
.Arch
, Module
.ToolChain
, Module
.BuildTarget
)
1590 OutputImageFile
= ''
1591 for ResultFile
in Module
.CodaTargetList
:
1592 if str(ResultFile
.Target
).endswith('.efi'):
1594 # module list for PEI, DXE, RUNTIME and SMM
1596 OutputImageFile
= os
.path
.join(Module
.OutputDir
, Module
.Name
+ '.efi')
1597 ImageClass
= PeImageClass (OutputImageFile
)
1598 if not ImageClass
.IsValid
:
1599 EdkLogger
.error("build", FILE_PARSE_FAILURE
, ExtraData
=ImageClass
.ErrorInfo
)
1600 ImageInfo
= PeImageInfo(Module
.Name
, Module
.Guid
, Module
.Arch
, Module
.OutputDir
, Module
.DebugDir
, ImageClass
)
1601 if Module
.ModuleType
in [SUP_MODULE_PEI_CORE
, SUP_MODULE_PEIM
, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER
, EDK_COMPONENT_TYPE_PIC_PEIM
, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM
, SUP_MODULE_DXE_CORE
]:
1602 PeiModuleList
[Module
.MetaFile
] = ImageInfo
1603 PeiSize
+= ImageInfo
.Image
.Size
1604 elif Module
.ModuleType
in [EDK_COMPONENT_TYPE_BS_DRIVER
, SUP_MODULE_DXE_DRIVER
, SUP_MODULE_UEFI_DRIVER
]:
1605 BtModuleList
[Module
.MetaFile
] = ImageInfo
1606 BtSize
+= ImageInfo
.Image
.Size
1607 elif Module
.ModuleType
in [SUP_MODULE_DXE_RUNTIME_DRIVER
, EDK_COMPONENT_TYPE_RT_DRIVER
, SUP_MODULE_DXE_SAL_DRIVER
, EDK_COMPONENT_TYPE_SAL_RT_DRIVER
]:
1608 RtModuleList
[Module
.MetaFile
] = ImageInfo
1609 RtSize
+= ImageInfo
.Image
.Size
1610 elif Module
.ModuleType
in [SUP_MODULE_SMM_CORE
, SUP_MODULE_DXE_SMM_DRIVER
, SUP_MODULE_MM_STANDALONE
, SUP_MODULE_MM_CORE_STANDALONE
]:
1611 SmmModuleList
[Module
.MetaFile
] = ImageInfo
1612 SmmSize
+= ImageInfo
.Image
.Size
1613 if Module
.ModuleType
== SUP_MODULE_DXE_SMM_DRIVER
:
1614 PiSpecVersion
= Module
.Module
.Specification
.get('PI_SPECIFICATION_VERSION', '0x00000000')
1615 # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1616 if int(PiSpecVersion
, 16) < 0x0001000A:
1617 BtModuleList
[Module
.MetaFile
] = ImageInfo
1618 BtSize
+= ImageInfo
.Image
.Size
1621 # EFI image is final target.
1622 # Check EFI image contains patchable FixAddress related PCDs.
1624 if OutputImageFile
!= '':
1625 ModuleIsPatch
= False
1626 for Pcd
in Module
.ModulePcdList
:
1627 if Pcd
.Type
== TAB_PCDS_PATCHABLE_IN_MODULE
and Pcd
.TokenCName
in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET
:
1628 ModuleIsPatch
= True
1630 if not ModuleIsPatch
:
1631 for Pcd
in Module
.LibraryPcdList
:
1632 if Pcd
.Type
== TAB_PCDS_PATCHABLE_IN_MODULE
and Pcd
.TokenCName
in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET
:
1633 ModuleIsPatch
= True
1636 if not ModuleIsPatch
:
1639 # Module includes the patchable load fix address PCDs.
1640 # It will be fixed up later.
1642 PatchEfiImageList
.append (OutputImageFile
)
1645 # Get Top Memory address
1647 ReservedRuntimeMemorySize
= 0
1648 TopMemoryAddress
= 0
1649 if self
.LoadFixAddress
== 0xFFFFFFFFFFFFFFFF:
1650 TopMemoryAddress
= 0
1652 TopMemoryAddress
= self
.LoadFixAddress
1653 if TopMemoryAddress
< RtSize
+ BtSize
+ PeiSize
:
1654 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1657 # Patch FixAddress related PCDs into EFI image
1659 for EfiImage
in PatchEfiImageList
:
1660 EfiImageMap
= EfiImage
.replace('.efi', '.map')
1661 if not os
.path
.exists(EfiImageMap
):
1664 # Get PCD offset in EFI image by GenPatchPcdTable function
1666 PcdTable
= parsePcdInfoFromMapFile(EfiImageMap
, EfiImage
)
1668 # Patch real PCD value by PatchPcdValue tool
1670 for PcdInfo
in PcdTable
:
1672 if PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE
:
1673 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE
, str (PeiSize
// 0x1000))
1674 elif PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE
:
1675 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE
, str (BtSize
// 0x1000))
1676 elif PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE
:
1677 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE
, str (RtSize
// 0x1000))
1678 elif PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE
and len (SmmModuleList
) > 0:
1679 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE
, str (SmmSize
// 0x1000))
1680 if ReturnValue
!= 0:
1681 EdkLogger
.error("build", PARAMETER_INVALID
, "Patch PCD value failed", ExtraData
=ErrorInfo
)
1683 MapBuffer
.append('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize
// 0x1000))
1684 MapBuffer
.append('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize
// 0x1000))
1685 MapBuffer
.append('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize
// 0x1000))
1686 if len (SmmModuleList
) > 0:
1687 MapBuffer
.append('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize
// 0x1000))
1689 PeiBaseAddr
= TopMemoryAddress
- RtSize
- BtSize
1690 BtBaseAddr
= TopMemoryAddress
- RtSize
1691 RtBaseAddr
= TopMemoryAddress
- ReservedRuntimeMemorySize
1693 self
._RebaseModule
(MapBuffer
, PeiBaseAddr
, PeiModuleList
, TopMemoryAddress
== 0)
1694 self
._RebaseModule
(MapBuffer
, BtBaseAddr
, BtModuleList
, TopMemoryAddress
== 0)
1695 self
._RebaseModule
(MapBuffer
, RtBaseAddr
, RtModuleList
, TopMemoryAddress
== 0)
1696 self
._RebaseModule
(MapBuffer
, 0x1000, SmmModuleList
, AddrIsOffset
=False, ModeIsSmm
=True)
1697 MapBuffer
.append('\n\n')
1698 sys
.stdout
.write ("\n")
1701 ## Save platform Map file
1703 def _SaveMapFile (self
, MapBuffer
, Wa
):
1705 # Map file path is got.
1707 MapFilePath
= os
.path
.join(Wa
.BuildDir
, Wa
.Name
+ '.map')
1709 # Save address map into MAP file.
1711 SaveFileOnChange(MapFilePath
, ''.join(MapBuffer
), False)
1712 if self
.LoadFixAddress
!= 0:
1713 sys
.stdout
.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath
))
1716 ## Build active platform for different build targets and different tool chains
1718 def _BuildPlatform(self
):
1719 SaveFileOnChange(self
.PlatformBuildPath
, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1720 for BuildTarget
in self
.BuildTargetList
:
1721 GlobalData
.gGlobalDefines
['TARGET'] = BuildTarget
1723 for ToolChain
in self
.ToolChainList
:
1724 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = ToolChain
1725 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = ToolChain
1726 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[index
]
1728 Wa
= WorkspaceAutoGen(
1745 self
.Fdf
= Wa
.FdfFile
1746 self
.LoadFixAddress
= Wa
.Platform
.LoadFixAddress
1747 self
.BuildReport
.AddPlatformReport(Wa
)
1748 self
.Progress
.Stop("done!")
1750 # Add ffs build to makefile
1752 if GlobalData
.gEnableGenfdsMultiThread
and self
.Fdf
:
1753 CmdListDict
= self
._GenFfsCmd
(Wa
.ArchList
)
1755 for Arch
in Wa
.ArchList
:
1757 GlobalData
.gGlobalDefines
['ARCH'] = Arch
1758 Pa
= PlatformAutoGen(Wa
, self
.PlatformFile
, BuildTarget
, ToolChain
, Arch
)
1759 for Module
in Pa
.Platform
.Modules
:
1760 # Get ModuleAutoGen object to generate C code file and makefile
1761 Ma
= ModuleAutoGen(Wa
, Module
, BuildTarget
, ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
1765 Ma
.PlatformInfo
= Pa
1767 PcdMaList
.append(Ma
)
1768 self
.BuildModules
.append(Ma
)
1769 Pa
.DataPipe
.DataContainer
= {"FfsCommand":CmdListDict
}
1770 Pa
.DataPipe
.DataContainer
= {"Workspace_timestamp": Wa
._SrcTimeStamp
}
1771 self
._BuildPa
(self
.Target
, Pa
, FfsCommand
=CmdListDict
,PcdMaList
=PcdMaList
)
1773 # Create MAP file when Load Fix Address is enabled.
1774 if self
.Target
in ["", "all", "fds"]:
1775 for Arch
in Wa
.ArchList
:
1776 GlobalData
.gGlobalDefines
['ARCH'] = Arch
1778 # Check whether the set fix address is above 4G for 32bit image.
1780 if (Arch
== 'IA32' or Arch
== 'ARM') and self
.LoadFixAddress
!= 0xFFFFFFFFFFFFFFFF and self
.LoadFixAddress
>= 0x100000000:
1781 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1786 for Pa
in Wa
.AutoGenObjectList
:
1787 for Ma
in Pa
.ModuleAutoGenList
:
1790 if not Ma
.IsLibrary
:
1791 ModuleList
[Ma
.Guid
.upper()] = Ma
1794 if self
.LoadFixAddress
!= 0:
1796 # Rebase module to the preferred memory address before GenFds
1798 self
._CollectModuleMapBuffer
(MapBuffer
, ModuleList
)
1801 # create FDS again for the updated EFI image
1803 self
._Build
("fds", Wa
)
1805 # Create MAP file for all platform FVs after GenFds.
1807 self
._CollectFvMapBuffer
(MapBuffer
, Wa
, ModuleList
)
1809 # Save MAP buffer into MAP file.
1811 self
._SaveMapFile
(MapBuffer
, Wa
)
1812 self
.CreateGuidedSectionToolsFile(Wa
)
1814 ## Build active module for different build targets, different tool chains and different archs
1816 def _BuildModule(self
):
1817 for BuildTarget
in self
.BuildTargetList
:
1818 GlobalData
.gGlobalDefines
['TARGET'] = BuildTarget
1820 for ToolChain
in self
.ToolChainList
:
1821 WorkspaceAutoGenTime
= time
.time()
1822 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = ToolChain
1823 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = ToolChain
1824 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[index
]
1827 # module build needs platform build information, so get platform
1830 Wa
= WorkspaceAutoGen(
1848 self
.Fdf
= Wa
.FdfFile
1849 self
.LoadFixAddress
= Wa
.Platform
.LoadFixAddress
1850 Wa
.CreateMakeFile(False)
1851 # Add ffs build to makefile
1853 if GlobalData
.gEnableGenfdsMultiThread
and self
.Fdf
:
1854 CmdListDict
= self
._GenFfsCmd
(Wa
.ArchList
)
1856 GlobalData
.file_lock
= mp
.Lock()
1857 GlobalData
.FfsCmd
= CmdListDict
1859 self
.Progress
.Stop("done!")
1861 ExitFlag
= threading
.Event()
1863 self
.AutoGenTime
+= int(round((time
.time() - WorkspaceAutoGenTime
)))
1864 for Arch
in Wa
.ArchList
:
1865 AutoGenStart
= time
.time()
1866 GlobalData
.gGlobalDefines
['ARCH'] = Arch
1867 Pa
= PlatformAutoGen(Wa
, self
.PlatformFile
, BuildTarget
, ToolChain
, Arch
)
1868 for Module
in Pa
.Platform
.Modules
:
1869 if self
.ModuleFile
.Dir
== Module
.Dir
and self
.ModuleFile
.Name
== Module
.Name
:
1870 Ma
= ModuleAutoGen(Wa
, Module
, BuildTarget
, ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
1874 Ma
.PlatformInfo
= Pa
1878 if GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheDest
and self
.Target
in [None, "", "all"]:
1879 if Ma
.CanSkipbyPreMakeCache():
1882 self
.PreMakeCacheMiss
.add(Ma
)
1884 # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1885 if self
.Target
not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1886 # for target which must generate AutoGen code and makefile
1887 if not self
.SkipAutoGen
or self
.Target
== 'genc':
1888 self
.Progress
.Start("Generating code")
1889 Ma
.CreateCodeFile(True)
1890 self
.Progress
.Stop("done!")
1891 if self
.Target
== "genc":
1893 if not self
.SkipAutoGen
or self
.Target
== 'genmake':
1894 self
.Progress
.Start("Generating makefile")
1895 if CmdListDict
and self
.Fdf
and (Module
.Path
, Arch
) in CmdListDict
:
1896 Ma
.CreateMakeFile(True, CmdListDict
[Module
.Path
, Arch
])
1897 del CmdListDict
[Module
.Path
, Arch
]
1899 Ma
.CreateMakeFile(True)
1900 self
.Progress
.Stop("done!")
1901 if self
.Target
== "genmake":
1904 if GlobalData
.gBinCacheSource
and self
.Target
in [None, "", "all"]:
1905 if Ma
.CanSkipbyMakeCache():
1908 self
.MakeCacheMiss
.add(Ma
)
1910 self
.BuildModules
.append(Ma
)
1911 self
.AutoGenTime
+= int(round((time
.time() - AutoGenStart
)))
1912 MakeStart
= time
.time()
1913 for Ma
in self
.BuildModules
:
1914 if not Ma
.IsBinaryModule
:
1915 Bt
= BuildTask
.New(ModuleMakeUnit(Ma
, Pa
.BuildCommand
,self
.Target
))
1916 # Break build if any build thread has error
1917 if BuildTask
.HasError():
1918 # we need a full version of makefile for platform
1920 BuildTask
.WaitForComplete()
1921 Pa
.CreateMakeFile(False)
1922 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
1923 # Start task scheduler
1924 if not BuildTask
.IsOnGoing():
1925 BuildTask
.StartScheduler(self
.ThreadNumber
, ExitFlag
)
1927 # in case there's an interruption. we need a full version of makefile for platform
1928 Pa
.CreateMakeFile(False)
1929 if BuildTask
.HasError():
1930 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
1931 self
.MakeTime
+= int(round((time
.time() - MakeStart
)))
1933 MakeContiue
= time
.time()
1935 BuildTask
.WaitForComplete()
1936 self
.CreateAsBuiltInf()
1937 if GlobalData
.gBinCacheDest
:
1939 elif GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheSource
:
1941 # Update PreMakeCacheChain files
1942 self
.GenLocalPreMakeCache()
1943 self
.BuildModules
= []
1944 self
.MakeTime
+= int(round((time
.time() - MakeContiue
)))
1945 if BuildTask
.HasError():
1946 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
1948 self
.BuildReport
.AddPlatformReport(Wa
, MaList
)
1953 "Module for [%s] is not a component of active platform."\
1954 " Please make sure that the ARCH and inf file path are"\
1955 " given in the same as in [%s]" % \
1956 (', '.join(Wa
.ArchList
), self
.PlatformFile
),
1957 ExtraData
=self
.ModuleFile
1959 # Create MAP file when Load Fix Address is enabled.
1960 if self
.Target
== "fds" and self
.Fdf
:
1961 for Arch
in Wa
.ArchList
:
1963 # Check whether the set fix address is above 4G for 32bit image.
1965 if (Arch
== 'IA32' or Arch
== 'ARM') and self
.LoadFixAddress
!= 0xFFFFFFFFFFFFFFFF and self
.LoadFixAddress
>= 0x100000000:
1966 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1971 for Pa
in Wa
.AutoGenObjectList
:
1972 for Ma
in Pa
.ModuleAutoGenList
:
1975 if not Ma
.IsLibrary
:
1976 ModuleList
[Ma
.Guid
.upper()] = Ma
1979 if self
.LoadFixAddress
!= 0:
1981 # Rebase module to the preferred memory address before GenFds
1983 self
._CollectModuleMapBuffer
(MapBuffer
, ModuleList
)
1985 # create FDS again for the updated EFI image
1987 GenFdsStart
= time
.time()
1988 self
._Build
("fds", Wa
)
1989 self
.GenFdsTime
+= int(round((time
.time() - GenFdsStart
)))
1991 # Create MAP file for all platform FVs after GenFds.
1993 self
._CollectFvMapBuffer
(MapBuffer
, Wa
, ModuleList
)
1995 # Save MAP buffer into MAP file.
1997 self
._SaveMapFile
(MapBuffer
, Wa
)
1999 def _GenFfsCmd(self
,ArchList
):
2000 # convert dictionary of Cmd:(Inf,Arch)
2001 # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
2002 CmdSetDict
= defaultdict(set)
2003 GenFfsDict
= GenFds
.GenFfsMakefile('', GlobalData
.gFdfParser
, self
, ArchList
, GlobalData
)
2004 for Cmd
in GenFfsDict
:
2005 tmpInf
, tmpArch
= GenFfsDict
[Cmd
]
2006 CmdSetDict
[tmpInf
, tmpArch
].add(Cmd
)
2008 def VerifyAutoGenFiles(self
):
2009 AutoGenIdFile
= os
.path
.join(GlobalData
.gConfDirectory
,".AutoGenIdFile.txt")
2011 with
open(AutoGenIdFile
) as fd
:
2012 lines
= fd
.readlines()
2017 ArchList
= line
.strip().split("=")[1].split("|")
2018 if "BuildDir" in line
:
2019 BuildDir
= line
.split("=")[1].strip()
2020 if "PlatformGuid" in line
:
2021 PlatformGuid
= line
.split("=")[1].strip()
2023 for arch
in ArchList
:
2024 global_var
= os
.path
.join(BuildDir
, "GlobalVar_%s_%s.bin" % (str(PlatformGuid
),arch
))
2025 if not os
.path
.exists(global_var
):
2027 GlobalVarList
.append(global_var
)
2028 for global_var
in GlobalVarList
:
2029 data_pipe
= MemoryDataPipe()
2030 data_pipe
.load(global_var
)
2031 target
= data_pipe
.Get("P_Info").get("Target")
2032 toolchain
= data_pipe
.Get("P_Info").get("ToolChain")
2033 archlist
= data_pipe
.Get("P_Info").get("ArchList")
2034 Arch
= data_pipe
.Get("P_Info").get("Arch")
2035 active_p
= data_pipe
.Get("P_Info").get("ActivePlatform")
2036 workspacedir
= data_pipe
.Get("P_Info").get("WorkspaceDir")
2037 PackagesPath
= os
.getenv("PACKAGES_PATH")
2038 mws
.setWs(workspacedir
, PackagesPath
)
2039 LibraryBuildDirectoryList
= data_pipe
.Get("LibraryBuildDirectoryList")
2040 ModuleBuildDirectoryList
= data_pipe
.Get("ModuleBuildDirectoryList")
2042 for m_build_dir
in LibraryBuildDirectoryList
:
2043 if not os
.path
.exists(os
.path
.join(m_build_dir
,self
.MakeFileName
)):
2045 for m_build_dir
in ModuleBuildDirectoryList
:
2046 if not os
.path
.exists(os
.path
.join(m_build_dir
,self
.MakeFileName
)):
2049 workspacedir
,active_p
,target
,toolchain
,archlist
2051 Pa
= PlatformInfo(Wa
, active_p
, target
, toolchain
, Arch
,data_pipe
)
2052 Wa
.AutoGenObjectList
.append(Pa
)
2054 def SetupMakeSetting(self
,Wa
):
2056 for Pa
in Wa
.AutoGenObjectList
:
2057 for m
in Pa
._MbList
:
2058 ma
= ModuleAutoGen(Wa
,m
.MetaFile
, Pa
.BuildTarget
, Wa
.ToolChain
, Pa
.Arch
, Pa
.MetaFile
,Pa
.DataPipe
)
2059 BuildModules
.append(ma
)
2060 fdf_file
= Wa
.FlashDefinition
2062 Fdf
= FdfParser(fdf_file
.Path
)
2064 GlobalData
.gFdfParser
= Fdf
2065 if Fdf
.CurrentFdName
and Fdf
.CurrentFdName
in Fdf
.Profile
.FdDict
:
2066 FdDict
= Fdf
.Profile
.FdDict
[Fdf
.CurrentFdName
]
2067 for FdRegion
in FdDict
.RegionList
:
2068 if str(FdRegion
.RegionType
) == 'FILE' and self
.Platform
.VpdToolGuid
in str(FdRegion
.RegionDataList
):
2069 if int(FdRegion
.Offset
) % 8 != 0:
2070 EdkLogger
.error("build", FORMAT_INVALID
, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion
.Offset
))
2071 Wa
.FdfProfile
= Fdf
.Profile
2077 ## Build a platform in multi-thread mode
2079 def PerformAutoGen(self
,BuildTarget
,ToolChain
):
2080 WorkspaceAutoGenTime
= time
.time()
2081 Wa
= WorkspaceAutoGen(
2098 self
.Fdf
= Wa
.FdfFile
2099 self
.LoadFixAddress
= Wa
.Platform
.LoadFixAddress
2100 self
.BuildReport
.AddPlatformReport(Wa
)
2101 Wa
.CreateMakeFile(False)
2103 # Add ffs build to makefile
2105 if GlobalData
.gEnableGenfdsMultiThread
and self
.Fdf
:
2106 CmdListDict
= self
._GenFfsCmd
(Wa
.ArchList
)
2108 self
.AutoGenTime
+= int(round((time
.time() - WorkspaceAutoGenTime
)))
2110 for Arch
in Wa
.ArchList
:
2112 AutoGenStart
= time
.time()
2113 GlobalData
.gGlobalDefines
['ARCH'] = Arch
2114 Pa
= PlatformAutoGen(Wa
, self
.PlatformFile
, BuildTarget
, ToolChain
, Arch
)
2118 for Inf
in Pa
.Platform
.Modules
:
2119 ModuleList
.append(Inf
)
2120 # Add the INF only list in FDF
2121 if GlobalData
.gFdfParser
is not None:
2122 for InfName
in GlobalData
.gFdfParser
.Profile
.InfList
:
2123 Inf
= PathClass(NormPath(InfName
), self
.WorkspaceDir
, Arch
)
2124 if Inf
in Pa
.Platform
.Modules
:
2126 ModuleList
.append(Inf
)
2127 Pa
.DataPipe
.DataContainer
= {"FfsCommand":CmdListDict
}
2128 Pa
.DataPipe
.DataContainer
= {"Workspace_timestamp": Wa
._SrcTimeStamp
}
2129 Pa
.DataPipe
.DataContainer
= {"CommandTarget": self
.Target
}
2130 Pa
.CreateLibModuelDirs()
2131 # Fetch the MakeFileName.
2132 self
.MakeFileName
= Pa
.MakeFileName
2133 if not self
.MakeFileName
:
2134 self
.MakeFileName
= Pa
.MakeFile
2136 Pa
.DataPipe
.DataContainer
= {"LibraryBuildDirectoryList":Pa
.LibraryBuildDirectoryList
}
2137 Pa
.DataPipe
.DataContainer
= {"ModuleBuildDirectoryList":Pa
.ModuleBuildDirectoryList
}
2138 Pa
.DataPipe
.DataContainer
= {"FdsCommandDict": Wa
.GenFdsCommandDict
}
2139 # Prepare the cache share data for multiprocessing
2140 Pa
.DataPipe
.DataContainer
= {"gPlatformHashFile":GlobalData
.gPlatformHashFile
}
2142 for ma
in Pa
.ModuleAutoGenList
:
2143 ModuleCodaFile
[(ma
.MetaFile
.File
,ma
.MetaFile
.Root
,ma
.Arch
,ma
.MetaFile
.Path
)] = [item
.Target
for item
in ma
.CodaTargetList
]
2144 Pa
.DataPipe
.DataContainer
= {"ModuleCodaFile":ModuleCodaFile
}
2145 # ModuleList contains all driver modules only
2146 for Module
in ModuleList
:
2147 # Get ModuleAutoGen object to generate C code file and makefile
2148 Ma
= ModuleAutoGen(Wa
, Module
, BuildTarget
, ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
2152 Ma
.PlatformInfo
= Pa
2154 PcdMaList
.append(Ma
)
2155 self
.AllDrivers
.add(Ma
)
2156 self
.AllModules
.add(Ma
)
2160 for m
in Pa
.GetAllModuleInfo
:
2162 module_file
,module_root
,module_path
,module_basename
,\
2163 module_originalpath
,module_arch
,IsLib
= m
2164 Ma
= ModuleAutoGen(Wa
, PathClass(module_path
, Wa
), BuildTarget
,\
2165 ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
2166 self
.AllModules
.add(Ma
)
2167 data_pipe_file
= os
.path
.join(Pa
.BuildDir
, "GlobalVar_%s_%s.bin" % (str(Pa
.Guid
),Pa
.Arch
))
2168 Pa
.DataPipe
.dump(data_pipe_file
)
2170 autogen_rt
, errorcode
= self
.StartAutoGen(mqueue
, Pa
.DataPipe
, self
.SkipAutoGen
, PcdMaList
, cqueue
)
2173 self
.AutoGenMgr
.TerminateWorkers()
2174 self
.AutoGenMgr
.join(1)
2175 raise FatalError(errorcode
)
2177 if GlobalData
.gUseHashCache
:
2178 for item
in GlobalData
.gModuleAllCacheStatus
:
2179 (MetaFilePath
, Arch
, CacheStr
, Status
) = item
2180 Ma
= ModuleAutoGen(Wa
, PathClass(MetaFilePath
, Wa
), BuildTarget
,\
2181 ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
2182 if CacheStr
== "PreMakeCache" and Status
== False:
2183 self
.PreMakeCacheMiss
.add(Ma
)
2184 if CacheStr
== "PreMakeCache" and Status
== True:
2185 self
.PreMakeCacheHit
.add(Ma
)
2186 GlobalData
.gModuleCacheHit
.add(Ma
)
2187 if CacheStr
== "MakeCache" and Status
== False:
2188 self
.MakeCacheMiss
.add(Ma
)
2189 if CacheStr
== "MakeCache" and Status
== True:
2190 self
.MakeCacheHit
.add(Ma
)
2191 GlobalData
.gModuleCacheHit
.add(Ma
)
2192 self
.AutoGenTime
+= int(round((time
.time() - AutoGenStart
)))
2193 AutoGenIdFile
= os
.path
.join(GlobalData
.gConfDirectory
,".AutoGenIdFile.txt")
2194 with
open(AutoGenIdFile
,"w") as fw
:
2195 fw
.write("Arch=%s\n" % "|".join((Wa
.ArchList
)))
2196 fw
.write("BuildDir=%s\n" % Wa
.BuildDir
)
2197 fw
.write("PlatformGuid=%s\n" % str(Wa
.AutoGenObjectList
[0].Guid
))
2199 if GlobalData
.gBinCacheSource
:
2200 BuildModules
.extend(self
.MakeCacheMiss
)
2201 elif GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheDest
:
2202 BuildModules
.extend(self
.PreMakeCacheMiss
)
2204 BuildModules
.extend(self
.AllDrivers
)
2206 self
.Progress
.Stop("done!")
2207 return Wa
, BuildModules
2209 def _MultiThreadBuildPlatform(self
):
2210 SaveFileOnChange(self
.PlatformBuildPath
, '# DO NOT EDIT \n# FILE auto-generated\n', False)
2211 for BuildTarget
in self
.BuildTargetList
:
2212 GlobalData
.gGlobalDefines
['TARGET'] = BuildTarget
2214 for ToolChain
in self
.ToolChainList
:
2215 resetFdsGlobalVariable()
2216 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = ToolChain
2217 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = ToolChain
2218 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[index
]
2220 ExitFlag
= threading
.Event()
2222 if self
.SkipAutoGen
:
2223 Wa
= self
.VerifyAutoGenFiles()
2225 self
.SkipAutoGen
= False
2226 Wa
, self
.BuildModules
= self
.PerformAutoGen(BuildTarget
,ToolChain
)
2228 GlobalData
.gAutoGenPhase
= True
2229 self
.BuildModules
= self
.SetupMakeSetting(Wa
)
2231 Wa
, self
.BuildModules
= self
.PerformAutoGen(BuildTarget
,ToolChain
)
2232 Pa
= Wa
.AutoGenObjectList
[0]
2233 GlobalData
.gAutoGenPhase
= False
2235 if GlobalData
.gBinCacheSource
:
2236 EdkLogger
.quiet("[cache Summary]: Total module num: %s" % len(self
.AllModules
))
2237 EdkLogger
.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self
.PreMakeCacheMiss
))
2238 EdkLogger
.quiet("[cache Summary]: Makecache miss num: %s " % len(self
.MakeCacheMiss
))
2240 for Arch
in Wa
.ArchList
:
2241 MakeStart
= time
.time()
2242 for Ma
in set(self
.BuildModules
):
2243 # Generate build task for the module
2244 if not Ma
.IsBinaryModule
:
2245 Bt
= BuildTask
.New(ModuleMakeUnit(Ma
, Pa
.BuildCommand
,self
.Target
))
2246 # Break build if any build thread has error
2247 if BuildTask
.HasError():
2248 # we need a full version of makefile for platform
2250 BuildTask
.WaitForComplete()
2251 Pa
.CreateMakeFile(False)
2252 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
2253 # Start task scheduler
2254 if not BuildTask
.IsOnGoing():
2255 BuildTask
.StartScheduler(self
.ThreadNumber
, ExitFlag
)
2257 # in case there's an interruption. we need a full version of makefile for platform
2259 if BuildTask
.HasError():
2260 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
2261 self
.MakeTime
+= int(round((time
.time() - MakeStart
)))
2263 MakeContiue
= time
.time()
2266 # All modules have been put in build tasks queue. Tell task scheduler
2267 # to exit if all tasks are completed
2270 BuildTask
.WaitForComplete()
2271 self
.CreateAsBuiltInf()
2272 if GlobalData
.gBinCacheDest
:
2274 elif GlobalData
.gUseHashCache
and not GlobalData
.gBinCacheSource
:
2276 # Update PreMakeCacheChain files
2277 self
.GenLocalPreMakeCache()
2281 ModuleList
= {ma
.Guid
.upper(): ma
for ma
in self
.BuildModules
}
2282 self
.BuildModules
= []
2283 self
.MakeTime
+= int(round((time
.time() - MakeContiue
)))
2285 # Check for build error, and raise exception if one
2286 # has been signaled.
2288 if BuildTask
.HasError():
2289 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
2291 # Create MAP file when Load Fix Address is enabled.
2292 if self
.Target
in ["", "all", "fds"]:
2293 for Arch
in Wa
.ArchList
:
2295 # Check whether the set fix address is above 4G for 32bit image.
2297 if (Arch
== 'IA32' or Arch
== 'ARM') and self
.LoadFixAddress
!= 0xFFFFFFFFFFFFFFFF and self
.LoadFixAddress
>= 0x100000000:
2298 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2301 # Rebase module to the preferred memory address before GenFds
2304 if self
.LoadFixAddress
!= 0:
2305 self
._CollectModuleMapBuffer
(MapBuffer
, ModuleList
)
2309 # Generate FD image if there's a FDF file found
2311 GenFdsStart
= time
.time()
2312 if GenFdsApi(Wa
.GenFdsCommandDict
, self
.Db
):
2313 EdkLogger
.error("build", COMMAND_FAILURE
)
2316 # Create MAP file for all platform FVs after GenFds.
2318 self
._CollectFvMapBuffer
(MapBuffer
, Wa
, ModuleList
)
2319 self
.GenFdsTime
+= int(round((time
.time() - GenFdsStart
)))
2321 # Save MAP buffer into MAP file.
2323 self
._SaveMapFile
(MapBuffer
, Wa
)
2324 self
.CreateGuidedSectionToolsFile(Wa
)
2325 ## Generate GuidedSectionTools.txt in the FV directories.
2327 def CreateGuidedSectionToolsFile(self
,Wa
):
2328 for BuildTarget
in self
.BuildTargetList
:
2329 for ToolChain
in self
.ToolChainList
:
2331 if not os
.path
.exists(FvDir
):
2334 for Arch
in self
.ArchList
:
2335 # Build up the list of supported architectures for this build
2336 prefix
= '%s_%s_%s_' % (BuildTarget
, ToolChain
, Arch
)
2338 # Look through the tool definitions for GUIDed tools
2340 for (attrib
, value
) in self
.ToolDef
.ToolsDefTxtDictionary
.items():
2341 if attrib
.upper().endswith('_GUID'):
2342 split
= attrib
.split('_')
2343 thisPrefix
= '_'.join(split
[0:3]) + '_'
2344 if thisPrefix
== prefix
:
2345 guid
= self
.ToolDef
.ToolsDefTxtDictionary
[attrib
]
2348 path
= '_'.join(split
[0:4]) + '_PATH'
2349 path
= self
.ToolDef
.ToolsDefTxtDictionary
[path
]
2350 path
= self
.GetRealPathOfTool(path
)
2351 guidAttribs
.append((guid
, toolName
, path
))
2353 # Write out GuidedSecTools.txt
2354 toolsFile
= os
.path
.join(FvDir
, 'GuidedSectionTools.txt')
2355 toolsFile
= open(toolsFile
, 'wt')
2356 for guidedSectionTool
in guidAttribs
:
2357 print(' '.join(guidedSectionTool
), file=toolsFile
)
2360 ## Returns the real path of the tool.
2362 def GetRealPathOfTool (self
, tool
):
2363 if os
.path
.exists(tool
):
2364 return os
.path
.realpath(tool
)
2367 ## Launch the module or platform build
2370 self
.AllDrivers
= set()
2371 self
.AllModules
= set()
2372 self
.PreMakeCacheMiss
= set()
2373 self
.PreMakeCacheHit
= set()
2374 self
.MakeCacheMiss
= set()
2375 self
.MakeCacheHit
= set()
2376 if not self
.ModuleFile
:
2377 if not self
.SpawnMode
or self
.Target
not in ["", "all"]:
2378 self
.SpawnMode
= False
2379 self
._BuildPlatform
()
2381 self
._MultiThreadBuildPlatform
()
2383 self
.SpawnMode
= False
2386 if self
.Target
== 'cleanall':
2387 RemoveDirectory(os
.path
.dirname(GlobalData
.gDatabasePath
), True)
2389 def CreateAsBuiltInf(self
):
2390 for Module
in self
.BuildModules
:
2391 Module
.CreateAsBuiltInf()
2393 def GenDestCache(self
):
2394 for Module
in self
.AllModules
:
2395 Module
.GenPreMakefileHashList()
2396 Module
.GenMakefileHashList()
2397 Module
.CopyModuleToCache()
2399 def GenLocalPreMakeCache(self
):
2400 for Module
in self
.PreMakeCacheMiss
:
2401 Module
.GenPreMakefileHashList()
2403 ## Do some clean-up works when error occurred
2404 def Relinquish(self
):
2405 OldLogLevel
= EdkLogger
.GetLevel()
2406 EdkLogger
.SetLevel(EdkLogger
.ERROR
)
2407 Utils
.Progressor
.Abort()
2408 if self
.SpawnMode
== True:
2410 EdkLogger
.SetLevel(OldLogLevel
)
2412 def ParseDefines(DefineList
=[]):
2414 if DefineList
is not None:
2415 for Define
in DefineList
:
2416 DefineTokenList
= Define
.split("=", 1)
2417 if not GlobalData
.gMacroNamePattern
.match(DefineTokenList
[0]):
2418 EdkLogger
.error('build', FORMAT_INVALID
,
2419 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2420 ExtraData
=DefineTokenList
[0])
2422 if len(DefineTokenList
) == 1:
2423 DefineDict
[DefineTokenList
[0]] = "TRUE"
2425 DefineDict
[DefineTokenList
[0]] = DefineTokenList
[1].strip()
2430 def LogBuildTime(Time
):
2433 TimeDur
= time
.gmtime(Time
)
2434 if TimeDur
.tm_yday
> 1:
2435 TimeDurStr
= time
.strftime("%H:%M:%S", TimeDur
) + ", %d day(s)" % (TimeDur
.tm_yday
- 1)
2437 TimeDurStr
= time
.strftime("%H:%M:%S", TimeDur
)
2442 OptionParser
= MyOptionParser()
2443 if not OptionParser
.BuildOption
and not OptionParser
.BuildTarget
:
2444 OptionParser
.GetOption()
2445 BuildOption
, BuildTarget
= OptionParser
.BuildOption
, OptionParser
.BuildTarget
2446 ThreadNumber
= BuildOption
.ThreadNumber
2447 GlobalData
.gCmdConfDir
= BuildOption
.ConfDirectory
2448 if ThreadNumber
is None:
2449 TargetObj
= TargetTxtDict()
2450 ThreadNumber
= TargetObj
.Target
.TargetTxtDictionary
[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER
]
2451 if ThreadNumber
== '':
2454 ThreadNumber
= int(ThreadNumber
, 0)
2456 if ThreadNumber
== 0:
2458 ThreadNumber
= multiprocessing
.cpu_count()
2459 except (ImportError, NotImplementedError):
2462 ## Tool entrance method
2464 # This method mainly dispatch specific methods per the command line options.
2465 # If no error found, return zero value so the caller of this tool can know
2466 # if it's executed successfully or not.
2468 # @retval 0 Tool was successful
2469 # @retval 1 Tool failed
2471 LogQMaxSize
= ThreadNum() * 10
2473 StartTime
= time
.time()
2476 # Create a log Queue
2478 LogQ
= mp
.Queue(LogQMaxSize
)
2479 # Initialize log system
2480 EdkLogger
.LogClientInitialize(LogQ
)
2481 GlobalData
.gCommand
= sys
.argv
[1:]
2483 # Parse the options and args
2485 OptionParser
= MyOptionParser()
2486 if not OptionParser
.BuildOption
and not OptionParser
.BuildTarget
:
2487 OptionParser
.GetOption()
2488 Option
, Target
= OptionParser
.BuildOption
, OptionParser
.BuildTarget
2489 GlobalData
.gOptions
= Option
2490 GlobalData
.gCaseInsensitive
= Option
.CaseInsensitive
2493 LogLevel
= EdkLogger
.INFO
2494 if Option
.verbose
is not None:
2495 EdkLogger
.SetLevel(EdkLogger
.VERBOSE
)
2496 LogLevel
= EdkLogger
.VERBOSE
2497 elif Option
.quiet
is not None:
2498 EdkLogger
.SetLevel(EdkLogger
.QUIET
)
2499 LogLevel
= EdkLogger
.QUIET
2500 elif Option
.debug
is not None:
2501 EdkLogger
.SetLevel(Option
.debug
+ 1)
2502 LogLevel
= Option
.debug
+ 1
2504 EdkLogger
.SetLevel(EdkLogger
.INFO
)
2506 if Option
.WarningAsError
== True:
2507 EdkLogger
.SetWarningAsError()
2508 Log_Agent
= LogAgent(LogQ
,LogLevel
,Option
.LogFile
)
2511 if platform
.platform().find("Windows") >= 0:
2512 GlobalData
.gIsWindows
= True
2514 GlobalData
.gIsWindows
= False
2516 EdkLogger
.quiet("Build environment: %s" % platform
.platform())
2517 EdkLogger
.quiet(time
.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time
.localtime()));
2522 if len(Target
) == 0:
2524 elif len(Target
) >= 2:
2525 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, "More than one targets are not supported.",
2526 ExtraData
="Please select one of: %s" % (' '.join(gSupportedTarget
)))
2528 Target
= Target
[0].lower()
2530 if Target
not in gSupportedTarget
:
2531 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, "Not supported target [%s]." % Target
,
2532 ExtraData
="Please select one of: %s" % (' '.join(gSupportedTarget
)))
2535 # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2538 GlobalData
.gCommandLineDefines
.update(ParseDefines(Option
.Macros
))
2540 Workspace
= os
.getenv("WORKSPACE")
2542 # Get files real name in workspace dir
2544 GlobalData
.gAllFiles
= Utils
.DirCache(Workspace
)
2546 WorkingDirectory
= os
.getcwd()
2547 if not Option
.ModuleFile
:
2548 FileList
= glob
.glob(os
.path
.normpath(os
.path
.join(WorkingDirectory
, '*.inf')))
2549 FileNum
= len(FileList
)
2551 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, "There are %d INF files in %s." % (FileNum
, WorkingDirectory
),
2552 ExtraData
="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2554 Option
.ModuleFile
= NormFile(FileList
[0], Workspace
)
2556 if Option
.ModuleFile
:
2557 if os
.path
.isabs (Option
.ModuleFile
):
2558 if os
.path
.normcase (os
.path
.normpath(Option
.ModuleFile
)).find (Workspace
) == 0:
2559 Option
.ModuleFile
= NormFile(os
.path
.normpath(Option
.ModuleFile
), Workspace
)
2560 Option
.ModuleFile
= PathClass(Option
.ModuleFile
, Workspace
)
2561 ErrorCode
, ErrorInfo
= Option
.ModuleFile
.Validate(".inf", False)
2563 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
2565 if Option
.PlatformFile
is not None:
2566 if os
.path
.isabs (Option
.PlatformFile
):
2567 if os
.path
.normcase (os
.path
.normpath(Option
.PlatformFile
)).find (Workspace
) == 0:
2568 Option
.PlatformFile
= NormFile(os
.path
.normpath(Option
.PlatformFile
), Workspace
)
2569 Option
.PlatformFile
= PathClass(Option
.PlatformFile
, Workspace
)
2571 if Option
.FdfFile
is not None:
2572 if os
.path
.isabs (Option
.FdfFile
):
2573 if os
.path
.normcase (os
.path
.normpath(Option
.FdfFile
)).find (Workspace
) == 0:
2574 Option
.FdfFile
= NormFile(os
.path
.normpath(Option
.FdfFile
), Workspace
)
2575 Option
.FdfFile
= PathClass(Option
.FdfFile
, Workspace
)
2576 ErrorCode
, ErrorInfo
= Option
.FdfFile
.Validate(".fdf", False)
2578 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
2580 if Option
.Flag
is not None and Option
.Flag
not in ['-c', '-s']:
2581 EdkLogger
.error("build", OPTION_VALUE_INVALID
, "UNI flag must be one of -c or -s")
2583 MyBuild
= Build(Target
, Workspace
, Option
,LogQ
)
2584 GlobalData
.gCommandLineDefines
['ARCH'] = ' '.join(MyBuild
.ArchList
)
2585 if not (MyBuild
.LaunchPrebuildFlag
and os
.path
.exists(MyBuild
.PlatformBuildPath
)):
2589 # All job done, no error found and no exception raised
2592 except FatalError
as X
:
2593 if MyBuild
is not None:
2594 # for multi-thread build exits safely
2595 MyBuild
.Relinquish()
2596 if Option
is not None and Option
.debug
is not None:
2597 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2598 ReturnCode
= X
.args
[0]
2599 except Warning as X
:
2600 # error from Fdf parser
2601 if MyBuild
is not None:
2602 # for multi-thread build exits safely
2603 MyBuild
.Relinquish()
2604 if Option
is not None and Option
.debug
is not None:
2605 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2607 EdkLogger
.error(X
.ToolName
, FORMAT_INVALID
, File
=X
.FileName
, Line
=X
.LineNumber
, ExtraData
=X
.Message
, RaiseError
=False)
2608 ReturnCode
= FORMAT_INVALID
2609 except KeyboardInterrupt:
2610 if MyBuild
is not None:
2612 # for multi-thread build exits safely
2613 MyBuild
.Relinquish()
2614 ReturnCode
= ABORT_ERROR
2615 if Option
is not None and Option
.debug
is not None:
2616 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2618 if MyBuild
is not None:
2619 # for multi-thread build exits safely
2620 MyBuild
.Relinquish()
2622 # try to get the meta-file from the object causing exception
2623 Tb
= sys
.exc_info()[-1]
2624 MetaFile
= GlobalData
.gProcessingFile
2625 while Tb
is not None:
2626 if 'self' in Tb
.tb_frame
.f_locals
and hasattr(Tb
.tb_frame
.f_locals
['self'], 'MetaFile'):
2627 MetaFile
= Tb
.tb_frame
.f_locals
['self'].MetaFile
2632 "Unknown fatal error when processing [%s]" % MetaFile
,
2633 ExtraData
="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR
,
2636 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2637 ReturnCode
= CODE_ERROR
2639 Utils
.Progressor
.Abort()
2640 Utils
.ClearDuplicatedInf()
2644 MyBuild
.LaunchPostbuild()
2647 Conclusion
= "Failed"
2648 elif ReturnCode
== ABORT_ERROR
:
2649 Conclusion
= "Aborted"
2651 Conclusion
= "Failed"
2652 FinishTime
= time
.time()
2653 BuildDuration
= time
.gmtime(int(round(FinishTime
- StartTime
)))
2654 BuildDurationStr
= ""
2655 if BuildDuration
.tm_yday
> 1:
2656 BuildDurationStr
= time
.strftime("%H:%M:%S", BuildDuration
) + ", %d day(s)" % (BuildDuration
.tm_yday
- 1)
2658 BuildDurationStr
= time
.strftime("%H:%M:%S", BuildDuration
)
2659 if MyBuild
is not None:
2661 MyBuild
.BuildReport
.GenerateReport(BuildDurationStr
, LogBuildTime(MyBuild
.AutoGenTime
), LogBuildTime(MyBuild
.MakeTime
), LogBuildTime(MyBuild
.GenFdsTime
))
2663 EdkLogger
.SetLevel(EdkLogger
.QUIET
)
2664 EdkLogger
.quiet("\n- %s -" % Conclusion
)
2665 EdkLogger
.quiet(time
.strftime("Build end time: %H:%M:%S, %b.%d %Y", time
.localtime()))
2666 EdkLogger
.quiet("Build total time: %s\n" % BuildDurationStr
)
2671 if __name__
== '__main__':
2673 mp
.set_start_method('spawn')
2677 ## 0-127 is a safe return range, and 1 is a standard default error
2678 if r
< 0 or r
> 127: r
= 1