2 # build a platform or a module
4 # Copyright (c) 2014, Hewlett-Packard Development Company, L.P.<BR>
5 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
6 # Copyright (c) 2018, Hewlett Packard Enterprise Development, L.P.<BR>
8 # SPDX-License-Identifier: BSD-2-Clause-Patent
14 from __future__
import print_function
15 from __future__
import absolute_import
16 import os
.path
as path
24 import multiprocessing
25 from threading
import Thread
,Event
,BoundedSemaphore
27 from subprocess
import Popen
,PIPE
28 from collections
import OrderedDict
, defaultdict
29 from Common
.buildoptions
import BuildOption
,BuildTarget
30 from AutoGen
.PlatformAutoGen
import PlatformAutoGen
31 from AutoGen
.ModuleAutoGen
import ModuleAutoGen
32 from AutoGen
.WorkspaceAutoGen
import WorkspaceAutoGen
33 from AutoGen
.AutoGenWorker
import AutoGenWorkerInProcess
,AutoGenManager
,\
35 from AutoGen
import GenMake
36 from Common
import Misc
as Utils
38 from Common
.TargetTxtClassObject
import TargetTxt
39 from Common
.ToolDefClassObject
import ToolDef
40 from Common
.Misc
import PathClass
,SaveFileOnChange
,RemoveDirectory
41 from Common
.StringUtils
import NormPath
42 from Common
.MultipleWorkspace
import MultipleWorkspace
as mws
43 from Common
.BuildToolError
import *
44 from Common
.DataType
import *
45 import Common
.EdkLogger
as EdkLogger
47 from Workspace
.WorkspaceDatabase
import BuildDB
49 from BuildReport
import BuildReport
50 from GenPatchPcdTable
.GenPatchPcdTable
import PeImageClass
,parsePcdInfoFromMapFile
51 from PatchPcdValue
.PatchPcdValue
import PatchBinaryFile
53 import Common
.GlobalData
as GlobalData
54 from GenFds
.GenFds
import GenFds
, GenFdsApi
55 import multiprocessing
as mp
56 from multiprocessing
import Manager
59 ## standard targets of build command
60 gSupportedTarget
= ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
62 ## build configuration file
63 gBuildConfiguration
= "target.txt"
64 gToolsDefinition
= "tools_def.txt"
66 TemporaryTablePattern
= re
.compile(r
'^_\d+_\d+_[a-fA-F0-9]+$')
69 ## Check environment PATH variable to make sure the specified tool is found
71 # If the tool is found in the PATH, then True is returned
72 # Otherwise, False is returned
74 def IsToolInPath(tool
):
75 if 'PATHEXT' in os
.environ
:
76 extns
= os
.environ
['PATHEXT'].split(os
.path
.pathsep
)
79 for pathDir
in os
.environ
['PATH'].split(os
.path
.pathsep
):
81 if os
.path
.exists(os
.path
.join(pathDir
, tool
+ ext
)):
85 ## Check environment variables
87 # Check environment variables that must be set for build. Currently they are
89 # WORKSPACE The directory all packages/platforms start from
90 # EDK_TOOLS_PATH The directory contains all tools needed by the build
91 # PATH $(EDK_TOOLS_PATH)/Bin/<sys> must be set in PATH
93 # If any of above environment variable is not set or has error, the build
96 def CheckEnvVariable():
98 if "WORKSPACE" not in os
.environ
:
99 EdkLogger
.error("build", ATTRIBUTE_NOT_AVAILABLE
, "Environment variable not found",
100 ExtraData
="WORKSPACE")
102 WorkspaceDir
= os
.path
.normcase(os
.path
.normpath(os
.environ
["WORKSPACE"]))
103 if not os
.path
.exists(WorkspaceDir
):
104 EdkLogger
.error("build", FILE_NOT_FOUND
, "WORKSPACE doesn't exist", ExtraData
=WorkspaceDir
)
105 elif ' ' in WorkspaceDir
:
106 EdkLogger
.error("build", FORMAT_NOT_SUPPORTED
, "No space is allowed in WORKSPACE path",
107 ExtraData
=WorkspaceDir
)
108 os
.environ
["WORKSPACE"] = WorkspaceDir
110 # set multiple workspace
111 PackagesPath
= os
.getenv("PACKAGES_PATH")
112 mws
.setWs(WorkspaceDir
, PackagesPath
)
113 if mws
.PACKAGES_PATH
:
114 for Path
in mws
.PACKAGES_PATH
:
115 if not os
.path
.exists(Path
):
116 EdkLogger
.error("build", FILE_NOT_FOUND
, "One Path in PACKAGES_PATH doesn't exist", ExtraData
=Path
)
118 EdkLogger
.error("build", FORMAT_NOT_SUPPORTED
, "No space is allowed in PACKAGES_PATH", ExtraData
=Path
)
121 os
.environ
["EDK_TOOLS_PATH"] = os
.path
.normcase(os
.environ
["EDK_TOOLS_PATH"])
123 # check EDK_TOOLS_PATH
124 if "EDK_TOOLS_PATH" not in os
.environ
:
125 EdkLogger
.error("build", ATTRIBUTE_NOT_AVAILABLE
, "Environment variable not found",
126 ExtraData
="EDK_TOOLS_PATH")
129 if "PATH" not in os
.environ
:
130 EdkLogger
.error("build", ATTRIBUTE_NOT_AVAILABLE
, "Environment variable not found",
133 GlobalData
.gWorkspace
= WorkspaceDir
135 GlobalData
.gGlobalDefines
["WORKSPACE"] = WorkspaceDir
136 GlobalData
.gGlobalDefines
["EDK_TOOLS_PATH"] = os
.environ
["EDK_TOOLS_PATH"]
138 ## Get normalized file path
140 # Convert the path to be local format, and remove the WORKSPACE path at the
141 # beginning if the file path is given in full path.
143 # @param FilePath File path to be normalized
144 # @param Workspace Workspace path which the FilePath will be checked against
146 # @retval string The normalized file path
148 def NormFile(FilePath
, Workspace
):
149 # check if the path is absolute or relative
150 if os
.path
.isabs(FilePath
):
151 FileFullPath
= os
.path
.normpath(FilePath
)
153 FileFullPath
= os
.path
.normpath(mws
.join(Workspace
, FilePath
))
154 Workspace
= mws
.getWs(Workspace
, FilePath
)
156 # check if the file path exists or not
157 if not os
.path
.isfile(FileFullPath
):
158 EdkLogger
.error("build", FILE_NOT_FOUND
, ExtraData
="\t%s (Please give file in absolute path or relative to WORKSPACE)" % FileFullPath
)
160 # remove workspace directory from the beginning part of the file path
161 if Workspace
[-1] in ["\\", "/"]:
162 return FileFullPath
[len(Workspace
):]
164 return FileFullPath
[(len(Workspace
) + 1):]
166 ## Get the output of an external program
168 # This is the entrance method of thread reading output of an external program and
169 # putting them in STDOUT/STDERR of current program.
171 # @param From The stream message read from
172 # @param To The stream message put on
173 # @param ExitFlag The flag used to indicate stopping reading
175 def ReadMessage(From
, To
, ExitFlag
):
177 # read one line a time
178 Line
= From
.readline()
179 # empty string means "end"
180 if Line
is not None and Line
!= b
"":
181 To(Line
.rstrip().decode(encoding
='utf-8', errors
='ignore'))
187 ## Launch an external program
189 # This method will call subprocess.Popen to execute an external program with
190 # given options in specified directory. Because of the dead-lock issue during
191 # redirecting output of the external program, threads are used to to do the
194 # @param Command A list or string containing the call of the program
195 # @param WorkingDir The directory in which the program will be running
197 def LaunchCommand(Command
, WorkingDir
):
198 BeginTime
= time
.time()
199 # if working directory doesn't exist, Popen() will raise an exception
200 if not os
.path
.isdir(WorkingDir
):
201 EdkLogger
.error("build", FILE_NOT_FOUND
, ExtraData
=WorkingDir
)
203 # Command is used as the first Argument in following Popen().
204 # It could be a string or sequence. We find that if command is a string in following Popen(),
205 # ubuntu may fail with an error message that the command is not found.
206 # So here we may need convert command from string to list instance.
207 if platform
.system() != 'Windows':
208 if not isinstance(Command
, list):
209 Command
= Command
.split()
210 Command
= ' '.join(Command
)
213 EndOfProcedure
= None
216 Proc
= Popen(Command
, stdout
=PIPE
, stderr
=PIPE
, env
=os
.environ
, cwd
=WorkingDir
, bufsize
=-1, shell
=True)
218 # launch two threads to read the STDOUT and STDERR
219 EndOfProcedure
= Event()
220 EndOfProcedure
.clear()
222 StdOutThread
= Thread(target
=ReadMessage
, args
=(Proc
.stdout
, EdkLogger
.info
, EndOfProcedure
))
223 StdOutThread
.setName("STDOUT-Redirector")
224 StdOutThread
.setDaemon(False)
228 StdErrThread
= Thread(target
=ReadMessage
, args
=(Proc
.stderr
, EdkLogger
.quiet
, EndOfProcedure
))
229 StdErrThread
.setName("STDERR-Redirector")
230 StdErrThread
.setDaemon(False)
233 # waiting for program exit
235 except: # in case of aborting
236 # terminate the threads redirecting the program output
237 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
238 if EndOfProcedure
is not None:
241 if not isinstance(Command
, type("")):
242 Command
= " ".join(Command
)
243 EdkLogger
.error("build", COMMAND_FAILURE
, "Failed to start command", ExtraData
="%s [%s]" % (Command
, WorkingDir
))
250 # check the return code of the program
251 if Proc
.returncode
!= 0:
252 if not isinstance(Command
, type("")):
253 Command
= " ".join(Command
)
254 # print out the Response file and its content when make failure
255 RespFile
= os
.path
.join(WorkingDir
, 'OUTPUT', 'respfilelist.txt')
256 if os
.path
.isfile(RespFile
):
258 RespContent
= f
.read()
260 EdkLogger
.info(RespContent
)
262 EdkLogger
.error("build", COMMAND_FAILURE
, ExtraData
="%s [%s]" % (Command
, WorkingDir
))
263 return "%dms" % (int(round((time
.time() - BeginTime
) * 1000)))
265 ## The smallest unit that can be built in multi-thread build mode
267 # This is the base class of build unit. The "Obj" parameter must provide
268 # __str__(), __eq__() and __hash__() methods. Otherwise there could be build units
271 # Currently the "Obj" should be only ModuleAutoGen or PlatformAutoGen objects.
276 # @param self The object pointer
277 # @param Obj The object the build is working on
278 # @param Target The build target name, one of gSupportedTarget
279 # @param Dependency The BuildUnit(s) which must be completed in advance
280 # @param WorkingDir The directory build command starts in
282 def __init__(self
, Obj
, BuildCommand
, Target
, Dependency
, WorkingDir
="."):
283 self
.BuildObject
= Obj
284 self
.Dependency
= Dependency
285 self
.WorkingDir
= WorkingDir
287 self
.BuildCommand
= BuildCommand
289 EdkLogger
.error("build", OPTION_MISSING
,
290 "No build command found for this module. "
291 "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
292 (Obj
.BuildTarget
, Obj
.ToolChain
, Obj
.Arch
),
298 # It just returns the string representation of self.BuildObject
300 # @param self The object pointer
303 return str(self
.BuildObject
)
305 ## "==" operator method
307 # It just compares self.BuildObject with "Other". So self.BuildObject must
308 # provide its own __eq__() method.
310 # @param self The object pointer
311 # @param Other The other BuildUnit object compared to
313 def __eq__(self
, Other
):
314 return Other
and self
.BuildObject
== Other
.BuildObject \
315 and Other
.BuildObject \
316 and self
.BuildObject
.Arch
== Other
.BuildObject
.Arch
320 # It just returns the hash value of self.BuildObject which must be hashable.
322 # @param self The object pointer
325 return hash(self
.BuildObject
) + hash(self
.BuildObject
.Arch
)
328 return repr(self
.BuildObject
)
330 ## The smallest module unit that can be built by nmake/make command in multi-thread build mode
332 # This class is for module build by nmake/make build system. The "Obj" parameter
333 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
334 # be make units missing build.
336 # Currently the "Obj" should be only ModuleAutoGen object.
338 class ModuleMakeUnit(BuildUnit
):
341 # @param self The object pointer
342 # @param Obj The ModuleAutoGen object the build is working on
343 # @param Target The build target name, one of gSupportedTarget
345 def __init__(self
, Obj
, BuildCommand
,Target
):
346 Dependency
= [ModuleMakeUnit(La
, BuildCommand
,Target
) for La
in Obj
.LibraryAutoGenList
]
347 BuildUnit
.__init
__(self
, Obj
, BuildCommand
, Target
, Dependency
, Obj
.MakeFileDir
)
348 if Target
in [None, "", "all"]:
349 self
.Target
= "tbuild"
351 ## The smallest platform unit that can be built by nmake/make command in multi-thread build mode
353 # This class is for platform build by nmake/make build system. The "Obj" parameter
354 # must provide __str__(), __eq__() and __hash__() methods. Otherwise there could
355 # be make units missing build.
357 # Currently the "Obj" should be only PlatformAutoGen object.
359 class PlatformMakeUnit(BuildUnit
):
362 # @param self The object pointer
363 # @param Obj The PlatformAutoGen object the build is working on
364 # @param Target The build target name, one of gSupportedTarget
366 def __init__(self
, Obj
, BuildCommand
, Target
):
367 Dependency
= [ModuleMakeUnit(Lib
, BuildCommand
, Target
) for Lib
in self
.BuildObject
.LibraryAutoGenList
]
368 Dependency
.extend([ModuleMakeUnit(Mod
, BuildCommand
,Target
) for Mod
in self
.BuildObject
.ModuleAutoGenList
])
369 BuildUnit
.__init
__(self
, Obj
, BuildCommand
, Target
, Dependency
, Obj
.MakeFileDir
)
371 ## The class representing the task of a module build or platform build
373 # This class manages the build tasks in multi-thread build mode. Its jobs include
374 # scheduling thread running, catching thread error, monitor the thread status, etc.
377 # queue for tasks waiting for schedule
378 _PendingQueue
= OrderedDict()
379 _PendingQueueLock
= threading
.Lock()
381 # queue for tasks ready for running
382 _ReadyQueue
= OrderedDict()
383 _ReadyQueueLock
= threading
.Lock()
385 # queue for run tasks
386 _RunningQueue
= OrderedDict()
387 _RunningQueueLock
= threading
.Lock()
389 # queue containing all build tasks, in case duplicate build
390 _TaskQueue
= OrderedDict()
392 # flag indicating error occurs in a running thread
393 _ErrorFlag
= threading
.Event()
397 # BoundedSemaphore object used to control the number of running threads
400 # flag indicating if the scheduler is started or not
401 _SchedulerStopped
= threading
.Event()
402 _SchedulerStopped
.set()
404 ## Start the task scheduler thread
406 # @param MaxThreadNumber The maximum thread number
407 # @param ExitFlag Flag used to end the scheduler
410 def StartScheduler(MaxThreadNumber
, ExitFlag
):
411 SchedulerThread
= Thread(target
=BuildTask
.Scheduler
, args
=(MaxThreadNumber
, ExitFlag
))
412 SchedulerThread
.setName("Build-Task-Scheduler")
413 SchedulerThread
.setDaemon(False)
414 SchedulerThread
.start()
415 # wait for the scheduler to be started, especially useful in Linux
416 while not BuildTask
.IsOnGoing():
421 # @param MaxThreadNumber The maximum thread number
422 # @param ExitFlag Flag used to end the scheduler
425 def Scheduler(MaxThreadNumber
, ExitFlag
):
426 BuildTask
._SchedulerStopped
.clear()
428 # use BoundedSemaphore to control the maximum running threads
429 BuildTask
._Thread
= BoundedSemaphore(MaxThreadNumber
)
431 # scheduling loop, which will exits when no pending/ready task and
432 # indicated to do so, or there's error in running thread
434 while (len(BuildTask
._PendingQueue
) > 0 or len(BuildTask
._ReadyQueue
) > 0 \
435 or not ExitFlag
.isSet()) and not BuildTask
._ErrorFlag
.isSet():
436 EdkLogger
.debug(EdkLogger
.DEBUG_8
, "Pending Queue (%d), Ready Queue (%d)"
437 % (len(BuildTask
._PendingQueue
), len(BuildTask
._ReadyQueue
)))
439 # get all pending tasks
440 BuildTask
._PendingQueueLock
.acquire()
441 BuildObjectList
= list(BuildTask
._PendingQueue
.keys())
443 # check if their dependency is resolved, and if true, move them
446 for BuildObject
in BuildObjectList
:
447 Bt
= BuildTask
._PendingQueue
[BuildObject
]
449 BuildTask
._ReadyQueue
[BuildObject
] = BuildTask
._PendingQueue
.pop(BuildObject
)
450 BuildTask
._PendingQueueLock
.release()
452 # launch build thread until the maximum number of threads is reached
453 while not BuildTask
._ErrorFlag
.isSet():
454 # empty ready queue, do nothing further
455 if len(BuildTask
._ReadyQueue
) == 0:
458 # wait for active thread(s) exit
459 BuildTask
._Thread
.acquire(True)
461 # start a new build thread
462 Bo
, Bt
= BuildTask
._ReadyQueue
.popitem()
464 # move into running queue
465 BuildTask
._RunningQueueLock
.acquire()
466 BuildTask
._RunningQueue
[Bo
] = Bt
467 BuildTask
._RunningQueueLock
.release()
476 # wait for all running threads exit
477 if BuildTask
._ErrorFlag
.isSet():
478 EdkLogger
.quiet("\nWaiting for all build threads exit...")
479 # while not BuildTask._ErrorFlag.isSet() and \
480 while len(BuildTask
._RunningQueue
) > 0:
481 EdkLogger
.verbose("Waiting for thread ending...(%d)" % len(BuildTask
._RunningQueue
))
482 EdkLogger
.debug(EdkLogger
.DEBUG_8
, "Threads [%s]" % ", ".join(Th
.getName() for Th
in threading
.enumerate()))
485 except BaseException
as X
:
487 # TRICK: hide the output of threads left running, so that the user can
488 # catch the error message easily
490 EdkLogger
.SetLevel(EdkLogger
.ERROR
)
491 BuildTask
._ErrorFlag
.set()
492 BuildTask
._ErrorMessage
= "build thread scheduler error\n\t%s" % str(X
)
494 BuildTask
._PendingQueue
.clear()
495 BuildTask
._ReadyQueue
.clear()
496 BuildTask
._RunningQueue
.clear()
497 BuildTask
._TaskQueue
.clear()
498 BuildTask
._SchedulerStopped
.set()
500 ## Wait for all running method exit
503 def WaitForComplete():
504 BuildTask
._SchedulerStopped
.wait()
506 ## Check if the scheduler is running or not
510 return not BuildTask
._SchedulerStopped
.isSet()
515 if BuildTask
.IsOnGoing():
516 BuildTask
._ErrorFlag
.set()
517 BuildTask
.WaitForComplete()
519 ## Check if there's error in running thread
521 # Since the main thread cannot catch exceptions in other thread, we have to
522 # use threading.Event to communicate this formation to main thread.
526 return BuildTask
._ErrorFlag
.isSet()
528 ## Get error message in running thread
530 # Since the main thread cannot catch exceptions in other thread, we have to
531 # use a static variable to communicate this message to main thread.
534 def GetErrorMessage():
535 return BuildTask
._ErrorMessage
537 ## Factory method to create a BuildTask object
539 # This method will check if a module is building or has been built. And if
540 # true, just return the associated BuildTask object in the _TaskQueue. If
541 # not, create and return a new BuildTask object. The new BuildTask object
542 # will be appended to the _PendingQueue for scheduling later.
544 # @param BuildItem A BuildUnit object representing a build object
545 # @param Dependency The dependent build object of BuildItem
548 def New(BuildItem
, Dependency
=None):
549 if BuildItem
in BuildTask
._TaskQueue
:
550 Bt
= BuildTask
._TaskQueue
[BuildItem
]
554 Bt
._Init
(BuildItem
, Dependency
)
555 BuildTask
._TaskQueue
[BuildItem
] = Bt
557 BuildTask
._PendingQueueLock
.acquire()
558 BuildTask
._PendingQueue
[BuildItem
] = Bt
559 BuildTask
._PendingQueueLock
.release()
563 ## The real constructor of BuildTask
565 # @param BuildItem A BuildUnit object representing a build object
566 # @param Dependency The dependent build object of BuildItem
568 def _Init(self
, BuildItem
, Dependency
=None):
569 self
.BuildItem
= BuildItem
571 self
.DependencyList
= []
572 if Dependency
is None:
573 Dependency
= BuildItem
.Dependency
575 Dependency
.extend(BuildItem
.Dependency
)
576 self
.AddDependency(Dependency
)
577 # flag indicating build completes, used to avoid unnecessary re-build
578 self
.CompleteFlag
= False
580 ## Check if all dependent build tasks are completed or not
584 for Dep
in self
.DependencyList
:
585 if Dep
.CompleteFlag
== True:
592 ## Add dependent build task
594 # @param Dependency The list of dependent build objects
596 def AddDependency(self
, Dependency
):
597 for Dep
in Dependency
:
598 if not Dep
.BuildObject
.IsBinaryModule
and not Dep
.BuildObject
.CanSkipbyCache(GlobalData
.gCacheIR
):
599 self
.DependencyList
.append(BuildTask
.New(Dep
)) # BuildTask list
601 ## The thread wrapper of LaunchCommand function
603 # @param Command A list or string contains the call of the command
604 # @param WorkingDir The directory in which the program will be running
606 def _CommandThread(self
, Command
, WorkingDir
):
608 self
.BuildItem
.BuildObject
.BuildTime
= LaunchCommand(Command
, WorkingDir
)
609 self
.CompleteFlag
= True
611 # Run hash operation post dependency, to account for libs
612 if GlobalData
.gUseHashCache
and self
.BuildItem
.BuildObject
.IsLibrary
:
613 HashFile
= path
.join(self
.BuildItem
.BuildObject
.BuildDir
, self
.BuildItem
.BuildObject
.Name
+ ".hash")
614 SaveFileOnChange(HashFile
, self
.BuildItem
.BuildObject
.GenModuleHash(), True)
617 # TRICK: hide the output of threads left running, so that the user can
618 # catch the error message easily
620 if not BuildTask
._ErrorFlag
.isSet():
621 GlobalData
.gBuildingModule
= "%s [%s, %s, %s]" % (str(self
.BuildItem
.BuildObject
),
622 self
.BuildItem
.BuildObject
.Arch
,
623 self
.BuildItem
.BuildObject
.ToolChain
,
624 self
.BuildItem
.BuildObject
.BuildTarget
626 EdkLogger
.SetLevel(EdkLogger
.ERROR
)
627 BuildTask
._ErrorFlag
.set()
628 BuildTask
._ErrorMessage
= "%s broken\n %s [%s]" % \
629 (threading
.currentThread().getName(), Command
, WorkingDir
)
631 # Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'
632 # If Module or Lib is being tracked, it did not fail header check test, and built successfully
633 if (self
.BuildItem
.BuildObject
.Arch
in GlobalData
.gModuleBuildTracking
and
634 self
.BuildItem
.BuildObject
in GlobalData
.gModuleBuildTracking
[self
.BuildItem
.BuildObject
.Arch
] and
635 GlobalData
.gModuleBuildTracking
[self
.BuildItem
.BuildObject
.Arch
][self
.BuildItem
.BuildObject
] != 'FAIL_METAFILE' and
636 not BuildTask
._ErrorFlag
.isSet()
638 GlobalData
.gModuleBuildTracking
[self
.BuildItem
.BuildObject
.Arch
][self
.BuildItem
.BuildObject
] = 'SUCCESS'
640 # indicate there's a thread is available for another build task
641 BuildTask
._RunningQueueLock
.acquire()
642 BuildTask
._RunningQueue
.pop(self
.BuildItem
)
643 BuildTask
._RunningQueueLock
.release()
644 BuildTask
._Thread
.release()
646 ## Start build task thread
649 EdkLogger
.quiet("Building ... %s" % repr(self
.BuildItem
))
650 Command
= self
.BuildItem
.BuildCommand
+ [self
.BuildItem
.Target
]
651 self
.BuildTread
= Thread(target
=self
._CommandThread
, args
=(Command
, self
.BuildItem
.WorkingDir
))
652 self
.BuildTread
.setName("build thread")
653 self
.BuildTread
.setDaemon(False)
654 self
.BuildTread
.start()
656 ## The class contains the information related to EFI image
661 # Constructor will load all required image information.
663 # @param BaseName The full file path of image.
664 # @param Guid The GUID for image.
665 # @param Arch Arch of this image.
666 # @param OutputDir The output directory for image.
667 # @param DebugDir The debug directory for image.
668 # @param ImageClass PeImage Information
670 def __init__(self
, BaseName
, Guid
, Arch
, OutputDir
, DebugDir
, ImageClass
):
671 self
.BaseName
= BaseName
674 self
.OutputDir
= OutputDir
675 self
.DebugDir
= DebugDir
676 self
.Image
= ImageClass
677 self
.Image
.Size
= (self
.Image
.Size
// 0x1000 + 1) * 0x1000
679 ## The class implementing the EDK2 build process
681 # The build process includes:
682 # 1. Load configuration from target.txt and tools_def.txt in $(WORKSPACE)/Conf
683 # 2. Parse DSC file of active platform
684 # 3. Parse FDF file if any
685 # 4. Establish build database, including parse all other files (module, package)
686 # 5. Create AutoGen files (C code file, depex file, makefile) if necessary
687 # 6. Call build command
692 # Constructor will load all necessary configurations, parse platform, modules
693 # and packages and the establish a database for AutoGen.
695 # @param Target The build command target, one of gSupportedTarget
696 # @param WorkspaceDir The directory of workspace
697 # @param BuildOptions Build options passed from command line
699 def __init__(self
, Target
, WorkspaceDir
, BuildOptions
,log_q
):
700 self
.WorkspaceDir
= WorkspaceDir
702 self
.PlatformFile
= BuildOptions
.PlatformFile
703 self
.ModuleFile
= BuildOptions
.ModuleFile
704 self
.ArchList
= BuildOptions
.TargetArch
705 self
.ToolChainList
= BuildOptions
.ToolChain
706 self
.BuildTargetList
= BuildOptions
.BuildTarget
707 self
.Fdf
= BuildOptions
.FdfFile
708 self
.FdList
= BuildOptions
.RomImage
709 self
.FvList
= BuildOptions
.FvImage
710 self
.CapList
= BuildOptions
.CapName
711 self
.SilentMode
= BuildOptions
.SilentMode
712 self
.ThreadNumber
= 1
713 self
.SkipAutoGen
= BuildOptions
.SkipAutoGen
714 self
.Reparse
= BuildOptions
.Reparse
715 self
.SkuId
= BuildOptions
.SkuId
717 GlobalData
.gSKUID_CMD
= self
.SkuId
718 self
.ConfDirectory
= BuildOptions
.ConfDirectory
719 self
.SpawnMode
= True
720 self
.BuildReport
= BuildReport(BuildOptions
.ReportFile
, BuildOptions
.ReportType
)
721 self
.TargetTxt
= TargetTxt
722 self
.ToolDef
= ToolDef
726 GlobalData
.BuildOptionPcd
= BuildOptions
.OptionPcd
if BuildOptions
.OptionPcd
else []
727 #Set global flag for build mode
728 GlobalData
.gIgnoreSource
= BuildOptions
.IgnoreSources
729 GlobalData
.gUseHashCache
= BuildOptions
.UseHashCache
730 GlobalData
.gBinCacheDest
= BuildOptions
.BinCacheDest
731 GlobalData
.gBinCacheSource
= BuildOptions
.BinCacheSource
732 GlobalData
.gEnableGenfdsMultiThread
= BuildOptions
.GenfdsMultiThread
733 GlobalData
.gDisableIncludePathCheck
= BuildOptions
.DisableIncludePathCheck
735 if GlobalData
.gBinCacheDest
and not GlobalData
.gUseHashCache
:
736 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, ExtraData
="--binary-destination must be used together with --hash.")
738 if GlobalData
.gBinCacheSource
and not GlobalData
.gUseHashCache
:
739 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, ExtraData
="--binary-source must be used together with --hash.")
741 if GlobalData
.gBinCacheDest
and GlobalData
.gBinCacheSource
:
742 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, ExtraData
="--binary-destination can not be used together with --binary-source.")
744 if GlobalData
.gBinCacheSource
:
745 BinCacheSource
= os
.path
.normpath(GlobalData
.gBinCacheSource
)
746 if not os
.path
.isabs(BinCacheSource
):
747 BinCacheSource
= mws
.join(self
.WorkspaceDir
, BinCacheSource
)
748 GlobalData
.gBinCacheSource
= BinCacheSource
750 if GlobalData
.gBinCacheSource
is not None:
751 EdkLogger
.error("build", OPTION_VALUE_INVALID
, ExtraData
="Invalid value of option --binary-source.")
753 if GlobalData
.gBinCacheDest
:
754 BinCacheDest
= os
.path
.normpath(GlobalData
.gBinCacheDest
)
755 if not os
.path
.isabs(BinCacheDest
):
756 BinCacheDest
= mws
.join(self
.WorkspaceDir
, BinCacheDest
)
757 GlobalData
.gBinCacheDest
= BinCacheDest
759 if GlobalData
.gBinCacheDest
is not None:
760 EdkLogger
.error("build", OPTION_VALUE_INVALID
, ExtraData
="Invalid value of option --binary-destination.")
762 GlobalData
.gDatabasePath
= os
.path
.normpath(os
.path
.join(GlobalData
.gConfDirectory
, GlobalData
.gDatabasePath
))
763 if not os
.path
.exists(os
.path
.join(GlobalData
.gConfDirectory
, '.cache')):
764 os
.makedirs(os
.path
.join(GlobalData
.gConfDirectory
, '.cache'))
766 self
.BuildDatabase
= self
.Db
.BuildObject
768 self
.ToolChainFamily
= None
769 self
.LoadFixAddress
= 0
770 self
.UniFlag
= BuildOptions
.Flag
771 self
.BuildModules
= []
772 self
.HashSkipModules
= []
774 self
.LaunchPrebuildFlag
= False
775 self
.PlatformBuildPath
= os
.path
.join(GlobalData
.gConfDirectory
, '.cache', '.PlatformBuild')
776 if BuildOptions
.CommandLength
:
777 GlobalData
.gCommandMaxLength
= BuildOptions
.CommandLength
779 # print dot character during doing some time-consuming work
780 self
.Progress
= Utils
.Progressor()
781 # print current build environment and configuration
782 EdkLogger
.quiet("%-16s = %s" % ("WORKSPACE", os
.environ
["WORKSPACE"]))
783 if "PACKAGES_PATH" in os
.environ
:
784 # WORKSPACE env has been converted before. Print the same path style with WORKSPACE env.
785 EdkLogger
.quiet("%-16s = %s" % ("PACKAGES_PATH", os
.path
.normcase(os
.path
.normpath(os
.environ
["PACKAGES_PATH"]))))
786 EdkLogger
.quiet("%-16s = %s" % ("EDK_TOOLS_PATH", os
.environ
["EDK_TOOLS_PATH"]))
787 if "EDK_TOOLS_BIN" in os
.environ
:
788 # Print the same path style with WORKSPACE env.
789 EdkLogger
.quiet("%-16s = %s" % ("EDK_TOOLS_BIN", os
.path
.normcase(os
.path
.normpath(os
.environ
["EDK_TOOLS_BIN"]))))
790 EdkLogger
.quiet("%-16s = %s" % ("CONF_PATH", GlobalData
.gConfDirectory
))
791 if "PYTHON3_ENABLE" in os
.environ
:
792 PYTHON3_ENABLE
= os
.environ
["PYTHON3_ENABLE"]
793 if PYTHON3_ENABLE
!= "TRUE":
794 PYTHON3_ENABLE
= "FALSE"
795 EdkLogger
.quiet("%-16s = %s" % ("PYTHON3_ENABLE", PYTHON3_ENABLE
))
796 if "PYTHON_COMMAND" in os
.environ
:
797 EdkLogger
.quiet("%-16s = %s" % ("PYTHON_COMMAND", os
.environ
["PYTHON_COMMAND"]))
801 EdkLogger
.quiet("%-16s = %s" % ("PREBUILD", self
.Prebuild
))
803 EdkLogger
.quiet("%-16s = %s" % ("POSTBUILD", self
.Postbuild
))
805 self
.LaunchPrebuild()
806 self
.TargetTxt
= TargetTxt
807 self
.ToolDef
= ToolDef
808 if not (self
.LaunchPrebuildFlag
and os
.path
.exists(self
.PlatformBuildPath
)):
811 self
.AutoGenMgr
= None
813 os
.chdir(self
.WorkspaceDir
)
814 GlobalData
.gCacheIR
= Manager().dict()
816 def StartAutoGen(self
,mqueue
, DataPipe
,SkipAutoGen
,PcdMaList
,share_data
):
820 feedback_q
= mp
.Queue()
821 file_lock
= mp
.Lock()
822 error_event
= mp
.Event()
823 GlobalData
.file_lock
= file_lock
824 FfsCmd
= DataPipe
.Get("FfsCommand")
827 GlobalData
.FfsCmd
= FfsCmd
828 GlobalData
.libConstPcd
= DataPipe
.Get("LibConstPcd")
829 GlobalData
.Refes
= DataPipe
.Get("REFS")
830 auto_workers
= [AutoGenWorkerInProcess(mqueue
,DataPipe
.dump_file
,feedback_q
,file_lock
,share_data
,self
.log_q
,error_event
) for _
in range(self
.ThreadNumber
)]
831 self
.AutoGenMgr
= AutoGenManager(auto_workers
,feedback_q
,error_event
)
832 self
.AutoGenMgr
.start()
833 for w
in auto_workers
:
835 if PcdMaList
is not None:
836 for PcdMa
in PcdMaList
:
837 if GlobalData
.gBinCacheSource
and self
.Target
in [None, "", "all"]:
838 PcdMa
.GenModuleFilesHash(share_data
)
839 PcdMa
.GenPreMakefileHash(share_data
)
840 if PcdMa
.CanSkipbyPreMakefileCache(share_data
):
843 PcdMa
.CreateCodeFile(False)
844 PcdMa
.CreateMakeFile(False,GenFfsList
= DataPipe
.Get("FfsCommand").get((PcdMa
.MetaFile
.File
, PcdMa
.Arch
),[]))
846 if GlobalData
.gBinCacheSource
and self
.Target
in [None, "", "all"]:
847 PcdMa
.GenMakeHeaderFilesHash(share_data
)
848 PcdMa
.GenMakeHash(share_data
)
849 if PcdMa
.CanSkipbyMakeCache(share_data
):
852 self
.AutoGenMgr
.join()
853 rt
= self
.AutoGenMgr
.Status
855 except FatalError
as e
:
856 return False, e
.args
[0]
858 return False, UNKNOWN_ERROR
860 ## Load configuration
862 # This method will parse target.txt and get the build configurations.
864 def LoadConfiguration(self
):
866 # if no ARCH given in command line, get it from target.txt
867 if not self
.ArchList
:
868 self
.ArchList
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_TARGET_ARCH
]
869 self
.ArchList
= tuple(self
.ArchList
)
871 # if no build target given in command line, get it from target.txt
872 if not self
.BuildTargetList
:
873 self
.BuildTargetList
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_TARGET
]
875 # if no tool chain given in command line, get it from target.txt
876 if not self
.ToolChainList
:
877 self
.ToolChainList
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_TOOL_CHAIN_TAG
]
878 if self
.ToolChainList
is None or len(self
.ToolChainList
) == 0:
879 EdkLogger
.error("build", RESOURCE_NOT_AVAILABLE
, ExtraData
="No toolchain given. Don't know how to build.\n")
881 # check if the tool chains are defined or not
882 NewToolChainList
= []
883 for ToolChain
in self
.ToolChainList
:
884 if ToolChain
not in self
.ToolDef
.ToolsDefTxtDatabase
[TAB_TOD_DEFINES_TOOL_CHAIN_TAG
]:
885 EdkLogger
.warn("build", "Tool chain [%s] is not defined" % ToolChain
)
887 NewToolChainList
.append(ToolChain
)
888 # if no tool chain available, break the build
889 if len(NewToolChainList
) == 0:
890 EdkLogger
.error("build", RESOURCE_NOT_AVAILABLE
,
891 ExtraData
="[%s] not defined. No toolchain available for build!\n" % ", ".join(self
.ToolChainList
))
893 self
.ToolChainList
= NewToolChainList
896 ToolDefinition
= self
.ToolDef
.ToolsDefTxtDatabase
897 for Tool
in self
.ToolChainList
:
898 if TAB_TOD_DEFINES_FAMILY
not in ToolDefinition
or Tool
not in ToolDefinition
[TAB_TOD_DEFINES_FAMILY
] \
899 or not ToolDefinition
[TAB_TOD_DEFINES_FAMILY
][Tool
]:
900 EdkLogger
.warn("build", "No tool chain family found in configuration for %s. Default to MSFT." % Tool
)
901 ToolChainFamily
.append(TAB_COMPILER_MSFT
)
903 ToolChainFamily
.append(ToolDefinition
[TAB_TOD_DEFINES_FAMILY
][Tool
])
904 self
.ToolChainFamily
= ToolChainFamily
906 if not self
.PlatformFile
:
907 PlatformFile
= self
.TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_ACTIVE_PLATFORM
]
909 # Try to find one in current directory
910 WorkingDirectory
= os
.getcwd()
911 FileList
= glob
.glob(os
.path
.normpath(os
.path
.join(WorkingDirectory
, '*.dsc')))
912 FileNum
= len(FileList
)
914 EdkLogger
.error("build", OPTION_MISSING
,
915 ExtraData
="There are %d DSC files in %s. Use '-p' to specify one.\n" % (FileNum
, WorkingDirectory
))
917 PlatformFile
= FileList
[0]
919 EdkLogger
.error("build", RESOURCE_NOT_AVAILABLE
,
920 ExtraData
="No active platform specified in target.txt or command line! Nothing can be built.\n")
922 self
.PlatformFile
= PathClass(NormFile(PlatformFile
, self
.WorkspaceDir
), self
.WorkspaceDir
)
923 self
.ThreadNumber
= ThreadNum()
924 ## Initialize build configuration
926 # This method will parse DSC file and merge the configurations from
927 # command line and target.txt, then get the final build configurations.
930 # parse target.txt, tools_def.txt, and platform file
931 self
.LoadConfiguration()
933 # Allow case-insensitive for those from command line or configuration file
934 ErrorCode
, ErrorInfo
= self
.PlatformFile
.Validate(".dsc", False)
936 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
939 def InitPreBuild(self
):
940 self
.LoadConfiguration()
941 ErrorCode
, ErrorInfo
= self
.PlatformFile
.Validate(".dsc", False)
943 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
944 if self
.BuildTargetList
:
945 GlobalData
.gGlobalDefines
['TARGET'] = self
.BuildTargetList
[0]
947 GlobalData
.gGlobalDefines
['ARCH'] = self
.ArchList
[0]
948 if self
.ToolChainList
:
949 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = self
.ToolChainList
[0]
950 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = self
.ToolChainList
[0]
951 if self
.ToolChainFamily
:
952 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[0]
953 if 'PREBUILD' in GlobalData
.gCommandLineDefines
:
954 self
.Prebuild
= GlobalData
.gCommandLineDefines
.get('PREBUILD')
957 Platform
= self
.Db
.MapPlatform(str(self
.PlatformFile
))
958 self
.Prebuild
= str(Platform
.Prebuild
)
962 # Evaluate all arguments and convert arguments that are WORKSPACE
963 # relative paths to absolute paths. Filter arguments that look like
964 # flags or do not follow the file/dir naming rules to avoid false
965 # positives on this conversion.
967 for Arg
in self
.Prebuild
.split():
969 # Do not modify Arg if it looks like a flag or an absolute file path
971 if Arg
.startswith('-') or os
.path
.isabs(Arg
):
972 PrebuildList
.append(Arg
)
975 # Do not modify Arg if it does not look like a Workspace relative
976 # path that starts with a valid package directory name
978 if not Arg
[0].isalpha() or os
.path
.dirname(Arg
) == '':
979 PrebuildList
.append(Arg
)
982 # If Arg looks like a WORKSPACE relative path, then convert to an
983 # absolute path and check to see if the file exists.
985 Temp
= mws
.join(self
.WorkspaceDir
, Arg
)
986 if os
.path
.isfile(Temp
):
988 PrebuildList
.append(Arg
)
989 self
.Prebuild
= ' '.join(PrebuildList
)
990 self
.Prebuild
+= self
.PassCommandOption(self
.BuildTargetList
, self
.ArchList
, self
.ToolChainList
, self
.PlatformFile
, self
.Target
)
992 def InitPostBuild(self
):
993 if 'POSTBUILD' in GlobalData
.gCommandLineDefines
:
994 self
.Postbuild
= GlobalData
.gCommandLineDefines
.get('POSTBUILD')
996 Platform
= self
.Db
.MapPlatform(str(self
.PlatformFile
))
997 self
.Postbuild
= str(Platform
.Postbuild
)
1001 # Evaluate all arguments and convert arguments that are WORKSPACE
1002 # relative paths to absolute paths. Filter arguments that look like
1003 # flags or do not follow the file/dir naming rules to avoid false
1004 # positives on this conversion.
1006 for Arg
in self
.Postbuild
.split():
1008 # Do not modify Arg if it looks like a flag or an absolute file path
1010 if Arg
.startswith('-') or os
.path
.isabs(Arg
):
1011 PostbuildList
.append(Arg
)
1014 # Do not modify Arg if it does not look like a Workspace relative
1015 # path that starts with a valid package directory name
1017 if not Arg
[0].isalpha() or os
.path
.dirname(Arg
) == '':
1018 PostbuildList
.append(Arg
)
1021 # If Arg looks like a WORKSPACE relative path, then convert to an
1022 # absolute path and check to see if the file exists.
1024 Temp
= mws
.join(self
.WorkspaceDir
, Arg
)
1025 if os
.path
.isfile(Temp
):
1027 PostbuildList
.append(Arg
)
1028 self
.Postbuild
= ' '.join(PostbuildList
)
1029 self
.Postbuild
+= self
.PassCommandOption(self
.BuildTargetList
, self
.ArchList
, self
.ToolChainList
, self
.PlatformFile
, self
.Target
)
1031 def PassCommandOption(self
, BuildTarget
, TargetArch
, ToolChain
, PlatformFile
, Target
):
1033 if GlobalData
.gCommand
and isinstance(GlobalData
.gCommand
, list):
1034 BuildStr
+= ' ' + ' '.join(GlobalData
.gCommand
)
1037 ToolChainFlag
= False
1038 PlatformFileFlag
= False
1040 if GlobalData
.gOptions
and not GlobalData
.gOptions
.BuildTarget
:
1042 if GlobalData
.gOptions
and not GlobalData
.gOptions
.TargetArch
:
1044 if GlobalData
.gOptions
and not GlobalData
.gOptions
.ToolChain
:
1045 ToolChainFlag
= True
1046 if GlobalData
.gOptions
and not GlobalData
.gOptions
.PlatformFile
:
1047 PlatformFileFlag
= True
1049 if TargetFlag
and BuildTarget
:
1050 if isinstance(BuildTarget
, list) or isinstance(BuildTarget
, tuple):
1051 BuildStr
+= ' -b ' + ' -b '.join(BuildTarget
)
1052 elif isinstance(BuildTarget
, str):
1053 BuildStr
+= ' -b ' + BuildTarget
1054 if ArchFlag
and TargetArch
:
1055 if isinstance(TargetArch
, list) or isinstance(TargetArch
, tuple):
1056 BuildStr
+= ' -a ' + ' -a '.join(TargetArch
)
1057 elif isinstance(TargetArch
, str):
1058 BuildStr
+= ' -a ' + TargetArch
1059 if ToolChainFlag
and ToolChain
:
1060 if isinstance(ToolChain
, list) or isinstance(ToolChain
, tuple):
1061 BuildStr
+= ' -t ' + ' -t '.join(ToolChain
)
1062 elif isinstance(ToolChain
, str):
1063 BuildStr
+= ' -t ' + ToolChain
1064 if PlatformFileFlag
and PlatformFile
:
1065 if isinstance(PlatformFile
, list) or isinstance(PlatformFile
, tuple):
1066 BuildStr
+= ' -p ' + ' -p '.join(PlatformFile
)
1067 elif isinstance(PlatformFile
, str):
1068 BuildStr
+= ' -p' + PlatformFile
1069 BuildStr
+= ' --conf=' + GlobalData
.gConfDirectory
1071 BuildStr
+= ' ' + Target
1075 def LaunchPrebuild(self
):
1077 EdkLogger
.info("\n- Prebuild Start -\n")
1078 self
.LaunchPrebuildFlag
= True
1080 # The purpose of .PrebuildEnv file is capture environment variable settings set by the prebuild script
1081 # and preserve them for the rest of the main build step, because the child process environment will
1082 # evaporate as soon as it exits, we cannot get it in build step.
1084 PrebuildEnvFile
= os
.path
.join(GlobalData
.gConfDirectory
, '.cache', '.PrebuildEnv')
1085 if os
.path
.isfile(PrebuildEnvFile
):
1086 os
.remove(PrebuildEnvFile
)
1087 if os
.path
.isfile(self
.PlatformBuildPath
):
1088 os
.remove(self
.PlatformBuildPath
)
1089 if sys
.platform
== "win32":
1090 args
= ' && '.join((self
.Prebuild
, 'set > ' + PrebuildEnvFile
))
1091 Process
= Popen(args
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1093 args
= ' && '.join((self
.Prebuild
, 'env > ' + PrebuildEnvFile
))
1094 Process
= Popen(args
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1096 # launch two threads to read the STDOUT and STDERR
1097 EndOfProcedure
= Event()
1098 EndOfProcedure
.clear()
1100 StdOutThread
= Thread(target
=ReadMessage
, args
=(Process
.stdout
, EdkLogger
.info
, EndOfProcedure
))
1101 StdOutThread
.setName("STDOUT-Redirector")
1102 StdOutThread
.setDaemon(False)
1103 StdOutThread
.start()
1106 StdErrThread
= Thread(target
=ReadMessage
, args
=(Process
.stderr
, EdkLogger
.quiet
, EndOfProcedure
))
1107 StdErrThread
.setName("STDERR-Redirector")
1108 StdErrThread
.setDaemon(False)
1109 StdErrThread
.start()
1110 # waiting for program exit
1117 if Process
.returncode
!= 0 :
1118 EdkLogger
.error("Prebuild", PREBUILD_ERROR
, 'Prebuild process is not success!')
1120 if os
.path
.exists(PrebuildEnvFile
):
1121 f
= open(PrebuildEnvFile
)
1122 envs
= f
.readlines()
1124 envs
= [l
.split("=", 1) for l
in envs
]
1125 envs
= [[I
.strip() for I
in item
] for item
in envs
if len(item
) == 2]
1126 os
.environ
.update(dict(envs
))
1127 EdkLogger
.info("\n- Prebuild Done -\n")
1129 def LaunchPostbuild(self
):
1131 EdkLogger
.info("\n- Postbuild Start -\n")
1132 if sys
.platform
== "win32":
1133 Process
= Popen(self
.Postbuild
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1135 Process
= Popen(self
.Postbuild
, stdout
=PIPE
, stderr
=PIPE
, shell
=True)
1136 # launch two threads to read the STDOUT and STDERR
1137 EndOfProcedure
= Event()
1138 EndOfProcedure
.clear()
1140 StdOutThread
= Thread(target
=ReadMessage
, args
=(Process
.stdout
, EdkLogger
.info
, EndOfProcedure
))
1141 StdOutThread
.setName("STDOUT-Redirector")
1142 StdOutThread
.setDaemon(False)
1143 StdOutThread
.start()
1146 StdErrThread
= Thread(target
=ReadMessage
, args
=(Process
.stderr
, EdkLogger
.quiet
, EndOfProcedure
))
1147 StdErrThread
.setName("STDERR-Redirector")
1148 StdErrThread
.setDaemon(False)
1149 StdErrThread
.start()
1150 # waiting for program exit
1157 if Process
.returncode
!= 0 :
1158 EdkLogger
.error("Postbuild", POSTBUILD_ERROR
, 'Postbuild process is not success!')
1159 EdkLogger
.info("\n- Postbuild Done -\n")
1161 ## Error handling for hash feature
1163 # On BuildTask error, iterate through the Module Build tracking
1164 # dictionary to determine wheather a module failed to build. Invalidate
1165 # the hash associated with that module by removing it from storage.
1168 def invalidateHash(self
):
1169 # Only for hashing feature
1170 if not GlobalData
.gUseHashCache
:
1173 # GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash
1174 for moduleAutoGenObjArch
in GlobalData
.gModuleBuildTracking
.keys():
1175 for moduleAutoGenObj
in GlobalData
.gModuleBuildTracking
[moduleAutoGenObjArch
].keys():
1176 # Skip invalidating for Successful Module/Lib builds
1177 if GlobalData
.gModuleBuildTracking
[moduleAutoGenObjArch
][moduleAutoGenObj
] == 'SUCCESS':
1180 # The module failed to build, failed to start building, or failed the header check test from this point on
1182 # Remove .hash from build
1183 ModuleHashFile
= os
.path
.join(moduleAutoGenObj
.BuildDir
, moduleAutoGenObj
.Name
+ ".hash")
1184 if os
.path
.exists(ModuleHashFile
):
1185 os
.remove(ModuleHashFile
)
1187 # Remove .hash file from cache
1188 if GlobalData
.gBinCacheDest
:
1189 FileDir
= os
.path
.join(GlobalData
.gBinCacheDest
, moduleAutoGenObj
.Arch
, moduleAutoGenObj
.SourceDir
, moduleAutoGenObj
.MetaFile
.BaseName
)
1190 HashFile
= os
.path
.join(FileDir
, moduleAutoGenObj
.Name
+ '.hash')
1191 if os
.path
.exists(HashFile
):
1194 ## Build a module or platform
1196 # Create autogen code and makefile for a module or platform, and the launch
1197 # "make" command to build it
1199 # @param Target The target of build command
1200 # @param Platform The platform file
1201 # @param Module The module file
1202 # @param BuildTarget The name of build target, one of "DEBUG", "RELEASE"
1203 # @param ToolChain The name of toolchain to build
1204 # @param Arch The arch of the module/platform
1205 # @param CreateDepModuleCodeFile Flag used to indicate creating code
1206 # for dependent modules/Libraries
1207 # @param CreateDepModuleMakeFile Flag used to indicate creating makefile
1208 # for dependent modules/Libraries
1210 def _BuildPa(self
, Target
, AutoGenObject
, CreateDepsCodeFile
=True, CreateDepsMakeFile
=True, BuildModule
=False, FfsCommand
=None, PcdMaList
=None):
1211 if AutoGenObject
is None:
1213 if FfsCommand
is None:
1215 # skip file generation for cleanxxx targets, run and fds target
1216 if Target
not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1217 # for target which must generate AutoGen code and makefile
1219 for m
in AutoGenObject
.GetAllModuleInfo
:
1222 AutoGenObject
.DataPipe
.DataContainer
= {"FfsCommand":FfsCommand
}
1223 AutoGenObject
.DataPipe
.DataContainer
= {"CommandTarget": self
.Target
}
1224 self
.Progress
.Start("Generating makefile and code")
1225 data_pipe_file
= os
.path
.join(AutoGenObject
.BuildDir
, "GlobalVar_%s_%s.bin" % (str(AutoGenObject
.Guid
),AutoGenObject
.Arch
))
1226 AutoGenObject
.DataPipe
.dump(data_pipe_file
)
1227 autogen_rt
,errorcode
= self
.StartAutoGen(mqueue
, AutoGenObject
.DataPipe
, self
.SkipAutoGen
, PcdMaList
, GlobalData
.gCacheIR
)
1228 self
.Progress
.Stop("done!")
1230 self
.AutoGenMgr
.TerminateWorkers()
1231 self
.AutoGenMgr
.join(0.1)
1232 raise FatalError(errorcode
)
1233 AutoGenObject
.CreateCodeFile(False)
1234 AutoGenObject
.CreateMakeFile(False)
1236 # always recreate top/platform makefile when clean, just in case of inconsistency
1237 AutoGenObject
.CreateCodeFile(True)
1238 AutoGenObject
.CreateMakeFile(True)
1240 if EdkLogger
.GetLevel() == EdkLogger
.QUIET
:
1241 EdkLogger
.quiet("Building ... %s" % repr(AutoGenObject
))
1243 BuildCommand
= AutoGenObject
.BuildCommand
1244 if BuildCommand
is None or len(BuildCommand
) == 0:
1245 EdkLogger
.error("build", OPTION_MISSING
,
1246 "No build command found for this module. "
1247 "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1248 (AutoGenObject
.BuildTarget
, AutoGenObject
.ToolChain
, AutoGenObject
.Arch
),
1249 ExtraData
=str(AutoGenObject
))
1251 makefile
= GenMake
.BuildFile(AutoGenObject
)._FILE
_NAME
_[GenMake
.gMakeType
]
1259 BuildCommand
= BuildCommand
+ [Target
]
1260 LaunchCommand(BuildCommand
, AutoGenObject
.MakeFileDir
)
1261 self
.CreateAsBuiltInf()
1262 if GlobalData
.gBinCacheDest
:
1263 self
.UpdateBuildCache()
1264 self
.BuildModules
= []
1268 if Target
== 'libraries':
1269 for Lib
in AutoGenObject
.LibraryBuildDirectoryList
:
1270 NewBuildCommand
= BuildCommand
+ ['-f', os
.path
.normpath(os
.path
.join(Lib
, makefile
)), 'pbuild']
1271 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1275 if Target
== 'modules':
1276 for Lib
in AutoGenObject
.LibraryBuildDirectoryList
:
1277 NewBuildCommand
= BuildCommand
+ ['-f', os
.path
.normpath(os
.path
.join(Lib
, makefile
)), 'pbuild']
1278 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1279 for Mod
in AutoGenObject
.ModuleBuildDirectoryList
:
1280 NewBuildCommand
= BuildCommand
+ ['-f', os
.path
.normpath(os
.path
.join(Mod
, makefile
)), 'pbuild']
1281 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1282 self
.CreateAsBuiltInf()
1283 if GlobalData
.gBinCacheDest
:
1284 self
.UpdateBuildCache()
1285 self
.BuildModules
= []
1289 if Target
== 'cleanlib':
1290 for Lib
in AutoGenObject
.LibraryBuildDirectoryList
:
1291 LibMakefile
= os
.path
.normpath(os
.path
.join(Lib
, makefile
))
1292 if os
.path
.exists(LibMakefile
):
1293 NewBuildCommand
= BuildCommand
+ ['-f', LibMakefile
, 'cleanall']
1294 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1298 if Target
== 'clean':
1299 for Mod
in AutoGenObject
.ModuleBuildDirectoryList
:
1300 ModMakefile
= os
.path
.normpath(os
.path
.join(Mod
, makefile
))
1301 if os
.path
.exists(ModMakefile
):
1302 NewBuildCommand
= BuildCommand
+ ['-f', ModMakefile
, 'cleanall']
1303 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1304 for Lib
in AutoGenObject
.LibraryBuildDirectoryList
:
1305 LibMakefile
= os
.path
.normpath(os
.path
.join(Lib
, makefile
))
1306 if os
.path
.exists(LibMakefile
):
1307 NewBuildCommand
= BuildCommand
+ ['-f', LibMakefile
, 'cleanall']
1308 LaunchCommand(NewBuildCommand
, AutoGenObject
.MakeFileDir
)
1312 if Target
== 'cleanall':
1314 #os.rmdir(AutoGenObject.BuildDir)
1315 RemoveDirectory(AutoGenObject
.BuildDir
, True)
1316 except WindowsError as X
:
1317 EdkLogger
.error("build", FILE_DELETE_FAILURE
, ExtraData
=str(X
))
1320 ## Build a module or platform
1322 # Create autogen code and makefile for a module or platform, and the launch
1323 # "make" command to build it
1325 # @param Target The target of build command
1326 # @param Platform The platform file
1327 # @param Module The module file
1328 # @param BuildTarget The name of build target, one of "DEBUG", "RELEASE"
1329 # @param ToolChain The name of toolchain to build
1330 # @param Arch The arch of the module/platform
1331 # @param CreateDepModuleCodeFile Flag used to indicate creating code
1332 # for dependent modules/Libraries
1333 # @param CreateDepModuleMakeFile Flag used to indicate creating makefile
1334 # for dependent modules/Libraries
1336 def _Build(self
, Target
, AutoGenObject
, CreateDepsCodeFile
=True, CreateDepsMakeFile
=True, BuildModule
=False):
1337 if AutoGenObject
is None:
1340 # skip file generation for cleanxxx targets, run and fds target
1341 if Target
not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1342 # for target which must generate AutoGen code and makefile
1343 if not self
.SkipAutoGen
or Target
== 'genc':
1344 self
.Progress
.Start("Generating code")
1345 AutoGenObject
.CreateCodeFile(CreateDepsCodeFile
)
1346 self
.Progress
.Stop("done!")
1347 if Target
== "genc":
1350 if not self
.SkipAutoGen
or Target
== 'genmake':
1351 self
.Progress
.Start("Generating makefile")
1352 AutoGenObject
.CreateMakeFile(CreateDepsMakeFile
)
1353 #AutoGenObject.CreateAsBuiltInf()
1354 self
.Progress
.Stop("done!")
1355 if Target
== "genmake":
1358 # always recreate top/platform makefile when clean, just in case of inconsistency
1359 AutoGenObject
.CreateCodeFile(True)
1360 AutoGenObject
.CreateMakeFile(True)
1362 if EdkLogger
.GetLevel() == EdkLogger
.QUIET
:
1363 EdkLogger
.quiet("Building ... %s" % repr(AutoGenObject
))
1365 BuildCommand
= AutoGenObject
.BuildCommand
1366 if BuildCommand
is None or len(BuildCommand
) == 0:
1367 EdkLogger
.error("build", OPTION_MISSING
,
1368 "No build command found for this module. "
1369 "Please check your setting of %s_%s_%s_MAKE_PATH in Conf/tools_def.txt file." %
1370 (AutoGenObject
.BuildTarget
, AutoGenObject
.ToolChain
, AutoGenObject
.Arch
),
1371 ExtraData
=str(AutoGenObject
))
1376 BuildCommand
= BuildCommand
+ [Target
]
1377 AutoGenObject
.BuildTime
= LaunchCommand(BuildCommand
, AutoGenObject
.MakeFileDir
)
1378 self
.CreateAsBuiltInf()
1379 if GlobalData
.gBinCacheDest
:
1380 self
.UpdateBuildCache()
1381 self
.BuildModules
= []
1386 if GenFdsApi(AutoGenObject
.GenFdsCommandDict
, self
.Db
):
1387 EdkLogger
.error("build", COMMAND_FAILURE
)
1395 if Target
== 'libraries':
1402 if Target
== 'cleanall':
1404 #os.rmdir(AutoGenObject.BuildDir)
1405 RemoveDirectory(AutoGenObject
.BuildDir
, True)
1406 except WindowsError as X
:
1407 EdkLogger
.error("build", FILE_DELETE_FAILURE
, ExtraData
=str(X
))
1410 ## Rebase module image and Get function address for the input module list.
1412 def _RebaseModule (self
, MapBuffer
, BaseAddress
, ModuleList
, AddrIsOffset
= True, ModeIsSmm
= False):
1414 AddrIsOffset
= False
1415 for InfFile
in ModuleList
:
1416 sys
.stdout
.write (".")
1418 ModuleInfo
= ModuleList
[InfFile
]
1419 ModuleName
= ModuleInfo
.BaseName
1420 ModuleOutputImage
= ModuleInfo
.Image
.FileName
1421 ModuleDebugImage
= os
.path
.join(ModuleInfo
.DebugDir
, ModuleInfo
.BaseName
+ '.efi')
1422 ## for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1424 BaseAddress
= BaseAddress
- ModuleInfo
.Image
.Size
1426 # Update Image to new BaseAddress by GenFw tool
1428 LaunchCommand(["GenFw", "--rebase", str(BaseAddress
), "-r", ModuleOutputImage
], ModuleInfo
.OutputDir
)
1429 LaunchCommand(["GenFw", "--rebase", str(BaseAddress
), "-r", ModuleDebugImage
], ModuleInfo
.DebugDir
)
1432 # Set new address to the section header only for SMM driver.
1434 LaunchCommand(["GenFw", "--address", str(BaseAddress
), "-r", ModuleOutputImage
], ModuleInfo
.OutputDir
)
1435 LaunchCommand(["GenFw", "--address", str(BaseAddress
), "-r", ModuleDebugImage
], ModuleInfo
.DebugDir
)
1437 # Collect function address from Map file
1439 ImageMapTable
= ModuleOutputImage
.replace('.efi', '.map')
1441 if os
.path
.exists(ImageMapTable
):
1442 OrigImageBaseAddress
= 0
1443 ImageMap
= open(ImageMapTable
, 'r')
1444 for LinStr
in ImageMap
:
1445 if len (LinStr
.strip()) == 0:
1448 # Get the preferred address set on link time.
1450 if LinStr
.find ('Preferred load address is') != -1:
1451 StrList
= LinStr
.split()
1452 OrigImageBaseAddress
= int (StrList
[len(StrList
) - 1], 16)
1454 StrList
= LinStr
.split()
1455 if len (StrList
) > 4:
1456 if StrList
[3] == 'f' or StrList
[3] == 'F':
1458 RelativeAddress
= int (StrList
[2], 16) - OrigImageBaseAddress
1459 FunctionList
.append ((Name
, RelativeAddress
))
1463 # Add general information.
1466 MapBuffer
.append('\n\n%s (Fixed SMRAM Offset, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName
, BaseAddress
, BaseAddress
+ ModuleInfo
.Image
.EntryPoint
))
1468 MapBuffer
.append('\n\n%s (Fixed Memory Offset, BaseAddress=-0x%010X, EntryPoint=-0x%010X)\n' % (ModuleName
, 0 - BaseAddress
, 0 - (BaseAddress
+ ModuleInfo
.Image
.EntryPoint
)))
1470 MapBuffer
.append('\n\n%s (Fixed Memory Address, BaseAddress=0x%010X, EntryPoint=0x%010X)\n' % (ModuleName
, BaseAddress
, BaseAddress
+ ModuleInfo
.Image
.EntryPoint
))
1472 # Add guid and general seciton section.
1474 TextSectionAddress
= 0
1475 DataSectionAddress
= 0
1476 for SectionHeader
in ModuleInfo
.Image
.SectionHeaderList
:
1477 if SectionHeader
[0] == '.text':
1478 TextSectionAddress
= SectionHeader
[1]
1479 elif SectionHeader
[0] in ['.data', '.sdata']:
1480 DataSectionAddress
= SectionHeader
[1]
1482 MapBuffer
.append('(GUID=%s, .textbaseaddress=-0x%010X, .databaseaddress=-0x%010X)\n' % (ModuleInfo
.Guid
, 0 - (BaseAddress
+ TextSectionAddress
), 0 - (BaseAddress
+ DataSectionAddress
)))
1484 MapBuffer
.append('(GUID=%s, .textbaseaddress=0x%010X, .databaseaddress=0x%010X)\n' % (ModuleInfo
.Guid
, BaseAddress
+ TextSectionAddress
, BaseAddress
+ DataSectionAddress
))
1486 # Add debug image full path.
1488 MapBuffer
.append('(IMAGE=%s)\n\n' % (ModuleDebugImage
))
1490 # Add function address
1492 for Function
in FunctionList
:
1494 MapBuffer
.append(' -0x%010X %s\n' % (0 - (BaseAddress
+ Function
[1]), Function
[0]))
1496 MapBuffer
.append(' 0x%010X %s\n' % (BaseAddress
+ Function
[1], Function
[0]))
1500 # for SMM module in SMRAM, the SMRAM will be allocated from base to top.
1503 BaseAddress
= BaseAddress
+ ModuleInfo
.Image
.Size
1505 ## Collect MAP information of all FVs
1507 def _CollectFvMapBuffer (self
, MapBuffer
, Wa
, ModuleList
):
1509 # First get the XIP base address for FV map file.
1510 GuidPattern
= re
.compile("[-a-fA-F0-9]+")
1511 GuidName
= re
.compile(r
"\(GUID=[-a-fA-F0-9]+")
1512 for FvName
in Wa
.FdfProfile
.FvDict
:
1513 FvMapBuffer
= os
.path
.join(Wa
.FvDir
, FvName
+ '.Fv.map')
1514 if not os
.path
.exists(FvMapBuffer
):
1516 FvMap
= open(FvMapBuffer
, 'r')
1517 #skip FV size information
1523 MatchGuid
= GuidPattern
.match(Line
)
1524 if MatchGuid
is not None:
1526 # Replace GUID with module name
1528 GuidString
= MatchGuid
.group()
1529 if GuidString
.upper() in ModuleList
:
1530 Line
= Line
.replace(GuidString
, ModuleList
[GuidString
.upper()].Name
)
1531 MapBuffer
.append(Line
)
1533 # Add the debug image full path.
1535 MatchGuid
= GuidName
.match(Line
)
1536 if MatchGuid
is not None:
1537 GuidString
= MatchGuid
.group().split("=")[1]
1538 if GuidString
.upper() in ModuleList
:
1539 MapBuffer
.append('(IMAGE=%s)\n' % (os
.path
.join(ModuleList
[GuidString
.upper()].DebugDir
, ModuleList
[GuidString
.upper()].Name
+ '.efi')))
1543 ## Collect MAP information of all modules
1545 def _CollectModuleMapBuffer (self
, MapBuffer
, ModuleList
):
1546 sys
.stdout
.write ("Generate Load Module At Fix Address Map")
1548 PatchEfiImageList
= []
1556 # reserve 4K size in SMRAM to make SMM module address not from 0.
1558 for ModuleGuid
in ModuleList
:
1559 Module
= ModuleList
[ModuleGuid
]
1560 GlobalData
.gProcessingFile
= "%s [%s, %s, %s]" % (Module
.MetaFile
, Module
.Arch
, Module
.ToolChain
, Module
.BuildTarget
)
1562 OutputImageFile
= ''
1563 for ResultFile
in Module
.CodaTargetList
:
1564 if str(ResultFile
.Target
).endswith('.efi'):
1566 # module list for PEI, DXE, RUNTIME and SMM
1568 OutputImageFile
= os
.path
.join(Module
.OutputDir
, Module
.Name
+ '.efi')
1569 ImageClass
= PeImageClass (OutputImageFile
)
1570 if not ImageClass
.IsValid
:
1571 EdkLogger
.error("build", FILE_PARSE_FAILURE
, ExtraData
=ImageClass
.ErrorInfo
)
1572 ImageInfo
= PeImageInfo(Module
.Name
, Module
.Guid
, Module
.Arch
, Module
.OutputDir
, Module
.DebugDir
, ImageClass
)
1573 if Module
.ModuleType
in [SUP_MODULE_PEI_CORE
, SUP_MODULE_PEIM
, EDK_COMPONENT_TYPE_COMBINED_PEIM_DRIVER
, EDK_COMPONENT_TYPE_PIC_PEIM
, EDK_COMPONENT_TYPE_RELOCATABLE_PEIM
, SUP_MODULE_DXE_CORE
]:
1574 PeiModuleList
[Module
.MetaFile
] = ImageInfo
1575 PeiSize
+= ImageInfo
.Image
.Size
1576 elif Module
.ModuleType
in [EDK_COMPONENT_TYPE_BS_DRIVER
, SUP_MODULE_DXE_DRIVER
, SUP_MODULE_UEFI_DRIVER
]:
1577 BtModuleList
[Module
.MetaFile
] = ImageInfo
1578 BtSize
+= ImageInfo
.Image
.Size
1579 elif Module
.ModuleType
in [SUP_MODULE_DXE_RUNTIME_DRIVER
, EDK_COMPONENT_TYPE_RT_DRIVER
, SUP_MODULE_DXE_SAL_DRIVER
, EDK_COMPONENT_TYPE_SAL_RT_DRIVER
]:
1580 RtModuleList
[Module
.MetaFile
] = ImageInfo
1581 RtSize
+= ImageInfo
.Image
.Size
1582 elif Module
.ModuleType
in [SUP_MODULE_SMM_CORE
, SUP_MODULE_DXE_SMM_DRIVER
, SUP_MODULE_MM_STANDALONE
, SUP_MODULE_MM_CORE_STANDALONE
]:
1583 SmmModuleList
[Module
.MetaFile
] = ImageInfo
1584 SmmSize
+= ImageInfo
.Image
.Size
1585 if Module
.ModuleType
== SUP_MODULE_DXE_SMM_DRIVER
:
1586 PiSpecVersion
= Module
.Module
.Specification
.get('PI_SPECIFICATION_VERSION', '0x00000000')
1587 # for PI specification < PI1.1, DXE_SMM_DRIVER also runs as BOOT time driver.
1588 if int(PiSpecVersion
, 16) < 0x0001000A:
1589 BtModuleList
[Module
.MetaFile
] = ImageInfo
1590 BtSize
+= ImageInfo
.Image
.Size
1593 # EFI image is final target.
1594 # Check EFI image contains patchable FixAddress related PCDs.
1596 if OutputImageFile
!= '':
1597 ModuleIsPatch
= False
1598 for Pcd
in Module
.ModulePcdList
:
1599 if Pcd
.Type
== TAB_PCDS_PATCHABLE_IN_MODULE
and Pcd
.TokenCName
in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET
:
1600 ModuleIsPatch
= True
1602 if not ModuleIsPatch
:
1603 for Pcd
in Module
.LibraryPcdList
:
1604 if Pcd
.Type
== TAB_PCDS_PATCHABLE_IN_MODULE
and Pcd
.TokenCName
in TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SET
:
1605 ModuleIsPatch
= True
1608 if not ModuleIsPatch
:
1611 # Module includes the patchable load fix address PCDs.
1612 # It will be fixed up later.
1614 PatchEfiImageList
.append (OutputImageFile
)
1617 # Get Top Memory address
1619 ReservedRuntimeMemorySize
= 0
1620 TopMemoryAddress
= 0
1621 if self
.LoadFixAddress
== 0xFFFFFFFFFFFFFFFF:
1622 TopMemoryAddress
= 0
1624 TopMemoryAddress
= self
.LoadFixAddress
1625 if TopMemoryAddress
< RtSize
+ BtSize
+ PeiSize
:
1626 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS is too low to load driver")
1629 # Patch FixAddress related PCDs into EFI image
1631 for EfiImage
in PatchEfiImageList
:
1632 EfiImageMap
= EfiImage
.replace('.efi', '.map')
1633 if not os
.path
.exists(EfiImageMap
):
1636 # Get PCD offset in EFI image by GenPatchPcdTable function
1638 PcdTable
= parsePcdInfoFromMapFile(EfiImageMap
, EfiImage
)
1640 # Patch real PCD value by PatchPcdValue tool
1642 for PcdInfo
in PcdTable
:
1644 if PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE
:
1645 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_PEI_PAGE_SIZE_DATA_TYPE
, str (PeiSize
// 0x1000))
1646 elif PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE
:
1647 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_DXE_PAGE_SIZE_DATA_TYPE
, str (BtSize
// 0x1000))
1648 elif PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE
:
1649 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_RUNTIME_PAGE_SIZE_DATA_TYPE
, str (RtSize
// 0x1000))
1650 elif PcdInfo
[0] == TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE
and len (SmmModuleList
) > 0:
1651 ReturnValue
, ErrorInfo
= PatchBinaryFile (EfiImage
, PcdInfo
[1], TAB_PCDS_PATCHABLE_LOAD_FIX_ADDRESS_SMM_PAGE_SIZE_DATA_TYPE
, str (SmmSize
// 0x1000))
1652 if ReturnValue
!= 0:
1653 EdkLogger
.error("build", PARAMETER_INVALID
, "Patch PCD value failed", ExtraData
=ErrorInfo
)
1655 MapBuffer
.append('PEI_CODE_PAGE_NUMBER = 0x%x\n' % (PeiSize
// 0x1000))
1656 MapBuffer
.append('BOOT_CODE_PAGE_NUMBER = 0x%x\n' % (BtSize
// 0x1000))
1657 MapBuffer
.append('RUNTIME_CODE_PAGE_NUMBER = 0x%x\n' % (RtSize
// 0x1000))
1658 if len (SmmModuleList
) > 0:
1659 MapBuffer
.append('SMM_CODE_PAGE_NUMBER = 0x%x\n' % (SmmSize
// 0x1000))
1661 PeiBaseAddr
= TopMemoryAddress
- RtSize
- BtSize
1662 BtBaseAddr
= TopMemoryAddress
- RtSize
1663 RtBaseAddr
= TopMemoryAddress
- ReservedRuntimeMemorySize
1665 self
._RebaseModule
(MapBuffer
, PeiBaseAddr
, PeiModuleList
, TopMemoryAddress
== 0)
1666 self
._RebaseModule
(MapBuffer
, BtBaseAddr
, BtModuleList
, TopMemoryAddress
== 0)
1667 self
._RebaseModule
(MapBuffer
, RtBaseAddr
, RtModuleList
, TopMemoryAddress
== 0)
1668 self
._RebaseModule
(MapBuffer
, 0x1000, SmmModuleList
, AddrIsOffset
=False, ModeIsSmm
=True)
1669 MapBuffer
.append('\n\n')
1670 sys
.stdout
.write ("\n")
1673 ## Save platform Map file
1675 def _SaveMapFile (self
, MapBuffer
, Wa
):
1677 # Map file path is got.
1679 MapFilePath
= os
.path
.join(Wa
.BuildDir
, Wa
.Name
+ '.map')
1681 # Save address map into MAP file.
1683 SaveFileOnChange(MapFilePath
, ''.join(MapBuffer
), False)
1684 if self
.LoadFixAddress
!= 0:
1685 sys
.stdout
.write ("\nLoad Module At Fix Address Map file can be found at %s\n" % (MapFilePath
))
1688 ## Build active platform for different build targets and different tool chains
1690 def _BuildPlatform(self
):
1691 SaveFileOnChange(self
.PlatformBuildPath
, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1692 for BuildTarget
in self
.BuildTargetList
:
1693 GlobalData
.gGlobalDefines
['TARGET'] = BuildTarget
1695 for ToolChain
in self
.ToolChainList
:
1696 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = ToolChain
1697 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = ToolChain
1698 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[index
]
1700 Wa
= WorkspaceAutoGen(
1717 self
.Fdf
= Wa
.FdfFile
1718 self
.LoadFixAddress
= Wa
.Platform
.LoadFixAddress
1719 self
.BuildReport
.AddPlatformReport(Wa
)
1720 self
.Progress
.Stop("done!")
1722 # Add ffs build to makefile
1724 if GlobalData
.gEnableGenfdsMultiThread
and self
.Fdf
:
1725 CmdListDict
= self
._GenFfsCmd
(Wa
.ArchList
)
1727 for Arch
in Wa
.ArchList
:
1729 GlobalData
.gGlobalDefines
['ARCH'] = Arch
1730 Pa
= PlatformAutoGen(Wa
, self
.PlatformFile
, BuildTarget
, ToolChain
, Arch
)
1731 for Module
in Pa
.Platform
.Modules
:
1732 # Get ModuleAutoGen object to generate C code file and makefile
1733 Ma
= ModuleAutoGen(Wa
, Module
, BuildTarget
, ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
1737 Ma
.PlatformInfo
= Pa
1739 PcdMaList
.append(Ma
)
1740 self
.BuildModules
.append(Ma
)
1741 self
._BuildPa
(self
.Target
, Pa
, FfsCommand
=CmdListDict
,PcdMaList
=PcdMaList
)
1743 # Create MAP file when Load Fix Address is enabled.
1744 if self
.Target
in ["", "all", "fds"]:
1745 for Arch
in Wa
.ArchList
:
1746 GlobalData
.gGlobalDefines
['ARCH'] = Arch
1748 # Check whether the set fix address is above 4G for 32bit image.
1750 if (Arch
== 'IA32' or Arch
== 'ARM') and self
.LoadFixAddress
!= 0xFFFFFFFFFFFFFFFF and self
.LoadFixAddress
>= 0x100000000:
1751 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platform with IA32 or ARM arch modules")
1756 for Pa
in Wa
.AutoGenObjectList
:
1757 for Ma
in Pa
.ModuleAutoGenList
:
1760 if not Ma
.IsLibrary
:
1761 ModuleList
[Ma
.Guid
.upper()] = Ma
1764 if self
.LoadFixAddress
!= 0:
1766 # Rebase module to the preferred memory address before GenFds
1768 self
._CollectModuleMapBuffer
(MapBuffer
, ModuleList
)
1771 # create FDS again for the updated EFI image
1773 self
._Build
("fds", Wa
)
1775 # Create MAP file for all platform FVs after GenFds.
1777 self
._CollectFvMapBuffer
(MapBuffer
, Wa
, ModuleList
)
1779 # Save MAP buffer into MAP file.
1781 self
._SaveMapFile
(MapBuffer
, Wa
)
1783 ## Build active module for different build targets, different tool chains and different archs
1785 def _BuildModule(self
):
1786 for BuildTarget
in self
.BuildTargetList
:
1787 GlobalData
.gGlobalDefines
['TARGET'] = BuildTarget
1789 for ToolChain
in self
.ToolChainList
:
1790 WorkspaceAutoGenTime
= time
.time()
1791 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = ToolChain
1792 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = ToolChain
1793 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[index
]
1796 # module build needs platform build information, so get platform
1799 Wa
= WorkspaceAutoGen(
1817 self
.Fdf
= Wa
.FdfFile
1818 self
.LoadFixAddress
= Wa
.Platform
.LoadFixAddress
1819 Wa
.CreateMakeFile(False)
1820 # Add ffs build to makefile
1822 if GlobalData
.gEnableGenfdsMultiThread
and self
.Fdf
:
1823 CmdListDict
= self
._GenFfsCmd
(Wa
.ArchList
)
1825 # Add Platform and Package level hash in share_data for module hash calculation later
1826 if GlobalData
.gBinCacheSource
or GlobalData
.gBinCacheDest
:
1827 GlobalData
.gCacheIR
[('PlatformHash')] = GlobalData
.gPlatformHash
1828 for PkgName
in GlobalData
.gPackageHash
.keys():
1829 GlobalData
.gCacheIR
[(PkgName
, 'PackageHash')] = GlobalData
.gPackageHash
[PkgName
]
1830 GlobalData
.file_lock
= mp
.Lock()
1831 GlobalData
.FfsCmd
= CmdListDict
1833 self
.Progress
.Stop("done!")
1835 ExitFlag
= threading
.Event()
1837 self
.AutoGenTime
+= int(round((time
.time() - WorkspaceAutoGenTime
)))
1838 for Arch
in Wa
.ArchList
:
1839 AutoGenStart
= time
.time()
1840 GlobalData
.gGlobalDefines
['ARCH'] = Arch
1841 Pa
= PlatformAutoGen(Wa
, self
.PlatformFile
, BuildTarget
, ToolChain
, Arch
)
1842 GlobalData
.libConstPcd
= Pa
.DataPipe
.Get("LibConstPcd")
1843 GlobalData
.Refes
= Pa
.DataPipe
.Get("REFS")
1844 for Module
in Pa
.Platform
.Modules
:
1845 if self
.ModuleFile
.Dir
== Module
.Dir
and self
.ModuleFile
.Name
== Module
.Name
:
1846 Ma
= ModuleAutoGen(Wa
, Module
, BuildTarget
, ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
1851 if GlobalData
.gBinCacheSource
and self
.Target
in [None, "", "all"]:
1852 Ma
.GenModuleFilesHash(GlobalData
.gCacheIR
)
1853 Ma
.GenPreMakefileHash(GlobalData
.gCacheIR
)
1854 if Ma
.CanSkipbyPreMakefileCache(GlobalData
.gCacheIR
):
1855 self
.HashSkipModules
.append(Ma
)
1856 EdkLogger
.quiet("cache hit: %s[%s]" % (Ma
.MetaFile
.Path
, Ma
.Arch
))
1859 # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
1860 if self
.Target
not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
1861 # for target which must generate AutoGen code and makefile
1862 if not self
.SkipAutoGen
or self
.Target
== 'genc':
1863 self
.Progress
.Start("Generating code")
1864 Ma
.CreateCodeFile(True)
1865 self
.Progress
.Stop("done!")
1866 if self
.Target
== "genc":
1868 if not self
.SkipAutoGen
or self
.Target
== 'genmake':
1869 self
.Progress
.Start("Generating makefile")
1870 if CmdListDict
and self
.Fdf
and (Module
.File
, Arch
) in CmdListDict
:
1871 Ma
.CreateMakeFile(True, CmdListDict
[Module
.File
, Arch
])
1872 del CmdListDict
[Module
.File
, Arch
]
1874 Ma
.CreateMakeFile(True)
1875 self
.Progress
.Stop("done!")
1876 if self
.Target
== "genmake":
1879 if GlobalData
.gBinCacheSource
and self
.Target
in [None, "", "all"]:
1880 Ma
.GenMakeHeaderFilesHash(GlobalData
.gCacheIR
)
1881 Ma
.GenMakeHash(GlobalData
.gCacheIR
)
1882 if Ma
.CanSkipbyMakeCache(GlobalData
.gCacheIR
):
1883 self
.HashSkipModules
.append(Ma
)
1884 EdkLogger
.quiet("cache hit: %s[%s]" % (Ma
.MetaFile
.Path
, Ma
.Arch
))
1887 EdkLogger
.quiet("cache miss: %s[%s]" % (Ma
.MetaFile
.Path
, Ma
.Arch
))
1888 Ma
.PrintFirstMakeCacheMissFile(GlobalData
.gCacheIR
)
1890 self
.BuildModules
.append(Ma
)
1891 # Initialize all modules in tracking to 'FAIL'
1892 if Ma
.Arch
not in GlobalData
.gModuleBuildTracking
:
1893 GlobalData
.gModuleBuildTracking
[Ma
.Arch
] = dict()
1894 if Ma
not in GlobalData
.gModuleBuildTracking
[Ma
.Arch
]:
1895 GlobalData
.gModuleBuildTracking
[Ma
.Arch
][Ma
] = 'FAIL'
1896 self
.AutoGenTime
+= int(round((time
.time() - AutoGenStart
)))
1897 MakeStart
= time
.time()
1898 for Ma
in self
.BuildModules
:
1899 if not Ma
.IsBinaryModule
:
1900 Bt
= BuildTask
.New(ModuleMakeUnit(Ma
, Pa
.BuildCommand
,self
.Target
))
1901 # Break build if any build thread has error
1902 if BuildTask
.HasError():
1903 # we need a full version of makefile for platform
1905 BuildTask
.WaitForComplete()
1906 self
.invalidateHash()
1907 Pa
.CreateMakeFile(False)
1908 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
1909 # Start task scheduler
1910 if not BuildTask
.IsOnGoing():
1911 BuildTask
.StartScheduler(self
.ThreadNumber
, ExitFlag
)
1913 # in case there's an interruption. we need a full version of makefile for platform
1914 Pa
.CreateMakeFile(False)
1915 if BuildTask
.HasError():
1916 self
.invalidateHash()
1917 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
1918 self
.MakeTime
+= int(round((time
.time() - MakeStart
)))
1920 MakeContiue
= time
.time()
1922 BuildTask
.WaitForComplete()
1923 self
.CreateAsBuiltInf()
1924 if GlobalData
.gBinCacheDest
:
1925 self
.UpdateBuildCache()
1926 self
.BuildModules
= []
1927 self
.MakeTime
+= int(round((time
.time() - MakeContiue
)))
1928 if BuildTask
.HasError():
1929 self
.invalidateHash()
1930 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
1932 self
.BuildReport
.AddPlatformReport(Wa
, MaList
)
1937 "Module for [%s] is not a component of active platform."\
1938 " Please make sure that the ARCH and inf file path are"\
1939 " given in the same as in [%s]" % \
1940 (', '.join(Wa
.ArchList
), self
.PlatformFile
),
1941 ExtraData
=self
.ModuleFile
1943 # Create MAP file when Load Fix Address is enabled.
1944 if self
.Target
== "fds" and self
.Fdf
:
1945 for Arch
in Wa
.ArchList
:
1947 # Check whether the set fix address is above 4G for 32bit image.
1949 if (Arch
== 'IA32' or Arch
== 'ARM') and self
.LoadFixAddress
!= 0xFFFFFFFFFFFFFFFF and self
.LoadFixAddress
>= 0x100000000:
1950 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
1955 for Pa
in Wa
.AutoGenObjectList
:
1956 for Ma
in Pa
.ModuleAutoGenList
:
1959 if not Ma
.IsLibrary
:
1960 ModuleList
[Ma
.Guid
.upper()] = Ma
1963 if self
.LoadFixAddress
!= 0:
1965 # Rebase module to the preferred memory address before GenFds
1967 self
._CollectModuleMapBuffer
(MapBuffer
, ModuleList
)
1969 # create FDS again for the updated EFI image
1971 GenFdsStart
= time
.time()
1972 self
._Build
("fds", Wa
)
1973 self
.GenFdsTime
+= int(round((time
.time() - GenFdsStart
)))
1975 # Create MAP file for all platform FVs after GenFds.
1977 self
._CollectFvMapBuffer
(MapBuffer
, Wa
, ModuleList
)
1979 # Save MAP buffer into MAP file.
1981 self
._SaveMapFile
(MapBuffer
, Wa
)
1982 self
.invalidateHash()
1984 def _GenFfsCmd(self
,ArchList
):
1985 # convert dictionary of Cmd:(Inf,Arch)
1986 # to a new dictionary of (Inf,Arch):Cmd,Cmd,Cmd...
1987 CmdSetDict
= defaultdict(set)
1988 GenFfsDict
= GenFds
.GenFfsMakefile('', GlobalData
.gFdfParser
, self
, ArchList
, GlobalData
)
1989 for Cmd
in GenFfsDict
:
1990 tmpInf
, tmpArch
= GenFfsDict
[Cmd
]
1991 CmdSetDict
[tmpInf
, tmpArch
].add(Cmd
)
1994 ## Build a platform in multi-thread mode
1996 def _MultiThreadBuildPlatform(self
):
1997 SaveFileOnChange(self
.PlatformBuildPath
, '# DO NOT EDIT \n# FILE auto-generated\n', False)
1998 for BuildTarget
in self
.BuildTargetList
:
1999 GlobalData
.gGlobalDefines
['TARGET'] = BuildTarget
2001 for ToolChain
in self
.ToolChainList
:
2002 WorkspaceAutoGenTime
= time
.time()
2003 GlobalData
.gGlobalDefines
['TOOLCHAIN'] = ToolChain
2004 GlobalData
.gGlobalDefines
['TOOL_CHAIN_TAG'] = ToolChain
2005 GlobalData
.gGlobalDefines
['FAMILY'] = self
.ToolChainFamily
[index
]
2007 Wa
= WorkspaceAutoGen(
2024 self
.Fdf
= Wa
.FdfFile
2025 self
.LoadFixAddress
= Wa
.Platform
.LoadFixAddress
2026 self
.BuildReport
.AddPlatformReport(Wa
)
2027 Wa
.CreateMakeFile(False)
2029 # Add ffs build to makefile
2031 if GlobalData
.gEnableGenfdsMultiThread
and self
.Fdf
:
2032 CmdListDict
= self
._GenFfsCmd
(Wa
.ArchList
)
2034 # Add Platform and Package level hash in share_data for module hash calculation later
2035 if GlobalData
.gBinCacheSource
or GlobalData
.gBinCacheDest
:
2036 GlobalData
.gCacheIR
[('PlatformHash')] = GlobalData
.gPlatformHash
2037 for PkgName
in GlobalData
.gPackageHash
.keys():
2038 GlobalData
.gCacheIR
[(PkgName
, 'PackageHash')] = GlobalData
.gPackageHash
[PkgName
]
2040 # multi-thread exit flag
2041 ExitFlag
= threading
.Event()
2043 self
.AutoGenTime
+= int(round((time
.time() - WorkspaceAutoGenTime
)))
2044 self
.BuildModules
= []
2046 for Arch
in Wa
.ArchList
:
2048 AutoGenStart
= time
.time()
2049 GlobalData
.gGlobalDefines
['ARCH'] = Arch
2050 Pa
= PlatformAutoGen(Wa
, self
.PlatformFile
, BuildTarget
, ToolChain
, Arch
)
2054 for Inf
in Pa
.Platform
.Modules
:
2055 ModuleList
.append(Inf
)
2056 # Add the INF only list in FDF
2057 if GlobalData
.gFdfParser
is not None:
2058 for InfName
in GlobalData
.gFdfParser
.Profile
.InfList
:
2059 Inf
= PathClass(NormPath(InfName
), self
.WorkspaceDir
, Arch
)
2060 if Inf
in Pa
.Platform
.Modules
:
2062 ModuleList
.append(Inf
)
2063 Pa
.DataPipe
.DataContainer
= {"FfsCommand":CmdListDict
}
2064 Pa
.DataPipe
.DataContainer
= {"Workspace_timestamp": Wa
._SrcTimeStamp
}
2065 Pa
.DataPipe
.DataContainer
= {"CommandTarget": self
.Target
}
2066 for Module
in ModuleList
:
2067 # Get ModuleAutoGen object to generate C code file and makefile
2068 Ma
= ModuleAutoGen(Wa
, Module
, BuildTarget
, ToolChain
, Arch
, self
.PlatformFile
,Pa
.DataPipe
)
2073 Ma
.PlatformInfo
= Pa
2075 PcdMaList
.append(Ma
)
2076 TotalModules
.append(Ma
)
2077 # Initialize all modules in tracking to 'FAIL'
2078 if Ma
.Arch
not in GlobalData
.gModuleBuildTracking
:
2079 GlobalData
.gModuleBuildTracking
[Ma
.Arch
] = dict()
2080 if Ma
not in GlobalData
.gModuleBuildTracking
[Ma
.Arch
]:
2081 GlobalData
.gModuleBuildTracking
[Ma
.Arch
][Ma
] = 'FAIL'
2084 for m
in Pa
.GetAllModuleInfo
:
2086 data_pipe_file
= os
.path
.join(Pa
.BuildDir
, "GlobalVar_%s_%s.bin" % (str(Pa
.Guid
),Pa
.Arch
))
2087 Pa
.DataPipe
.dump(data_pipe_file
)
2088 autogen_rt
, errorcode
= self
.StartAutoGen(mqueue
, Pa
.DataPipe
, self
.SkipAutoGen
, PcdMaList
, GlobalData
.gCacheIR
)
2090 # Skip cache hit modules
2091 if GlobalData
.gBinCacheSource
:
2092 for Ma
in TotalModules
:
2093 if (Ma
.MetaFile
.Path
, Ma
.Arch
) in GlobalData
.gCacheIR
and \
2094 GlobalData
.gCacheIR
[(Ma
.MetaFile
.Path
, Ma
.Arch
)].PreMakeCacheHit
:
2095 self
.HashSkipModules
.append(Ma
)
2097 if (Ma
.MetaFile
.Path
, Ma
.Arch
) in GlobalData
.gCacheIR
and \
2098 GlobalData
.gCacheIR
[(Ma
.MetaFile
.Path
, Ma
.Arch
)].MakeCacheHit
:
2099 self
.HashSkipModules
.append(Ma
)
2101 self
.BuildModules
.append(Ma
)
2103 self
.BuildModules
.extend(TotalModules
)
2106 self
.AutoGenMgr
.TerminateWorkers()
2107 self
.AutoGenMgr
.join(0.1)
2108 raise FatalError(errorcode
)
2109 self
.AutoGenTime
+= int(round((time
.time() - AutoGenStart
)))
2110 self
.Progress
.Stop("done!")
2112 if GlobalData
.gBinCacheSource
:
2113 EdkLogger
.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self
.HashSkipModules
)), len(set(self
.BuildModules
))))
2115 CacheNotHitMa
= set()
2116 for IR
in GlobalData
.gCacheIR
.keys():
2117 if 'PlatformHash' in IR
or 'PackageHash' in IR
:
2119 if GlobalData
.gCacheIR
[IR
].PreMakeCacheHit
or GlobalData
.gCacheIR
[IR
].MakeCacheHit
:
2122 # There might be binary module or module which has .inc files, not count for cache miss
2123 CacheNotHitMa
.add(IR
)
2124 EdkLogger
.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa
)+len(CacheNotHitMa
), len(CacheHitMa
)))
2126 for Arch
in Wa
.ArchList
:
2127 MakeStart
= time
.time()
2128 for Ma
in set(self
.BuildModules
):
2129 # Generate build task for the module
2130 if not Ma
.IsBinaryModule
:
2131 Bt
= BuildTask
.New(ModuleMakeUnit(Ma
, Pa
.BuildCommand
,self
.Target
))
2132 # Break build if any build thread has error
2133 if BuildTask
.HasError():
2134 # we need a full version of makefile for platform
2136 BuildTask
.WaitForComplete()
2137 self
.invalidateHash()
2138 Pa
.CreateMakeFile(False)
2139 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
2140 # Start task scheduler
2141 if not BuildTask
.IsOnGoing():
2142 BuildTask
.StartScheduler(self
.ThreadNumber
, ExitFlag
)
2144 # in case there's an interruption. we need a full version of makefile for platform
2145 Pa
.CreateMakeFile(False)
2146 if BuildTask
.HasError():
2147 self
.invalidateHash()
2148 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
2149 self
.MakeTime
+= int(round((time
.time() - MakeStart
)))
2151 MakeContiue
= time
.time()
2155 # All modules have been put in build tasks queue. Tell task scheduler
2156 # to exit if all tasks are completed
2159 BuildTask
.WaitForComplete()
2160 self
.CreateAsBuiltInf()
2161 if GlobalData
.gBinCacheDest
:
2162 self
.UpdateBuildCache()
2163 self
.BuildModules
= []
2164 self
.MakeTime
+= int(round((time
.time() - MakeContiue
)))
2166 # Check for build error, and raise exception if one
2167 # has been signaled.
2169 if BuildTask
.HasError():
2170 self
.invalidateHash()
2171 EdkLogger
.error("build", BUILD_ERROR
, "Failed to build module", ExtraData
=GlobalData
.gBuildingModule
)
2173 # Create MAP file when Load Fix Address is enabled.
2174 if self
.Target
in ["", "all", "fds"]:
2175 for Arch
in Wa
.ArchList
:
2177 # Check whether the set fix address is above 4G for 32bit image.
2179 if (Arch
== 'IA32' or Arch
== 'ARM') and self
.LoadFixAddress
!= 0xFFFFFFFFFFFFFFFF and self
.LoadFixAddress
>= 0x100000000:
2180 EdkLogger
.error("build", PARAMETER_INVALID
, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
2185 for Pa
in Wa
.AutoGenObjectList
:
2186 for Ma
in Pa
.ModuleAutoGenList
:
2189 if not Ma
.IsLibrary
:
2190 ModuleList
[Ma
.Guid
.upper()] = Ma
2192 # Rebase module to the preferred memory address before GenFds
2195 if self
.LoadFixAddress
!= 0:
2196 self
._CollectModuleMapBuffer
(MapBuffer
, ModuleList
)
2200 # Generate FD image if there's a FDF file found
2202 GenFdsStart
= time
.time()
2203 if GenFdsApi(Wa
.GenFdsCommandDict
, self
.Db
):
2204 EdkLogger
.error("build", COMMAND_FAILURE
)
2207 # Create MAP file for all platform FVs after GenFds.
2209 self
._CollectFvMapBuffer
(MapBuffer
, Wa
, ModuleList
)
2210 self
.GenFdsTime
+= int(round((time
.time() - GenFdsStart
)))
2212 # Save MAP buffer into MAP file.
2214 self
._SaveMapFile
(MapBuffer
, Wa
)
2215 self
.invalidateHash()
2217 ## Generate GuidedSectionTools.txt in the FV directories.
2219 def CreateGuidedSectionToolsFile(self
):
2220 for BuildTarget
in self
.BuildTargetList
:
2221 for ToolChain
in self
.ToolChainList
:
2222 Wa
= WorkspaceAutoGen(
2239 if not os
.path
.exists(FvDir
):
2242 for Arch
in self
.ArchList
:
2243 # Build up the list of supported architectures for this build
2244 prefix
= '%s_%s_%s_' % (BuildTarget
, ToolChain
, Arch
)
2246 # Look through the tool definitions for GUIDed tools
2248 for (attrib
, value
) in self
.ToolDef
.ToolsDefTxtDictionary
.items():
2249 if attrib
.upper().endswith('_GUID'):
2250 split
= attrib
.split('_')
2251 thisPrefix
= '_'.join(split
[0:3]) + '_'
2252 if thisPrefix
== prefix
:
2253 guid
= self
.ToolDef
.ToolsDefTxtDictionary
[attrib
]
2256 path
= '_'.join(split
[0:4]) + '_PATH'
2257 path
= self
.ToolDef
.ToolsDefTxtDictionary
[path
]
2258 path
= self
.GetFullPathOfTool(path
)
2259 guidAttribs
.append((guid
, toolName
, path
))
2261 # Write out GuidedSecTools.txt
2262 toolsFile
= os
.path
.join(FvDir
, 'GuidedSectionTools.txt')
2263 toolsFile
= open(toolsFile
, 'wt')
2264 for guidedSectionTool
in guidAttribs
:
2265 print(' '.join(guidedSectionTool
), file=toolsFile
)
2268 ## Returns the full path of the tool.
2270 def GetFullPathOfTool (self
, tool
):
2271 if os
.path
.exists(tool
):
2272 return os
.path
.realpath(tool
)
2274 # We need to search for the tool using the
2275 # PATH environment variable.
2276 for dirInPath
in os
.environ
['PATH'].split(os
.pathsep
):
2277 foundPath
= os
.path
.join(dirInPath
, tool
)
2278 if os
.path
.exists(foundPath
):
2279 return os
.path
.realpath(foundPath
)
2281 # If the tool was not found in the path then we just return
2285 ## Launch the module or platform build
2288 if not self
.ModuleFile
:
2289 if not self
.SpawnMode
or self
.Target
not in ["", "all"]:
2290 self
.SpawnMode
= False
2291 self
._BuildPlatform
()
2293 self
._MultiThreadBuildPlatform
()
2294 self
.CreateGuidedSectionToolsFile()
2296 self
.SpawnMode
= False
2299 if self
.Target
== 'cleanall':
2300 RemoveDirectory(os
.path
.dirname(GlobalData
.gDatabasePath
), True)
2302 def CreateAsBuiltInf(self
):
2303 for Module
in self
.BuildModules
:
2304 Module
.CreateAsBuiltInf()
2306 def UpdateBuildCache(self
):
2309 for Module
in self
.BuildModules
:
2310 Module
.CopyModuleToCache()
2311 all_mod_set
.add(Module
)
2312 for Module
in self
.HashSkipModules
:
2313 Module
.CopyModuleToCache()
2314 all_mod_set
.add(Module
)
2315 for Module
in all_mod_set
:
2316 for lib
in Module
.LibraryAutoGenList
:
2317 all_lib_set
.add(lib
)
2318 for lib
in all_lib_set
:
2319 lib
.CopyModuleToCache()
2322 self
.HashSkipModules
= []
2323 ## Do some clean-up works when error occurred
2324 def Relinquish(self
):
2325 OldLogLevel
= EdkLogger
.GetLevel()
2326 EdkLogger
.SetLevel(EdkLogger
.ERROR
)
2327 Utils
.Progressor
.Abort()
2328 if self
.SpawnMode
== True:
2330 EdkLogger
.SetLevel(OldLogLevel
)
2332 def ParseDefines(DefineList
=[]):
2334 if DefineList
is not None:
2335 for Define
in DefineList
:
2336 DefineTokenList
= Define
.split("=", 1)
2337 if not GlobalData
.gMacroNamePattern
.match(DefineTokenList
[0]):
2338 EdkLogger
.error('build', FORMAT_INVALID
,
2339 "The macro name must be in the pattern [A-Z][A-Z0-9_]*",
2340 ExtraData
=DefineTokenList
[0])
2342 if len(DefineTokenList
) == 1:
2343 DefineDict
[DefineTokenList
[0]] = "TRUE"
2345 DefineDict
[DefineTokenList
[0]] = DefineTokenList
[1].strip()
2350 def LogBuildTime(Time
):
2353 TimeDur
= time
.gmtime(Time
)
2354 if TimeDur
.tm_yday
> 1:
2355 TimeDurStr
= time
.strftime("%H:%M:%S", TimeDur
) + ", %d day(s)" % (TimeDur
.tm_yday
- 1)
2357 TimeDurStr
= time
.strftime("%H:%M:%S", TimeDur
)
2362 ThreadNumber
= BuildOption
.ThreadNumber
2363 if ThreadNumber
is None:
2364 ThreadNumber
= TargetTxt
.TargetTxtDictionary
[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER
]
2365 if ThreadNumber
== '':
2368 ThreadNumber
= int(ThreadNumber
, 0)
2370 if ThreadNumber
== 0:
2372 ThreadNumber
= multiprocessing
.cpu_count()
2373 except (ImportError, NotImplementedError):
2376 ## Tool entrance method
2378 # This method mainly dispatch specific methods per the command line options.
2379 # If no error found, return zero value so the caller of this tool can know
2380 # if it's executed successfully or not.
2382 # @retval 0 Tool was successful
2383 # @retval 1 Tool failed
2385 LogQMaxSize
= ThreadNum() * 10
2387 StartTime
= time
.time()
2390 # Create a log Queue
2392 LogQ
= mp
.Queue(LogQMaxSize
)
2393 # Initialize log system
2394 EdkLogger
.LogClientInitialize(LogQ
)
2395 GlobalData
.gCommand
= sys
.argv
[1:]
2397 # Parse the options and args
2399 Option
, Target
= BuildOption
, BuildTarget
2400 GlobalData
.gOptions
= Option
2401 GlobalData
.gCaseInsensitive
= Option
.CaseInsensitive
2404 LogLevel
= EdkLogger
.INFO
2405 if Option
.verbose
is not None:
2406 EdkLogger
.SetLevel(EdkLogger
.VERBOSE
)
2407 LogLevel
= EdkLogger
.VERBOSE
2408 elif Option
.quiet
is not None:
2409 EdkLogger
.SetLevel(EdkLogger
.QUIET
)
2410 LogLevel
= EdkLogger
.QUIET
2411 elif Option
.debug
is not None:
2412 EdkLogger
.SetLevel(Option
.debug
+ 1)
2413 LogLevel
= Option
.debug
+ 1
2415 EdkLogger
.SetLevel(EdkLogger
.INFO
)
2417 if Option
.WarningAsError
== True:
2418 EdkLogger
.SetWarningAsError()
2419 Log_Agent
= LogAgent(LogQ
,LogLevel
,Option
.LogFile
)
2422 if platform
.platform().find("Windows") >= 0:
2423 GlobalData
.gIsWindows
= True
2425 GlobalData
.gIsWindows
= False
2427 EdkLogger
.quiet("Build environment: %s" % platform
.platform())
2428 EdkLogger
.quiet(time
.strftime("Build start time: %H:%M:%S, %b.%d %Y\n", time
.localtime()));
2433 if len(Target
) == 0:
2435 elif len(Target
) >= 2:
2436 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, "More than one targets are not supported.",
2437 ExtraData
="Please select one of: %s" % (' '.join(gSupportedTarget
)))
2439 Target
= Target
[0].lower()
2441 if Target
not in gSupportedTarget
:
2442 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, "Not supported target [%s]." % Target
,
2443 ExtraData
="Please select one of: %s" % (' '.join(gSupportedTarget
)))
2446 # Check environment variable: EDK_TOOLS_PATH, WORKSPACE, PATH
2449 GlobalData
.gCommandLineDefines
.update(ParseDefines(Option
.Macros
))
2451 Workspace
= os
.getenv("WORKSPACE")
2453 # Get files real name in workspace dir
2455 GlobalData
.gAllFiles
= Utils
.DirCache(Workspace
)
2457 WorkingDirectory
= os
.getcwd()
2458 if not Option
.ModuleFile
:
2459 FileList
= glob
.glob(os
.path
.normpath(os
.path
.join(WorkingDirectory
, '*.inf')))
2460 FileNum
= len(FileList
)
2462 EdkLogger
.error("build", OPTION_NOT_SUPPORTED
, "There are %d INF files in %s." % (FileNum
, WorkingDirectory
),
2463 ExtraData
="Please use '-m <INF_FILE_PATH>' switch to choose one.")
2465 Option
.ModuleFile
= NormFile(FileList
[0], Workspace
)
2467 if Option
.ModuleFile
:
2468 if os
.path
.isabs (Option
.ModuleFile
):
2469 if os
.path
.normcase (os
.path
.normpath(Option
.ModuleFile
)).find (Workspace
) == 0:
2470 Option
.ModuleFile
= NormFile(os
.path
.normpath(Option
.ModuleFile
), Workspace
)
2471 Option
.ModuleFile
= PathClass(Option
.ModuleFile
, Workspace
)
2472 ErrorCode
, ErrorInfo
= Option
.ModuleFile
.Validate(".inf", False)
2474 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
2476 if Option
.PlatformFile
is not None:
2477 if os
.path
.isabs (Option
.PlatformFile
):
2478 if os
.path
.normcase (os
.path
.normpath(Option
.PlatformFile
)).find (Workspace
) == 0:
2479 Option
.PlatformFile
= NormFile(os
.path
.normpath(Option
.PlatformFile
), Workspace
)
2480 Option
.PlatformFile
= PathClass(Option
.PlatformFile
, Workspace
)
2482 if Option
.FdfFile
is not None:
2483 if os
.path
.isabs (Option
.FdfFile
):
2484 if os
.path
.normcase (os
.path
.normpath(Option
.FdfFile
)).find (Workspace
) == 0:
2485 Option
.FdfFile
= NormFile(os
.path
.normpath(Option
.FdfFile
), Workspace
)
2486 Option
.FdfFile
= PathClass(Option
.FdfFile
, Workspace
)
2487 ErrorCode
, ErrorInfo
= Option
.FdfFile
.Validate(".fdf", False)
2489 EdkLogger
.error("build", ErrorCode
, ExtraData
=ErrorInfo
)
2491 if Option
.Flag
is not None and Option
.Flag
not in ['-c', '-s']:
2492 EdkLogger
.error("build", OPTION_VALUE_INVALID
, "UNI flag must be one of -c or -s")
2494 MyBuild
= Build(Target
, Workspace
, Option
,LogQ
)
2495 GlobalData
.gCommandLineDefines
['ARCH'] = ' '.join(MyBuild
.ArchList
)
2496 if not (MyBuild
.LaunchPrebuildFlag
and os
.path
.exists(MyBuild
.PlatformBuildPath
)):
2500 # All job done, no error found and no exception raised
2503 except FatalError
as X
:
2504 if MyBuild
is not None:
2505 # for multi-thread build exits safely
2506 MyBuild
.Relinquish()
2507 if Option
is not None and Option
.debug
is not None:
2508 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2509 ReturnCode
= X
.args
[0]
2510 except Warning as X
:
2511 # error from Fdf parser
2512 if MyBuild
is not None:
2513 # for multi-thread build exits safely
2514 MyBuild
.Relinquish()
2515 if Option
is not None and Option
.debug
is not None:
2516 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2518 EdkLogger
.error(X
.ToolName
, FORMAT_INVALID
, File
=X
.FileName
, Line
=X
.LineNumber
, ExtraData
=X
.Message
, RaiseError
=False)
2519 ReturnCode
= FORMAT_INVALID
2520 except KeyboardInterrupt:
2521 if MyBuild
is not None:
2523 # for multi-thread build exits safely
2524 MyBuild
.Relinquish()
2525 ReturnCode
= ABORT_ERROR
2526 if Option
is not None and Option
.debug
is not None:
2527 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2529 if MyBuild
is not None:
2530 # for multi-thread build exits safely
2531 MyBuild
.Relinquish()
2533 # try to get the meta-file from the object causing exception
2534 Tb
= sys
.exc_info()[-1]
2535 MetaFile
= GlobalData
.gProcessingFile
2536 while Tb
is not None:
2537 if 'self' in Tb
.tb_frame
.f_locals
and hasattr(Tb
.tb_frame
.f_locals
['self'], 'MetaFile'):
2538 MetaFile
= Tb
.tb_frame
.f_locals
['self'].MetaFile
2543 "Unknown fatal error when processing [%s]" % MetaFile
,
2544 ExtraData
="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR
,
2547 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
2548 ReturnCode
= CODE_ERROR
2550 Utils
.Progressor
.Abort()
2551 Utils
.ClearDuplicatedInf()
2555 MyBuild
.LaunchPostbuild()
2558 Conclusion
= "Failed"
2559 elif ReturnCode
== ABORT_ERROR
:
2560 Conclusion
= "Aborted"
2562 Conclusion
= "Failed"
2563 FinishTime
= time
.time()
2564 BuildDuration
= time
.gmtime(int(round(FinishTime
- StartTime
)))
2565 BuildDurationStr
= ""
2566 if BuildDuration
.tm_yday
> 1:
2567 BuildDurationStr
= time
.strftime("%H:%M:%S", BuildDuration
) + ", %d day(s)" % (BuildDuration
.tm_yday
- 1)
2569 BuildDurationStr
= time
.strftime("%H:%M:%S", BuildDuration
)
2570 if MyBuild
is not None:
2572 MyBuild
.BuildReport
.GenerateReport(BuildDurationStr
, LogBuildTime(MyBuild
.AutoGenTime
), LogBuildTime(MyBuild
.MakeTime
), LogBuildTime(MyBuild
.GenFdsTime
))
2574 EdkLogger
.SetLevel(EdkLogger
.QUIET
)
2575 EdkLogger
.quiet("\n- %s -" % Conclusion
)
2576 EdkLogger
.quiet(time
.strftime("Build end time: %H:%M:%S, %b.%d %Y", time
.localtime()))
2577 EdkLogger
.quiet("Build total time: %s\n" % BuildDurationStr
)
2582 if __name__
== '__main__':
2584 mp
.set_start_method('spawn')
2588 ## 0-127 is a safe return range, and 1 is a standard default error
2589 if r
< 0 or r
> 127: r
= 1