]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/AutoGen/BuildEngine.py
BaseTools: Replace BSD License with BSD+Patent License
[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / BuildEngine.py
1 ## @file
2 # The engine for building files
3 #
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # SPDX-License-Identifier: BSD-2-Clause-Patent
6 #
7
8 ##
9 # Import Modules
10 #
11 from __future__ import print_function
12 import Common.LongFilePathOs as os
13 import re
14 import copy
15 import string
16 from Common.LongFilePathSupport import OpenLongFilePath as open
17
18 from Common.GlobalData import *
19 from Common.BuildToolError import *
20 from Common.Misc import tdict, PathClass
21 from Common.StringUtils import NormPath
22 from Common.DataType import *
23
24 import Common.EdkLogger as EdkLogger
25
26 ## Convert file type to file list macro name
27 #
28 # @param FileType The name of file type
29 #
30 # @retval string The name of macro
31 #
32 def FileListMacro(FileType):
33 return "%sS" % FileType.replace("-", "_").upper()
34
35 ## Convert file type to list file macro name
36 #
37 # @param FileType The name of file type
38 #
39 # @retval string The name of macro
40 #
41 def ListFileMacro(FileType):
42 return "%s_LIST" % FileListMacro(FileType)
43
44 class TargetDescBlock(object):
45 def __init__(self, Inputs, Outputs, Commands, Dependencies):
46 self.InitWorker(Inputs, Outputs, Commands, Dependencies)
47
48 def InitWorker(self, Inputs, Outputs, Commands, Dependencies):
49 self.Inputs = Inputs
50 self.Outputs = Outputs
51 self.Commands = Commands
52 self.Dependencies = Dependencies
53 if self.Outputs:
54 self.Target = self.Outputs[0]
55 else:
56 self.Target = None
57
58 def __str__(self):
59 return self.Target.Path
60
61 def __hash__(self):
62 return hash(self.Target.Path)
63
64 def __eq__(self, Other):
65 if isinstance(Other, type(self)):
66 return Other.Target.Path == self.Target.Path
67 else:
68 return str(Other) == self.Target.Path
69
70 def AddInput(self, Input):
71 if Input not in self.Inputs:
72 self.Inputs.append(Input)
73
74 def IsMultipleInput(self):
75 return len(self.Inputs) > 1
76
77 ## Class for one build rule
78 #
79 # This represents a build rule which can give out corresponding command list for
80 # building the given source file(s). The result can be used for generating the
81 # target for makefile.
82 #
83 class FileBuildRule:
84 INC_LIST_MACRO = "INC_LIST"
85 INC_MACRO = "INC"
86
87 ## constructor
88 #
89 # @param Input The dictionary representing input file(s) for a rule
90 # @param Output The list representing output file(s) for a rule
91 # @param Command The list containing commands to generate the output from input
92 #
93 def __init__(self, Type, Input, Output, Command, ExtraDependency=None):
94 # The Input should not be empty
95 if not Input:
96 Input = []
97 if not Output:
98 Output = []
99 if not Command:
100 Command = []
101
102 self.FileListMacro = FileListMacro(Type)
103 self.ListFileMacro = ListFileMacro(Type)
104 self.IncListFileMacro = self.INC_LIST_MACRO
105
106 self.SourceFileType = Type
107 # source files listed not in TAB_STAR or "?" pattern format
108 if not ExtraDependency:
109 self.ExtraSourceFileList = []
110 else:
111 self.ExtraSourceFileList = ExtraDependency
112
113 #
114 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST.
115 # If found, generate a file to keep the input files used to get over the
116 # limitation of command line length
117 #
118 self.MacroList = []
119 self.CommandList = []
120 for CmdLine in Command:
121 self.MacroList.extend(gMacroRefPattern.findall(CmdLine))
122 # replace path separator with native one
123 self.CommandList.append(CmdLine)
124
125 # Indicate what should be generated
126 if self.FileListMacro in self.MacroList:
127 self.GenFileListMacro = True
128 else:
129 self.GenFileListMacro = False
130
131 if self.ListFileMacro in self.MacroList:
132 self.GenListFile = True
133 self.GenFileListMacro = True
134 else:
135 self.GenListFile = False
136
137 if self.INC_LIST_MACRO in self.MacroList:
138 self.GenIncListFile = True
139 else:
140 self.GenIncListFile = False
141
142 # Check input files
143 self.IsMultipleInput = False
144 self.SourceFileExtList = set()
145 for File in Input:
146 Base, Ext = os.path.splitext(File)
147 if Base.find(TAB_STAR) >= 0:
148 # There's TAB_STAR in the file name
149 self.IsMultipleInput = True
150 self.GenFileListMacro = True
151 elif Base.find("?") < 0:
152 # There's no TAB_STAR and "?" in file name
153 self.ExtraSourceFileList.append(File)
154 continue
155 self.SourceFileExtList.add(Ext)
156
157 # Check output files
158 self.DestFileList = []
159 for File in Output:
160 self.DestFileList.append(File)
161
162 # All build targets generated by this rule for a module
163 self.BuildTargets = {}
164
165 ## str() function support
166 #
167 # @retval string
168 #
169 def __str__(self):
170 SourceString = ""
171 SourceString += " %s %s %s" % (self.SourceFileType, " ".join(self.SourceFileExtList), self.ExtraSourceFileList)
172 DestString = ", ".join(self.DestFileList)
173 CommandString = "\n\t".join(self.CommandList)
174 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString)
175
176 def Instantiate(self, Macros={}):
177 NewRuleObject = copy.copy(self)
178 NewRuleObject.BuildTargets = {}
179 NewRuleObject.DestFileList = []
180 for File in self.DestFileList:
181 NewRuleObject.DestFileList.append(PathClass(NormPath(File, Macros)))
182 return NewRuleObject
183
184 ## Apply the rule to given source file(s)
185 #
186 # @param SourceFile One file or a list of files to be built
187 # @param RelativeToDir The relative path of the source file
188 # @param PathSeparator Path separator
189 #
190 # @retval tuple (Source file in full path, List of individual sourcefiles, Destination file, List of build commands)
191 #
192 def Apply(self, SourceFile, BuildRuleOrder=None):
193 if not self.CommandList or not self.DestFileList:
194 return None
195
196 # source file
197 if self.IsMultipleInput:
198 SrcFileName = ""
199 SrcFileBase = ""
200 SrcFileExt = ""
201 SrcFileDir = ""
202 SrcPath = ""
203 # SourceFile must be a list
204 SrcFile = "$(%s)" % self.FileListMacro
205 else:
206 SrcFileName, SrcFileBase, SrcFileExt = SourceFile.Name, SourceFile.BaseName, SourceFile.Ext
207 if SourceFile.Root:
208 SrcFileDir = SourceFile.SubDir
209 if SrcFileDir == "":
210 SrcFileDir = "."
211 else:
212 SrcFileDir = "."
213 SrcFile = SourceFile.Path
214 SrcPath = SourceFile.Dir
215
216 # destination file (the first one)
217 if self.DestFileList:
218 DestFile = self.DestFileList[0].Path
219 DestPath = self.DestFileList[0].Dir
220 DestFileName = self.DestFileList[0].Name
221 DestFileBase, DestFileExt = self.DestFileList[0].BaseName, self.DestFileList[0].Ext
222 else:
223 DestFile = ""
224 DestPath = ""
225 DestFileName = ""
226 DestFileBase = ""
227 DestFileExt = ""
228
229 BuildRulePlaceholderDict = {
230 # source file
231 "src" : SrcFile,
232 "s_path" : SrcPath,
233 "s_dir" : SrcFileDir,
234 "s_name" : SrcFileName,
235 "s_base" : SrcFileBase,
236 "s_ext" : SrcFileExt,
237 # destination file
238 "dst" : DestFile,
239 "d_path" : DestPath,
240 "d_name" : DestFileName,
241 "d_base" : DestFileBase,
242 "d_ext" : DestFileExt,
243 }
244
245 DstFile = []
246 for File in self.DestFileList:
247 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
248 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
249 DstFile.append(PathClass(File, IsBinary=True))
250
251 if DstFile[0] in self.BuildTargets:
252 TargetDesc = self.BuildTargets[DstFile[0]]
253 if BuildRuleOrder and SourceFile.Ext in BuildRuleOrder:
254 Index = BuildRuleOrder.index(SourceFile.Ext)
255 for Input in TargetDesc.Inputs:
256 if Input.Ext not in BuildRuleOrder or BuildRuleOrder.index(Input.Ext) > Index:
257 #
258 # Command line should be regenerated since some macros are different
259 #
260 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
261 TargetDesc.InitWorker([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
262 break
263 else:
264 TargetDesc.AddInput(SourceFile)
265 else:
266 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
267 TargetDesc = TargetDescBlock([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
268 TargetDesc.ListFileMacro = self.ListFileMacro
269 TargetDesc.FileListMacro = self.FileListMacro
270 TargetDesc.IncListFileMacro = self.IncListFileMacro
271 TargetDesc.GenFileListMacro = self.GenFileListMacro
272 TargetDesc.GenListFile = self.GenListFile
273 TargetDesc.GenIncListFile = self.GenIncListFile
274 self.BuildTargets[DstFile[0]] = TargetDesc
275 return TargetDesc
276
277 def _BuildCommand(self, Macros):
278 CommandList = []
279 for CommandString in self.CommandList:
280 CommandString = string.Template(CommandString).safe_substitute(Macros)
281 CommandString = string.Template(CommandString).safe_substitute(Macros)
282 CommandList.append(CommandString)
283 return CommandList
284
285 ## Class for build rules
286 #
287 # BuildRule class parses rules defined in a file or passed by caller, and converts
288 # the rule into FileBuildRule object.
289 #
290 class BuildRule:
291 _SectionHeader = "SECTIONHEADER"
292 _Section = "SECTION"
293 _SubSectionHeader = "SUBSECTIONHEADER"
294 _SubSection = "SUBSECTION"
295 _InputFile = "INPUTFILE"
296 _OutputFile = "OUTPUTFILE"
297 _ExtraDependency = "EXTRADEPENDENCY"
298 _Command = "COMMAND"
299 _UnknownSection = "UNKNOWNSECTION"
300
301 _SubSectionList = [_InputFile, _OutputFile, _Command]
302
303 _PATH_SEP = "(+)"
304 _FileTypePattern = re.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$")
305 _BinaryFileRule = FileBuildRule(TAB_DEFAULT_BINARY_FILE, [], [os.path.join("$(OUTPUT_DIR)", "${s_name}")],
306 ["$(CP) ${src} ${dst}"], [])
307
308 ## Constructor
309 #
310 # @param File The file containing build rules in a well defined format
311 # @param Content The string list of build rules in a well defined format
312 # @param LineIndex The line number from which the parsing will begin
313 # @param SupportedFamily The list of supported tool chain families
314 #
315 def __init__(self, File=None, Content=None, LineIndex=0, SupportedFamily=[TAB_COMPILER_MSFT, "INTEL", "GCC", "RVCT"]):
316 self.RuleFile = File
317 # Read build rules from file if it's not none
318 if File is not None:
319 try:
320 self.RuleContent = open(File, 'r').readlines()
321 except:
322 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File)
323 elif Content is not None:
324 self.RuleContent = Content
325 else:
326 EdkLogger.error("build", PARAMETER_MISSING, ExtraData="No rule file or string given")
327
328 self.SupportedToolChainFamilyList = SupportedFamily
329 self.RuleDatabase = tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object}
330 self.Ext2FileType = {} # {ext : file-type}
331 self.FileTypeList = set()
332
333 self._LineIndex = LineIndex
334 self._State = ""
335 self._RuleInfo = tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}}
336 self._FileType = ''
337 self._BuildTypeList = set()
338 self._ArchList = set()
339 self._FamilyList = []
340 self._TotalToolChainFamilySet = set()
341 self._RuleObjectList = [] # FileBuildRule object list
342 self._FileVersion = ""
343
344 self.Parse()
345
346 # some intrinsic rules
347 self.RuleDatabase[TAB_DEFAULT_BINARY_FILE, TAB_COMMON, TAB_COMMON, TAB_COMMON] = self._BinaryFileRule
348 self.FileTypeList.add(TAB_DEFAULT_BINARY_FILE)
349
350 ## Parse the build rule strings
351 def Parse(self):
352 self._State = self._Section
353 for Index in range(self._LineIndex, len(self.RuleContent)):
354 # Clean up the line and replace path separator with native one
355 Line = self.RuleContent[Index].strip().replace(self._PATH_SEP, os.path.sep)
356 self.RuleContent[Index] = Line
357
358 # find the build_rule_version
359 if Line and Line[0] == "#" and Line.find(TAB_BUILD_RULE_VERSION) != -1:
360 if Line.find("=") != -1 and Line.find("=") < (len(Line) - 1) and (Line[(Line.find("=") + 1):]).split():
361 self._FileVersion = (Line[(Line.find("=") + 1):]).split()[0]
362 # skip empty or comment line
363 if Line == "" or Line[0] == "#":
364 continue
365
366 # find out section header, enclosed by []
367 if Line[0] == '[' and Line[-1] == ']':
368 # merge last section information into rule database
369 self.EndOfSection()
370 self._State = self._SectionHeader
371 # find out sub-section header, enclosed by <>
372 elif Line[0] == '<' and Line[-1] == '>':
373 if self._State != self._UnknownSection:
374 self._State = self._SubSectionHeader
375
376 # call section handler to parse each (sub)section
377 self._StateHandler[self._State](self, Index)
378 # merge last section information into rule database
379 self.EndOfSection()
380
381 ## Parse definitions under a section
382 #
383 # @param LineIndex The line index of build rule text
384 #
385 def ParseSection(self, LineIndex):
386 pass
387
388 ## Parse definitions under a subsection
389 #
390 # @param LineIndex The line index of build rule text
391 #
392 def ParseSubSection(self, LineIndex):
393 # currently nothing here
394 pass
395
396 ## Placeholder for not supported sections
397 #
398 # @param LineIndex The line index of build rule text
399 #
400 def SkipSection(self, LineIndex):
401 pass
402
403 ## Merge section information just got into rule database
404 def EndOfSection(self):
405 Database = self.RuleDatabase
406 # if there's specific toolchain family, 'COMMON' doesn't make sense any more
407 if len(self._TotalToolChainFamilySet) > 1 and TAB_COMMON in self._TotalToolChainFamilySet:
408 self._TotalToolChainFamilySet.remove(TAB_COMMON)
409 for Family in self._TotalToolChainFamilySet:
410 Input = self._RuleInfo[Family, self._InputFile]
411 Output = self._RuleInfo[Family, self._OutputFile]
412 Command = self._RuleInfo[Family, self._Command]
413 ExtraDependency = self._RuleInfo[Family, self._ExtraDependency]
414
415 BuildRule = FileBuildRule(self._FileType, Input, Output, Command, ExtraDependency)
416 for BuildType in self._BuildTypeList:
417 for Arch in self._ArchList:
418 Database[self._FileType, BuildType, Arch, Family] = BuildRule
419 for FileExt in BuildRule.SourceFileExtList:
420 self.Ext2FileType[FileExt] = self._FileType
421
422 ## Parse section header
423 #
424 # @param LineIndex The line index of build rule text
425 #
426 def ParseSectionHeader(self, LineIndex):
427 self._RuleInfo = tdict(True, 2)
428 self._BuildTypeList = set()
429 self._ArchList = set()
430 self._FamilyList = []
431 self._TotalToolChainFamilySet = set()
432 FileType = ''
433 RuleNameList = self.RuleContent[LineIndex][1:-1].split(',')
434 for RuleName in RuleNameList:
435 Arch = TAB_COMMON
436 BuildType = TAB_COMMON
437 TokenList = [Token.strip().upper() for Token in RuleName.split('.')]
438 # old format: Build.File-Type
439 if TokenList[0] == "BUILD":
440 if len(TokenList) == 1:
441 EdkLogger.error("build", FORMAT_INVALID, "Invalid rule section",
442 File=self.RuleFile, Line=LineIndex + 1,
443 ExtraData=self.RuleContent[LineIndex])
444
445 FileType = TokenList[1]
446 if FileType == '':
447 EdkLogger.error("build", FORMAT_INVALID, "No file type given",
448 File=self.RuleFile, Line=LineIndex + 1,
449 ExtraData=self.RuleContent[LineIndex])
450 if self._FileTypePattern.match(FileType) is None:
451 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
452 ExtraData="Only character, number (non-first character), '_' and '-' are allowed in file type")
453 # new format: File-Type.Build-Type.Arch
454 else:
455 if FileType == '':
456 FileType = TokenList[0]
457 elif FileType != TokenList[0]:
458 EdkLogger.error("build", FORMAT_INVALID,
459 "Different file types are not allowed in the same rule section",
460 File=self.RuleFile, Line=LineIndex + 1,
461 ExtraData=self.RuleContent[LineIndex])
462 if len(TokenList) > 1:
463 BuildType = TokenList[1]
464 if len(TokenList) > 2:
465 Arch = TokenList[2]
466 self._BuildTypeList.add(BuildType)
467 self._ArchList.add(Arch)
468
469 if TAB_COMMON in self._BuildTypeList and len(self._BuildTypeList) > 1:
470 EdkLogger.error("build", FORMAT_INVALID,
471 "Specific build types must not be mixed with common one",
472 File=self.RuleFile, Line=LineIndex + 1,
473 ExtraData=self.RuleContent[LineIndex])
474 if TAB_COMMON in self._ArchList and len(self._ArchList) > 1:
475 EdkLogger.error("build", FORMAT_INVALID,
476 "Specific ARCH must not be mixed with common one",
477 File=self.RuleFile, Line=LineIndex + 1,
478 ExtraData=self.RuleContent[LineIndex])
479
480 self._FileType = FileType
481 self._State = self._Section
482 self.FileTypeList.add(FileType)
483
484 ## Parse sub-section header
485 #
486 # @param LineIndex The line index of build rule text
487 #
488 def ParseSubSectionHeader(self, LineIndex):
489 SectionType = ""
490 List = self.RuleContent[LineIndex][1:-1].split(',')
491 FamilyList = []
492 for Section in List:
493 TokenList = Section.split('.')
494 Type = TokenList[0].strip().upper()
495
496 if SectionType == "":
497 SectionType = Type
498 elif SectionType != Type:
499 EdkLogger.error("build", FORMAT_INVALID,
500 "Two different section types are not allowed in the same sub-section",
501 File=self.RuleFile, Line=LineIndex + 1,
502 ExtraData=self.RuleContent[LineIndex])
503
504 if len(TokenList) > 1:
505 Family = TokenList[1].strip().upper()
506 else:
507 Family = TAB_COMMON
508
509 if Family not in FamilyList:
510 FamilyList.append(Family)
511
512 self._FamilyList = FamilyList
513 self._TotalToolChainFamilySet.update(FamilyList)
514 self._State = SectionType.upper()
515 if TAB_COMMON in FamilyList and len(FamilyList) > 1:
516 EdkLogger.error("build", FORMAT_INVALID,
517 "Specific tool chain family should not be mixed with general one",
518 File=self.RuleFile, Line=LineIndex + 1,
519 ExtraData=self.RuleContent[LineIndex])
520 if self._State not in self._StateHandler:
521 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
522 ExtraData="Unknown subsection: %s" % self.RuleContent[LineIndex])
523 ## Parse <InputFile> sub-section
524 #
525 # @param LineIndex The line index of build rule text
526 #
527 def ParseInputFileSubSection(self, LineIndex):
528 FileList = [File.strip() for File in self.RuleContent[LineIndex].split(",")]
529 for ToolChainFamily in self._FamilyList:
530 if self._RuleInfo[ToolChainFamily, self._State] is None:
531 self._RuleInfo[ToolChainFamily, self._State] = []
532 self._RuleInfo[ToolChainFamily, self._State].extend(FileList)
533
534 ## Parse <ExtraDependency> sub-section
535 ## Parse <OutputFile> sub-section
536 ## Parse <Command> sub-section
537 #
538 # @param LineIndex The line index of build rule text
539 #
540 def ParseCommonSubSection(self, LineIndex):
541 for ToolChainFamily in self._FamilyList:
542 if self._RuleInfo[ToolChainFamily, self._State] is None:
543 self._RuleInfo[ToolChainFamily, self._State] = []
544 self._RuleInfo[ToolChainFamily, self._State].append(self.RuleContent[LineIndex])
545
546 ## Get a build rule via [] operator
547 #
548 # @param FileExt The extension of a file
549 # @param ToolChainFamily The tool chain family name
550 # @param BuildVersion The build version number. TAB_STAR means any rule
551 # is applicable.
552 #
553 # @retval FileType The file type string
554 # @retval FileBuildRule The object of FileBuildRule
555 #
556 # Key = (FileExt, ModuleType, Arch, ToolChainFamily)
557 def __getitem__(self, Key):
558 if not Key:
559 return None
560
561 if Key[0] in self.Ext2FileType:
562 Type = self.Ext2FileType[Key[0]]
563 elif Key[0].upper() in self.FileTypeList:
564 Type = Key[0].upper()
565 else:
566 return None
567
568 if len(Key) > 1:
569 Key = (Type,) + Key[1:]
570 else:
571 Key = (Type,)
572 return self.RuleDatabase[Key]
573
574 _StateHandler = {
575 _SectionHeader : ParseSectionHeader,
576 _Section : ParseSection,
577 _SubSectionHeader : ParseSubSectionHeader,
578 _SubSection : ParseSubSection,
579 _InputFile : ParseInputFileSubSection,
580 _OutputFile : ParseCommonSubSection,
581 _ExtraDependency : ParseCommonSubSection,
582 _Command : ParseCommonSubSection,
583 _UnknownSection : SkipSection,
584 }
585
586 # This acts like the main() function for the script, unless it is 'import'ed into another
587 # script.
588 if __name__ == '__main__':
589 import sys
590 EdkLogger.Initialize()
591 if len(sys.argv) > 1:
592 Br = BuildRule(sys.argv[1])
593 print(str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", TAB_COMPILER_MSFT][1]))
594 print()
595 print(str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "INTEL"][1]))
596 print()
597 print(str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "GCC"][1]))
598 print()
599 print(str(Br[".ac", "ACPI_TABLE", "IA32", TAB_COMPILER_MSFT][1]))
600 print()
601 print(str(Br[".h", "ACPI_TABLE", "IA32", "INTEL"][1]))
602 print()
603 print(str(Br[".ac", "ACPI_TABLE", "IA32", TAB_COMPILER_MSFT][1]))
604 print()
605 print(str(Br[".s", SUP_MODULE_SEC, "IPF", "COMMON"][1]))
606 print()
607 print(str(Br[".s", SUP_MODULE_SEC][1]))
608