]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/AutoGen/BuildEngine.py
BaseTools: use predefined constants instead of local strings
[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / BuildEngine.py
1 ## @file
2 # The engine for building files
3 #
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import Common.LongFilePathOs as os
18 import re
19 import copy
20 import string
21 from Common.LongFilePathSupport import OpenLongFilePath as open
22
23 from Common.GlobalData import *
24 from Common.BuildToolError import *
25 from Common.Misc import tdict, PathClass
26 from Common.String import NormPath
27 from Common.DataType import *
28
29 import Common.EdkLogger as EdkLogger
30
31 ## Convert file type to file list macro name
32 #
33 # @param FileType The name of file type
34 #
35 # @retval string The name of macro
36 #
37 def FileListMacro(FileType):
38 return "%sS" % FileType.replace("-", "_").upper()
39
40 ## Convert file type to list file macro name
41 #
42 # @param FileType The name of file type
43 #
44 # @retval string The name of macro
45 #
46 def ListFileMacro(FileType):
47 return "%s_LIST" % FileListMacro(FileType)
48
49 class TargetDescBlock(object):
50 _Cache_ = {} # {TargetFile : TargetDescBlock object}
51
52 # Factory method
53 def __new__(Class, Inputs, Outputs, Commands, Dependencies):
54 if Outputs[0] in Class._Cache_:
55 Tdb = Class._Cache_[Outputs[0]]
56 for File in Inputs:
57 Tdb.AddInput(File)
58 else:
59 Tdb = super(TargetDescBlock, Class).__new__(Class)
60 Tdb._Init(Inputs, Outputs, Commands, Dependencies)
61 #Class._Cache_[Outputs[0]] = Tdb
62 return Tdb
63
64 def _Init(self, Inputs, Outputs, Commands, Dependencies):
65 self.Inputs = Inputs
66 self.Outputs = Outputs
67 self.Commands = Commands
68 self.Dependencies = Dependencies
69 if self.Outputs:
70 self.Target = self.Outputs[0]
71 else:
72 self.Target = None
73
74 def __str__(self):
75 return self.Target.Path
76
77 def __hash__(self):
78 return hash(self.Target.Path)
79
80 def __eq__(self, Other):
81 if type(Other) == type(self):
82 return Other.Target.Path == self.Target.Path
83 else:
84 return str(Other) == self.Target.Path
85
86 def AddInput(self, Input):
87 if Input not in self.Inputs:
88 self.Inputs.append(Input)
89
90 def IsMultipleInput(self):
91 return len(self.Inputs) > 1
92
93 @staticmethod
94 def Renew():
95 TargetDescBlock._Cache_ = {}
96
97 ## Class for one build rule
98 #
99 # This represents a build rule which can give out corresponding command list for
100 # building the given source file(s). The result can be used for generating the
101 # target for makefile.
102 #
103 class FileBuildRule:
104 INC_LIST_MACRO = "INC_LIST"
105 INC_MACRO = "INC"
106
107 ## constructor
108 #
109 # @param Input The dictionary represeting input file(s) for a rule
110 # @param Output The list represeting output file(s) for a rule
111 # @param Command The list containing commands to generate the output from input
112 #
113 def __init__(self, Type, Input, Output, Command, ExtraDependency=None):
114 # The Input should not be empty
115 if not Input:
116 Input = []
117 if not Output:
118 Output = []
119 if not Command:
120 Command = []
121
122 self.FileListMacro = FileListMacro(Type)
123 self.ListFileMacro = ListFileMacro(Type)
124 self.IncListFileMacro = self.INC_LIST_MACRO
125
126 self.SourceFileType = Type
127 # source files listed not in "*" or "?" pattern format
128 if not ExtraDependency:
129 self.ExtraSourceFileList = []
130 else:
131 self.ExtraSourceFileList = ExtraDependency
132
133 #
134 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST.
135 # If found, generate a file to keep the input files used to get over the
136 # limitation of command line length
137 #
138 self.MacroList = []
139 self.CommandList = []
140 for CmdLine in Command:
141 self.MacroList.extend(gMacroRefPattern.findall(CmdLine))
142 # replace path separator with native one
143 self.CommandList.append(CmdLine)
144
145 # Indicate what should be generated
146 if self.FileListMacro in self.MacroList:
147 self.GenFileListMacro = True
148 else:
149 self.GenFileListMacro = False
150
151 if self.ListFileMacro in self.MacroList:
152 self.GenListFile = True
153 self.GenFileListMacro = True
154 else:
155 self.GenListFile = False
156
157 if self.INC_LIST_MACRO in self.MacroList:
158 self.GenIncListFile = True
159 else:
160 self.GenIncListFile = False
161
162 # Check input files
163 self.IsMultipleInput = False
164 self.SourceFileExtList = set()
165 for File in Input:
166 Base, Ext = os.path.splitext(File)
167 if Base.find("*") >= 0:
168 # There's "*" in the file name
169 self.IsMultipleInput = True
170 self.GenFileListMacro = True
171 elif Base.find("?") < 0:
172 # There's no "*" and "?" in file name
173 self.ExtraSourceFileList.append(File)
174 continue
175 self.SourceFileExtList.add(Ext)
176
177 # Check output files
178 self.DestFileList = []
179 for File in Output:
180 self.DestFileList.append(File)
181
182 # All build targets generated by this rule for a module
183 self.BuildTargets = {}
184
185 ## str() function support
186 #
187 # @retval string
188 #
189 def __str__(self):
190 SourceString = ""
191 SourceString += " %s %s %s" % (self.SourceFileType, " ".join(self.SourceFileExtList), self.ExtraSourceFileList)
192 DestString = ", ".join(self.DestFileList)
193 CommandString = "\n\t".join(self.CommandList)
194 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString)
195
196 def Instantiate(self, Macros={}):
197 NewRuleObject = copy.copy(self)
198 NewRuleObject.BuildTargets = {}
199 NewRuleObject.DestFileList = []
200 for File in self.DestFileList:
201 NewRuleObject.DestFileList.append(PathClass(NormPath(File, Macros)))
202 return NewRuleObject
203
204 ## Apply the rule to given source file(s)
205 #
206 # @param SourceFile One file or a list of files to be built
207 # @param RelativeToDir The relative path of the source file
208 # @param PathSeparator Path separator
209 #
210 # @retval tuple (Source file in full path, List of individual sourcefiles, Destionation file, List of build commands)
211 #
212 def Apply(self, SourceFile, BuildRuleOrder=None):
213 if not self.CommandList or not self.DestFileList:
214 return None
215
216 # source file
217 if self.IsMultipleInput:
218 SrcFileName = ""
219 SrcFileBase = ""
220 SrcFileExt = ""
221 SrcFileDir = ""
222 SrcPath = ""
223 # SourceFile must be a list
224 SrcFile = "$(%s)" % self.FileListMacro
225 else:
226 SrcFileName, SrcFileBase, SrcFileExt = SourceFile.Name, SourceFile.BaseName, SourceFile.Ext
227 if SourceFile.Root:
228 SrcFileDir = SourceFile.SubDir
229 if SrcFileDir == "":
230 SrcFileDir = "."
231 else:
232 SrcFileDir = "."
233 SrcFile = SourceFile.Path
234 SrcPath = SourceFile.Dir
235
236 # destination file (the first one)
237 if self.DestFileList:
238 DestFile = self.DestFileList[0].Path
239 DestPath = self.DestFileList[0].Dir
240 DestFileName = self.DestFileList[0].Name
241 DestFileBase, DestFileExt = self.DestFileList[0].BaseName, self.DestFileList[0].Ext
242 else:
243 DestFile = ""
244 DestPath = ""
245 DestFileName = ""
246 DestFileBase = ""
247 DestFileExt = ""
248
249 BuildRulePlaceholderDict = {
250 # source file
251 "src" : SrcFile,
252 "s_path" : SrcPath,
253 "s_dir" : SrcFileDir,
254 "s_name" : SrcFileName,
255 "s_base" : SrcFileBase,
256 "s_ext" : SrcFileExt,
257 # destination file
258 "dst" : DestFile,
259 "d_path" : DestPath,
260 "d_name" : DestFileName,
261 "d_base" : DestFileBase,
262 "d_ext" : DestFileExt,
263 }
264
265 DstFile = []
266 for File in self.DestFileList:
267 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
268 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
269 DstFile.append(PathClass(File, IsBinary=True))
270
271 if DstFile[0] in self.BuildTargets:
272 TargetDesc = self.BuildTargets[DstFile[0]]
273 if BuildRuleOrder and SourceFile.Ext in BuildRuleOrder:
274 Index = BuildRuleOrder.index(SourceFile.Ext)
275 for Input in TargetDesc.Inputs:
276 if Input.Ext not in BuildRuleOrder or BuildRuleOrder.index(Input.Ext) > Index:
277 #
278 # Command line should be regenerated since some macros are different
279 #
280 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
281 TargetDesc._Init([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
282 break
283 else:
284 TargetDesc.AddInput(SourceFile)
285 else:
286 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
287 TargetDesc = TargetDescBlock([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
288 TargetDesc.ListFileMacro = self.ListFileMacro
289 TargetDesc.FileListMacro = self.FileListMacro
290 TargetDesc.IncListFileMacro = self.IncListFileMacro
291 TargetDesc.GenFileListMacro = self.GenFileListMacro
292 TargetDesc.GenListFile = self.GenListFile
293 TargetDesc.GenIncListFile = self.GenIncListFile
294 self.BuildTargets[DstFile[0]] = TargetDesc
295 return TargetDesc
296
297 def _BuildCommand(self, Macros):
298 CommandList = []
299 for CommandString in self.CommandList:
300 CommandString = string.Template(CommandString).safe_substitute(Macros)
301 CommandString = string.Template(CommandString).safe_substitute(Macros)
302 CommandList.append(CommandString)
303 return CommandList
304
305 ## Class for build rules
306 #
307 # BuildRule class parses rules defined in a file or passed by caller, and converts
308 # the rule into FileBuildRule object.
309 #
310 class BuildRule:
311 _SectionHeader = "SECTIONHEADER"
312 _Section = "SECTION"
313 _SubSectionHeader = "SUBSECTIONHEADER"
314 _SubSection = "SUBSECTION"
315 _InputFile = "INPUTFILE"
316 _OutputFile = "OUTPUTFILE"
317 _ExtraDependency = "EXTRADEPENDENCY"
318 _Command = "COMMAND"
319 _UnknownSection = "UNKNOWNSECTION"
320
321 _SubSectionList = [_InputFile, _OutputFile, _Command]
322
323 _PATH_SEP = "(+)"
324 _FileTypePattern = re.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$")
325 _BinaryFileRule = FileBuildRule(TAB_DEFAULT_BINARY_FILE, [], [os.path.join("$(OUTPUT_DIR)", "${s_name}")],
326 ["$(CP) ${src} ${dst}"], [])
327
328 ## Constructor
329 #
330 # @param File The file containing build rules in a well defined format
331 # @param Content The string list of build rules in a well defined format
332 # @param LineIndex The line number from which the parsing will begin
333 # @param SupportedFamily The list of supported tool chain families
334 #
335 def __init__(self, File=None, Content=None, LineIndex=0, SupportedFamily=["MSFT", "INTEL", "GCC", "RVCT"]):
336 self.RuleFile = File
337 # Read build rules from file if it's not none
338 if File is not None:
339 try:
340 self.RuleContent = open(File, 'r').readlines()
341 except:
342 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File)
343 elif Content is not None:
344 self.RuleContent = Content
345 else:
346 EdkLogger.error("build", PARAMETER_MISSING, ExtraData="No rule file or string given")
347
348 self.SupportedToolChainFamilyList = SupportedFamily
349 self.RuleDatabase = tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object}
350 self.Ext2FileType = {} # {ext : file-type}
351 self.FileTypeList = set()
352
353 self._LineIndex = LineIndex
354 self._State = ""
355 self._RuleInfo = tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}}
356 self._FileType = ''
357 self._BuildTypeList = set()
358 self._ArchList = set()
359 self._FamilyList = []
360 self._TotalToolChainFamilySet = set()
361 self._RuleObjectList = [] # FileBuildRule object list
362 self._FileVersion = ""
363
364 self.Parse()
365
366 # some intrinsic rules
367 self.RuleDatabase[TAB_DEFAULT_BINARY_FILE, TAB_COMMON, TAB_COMMON, TAB_COMMON] = self._BinaryFileRule
368 self.FileTypeList.add(TAB_DEFAULT_BINARY_FILE)
369
370 ## Parse the build rule strings
371 def Parse(self):
372 self._State = self._Section
373 for Index in range(self._LineIndex, len(self.RuleContent)):
374 # Clean up the line and replace path separator with native one
375 Line = self.RuleContent[Index].strip().replace(self._PATH_SEP, os.path.sep)
376 self.RuleContent[Index] = Line
377
378 # find the build_rule_version
379 if Line and Line[0] == "#" and Line.find(TAB_BUILD_RULE_VERSION) <> -1:
380 if Line.find("=") <> -1 and Line.find("=") < (len(Line) - 1) and (Line[(Line.find("=") + 1):]).split():
381 self._FileVersion = (Line[(Line.find("=") + 1):]).split()[0]
382 # skip empty or comment line
383 if Line == "" or Line[0] == "#":
384 continue
385
386 # find out section header, enclosed by []
387 if Line[0] == '[' and Line[-1] == ']':
388 # merge last section information into rule database
389 self.EndOfSection()
390 self._State = self._SectionHeader
391 # find out sub-section header, enclosed by <>
392 elif Line[0] == '<' and Line[-1] == '>':
393 if self._State != self._UnknownSection:
394 self._State = self._SubSectionHeader
395
396 # call section handler to parse each (sub)section
397 self._StateHandler[self._State](self, Index)
398 # merge last section information into rule database
399 self.EndOfSection()
400
401 ## Parse definitions under a section
402 #
403 # @param LineIndex The line index of build rule text
404 #
405 def ParseSection(self, LineIndex):
406 pass
407
408 ## Parse definitions under a subsection
409 #
410 # @param LineIndex The line index of build rule text
411 #
412 def ParseSubSection(self, LineIndex):
413 # currenly nothing here
414 pass
415
416 ## Placeholder for not supported sections
417 #
418 # @param LineIndex The line index of build rule text
419 #
420 def SkipSection(self, LineIndex):
421 pass
422
423 ## Merge section information just got into rule database
424 def EndOfSection(self):
425 Database = self.RuleDatabase
426 # if there's specific toochain family, 'COMMON' doesn't make sense any more
427 if len(self._TotalToolChainFamilySet) > 1 and TAB_COMMON in self._TotalToolChainFamilySet:
428 self._TotalToolChainFamilySet.remove(TAB_COMMON)
429 for Family in self._TotalToolChainFamilySet:
430 Input = self._RuleInfo[Family, self._InputFile]
431 Output = self._RuleInfo[Family, self._OutputFile]
432 Command = self._RuleInfo[Family, self._Command]
433 ExtraDependency = self._RuleInfo[Family, self._ExtraDependency]
434
435 BuildRule = FileBuildRule(self._FileType, Input, Output, Command, ExtraDependency)
436 for BuildType in self._BuildTypeList:
437 for Arch in self._ArchList:
438 Database[self._FileType, BuildType, Arch, Family] = BuildRule
439 for FileExt in BuildRule.SourceFileExtList:
440 self.Ext2FileType[FileExt] = self._FileType
441
442 ## Parse section header
443 #
444 # @param LineIndex The line index of build rule text
445 #
446 def ParseSectionHeader(self, LineIndex):
447 self._RuleInfo = tdict(True, 2)
448 self._BuildTypeList = set()
449 self._ArchList = set()
450 self._FamilyList = []
451 self._TotalToolChainFamilySet = set()
452 FileType = ''
453 RuleNameList = self.RuleContent[LineIndex][1:-1].split(',')
454 for RuleName in RuleNameList:
455 Arch = TAB_COMMON
456 BuildType = TAB_COMMON
457 TokenList = [Token.strip().upper() for Token in RuleName.split('.')]
458 # old format: Build.File-Type
459 if TokenList[0] == "BUILD":
460 if len(TokenList) == 1:
461 EdkLogger.error("build", FORMAT_INVALID, "Invalid rule section",
462 File=self.RuleFile, Line=LineIndex + 1,
463 ExtraData=self.RuleContent[LineIndex])
464
465 FileType = TokenList[1]
466 if FileType == '':
467 EdkLogger.error("build", FORMAT_INVALID, "No file type given",
468 File=self.RuleFile, Line=LineIndex + 1,
469 ExtraData=self.RuleContent[LineIndex])
470 if self._FileTypePattern.match(FileType) is None:
471 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
472 ExtraData="Only character, number (non-first character), '_' and '-' are allowed in file type")
473 # new format: File-Type.Build-Type.Arch
474 else:
475 if FileType == '':
476 FileType = TokenList[0]
477 elif FileType != TokenList[0]:
478 EdkLogger.error("build", FORMAT_INVALID,
479 "Different file types are not allowed in the same rule section",
480 File=self.RuleFile, Line=LineIndex + 1,
481 ExtraData=self.RuleContent[LineIndex])
482 if len(TokenList) > 1:
483 BuildType = TokenList[1]
484 if len(TokenList) > 2:
485 Arch = TokenList[2]
486 self._BuildTypeList.add(BuildType)
487 self._ArchList.add(Arch)
488
489 if TAB_COMMON in self._BuildTypeList and len(self._BuildTypeList) > 1:
490 EdkLogger.error("build", FORMAT_INVALID,
491 "Specific build types must not be mixed with common one",
492 File=self.RuleFile, Line=LineIndex + 1,
493 ExtraData=self.RuleContent[LineIndex])
494 if TAB_COMMON in self._ArchList and len(self._ArchList) > 1:
495 EdkLogger.error("build", FORMAT_INVALID,
496 "Specific ARCH must not be mixed with common one",
497 File=self.RuleFile, Line=LineIndex + 1,
498 ExtraData=self.RuleContent[LineIndex])
499
500 self._FileType = FileType
501 self._State = self._Section
502 self.FileTypeList.add(FileType)
503
504 ## Parse sub-section header
505 #
506 # @param LineIndex The line index of build rule text
507 #
508 def ParseSubSectionHeader(self, LineIndex):
509 SectionType = ""
510 List = self.RuleContent[LineIndex][1:-1].split(',')
511 FamilyList = []
512 for Section in List:
513 TokenList = Section.split('.')
514 Type = TokenList[0].strip().upper()
515
516 if SectionType == "":
517 SectionType = Type
518 elif SectionType != Type:
519 EdkLogger.error("build", FORMAT_INVALID,
520 "Two different section types are not allowed in the same sub-section",
521 File=self.RuleFile, Line=LineIndex + 1,
522 ExtraData=self.RuleContent[LineIndex])
523
524 if len(TokenList) > 1:
525 Family = TokenList[1].strip().upper()
526 else:
527 Family = TAB_COMMON
528
529 if Family not in FamilyList:
530 FamilyList.append(Family)
531
532 self._FamilyList = FamilyList
533 self._TotalToolChainFamilySet.update(FamilyList)
534 self._State = SectionType.upper()
535 if TAB_COMMON in FamilyList and len(FamilyList) > 1:
536 EdkLogger.error("build", FORMAT_INVALID,
537 "Specific tool chain family should not be mixed with general one",
538 File=self.RuleFile, Line=LineIndex + 1,
539 ExtraData=self.RuleContent[LineIndex])
540 if self._State not in self._StateHandler:
541 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
542 ExtraData="Unknown subsection: %s" % self.RuleContent[LineIndex])
543 ## Parse <InputFile> sub-section
544 #
545 # @param LineIndex The line index of build rule text
546 #
547 def ParseInputFile(self, LineIndex):
548 FileList = [File.strip() for File in self.RuleContent[LineIndex].split(",")]
549 for ToolChainFamily in self._FamilyList:
550 InputFiles = self._RuleInfo[ToolChainFamily, self._State]
551 if InputFiles is None:
552 InputFiles = []
553 self._RuleInfo[ToolChainFamily, self._State] = InputFiles
554 InputFiles.extend(FileList)
555
556 ## Parse <ExtraDependency> sub-section
557 #
558 # @param LineIndex The line index of build rule text
559 #
560 def ParseCommon(self, LineIndex):
561 for ToolChainFamily in self._FamilyList:
562 Items = self._RuleInfo[ToolChainFamily, self._State]
563 if Items is None:
564 Items = []
565 self._RuleInfo[ToolChainFamily, self._State] = Items
566 Items.append(self.RuleContent[LineIndex])
567
568 ## Get a build rule via [] operator
569 #
570 # @param FileExt The extension of a file
571 # @param ToolChainFamily The tool chain family name
572 # @param BuildVersion The build version number. "*" means any rule
573 # is applicalbe.
574 #
575 # @retval FileType The file type string
576 # @retval FileBuildRule The object of FileBuildRule
577 #
578 # Key = (FileExt, ModuleType, Arch, ToolChainFamily)
579 def __getitem__(self, Key):
580 if not Key:
581 return None
582
583 if Key[0] in self.Ext2FileType:
584 Type = self.Ext2FileType[Key[0]]
585 elif Key[0].upper() in self.FileTypeList:
586 Type = Key[0].upper()
587 else:
588 return None
589
590 if len(Key) > 1:
591 Key = (Type,) + Key[1:]
592 else:
593 Key = (Type,)
594 return self.RuleDatabase[Key]
595
596 _StateHandler = {
597 _SectionHeader : ParseSectionHeader,
598 _Section : ParseSection,
599 _SubSectionHeader : ParseSubSectionHeader,
600 _SubSection : ParseSubSection,
601 _InputFile : ParseInputFile,
602 _OutputFile : ParseCommon,
603 _ExtraDependency : ParseCommon,
604 _Command : ParseCommon,
605 _UnknownSection : SkipSection,
606 }
607
608 # This acts like the main() function for the script, unless it is 'import'ed into another
609 # script.
610 if __name__ == '__main__':
611 import sys
612 EdkLogger.Initialize()
613 if len(sys.argv) > 1:
614 Br = BuildRule(sys.argv[1])
615 print str(Br[".c", "DXE_DRIVER", "IA32", "MSFT"][1])
616 print
617 print str(Br[".c", "DXE_DRIVER", "IA32", "INTEL"][1])
618 print
619 print str(Br[".c", "DXE_DRIVER", "IA32", "GCC"][1])
620 print
621 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1])
622 print
623 print str(Br[".h", "ACPI_TABLE", "IA32", "INTEL"][1])
624 print
625 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1])
626 print
627 print str(Br[".s", "SEC", "IPF", "COMMON"][1])
628 print
629 print str(Br[".s", "SEC"][1])
630