]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/AutoGen/BuildEngine.py
BaseTools: Clean up source files
[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / BuildEngine.py
1 ## @file
2 # The engine for building files
3 #
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 from __future__ import print_function
18 import Common.LongFilePathOs as os
19 import re
20 import copy
21 import string
22 from Common.LongFilePathSupport import OpenLongFilePath as open
23
24 from Common.GlobalData import *
25 from Common.BuildToolError import *
26 from Common.Misc import tdict, PathClass
27 from Common.StringUtils import NormPath
28 from Common.DataType import *
29
30 import Common.EdkLogger as EdkLogger
31
32 ## Convert file type to file list macro name
33 #
34 # @param FileType The name of file type
35 #
36 # @retval string The name of macro
37 #
38 def FileListMacro(FileType):
39 return "%sS" % FileType.replace("-", "_").upper()
40
41 ## Convert file type to list file macro name
42 #
43 # @param FileType The name of file type
44 #
45 # @retval string The name of macro
46 #
47 def ListFileMacro(FileType):
48 return "%s_LIST" % FileListMacro(FileType)
49
50 class TargetDescBlock(object):
51 def __init__(self, Inputs, Outputs, Commands, Dependencies):
52 self.InitWorker(Inputs, Outputs, Commands, Dependencies)
53
54 def InitWorker(self, Inputs, Outputs, Commands, Dependencies):
55 self.Inputs = Inputs
56 self.Outputs = Outputs
57 self.Commands = Commands
58 self.Dependencies = Dependencies
59 if self.Outputs:
60 self.Target = self.Outputs[0]
61 else:
62 self.Target = None
63
64 def __str__(self):
65 return self.Target.Path
66
67 def __hash__(self):
68 return hash(self.Target.Path)
69
70 def __eq__(self, Other):
71 if isinstance(Other, type(self)):
72 return Other.Target.Path == self.Target.Path
73 else:
74 return str(Other) == self.Target.Path
75
76 def AddInput(self, Input):
77 if Input not in self.Inputs:
78 self.Inputs.append(Input)
79
80 def IsMultipleInput(self):
81 return len(self.Inputs) > 1
82
83 ## Class for one build rule
84 #
85 # This represents a build rule which can give out corresponding command list for
86 # building the given source file(s). The result can be used for generating the
87 # target for makefile.
88 #
89 class FileBuildRule:
90 INC_LIST_MACRO = "INC_LIST"
91 INC_MACRO = "INC"
92
93 ## constructor
94 #
95 # @param Input The dictionary represeting input file(s) for a rule
96 # @param Output The list represeting output file(s) for a rule
97 # @param Command The list containing commands to generate the output from input
98 #
99 def __init__(self, Type, Input, Output, Command, ExtraDependency=None):
100 # The Input should not be empty
101 if not Input:
102 Input = []
103 if not Output:
104 Output = []
105 if not Command:
106 Command = []
107
108 self.FileListMacro = FileListMacro(Type)
109 self.ListFileMacro = ListFileMacro(Type)
110 self.IncListFileMacro = self.INC_LIST_MACRO
111
112 self.SourceFileType = Type
113 # source files listed not in "*" or "?" pattern format
114 if not ExtraDependency:
115 self.ExtraSourceFileList = []
116 else:
117 self.ExtraSourceFileList = ExtraDependency
118
119 #
120 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST.
121 # If found, generate a file to keep the input files used to get over the
122 # limitation of command line length
123 #
124 self.MacroList = []
125 self.CommandList = []
126 for CmdLine in Command:
127 self.MacroList.extend(gMacroRefPattern.findall(CmdLine))
128 # replace path separator with native one
129 self.CommandList.append(CmdLine)
130
131 # Indicate what should be generated
132 if self.FileListMacro in self.MacroList:
133 self.GenFileListMacro = True
134 else:
135 self.GenFileListMacro = False
136
137 if self.ListFileMacro in self.MacroList:
138 self.GenListFile = True
139 self.GenFileListMacro = True
140 else:
141 self.GenListFile = False
142
143 if self.INC_LIST_MACRO in self.MacroList:
144 self.GenIncListFile = True
145 else:
146 self.GenIncListFile = False
147
148 # Check input files
149 self.IsMultipleInput = False
150 self.SourceFileExtList = set()
151 for File in Input:
152 Base, Ext = os.path.splitext(File)
153 if Base.find("*") >= 0:
154 # There's "*" in the file name
155 self.IsMultipleInput = True
156 self.GenFileListMacro = True
157 elif Base.find("?") < 0:
158 # There's no "*" and "?" in file name
159 self.ExtraSourceFileList.append(File)
160 continue
161 self.SourceFileExtList.add(Ext)
162
163 # Check output files
164 self.DestFileList = []
165 for File in Output:
166 self.DestFileList.append(File)
167
168 # All build targets generated by this rule for a module
169 self.BuildTargets = {}
170
171 ## str() function support
172 #
173 # @retval string
174 #
175 def __str__(self):
176 SourceString = ""
177 SourceString += " %s %s %s" % (self.SourceFileType, " ".join(self.SourceFileExtList), self.ExtraSourceFileList)
178 DestString = ", ".join(self.DestFileList)
179 CommandString = "\n\t".join(self.CommandList)
180 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString)
181
182 def Instantiate(self, Macros={}):
183 NewRuleObject = copy.copy(self)
184 NewRuleObject.BuildTargets = {}
185 NewRuleObject.DestFileList = []
186 for File in self.DestFileList:
187 NewRuleObject.DestFileList.append(PathClass(NormPath(File, Macros)))
188 return NewRuleObject
189
190 ## Apply the rule to given source file(s)
191 #
192 # @param SourceFile One file or a list of files to be built
193 # @param RelativeToDir The relative path of the source file
194 # @param PathSeparator Path separator
195 #
196 # @retval tuple (Source file in full path, List of individual sourcefiles, Destionation file, List of build commands)
197 #
198 def Apply(self, SourceFile, BuildRuleOrder=None):
199 if not self.CommandList or not self.DestFileList:
200 return None
201
202 # source file
203 if self.IsMultipleInput:
204 SrcFileName = ""
205 SrcFileBase = ""
206 SrcFileExt = ""
207 SrcFileDir = ""
208 SrcPath = ""
209 # SourceFile must be a list
210 SrcFile = "$(%s)" % self.FileListMacro
211 else:
212 SrcFileName, SrcFileBase, SrcFileExt = SourceFile.Name, SourceFile.BaseName, SourceFile.Ext
213 if SourceFile.Root:
214 SrcFileDir = SourceFile.SubDir
215 if SrcFileDir == "":
216 SrcFileDir = "."
217 else:
218 SrcFileDir = "."
219 SrcFile = SourceFile.Path
220 SrcPath = SourceFile.Dir
221
222 # destination file (the first one)
223 if self.DestFileList:
224 DestFile = self.DestFileList[0].Path
225 DestPath = self.DestFileList[0].Dir
226 DestFileName = self.DestFileList[0].Name
227 DestFileBase, DestFileExt = self.DestFileList[0].BaseName, self.DestFileList[0].Ext
228 else:
229 DestFile = ""
230 DestPath = ""
231 DestFileName = ""
232 DestFileBase = ""
233 DestFileExt = ""
234
235 BuildRulePlaceholderDict = {
236 # source file
237 "src" : SrcFile,
238 "s_path" : SrcPath,
239 "s_dir" : SrcFileDir,
240 "s_name" : SrcFileName,
241 "s_base" : SrcFileBase,
242 "s_ext" : SrcFileExt,
243 # destination file
244 "dst" : DestFile,
245 "d_path" : DestPath,
246 "d_name" : DestFileName,
247 "d_base" : DestFileBase,
248 "d_ext" : DestFileExt,
249 }
250
251 DstFile = []
252 for File in self.DestFileList:
253 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
254 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
255 DstFile.append(PathClass(File, IsBinary=True))
256
257 if DstFile[0] in self.BuildTargets:
258 TargetDesc = self.BuildTargets[DstFile[0]]
259 if BuildRuleOrder and SourceFile.Ext in BuildRuleOrder:
260 Index = BuildRuleOrder.index(SourceFile.Ext)
261 for Input in TargetDesc.Inputs:
262 if Input.Ext not in BuildRuleOrder or BuildRuleOrder.index(Input.Ext) > Index:
263 #
264 # Command line should be regenerated since some macros are different
265 #
266 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
267 TargetDesc.InitWorker([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
268 break
269 else:
270 TargetDesc.AddInput(SourceFile)
271 else:
272 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
273 TargetDesc = TargetDescBlock([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
274 TargetDesc.ListFileMacro = self.ListFileMacro
275 TargetDesc.FileListMacro = self.FileListMacro
276 TargetDesc.IncListFileMacro = self.IncListFileMacro
277 TargetDesc.GenFileListMacro = self.GenFileListMacro
278 TargetDesc.GenListFile = self.GenListFile
279 TargetDesc.GenIncListFile = self.GenIncListFile
280 self.BuildTargets[DstFile[0]] = TargetDesc
281 return TargetDesc
282
283 def _BuildCommand(self, Macros):
284 CommandList = []
285 for CommandString in self.CommandList:
286 CommandString = string.Template(CommandString).safe_substitute(Macros)
287 CommandString = string.Template(CommandString).safe_substitute(Macros)
288 CommandList.append(CommandString)
289 return CommandList
290
291 ## Class for build rules
292 #
293 # BuildRule class parses rules defined in a file or passed by caller, and converts
294 # the rule into FileBuildRule object.
295 #
296 class BuildRule:
297 _SectionHeader = "SECTIONHEADER"
298 _Section = "SECTION"
299 _SubSectionHeader = "SUBSECTIONHEADER"
300 _SubSection = "SUBSECTION"
301 _InputFile = "INPUTFILE"
302 _OutputFile = "OUTPUTFILE"
303 _ExtraDependency = "EXTRADEPENDENCY"
304 _Command = "COMMAND"
305 _UnknownSection = "UNKNOWNSECTION"
306
307 _SubSectionList = [_InputFile, _OutputFile, _Command]
308
309 _PATH_SEP = "(+)"
310 _FileTypePattern = re.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$")
311 _BinaryFileRule = FileBuildRule(TAB_DEFAULT_BINARY_FILE, [], [os.path.join("$(OUTPUT_DIR)", "${s_name}")],
312 ["$(CP) ${src} ${dst}"], [])
313
314 ## Constructor
315 #
316 # @param File The file containing build rules in a well defined format
317 # @param Content The string list of build rules in a well defined format
318 # @param LineIndex The line number from which the parsing will begin
319 # @param SupportedFamily The list of supported tool chain families
320 #
321 def __init__(self, File=None, Content=None, LineIndex=0, SupportedFamily=["MSFT", "INTEL", "GCC", "RVCT"]):
322 self.RuleFile = File
323 # Read build rules from file if it's not none
324 if File is not None:
325 try:
326 self.RuleContent = open(File, 'r').readlines()
327 except:
328 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File)
329 elif Content is not None:
330 self.RuleContent = Content
331 else:
332 EdkLogger.error("build", PARAMETER_MISSING, ExtraData="No rule file or string given")
333
334 self.SupportedToolChainFamilyList = SupportedFamily
335 self.RuleDatabase = tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object}
336 self.Ext2FileType = {} # {ext : file-type}
337 self.FileTypeList = set()
338
339 self._LineIndex = LineIndex
340 self._State = ""
341 self._RuleInfo = tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}}
342 self._FileType = ''
343 self._BuildTypeList = set()
344 self._ArchList = set()
345 self._FamilyList = []
346 self._TotalToolChainFamilySet = set()
347 self._RuleObjectList = [] # FileBuildRule object list
348 self._FileVersion = ""
349
350 self.Parse()
351
352 # some intrinsic rules
353 self.RuleDatabase[TAB_DEFAULT_BINARY_FILE, TAB_COMMON, TAB_COMMON, TAB_COMMON] = self._BinaryFileRule
354 self.FileTypeList.add(TAB_DEFAULT_BINARY_FILE)
355
356 ## Parse the build rule strings
357 def Parse(self):
358 self._State = self._Section
359 for Index in range(self._LineIndex, len(self.RuleContent)):
360 # Clean up the line and replace path separator with native one
361 Line = self.RuleContent[Index].strip().replace(self._PATH_SEP, os.path.sep)
362 self.RuleContent[Index] = Line
363
364 # find the build_rule_version
365 if Line and Line[0] == "#" and Line.find(TAB_BUILD_RULE_VERSION) != -1:
366 if Line.find("=") != -1 and Line.find("=") < (len(Line) - 1) and (Line[(Line.find("=") + 1):]).split():
367 self._FileVersion = (Line[(Line.find("=") + 1):]).split()[0]
368 # skip empty or comment line
369 if Line == "" or Line[0] == "#":
370 continue
371
372 # find out section header, enclosed by []
373 if Line[0] == '[' and Line[-1] == ']':
374 # merge last section information into rule database
375 self.EndOfSection()
376 self._State = self._SectionHeader
377 # find out sub-section header, enclosed by <>
378 elif Line[0] == '<' and Line[-1] == '>':
379 if self._State != self._UnknownSection:
380 self._State = self._SubSectionHeader
381
382 # call section handler to parse each (sub)section
383 self._StateHandler[self._State](self, Index)
384 # merge last section information into rule database
385 self.EndOfSection()
386
387 ## Parse definitions under a section
388 #
389 # @param LineIndex The line index of build rule text
390 #
391 def ParseSection(self, LineIndex):
392 pass
393
394 ## Parse definitions under a subsection
395 #
396 # @param LineIndex The line index of build rule text
397 #
398 def ParseSubSection(self, LineIndex):
399 # currenly nothing here
400 pass
401
402 ## Placeholder for not supported sections
403 #
404 # @param LineIndex The line index of build rule text
405 #
406 def SkipSection(self, LineIndex):
407 pass
408
409 ## Merge section information just got into rule database
410 def EndOfSection(self):
411 Database = self.RuleDatabase
412 # if there's specific toochain family, 'COMMON' doesn't make sense any more
413 if len(self._TotalToolChainFamilySet) > 1 and TAB_COMMON in self._TotalToolChainFamilySet:
414 self._TotalToolChainFamilySet.remove(TAB_COMMON)
415 for Family in self._TotalToolChainFamilySet:
416 Input = self._RuleInfo[Family, self._InputFile]
417 Output = self._RuleInfo[Family, self._OutputFile]
418 Command = self._RuleInfo[Family, self._Command]
419 ExtraDependency = self._RuleInfo[Family, self._ExtraDependency]
420
421 BuildRule = FileBuildRule(self._FileType, Input, Output, Command, ExtraDependency)
422 for BuildType in self._BuildTypeList:
423 for Arch in self._ArchList:
424 Database[self._FileType, BuildType, Arch, Family] = BuildRule
425 for FileExt in BuildRule.SourceFileExtList:
426 self.Ext2FileType[FileExt] = self._FileType
427
428 ## Parse section header
429 #
430 # @param LineIndex The line index of build rule text
431 #
432 def ParseSectionHeader(self, LineIndex):
433 self._RuleInfo = tdict(True, 2)
434 self._BuildTypeList = set()
435 self._ArchList = set()
436 self._FamilyList = []
437 self._TotalToolChainFamilySet = set()
438 FileType = ''
439 RuleNameList = self.RuleContent[LineIndex][1:-1].split(',')
440 for RuleName in RuleNameList:
441 Arch = TAB_COMMON
442 BuildType = TAB_COMMON
443 TokenList = [Token.strip().upper() for Token in RuleName.split('.')]
444 # old format: Build.File-Type
445 if TokenList[0] == "BUILD":
446 if len(TokenList) == 1:
447 EdkLogger.error("build", FORMAT_INVALID, "Invalid rule section",
448 File=self.RuleFile, Line=LineIndex + 1,
449 ExtraData=self.RuleContent[LineIndex])
450
451 FileType = TokenList[1]
452 if FileType == '':
453 EdkLogger.error("build", FORMAT_INVALID, "No file type given",
454 File=self.RuleFile, Line=LineIndex + 1,
455 ExtraData=self.RuleContent[LineIndex])
456 if self._FileTypePattern.match(FileType) is None:
457 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
458 ExtraData="Only character, number (non-first character), '_' and '-' are allowed in file type")
459 # new format: File-Type.Build-Type.Arch
460 else:
461 if FileType == '':
462 FileType = TokenList[0]
463 elif FileType != TokenList[0]:
464 EdkLogger.error("build", FORMAT_INVALID,
465 "Different file types are not allowed in the same rule section",
466 File=self.RuleFile, Line=LineIndex + 1,
467 ExtraData=self.RuleContent[LineIndex])
468 if len(TokenList) > 1:
469 BuildType = TokenList[1]
470 if len(TokenList) > 2:
471 Arch = TokenList[2]
472 self._BuildTypeList.add(BuildType)
473 self._ArchList.add(Arch)
474
475 if TAB_COMMON in self._BuildTypeList and len(self._BuildTypeList) > 1:
476 EdkLogger.error("build", FORMAT_INVALID,
477 "Specific build types must not be mixed with common one",
478 File=self.RuleFile, Line=LineIndex + 1,
479 ExtraData=self.RuleContent[LineIndex])
480 if TAB_COMMON in self._ArchList and len(self._ArchList) > 1:
481 EdkLogger.error("build", FORMAT_INVALID,
482 "Specific ARCH must not be mixed with common one",
483 File=self.RuleFile, Line=LineIndex + 1,
484 ExtraData=self.RuleContent[LineIndex])
485
486 self._FileType = FileType
487 self._State = self._Section
488 self.FileTypeList.add(FileType)
489
490 ## Parse sub-section header
491 #
492 # @param LineIndex The line index of build rule text
493 #
494 def ParseSubSectionHeader(self, LineIndex):
495 SectionType = ""
496 List = self.RuleContent[LineIndex][1:-1].split(',')
497 FamilyList = []
498 for Section in List:
499 TokenList = Section.split('.')
500 Type = TokenList[0].strip().upper()
501
502 if SectionType == "":
503 SectionType = Type
504 elif SectionType != Type:
505 EdkLogger.error("build", FORMAT_INVALID,
506 "Two different section types are not allowed in the same sub-section",
507 File=self.RuleFile, Line=LineIndex + 1,
508 ExtraData=self.RuleContent[LineIndex])
509
510 if len(TokenList) > 1:
511 Family = TokenList[1].strip().upper()
512 else:
513 Family = TAB_COMMON
514
515 if Family not in FamilyList:
516 FamilyList.append(Family)
517
518 self._FamilyList = FamilyList
519 self._TotalToolChainFamilySet.update(FamilyList)
520 self._State = SectionType.upper()
521 if TAB_COMMON in FamilyList and len(FamilyList) > 1:
522 EdkLogger.error("build", FORMAT_INVALID,
523 "Specific tool chain family should not be mixed with general one",
524 File=self.RuleFile, Line=LineIndex + 1,
525 ExtraData=self.RuleContent[LineIndex])
526 if self._State not in self._StateHandler:
527 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
528 ExtraData="Unknown subsection: %s" % self.RuleContent[LineIndex])
529 ## Parse <InputFile> sub-section
530 #
531 # @param LineIndex The line index of build rule text
532 #
533 def ParseInputFile(self, LineIndex):
534 FileList = [File.strip() for File in self.RuleContent[LineIndex].split(",")]
535 for ToolChainFamily in self._FamilyList:
536 InputFiles = self._RuleInfo[ToolChainFamily, self._State]
537 if InputFiles is None:
538 InputFiles = []
539 self._RuleInfo[ToolChainFamily, self._State] = InputFiles
540 InputFiles.extend(FileList)
541
542 ## Parse <ExtraDependency> sub-section
543 #
544 # @param LineIndex The line index of build rule text
545 #
546 def ParseCommon(self, LineIndex):
547 for ToolChainFamily in self._FamilyList:
548 Items = self._RuleInfo[ToolChainFamily, self._State]
549 if Items is None:
550 Items = []
551 self._RuleInfo[ToolChainFamily, self._State] = Items
552 Items.append(self.RuleContent[LineIndex])
553
554 ## Get a build rule via [] operator
555 #
556 # @param FileExt The extension of a file
557 # @param ToolChainFamily The tool chain family name
558 # @param BuildVersion The build version number. "*" means any rule
559 # is applicalbe.
560 #
561 # @retval FileType The file type string
562 # @retval FileBuildRule The object of FileBuildRule
563 #
564 # Key = (FileExt, ModuleType, Arch, ToolChainFamily)
565 def __getitem__(self, Key):
566 if not Key:
567 return None
568
569 if Key[0] in self.Ext2FileType:
570 Type = self.Ext2FileType[Key[0]]
571 elif Key[0].upper() in self.FileTypeList:
572 Type = Key[0].upper()
573 else:
574 return None
575
576 if len(Key) > 1:
577 Key = (Type,) + Key[1:]
578 else:
579 Key = (Type,)
580 return self.RuleDatabase[Key]
581
582 _StateHandler = {
583 _SectionHeader : ParseSectionHeader,
584 _Section : ParseSection,
585 _SubSectionHeader : ParseSubSectionHeader,
586 _SubSection : ParseSubSection,
587 _InputFile : ParseInputFile,
588 _OutputFile : ParseCommon,
589 _ExtraDependency : ParseCommon,
590 _Command : ParseCommon,
591 _UnknownSection : SkipSection,
592 }
593
594 # This acts like the main() function for the script, unless it is 'import'ed into another
595 # script.
596 if __name__ == '__main__':
597 import sys
598 EdkLogger.Initialize()
599 if len(sys.argv) > 1:
600 Br = BuildRule(sys.argv[1])
601 print(str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "MSFT"][1]))
602 print()
603 print(str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "INTEL"][1]))
604 print()
605 print(str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "GCC"][1]))
606 print()
607 print(str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1]))
608 print()
609 print(str(Br[".h", "ACPI_TABLE", "IA32", "INTEL"][1]))
610 print()
611 print(str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1]))
612 print()
613 print(str(Br[".s", SUP_MODULE_SEC, "IPF", "COMMON"][1]))
614 print()
615 print(str(Br[".s", SUP_MODULE_SEC][1]))
616