]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/AutoGen/BuildEngine.py
BaseTools: Rename String to StringUtils.
[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / BuildEngine.py
1 ## @file
2 # The engine for building files
3 #
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import Common.LongFilePathOs as os
18 import re
19 import copy
20 import string
21 from Common.LongFilePathSupport import OpenLongFilePath as open
22
23 from Common.GlobalData import *
24 from Common.BuildToolError import *
25 from Common.Misc import tdict, PathClass
26 from Common.StringUtils import NormPath
27 from Common.DataType import *
28
29 import Common.EdkLogger as EdkLogger
30
31 ## Convert file type to file list macro name
32 #
33 # @param FileType The name of file type
34 #
35 # @retval string The name of macro
36 #
37 def FileListMacro(FileType):
38 return "%sS" % FileType.replace("-", "_").upper()
39
40 ## Convert file type to list file macro name
41 #
42 # @param FileType The name of file type
43 #
44 # @retval string The name of macro
45 #
46 def ListFileMacro(FileType):
47 return "%s_LIST" % FileListMacro(FileType)
48
49 class TargetDescBlock(object):
50 def __init__(self, Inputs, Outputs, Commands, Dependencies):
51 self.InitWorker(Inputs, Outputs, Commands, Dependencies)
52
53 def InitWorker(self, Inputs, Outputs, Commands, Dependencies):
54 self.Inputs = Inputs
55 self.Outputs = Outputs
56 self.Commands = Commands
57 self.Dependencies = Dependencies
58 if self.Outputs:
59 self.Target = self.Outputs[0]
60 else:
61 self.Target = None
62
63 def __str__(self):
64 return self.Target.Path
65
66 def __hash__(self):
67 return hash(self.Target.Path)
68
69 def __eq__(self, Other):
70 if type(Other) == type(self):
71 return Other.Target.Path == self.Target.Path
72 else:
73 return str(Other) == self.Target.Path
74
75 def AddInput(self, Input):
76 if Input not in self.Inputs:
77 self.Inputs.append(Input)
78
79 def IsMultipleInput(self):
80 return len(self.Inputs) > 1
81
82 ## Class for one build rule
83 #
84 # This represents a build rule which can give out corresponding command list for
85 # building the given source file(s). The result can be used for generating the
86 # target for makefile.
87 #
88 class FileBuildRule:
89 INC_LIST_MACRO = "INC_LIST"
90 INC_MACRO = "INC"
91
92 ## constructor
93 #
94 # @param Input The dictionary represeting input file(s) for a rule
95 # @param Output The list represeting output file(s) for a rule
96 # @param Command The list containing commands to generate the output from input
97 #
98 def __init__(self, Type, Input, Output, Command, ExtraDependency=None):
99 # The Input should not be empty
100 if not Input:
101 Input = []
102 if not Output:
103 Output = []
104 if not Command:
105 Command = []
106
107 self.FileListMacro = FileListMacro(Type)
108 self.ListFileMacro = ListFileMacro(Type)
109 self.IncListFileMacro = self.INC_LIST_MACRO
110
111 self.SourceFileType = Type
112 # source files listed not in "*" or "?" pattern format
113 if not ExtraDependency:
114 self.ExtraSourceFileList = []
115 else:
116 self.ExtraSourceFileList = ExtraDependency
117
118 #
119 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST.
120 # If found, generate a file to keep the input files used to get over the
121 # limitation of command line length
122 #
123 self.MacroList = []
124 self.CommandList = []
125 for CmdLine in Command:
126 self.MacroList.extend(gMacroRefPattern.findall(CmdLine))
127 # replace path separator with native one
128 self.CommandList.append(CmdLine)
129
130 # Indicate what should be generated
131 if self.FileListMacro in self.MacroList:
132 self.GenFileListMacro = True
133 else:
134 self.GenFileListMacro = False
135
136 if self.ListFileMacro in self.MacroList:
137 self.GenListFile = True
138 self.GenFileListMacro = True
139 else:
140 self.GenListFile = False
141
142 if self.INC_LIST_MACRO in self.MacroList:
143 self.GenIncListFile = True
144 else:
145 self.GenIncListFile = False
146
147 # Check input files
148 self.IsMultipleInput = False
149 self.SourceFileExtList = set()
150 for File in Input:
151 Base, Ext = os.path.splitext(File)
152 if Base.find("*") >= 0:
153 # There's "*" in the file name
154 self.IsMultipleInput = True
155 self.GenFileListMacro = True
156 elif Base.find("?") < 0:
157 # There's no "*" and "?" in file name
158 self.ExtraSourceFileList.append(File)
159 continue
160 self.SourceFileExtList.add(Ext)
161
162 # Check output files
163 self.DestFileList = []
164 for File in Output:
165 self.DestFileList.append(File)
166
167 # All build targets generated by this rule for a module
168 self.BuildTargets = {}
169
170 ## str() function support
171 #
172 # @retval string
173 #
174 def __str__(self):
175 SourceString = ""
176 SourceString += " %s %s %s" % (self.SourceFileType, " ".join(self.SourceFileExtList), self.ExtraSourceFileList)
177 DestString = ", ".join(self.DestFileList)
178 CommandString = "\n\t".join(self.CommandList)
179 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString)
180
181 def Instantiate(self, Macros={}):
182 NewRuleObject = copy.copy(self)
183 NewRuleObject.BuildTargets = {}
184 NewRuleObject.DestFileList = []
185 for File in self.DestFileList:
186 NewRuleObject.DestFileList.append(PathClass(NormPath(File, Macros)))
187 return NewRuleObject
188
189 ## Apply the rule to given source file(s)
190 #
191 # @param SourceFile One file or a list of files to be built
192 # @param RelativeToDir The relative path of the source file
193 # @param PathSeparator Path separator
194 #
195 # @retval tuple (Source file in full path, List of individual sourcefiles, Destionation file, List of build commands)
196 #
197 def Apply(self, SourceFile, BuildRuleOrder=None):
198 if not self.CommandList or not self.DestFileList:
199 return None
200
201 # source file
202 if self.IsMultipleInput:
203 SrcFileName = ""
204 SrcFileBase = ""
205 SrcFileExt = ""
206 SrcFileDir = ""
207 SrcPath = ""
208 # SourceFile must be a list
209 SrcFile = "$(%s)" % self.FileListMacro
210 else:
211 SrcFileName, SrcFileBase, SrcFileExt = SourceFile.Name, SourceFile.BaseName, SourceFile.Ext
212 if SourceFile.Root:
213 SrcFileDir = SourceFile.SubDir
214 if SrcFileDir == "":
215 SrcFileDir = "."
216 else:
217 SrcFileDir = "."
218 SrcFile = SourceFile.Path
219 SrcPath = SourceFile.Dir
220
221 # destination file (the first one)
222 if self.DestFileList:
223 DestFile = self.DestFileList[0].Path
224 DestPath = self.DestFileList[0].Dir
225 DestFileName = self.DestFileList[0].Name
226 DestFileBase, DestFileExt = self.DestFileList[0].BaseName, self.DestFileList[0].Ext
227 else:
228 DestFile = ""
229 DestPath = ""
230 DestFileName = ""
231 DestFileBase = ""
232 DestFileExt = ""
233
234 BuildRulePlaceholderDict = {
235 # source file
236 "src" : SrcFile,
237 "s_path" : SrcPath,
238 "s_dir" : SrcFileDir,
239 "s_name" : SrcFileName,
240 "s_base" : SrcFileBase,
241 "s_ext" : SrcFileExt,
242 # destination file
243 "dst" : DestFile,
244 "d_path" : DestPath,
245 "d_name" : DestFileName,
246 "d_base" : DestFileBase,
247 "d_ext" : DestFileExt,
248 }
249
250 DstFile = []
251 for File in self.DestFileList:
252 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
253 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
254 DstFile.append(PathClass(File, IsBinary=True))
255
256 if DstFile[0] in self.BuildTargets:
257 TargetDesc = self.BuildTargets[DstFile[0]]
258 if BuildRuleOrder and SourceFile.Ext in BuildRuleOrder:
259 Index = BuildRuleOrder.index(SourceFile.Ext)
260 for Input in TargetDesc.Inputs:
261 if Input.Ext not in BuildRuleOrder or BuildRuleOrder.index(Input.Ext) > Index:
262 #
263 # Command line should be regenerated since some macros are different
264 #
265 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
266 TargetDesc.InitWorker([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
267 break
268 else:
269 TargetDesc.AddInput(SourceFile)
270 else:
271 CommandList = self._BuildCommand(BuildRulePlaceholderDict)
272 TargetDesc = TargetDescBlock([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
273 TargetDesc.ListFileMacro = self.ListFileMacro
274 TargetDesc.FileListMacro = self.FileListMacro
275 TargetDesc.IncListFileMacro = self.IncListFileMacro
276 TargetDesc.GenFileListMacro = self.GenFileListMacro
277 TargetDesc.GenListFile = self.GenListFile
278 TargetDesc.GenIncListFile = self.GenIncListFile
279 self.BuildTargets[DstFile[0]] = TargetDesc
280 return TargetDesc
281
282 def _BuildCommand(self, Macros):
283 CommandList = []
284 for CommandString in self.CommandList:
285 CommandString = string.Template(CommandString).safe_substitute(Macros)
286 CommandString = string.Template(CommandString).safe_substitute(Macros)
287 CommandList.append(CommandString)
288 return CommandList
289
290 ## Class for build rules
291 #
292 # BuildRule class parses rules defined in a file or passed by caller, and converts
293 # the rule into FileBuildRule object.
294 #
295 class BuildRule:
296 _SectionHeader = "SECTIONHEADER"
297 _Section = "SECTION"
298 _SubSectionHeader = "SUBSECTIONHEADER"
299 _SubSection = "SUBSECTION"
300 _InputFile = "INPUTFILE"
301 _OutputFile = "OUTPUTFILE"
302 _ExtraDependency = "EXTRADEPENDENCY"
303 _Command = "COMMAND"
304 _UnknownSection = "UNKNOWNSECTION"
305
306 _SubSectionList = [_InputFile, _OutputFile, _Command]
307
308 _PATH_SEP = "(+)"
309 _FileTypePattern = re.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$")
310 _BinaryFileRule = FileBuildRule(TAB_DEFAULT_BINARY_FILE, [], [os.path.join("$(OUTPUT_DIR)", "${s_name}")],
311 ["$(CP) ${src} ${dst}"], [])
312
313 ## Constructor
314 #
315 # @param File The file containing build rules in a well defined format
316 # @param Content The string list of build rules in a well defined format
317 # @param LineIndex The line number from which the parsing will begin
318 # @param SupportedFamily The list of supported tool chain families
319 #
320 def __init__(self, File=None, Content=None, LineIndex=0, SupportedFamily=["MSFT", "INTEL", "GCC", "RVCT"]):
321 self.RuleFile = File
322 # Read build rules from file if it's not none
323 if File is not None:
324 try:
325 self.RuleContent = open(File, 'r').readlines()
326 except:
327 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File)
328 elif Content is not None:
329 self.RuleContent = Content
330 else:
331 EdkLogger.error("build", PARAMETER_MISSING, ExtraData="No rule file or string given")
332
333 self.SupportedToolChainFamilyList = SupportedFamily
334 self.RuleDatabase = tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object}
335 self.Ext2FileType = {} # {ext : file-type}
336 self.FileTypeList = set()
337
338 self._LineIndex = LineIndex
339 self._State = ""
340 self._RuleInfo = tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}}
341 self._FileType = ''
342 self._BuildTypeList = set()
343 self._ArchList = set()
344 self._FamilyList = []
345 self._TotalToolChainFamilySet = set()
346 self._RuleObjectList = [] # FileBuildRule object list
347 self._FileVersion = ""
348
349 self.Parse()
350
351 # some intrinsic rules
352 self.RuleDatabase[TAB_DEFAULT_BINARY_FILE, TAB_COMMON, TAB_COMMON, TAB_COMMON] = self._BinaryFileRule
353 self.FileTypeList.add(TAB_DEFAULT_BINARY_FILE)
354
355 ## Parse the build rule strings
356 def Parse(self):
357 self._State = self._Section
358 for Index in range(self._LineIndex, len(self.RuleContent)):
359 # Clean up the line and replace path separator with native one
360 Line = self.RuleContent[Index].strip().replace(self._PATH_SEP, os.path.sep)
361 self.RuleContent[Index] = Line
362
363 # find the build_rule_version
364 if Line and Line[0] == "#" and Line.find(TAB_BUILD_RULE_VERSION) <> -1:
365 if Line.find("=") <> -1 and Line.find("=") < (len(Line) - 1) and (Line[(Line.find("=") + 1):]).split():
366 self._FileVersion = (Line[(Line.find("=") + 1):]).split()[0]
367 # skip empty or comment line
368 if Line == "" or Line[0] == "#":
369 continue
370
371 # find out section header, enclosed by []
372 if Line[0] == '[' and Line[-1] == ']':
373 # merge last section information into rule database
374 self.EndOfSection()
375 self._State = self._SectionHeader
376 # find out sub-section header, enclosed by <>
377 elif Line[0] == '<' and Line[-1] == '>':
378 if self._State != self._UnknownSection:
379 self._State = self._SubSectionHeader
380
381 # call section handler to parse each (sub)section
382 self._StateHandler[self._State](self, Index)
383 # merge last section information into rule database
384 self.EndOfSection()
385
386 ## Parse definitions under a section
387 #
388 # @param LineIndex The line index of build rule text
389 #
390 def ParseSection(self, LineIndex):
391 pass
392
393 ## Parse definitions under a subsection
394 #
395 # @param LineIndex The line index of build rule text
396 #
397 def ParseSubSection(self, LineIndex):
398 # currenly nothing here
399 pass
400
401 ## Placeholder for not supported sections
402 #
403 # @param LineIndex The line index of build rule text
404 #
405 def SkipSection(self, LineIndex):
406 pass
407
408 ## Merge section information just got into rule database
409 def EndOfSection(self):
410 Database = self.RuleDatabase
411 # if there's specific toochain family, 'COMMON' doesn't make sense any more
412 if len(self._TotalToolChainFamilySet) > 1 and TAB_COMMON in self._TotalToolChainFamilySet:
413 self._TotalToolChainFamilySet.remove(TAB_COMMON)
414 for Family in self._TotalToolChainFamilySet:
415 Input = self._RuleInfo[Family, self._InputFile]
416 Output = self._RuleInfo[Family, self._OutputFile]
417 Command = self._RuleInfo[Family, self._Command]
418 ExtraDependency = self._RuleInfo[Family, self._ExtraDependency]
419
420 BuildRule = FileBuildRule(self._FileType, Input, Output, Command, ExtraDependency)
421 for BuildType in self._BuildTypeList:
422 for Arch in self._ArchList:
423 Database[self._FileType, BuildType, Arch, Family] = BuildRule
424 for FileExt in BuildRule.SourceFileExtList:
425 self.Ext2FileType[FileExt] = self._FileType
426
427 ## Parse section header
428 #
429 # @param LineIndex The line index of build rule text
430 #
431 def ParseSectionHeader(self, LineIndex):
432 self._RuleInfo = tdict(True, 2)
433 self._BuildTypeList = set()
434 self._ArchList = set()
435 self._FamilyList = []
436 self._TotalToolChainFamilySet = set()
437 FileType = ''
438 RuleNameList = self.RuleContent[LineIndex][1:-1].split(',')
439 for RuleName in RuleNameList:
440 Arch = TAB_COMMON
441 BuildType = TAB_COMMON
442 TokenList = [Token.strip().upper() for Token in RuleName.split('.')]
443 # old format: Build.File-Type
444 if TokenList[0] == "BUILD":
445 if len(TokenList) == 1:
446 EdkLogger.error("build", FORMAT_INVALID, "Invalid rule section",
447 File=self.RuleFile, Line=LineIndex + 1,
448 ExtraData=self.RuleContent[LineIndex])
449
450 FileType = TokenList[1]
451 if FileType == '':
452 EdkLogger.error("build", FORMAT_INVALID, "No file type given",
453 File=self.RuleFile, Line=LineIndex + 1,
454 ExtraData=self.RuleContent[LineIndex])
455 if self._FileTypePattern.match(FileType) is None:
456 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
457 ExtraData="Only character, number (non-first character), '_' and '-' are allowed in file type")
458 # new format: File-Type.Build-Type.Arch
459 else:
460 if FileType == '':
461 FileType = TokenList[0]
462 elif FileType != TokenList[0]:
463 EdkLogger.error("build", FORMAT_INVALID,
464 "Different file types are not allowed in the same rule section",
465 File=self.RuleFile, Line=LineIndex + 1,
466 ExtraData=self.RuleContent[LineIndex])
467 if len(TokenList) > 1:
468 BuildType = TokenList[1]
469 if len(TokenList) > 2:
470 Arch = TokenList[2]
471 self._BuildTypeList.add(BuildType)
472 self._ArchList.add(Arch)
473
474 if TAB_COMMON in self._BuildTypeList and len(self._BuildTypeList) > 1:
475 EdkLogger.error("build", FORMAT_INVALID,
476 "Specific build types must not be mixed with common one",
477 File=self.RuleFile, Line=LineIndex + 1,
478 ExtraData=self.RuleContent[LineIndex])
479 if TAB_COMMON in self._ArchList and len(self._ArchList) > 1:
480 EdkLogger.error("build", FORMAT_INVALID,
481 "Specific ARCH must not be mixed with common one",
482 File=self.RuleFile, Line=LineIndex + 1,
483 ExtraData=self.RuleContent[LineIndex])
484
485 self._FileType = FileType
486 self._State = self._Section
487 self.FileTypeList.add(FileType)
488
489 ## Parse sub-section header
490 #
491 # @param LineIndex The line index of build rule text
492 #
493 def ParseSubSectionHeader(self, LineIndex):
494 SectionType = ""
495 List = self.RuleContent[LineIndex][1:-1].split(',')
496 FamilyList = []
497 for Section in List:
498 TokenList = Section.split('.')
499 Type = TokenList[0].strip().upper()
500
501 if SectionType == "":
502 SectionType = Type
503 elif SectionType != Type:
504 EdkLogger.error("build", FORMAT_INVALID,
505 "Two different section types are not allowed in the same sub-section",
506 File=self.RuleFile, Line=LineIndex + 1,
507 ExtraData=self.RuleContent[LineIndex])
508
509 if len(TokenList) > 1:
510 Family = TokenList[1].strip().upper()
511 else:
512 Family = TAB_COMMON
513
514 if Family not in FamilyList:
515 FamilyList.append(Family)
516
517 self._FamilyList = FamilyList
518 self._TotalToolChainFamilySet.update(FamilyList)
519 self._State = SectionType.upper()
520 if TAB_COMMON in FamilyList and len(FamilyList) > 1:
521 EdkLogger.error("build", FORMAT_INVALID,
522 "Specific tool chain family should not be mixed with general one",
523 File=self.RuleFile, Line=LineIndex + 1,
524 ExtraData=self.RuleContent[LineIndex])
525 if self._State not in self._StateHandler:
526 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex + 1,
527 ExtraData="Unknown subsection: %s" % self.RuleContent[LineIndex])
528 ## Parse <InputFile> sub-section
529 #
530 # @param LineIndex The line index of build rule text
531 #
532 def ParseInputFile(self, LineIndex):
533 FileList = [File.strip() for File in self.RuleContent[LineIndex].split(",")]
534 for ToolChainFamily in self._FamilyList:
535 InputFiles = self._RuleInfo[ToolChainFamily, self._State]
536 if InputFiles is None:
537 InputFiles = []
538 self._RuleInfo[ToolChainFamily, self._State] = InputFiles
539 InputFiles.extend(FileList)
540
541 ## Parse <ExtraDependency> sub-section
542 #
543 # @param LineIndex The line index of build rule text
544 #
545 def ParseCommon(self, LineIndex):
546 for ToolChainFamily in self._FamilyList:
547 Items = self._RuleInfo[ToolChainFamily, self._State]
548 if Items is None:
549 Items = []
550 self._RuleInfo[ToolChainFamily, self._State] = Items
551 Items.append(self.RuleContent[LineIndex])
552
553 ## Get a build rule via [] operator
554 #
555 # @param FileExt The extension of a file
556 # @param ToolChainFamily The tool chain family name
557 # @param BuildVersion The build version number. "*" means any rule
558 # is applicalbe.
559 #
560 # @retval FileType The file type string
561 # @retval FileBuildRule The object of FileBuildRule
562 #
563 # Key = (FileExt, ModuleType, Arch, ToolChainFamily)
564 def __getitem__(self, Key):
565 if not Key:
566 return None
567
568 if Key[0] in self.Ext2FileType:
569 Type = self.Ext2FileType[Key[0]]
570 elif Key[0].upper() in self.FileTypeList:
571 Type = Key[0].upper()
572 else:
573 return None
574
575 if len(Key) > 1:
576 Key = (Type,) + Key[1:]
577 else:
578 Key = (Type,)
579 return self.RuleDatabase[Key]
580
581 _StateHandler = {
582 _SectionHeader : ParseSectionHeader,
583 _Section : ParseSection,
584 _SubSectionHeader : ParseSubSectionHeader,
585 _SubSection : ParseSubSection,
586 _InputFile : ParseInputFile,
587 _OutputFile : ParseCommon,
588 _ExtraDependency : ParseCommon,
589 _Command : ParseCommon,
590 _UnknownSection : SkipSection,
591 }
592
593 # This acts like the main() function for the script, unless it is 'import'ed into another
594 # script.
595 if __name__ == '__main__':
596 import sys
597 EdkLogger.Initialize()
598 if len(sys.argv) > 1:
599 Br = BuildRule(sys.argv[1])
600 print str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "MSFT"][1])
601 print
602 print str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "INTEL"][1])
603 print
604 print str(Br[".c", SUP_MODULE_DXE_DRIVER, "IA32", "GCC"][1])
605 print
606 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1])
607 print
608 print str(Br[".h", "ACPI_TABLE", "IA32", "INTEL"][1])
609 print
610 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1])
611 print
612 print str(Br[".s", SUP_MODULE_SEC, "IPF", "COMMON"][1])
613 print
614 print str(Br[".s", SUP_MODULE_SEC][1])
615