]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/AutoGen/BuildEngine.py
There is a limitation on WINDOWS OS for the length of entire file path can’t be large...
[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / BuildEngine.py
1 ## @file
2 # The engine for building files
3 #
4 # Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import Common.LongFilePathOs as os
18 import re
19 import copy
20 import string
21 from Common.LongFilePathSupport import OpenLongFilePath as open
22
23 from Common.GlobalData import *
24 from Common.BuildToolError import *
25 from Common.Misc import tdict, PathClass
26 from Common.String import NormPath
27 from Common.DataType import *
28
29 import Common.EdkLogger as EdkLogger
30
31 ## Convert file type to file list macro name
32 #
33 # @param FileType The name of file type
34 #
35 # @retval string The name of macro
36 #
37 def FileListMacro(FileType):
38 return "%sS" % FileType.replace("-", "_").upper()
39
40 ## Convert file type to list file macro name
41 #
42 # @param FileType The name of file type
43 #
44 # @retval string The name of macro
45 #
46 def ListFileMacro(FileType):
47 return "%s_LIST" % FileListMacro(FileType)
48
49 class TargetDescBlock(object):
50 _Cache_ = {} # {TargetFile : TargetDescBlock object}
51
52 # Factory method
53 def __new__(Class, Inputs, Outputs, Commands, Dependencies):
54 if Outputs[0] in Class._Cache_:
55 Tdb = Class._Cache_[Outputs[0]]
56 for File in Inputs:
57 Tdb.AddInput(File)
58 else:
59 Tdb = super(TargetDescBlock, Class).__new__(Class)
60 Tdb._Init(Inputs, Outputs, Commands, Dependencies)
61 #Class._Cache_[Outputs[0]] = Tdb
62 return Tdb
63
64 def _Init(self, Inputs, Outputs, Commands, Dependencies):
65 self.Inputs = Inputs
66 self.Outputs = Outputs
67 self.Commands = Commands
68 self.Dependencies = Dependencies
69 if self.Outputs:
70 self.Target = self.Outputs[0]
71 else:
72 self.Target = None
73
74 def __str__(self):
75 return self.Target.Path
76
77 def __hash__(self):
78 return hash(self.Target.Path)
79
80 def __eq__(self, Other):
81 if type(Other) == type(self):
82 return Other.Target.Path == self.Target.Path
83 else:
84 return str(Other) == self.Target.Path
85
86 def AddInput(self, Input):
87 if Input not in self.Inputs:
88 self.Inputs.append(Input)
89
90 def IsMultipleInput(self):
91 return len(self.Inputs) > 1
92
93 @staticmethod
94 def Renew():
95 TargetDescBlock._Cache_ = {}
96
97 ## Class for one build rule
98 #
99 # This represents a build rule which can give out corresponding command list for
100 # building the given source file(s). The result can be used for generating the
101 # target for makefile.
102 #
103 class FileBuildRule:
104 INC_LIST_MACRO = "INC_LIST"
105 INC_MACRO = "INC"
106
107 ## constructor
108 #
109 # @param Input The dictionary represeting input file(s) for a rule
110 # @param Output The list represeting output file(s) for a rule
111 # @param Command The list containing commands to generate the output from input
112 #
113 def __init__(self, Type, Input, Output, Command, ExtraDependency=None):
114 # The Input should not be empty
115 if not Input:
116 Input = []
117 if not Output:
118 Output = []
119 if not Command:
120 Command = []
121
122 self.FileListMacro = FileListMacro(Type)
123 self.ListFileMacro = ListFileMacro(Type)
124 self.IncListFileMacro = self.INC_LIST_MACRO
125
126 self.SourceFileType = Type
127 # source files listed not in "*" or "?" pattern format
128 if not ExtraDependency:
129 self.ExtraSourceFileList = []
130 else:
131 self.ExtraSourceFileList = ExtraDependency
132
133 #
134 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST.
135 # If found, generate a file to keep the input files used to get over the
136 # limitation of command line length
137 #
138 self.MacroList = []
139 self.CommandList = []
140 for CmdLine in Command:
141 self.MacroList.extend(gMacroRefPattern.findall(CmdLine))
142 # replace path separator with native one
143 self.CommandList.append(CmdLine)
144
145 # Indicate what should be generated
146 if self.FileListMacro in self.MacroList:
147 self.GenFileListMacro = True
148 else:
149 self.GenFileListMacro = False
150
151 if self.ListFileMacro in self.MacroList:
152 self.GenListFile = True
153 self.GenFileListMacro = True
154 else:
155 self.GenListFile = False
156
157 if self.INC_LIST_MACRO in self.MacroList:
158 self.GenIncListFile = True
159 else:
160 self.GenIncListFile = False
161
162 # Check input files
163 self.IsMultipleInput = False
164 self.SourceFileExtList = []
165 for File in Input:
166 Base, Ext = os.path.splitext(File)
167 if Base.find("*") >= 0:
168 # There's "*" in the file name
169 self.IsMultipleInput = True
170 self.GenFileListMacro = True
171 elif Base.find("?") < 0:
172 # There's no "*" and "?" in file name
173 self.ExtraSourceFileList.append(File)
174 continue
175 if Ext not in self.SourceFileExtList:
176 self.SourceFileExtList.append(Ext)
177
178 # Check output files
179 self.DestFileList = []
180 for File in Output:
181 self.DestFileList.append(File)
182
183 # All build targets generated by this rule for a module
184 self.BuildTargets = {}
185
186 ## str() function support
187 #
188 # @retval string
189 #
190 def __str__(self):
191 SourceString = ""
192 SourceString += " %s %s %s" % (self.SourceFileType, " ".join(self.SourceFileExtList), self.ExtraSourceFileList)
193 DestString = ", ".join(self.DestFileList)
194 CommandString = "\n\t".join(self.CommandList)
195 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString)
196
197 ## Check if given file extension is supported by this rule
198 #
199 # @param FileExt The extension of a file
200 #
201 # @retval True If the extension is supported
202 # @retval False If the extension is not supported
203 #
204 def IsSupported(self, FileExt):
205 return FileExt in self.SourceFileExtList
206
207 def Instantiate(self, Macros={}):
208 NewRuleObject = copy.copy(self)
209 NewRuleObject.BuildTargets = {}
210 NewRuleObject.DestFileList = []
211 for File in self.DestFileList:
212 NewRuleObject.DestFileList.append(PathClass(NormPath(File, Macros)))
213 return NewRuleObject
214
215 ## Apply the rule to given source file(s)
216 #
217 # @param SourceFile One file or a list of files to be built
218 # @param RelativeToDir The relative path of the source file
219 # @param PathSeparator Path separator
220 #
221 # @retval tuple (Source file in full path, List of individual sourcefiles, Destionation file, List of build commands)
222 #
223 def Apply(self, SourceFile):
224 if not self.CommandList or not self.DestFileList:
225 return None
226
227 # source file
228 if self.IsMultipleInput:
229 SrcFileName = ""
230 SrcFileBase = ""
231 SrcFileExt = ""
232 SrcFileDir = ""
233 SrcPath = ""
234 # SourceFile must be a list
235 SrcFile = "$(%s)" % self.FileListMacro
236 else:
237 SrcFileName, SrcFileBase, SrcFileExt = SourceFile.Name, SourceFile.BaseName, SourceFile.Ext
238 if SourceFile.Root:
239 SrcFileDir = SourceFile.SubDir
240 if SrcFileDir == "":
241 SrcFileDir = "."
242 else:
243 SrcFileDir = "."
244 SrcFile = SourceFile.Path
245 SrcPath = SourceFile.Dir
246
247 # destination file (the first one)
248 if self.DestFileList:
249 DestFile = self.DestFileList[0].Path
250 DestPath = self.DestFileList[0].Dir
251 DestFileName = self.DestFileList[0].Name
252 DestFileBase, DestFileExt = self.DestFileList[0].BaseName, self.DestFileList[0].Ext
253 else:
254 DestFile = ""
255 DestPath = ""
256 DestFileName = ""
257 DestFileBase = ""
258 DestFileExt = ""
259
260 BuildRulePlaceholderDict = {
261 # source file
262 "src" : SrcFile,
263 "s_path" : SrcPath,
264 "s_dir" : SrcFileDir,
265 "s_name" : SrcFileName,
266 "s_base" : SrcFileBase,
267 "s_ext" : SrcFileExt,
268 # destination file
269 "dst" : DestFile,
270 "d_path" : DestPath,
271 "d_name" : DestFileName,
272 "d_base" : DestFileBase,
273 "d_ext" : DestFileExt,
274 }
275
276 DstFile = []
277 for File in self.DestFileList:
278 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
279 File = string.Template(str(File)).safe_substitute(BuildRulePlaceholderDict)
280 DstFile.append(PathClass(File, IsBinary=True))
281
282 if DstFile[0] in self.BuildTargets:
283 TargetDesc = self.BuildTargets[DstFile[0]]
284 TargetDesc.AddInput(SourceFile)
285 else:
286 CommandList = []
287 for CommandString in self.CommandList:
288 CommandString = string.Template(CommandString).safe_substitute(BuildRulePlaceholderDict)
289 CommandString = string.Template(CommandString).safe_substitute(BuildRulePlaceholderDict)
290 CommandList.append(CommandString)
291 TargetDesc = TargetDescBlock([SourceFile], DstFile, CommandList, self.ExtraSourceFileList)
292 TargetDesc.ListFileMacro = self.ListFileMacro
293 TargetDesc.FileListMacro = self.FileListMacro
294 TargetDesc.IncListFileMacro = self.IncListFileMacro
295 TargetDesc.GenFileListMacro = self.GenFileListMacro
296 TargetDesc.GenListFile = self.GenListFile
297 TargetDesc.GenIncListFile = self.GenIncListFile
298 self.BuildTargets[DstFile[0]] = TargetDesc
299 return TargetDesc
300
301 ## Class for build rules
302 #
303 # BuildRule class parses rules defined in a file or passed by caller, and converts
304 # the rule into FileBuildRule object.
305 #
306 class BuildRule:
307 _SectionHeader = "SECTIONHEADER"
308 _Section = "SECTION"
309 _SubSectionHeader = "SUBSECTIONHEADER"
310 _SubSection = "SUBSECTION"
311 _InputFile = "INPUTFILE"
312 _OutputFile = "OUTPUTFILE"
313 _ExtraDependency = "EXTRADEPENDENCY"
314 _Command = "COMMAND"
315 _UnknownSection = "UNKNOWNSECTION"
316
317 _SubSectionList = [_InputFile, _OutputFile, _Command]
318
319 _PATH_SEP = "(+)"
320 _FileTypePattern = re.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$")
321 _BinaryFileRule = FileBuildRule(TAB_DEFAULT_BINARY_FILE, [], [os.path.join("$(OUTPUT_DIR)", "${s_name}")],
322 ["$(CP) ${src} ${dst}"], [])
323
324 ## Constructor
325 #
326 # @param File The file containing build rules in a well defined format
327 # @param Content The string list of build rules in a well defined format
328 # @param LineIndex The line number from which the parsing will begin
329 # @param SupportedFamily The list of supported tool chain families
330 #
331 def __init__(self, File=None, Content=None, LineIndex=0, SupportedFamily=["MSFT", "INTEL", "GCC", "RVCT"]):
332 self.RuleFile = File
333 # Read build rules from file if it's not none
334 if File != None:
335 try:
336 self.RuleContent = open(File, 'r').readlines()
337 except:
338 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=File)
339 elif Content != None:
340 self.RuleContent = Content
341 else:
342 EdkLogger.error("build", PARAMETER_MISSING, ExtraData="No rule file or string given")
343
344 self.SupportedToolChainFamilyList = SupportedFamily
345 self.RuleDatabase = tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object}
346 self.Ext2FileType = {} # {ext : file-type}
347 self.FileTypeList = set()
348
349 self._LineIndex = LineIndex
350 self._State = ""
351 self._RuleInfo = tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}}
352 self._FileType = ''
353 self._BuildTypeList = []
354 self._ArchList = []
355 self._FamilyList = []
356 self._TotalToolChainFamilySet = set()
357 self._RuleObjectList = [] # FileBuildRule object list
358 self._FileVersion = ""
359
360 self.Parse()
361
362 # some intrinsic rules
363 self.RuleDatabase[TAB_DEFAULT_BINARY_FILE, "COMMON", "COMMON", "COMMON"] = self._BinaryFileRule
364 self.FileTypeList.add(TAB_DEFAULT_BINARY_FILE)
365
366 ## Parse the build rule strings
367 def Parse(self):
368 self._State = self._Section
369 for Index in range(self._LineIndex, len(self.RuleContent)):
370 # Clean up the line and replace path separator with native one
371 Line = self.RuleContent[Index].strip().replace(self._PATH_SEP, os.path.sep)
372 self.RuleContent[Index] = Line
373
374 # find the build_rule_version
375 if Line and Line[0] == "#" and Line.find(TAB_BUILD_RULE_VERSION) <> -1:
376 if Line.find("=") <> -1 and Line.find("=") < (len(Line)-1) and (Line[(Line.find("=") + 1):]).split():
377 self._FileVersion = (Line[(Line.find("=") + 1):]).split()[0]
378 # skip empty or comment line
379 if Line == "" or Line[0] == "#":
380 continue
381
382 # find out section header, enclosed by []
383 if Line[0] == '[' and Line[-1] == ']':
384 # merge last section information into rule database
385 self.EndOfSection()
386 self._State = self._SectionHeader
387 # find out sub-section header, enclosed by <>
388 elif Line[0] == '<' and Line[-1] == '>':
389 if self._State != self._UnknownSection:
390 self._State = self._SubSectionHeader
391
392 # call section handler to parse each (sub)section
393 self._StateHandler[self._State](self, Index)
394 # merge last section information into rule database
395 self.EndOfSection()
396
397 ## Parse definitions under a section
398 #
399 # @param LineIndex The line index of build rule text
400 #
401 def ParseSection(self, LineIndex):
402 pass
403
404 ## Parse definitions under a subsection
405 #
406 # @param LineIndex The line index of build rule text
407 #
408 def ParseSubSection(self, LineIndex):
409 # currenly nothing here
410 pass
411
412 ## Placeholder for not supported sections
413 #
414 # @param LineIndex The line index of build rule text
415 #
416 def SkipSection(self, LineIndex):
417 pass
418
419 ## Merge section information just got into rule database
420 def EndOfSection(self):
421 Database = self.RuleDatabase
422 # if there's specific toochain family, 'COMMON' doesn't make sense any more
423 if len(self._TotalToolChainFamilySet) > 1 and 'COMMON' in self._TotalToolChainFamilySet:
424 self._TotalToolChainFamilySet.remove('COMMON')
425 for Family in self._TotalToolChainFamilySet:
426 Input = self._RuleInfo[Family, self._InputFile]
427 Output = self._RuleInfo[Family, self._OutputFile]
428 Command = self._RuleInfo[Family, self._Command]
429 ExtraDependency = self._RuleInfo[Family, self._ExtraDependency]
430
431 BuildRule = FileBuildRule(self._FileType, Input, Output, Command, ExtraDependency)
432 for BuildType in self._BuildTypeList:
433 for Arch in self._ArchList:
434 Database[self._FileType, BuildType, Arch, Family] = BuildRule
435 for FileExt in BuildRule.SourceFileExtList:
436 self.Ext2FileType[FileExt] = self._FileType
437
438 ## Parse section header
439 #
440 # @param LineIndex The line index of build rule text
441 #
442 def ParseSectionHeader(self, LineIndex):
443 self._RuleInfo = tdict(True, 2)
444 self._BuildTypeList = []
445 self._ArchList = []
446 self._FamilyList = []
447 self._TotalToolChainFamilySet = set()
448 FileType = ''
449 RuleNameList = self.RuleContent[LineIndex][1:-1].split(',')
450 for RuleName in RuleNameList:
451 Arch = 'COMMON'
452 BuildType = 'COMMON'
453 TokenList = [Token.strip().upper() for Token in RuleName.split('.')]
454 # old format: Build.File-Type
455 if TokenList[0] == "BUILD":
456 if len(TokenList) == 1:
457 EdkLogger.error("build", FORMAT_INVALID, "Invalid rule section",
458 File=self.RuleFile, Line=LineIndex+1,
459 ExtraData=self.RuleContent[LineIndex])
460
461 FileType = TokenList[1]
462 if FileType == '':
463 EdkLogger.error("build", FORMAT_INVALID, "No file type given",
464 File=self.RuleFile, Line=LineIndex+1,
465 ExtraData=self.RuleContent[LineIndex])
466 if self._FileTypePattern.match(FileType) == None:
467 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex+1,
468 ExtraData="Only character, number (non-first character), '_' and '-' are allowed in file type")
469 # new format: File-Type.Build-Type.Arch
470 else:
471 if FileType == '':
472 FileType = TokenList[0]
473 elif FileType != TokenList[0]:
474 EdkLogger.error("build", FORMAT_INVALID,
475 "Different file types are not allowed in the same rule section",
476 File=self.RuleFile, Line=LineIndex+1,
477 ExtraData=self.RuleContent[LineIndex])
478 if len(TokenList) > 1:
479 BuildType = TokenList[1]
480 if len(TokenList) > 2:
481 Arch = TokenList[2]
482 if BuildType not in self._BuildTypeList:
483 self._BuildTypeList.append(BuildType)
484 if Arch not in self._ArchList:
485 self._ArchList.append(Arch)
486
487 if 'COMMON' in self._BuildTypeList and len(self._BuildTypeList) > 1:
488 EdkLogger.error("build", FORMAT_INVALID,
489 "Specific build types must not be mixed with common one",
490 File=self.RuleFile, Line=LineIndex+1,
491 ExtraData=self.RuleContent[LineIndex])
492 if 'COMMON' in self._ArchList and len(self._ArchList) > 1:
493 EdkLogger.error("build", FORMAT_INVALID,
494 "Specific ARCH must not be mixed with common one",
495 File=self.RuleFile, Line=LineIndex+1,
496 ExtraData=self.RuleContent[LineIndex])
497
498 self._FileType = FileType
499 self._State = self._Section
500 self.FileTypeList.add(FileType)
501
502 ## Parse sub-section header
503 #
504 # @param LineIndex The line index of build rule text
505 #
506 def ParseSubSectionHeader(self, LineIndex):
507 SectionType = ""
508 List = self.RuleContent[LineIndex][1:-1].split(',')
509 FamilyList = []
510 for Section in List:
511 TokenList = Section.split('.')
512 Type = TokenList[0].strip().upper()
513
514 if SectionType == "":
515 SectionType = Type
516 elif SectionType != Type:
517 EdkLogger.error("build", FORMAT_INVALID,
518 "Two different section types are not allowed in the same sub-section",
519 File=self.RuleFile, Line=LineIndex+1,
520 ExtraData=self.RuleContent[LineIndex])
521
522 if len(TokenList) > 1:
523 Family = TokenList[1].strip().upper()
524 else:
525 Family = "COMMON"
526
527 if Family not in FamilyList:
528 FamilyList.append(Family)
529
530 self._FamilyList = FamilyList
531 self._TotalToolChainFamilySet.update(FamilyList)
532 self._State = SectionType.upper()
533 if 'COMMON' in FamilyList and len(FamilyList) > 1:
534 EdkLogger.error("build", FORMAT_INVALID,
535 "Specific tool chain family should not be mixed with general one",
536 File=self.RuleFile, Line=LineIndex+1,
537 ExtraData=self.RuleContent[LineIndex])
538 if self._State not in self._StateHandler:
539 EdkLogger.error("build", FORMAT_INVALID, File=self.RuleFile, Line=LineIndex+1,
540 ExtraData="Unknown subsection: %s" % self.RuleContent[LineIndex])
541 ## Parse <InputFile> sub-section
542 #
543 # @param LineIndex The line index of build rule text
544 #
545 def ParseInputFile(self, LineIndex):
546 FileList = [File.strip() for File in self.RuleContent[LineIndex].split(",")]
547 for ToolChainFamily in self._FamilyList:
548 InputFiles = self._RuleInfo[ToolChainFamily, self._State]
549 if InputFiles == None:
550 InputFiles = []
551 self._RuleInfo[ToolChainFamily, self._State] = InputFiles
552 InputFiles.extend(FileList)
553
554 ## Parse <ExtraDependency> sub-section
555 #
556 # @param LineIndex The line index of build rule text
557 #
558 def ParseCommon(self, LineIndex):
559 for ToolChainFamily in self._FamilyList:
560 Items = self._RuleInfo[ToolChainFamily, self._State]
561 if Items == None:
562 Items = []
563 self._RuleInfo[ToolChainFamily, self._State] = Items
564 Items.append(self.RuleContent[LineIndex])
565
566 ## Get a build rule via [] operator
567 #
568 # @param FileExt The extension of a file
569 # @param ToolChainFamily The tool chain family name
570 # @param BuildVersion The build version number. "*" means any rule
571 # is applicalbe.
572 #
573 # @retval FileType The file type string
574 # @retval FileBuildRule The object of FileBuildRule
575 #
576 # Key = (FileExt, ModuleType, Arch, ToolChainFamily)
577 def __getitem__(self, Key):
578 if not Key:
579 return None
580
581 if Key[0] in self.Ext2FileType:
582 Type = self.Ext2FileType[Key[0]]
583 elif Key[0].upper() in self.FileTypeList:
584 Type = Key[0].upper()
585 else:
586 return None
587
588 if len(Key) > 1:
589 Key = (Type,) + Key[1:]
590 else:
591 Key = (Type,)
592 return self.RuleDatabase[Key]
593
594 _StateHandler = {
595 _SectionHeader : ParseSectionHeader,
596 _Section : ParseSection,
597 _SubSectionHeader : ParseSubSectionHeader,
598 _SubSection : ParseSubSection,
599 _InputFile : ParseInputFile,
600 _OutputFile : ParseCommon,
601 _ExtraDependency : ParseCommon,
602 _Command : ParseCommon,
603 _UnknownSection : SkipSection,
604 }
605
606 # This acts like the main() function for the script, unless it is 'import'ed into another
607 # script.
608 if __name__ == '__main__':
609 import sys
610 EdkLogger.Initialize()
611 if len(sys.argv) > 1:
612 Br = BuildRule(sys.argv[1])
613 print str(Br[".c", "DXE_DRIVER", "IA32", "MSFT"][1])
614 print
615 print str(Br[".c", "DXE_DRIVER", "IA32", "INTEL"][1])
616 print
617 print str(Br[".c", "DXE_DRIVER", "IA32", "GCC"][1])
618 print
619 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1])
620 print
621 print str(Br[".h", "ACPI_TABLE", "IA32", "INTEL"][1])
622 print
623 print str(Br[".ac", "ACPI_TABLE", "IA32", "MSFT"][1])
624 print
625 print str(Br[".s", "SEC", "IPF", "COMMON"][1])
626 print
627 print str(Br[".s", "SEC"][1])
628