2 # The engine for building files
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 from __future__
import print_function
18 import Common
.LongFilePathOs
as os
22 from Common
.LongFilePathSupport
import OpenLongFilePath
as open
24 from Common
.GlobalData
import *
25 from Common
.BuildToolError
import *
26 from Common
.Misc
import tdict
, PathClass
27 from Common
.StringUtils
import NormPath
28 from Common
.DataType
import *
30 import Common
.EdkLogger
as EdkLogger
32 ## Convert file type to file list macro name
34 # @param FileType The name of file type
36 # @retval string The name of macro
38 def FileListMacro(FileType
):
39 return "%sS" % FileType
.replace("-", "_").upper()
41 ## Convert file type to list file macro name
43 # @param FileType The name of file type
45 # @retval string The name of macro
47 def ListFileMacro(FileType
):
48 return "%s_LIST" % FileListMacro(FileType
)
50 class TargetDescBlock(object):
51 def __init__(self
, Inputs
, Outputs
, Commands
, Dependencies
):
52 self
.InitWorker(Inputs
, Outputs
, Commands
, Dependencies
)
54 def InitWorker(self
, Inputs
, Outputs
, Commands
, Dependencies
):
56 self
.Outputs
= Outputs
57 self
.Commands
= Commands
58 self
.Dependencies
= Dependencies
60 self
.Target
= self
.Outputs
[0]
65 return self
.Target
.Path
68 return hash(self
.Target
.Path
)
70 def __eq__(self
, Other
):
71 if isinstance(Other
, type(self
)):
72 return Other
.Target
.Path
== self
.Target
.Path
74 return str(Other
) == self
.Target
.Path
76 def AddInput(self
, Input
):
77 if Input
not in self
.Inputs
:
78 self
.Inputs
.append(Input
)
80 def IsMultipleInput(self
):
81 return len(self
.Inputs
) > 1
83 ## Class for one build rule
85 # This represents a build rule which can give out corresponding command list for
86 # building the given source file(s). The result can be used for generating the
87 # target for makefile.
90 INC_LIST_MACRO
= "INC_LIST"
95 # @param Input The dictionary represeting input file(s) for a rule
96 # @param Output The list represeting output file(s) for a rule
97 # @param Command The list containing commands to generate the output from input
99 def __init__(self
, Type
, Input
, Output
, Command
, ExtraDependency
=None):
100 # The Input should not be empty
108 self
.FileListMacro
= FileListMacro(Type
)
109 self
.ListFileMacro
= ListFileMacro(Type
)
110 self
.IncListFileMacro
= self
.INC_LIST_MACRO
112 self
.SourceFileType
= Type
113 # source files listed not in "*" or "?" pattern format
114 if not ExtraDependency
:
115 self
.ExtraSourceFileList
= []
117 self
.ExtraSourceFileList
= ExtraDependency
120 # Search macros used in command lines for <FILE_TYPE>_LIST and INC_LIST.
121 # If found, generate a file to keep the input files used to get over the
122 # limitation of command line length
125 self
.CommandList
= []
126 for CmdLine
in Command
:
127 self
.MacroList
.extend(gMacroRefPattern
.findall(CmdLine
))
128 # replace path separator with native one
129 self
.CommandList
.append(CmdLine
)
131 # Indicate what should be generated
132 if self
.FileListMacro
in self
.MacroList
:
133 self
.GenFileListMacro
= True
135 self
.GenFileListMacro
= False
137 if self
.ListFileMacro
in self
.MacroList
:
138 self
.GenListFile
= True
139 self
.GenFileListMacro
= True
141 self
.GenListFile
= False
143 if self
.INC_LIST_MACRO
in self
.MacroList
:
144 self
.GenIncListFile
= True
146 self
.GenIncListFile
= False
149 self
.IsMultipleInput
= False
150 self
.SourceFileExtList
= set()
152 Base
, Ext
= os
.path
.splitext(File
)
153 if Base
.find("*") >= 0:
154 # There's "*" in the file name
155 self
.IsMultipleInput
= True
156 self
.GenFileListMacro
= True
157 elif Base
.find("?") < 0:
158 # There's no "*" and "?" in file name
159 self
.ExtraSourceFileList
.append(File
)
161 self
.SourceFileExtList
.add(Ext
)
164 self
.DestFileList
= []
166 self
.DestFileList
.append(File
)
168 # All build targets generated by this rule for a module
169 self
.BuildTargets
= {}
171 ## str() function support
177 SourceString
+= " %s %s %s" % (self
.SourceFileType
, " ".join(self
.SourceFileExtList
), self
.ExtraSourceFileList
)
178 DestString
= ", ".join(self
.DestFileList
)
179 CommandString
= "\n\t".join(self
.CommandList
)
180 return "%s : %s\n\t%s" % (DestString
, SourceString
, CommandString
)
182 def Instantiate(self
, Macros
={}):
183 NewRuleObject
= copy
.copy(self
)
184 NewRuleObject
.BuildTargets
= {}
185 NewRuleObject
.DestFileList
= []
186 for File
in self
.DestFileList
:
187 NewRuleObject
.DestFileList
.append(PathClass(NormPath(File
, Macros
)))
190 ## Apply the rule to given source file(s)
192 # @param SourceFile One file or a list of files to be built
193 # @param RelativeToDir The relative path of the source file
194 # @param PathSeparator Path separator
196 # @retval tuple (Source file in full path, List of individual sourcefiles, Destionation file, List of build commands)
198 def Apply(self
, SourceFile
, BuildRuleOrder
=None):
199 if not self
.CommandList
or not self
.DestFileList
:
203 if self
.IsMultipleInput
:
209 # SourceFile must be a list
210 SrcFile
= "$(%s)" % self
.FileListMacro
212 SrcFileName
, SrcFileBase
, SrcFileExt
= SourceFile
.Name
, SourceFile
.BaseName
, SourceFile
.Ext
214 SrcFileDir
= SourceFile
.SubDir
219 SrcFile
= SourceFile
.Path
220 SrcPath
= SourceFile
.Dir
222 # destination file (the first one)
223 if self
.DestFileList
:
224 DestFile
= self
.DestFileList
[0].Path
225 DestPath
= self
.DestFileList
[0].Dir
226 DestFileName
= self
.DestFileList
[0].Name
227 DestFileBase
, DestFileExt
= self
.DestFileList
[0].BaseName
, self
.DestFileList
[0].Ext
235 BuildRulePlaceholderDict
= {
239 "s_dir" : SrcFileDir
,
240 "s_name" : SrcFileName
,
241 "s_base" : SrcFileBase
,
242 "s_ext" : SrcFileExt
,
246 "d_name" : DestFileName
,
247 "d_base" : DestFileBase
,
248 "d_ext" : DestFileExt
,
252 for File
in self
.DestFileList
:
253 File
= string
.Template(str(File
)).safe_substitute(BuildRulePlaceholderDict
)
254 File
= string
.Template(str(File
)).safe_substitute(BuildRulePlaceholderDict
)
255 DstFile
.append(PathClass(File
, IsBinary
=True))
257 if DstFile
[0] in self
.BuildTargets
:
258 TargetDesc
= self
.BuildTargets
[DstFile
[0]]
259 if BuildRuleOrder
and SourceFile
.Ext
in BuildRuleOrder
:
260 Index
= BuildRuleOrder
.index(SourceFile
.Ext
)
261 for Input
in TargetDesc
.Inputs
:
262 if Input
.Ext
not in BuildRuleOrder
or BuildRuleOrder
.index(Input
.Ext
) > Index
:
264 # Command line should be regenerated since some macros are different
266 CommandList
= self
._BuildCommand
(BuildRulePlaceholderDict
)
267 TargetDesc
.InitWorker([SourceFile
], DstFile
, CommandList
, self
.ExtraSourceFileList
)
270 TargetDesc
.AddInput(SourceFile
)
272 CommandList
= self
._BuildCommand
(BuildRulePlaceholderDict
)
273 TargetDesc
= TargetDescBlock([SourceFile
], DstFile
, CommandList
, self
.ExtraSourceFileList
)
274 TargetDesc
.ListFileMacro
= self
.ListFileMacro
275 TargetDesc
.FileListMacro
= self
.FileListMacro
276 TargetDesc
.IncListFileMacro
= self
.IncListFileMacro
277 TargetDesc
.GenFileListMacro
= self
.GenFileListMacro
278 TargetDesc
.GenListFile
= self
.GenListFile
279 TargetDesc
.GenIncListFile
= self
.GenIncListFile
280 self
.BuildTargets
[DstFile
[0]] = TargetDesc
283 def _BuildCommand(self
, Macros
):
285 for CommandString
in self
.CommandList
:
286 CommandString
= string
.Template(CommandString
).safe_substitute(Macros
)
287 CommandString
= string
.Template(CommandString
).safe_substitute(Macros
)
288 CommandList
.append(CommandString
)
291 ## Class for build rules
293 # BuildRule class parses rules defined in a file or passed by caller, and converts
294 # the rule into FileBuildRule object.
297 _SectionHeader
= "SECTIONHEADER"
299 _SubSectionHeader
= "SUBSECTIONHEADER"
300 _SubSection
= "SUBSECTION"
301 _InputFile
= "INPUTFILE"
302 _OutputFile
= "OUTPUTFILE"
303 _ExtraDependency
= "EXTRADEPENDENCY"
305 _UnknownSection
= "UNKNOWNSECTION"
307 _SubSectionList
= [_InputFile
, _OutputFile
, _Command
]
310 _FileTypePattern
= re
.compile("^[_a-zA-Z][_\-0-9a-zA-Z]*$")
311 _BinaryFileRule
= FileBuildRule(TAB_DEFAULT_BINARY_FILE
, [], [os
.path
.join("$(OUTPUT_DIR)", "${s_name}")],
312 ["$(CP) ${src} ${dst}"], [])
316 # @param File The file containing build rules in a well defined format
317 # @param Content The string list of build rules in a well defined format
318 # @param LineIndex The line number from which the parsing will begin
319 # @param SupportedFamily The list of supported tool chain families
321 def __init__(self
, File
=None, Content
=None, LineIndex
=0, SupportedFamily
=["MSFT", "INTEL", "GCC", "RVCT"]):
323 # Read build rules from file if it's not none
326 self
.RuleContent
= open(File
, 'r').readlines()
328 EdkLogger
.error("build", FILE_OPEN_FAILURE
, ExtraData
=File
)
329 elif Content
is not None:
330 self
.RuleContent
= Content
332 EdkLogger
.error("build", PARAMETER_MISSING
, ExtraData
="No rule file or string given")
334 self
.SupportedToolChainFamilyList
= SupportedFamily
335 self
.RuleDatabase
= tdict(True, 4) # {FileExt, ModuleType, Arch, Family : FileBuildRule object}
336 self
.Ext2FileType
= {} # {ext : file-type}
337 self
.FileTypeList
= set()
339 self
._LineIndex
= LineIndex
341 self
._RuleInfo
= tdict(True, 2) # {toolchain family : {"InputFile": {}, "OutputFile" : [], "Command" : []}}
343 self
._BuildTypeList
= set()
344 self
._ArchList
= set()
345 self
._FamilyList
= []
346 self
._TotalToolChainFamilySet
= set()
347 self
._RuleObjectList
= [] # FileBuildRule object list
348 self
._FileVersion
= ""
352 # some intrinsic rules
353 self
.RuleDatabase
[TAB_DEFAULT_BINARY_FILE
, TAB_COMMON
, TAB_COMMON
, TAB_COMMON
] = self
._BinaryFileRule
354 self
.FileTypeList
.add(TAB_DEFAULT_BINARY_FILE
)
356 ## Parse the build rule strings
358 self
._State
= self
._Section
359 for Index
in range(self
._LineIndex
, len(self
.RuleContent
)):
360 # Clean up the line and replace path separator with native one
361 Line
= self
.RuleContent
[Index
].strip().replace(self
._PATH
_SEP
, os
.path
.sep
)
362 self
.RuleContent
[Index
] = Line
364 # find the build_rule_version
365 if Line
and Line
[0] == "#" and Line
.find(TAB_BUILD_RULE_VERSION
) != -1:
366 if Line
.find("=") != -1 and Line
.find("=") < (len(Line
) - 1) and (Line
[(Line
.find("=") + 1):]).split():
367 self
._FileVersion
= (Line
[(Line
.find("=") + 1):]).split()[0]
368 # skip empty or comment line
369 if Line
== "" or Line
[0] == "#":
372 # find out section header, enclosed by []
373 if Line
[0] == '[' and Line
[-1] == ']':
374 # merge last section information into rule database
376 self
._State
= self
._SectionHeader
377 # find out sub-section header, enclosed by <>
378 elif Line
[0] == '<' and Line
[-1] == '>':
379 if self
._State
!= self
._UnknownSection
:
380 self
._State
= self
._SubSectionHeader
382 # call section handler to parse each (sub)section
383 self
._StateHandler
[self
._State
](self
, Index
)
384 # merge last section information into rule database
387 ## Parse definitions under a section
389 # @param LineIndex The line index of build rule text
391 def ParseSection(self
, LineIndex
):
394 ## Parse definitions under a subsection
396 # @param LineIndex The line index of build rule text
398 def ParseSubSection(self
, LineIndex
):
399 # currenly nothing here
402 ## Placeholder for not supported sections
404 # @param LineIndex The line index of build rule text
406 def SkipSection(self
, LineIndex
):
409 ## Merge section information just got into rule database
410 def EndOfSection(self
):
411 Database
= self
.RuleDatabase
412 # if there's specific toochain family, 'COMMON' doesn't make sense any more
413 if len(self
._TotalToolChainFamilySet
) > 1 and TAB_COMMON
in self
._TotalToolChainFamilySet
:
414 self
._TotalToolChainFamilySet
.remove(TAB_COMMON
)
415 for Family
in self
._TotalToolChainFamilySet
:
416 Input
= self
._RuleInfo
[Family
, self
._InputFile
]
417 Output
= self
._RuleInfo
[Family
, self
._OutputFile
]
418 Command
= self
._RuleInfo
[Family
, self
._Command
]
419 ExtraDependency
= self
._RuleInfo
[Family
, self
._ExtraDependency
]
421 BuildRule
= FileBuildRule(self
._FileType
, Input
, Output
, Command
, ExtraDependency
)
422 for BuildType
in self
._BuildTypeList
:
423 for Arch
in self
._ArchList
:
424 Database
[self
._FileType
, BuildType
, Arch
, Family
] = BuildRule
425 for FileExt
in BuildRule
.SourceFileExtList
:
426 self
.Ext2FileType
[FileExt
] = self
._FileType
428 ## Parse section header
430 # @param LineIndex The line index of build rule text
432 def ParseSectionHeader(self
, LineIndex
):
433 self
._RuleInfo
= tdict(True, 2)
434 self
._BuildTypeList
= set()
435 self
._ArchList
= set()
436 self
._FamilyList
= []
437 self
._TotalToolChainFamilySet
= set()
439 RuleNameList
= self
.RuleContent
[LineIndex
][1:-1].split(',')
440 for RuleName
in RuleNameList
:
442 BuildType
= TAB_COMMON
443 TokenList
= [Token
.strip().upper() for Token
in RuleName
.split('.')]
444 # old format: Build.File-Type
445 if TokenList
[0] == "BUILD":
446 if len(TokenList
) == 1:
447 EdkLogger
.error("build", FORMAT_INVALID
, "Invalid rule section",
448 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
449 ExtraData
=self
.RuleContent
[LineIndex
])
451 FileType
= TokenList
[1]
453 EdkLogger
.error("build", FORMAT_INVALID
, "No file type given",
454 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
455 ExtraData
=self
.RuleContent
[LineIndex
])
456 if self
._FileTypePattern
.match(FileType
) is None:
457 EdkLogger
.error("build", FORMAT_INVALID
, File
=self
.RuleFile
, Line
=LineIndex
+ 1,
458 ExtraData
="Only character, number (non-first character), '_' and '-' are allowed in file type")
459 # new format: File-Type.Build-Type.Arch
462 FileType
= TokenList
[0]
463 elif FileType
!= TokenList
[0]:
464 EdkLogger
.error("build", FORMAT_INVALID
,
465 "Different file types are not allowed in the same rule section",
466 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
467 ExtraData
=self
.RuleContent
[LineIndex
])
468 if len(TokenList
) > 1:
469 BuildType
= TokenList
[1]
470 if len(TokenList
) > 2:
472 self
._BuildTypeList
.add(BuildType
)
473 self
._ArchList
.add(Arch
)
475 if TAB_COMMON
in self
._BuildTypeList
and len(self
._BuildTypeList
) > 1:
476 EdkLogger
.error("build", FORMAT_INVALID
,
477 "Specific build types must not be mixed with common one",
478 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
479 ExtraData
=self
.RuleContent
[LineIndex
])
480 if TAB_COMMON
in self
._ArchList
and len(self
._ArchList
) > 1:
481 EdkLogger
.error("build", FORMAT_INVALID
,
482 "Specific ARCH must not be mixed with common one",
483 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
484 ExtraData
=self
.RuleContent
[LineIndex
])
486 self
._FileType
= FileType
487 self
._State
= self
._Section
488 self
.FileTypeList
.add(FileType
)
490 ## Parse sub-section header
492 # @param LineIndex The line index of build rule text
494 def ParseSubSectionHeader(self
, LineIndex
):
496 List
= self
.RuleContent
[LineIndex
][1:-1].split(',')
499 TokenList
= Section
.split('.')
500 Type
= TokenList
[0].strip().upper()
502 if SectionType
== "":
504 elif SectionType
!= Type
:
505 EdkLogger
.error("build", FORMAT_INVALID
,
506 "Two different section types are not allowed in the same sub-section",
507 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
508 ExtraData
=self
.RuleContent
[LineIndex
])
510 if len(TokenList
) > 1:
511 Family
= TokenList
[1].strip().upper()
515 if Family
not in FamilyList
:
516 FamilyList
.append(Family
)
518 self
._FamilyList
= FamilyList
519 self
._TotalToolChainFamilySet
.update(FamilyList
)
520 self
._State
= SectionType
.upper()
521 if TAB_COMMON
in FamilyList
and len(FamilyList
) > 1:
522 EdkLogger
.error("build", FORMAT_INVALID
,
523 "Specific tool chain family should not be mixed with general one",
524 File
=self
.RuleFile
, Line
=LineIndex
+ 1,
525 ExtraData
=self
.RuleContent
[LineIndex
])
526 if self
._State
not in self
._StateHandler
:
527 EdkLogger
.error("build", FORMAT_INVALID
, File
=self
.RuleFile
, Line
=LineIndex
+ 1,
528 ExtraData
="Unknown subsection: %s" % self
.RuleContent
[LineIndex
])
529 ## Parse <InputFile> sub-section
531 # @param LineIndex The line index of build rule text
533 def ParseInputFile(self
, LineIndex
):
534 FileList
= [File
.strip() for File
in self
.RuleContent
[LineIndex
].split(",")]
535 for ToolChainFamily
in self
._FamilyList
:
536 InputFiles
= self
._RuleInfo
[ToolChainFamily
, self
._State
]
537 if InputFiles
is None:
539 self
._RuleInfo
[ToolChainFamily
, self
._State
] = InputFiles
540 InputFiles
.extend(FileList
)
542 ## Parse <ExtraDependency> sub-section
544 # @param LineIndex The line index of build rule text
546 def ParseCommon(self
, LineIndex
):
547 for ToolChainFamily
in self
._FamilyList
:
548 Items
= self
._RuleInfo
[ToolChainFamily
, self
._State
]
551 self
._RuleInfo
[ToolChainFamily
, self
._State
] = Items
552 Items
.append(self
.RuleContent
[LineIndex
])
554 ## Get a build rule via [] operator
556 # @param FileExt The extension of a file
557 # @param ToolChainFamily The tool chain family name
558 # @param BuildVersion The build version number. "*" means any rule
561 # @retval FileType The file type string
562 # @retval FileBuildRule The object of FileBuildRule
564 # Key = (FileExt, ModuleType, Arch, ToolChainFamily)
565 def __getitem__(self
, Key
):
569 if Key
[0] in self
.Ext2FileType
:
570 Type
= self
.Ext2FileType
[Key
[0]]
571 elif Key
[0].upper() in self
.FileTypeList
:
572 Type
= Key
[0].upper()
577 Key
= (Type
,) + Key
[1:]
580 return self
.RuleDatabase
[Key
]
583 _SectionHeader
: ParseSectionHeader
,
584 _Section
: ParseSection
,
585 _SubSectionHeader
: ParseSubSectionHeader
,
586 _SubSection
: ParseSubSection
,
587 _InputFile
: ParseInputFile
,
588 _OutputFile
: ParseCommon
,
589 _ExtraDependency
: ParseCommon
,
590 _Command
: ParseCommon
,
591 _UnknownSection
: SkipSection
,
594 # This acts like the main() function for the script, unless it is 'import'ed into another
596 if __name__
== '__main__':
598 EdkLogger
.Initialize()
599 if len(sys
.argv
) > 1:
600 Br
= BuildRule(sys
.argv
[1])
601 print(str(Br
[".c", SUP_MODULE_DXE_DRIVER
, "IA32", "MSFT"][1]))
603 print(str(Br
[".c", SUP_MODULE_DXE_DRIVER
, "IA32", "INTEL"][1]))
605 print(str(Br
[".c", SUP_MODULE_DXE_DRIVER
, "IA32", "GCC"][1]))
607 print(str(Br
[".ac", "ACPI_TABLE", "IA32", "MSFT"][1]))
609 print(str(Br
[".h", "ACPI_TABLE", "IA32", "INTEL"][1]))
611 print(str(Br
[".ac", "ACPI_TABLE", "IA32", "MSFT"][1]))
613 print(str(Br
[".s", SUP_MODULE_SEC
, "IPF", "COMMON"][1]))
615 print(str(Br
[".s", SUP_MODULE_SEC
][1]))