2 # Trim files preprocessed by compiler
4 # Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 import Common
.LongFilePathOs
as os
21 from optparse
import OptionParser
22 from optparse
import make_option
23 from Common
.BuildToolError
import *
24 from Common
.Misc
import *
25 from Common
.BuildVersion
import gBUILD_VERSION
26 import Common
.EdkLogger
as EdkLogger
27 from Common
.LongFilePathSupport
import OpenLongFilePath
as open
29 # Version and Copyright
30 __version_number__
= ("0.10" + " " + gBUILD_VERSION
)
31 __version__
= "%prog Version " + __version_number__
32 __copyright__
= "Copyright (c) 2007-2016, Intel Corporation. All rights reserved."
34 ## Regular expression for matching Line Control directive like "#line xxx"
35 gLineControlDirective
= re
.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
36 ## Regular expression for matching "typedef struct"
37 gTypedefPattern
= re
.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re
.MULTILINE
)
38 ## Regular expression for matching "#pragma pack"
39 gPragmaPattern
= re
.compile("^\s*#pragma\s+pack", re
.MULTILINE
)
40 ## Regular expression for matching "typedef"
41 gTypedef_SinglePattern
= re
.compile("^\s*typedef", re
.MULTILINE
)
42 ## Regular expression for matching "typedef struct, typedef union, struct, union"
43 gTypedef_MulPattern
= re
.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re
.MULTILINE
)
46 # The following number pattern match will only match if following criteria is met:
47 # There is leading non-(alphanumeric or _) character, and no following alphanumeric or _
48 # as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match
50 ## Regular expression for matching HEX number
51 gHexNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")
52 ## Regular expression for matching decimal number with 'U' postfix
53 gDecNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")
54 ## Regular expression for matching constant with 'ULL' 'LL' postfix
55 gLongNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")
57 ## Regular expression for matching "Include ()" in asl file
58 gAslIncludePattern
= re
.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re
.MULTILINE
)
59 ## Regular expression for matching C style #include "XXX.asl" in asl file
60 gAslCIncludePattern
= re
.compile(r
'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re
.MULTILINE
)
61 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
62 gImportCodePatterns
= [
64 re
.compile('^(\s*)\(\*\*PeiServices\)\.PciCfg\s*=\s*([^;\s]+);', re
.MULTILINE
),
66 \\1 STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
67 \\1 (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
68 \\1 &gEcpPeiPciCfgPpiGuid,
71 \\1 (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
76 re
.compile('^(\s*)\(\*PeiServices\)->PciCfg\s*=\s*([^;\s]+);', re
.MULTILINE
),
78 \\1 STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
79 \\1 (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
80 \\1 &gEcpPeiPciCfgPpiGuid,
83 \\1 (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
88 re
.compile("(\s*).+->Modify[\s\n]*\(", re
.MULTILINE
),
89 '\\1PeiLibPciCfgModify ('
93 re
.compile("(\W*)gRT->ReportStatusCode[\s\n]*\(", re
.MULTILINE
),
94 '\\1EfiLibReportStatusCode ('
98 re
.compile('#include\s+EFI_GUID_DEFINITION\s*\(FirmwareFileSystem\)', re
.MULTILINE
),
99 '#include EFI_GUID_DEFINITION (FirmwareFileSystem)\n#include EFI_GUID_DEFINITION (FirmwareFileSystem2)'
103 re
.compile('gEfiFirmwareFileSystemGuid', re
.MULTILINE
),
104 'gEfiFirmwareFileSystem2Guid'
108 re
.compile('EFI_FVH_REVISION', re
.MULTILINE
),
109 'EFI_FVH_PI_REVISION'
113 re
.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_READY_TO_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re
.MULTILINE
),
114 '\\1EfiCreateEventReadyToBoot (\\2\\3;'
118 re
.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_LEGACY_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re
.MULTILINE
),
119 '\\1EfiCreateEventLegacyBoot (\\2\\3;'
122 # re.compile("(\W)(PEI_PCI_CFG_PPI)(\W)", re.MULTILINE),
127 ## file cache to avoid circular include in ASL file
128 gIncludedAslFile
= []
130 ## Trim preprocessed source code
132 # Remove extra content made by preprocessor. The preprocessor must enable the
133 # line number generation option when preprocessing.
135 # @param Source File to be trimmed
136 # @param Target File to store the trimmed content
137 # @param Convert If True, convert standard HEX format to MASM format
139 def TrimPreprocessedFile(Source
, Target
, ConvertHex
, TrimLong
):
140 CreateDirectory(os
.path
.dirname(Target
))
142 f
= open (Source
, 'r')
144 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
147 Lines
= f
.readlines()
150 PreprocessedFile
= ""
152 LineIndexOfOriginalFile
= None
154 LineControlDirectiveFound
= False
155 for Index
in range(len(Lines
)):
158 # Find out the name of files injected by preprocessor from the lines
159 # with Line Control directive
161 MatchList
= gLineControlDirective
.findall(Line
)
163 MatchList
= MatchList
[0]
164 if len(MatchList
) == 2:
165 LineNumber
= int(MatchList
[0], 0)
166 InjectedFile
= MatchList
[1]
167 # The first injetcted file must be the preprocessed file itself
168 if PreprocessedFile
== "":
169 PreprocessedFile
= InjectedFile
170 LineControlDirectiveFound
= True
172 elif PreprocessedFile
== "" or InjectedFile
!= PreprocessedFile
:
175 if LineIndexOfOriginalFile
== None:
177 # Any non-empty lines must be from original preprocessed file.
178 # And this must be the first one.
180 LineIndexOfOriginalFile
= Index
181 EdkLogger
.verbose("Found original file content starting from line %d"
182 % (LineIndexOfOriginalFile
+ 1))
185 Line
= gLongNumberPattern
.sub(r
"\1", Line
)
186 # convert HEX number format if indicated
188 Line
= gHexNumberPattern
.sub(r
"0\2h", Line
)
190 Line
= gHexNumberPattern
.sub(r
"\1\2", Line
)
192 # convert Decimal number format
193 Line
= gDecNumberPattern
.sub(r
"\1", Line
)
195 if LineNumber
!= None:
196 EdkLogger
.verbose("Got line directive: line=%d" % LineNumber
)
197 # in case preprocessor removed some lines, like blank or comment lines
198 if LineNumber
<= len(NewLines
):
200 NewLines
[LineNumber
- 1] = Line
202 if LineNumber
> (len(NewLines
) + 1):
203 for LineIndex
in range(len(NewLines
), LineNumber
-1):
204 NewLines
.append(os
.linesep
)
205 NewLines
.append(Line
)
207 EdkLogger
.verbose("Now we have lines: %d" % len(NewLines
))
209 NewLines
.append(Line
)
211 # in case there's no line directive or linemarker found
212 if (not LineControlDirectiveFound
) and NewLines
== []:
213 MulPatternFlag
= False
214 SinglePatternFlag
= False
216 for Index
in range(len(Lines
)):
218 if MulPatternFlag
== False and gTypedef_MulPattern
.search(Line
) == None:
219 if SinglePatternFlag
== False and gTypedef_SinglePattern
.search(Line
) == None:
220 # remove "#pragram pack" directive
221 if gPragmaPattern
.search(Line
) == None:
222 NewLines
.append(Line
)
224 elif SinglePatternFlag
== False:
225 SinglePatternFlag
= True
226 if Line
.find(";") >= 0:
227 SinglePatternFlag
= False
228 elif MulPatternFlag
== False:
229 # found "typedef struct, typedef union, union, struct", keep its position and set a flag
230 MulPatternFlag
= True
232 # match { and } to find the end of typedef definition
233 if Line
.find("{") >= 0:
235 elif Line
.find("}") >= 0:
238 # "typedef struct, typedef union, union, struct" must end with a ";"
239 if Brace
== 0 and Line
.find(";") >= 0:
240 MulPatternFlag
= False
244 f
= open (Target
, 'wb')
246 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
247 f
.writelines(NewLines
)
250 ## Trim preprocessed VFR file
252 # Remove extra content made by preprocessor. The preprocessor doesn't need to
253 # enable line number generation option when preprocessing.
255 # @param Source File to be trimmed
256 # @param Target File to store the trimmed content
258 def TrimPreprocessedVfr(Source
, Target
):
259 CreateDirectory(os
.path
.dirname(Target
))
262 f
= open (Source
,'r')
264 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
266 Lines
= f
.readlines()
273 for Index
in range(len(Lines
)):
275 # don't trim the lines from "formset" definition to the end of file
276 if Line
.strip() == 'formset':
279 if FoundTypedef
== False and (Line
.find('#line') == 0 or Line
.find('# ') == 0):
280 # empty the line number directive if it's not aomong "typedef struct"
284 if FoundTypedef
== False and gTypedefPattern
.search(Line
) == None:
285 # keep "#pragram pack" directive
286 if gPragmaPattern
.search(Line
) == None:
289 elif FoundTypedef
== False:
290 # found "typedef struct", keept its position and set a flag
294 # match { and } to find the end of typedef definition
295 if Line
.find("{") >= 0:
297 elif Line
.find("}") >= 0:
300 # "typedef struct" must end with a ";"
301 if Brace
== 0 and Line
.find(";") >= 0:
304 # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
305 if Line
.strip("} ;\r\n") in ["GUID", "EFI_PLABEL", "PAL_CALL_RETURN"]:
306 for i
in range(TypedefStart
, TypedefEnd
+1):
309 # save all lines trimmed
311 f
= open (Target
,'w')
313 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
317 ## Read the content ASL file, including ASL included, recursively
319 # @param Source File to be read
320 # @param Indent Spaces before the Include() statement
321 # @param IncludePathList The list of external include file
322 # @param LocalSearchPath If LocalSearchPath is specified, this path will be searched
323 # first for the included file; otherwise, only the path specified
324 # in the IncludePathList will be searched.
326 def DoInclude(Source
, Indent
='', IncludePathList
=[], LocalSearchPath
=None):
331 # Search LocalSearchPath first if it is specified.
334 SearchPathList
= [LocalSearchPath
] + IncludePathList
336 SearchPathList
= IncludePathList
338 for IncludePath
in SearchPathList
:
339 IncludeFile
= os
.path
.join(IncludePath
, Source
)
340 if os
.path
.isfile(IncludeFile
):
341 F
= open(IncludeFile
, "r")
344 EdkLogger
.error("Trim", "Failed to find include file %s" % Source
)
346 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
349 # avoid A "include" B and B "include" A
350 IncludeFile
= os
.path
.abspath(os
.path
.normpath(IncludeFile
))
351 if IncludeFile
in gIncludedAslFile
:
352 EdkLogger
.warn("Trim", "Circular include",
353 ExtraData
= "%s -> %s" % (" -> ".join(gIncludedAslFile
), IncludeFile
))
355 gIncludedAslFile
.append(IncludeFile
)
358 LocalSearchPath
= None
359 Result
= gAslIncludePattern
.findall(Line
)
361 Result
= gAslCIncludePattern
.findall(Line
)
362 if len(Result
) == 0 or os
.path
.splitext(Result
[0][1])[1].lower() not in [".asl", ".asi"]:
363 NewFileContent
.append("%s%s" % (Indent
, Line
))
366 # We should first search the local directory if current file are using pattern #include "XXX"
368 if Result
[0][2] == '"':
369 LocalSearchPath
= os
.path
.dirname(IncludeFile
)
370 CurrentIndent
= Indent
+ Result
[0][0]
371 IncludedFile
= Result
[0][1]
372 NewFileContent
.extend(DoInclude(IncludedFile
, CurrentIndent
, IncludePathList
, LocalSearchPath
))
373 NewFileContent
.append("\n")
375 gIncludedAslFile
.pop()
378 return NewFileContent
383 # Replace ASL include statement with the content the included file
385 # @param Source File to be trimmed
386 # @param Target File to store the trimmed content
387 # @param IncludePathFile The file to log the external include path
389 def TrimAslFile(Source
, Target
, IncludePathFile
):
390 CreateDirectory(os
.path
.dirname(Target
))
392 SourceDir
= os
.path
.dirname(Source
)
397 # Add source directory as the first search directory
399 IncludePathList
= [SourceDir
]
402 # If additional include path file is specified, append them all
403 # to the search directory list.
408 for Line
in open(IncludePathFile
,'r'):
410 if Line
.startswith("/I") or Line
.startswith ("-I"):
411 IncludePathList
.append(Line
[2:].strip())
413 EdkLogger
.warn("Trim", "Invalid include line in include list file.", IncludePathFile
, LineNum
)
415 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=IncludePathFile
)
417 Lines
= DoInclude(Source
, '', IncludePathList
)
420 # Undef MIN and MAX to avoid collision in ASL source code
422 Lines
.insert(0, "#undef MIN\n#undef MAX\n")
424 # save all lines trimmed
426 f
= open (Target
,'w')
428 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
433 ## Trim EDK source code file(s)
436 # @param Source File or directory to be trimmed
437 # @param Target File or directory to store the trimmed content
439 def TrimEdkSources(Source
, Target
):
440 if os
.path
.isdir(Source
):
441 for CurrentDir
, Dirs
, Files
in os
.walk(Source
):
447 for FileName
in Files
:
448 Dummy
, Ext
= os
.path
.splitext(FileName
)
449 if Ext
.upper() not in ['.C', '.H']: continue
450 if Target
== None or Target
== '':
452 os
.path
.join(CurrentDir
, FileName
),
453 os
.path
.join(CurrentDir
, FileName
)
457 os
.path
.join(CurrentDir
, FileName
),
458 os
.path
.join(Target
, CurrentDir
[len(Source
)+1:], FileName
)
461 TrimEdkSourceCode(Source
, Target
)
463 ## Trim one EDK source code file
465 # Do following replacement:
467 # (**PeiServices\).PciCfg = <*>;
469 # STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
470 # (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
471 # &gEcpPeiPciCfgPpiGuid,
474 # (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
477 # => PeiLibPciCfgModify (<*>)
479 # gRT->ReportStatusCode (<*>)
480 # => EfiLibReportStatusCode (<*>)
482 # #include <LoadFile\.h>
483 # => #include <FvLoadFile.h>
485 # CreateEvent (EFI_EVENT_SIGNAL_READY_TO_BOOT, <*>)
486 # => EfiCreateEventReadyToBoot (<*>)
488 # CreateEvent (EFI_EVENT_SIGNAL_LEGACY_BOOT, <*>)
489 # => EfiCreateEventLegacyBoot (<*>)
491 # @param Source File to be trimmed
492 # @param Target File to store the trimmed content
494 def TrimEdkSourceCode(Source
, Target
):
495 EdkLogger
.verbose("\t%s -> %s" % (Source
, Target
))
496 CreateDirectory(os
.path
.dirname(Target
))
499 f
= open (Source
,'rb')
501 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
507 for Re
,Repl
in gImportCodePatterns
:
509 NewLines
= Re
.sub(Repl
, Lines
)
511 NewLines
= Re
.sub(Repl
, NewLines
)
513 # save all lines if trimmed
514 if Source
== Target
and NewLines
== Lines
:
518 f
= open (Target
,'wb')
520 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
525 ## Parse command line options
527 # Using standard Python module optparse to parse command line option of this tool.
529 # @retval Options A optparse.Values object containing the parsed options
530 # @retval InputFile Path of file to be trimmed
534 make_option("-s", "--source-code", dest
="FileType", const
="SourceCode", action
="store_const",
535 help="The input file is preprocessed source code, including C or assembly code"),
536 make_option("-r", "--vfr-file", dest
="FileType", const
="Vfr", action
="store_const",
537 help="The input file is preprocessed VFR file"),
538 make_option("-a", "--asl-file", dest
="FileType", const
="Asl", action
="store_const",
539 help="The input file is ASL file"),
540 make_option("-8", "--Edk-source-code", dest
="FileType", const
="EdkSourceCode", action
="store_const",
541 help="The input file is source code for Edk to be trimmed for ECP"),
543 make_option("-c", "--convert-hex", dest
="ConvertHex", action
="store_true",
544 help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
546 make_option("-l", "--trim-long", dest
="TrimLong", action
="store_true",
547 help="Remove postfix of long number"),
548 make_option("-i", "--include-path-file", dest
="IncludePathFile",
549 help="The input file is include path list to search for ASL include file"),
550 make_option("-o", "--output", dest
="OutputFile",
551 help="File to store the trimmed content"),
552 make_option("-v", "--verbose", dest
="LogLevel", action
="store_const", const
=EdkLogger
.VERBOSE
,
553 help="Run verbosely"),
554 make_option("-d", "--debug", dest
="LogLevel", type="int",
555 help="Run with debug information"),
556 make_option("-q", "--quiet", dest
="LogLevel", action
="store_const", const
=EdkLogger
.QUIET
,
558 make_option("-?", action
="help", help="show this help message and exit"),
561 # use clearer usage to override default usage message
562 UsageString
= "%prog [-s|-r|-a] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] <input_file>"
564 Parser
= OptionParser(description
=__copyright__
, version
=__version__
, option_list
=OptionList
, usage
=UsageString
)
565 Parser
.set_defaults(FileType
="Vfr")
566 Parser
.set_defaults(ConvertHex
=False)
567 Parser
.set_defaults(LogLevel
=EdkLogger
.INFO
)
569 Options
, Args
= Parser
.parse_args()
573 EdkLogger
.error("Trim", OPTION_MISSING
, ExtraData
=Parser
.get_usage())
575 EdkLogger
.error("Trim", OPTION_NOT_SUPPORTED
, ExtraData
=Parser
.get_usage())
578 return Options
, InputFile
582 # This method mainly dispatch specific methods per the command line options.
583 # If no error found, return zero value so the caller of this tool can know
584 # if it's executed successfully or not.
586 # @retval 0 Tool was successful
587 # @retval 1 Tool failed
591 EdkLogger
.Initialize()
592 CommandOptions
, InputFile
= Options()
593 if CommandOptions
.LogLevel
< EdkLogger
.DEBUG_9
:
594 EdkLogger
.SetLevel(CommandOptions
.LogLevel
+ 1)
596 EdkLogger
.SetLevel(CommandOptions
.LogLevel
)
597 except FatalError
, X
:
601 if CommandOptions
.FileType
== "Vfr":
602 if CommandOptions
.OutputFile
== None:
603 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
604 TrimPreprocessedVfr(InputFile
, CommandOptions
.OutputFile
)
605 elif CommandOptions
.FileType
== "Asl":
606 if CommandOptions
.OutputFile
== None:
607 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
608 TrimAslFile(InputFile
, CommandOptions
.OutputFile
, CommandOptions
.IncludePathFile
)
609 elif CommandOptions
.FileType
== "EdkSourceCode":
610 TrimEdkSources(InputFile
, CommandOptions
.OutputFile
)
612 if CommandOptions
.OutputFile
== None:
613 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
614 TrimPreprocessedFile(InputFile
, CommandOptions
.OutputFile
, CommandOptions
.ConvertHex
, CommandOptions
.TrimLong
)
615 except FatalError
, X
:
618 if CommandOptions
!= None and CommandOptions
.LogLevel
<= EdkLogger
.DEBUG_9
:
619 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
627 "Unknown fatal error when trimming [%s]" % InputFile
,
628 ExtraData
="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n",
631 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
636 if __name__
== '__main__':
638 ## 0-127 is a safe return range, and 1 is a standard default error
639 if r
< 0 or r
> 127: r
= 1