2 # Trim files preprocessed by compiler
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 import Common
.LongFilePathOs
as os
20 from io
import BytesIO
22 from optparse
import OptionParser
23 from optparse
import make_option
24 from Common
.BuildToolError
import *
25 from Common
.Misc
import *
26 from Common
.DataType
import *
27 from Common
.BuildVersion
import gBUILD_VERSION
28 import Common
.EdkLogger
as EdkLogger
29 from Common
.LongFilePathSupport
import OpenLongFilePath
as open
31 # Version and Copyright
32 __version_number__
= ("0.10" + " " + gBUILD_VERSION
)
33 __version__
= "%prog Version " + __version_number__
34 __copyright__
= "Copyright (c) 2007-2018, Intel Corporation. All rights reserved."
36 ## Regular expression for matching Line Control directive like "#line xxx"
37 gLineControlDirective
= re
.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
38 ## Regular expression for matching "typedef struct"
39 gTypedefPattern
= re
.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re
.MULTILINE
)
40 ## Regular expression for matching "#pragma pack"
41 gPragmaPattern
= re
.compile("^\s*#pragma\s+pack", re
.MULTILINE
)
42 ## Regular expression for matching "typedef"
43 gTypedef_SinglePattern
= re
.compile("^\s*typedef", re
.MULTILINE
)
44 ## Regular expression for matching "typedef struct, typedef union, struct, union"
45 gTypedef_MulPattern
= re
.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re
.MULTILINE
)
48 # The following number pattern match will only match if following criteria is met:
49 # There is leading non-(alphanumeric or _) character, and no following alphanumeric or _
50 # as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match
52 ## Regular expression for matching HEX number
53 gHexNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")
54 ## Regular expression for matching decimal number with 'U' postfix
55 gDecNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")
56 ## Regular expression for matching constant with 'ULL' 'LL' postfix
57 gLongNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")
59 ## Regular expression for matching "Include ()" in asl file
60 gAslIncludePattern
= re
.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re
.MULTILINE
)
61 ## Regular expression for matching C style #include "XXX.asl" in asl file
62 gAslCIncludePattern
= re
.compile(r
'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re
.MULTILINE
)
63 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
65 ## file cache to avoid circular include in ASL file
68 ## Trim preprocessed source code
70 # Remove extra content made by preprocessor. The preprocessor must enable the
71 # line number generation option when preprocessing.
73 # @param Source File to be trimmed
74 # @param Target File to store the trimmed content
75 # @param Convert If True, convert standard HEX format to MASM format
77 def TrimPreprocessedFile(Source
, Target
, ConvertHex
, TrimLong
):
78 CreateDirectory(os
.path
.dirname(Target
))
80 f
= open (Source
, 'r')
82 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
90 LineIndexOfOriginalFile
= None
92 LineControlDirectiveFound
= False
93 for Index
in range(len(Lines
)):
96 # Find out the name of files injected by preprocessor from the lines
97 # with Line Control directive
99 MatchList
= gLineControlDirective
.findall(Line
)
101 MatchList
= MatchList
[0]
102 if len(MatchList
) == 2:
103 LineNumber
= int(MatchList
[0], 0)
104 InjectedFile
= MatchList
[1]
105 InjectedFile
= os
.path
.normpath(InjectedFile
)
106 InjectedFile
= os
.path
.normcase(InjectedFile
)
107 # The first injected file must be the preprocessed file itself
108 if PreprocessedFile
== "":
109 PreprocessedFile
= InjectedFile
110 LineControlDirectiveFound
= True
112 elif PreprocessedFile
== "" or InjectedFile
!= PreprocessedFile
:
115 if LineIndexOfOriginalFile
is None:
117 # Any non-empty lines must be from original preprocessed file.
118 # And this must be the first one.
120 LineIndexOfOriginalFile
= Index
121 EdkLogger
.verbose("Found original file content starting from line %d"
122 % (LineIndexOfOriginalFile
+ 1))
125 Line
= gLongNumberPattern
.sub(r
"\1", Line
)
126 # convert HEX number format if indicated
128 Line
= gHexNumberPattern
.sub(r
"0\2h", Line
)
130 Line
= gHexNumberPattern
.sub(r
"\1\2", Line
)
132 # convert Decimal number format
133 Line
= gDecNumberPattern
.sub(r
"\1", Line
)
135 if LineNumber
is not None:
136 EdkLogger
.verbose("Got line directive: line=%d" % LineNumber
)
137 # in case preprocessor removed some lines, like blank or comment lines
138 if LineNumber
<= len(NewLines
):
140 NewLines
[LineNumber
- 1] = Line
142 if LineNumber
> (len(NewLines
) + 1):
143 for LineIndex
in range(len(NewLines
), LineNumber
-1):
144 NewLines
.append(TAB_LINE_BREAK
)
145 NewLines
.append(Line
)
147 EdkLogger
.verbose("Now we have lines: %d" % len(NewLines
))
149 NewLines
.append(Line
)
151 # in case there's no line directive or linemarker found
152 if (not LineControlDirectiveFound
) and NewLines
== []:
153 MulPatternFlag
= False
154 SinglePatternFlag
= False
156 for Index
in range(len(Lines
)):
158 if MulPatternFlag
== False and gTypedef_MulPattern
.search(Line
) is None:
159 if SinglePatternFlag
== False and gTypedef_SinglePattern
.search(Line
) is None:
160 # remove "#pragram pack" directive
161 if gPragmaPattern
.search(Line
) is None:
162 NewLines
.append(Line
)
164 elif SinglePatternFlag
== False:
165 SinglePatternFlag
= True
166 if Line
.find(";") >= 0:
167 SinglePatternFlag
= False
168 elif MulPatternFlag
== False:
169 # found "typedef struct, typedef union, union, struct", keep its position and set a flag
170 MulPatternFlag
= True
172 # match { and } to find the end of typedef definition
173 if Line
.find("{") >= 0:
175 elif Line
.find("}") >= 0:
178 # "typedef struct, typedef union, union, struct" must end with a ";"
179 if Brace
== 0 and Line
.find(";") >= 0:
180 MulPatternFlag
= False
184 f
= open (Target
, 'w')
186 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
187 f
.writelines(NewLines
)
190 ## Trim preprocessed VFR file
192 # Remove extra content made by preprocessor. The preprocessor doesn't need to
193 # enable line number generation option when preprocessing.
195 # @param Source File to be trimmed
196 # @param Target File to store the trimmed content
198 def TrimPreprocessedVfr(Source
, Target
):
199 CreateDirectory(os
.path
.dirname(Target
))
202 f
= open (Source
, 'r')
204 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
206 Lines
= f
.readlines()
213 for Index
in range(len(Lines
)):
215 # don't trim the lines from "formset" definition to the end of file
216 if Line
.strip() == 'formset':
219 if FoundTypedef
== False and (Line
.find('#line') == 0 or Line
.find('# ') == 0):
220 # empty the line number directive if it's not aomong "typedef struct"
224 if FoundTypedef
== False and gTypedefPattern
.search(Line
) is None:
225 # keep "#pragram pack" directive
226 if gPragmaPattern
.search(Line
) is None:
229 elif FoundTypedef
== False:
230 # found "typedef struct", keept its position and set a flag
234 # match { and } to find the end of typedef definition
235 if Line
.find("{") >= 0:
237 elif Line
.find("}") >= 0:
240 # "typedef struct" must end with a ";"
241 if Brace
== 0 and Line
.find(";") >= 0:
244 # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
245 if Line
.strip("} ;\r\n") in [TAB_GUID
, "EFI_PLABEL", "PAL_CALL_RETURN"]:
246 for i
in range(TypedefStart
, TypedefEnd
+1):
249 # save all lines trimmed
251 f
= open (Target
, 'w')
253 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
257 ## Read the content ASL file, including ASL included, recursively
259 # @param Source File to be read
260 # @param Indent Spaces before the Include() statement
261 # @param IncludePathList The list of external include file
262 # @param LocalSearchPath If LocalSearchPath is specified, this path will be searched
263 # first for the included file; otherwise, only the path specified
264 # in the IncludePathList will be searched.
266 def DoInclude(Source
, Indent
='', IncludePathList
=[], LocalSearchPath
=None):
271 # Search LocalSearchPath first if it is specified.
274 SearchPathList
= [LocalSearchPath
] + IncludePathList
276 SearchPathList
= IncludePathList
278 for IncludePath
in SearchPathList
:
279 IncludeFile
= os
.path
.join(IncludePath
, Source
)
280 if os
.path
.isfile(IncludeFile
):
281 F
= open(IncludeFile
, "r")
284 EdkLogger
.error("Trim", "Failed to find include file %s" % Source
)
286 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
289 # avoid A "include" B and B "include" A
290 IncludeFile
= os
.path
.abspath(os
.path
.normpath(IncludeFile
))
291 if IncludeFile
in gIncludedAslFile
:
292 EdkLogger
.warn("Trim", "Circular include",
293 ExtraData
= "%s -> %s" % (" -> ".join(gIncludedAslFile
), IncludeFile
))
295 gIncludedAslFile
.append(IncludeFile
)
298 LocalSearchPath
= None
299 Result
= gAslIncludePattern
.findall(Line
)
301 Result
= gAslCIncludePattern
.findall(Line
)
302 if len(Result
) == 0 or os
.path
.splitext(Result
[0][1])[1].lower() not in [".asl", ".asi"]:
303 NewFileContent
.append("%s%s" % (Indent
, Line
))
306 # We should first search the local directory if current file are using pattern #include "XXX"
308 if Result
[0][2] == '"':
309 LocalSearchPath
= os
.path
.dirname(IncludeFile
)
310 CurrentIndent
= Indent
+ Result
[0][0]
311 IncludedFile
= Result
[0][1]
312 NewFileContent
.extend(DoInclude(IncludedFile
, CurrentIndent
, IncludePathList
, LocalSearchPath
))
313 NewFileContent
.append("\n")
315 gIncludedAslFile
.pop()
318 return NewFileContent
323 # Replace ASL include statement with the content the included file
325 # @param Source File to be trimmed
326 # @param Target File to store the trimmed content
327 # @param IncludePathFile The file to log the external include path
329 def TrimAslFile(Source
, Target
, IncludePathFile
):
330 CreateDirectory(os
.path
.dirname(Target
))
332 SourceDir
= os
.path
.dirname(Source
)
337 # Add source directory as the first search directory
339 IncludePathList
= [SourceDir
]
342 # If additional include path file is specified, append them all
343 # to the search directory list.
348 for Line
in open(IncludePathFile
, 'r'):
350 if Line
.startswith("/I") or Line
.startswith ("-I"):
351 IncludePathList
.append(Line
[2:].strip())
353 EdkLogger
.warn("Trim", "Invalid include line in include list file.", IncludePathFile
, LineNum
)
355 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=IncludePathFile
)
357 Lines
= DoInclude(Source
, '', IncludePathList
)
360 # Undef MIN and MAX to avoid collision in ASL source code
362 Lines
.insert(0, "#undef MIN\n#undef MAX\n")
364 # save all lines trimmed
366 f
= open (Target
, 'w')
368 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
373 def GenerateVfrBinSec(ModuleName
, DebugDir
, OutputFile
):
375 if os
.path
.isdir(DebugDir
):
376 for CurrentDir
, Dirs
, Files
in os
.walk(DebugDir
):
377 for FileName
in Files
:
378 Name
, Ext
= os
.path
.splitext(FileName
)
379 if Ext
== '.c' and Name
!= 'AutoGen':
380 VfrNameList
.append (Name
+ 'Bin')
382 VfrNameList
.append (ModuleName
+ 'Strings')
384 EfiFileName
= os
.path
.join(DebugDir
, ModuleName
+ '.efi')
385 MapFileName
= os
.path
.join(DebugDir
, ModuleName
+ '.map')
386 VfrUniOffsetList
= GetVariableOffset(MapFileName
, EfiFileName
, VfrNameList
)
388 if not VfrUniOffsetList
:
392 fInputfile
= open(OutputFile
, "wb+", 0)
394 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, "File open failed for %s" %OutputFile
, None)
396 # Use a instance of BytesIO to cache data
397 fStringIO
= BytesIO()
399 for Item
in VfrUniOffsetList
:
400 if (Item
[0].find("Strings") != -1):
402 # UNI offset in image.
404 # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }
406 UniGuid
= b
'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'
407 fStringIO
.write(UniGuid
)
408 UniValue
= pack ('Q', int (Item
[1], 16))
409 fStringIO
.write (UniValue
)
412 # VFR binary offset in image.
414 # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };
416 VfrGuid
= b
'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'
417 fStringIO
.write(VfrGuid
)
419 VfrValue
= pack ('Q', int (Item
[1], 16))
420 fStringIO
.write (VfrValue
)
423 # write data into file.
426 fInputfile
.write (fStringIO
.getvalue())
428 EdkLogger
.error("Trim", FILE_WRITE_FAILURE
, "Write data to file %s failed, please check whether the file been locked or using by other applications." %OutputFile
, None)
434 ## Parse command line options
436 # Using standard Python module optparse to parse command line option of this tool.
438 # @retval Options A optparse.Values object containing the parsed options
439 # @retval InputFile Path of file to be trimmed
443 make_option("-s", "--source-code", dest
="FileType", const
="SourceCode", action
="store_const",
444 help="The input file is preprocessed source code, including C or assembly code"),
445 make_option("-r", "--vfr-file", dest
="FileType", const
="Vfr", action
="store_const",
446 help="The input file is preprocessed VFR file"),
447 make_option("--Vfr-Uni-Offset", dest
="FileType", const
="VfrOffsetBin", action
="store_const",
448 help="The input file is EFI image"),
449 make_option("-a", "--asl-file", dest
="FileType", const
="Asl", action
="store_const",
450 help="The input file is ASL file"),
451 make_option("-c", "--convert-hex", dest
="ConvertHex", action
="store_true",
452 help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
454 make_option("-l", "--trim-long", dest
="TrimLong", action
="store_true",
455 help="Remove postfix of long number"),
456 make_option("-i", "--include-path-file", dest
="IncludePathFile",
457 help="The input file is include path list to search for ASL include file"),
458 make_option("-o", "--output", dest
="OutputFile",
459 help="File to store the trimmed content"),
460 make_option("--ModuleName", dest
="ModuleName", help="The module's BASE_NAME"),
461 make_option("--DebugDir", dest
="DebugDir",
462 help="Debug Output directory to store the output files"),
463 make_option("-v", "--verbose", dest
="LogLevel", action
="store_const", const
=EdkLogger
.VERBOSE
,
464 help="Run verbosely"),
465 make_option("-d", "--debug", dest
="LogLevel", type="int",
466 help="Run with debug information"),
467 make_option("-q", "--quiet", dest
="LogLevel", action
="store_const", const
=EdkLogger
.QUIET
,
469 make_option("-?", action
="help", help="show this help message and exit"),
472 # use clearer usage to override default usage message
473 UsageString
= "%prog [-s|-r|-a|--Vfr-Uni-Offset] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] [--ModuleName <ModuleName>] [--DebugDir <DebugDir>] [<input_file>]"
475 Parser
= OptionParser(description
=__copyright__
, version
=__version__
, option_list
=OptionList
, usage
=UsageString
)
476 Parser
.set_defaults(FileType
="Vfr")
477 Parser
.set_defaults(ConvertHex
=False)
478 Parser
.set_defaults(LogLevel
=EdkLogger
.INFO
)
480 Options
, Args
= Parser
.parse_args()
483 if Options
.FileType
== 'VfrOffsetBin':
487 EdkLogger
.error("Trim", OPTION_NOT_SUPPORTED
, ExtraData
=Parser
.get_usage())
489 EdkLogger
.error("Trim", OPTION_MISSING
, ExtraData
=Parser
.get_usage())
491 EdkLogger
.error("Trim", OPTION_NOT_SUPPORTED
, ExtraData
=Parser
.get_usage())
494 return Options
, InputFile
498 # This method mainly dispatch specific methods per the command line options.
499 # If no error found, return zero value so the caller of this tool can know
500 # if it's executed successfully or not.
502 # @retval 0 Tool was successful
503 # @retval 1 Tool failed
507 EdkLogger
.Initialize()
508 CommandOptions
, InputFile
= Options()
509 if CommandOptions
.LogLevel
< EdkLogger
.DEBUG_9
:
510 EdkLogger
.SetLevel(CommandOptions
.LogLevel
+ 1)
512 EdkLogger
.SetLevel(CommandOptions
.LogLevel
)
513 except FatalError
as X
:
517 if CommandOptions
.FileType
== "Vfr":
518 if CommandOptions
.OutputFile
is None:
519 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
520 TrimPreprocessedVfr(InputFile
, CommandOptions
.OutputFile
)
521 elif CommandOptions
.FileType
== "Asl":
522 if CommandOptions
.OutputFile
is None:
523 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
524 TrimAslFile(InputFile
, CommandOptions
.OutputFile
, CommandOptions
.IncludePathFile
)
525 elif CommandOptions
.FileType
== "VfrOffsetBin":
526 GenerateVfrBinSec(CommandOptions
.ModuleName
, CommandOptions
.DebugDir
, CommandOptions
.OutputFile
)
528 if CommandOptions
.OutputFile
is None:
529 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
530 TrimPreprocessedFile(InputFile
, CommandOptions
.OutputFile
, CommandOptions
.ConvertHex
, CommandOptions
.TrimLong
)
531 except FatalError
as X
:
534 if CommandOptions
is not None and CommandOptions
.LogLevel
<= EdkLogger
.DEBUG_9
:
535 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
543 "Unknown fatal error when trimming [%s]" % InputFile
,
544 ExtraData
="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n",
547 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
552 if __name__
== '__main__':
554 ## 0-127 is a safe return range, and 1 is a standard default error
555 if r
< 0 or r
> 127: r
= 1