2 # Trim files preprocessed by compiler
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # SPDX-License-Identifier: BSD-2-Clause-Patent
11 import Common
.LongFilePathOs
as os
14 from io
import BytesIO
16 from optparse
import OptionParser
17 from optparse
import make_option
18 from Common
.BuildToolError
import *
19 from Common
.Misc
import *
20 from Common
.DataType
import *
21 from Common
.BuildVersion
import gBUILD_VERSION
22 import Common
.EdkLogger
as EdkLogger
23 from Common
.LongFilePathSupport
import OpenLongFilePath
as open
25 # Version and Copyright
26 __version_number__
= ("0.10" + " " + gBUILD_VERSION
)
27 __version__
= "%prog Version " + __version_number__
28 __copyright__
= "Copyright (c) 2007-2018, Intel Corporation. All rights reserved."
30 ## Regular expression for matching Line Control directive like "#line xxx"
31 gLineControlDirective
= re
.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
32 ## Regular expression for matching "typedef struct"
33 gTypedefPattern
= re
.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re
.MULTILINE
)
34 ## Regular expression for matching "#pragma pack"
35 gPragmaPattern
= re
.compile("^\s*#pragma\s+pack", re
.MULTILINE
)
36 ## Regular expression for matching "typedef"
37 gTypedef_SinglePattern
= re
.compile("^\s*typedef", re
.MULTILINE
)
38 ## Regular expression for matching "typedef struct, typedef union, struct, union"
39 gTypedef_MulPattern
= re
.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re
.MULTILINE
)
42 # The following number pattern match will only match if following criteria is met:
43 # There is leading non-(alphanumeric or _) character, and no following alphanumeric or _
44 # as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match
46 ## Regular expression for matching HEX number
47 gHexNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")
48 ## Regular expression for matching decimal number with 'U' postfix
49 gDecNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")
50 ## Regular expression for matching constant with 'ULL' 'LL' postfix
51 gLongNumberPattern
= re
.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")
53 ## Regular expression for matching "Include ()" in asl file
54 gAslIncludePattern
= re
.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re
.MULTILINE
)
55 ## Regular expression for matching C style #include "XXX.asl" in asl file
56 gAslCIncludePattern
= re
.compile(r
'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re
.MULTILINE
)
57 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
59 ## file cache to avoid circular include in ASL file
62 ## Trim preprocessed source code
64 # Remove extra content made by preprocessor. The preprocessor must enable the
65 # line number generation option when preprocessing.
67 # @param Source File to be trimmed
68 # @param Target File to store the trimmed content
69 # @param Convert If True, convert standard HEX format to MASM format
71 def TrimPreprocessedFile(Source
, Target
, ConvertHex
, TrimLong
):
72 CreateDirectory(os
.path
.dirname(Target
))
74 with
open(Source
, "r") as File
:
75 Lines
= File
.readlines()
77 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
79 EdkLogger
.error("Trim", AUTOGEN_ERROR
, "TrimPreprocessedFile: Error while processing file", File
=Source
)
83 LineIndexOfOriginalFile
= None
85 LineControlDirectiveFound
= False
86 for Index
in range(len(Lines
)):
89 # Find out the name of files injected by preprocessor from the lines
90 # with Line Control directive
92 MatchList
= gLineControlDirective
.findall(Line
)
94 MatchList
= MatchList
[0]
95 if len(MatchList
) == 2:
96 LineNumber
= int(MatchList
[0], 0)
97 InjectedFile
= MatchList
[1]
98 InjectedFile
= os
.path
.normpath(InjectedFile
)
99 InjectedFile
= os
.path
.normcase(InjectedFile
)
100 # The first injected file must be the preprocessed file itself
101 if PreprocessedFile
== "":
102 PreprocessedFile
= InjectedFile
103 LineControlDirectiveFound
= True
105 elif PreprocessedFile
== "" or InjectedFile
!= PreprocessedFile
:
108 if LineIndexOfOriginalFile
is None:
110 # Any non-empty lines must be from original preprocessed file.
111 # And this must be the first one.
113 LineIndexOfOriginalFile
= Index
114 EdkLogger
.verbose("Found original file content starting from line %d"
115 % (LineIndexOfOriginalFile
+ 1))
118 Line
= gLongNumberPattern
.sub(r
"\1", Line
)
119 # convert HEX number format if indicated
121 Line
= gHexNumberPattern
.sub(r
"0\2h", Line
)
123 Line
= gHexNumberPattern
.sub(r
"\1\2", Line
)
125 # convert Decimal number format
126 Line
= gDecNumberPattern
.sub(r
"\1", Line
)
128 if LineNumber
is not None:
129 EdkLogger
.verbose("Got line directive: line=%d" % LineNumber
)
130 # in case preprocessor removed some lines, like blank or comment lines
131 if LineNumber
<= len(NewLines
):
133 NewLines
[LineNumber
- 1] = Line
135 if LineNumber
> (len(NewLines
) + 1):
136 for LineIndex
in range(len(NewLines
), LineNumber
-1):
137 NewLines
.append(TAB_LINE_BREAK
)
138 NewLines
.append(Line
)
140 EdkLogger
.verbose("Now we have lines: %d" % len(NewLines
))
142 NewLines
.append(Line
)
144 # in case there's no line directive or linemarker found
145 if (not LineControlDirectiveFound
) and NewLines
== []:
146 MulPatternFlag
= False
147 SinglePatternFlag
= False
149 for Index
in range(len(Lines
)):
151 if MulPatternFlag
== False and gTypedef_MulPattern
.search(Line
) is None:
152 if SinglePatternFlag
== False and gTypedef_SinglePattern
.search(Line
) is None:
153 # remove "#pragram pack" directive
154 if gPragmaPattern
.search(Line
) is None:
155 NewLines
.append(Line
)
157 elif SinglePatternFlag
== False:
158 SinglePatternFlag
= True
159 if Line
.find(";") >= 0:
160 SinglePatternFlag
= False
161 elif MulPatternFlag
== False:
162 # found "typedef struct, typedef union, union, struct", keep its position and set a flag
163 MulPatternFlag
= True
165 # match { and } to find the end of typedef definition
166 if Line
.find("{") >= 0:
168 elif Line
.find("}") >= 0:
171 # "typedef struct, typedef union, union, struct" must end with a ";"
172 if Brace
== 0 and Line
.find(";") >= 0:
173 MulPatternFlag
= False
177 with
open(Target
, 'w') as File
:
178 File
.writelines(NewLines
)
180 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
182 ## Trim preprocessed VFR file
184 # Remove extra content made by preprocessor. The preprocessor doesn't need to
185 # enable line number generation option when preprocessing.
187 # @param Source File to be trimmed
188 # @param Target File to store the trimmed content
190 def TrimPreprocessedVfr(Source
, Target
):
191 CreateDirectory(os
.path
.dirname(Target
))
194 with
open(Source
, "r") as File
:
195 Lines
= File
.readlines()
197 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
204 for Index
in range(len(Lines
)):
206 # don't trim the lines from "formset" definition to the end of file
207 if Line
.strip() == 'formset':
210 if FoundTypedef
== False and (Line
.find('#line') == 0 or Line
.find('# ') == 0):
211 # empty the line number directive if it's not aomong "typedef struct"
215 if FoundTypedef
== False and gTypedefPattern
.search(Line
) is None:
216 # keep "#pragram pack" directive
217 if gPragmaPattern
.search(Line
) is None:
220 elif FoundTypedef
== False:
221 # found "typedef struct", keept its position and set a flag
225 # match { and } to find the end of typedef definition
226 if Line
.find("{") >= 0:
228 elif Line
.find("}") >= 0:
231 # "typedef struct" must end with a ";"
232 if Brace
== 0 and Line
.find(";") >= 0:
235 # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
236 if Line
.strip("} ;\r\n") in [TAB_GUID
, "EFI_PLABEL", "PAL_CALL_RETURN"]:
237 for i
in range(TypedefStart
, TypedefEnd
+1):
240 # save all lines trimmed
242 with
open(Target
, 'w') as File
:
243 File
.writelines(Lines
)
245 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
247 ## Read the content ASL file, including ASL included, recursively
249 # @param Source File to be read
250 # @param Indent Spaces before the Include() statement
251 # @param IncludePathList The list of external include file
252 # @param LocalSearchPath If LocalSearchPath is specified, this path will be searched
253 # first for the included file; otherwise, only the path specified
254 # in the IncludePathList will be searched.
256 def DoInclude(Source
, Indent
='', IncludePathList
=[], LocalSearchPath
=None):
261 # Search LocalSearchPath first if it is specified.
264 SearchPathList
= [LocalSearchPath
] + IncludePathList
266 SearchPathList
= IncludePathList
268 for IncludePath
in SearchPathList
:
269 IncludeFile
= os
.path
.join(IncludePath
, Source
)
270 if os
.path
.isfile(IncludeFile
):
272 with
open(IncludeFile
, "r") as File
:
275 with codecs
.open(IncludeFile
, "r", encoding
='utf-8') as File
:
279 EdkLogger
.error("Trim", "Failed to find include file %s" % Source
)
281 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Source
)
284 # avoid A "include" B and B "include" A
285 IncludeFile
= os
.path
.abspath(os
.path
.normpath(IncludeFile
))
286 if IncludeFile
in gIncludedAslFile
:
287 EdkLogger
.warn("Trim", "Circular include",
288 ExtraData
= "%s -> %s" % (" -> ".join(gIncludedAslFile
), IncludeFile
))
290 gIncludedAslFile
.append(IncludeFile
)
293 LocalSearchPath
= None
294 Result
= gAslIncludePattern
.findall(Line
)
296 Result
= gAslCIncludePattern
.findall(Line
)
297 if len(Result
) == 0 or os
.path
.splitext(Result
[0][1])[1].lower() not in [".asl", ".asi"]:
298 NewFileContent
.append("%s%s" % (Indent
, Line
))
301 # We should first search the local directory if current file are using pattern #include "XXX"
303 if Result
[0][2] == '"':
304 LocalSearchPath
= os
.path
.dirname(IncludeFile
)
305 CurrentIndent
= Indent
+ Result
[0][0]
306 IncludedFile
= Result
[0][1]
307 NewFileContent
.extend(DoInclude(IncludedFile
, CurrentIndent
, IncludePathList
, LocalSearchPath
))
308 NewFileContent
.append("\n")
310 gIncludedAslFile
.pop()
312 return NewFileContent
317 # Replace ASL include statement with the content the included file
319 # @param Source File to be trimmed
320 # @param Target File to store the trimmed content
321 # @param IncludePathFile The file to log the external include path
323 def TrimAslFile(Source
, Target
, IncludePathFile
):
324 CreateDirectory(os
.path
.dirname(Target
))
326 SourceDir
= os
.path
.dirname(Source
)
331 # Add source directory as the first search directory
333 IncludePathList
= [SourceDir
]
336 # If additional include path file is specified, append them all
337 # to the search directory list.
342 with
open(IncludePathFile
, 'r') as File
:
343 FileLines
= File
.readlines()
344 for Line
in FileLines
:
346 if Line
.startswith("/I") or Line
.startswith ("-I"):
347 IncludePathList
.append(Line
[2:].strip())
349 EdkLogger
.warn("Trim", "Invalid include line in include list file.", IncludePathFile
, LineNum
)
351 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=IncludePathFile
)
353 Lines
= DoInclude(Source
, '', IncludePathList
)
356 # Undef MIN and MAX to avoid collision in ASL source code
358 Lines
.insert(0, "#undef MIN\n#undef MAX\n")
360 # save all lines trimmed
362 with
open(Target
, 'w') as File
:
363 File
.writelines(Lines
)
365 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, ExtraData
=Target
)
367 def GenerateVfrBinSec(ModuleName
, DebugDir
, OutputFile
):
369 if os
.path
.isdir(DebugDir
):
370 for CurrentDir
, Dirs
, Files
in os
.walk(DebugDir
):
371 for FileName
in Files
:
372 Name
, Ext
= os
.path
.splitext(FileName
)
373 if Ext
== '.c' and Name
!= 'AutoGen':
374 VfrNameList
.append (Name
+ 'Bin')
376 VfrNameList
.append (ModuleName
+ 'Strings')
378 EfiFileName
= os
.path
.join(DebugDir
, ModuleName
+ '.efi')
379 MapFileName
= os
.path
.join(DebugDir
, ModuleName
+ '.map')
380 VfrUniOffsetList
= GetVariableOffset(MapFileName
, EfiFileName
, VfrNameList
)
382 if not VfrUniOffsetList
:
386 fInputfile
= open(OutputFile
, "wb+")
388 EdkLogger
.error("Trim", FILE_OPEN_FAILURE
, "File open failed for %s" %OutputFile
, None)
390 # Use a instance of BytesIO to cache data
391 fStringIO
= BytesIO()
393 for Item
in VfrUniOffsetList
:
394 if (Item
[0].find("Strings") != -1):
396 # UNI offset in image.
398 # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }
400 UniGuid
= b
'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'
401 fStringIO
.write(UniGuid
)
402 UniValue
= pack ('Q', int (Item
[1], 16))
403 fStringIO
.write (UniValue
)
406 # VFR binary offset in image.
408 # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };
410 VfrGuid
= b
'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'
411 fStringIO
.write(VfrGuid
)
413 VfrValue
= pack ('Q', int (Item
[1], 16))
414 fStringIO
.write (VfrValue
)
417 # write data into file.
420 fInputfile
.write (fStringIO
.getvalue())
422 EdkLogger
.error("Trim", FILE_WRITE_FAILURE
, "Write data to file %s failed, please check whether the file been locked or using by other applications." %OutputFile
, None)
428 ## Parse command line options
430 # Using standard Python module optparse to parse command line option of this tool.
432 # @retval Options A optparse.Values object containing the parsed options
433 # @retval InputFile Path of file to be trimmed
437 make_option("-s", "--source-code", dest
="FileType", const
="SourceCode", action
="store_const",
438 help="The input file is preprocessed source code, including C or assembly code"),
439 make_option("-r", "--vfr-file", dest
="FileType", const
="Vfr", action
="store_const",
440 help="The input file is preprocessed VFR file"),
441 make_option("--Vfr-Uni-Offset", dest
="FileType", const
="VfrOffsetBin", action
="store_const",
442 help="The input file is EFI image"),
443 make_option("-a", "--asl-file", dest
="FileType", const
="Asl", action
="store_const",
444 help="The input file is ASL file"),
445 make_option("-c", "--convert-hex", dest
="ConvertHex", action
="store_true",
446 help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
448 make_option("-l", "--trim-long", dest
="TrimLong", action
="store_true",
449 help="Remove postfix of long number"),
450 make_option("-i", "--include-path-file", dest
="IncludePathFile",
451 help="The input file is include path list to search for ASL include file"),
452 make_option("-o", "--output", dest
="OutputFile",
453 help="File to store the trimmed content"),
454 make_option("--ModuleName", dest
="ModuleName", help="The module's BASE_NAME"),
455 make_option("--DebugDir", dest
="DebugDir",
456 help="Debug Output directory to store the output files"),
457 make_option("-v", "--verbose", dest
="LogLevel", action
="store_const", const
=EdkLogger
.VERBOSE
,
458 help="Run verbosely"),
459 make_option("-d", "--debug", dest
="LogLevel", type="int",
460 help="Run with debug information"),
461 make_option("-q", "--quiet", dest
="LogLevel", action
="store_const", const
=EdkLogger
.QUIET
,
463 make_option("-?", action
="help", help="show this help message and exit"),
466 # use clearer usage to override default usage message
467 UsageString
= "%prog [-s|-r|-a|--Vfr-Uni-Offset] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] [--ModuleName <ModuleName>] [--DebugDir <DebugDir>] [<input_file>]"
469 Parser
= OptionParser(description
=__copyright__
, version
=__version__
, option_list
=OptionList
, usage
=UsageString
)
470 Parser
.set_defaults(FileType
="Vfr")
471 Parser
.set_defaults(ConvertHex
=False)
472 Parser
.set_defaults(LogLevel
=EdkLogger
.INFO
)
474 Options
, Args
= Parser
.parse_args()
477 if Options
.FileType
== 'VfrOffsetBin':
481 EdkLogger
.error("Trim", OPTION_NOT_SUPPORTED
, ExtraData
=Parser
.get_usage())
483 EdkLogger
.error("Trim", OPTION_MISSING
, ExtraData
=Parser
.get_usage())
485 EdkLogger
.error("Trim", OPTION_NOT_SUPPORTED
, ExtraData
=Parser
.get_usage())
488 return Options
, InputFile
492 # This method mainly dispatch specific methods per the command line options.
493 # If no error found, return zero value so the caller of this tool can know
494 # if it's executed successfully or not.
496 # @retval 0 Tool was successful
497 # @retval 1 Tool failed
501 EdkLogger
.Initialize()
502 CommandOptions
, InputFile
= Options()
503 if CommandOptions
.LogLevel
< EdkLogger
.DEBUG_9
:
504 EdkLogger
.SetLevel(CommandOptions
.LogLevel
+ 1)
506 EdkLogger
.SetLevel(CommandOptions
.LogLevel
)
507 except FatalError
as X
:
511 if CommandOptions
.FileType
== "Vfr":
512 if CommandOptions
.OutputFile
is None:
513 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
514 TrimPreprocessedVfr(InputFile
, CommandOptions
.OutputFile
)
515 elif CommandOptions
.FileType
== "Asl":
516 if CommandOptions
.OutputFile
is None:
517 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
518 TrimAslFile(InputFile
, CommandOptions
.OutputFile
, CommandOptions
.IncludePathFile
)
519 elif CommandOptions
.FileType
== "VfrOffsetBin":
520 GenerateVfrBinSec(CommandOptions
.ModuleName
, CommandOptions
.DebugDir
, CommandOptions
.OutputFile
)
522 if CommandOptions
.OutputFile
is None:
523 CommandOptions
.OutputFile
= os
.path
.splitext(InputFile
)[0] + '.iii'
524 TrimPreprocessedFile(InputFile
, CommandOptions
.OutputFile
, CommandOptions
.ConvertHex
, CommandOptions
.TrimLong
)
525 except FatalError
as X
:
528 if CommandOptions
is not None and CommandOptions
.LogLevel
<= EdkLogger
.DEBUG_9
:
529 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
537 "Unknown fatal error when trimming [%s]" % InputFile
,
538 ExtraData
="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR
,
541 EdkLogger
.quiet("(Python %s on %s) " % (platform
.python_version(), sys
.platform
) + traceback
.format_exc())
546 if __name__
== '__main__':
548 ## 0-127 is a safe return range, and 1 is a standard default error
549 if r
< 0 or r
> 127: r
= 1