]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/Trim/Trim.py
BaseTools: Fixed a typo in Trim.py
[mirror_edk2.git] / BaseTools / Source / Python / Trim / Trim.py
1 ## @file
2 # Trim files preprocessed by compiler
3 #
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # SPDX-License-Identifier: BSD-2-Clause-Patent
6 #
7
8 ##
9 # Import Modules
10 #
11 import Common.LongFilePathOs as os
12 import sys
13 import re
14 from io import BytesIO
15 import codecs
16 from optparse import OptionParser
17 from optparse import make_option
18 from Common.BuildToolError import *
19 from Common.Misc import *
20 from Common.DataType import *
21 from Common.BuildVersion import gBUILD_VERSION
22 import Common.EdkLogger as EdkLogger
23 from Common.LongFilePathSupport import OpenLongFilePath as open
24
25 # Version and Copyright
26 __version_number__ = ("0.10" + " " + gBUILD_VERSION)
27 __version__ = "%prog Version " + __version_number__
28 __copyright__ = "Copyright (c) 2007-2018, Intel Corporation. All rights reserved."
29
30 ## Regular expression for matching Line Control directive like "#line xxx"
31 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
32 ## Regular expression for matching "typedef struct"
33 gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)
34 ## Regular expression for matching "#pragma pack"
35 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
36 ## Regular expression for matching "typedef"
37 gTypedef_SinglePattern = re.compile("^\s*typedef", re.MULTILINE)
38 ## Regular expression for matching "typedef struct, typedef union, struct, union"
39 gTypedef_MulPattern = re.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re.MULTILINE)
40
41 #
42 # The following number pattern match will only match if following criteria is met:
43 # There is leading non-(alphanumeric or _) character, and no following alphanumeric or _
44 # as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match
45 #
46 ## Regular expression for matching HEX number
47 gHexNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")
48 ## Regular expression for matching decimal number with 'U' postfix
49 gDecNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")
50 ## Regular expression for matching constant with 'ULL' 'LL' postfix
51 gLongNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")
52
53 ## Regular expression for matching "Include ()" in asl file
54 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
55 ## Regular expression for matching C style #include "XXX.asl" in asl file
56 gAslCIncludePattern = re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re.MULTILINE)
57 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
58
59 ## file cache to avoid circular include in ASL file
60 gIncludedAslFile = []
61
62 ## Trim preprocessed source code
63 #
64 # Remove extra content made by preprocessor. The preprocessor must enable the
65 # line number generation option when preprocessing.
66 #
67 # @param Source File to be trimmed
68 # @param Target File to store the trimmed content
69 # @param Convert If True, convert standard HEX format to MASM format
70 #
71 def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
72 CreateDirectory(os.path.dirname(Target))
73 try:
74 with open(Source, "r") as File:
75 Lines = File.readlines()
76 except IOError:
77 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
78 except:
79 EdkLogger.error("Trim", AUTOGEN_ERROR, "TrimPreprocessedFile: Error while processing file", File=Source)
80
81 PreprocessedFile = ""
82 InjectedFile = ""
83 LineIndexOfOriginalFile = None
84 NewLines = []
85 LineControlDirectiveFound = False
86 for Index in range(len(Lines)):
87 Line = Lines[Index]
88 #
89 # Find out the name of files injected by preprocessor from the lines
90 # with Line Control directive
91 #
92 MatchList = gLineControlDirective.findall(Line)
93 if MatchList != []:
94 MatchList = MatchList[0]
95 if len(MatchList) == 2:
96 LineNumber = int(MatchList[0], 0)
97 InjectedFile = MatchList[1]
98 InjectedFile = os.path.normpath(InjectedFile)
99 InjectedFile = os.path.normcase(InjectedFile)
100 # The first injected file must be the preprocessed file itself
101 if PreprocessedFile == "":
102 PreprocessedFile = InjectedFile
103 LineControlDirectiveFound = True
104 continue
105 elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
106 continue
107
108 if LineIndexOfOriginalFile is None:
109 #
110 # Any non-empty lines must be from original preprocessed file.
111 # And this must be the first one.
112 #
113 LineIndexOfOriginalFile = Index
114 EdkLogger.verbose("Found original file content starting from line %d"
115 % (LineIndexOfOriginalFile + 1))
116
117 if TrimLong:
118 Line = gLongNumberPattern.sub(r"\1", Line)
119 # convert HEX number format if indicated
120 if ConvertHex:
121 Line = gHexNumberPattern.sub(r"0\2h", Line)
122 else:
123 Line = gHexNumberPattern.sub(r"\1\2", Line)
124
125 # convert Decimal number format
126 Line = gDecNumberPattern.sub(r"\1", Line)
127
128 if LineNumber is not None:
129 EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
130 # in case preprocessor removed some lines, like blank or comment lines
131 if LineNumber <= len(NewLines):
132 # possible?
133 NewLines[LineNumber - 1] = Line
134 else:
135 if LineNumber > (len(NewLines) + 1):
136 for LineIndex in range(len(NewLines), LineNumber-1):
137 NewLines.append(TAB_LINE_BREAK)
138 NewLines.append(Line)
139 LineNumber = None
140 EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
141 else:
142 NewLines.append(Line)
143
144 # in case there's no line directive or linemarker found
145 if (not LineControlDirectiveFound) and NewLines == []:
146 MulPatternFlag = False
147 SinglePatternFlag = False
148 Brace = 0
149 for Index in range(len(Lines)):
150 Line = Lines[Index]
151 if MulPatternFlag == False and gTypedef_MulPattern.search(Line) is None:
152 if SinglePatternFlag == False and gTypedef_SinglePattern.search(Line) is None:
153 # remove "#pragram pack" directive
154 if gPragmaPattern.search(Line) is None:
155 NewLines.append(Line)
156 continue
157 elif SinglePatternFlag == False:
158 SinglePatternFlag = True
159 if Line.find(";") >= 0:
160 SinglePatternFlag = False
161 elif MulPatternFlag == False:
162 # found "typedef struct, typedef union, union, struct", keep its position and set a flag
163 MulPatternFlag = True
164
165 # match { and } to find the end of typedef definition
166 if Line.find("{") >= 0:
167 Brace += 1
168 elif Line.find("}") >= 0:
169 Brace -= 1
170
171 # "typedef struct, typedef union, union, struct" must end with a ";"
172 if Brace == 0 and Line.find(";") >= 0:
173 MulPatternFlag = False
174
175 # save to file
176 try:
177 with open(Target, 'w') as File:
178 File.writelines(NewLines)
179 except:
180 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
181
182 ## Trim preprocessed VFR file
183 #
184 # Remove extra content made by preprocessor. The preprocessor doesn't need to
185 # enable line number generation option when preprocessing.
186 #
187 # @param Source File to be trimmed
188 # @param Target File to store the trimmed content
189 #
190 def TrimPreprocessedVfr(Source, Target):
191 CreateDirectory(os.path.dirname(Target))
192
193 try:
194 with open(Source, "r") as File:
195 Lines = File.readlines()
196 except:
197 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
198 # read whole file
199
200 FoundTypedef = False
201 Brace = 0
202 TypedefStart = 0
203 TypedefEnd = 0
204 for Index in range(len(Lines)):
205 Line = Lines[Index]
206 # don't trim the lines from "formset" definition to the end of file
207 if Line.strip() == 'formset':
208 break
209
210 if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
211 # empty the line number directive if it's not aomong "typedef struct"
212 Lines[Index] = "\n"
213 continue
214
215 if FoundTypedef == False and gTypedefPattern.search(Line) is None:
216 # keep "#pragram pack" directive
217 if gPragmaPattern.search(Line) is None:
218 Lines[Index] = "\n"
219 continue
220 elif FoundTypedef == False:
221 # found "typedef struct", keept its position and set a flag
222 FoundTypedef = True
223 TypedefStart = Index
224
225 # match { and } to find the end of typedef definition
226 if Line.find("{") >= 0:
227 Brace += 1
228 elif Line.find("}") >= 0:
229 Brace -= 1
230
231 # "typedef struct" must end with a ";"
232 if Brace == 0 and Line.find(";") >= 0:
233 FoundTypedef = False
234 TypedefEnd = Index
235 # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
236 if Line.strip("} ;\r\n") in [TAB_GUID, "EFI_PLABEL", "PAL_CALL_RETURN"]:
237 for i in range(TypedefStart, TypedefEnd+1):
238 Lines[i] = "\n"
239
240 # save all lines trimmed
241 try:
242 with open(Target, 'w') as File:
243 File.writelines(Lines)
244 except:
245 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
246
247 ## Read the content ASL file, including ASL included, recursively
248 #
249 # @param Source File to be read
250 # @param Indent Spaces before the Include() statement
251 # @param IncludePathList The list of external include file
252 # @param LocalSearchPath If LocalSearchPath is specified, this path will be searched
253 # first for the included file; otherwise, only the path specified
254 # in the IncludePathList will be searched.
255 #
256 def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
257 NewFileContent = []
258
259 try:
260 #
261 # Search LocalSearchPath first if it is specified.
262 #
263 if LocalSearchPath:
264 SearchPathList = [LocalSearchPath] + IncludePathList
265 else:
266 SearchPathList = IncludePathList
267
268 for IncludePath in SearchPathList:
269 IncludeFile = os.path.join(IncludePath, Source)
270 if os.path.isfile(IncludeFile):
271 try:
272 with open(IncludeFile, "r") as File:
273 F = File.readlines()
274 except:
275 with codecs.open(IncludeFile, "r", encoding='utf-8') as File:
276 F = File.readlines()
277 break
278 else:
279 EdkLogger.error("Trim", "Failed to find include file %s" % Source)
280 except:
281 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
282
283
284 # avoid A "include" B and B "include" A
285 IncludeFile = os.path.abspath(os.path.normpath(IncludeFile))
286 if IncludeFile in gIncludedAslFile:
287 EdkLogger.warn("Trim", "Circular include",
288 ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), IncludeFile))
289 return []
290 gIncludedAslFile.append(IncludeFile)
291
292 for Line in F:
293 LocalSearchPath = None
294 Result = gAslIncludePattern.findall(Line)
295 if len(Result) == 0:
296 Result = gAslCIncludePattern.findall(Line)
297 if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:
298 NewFileContent.append("%s%s" % (Indent, Line))
299 continue
300 #
301 # We should first search the local directory if current file are using pattern #include "XXX"
302 #
303 if Result[0][2] == '"':
304 LocalSearchPath = os.path.dirname(IncludeFile)
305 CurrentIndent = Indent + Result[0][0]
306 IncludedFile = Result[0][1]
307 NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, IncludePathList, LocalSearchPath))
308 NewFileContent.append("\n")
309
310 gIncludedAslFile.pop()
311
312 return NewFileContent
313
314
315 ## Trim ASL file
316 #
317 # Replace ASL include statement with the content the included file
318 #
319 # @param Source File to be trimmed
320 # @param Target File to store the trimmed content
321 # @param IncludePathFile The file to log the external include path
322 #
323 def TrimAslFile(Source, Target, IncludePathFile):
324 CreateDirectory(os.path.dirname(Target))
325
326 SourceDir = os.path.dirname(Source)
327 if SourceDir == '':
328 SourceDir = '.'
329
330 #
331 # Add source directory as the first search directory
332 #
333 IncludePathList = [SourceDir]
334
335 #
336 # If additional include path file is specified, append them all
337 # to the search directory list.
338 #
339 if IncludePathFile:
340 try:
341 LineNum = 0
342 with open(IncludePathFile, 'r') as File:
343 FileLines = File.readlines()
344 for Line in FileLines:
345 LineNum += 1
346 if Line.startswith("/I") or Line.startswith ("-I"):
347 IncludePathList.append(Line[2:].strip())
348 else:
349 EdkLogger.warn("Trim", "Invalid include line in include list file.", IncludePathFile, LineNum)
350 except:
351 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=IncludePathFile)
352
353 Lines = DoInclude(Source, '', IncludePathList)
354
355 #
356 # Undef MIN and MAX to avoid collision in ASL source code
357 #
358 Lines.insert(0, "#undef MIN\n#undef MAX\n")
359
360 # save all lines trimmed
361 try:
362 with open(Target, 'w') as File:
363 File.writelines(Lines)
364 except:
365 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
366
367 def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
368 VfrNameList = []
369 if os.path.isdir(DebugDir):
370 for CurrentDir, Dirs, Files in os.walk(DebugDir):
371 for FileName in Files:
372 Name, Ext = os.path.splitext(FileName)
373 if Ext == '.c' and Name != 'AutoGen':
374 VfrNameList.append (Name + 'Bin')
375
376 VfrNameList.append (ModuleName + 'Strings')
377
378 EfiFileName = os.path.join(DebugDir, ModuleName + '.efi')
379 MapFileName = os.path.join(DebugDir, ModuleName + '.map')
380 VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, VfrNameList)
381
382 if not VfrUniOffsetList:
383 return
384
385 try:
386 fInputfile = open(OutputFile, "wb+")
387 except:
388 EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)
389
390 # Use a instance of BytesIO to cache data
391 fStringIO = BytesIO()
392
393 for Item in VfrUniOffsetList:
394 if (Item[0].find("Strings") != -1):
395 #
396 # UNI offset in image.
397 # GUID + Offset
398 # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }
399 #
400 UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'
401 fStringIO.write(UniGuid)
402 UniValue = pack ('Q', int (Item[1], 16))
403 fStringIO.write (UniValue)
404 else:
405 #
406 # VFR binary offset in image.
407 # GUID + Offset
408 # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };
409 #
410 VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'
411 fStringIO.write(VfrGuid)
412 type (Item[1])
413 VfrValue = pack ('Q', int (Item[1], 16))
414 fStringIO.write (VfrValue)
415
416 #
417 # write data into file.
418 #
419 try :
420 fInputfile.write (fStringIO.getvalue())
421 except:
422 EdkLogger.error("Trim", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the file been locked or using by other applications." %OutputFile, None)
423
424 fStringIO.close ()
425 fInputfile.close ()
426
427
428 ## Parse command line options
429 #
430 # Using standard Python module optparse to parse command line option of this tool.
431 #
432 # @retval Options A optparse.Values object containing the parsed options
433 # @retval InputFile Path of file to be trimmed
434 #
435 def Options():
436 OptionList = [
437 make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
438 help="The input file is preprocessed source code, including C or assembly code"),
439 make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
440 help="The input file is preprocessed VFR file"),
441 make_option("--Vfr-Uni-Offset", dest="FileType", const="VfrOffsetBin", action="store_const",
442 help="The input file is EFI image"),
443 make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
444 help="The input file is ASL file"),
445 make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
446 help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
447
448 make_option("-l", "--trim-long", dest="TrimLong", action="store_true",
449 help="Remove postfix of long number"),
450 make_option("-i", "--include-path-file", dest="IncludePathFile",
451 help="The input file is include path list to search for ASL include file"),
452 make_option("-o", "--output", dest="OutputFile",
453 help="File to store the trimmed content"),
454 make_option("--ModuleName", dest="ModuleName", help="The module's BASE_NAME"),
455 make_option("--DebugDir", dest="DebugDir",
456 help="Debug Output directory to store the output files"),
457 make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
458 help="Run verbosely"),
459 make_option("-d", "--debug", dest="LogLevel", type="int",
460 help="Run with debug information"),
461 make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
462 help="Run quietly"),
463 make_option("-?", action="help", help="show this help message and exit"),
464 ]
465
466 # use clearer usage to override default usage message
467 UsageString = "%prog [-s|-r|-a|--Vfr-Uni-Offset] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] [--ModuleName <ModuleName>] [--DebugDir <DebugDir>] [<input_file>]"
468
469 Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
470 Parser.set_defaults(FileType="Vfr")
471 Parser.set_defaults(ConvertHex=False)
472 Parser.set_defaults(LogLevel=EdkLogger.INFO)
473
474 Options, Args = Parser.parse_args()
475
476 # error check
477 if Options.FileType == 'VfrOffsetBin':
478 if len(Args) == 0:
479 return Options, ''
480 elif len(Args) > 1:
481 EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
482 if len(Args) == 0:
483 EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
484 if len(Args) > 1:
485 EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
486
487 InputFile = Args[0]
488 return Options, InputFile
489
490 ## Entrance method
491 #
492 # This method mainly dispatch specific methods per the command line options.
493 # If no error found, return zero value so the caller of this tool can know
494 # if it's executed successfully or not.
495 #
496 # @retval 0 Tool was successful
497 # @retval 1 Tool failed
498 #
499 def Main():
500 try:
501 EdkLogger.Initialize()
502 CommandOptions, InputFile = Options()
503 if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
504 EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
505 else:
506 EdkLogger.SetLevel(CommandOptions.LogLevel)
507 except FatalError as X:
508 return 1
509
510 try:
511 if CommandOptions.FileType == "Vfr":
512 if CommandOptions.OutputFile is None:
513 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
514 TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
515 elif CommandOptions.FileType == "Asl":
516 if CommandOptions.OutputFile is None:
517 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
518 TrimAslFile(InputFile, CommandOptions.OutputFile, CommandOptions.IncludePathFile)
519 elif CommandOptions.FileType == "VfrOffsetBin":
520 GenerateVfrBinSec(CommandOptions.ModuleName, CommandOptions.DebugDir, CommandOptions.OutputFile)
521 else :
522 if CommandOptions.OutputFile is None:
523 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
524 TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex, CommandOptions.TrimLong)
525 except FatalError as X:
526 import platform
527 import traceback
528 if CommandOptions is not None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
529 EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
530 return 1
531 except:
532 import traceback
533 import platform
534 EdkLogger.error(
535 "\nTrim",
536 CODE_ERROR,
537 "Unknown fatal error when trimming [%s]" % InputFile,
538 ExtraData="\n(Please send email to %s for help, attaching following call stack trace!)\n" % MSG_EDKII_MAIL_ADDR,
539 RaiseError=False
540 )
541 EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
542 return 1
543
544 return 0
545
546 if __name__ == '__main__':
547 r = Main()
548 ## 0-127 is a safe return range, and 1 is a standard default error
549 if r < 0 or r > 127: r = 1
550 sys.exit(r)
551