]> git.proxmox.com Git - mirror_edk2.git/blob - BaseTools/Source/Python/Trim/Trim.py
228779b5a9a16f22cbad12e4c8770f0dc6569d29
[mirror_edk2.git] / BaseTools / Source / Python / Trim / Trim.py
1 ## @file
2 # Trim files preprocessed by compiler
3 #
4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import Common.LongFilePathOs as os
18 import sys
19 import re
20 from io import BytesIO
21
22 from optparse import OptionParser
23 from optparse import make_option
24 from Common.BuildToolError import *
25 from Common.Misc import *
26 from Common.DataType import *
27 from Common.BuildVersion import gBUILD_VERSION
28 import Common.EdkLogger as EdkLogger
29 from Common.LongFilePathSupport import OpenLongFilePath as open
30
31 # Version and Copyright
32 __version_number__ = ("0.10" + " " + gBUILD_VERSION)
33 __version__ = "%prog Version " + __version_number__
34 __copyright__ = "Copyright (c) 2007-2018, Intel Corporation. All rights reserved."
35
36 ## Regular expression for matching Line Control directive like "#line xxx"
37 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
38 ## Regular expression for matching "typedef struct"
39 gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)
40 ## Regular expression for matching "#pragma pack"
41 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
42 ## Regular expression for matching "typedef"
43 gTypedef_SinglePattern = re.compile("^\s*typedef", re.MULTILINE)
44 ## Regular expression for matching "typedef struct, typedef union, struct, union"
45 gTypedef_MulPattern = re.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re.MULTILINE)
46
47 #
48 # The following number pattern match will only match if following criteria is met:
49 # There is leading non-(alphanumeric or _) character, and no following alphanumeric or _
50 # as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match
51 #
52 ## Regular expression for matching HEX number
53 gHexNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")
54 ## Regular expression for matching decimal number with 'U' postfix
55 gDecNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")
56 ## Regular expression for matching constant with 'ULL' 'LL' postfix
57 gLongNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")
58
59 ## Regular expression for matching "Include ()" in asl file
60 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
61 ## Regular expression for matching C style #include "XXX.asl" in asl file
62 gAslCIncludePattern = re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re.MULTILINE)
63 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
64
65 ## file cache to avoid circular include in ASL file
66 gIncludedAslFile = []
67
68 ## Trim preprocessed source code
69 #
70 # Remove extra content made by preprocessor. The preprocessor must enable the
71 # line number generation option when preprocessing.
72 #
73 # @param Source File to be trimmed
74 # @param Target File to store the trimmed content
75 # @param Convert If True, convert standard HEX format to MASM format
76 #
77 def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
78 CreateDirectory(os.path.dirname(Target))
79 try:
80 f = open (Source, 'r')
81 except:
82 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
83
84 # read whole file
85 Lines = f.readlines()
86 f.close()
87
88 PreprocessedFile = ""
89 InjectedFile = ""
90 LineIndexOfOriginalFile = None
91 NewLines = []
92 LineControlDirectiveFound = False
93 for Index in range(len(Lines)):
94 Line = Lines[Index]
95 #
96 # Find out the name of files injected by preprocessor from the lines
97 # with Line Control directive
98 #
99 MatchList = gLineControlDirective.findall(Line)
100 if MatchList != []:
101 MatchList = MatchList[0]
102 if len(MatchList) == 2:
103 LineNumber = int(MatchList[0], 0)
104 InjectedFile = MatchList[1]
105 InjectedFile = os.path.normpath(InjectedFile)
106 InjectedFile = os.path.normcase(InjectedFile)
107 # The first injected file must be the preprocessed file itself
108 if PreprocessedFile == "":
109 PreprocessedFile = InjectedFile
110 LineControlDirectiveFound = True
111 continue
112 elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
113 continue
114
115 if LineIndexOfOriginalFile is None:
116 #
117 # Any non-empty lines must be from original preprocessed file.
118 # And this must be the first one.
119 #
120 LineIndexOfOriginalFile = Index
121 EdkLogger.verbose("Found original file content starting from line %d"
122 % (LineIndexOfOriginalFile + 1))
123
124 if TrimLong:
125 Line = gLongNumberPattern.sub(r"\1", Line)
126 # convert HEX number format if indicated
127 if ConvertHex:
128 Line = gHexNumberPattern.sub(r"0\2h", Line)
129 else:
130 Line = gHexNumberPattern.sub(r"\1\2", Line)
131
132 # convert Decimal number format
133 Line = gDecNumberPattern.sub(r"\1", Line)
134
135 if LineNumber is not None:
136 EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
137 # in case preprocessor removed some lines, like blank or comment lines
138 if LineNumber <= len(NewLines):
139 # possible?
140 NewLines[LineNumber - 1] = Line
141 else:
142 if LineNumber > (len(NewLines) + 1):
143 for LineIndex in range(len(NewLines), LineNumber-1):
144 NewLines.append(TAB_LINE_BREAK)
145 NewLines.append(Line)
146 LineNumber = None
147 EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
148 else:
149 NewLines.append(Line)
150
151 # in case there's no line directive or linemarker found
152 if (not LineControlDirectiveFound) and NewLines == []:
153 MulPatternFlag = False
154 SinglePatternFlag = False
155 Brace = 0
156 for Index in range(len(Lines)):
157 Line = Lines[Index]
158 if MulPatternFlag == False and gTypedef_MulPattern.search(Line) is None:
159 if SinglePatternFlag == False and gTypedef_SinglePattern.search(Line) is None:
160 # remove "#pragram pack" directive
161 if gPragmaPattern.search(Line) is None:
162 NewLines.append(Line)
163 continue
164 elif SinglePatternFlag == False:
165 SinglePatternFlag = True
166 if Line.find(";") >= 0:
167 SinglePatternFlag = False
168 elif MulPatternFlag == False:
169 # found "typedef struct, typedef union, union, struct", keep its position and set a flag
170 MulPatternFlag = True
171
172 # match { and } to find the end of typedef definition
173 if Line.find("{") >= 0:
174 Brace += 1
175 elif Line.find("}") >= 0:
176 Brace -= 1
177
178 # "typedef struct, typedef union, union, struct" must end with a ";"
179 if Brace == 0 and Line.find(";") >= 0:
180 MulPatternFlag = False
181
182 # save to file
183 try:
184 f = open (Target, 'w')
185 except:
186 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
187 f.writelines(NewLines)
188 f.close()
189
190 ## Trim preprocessed VFR file
191 #
192 # Remove extra content made by preprocessor. The preprocessor doesn't need to
193 # enable line number generation option when preprocessing.
194 #
195 # @param Source File to be trimmed
196 # @param Target File to store the trimmed content
197 #
198 def TrimPreprocessedVfr(Source, Target):
199 CreateDirectory(os.path.dirname(Target))
200
201 try:
202 f = open (Source, 'r')
203 except:
204 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
205 # read whole file
206 Lines = f.readlines()
207 f.close()
208
209 FoundTypedef = False
210 Brace = 0
211 TypedefStart = 0
212 TypedefEnd = 0
213 for Index in range(len(Lines)):
214 Line = Lines[Index]
215 # don't trim the lines from "formset" definition to the end of file
216 if Line.strip() == 'formset':
217 break
218
219 if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
220 # empty the line number directive if it's not aomong "typedef struct"
221 Lines[Index] = "\n"
222 continue
223
224 if FoundTypedef == False and gTypedefPattern.search(Line) is None:
225 # keep "#pragram pack" directive
226 if gPragmaPattern.search(Line) is None:
227 Lines[Index] = "\n"
228 continue
229 elif FoundTypedef == False:
230 # found "typedef struct", keept its position and set a flag
231 FoundTypedef = True
232 TypedefStart = Index
233
234 # match { and } to find the end of typedef definition
235 if Line.find("{") >= 0:
236 Brace += 1
237 elif Line.find("}") >= 0:
238 Brace -= 1
239
240 # "typedef struct" must end with a ";"
241 if Brace == 0 and Line.find(";") >= 0:
242 FoundTypedef = False
243 TypedefEnd = Index
244 # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
245 if Line.strip("} ;\r\n") in [TAB_GUID, "EFI_PLABEL", "PAL_CALL_RETURN"]:
246 for i in range(TypedefStart, TypedefEnd+1):
247 Lines[i] = "\n"
248
249 # save all lines trimmed
250 try:
251 f = open (Target, 'w')
252 except:
253 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
254 f.writelines(Lines)
255 f.close()
256
257 ## Read the content ASL file, including ASL included, recursively
258 #
259 # @param Source File to be read
260 # @param Indent Spaces before the Include() statement
261 # @param IncludePathList The list of external include file
262 # @param LocalSearchPath If LocalSearchPath is specified, this path will be searched
263 # first for the included file; otherwise, only the path specified
264 # in the IncludePathList will be searched.
265 #
266 def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
267 NewFileContent = []
268
269 try:
270 #
271 # Search LocalSearchPath first if it is specified.
272 #
273 if LocalSearchPath:
274 SearchPathList = [LocalSearchPath] + IncludePathList
275 else:
276 SearchPathList = IncludePathList
277
278 for IncludePath in SearchPathList:
279 IncludeFile = os.path.join(IncludePath, Source)
280 if os.path.isfile(IncludeFile):
281 F = open(IncludeFile, "r")
282 break
283 else:
284 EdkLogger.error("Trim", "Failed to find include file %s" % Source)
285 except:
286 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
287
288
289 # avoid A "include" B and B "include" A
290 IncludeFile = os.path.abspath(os.path.normpath(IncludeFile))
291 if IncludeFile in gIncludedAslFile:
292 EdkLogger.warn("Trim", "Circular include",
293 ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), IncludeFile))
294 return []
295 gIncludedAslFile.append(IncludeFile)
296
297 for Line in F:
298 LocalSearchPath = None
299 Result = gAslIncludePattern.findall(Line)
300 if len(Result) == 0:
301 Result = gAslCIncludePattern.findall(Line)
302 if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:
303 NewFileContent.append("%s%s" % (Indent, Line))
304 continue
305 #
306 # We should first search the local directory if current file are using pattern #include "XXX"
307 #
308 if Result[0][2] == '"':
309 LocalSearchPath = os.path.dirname(IncludeFile)
310 CurrentIndent = Indent + Result[0][0]
311 IncludedFile = Result[0][1]
312 NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, IncludePathList, LocalSearchPath))
313 NewFileContent.append("\n")
314
315 gIncludedAslFile.pop()
316 F.close()
317
318 return NewFileContent
319
320
321 ## Trim ASL file
322 #
323 # Replace ASL include statement with the content the included file
324 #
325 # @param Source File to be trimmed
326 # @param Target File to store the trimmed content
327 # @param IncludePathFile The file to log the external include path
328 #
329 def TrimAslFile(Source, Target, IncludePathFile):
330 CreateDirectory(os.path.dirname(Target))
331
332 SourceDir = os.path.dirname(Source)
333 if SourceDir == '':
334 SourceDir = '.'
335
336 #
337 # Add source directory as the first search directory
338 #
339 IncludePathList = [SourceDir]
340
341 #
342 # If additional include path file is specified, append them all
343 # to the search directory list.
344 #
345 if IncludePathFile:
346 try:
347 LineNum = 0
348 for Line in open(IncludePathFile, 'r'):
349 LineNum += 1
350 if Line.startswith("/I") or Line.startswith ("-I"):
351 IncludePathList.append(Line[2:].strip())
352 else:
353 EdkLogger.warn("Trim", "Invalid include line in include list file.", IncludePathFile, LineNum)
354 except:
355 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=IncludePathFile)
356
357 Lines = DoInclude(Source, '', IncludePathList)
358
359 #
360 # Undef MIN and MAX to avoid collision in ASL source code
361 #
362 Lines.insert(0, "#undef MIN\n#undef MAX\n")
363
364 # save all lines trimmed
365 try:
366 f = open (Target, 'w')
367 except:
368 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
369
370 f.writelines(Lines)
371 f.close()
372
373 def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
374 VfrNameList = []
375 if os.path.isdir(DebugDir):
376 for CurrentDir, Dirs, Files in os.walk(DebugDir):
377 for FileName in Files:
378 Name, Ext = os.path.splitext(FileName)
379 if Ext == '.c' and Name != 'AutoGen':
380 VfrNameList.append (Name + 'Bin')
381
382 VfrNameList.append (ModuleName + 'Strings')
383
384 EfiFileName = os.path.join(DebugDir, ModuleName + '.efi')
385 MapFileName = os.path.join(DebugDir, ModuleName + '.map')
386 VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, VfrNameList)
387
388 if not VfrUniOffsetList:
389 return
390
391 try:
392 fInputfile = open(OutputFile, "wb+", 0)
393 except:
394 EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)
395
396 # Use a instance of BytesIO to cache data
397 fStringIO = BytesIO()
398
399 for Item in VfrUniOffsetList:
400 if (Item[0].find("Strings") != -1):
401 #
402 # UNI offset in image.
403 # GUID + Offset
404 # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }
405 #
406 UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'
407 fStringIO.write(UniGuid)
408 UniValue = pack ('Q', int (Item[1], 16))
409 fStringIO.write (UniValue)
410 else:
411 #
412 # VFR binary offset in image.
413 # GUID + Offset
414 # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };
415 #
416 VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'
417 fStringIO.write(VfrGuid)
418 type (Item[1])
419 VfrValue = pack ('Q', int (Item[1], 16))
420 fStringIO.write (VfrValue)
421
422 #
423 # write data into file.
424 #
425 try :
426 fInputfile.write (fStringIO.getvalue())
427 except:
428 EdkLogger.error("Trim", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the file been locked or using by other applications." %OutputFile, None)
429
430 fStringIO.close ()
431 fInputfile.close ()
432
433
434 ## Parse command line options
435 #
436 # Using standard Python module optparse to parse command line option of this tool.
437 #
438 # @retval Options A optparse.Values object containing the parsed options
439 # @retval InputFile Path of file to be trimmed
440 #
441 def Options():
442 OptionList = [
443 make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
444 help="The input file is preprocessed source code, including C or assembly code"),
445 make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
446 help="The input file is preprocessed VFR file"),
447 make_option("--Vfr-Uni-Offset", dest="FileType", const="VfrOffsetBin", action="store_const",
448 help="The input file is EFI image"),
449 make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
450 help="The input file is ASL file"),
451 make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
452 help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
453
454 make_option("-l", "--trim-long", dest="TrimLong", action="store_true",
455 help="Remove postfix of long number"),
456 make_option("-i", "--include-path-file", dest="IncludePathFile",
457 help="The input file is include path list to search for ASL include file"),
458 make_option("-o", "--output", dest="OutputFile",
459 help="File to store the trimmed content"),
460 make_option("--ModuleName", dest="ModuleName", help="The module's BASE_NAME"),
461 make_option("--DebugDir", dest="DebugDir",
462 help="Debug Output directory to store the output files"),
463 make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
464 help="Run verbosely"),
465 make_option("-d", "--debug", dest="LogLevel", type="int",
466 help="Run with debug information"),
467 make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
468 help="Run quietly"),
469 make_option("-?", action="help", help="show this help message and exit"),
470 ]
471
472 # use clearer usage to override default usage message
473 UsageString = "%prog [-s|-r|-a|--Vfr-Uni-Offset] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] [--ModuleName <ModuleName>] [--DebugDir <DebugDir>] [<input_file>]"
474
475 Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
476 Parser.set_defaults(FileType="Vfr")
477 Parser.set_defaults(ConvertHex=False)
478 Parser.set_defaults(LogLevel=EdkLogger.INFO)
479
480 Options, Args = Parser.parse_args()
481
482 # error check
483 if Options.FileType == 'VfrOffsetBin':
484 if len(Args) == 0:
485 return Options, ''
486 elif len(Args) > 1:
487 EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
488 if len(Args) == 0:
489 EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
490 if len(Args) > 1:
491 EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
492
493 InputFile = Args[0]
494 return Options, InputFile
495
496 ## Entrance method
497 #
498 # This method mainly dispatch specific methods per the command line options.
499 # If no error found, return zero value so the caller of this tool can know
500 # if it's executed successfully or not.
501 #
502 # @retval 0 Tool was successful
503 # @retval 1 Tool failed
504 #
505 def Main():
506 try:
507 EdkLogger.Initialize()
508 CommandOptions, InputFile = Options()
509 if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
510 EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
511 else:
512 EdkLogger.SetLevel(CommandOptions.LogLevel)
513 except FatalError as X:
514 return 1
515
516 try:
517 if CommandOptions.FileType == "Vfr":
518 if CommandOptions.OutputFile is None:
519 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
520 TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
521 elif CommandOptions.FileType == "Asl":
522 if CommandOptions.OutputFile is None:
523 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
524 TrimAslFile(InputFile, CommandOptions.OutputFile, CommandOptions.IncludePathFile)
525 elif CommandOptions.FileType == "VfrOffsetBin":
526 GenerateVfrBinSec(CommandOptions.ModuleName, CommandOptions.DebugDir, CommandOptions.OutputFile)
527 else :
528 if CommandOptions.OutputFile is None:
529 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
530 TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex, CommandOptions.TrimLong)
531 except FatalError as X:
532 import platform
533 import traceback
534 if CommandOptions is not None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
535 EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
536 return 1
537 except:
538 import traceback
539 import platform
540 EdkLogger.error(
541 "\nTrim",
542 CODE_ERROR,
543 "Unknown fatal error when trimming [%s]" % InputFile,
544 ExtraData="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n",
545 RaiseError=False
546 )
547 EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
548 return 1
549
550 return 0
551
552 if __name__ == '__main__':
553 r = Main()
554 ## 0-127 is a safe return range, and 1 is a standard default error
555 if r < 0 or r > 127: r = 1
556 sys.exit(r)
557