Sync EDKII BaseTools to BaseTools project r2065.
[mirror_edk2.git] / BaseTools / Source / Python / Trim / Trim.py
1 ## @file
2 # Trim files preprocessed by compiler
3 #
4 # Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
5 # This program and the accompanying materials
6 # are licensed and made available under the terms and conditions of the BSD License
7 # which accompanies this distribution. The full text of the license may be found at
8 # http://opensource.org/licenses/bsd-license.php
9 #
10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12 #
13
14 ##
15 # Import Modules
16 #
17 import os
18 import sys
19 import re
20
21 from optparse import OptionParser
22 from optparse import make_option
23 from Common.BuildToolError import *
24 from Common.Misc import *
25
26 import Common.EdkLogger as EdkLogger
27
28 # Version and Copyright
29 __version_number__ = "0.10"
30 __version__ = "%prog Version " + __version_number__
31 __copyright__ = "Copyright (c) 2007-2010, Intel Corporation. All rights reserved."
32
33 ## Regular expression for matching Line Control directive like "#line xxx"
34 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
35 ## Regular expression for matching "typedef struct"
36 gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)
37 ## Regular expression for matching "#pragma pack"
38 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
39 ## Regular expression for matching HEX number
40 gHexNumberPattern = re.compile("0[xX]([0-9a-fA-F]+)")
41 ## Regular expression for matching "Include ()" in asl file
42 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
43 ## Regular expression for matching C style #include "XXX.asl" in asl file
44 gAslCIncludePattern = re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*[>"]', re.MULTILINE)
45 ## Regular expression for matching constant with 'ULL' and 'UL', 'LL', 'L' postfix
46 gLongNumberPattern = re.compile("(0[xX][0-9a-fA-F]+|[0-9]+)U?LL", re.MULTILINE)
47 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
48 gImportCodePatterns = [
49 [
50 re.compile('^(\s*)\(\*\*PeiServices\)\.PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
51 '''\\1{
52 \\1 STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
53 \\1 (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
54 \\1 &gEcpPeiPciCfgPpiGuid,
55 \\1 \\2
56 \\1 };
57 \\1 (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
58 \\1}'''
59 ],
60
61 [
62 re.compile('^(\s*)\(\*PeiServices\)->PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
63 '''\\1{
64 \\1 STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
65 \\1 (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
66 \\1 &gEcpPeiPciCfgPpiGuid,
67 \\1 \\2
68 \\1 };
69 \\1 (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
70 \\1}'''
71 ],
72
73 [
74 re.compile("(\s*).+->Modify[\s\n]*\(", re.MULTILINE),
75 '\\1PeiLibPciCfgModify ('
76 ],
77
78 [
79 re.compile("(\W*)gRT->ReportStatusCode[\s\n]*\(", re.MULTILINE),
80 '\\1EfiLibReportStatusCode ('
81 ],
82
83 [
84 re.compile('#include\s+EFI_GUID_DEFINITION\s*\(FirmwareFileSystem\)', re.MULTILINE),
85 '#include EFI_GUID_DEFINITION (FirmwareFileSystem)\n#include EFI_GUID_DEFINITION (FirmwareFileSystem2)'
86 ],
87
88 [
89 re.compile('gEfiFirmwareFileSystemGuid', re.MULTILINE),
90 'gEfiFirmwareFileSystem2Guid'
91 ],
92
93 [
94 re.compile('EFI_FVH_REVISION', re.MULTILINE),
95 'EFI_FVH_PI_REVISION'
96 ],
97
98 [
99 re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_READY_TO_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
100 '\\1EfiCreateEventReadyToBoot (\\2\\3;'
101 ],
102
103 [
104 re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_LEGACY_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
105 '\\1EfiCreateEventLegacyBoot (\\2\\3;'
106 ],
107 # [
108 # re.compile("(\W)(PEI_PCI_CFG_PPI)(\W)", re.MULTILINE),
109 # '\\1ECP_\\2\\3'
110 # ]
111 ]
112
113 ## file cache to avoid circular include in ASL file
114 gIncludedAslFile = []
115
116 ## Trim preprocessed source code
117 #
118 # Remove extra content made by preprocessor. The preprocessor must enable the
119 # line number generation option when preprocessing.
120 #
121 # @param Source File to be trimmed
122 # @param Target File to store the trimmed content
123 # @param Convert If True, convert standard HEX format to MASM format
124 #
125 def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
126 CreateDirectory(os.path.dirname(Target))
127 try:
128 f = open (Source, 'r')
129 except:
130 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
131
132 # read whole file
133 Lines = f.readlines()
134 f.close()
135
136 PreprocessedFile = ""
137 InjectedFile = ""
138 LineIndexOfOriginalFile = None
139 NewLines = []
140 LineControlDirectiveFound = False
141 for Index in range(len(Lines)):
142 Line = Lines[Index]
143 #
144 # Find out the name of files injected by preprocessor from the lines
145 # with Line Control directive
146 #
147 MatchList = gLineControlDirective.findall(Line)
148 if MatchList != []:
149 MatchList = MatchList[0]
150 if len(MatchList) == 2:
151 LineNumber = int(MatchList[0], 0)
152 InjectedFile = MatchList[1]
153 # The first injetcted file must be the preprocessed file itself
154 if PreprocessedFile == "":
155 PreprocessedFile = InjectedFile
156 LineControlDirectiveFound = True
157 continue
158 elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
159 continue
160
161 if LineIndexOfOriginalFile == None:
162 #
163 # Any non-empty lines must be from original preprocessed file.
164 # And this must be the first one.
165 #
166 LineIndexOfOriginalFile = Index
167 EdkLogger.verbose("Found original file content starting from line %d"
168 % (LineIndexOfOriginalFile + 1))
169
170 # convert HEX number format if indicated
171 if ConvertHex:
172 Line = gHexNumberPattern.sub(r"0\1h", Line)
173 if TrimLong:
174 Line = gLongNumberPattern.sub(r"\1", Line)
175
176 if LineNumber != None:
177 EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
178 # in case preprocessor removed some lines, like blank or comment lines
179 if LineNumber <= len(NewLines):
180 # possible?
181 NewLines[LineNumber - 1] = Line
182 else:
183 if LineNumber > (len(NewLines) + 1):
184 for LineIndex in range(len(NewLines), LineNumber-1):
185 NewLines.append(os.linesep)
186 NewLines.append(Line)
187 LineNumber = None
188 EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
189 else:
190 NewLines.append(Line)
191
192 # in case there's no line directive or linemarker found
193 if (not LineControlDirectiveFound) and NewLines == []:
194 NewLines = Lines
195
196 # save to file
197 try:
198 f = open (Target, 'wb')
199 except:
200 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
201 f.writelines(NewLines)
202 f.close()
203
204 ## Trim preprocessed VFR file
205 #
206 # Remove extra content made by preprocessor. The preprocessor doesn't need to
207 # enable line number generation option when preprocessing.
208 #
209 # @param Source File to be trimmed
210 # @param Target File to store the trimmed content
211 #
212 def TrimPreprocessedVfr(Source, Target):
213 CreateDirectory(os.path.dirname(Target))
214
215 try:
216 f = open (Source,'r')
217 except:
218 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
219 # read whole file
220 Lines = f.readlines()
221 f.close()
222
223 FoundTypedef = False
224 Brace = 0
225 TypedefStart = 0
226 TypedefEnd = 0
227 for Index in range(len(Lines)):
228 Line = Lines[Index]
229 # don't trim the lines from "formset" definition to the end of file
230 if Line.strip() == 'formset':
231 break
232
233 if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
234 # empty the line number directive if it's not aomong "typedef struct"
235 Lines[Index] = "\n"
236 continue
237
238 if FoundTypedef == False and gTypedefPattern.search(Line) == None:
239 # keep "#pragram pack" directive
240 if gPragmaPattern.search(Line) == None:
241 Lines[Index] = "\n"
242 continue
243 elif FoundTypedef == False:
244 # found "typedef struct", keept its position and set a flag
245 FoundTypedef = True
246 TypedefStart = Index
247
248 # match { and } to find the end of typedef definition
249 if Line.find("{") >= 0:
250 Brace += 1
251 elif Line.find("}") >= 0:
252 Brace -= 1
253
254 # "typedef struct" must end with a ";"
255 if Brace == 0 and Line.find(";") >= 0:
256 FoundTypedef = False
257 TypedefEnd = Index
258 # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
259 if Line.strip("} ;\r\n") in ["GUID", "EFI_PLABEL", "PAL_CALL_RETURN"]:
260 for i in range(TypedefStart, TypedefEnd+1):
261 Lines[i] = "\n"
262
263 # save all lines trimmed
264 try:
265 f = open (Target,'w')
266 except:
267 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
268 f.writelines(Lines)
269 f.close()
270
271 ## Read the content ASL file, including ASL included, recursively
272 #
273 # @param Source File to be read
274 # @param Indent Spaces before the Include() statement
275 # @param IncludePathList The list of external include file
276 #
277 def DoInclude(Source, Indent='', IncludePathList=[]):
278 NewFileContent = []
279
280 try:
281 for IncludePath in IncludePathList:
282 IncludeFile = os.path.join(IncludePath, Source)
283 if os.path.isfile(IncludeFile):
284 F = open(IncludeFile, "r")
285 break
286 else:
287 EdkLogger.error("Trim", "Failed to find include file %s" % Source)
288 except:
289 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
290
291
292 # avoid A "include" B and B "include" A
293 IncludeFile = os.path.abspath(os.path.normpath(IncludeFile))
294 if IncludeFile in gIncludedAslFile:
295 EdkLogger.warn("Trim", "Circular include",
296 ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), IncludeFile))
297 return []
298 gIncludedAslFile.append(IncludeFile)
299
300 for Line in F:
301 Result = gAslIncludePattern.findall(Line)
302 if len(Result) == 0:
303 Result = gAslCIncludePattern.findall(Line)
304 if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:
305 NewFileContent.append("%s%s" % (Indent, Line))
306 continue
307 CurrentIndent = Indent + Result[0][0]
308 IncludedFile = Result[0][1]
309 NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, IncludePathList))
310 NewFileContent.append("\n")
311
312 gIncludedAslFile.pop()
313 F.close()
314
315 return NewFileContent
316
317
318 ## Trim ASL file
319 #
320 # Replace ASL include statement with the content the included file
321 #
322 # @param Source File to be trimmed
323 # @param Target File to store the trimmed content
324 # @param IncludePathFile The file to log the external include path
325 #
326 def TrimAslFile(Source, Target, IncludePathFile):
327 CreateDirectory(os.path.dirname(Target))
328
329 SourceDir = os.path.dirname(Source)
330 if SourceDir == '':
331 SourceDir = '.'
332
333 #
334 # Add source directory as the first search directory
335 #
336 IncludePathList = [SourceDir]
337
338 #
339 # If additional include path file is specified, append them all
340 # to the search directory list.
341 #
342 if IncludePathFile:
343 try:
344 LineNum = 0
345 for Line in open(IncludePathFile,'r'):
346 LineNum += 1
347 if Line.startswith("/I") or Line.startswith ("-I"):
348 IncludePathList.append(Line[2:].strip())
349 else:
350 EdkLogger.warn("Trim", "Invalid include line in include list file.", IncludePathFile, LineNum)
351 except:
352 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=IncludePathFile)
353
354 Lines = DoInclude(Source, '', IncludePathList)
355
356 #
357 # Undef MIN and MAX to avoid collision in ASL source code
358 #
359 Lines.insert(0, "#undef MIN\n#undef MAX\n")
360
361 # save all lines trimmed
362 try:
363 f = open (Target,'w')
364 except:
365 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
366
367 f.writelines(Lines)
368 f.close()
369
370 ## Trim EDK source code file(s)
371 #
372 #
373 # @param Source File or directory to be trimmed
374 # @param Target File or directory to store the trimmed content
375 #
376 def TrimR8Sources(Source, Target):
377 if os.path.isdir(Source):
378 for CurrentDir, Dirs, Files in os.walk(Source):
379 if '.svn' in Dirs:
380 Dirs.remove('.svn')
381 elif "CVS" in Dirs:
382 Dirs.remove("CVS")
383
384 for FileName in Files:
385 Dummy, Ext = os.path.splitext(FileName)
386 if Ext.upper() not in ['.C', '.H']: continue
387 if Target == None or Target == '':
388 TrimR8SourceCode(
389 os.path.join(CurrentDir, FileName),
390 os.path.join(CurrentDir, FileName)
391 )
392 else:
393 TrimR8SourceCode(
394 os.path.join(CurrentDir, FileName),
395 os.path.join(Target, CurrentDir[len(Source)+1:], FileName)
396 )
397 else:
398 TrimR8SourceCode(Source, Target)
399
400 ## Trim one EDK source code file
401 #
402 # Do following replacement:
403 #
404 # (**PeiServices\).PciCfg = <*>;
405 # => {
406 # STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
407 # (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
408 # &gEcpPeiPciCfgPpiGuid,
409 # <*>
410 # };
411 # (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
412 #
413 # <*>Modify(<*>)
414 # => PeiLibPciCfgModify (<*>)
415 #
416 # gRT->ReportStatusCode (<*>)
417 # => EfiLibReportStatusCode (<*>)
418 #
419 # #include <LoadFile\.h>
420 # => #include <FvLoadFile.h>
421 #
422 # CreateEvent (EFI_EVENT_SIGNAL_READY_TO_BOOT, <*>)
423 # => EfiCreateEventReadyToBoot (<*>)
424 #
425 # CreateEvent (EFI_EVENT_SIGNAL_LEGACY_BOOT, <*>)
426 # => EfiCreateEventLegacyBoot (<*>)
427 #
428 # @param Source File to be trimmed
429 # @param Target File to store the trimmed content
430 #
431 def TrimR8SourceCode(Source, Target):
432 EdkLogger.verbose("\t%s -> %s" % (Source, Target))
433 CreateDirectory(os.path.dirname(Target))
434
435 try:
436 f = open (Source,'rb')
437 except:
438 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
439 # read whole file
440 Lines = f.read()
441 f.close()
442
443 NewLines = None
444 for Re,Repl in gImportCodePatterns:
445 if NewLines == None:
446 NewLines = Re.sub(Repl, Lines)
447 else:
448 NewLines = Re.sub(Repl, NewLines)
449
450 # save all lines if trimmed
451 if Source == Target and NewLines == Lines:
452 return
453
454 try:
455 f = open (Target,'wb')
456 except:
457 EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
458 f.write(NewLines)
459 f.close()
460
461
462 ## Parse command line options
463 #
464 # Using standard Python module optparse to parse command line option of this tool.
465 #
466 # @retval Options A optparse.Values object containing the parsed options
467 # @retval InputFile Path of file to be trimmed
468 #
469 def Options():
470 OptionList = [
471 make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
472 help="The input file is preprocessed source code, including C or assembly code"),
473 make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
474 help="The input file is preprocessed VFR file"),
475 make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
476 help="The input file is ASL file"),
477 make_option("-8", "--r8-source-code", dest="FileType", const="R8SourceCode", action="store_const",
478 help="The input file is source code for R8 to be trimmed for ECP"),
479
480 make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
481 help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
482
483 make_option("-l", "--trim-long", dest="TrimLong", action="store_true",
484 help="Remove postfix of long number"),
485 make_option("-i", "--include-path-file", dest="IncludePathFile",
486 help="The input file is include path list to search for ASL include file"),
487 make_option("-o", "--output", dest="OutputFile",
488 help="File to store the trimmed content"),
489 make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
490 help="Run verbosely"),
491 make_option("-d", "--debug", dest="LogLevel", type="int",
492 help="Run with debug information"),
493 make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
494 help="Run quietly"),
495 make_option("-?", action="help", help="show this help message and exit"),
496 ]
497
498 # use clearer usage to override default usage message
499 UsageString = "%prog [-s|-r|-a] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] <input_file>"
500
501 Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
502 Parser.set_defaults(FileType="Vfr")
503 Parser.set_defaults(ConvertHex=False)
504 Parser.set_defaults(LogLevel=EdkLogger.INFO)
505
506 Options, Args = Parser.parse_args()
507
508 # error check
509 if len(Args) == 0:
510 EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
511 if len(Args) > 1:
512 EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
513
514 InputFile = Args[0]
515 return Options, InputFile
516
517 ## Entrance method
518 #
519 # This method mainly dispatch specific methods per the command line options.
520 # If no error found, return zero value so the caller of this tool can know
521 # if it's executed successfully or not.
522 #
523 # @retval 0 Tool was successful
524 # @retval 1 Tool failed
525 #
526 def Main():
527 try:
528 EdkLogger.Initialize()
529 CommandOptions, InputFile = Options()
530 if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
531 EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
532 else:
533 EdkLogger.SetLevel(CommandOptions.LogLevel)
534 except FatalError, X:
535 return 1
536
537 try:
538 if CommandOptions.FileType == "Vfr":
539 if CommandOptions.OutputFile == None:
540 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
541 TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
542 elif CommandOptions.FileType == "Asl":
543 if CommandOptions.OutputFile == None:
544 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
545 TrimAslFile(InputFile, CommandOptions.OutputFile, CommandOptions.IncludePathFile)
546 elif CommandOptions.FileType == "R8SourceCode":
547 TrimR8Sources(InputFile, CommandOptions.OutputFile)
548 else :
549 if CommandOptions.OutputFile == None:
550 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
551 TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex, CommandOptions.TrimLong)
552 except FatalError, X:
553 import platform
554 import traceback
555 if CommandOptions != None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
556 EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
557 return 1
558 except:
559 import traceback
560 import platform
561 EdkLogger.error(
562 "\nTrim",
563 CODE_ERROR,
564 "Unknown fatal error when trimming [%s]" % InputFile,
565 ExtraData="\n(Please send email to edk2-buildtools-devel@lists.sourceforge.net for help, attaching following call stack trace!)\n",
566 RaiseError=False
567 )
568 EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
569 return 1
570
571 return 0
572
573 if __name__ == '__main__':
574 r = Main()
575 ## 0-127 is a safe return range, and 1 is a standard default error
576 if r < 0 or r > 127: r = 1
577 sys.exit(r)
578