BaseTools/Source/Python/Trim/Trim.py

   1 ## @file
   2 # Trim files preprocessed by compiler
   3 #
   4 # Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os
  18 import sys
  19 import re
  20
  21 from optparse import OptionParser
  22 from optparse import make_option
  23 from Common.BuildToolError import *
  24 from Common.Misc import *
  25
  26 import Common.EdkLogger as EdkLogger
  27
  28 # Version and Copyright
  29 __version_number__ = "0.10"
  30 __version__ = "%prog Version " + __version_number__
  31 __copyright__ = "Copyright (c) 2007-2010, Intel Corporation. All rights reserved."
  32
  33 ## Regular expression for matching Line Control directive like "#line xxx"
  34 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
  35 ## Regular expression for matching "typedef struct"
  36 gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)
  37 ## Regular expression for matching "#pragma pack"
  38 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
  39 ## Regular expression for matching HEX number
  40 gHexNumberPattern = re.compile("0[xX]([0-9a-fA-F]+)")
  41 ## Regular expression for matching "Include ()" in asl file
  42 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
  43 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
  44 gImportCodePatterns = [
  45     [
  46         re.compile('^(\s*)\(\*\*PeiServices\)\.PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
  47         '''\\1{
  48 \\1  STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
  49 \\1    (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
  50 \\1    &gEcpPeiPciCfgPpiGuid,
  51 \\1    \\2
  52 \\1  };
  53 \\1  (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
  54 \\1}'''
  55     ],
  56
  57     [
  58         re.compile('^(\s*)\(\*PeiServices\)->PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
  59         '''\\1{
  60 \\1  STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
  61 \\1    (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
  62 \\1    &gEcpPeiPciCfgPpiGuid,
  63 \\1    \\2
  64 \\1  };
  65 \\1  (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
  66 \\1}'''
  67     ],
  68
  69     [
  70         re.compile("(\s*).+->Modify[\s\n]*\(", re.MULTILINE),
  71         '\\1PeiLibPciCfgModify ('
  72     ],
  73
  74     [
  75         re.compile("(\W*)gRT->ReportStatusCode[\s\n]*\(", re.MULTILINE),
  76         '\\1EfiLibReportStatusCode ('
  77     ],
  78
  79     [
  80         re.compile('#include\s+["<]LoadFile\.h[">]', re.MULTILINE),
  81         '#include <FvLoadFile.h>'
  82     ],
  83
  84     [
  85         re.compile('#include\s+EFI_GUID_DEFINITION\s*\(FirmwareFileSystem\)', re.MULTILINE),
  86         '#include EFI_GUID_DEFINITION (FirmwareFileSystem)\n#include EFI_GUID_DEFINITION (FirmwareFileSystem2)'
  87     ],
  88
  89     [
  90         re.compile('gEfiFirmwareFileSystemGuid', re.MULTILINE),
  91         'gEfiFirmwareFileSystem2Guid'
  92     ],
  93
  94     [
  95         re.compile('EFI_FVH_REVISION', re.MULTILINE),
  96         'EFI_FVH_PI_REVISION'
  97     ],
  98
  99     [
 100         re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_READY_TO_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
 101         '\\1EfiCreateEventReadyToBoot (\\2\\3;'
 102     ],
 103
 104     [
 105         re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_LEGACY_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
 106         '\\1EfiCreateEventLegacyBoot (\\2\\3;'
 107     ],
 108 #    [
 109 #        re.compile("(\W)(PEI_PCI_CFG_PPI)(\W)", re.MULTILINE),
 110 #        '\\1ECP_\\2\\3'
 111 #    ]
 112 ]
 113
 114 ## file cache to avoid circular include in ASL file
 115 gIncludedAslFile = []
 116
 117 ## Trim preprocessed source code
 118 #
 119 # Remove extra content made by preprocessor. The preprocessor must enable the
 120 # line number generation option when preprocessing.
 121 #
 122 # @param  Source    File to be trimmed
 123 # @param  Target    File to store the trimmed content
 124 # @param  Convert   If True, convert standard HEX format to MASM format
 125 #
 126 def TrimPreprocessedFile(Source, Target, Convert):
 127     CreateDirectory(os.path.dirname(Target))
 128     try:
 129         f = open (Source, 'r')
 130     except:
 131         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 132
 133     # read whole file
 134     Lines = f.readlines()
 135     f.close()
 136
 137     PreprocessedFile = ""
 138     InjectedFile = ""
 139     LineIndexOfOriginalFile = None
 140     NewLines = []
 141     LineControlDirectiveFound = False
 142     for Index in range(len(Lines)):
 143         Line = Lines[Index]
 144         #
 145         # Find out the name of files injected by preprocessor from the lines
 146         # with Line Control directive
 147         #
 148         MatchList = gLineControlDirective.findall(Line)
 149         if MatchList != []:
 150             MatchList = MatchList[0]
 151             if len(MatchList) == 2:
 152                 LineNumber = int(MatchList[0], 0)
 153                 InjectedFile = MatchList[1]
 154                 # The first injetcted file must be the preprocessed file itself
 155                 if PreprocessedFile == "":
 156                     PreprocessedFile = InjectedFile
 157             LineControlDirectiveFound = True
 158             continue
 159         elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
 160             continue
 161
 162         if LineIndexOfOriginalFile == None:
 163             #
 164             # Any non-empty lines must be from original preprocessed file.
 165             # And this must be the first one.
 166             #
 167             LineIndexOfOriginalFile = Index
 168             EdkLogger.verbose("Found original file content starting from line %d"
 169                               % (LineIndexOfOriginalFile + 1))
 170
 171         # convert HEX number format if indicated
 172         if Convert:
 173             Line = gHexNumberPattern.sub(r"0\1h", Line)
 174
 175         if LineNumber != None:
 176             EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
 177             # in case preprocessor removed some lines, like blank or comment lines
 178             if LineNumber <= len(NewLines):
 179                 # possible?
 180                 NewLines[LineNumber - 1] = Line
 181             else:
 182                 if LineNumber > (len(NewLines) + 1):
 183                     for LineIndex in range(len(NewLines), LineNumber-1):
 184                         NewLines.append(os.linesep)
 185                 NewLines.append(Line)
 186             LineNumber = None
 187             EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
 188         else:
 189             NewLines.append(Line)
 190
 191     # in case there's no line directive or linemarker found
 192     if (not LineControlDirectiveFound) and NewLines == []:
 193         NewLines = Lines
 194
 195     # save to file
 196     try:
 197         f = open (Target, 'wb')
 198     except:
 199         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 200     f.writelines(NewLines)
 201     f.close()
 202
 203 ## Trim preprocessed VFR file
 204 #
 205 # Remove extra content made by preprocessor. The preprocessor doesn't need to
 206 # enable line number generation option when preprocessing.
 207 #
 208 # @param  Source    File to be trimmed
 209 # @param  Target    File to store the trimmed content
 210 #
 211 def TrimPreprocessedVfr(Source, Target):
 212     CreateDirectory(os.path.dirname(Target))
 213
 214     try:
 215         f = open (Source,'r')
 216     except:
 217         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 218     # read whole file
 219     Lines = f.readlines()
 220     f.close()
 221
 222     FoundTypedef = False
 223     Brace = 0
 224     TypedefStart = 0
 225     TypedefEnd = 0
 226     for Index in range(len(Lines)):
 227         Line = Lines[Index]
 228         # don't trim the lines from "formset" definition to the end of file
 229         if Line.strip() == 'formset':
 230             break
 231
 232         if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
 233             # empty the line number directive if it's not aomong "typedef struct"
 234             Lines[Index] = "\n"
 235             continue
 236
 237         if FoundTypedef == False and gTypedefPattern.search(Line) == None:
 238             # keep "#pragram pack" directive
 239             if gPragmaPattern.search(Line) == None:
 240                 Lines[Index] = "\n"
 241             continue
 242         elif FoundTypedef == False:
 243             # found "typedef struct", keept its position and set a flag
 244             FoundTypedef = True
 245             TypedefStart = Index
 246
 247         # match { and } to find the end of typedef definition
 248         if Line.find("{") >= 0:
 249             Brace += 1
 250         elif Line.find("}") >= 0:
 251             Brace -= 1
 252
 253         # "typedef struct" must end with a ";"
 254         if Brace == 0 and Line.find(";") >= 0:
 255             FoundTypedef = False
 256             TypedefEnd = Index
 257             # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
 258             if Line.strip("} ;\r\n") in ["GUID", "EFI_PLABEL", "PAL_CALL_RETURN"]:
 259                 for i in range(TypedefStart, TypedefEnd+1):
 260                     Lines[i] = "\n"
 261
 262     # save all lines trimmed
 263     try:
 264         f = open (Target,'w')
 265     except:
 266         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 267     f.writelines(Lines)
 268     f.close()
 269
 270 ## Read the content  ASL file, including ASL included, recursively
 271 #
 272 # @param  Source    File to be read
 273 # @param  Indent    Spaces before the Include() statement
 274 #
 275 def DoInclude(Source, Indent=''):
 276     NewFileContent = []
 277     # avoid A "include" B and B "include" A
 278     if Source in gIncludedAslFile:
 279         EdkLogger.warn("Trim", "Circular include",
 280                        ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), Source))
 281         return []
 282     gIncludedAslFile.append(Source)
 283
 284     try:
 285         F = open(Source,'r')
 286     except:
 287         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 288
 289     for Line in F:
 290         Result = gAslIncludePattern.findall(Line)
 291         if len(Result) == 0:
 292             NewFileContent.append("%s%s" % (Indent, Line))
 293             continue
 294         CurrentIndent = Indent + Result[0][0]
 295         IncludedFile = Result[0][1]
 296         NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent))
 297         NewFileContent.append("\n")
 298
 299     gIncludedAslFile.pop()
 300     F.close()
 301
 302     return NewFileContent
 303
 304
 305 ## Trim ASL file
 306 #
 307 # Replace ASL include statement with the content the included file
 308 #
 309 # @param  Source    File to be trimmed
 310 # @param  Target    File to store the trimmed content
 311 #
 312 def TrimAslFile(Source, Target):
 313     CreateDirectory(os.path.dirname(Target))
 314
 315     Cwd = os.getcwd()
 316     SourceDir = os.path.dirname(Source)
 317     if SourceDir == '':
 318         SourceDir = '.'
 319     os.chdir(SourceDir)
 320     Lines = DoInclude(Source)
 321     os.chdir(Cwd)
 322
 323     # save all lines trimmed
 324     try:
 325         f = open (Target,'w')
 326     except:
 327         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 328
 329     f.writelines(Lines)
 330     f.close()
 331
 332 ## Trim EDK source code file(s)
 333 #
 334 #
 335 # @param  Source    File or directory to be trimmed
 336 # @param  Target    File or directory to store the trimmed content
 337 #
 338 def TrimR8Sources(Source, Target):
 339     if os.path.isdir(Source):
 340         for CurrentDir, Dirs, Files in os.walk(Source):
 341             if '.svn' in Dirs:
 342                 Dirs.remove('.svn')
 343             elif "CVS" in Dirs:
 344                 Dirs.remove("CVS")
 345
 346             for FileName in Files:
 347                 Dummy, Ext = os.path.splitext(FileName)
 348                 if Ext.upper() not in ['.C', '.H']: continue
 349                 if Target == None or Target == '':
 350                     TrimR8SourceCode(
 351                         os.path.join(CurrentDir, FileName),
 352                         os.path.join(CurrentDir, FileName)
 353                         )
 354                 else:
 355                     TrimR8SourceCode(
 356                         os.path.join(CurrentDir, FileName),
 357                         os.path.join(Target, CurrentDir[len(Source)+1:], FileName)
 358                         )
 359     else:
 360         TrimR8SourceCode(Source, Target)
 361
 362 ## Trim one EDK source code file
 363 #
 364 # Do following replacement:
 365 #
 366 #   (**PeiServices\).PciCfg = <*>;
 367 #   =>  {
 368 #         STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
 369 #         (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
 370 #         &gEcpPeiPciCfgPpiGuid,
 371 #         <*>
 372 #       };
 373 #       (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
 374 #
 375 #   <*>Modify(<*>)
 376 #   =>  PeiLibPciCfgModify (<*>)
 377 #
 378 #   gRT->ReportStatusCode (<*>)
 379 #   => EfiLibReportStatusCode (<*>)
 380 #
 381 #   #include <LoadFile\.h>
 382 #   =>  #include <FvLoadFile.h>
 383 #
 384 #   CreateEvent (EFI_EVENT_SIGNAL_READY_TO_BOOT, <*>)
 385 #   => EfiCreateEventReadyToBoot (<*>)
 386 #
 387 #   CreateEvent (EFI_EVENT_SIGNAL_LEGACY_BOOT, <*>)
 388 #   =>  EfiCreateEventLegacyBoot (<*>)
 389 #
 390 # @param  Source    File to be trimmed
 391 # @param  Target    File to store the trimmed content
 392 #
 393 def TrimR8SourceCode(Source, Target):
 394     EdkLogger.verbose("\t%s -> %s" % (Source, Target))
 395     CreateDirectory(os.path.dirname(Target))
 396
 397     try:
 398         f = open (Source,'rb')
 399     except:
 400         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 401     # read whole file
 402     Lines = f.read()
 403     f.close()
 404
 405     NewLines = None
 406     for Re,Repl in gImportCodePatterns:
 407         if NewLines == None:
 408             NewLines = Re.sub(Repl, Lines)
 409         else:
 410             NewLines = Re.sub(Repl, NewLines)
 411
 412     # save all lines if trimmed
 413     if Source == Target and NewLines == Lines:
 414         return
 415
 416     try:
 417         f = open (Target,'wb')
 418     except:
 419         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 420     f.write(NewLines)
 421     f.close()
 422
 423
 424 ## Parse command line options
 425 #
 426 # Using standard Python module optparse to parse command line option of this tool.
 427 #
 428 # @retval Options   A optparse.Values object containing the parsed options
 429 # @retval InputFile Path of file to be trimmed
 430 #
 431 def Options():
 432     OptionList = [
 433         make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
 434                           help="The input file is preprocessed source code, including C or assembly code"),
 435         make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
 436                           help="The input file is preprocessed VFR file"),
 437         make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
 438                           help="The input file is ASL file"),
 439         make_option("-8", "--r8-source-code", dest="FileType", const="R8SourceCode", action="store_const",
 440                           help="The input file is source code for R8 to be trimmed for ECP"),
 441
 442         make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
 443                           help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
 444
 445         make_option("-o", "--output", dest="OutputFile",
 446                           help="File to store the trimmed content"),
 447         make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
 448                           help="Run verbosely"),
 449         make_option("-d", "--debug", dest="LogLevel", type="int",
 450                           help="Run with debug information"),
 451         make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
 452                           help="Run quietly"),
 453         make_option("-?", action="help", help="show this help message and exit"),
 454     ]
 455
 456     # use clearer usage to override default usage message
 457     UsageString = "%prog [-s|-r|-a] [-c] [-v|-d <debug_level>|-q] [-o <output_file>] <input_file>"
 458
 459     Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
 460     Parser.set_defaults(FileType="Vfr")
 461     Parser.set_defaults(ConvertHex=False)
 462     Parser.set_defaults(LogLevel=EdkLogger.INFO)
 463
 464     Options, Args = Parser.parse_args()
 465
 466     # error check
 467     if len(Args) == 0:
 468         EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
 469     if len(Args) > 1:
 470         EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
 471
 472     InputFile = Args[0]
 473     return Options, InputFile
 474
 475 ## Entrance method
 476 #
 477 # This method mainly dispatch specific methods per the command line options.
 478 # If no error found, return zero value so the caller of this tool can know
 479 # if it's executed successfully or not.
 480 #
 481 # @retval 0     Tool was successful
 482 # @retval 1     Tool failed
 483 #
 484 def Main():
 485     try:
 486         EdkLogger.Initialize()
 487         CommandOptions, InputFile = Options()
 488         if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
 489             EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
 490         else:
 491             EdkLogger.SetLevel(CommandOptions.LogLevel)
 492     except FatalError, X:
 493         return 1
 494
 495     try:
 496         if CommandOptions.FileType == "Vfr":
 497             if CommandOptions.OutputFile == None:
 498                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 499             TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
 500         elif CommandOptions.FileType == "Asl":
 501             if CommandOptions.OutputFile == None:
 502                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 503             TrimAslFile(InputFile, CommandOptions.OutputFile)
 504         elif CommandOptions.FileType == "R8SourceCode":
 505             TrimR8Sources(InputFile, CommandOptions.OutputFile)
 506         else :
 507             if CommandOptions.OutputFile == None:
 508                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 509             TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex)
 510     except FatalError, X:
 511         import platform
 512         import traceback
 513         if CommandOptions != None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
 514             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 515         return 1
 516     except:
 517         import traceback
 518         import platform
 519         EdkLogger.error(
 520                     "\nTrim",
 521                     CODE_ERROR,
 522                     "Unknown fatal error when trimming [%s]" % InputFile,
 523                     ExtraData="\n(Please send email to edk2-buildtools-devel@lists.sourceforge.net for help, attaching following call stack trace!)\n",
 524                     RaiseError=False
 525                     )
 526         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 527         return 1
 528
 529     return 0
 530
 531 if __name__ == '__main__':
 532     r = Main()
 533     ## 0-127 is a safe return range, and 1 is a standard default error
 534     if r < 0 or r > 127: r = 1
 535     sys.exit(r)
 536