BaseTools/Source/Python/Trim/Trim.py

   1 ## @file
   2 # Trim files preprocessed by compiler
   3 #
   4 # Copyright (c) 2007 - 2010, Intel Corporation
   5 # All rights reserved. This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os
  18 import sys
  19 import re
  20
  21 from optparse import OptionParser
  22 from optparse import make_option
  23 from Common.BuildToolError import *
  24 from Common.Misc import *
  25
  26 import Common.EdkLogger as EdkLogger
  27
  28 # Version and Copyright
  29 __version_number__ = "0.10"
  30 __version__ = "%prog Version " + __version_number__
  31 __copyright__ = "Copyright (c) 2007-2010, Intel Corporation. All rights reserved."
  32
  33 ## Regular expression for matching Line Control directive like "#line xxx"
  34 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
  35 ## Regular expression for matching "typedef struct"
  36 gTypedefPattern = re.compile("^\s*typedef\s+struct\s*[{]*$", re.MULTILINE)
  37 ## Regular expression for matching "#pragma pack"
  38 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
  39 ## Regular expression for matching HEX number
  40 gHexNumberPattern = re.compile("0[xX]([0-9a-fA-F]+)")
  41 ## Regular expression for matching "Include ()" in asl file
  42 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
  43 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
  44 gImportCodePatterns = [
  45     [
  46         re.compile('^(\s*)\(\*\*PeiServices\)\.PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
  47         '''\\1{
  48 \\1  STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
  49 \\1    (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
  50 \\1    &gEcpPeiPciCfgPpiGuid,
  51 \\1    \\2
  52 \\1  };
  53 \\1  (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
  54 \\1}'''
  55     ],
  56
  57     [
  58         re.compile('^(\s*)\(\*PeiServices\)->PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
  59         '''\\1{
  60 \\1  STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
  61 \\1    (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
  62 \\1    &gEcpPeiPciCfgPpiGuid,
  63 \\1    \\2
  64 \\1  };
  65 \\1  (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
  66 \\1}'''
  67     ],
  68
  69     [
  70         re.compile("(\s*).+->Modify[\s\n]*\(", re.MULTILINE),
  71         '\\1PeiLibPciCfgModify ('
  72     ],
  73
  74     [
  75         re.compile("(\W*)gRT->ReportStatusCode[\s\n]*\(", re.MULTILINE),
  76         '\\1EfiLibReportStatusCode ('
  77     ],
  78
  79     [
  80         re.compile('#include\s+["<]LoadFile\.h[">]', re.MULTILINE),
  81         '#include <FvLoadFile.h>'
  82     ],
  83
  84     [
  85         re.compile('#include\s+EFI_GUID_DEFINITION\s*\(FirmwareFileSystem\)', re.MULTILINE),
  86         '#include EFI_GUID_DEFINITION (FirmwareFileSystem)\n#include EFI_GUID_DEFINITION (FirmwareFileSystem2)'
  87     ],
  88
  89     [
  90         re.compile('gEfiFirmwareFileSystemGuid', re.MULTILINE),
  91         'gEfiFirmwareFileSystem2Guid'
  92     ],
  93
  94     [
  95         re.compile('EFI_FVH_REVISION', re.MULTILINE),
  96         'EFI_FVH_PI_REVISION'
  97     ],
  98
  99     [
 100         re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_READY_TO_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
 101         '\\1EfiCreateEventReadyToBoot (\\2\\3;'
 102     ],
 103
 104     [
 105         re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_LEGACY_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
 106         '\\1EfiCreateEventLegacyBoot (\\2\\3;'
 107     ],
 108 #    [
 109 #        re.compile("(\W)(PEI_PCI_CFG_PPI)(\W)", re.MULTILINE),
 110 #        '\\1ECP_\\2\\3'
 111 #    ]
 112 ]
 113
 114 ## file cache to avoid circular include in ASL file
 115 gIncludedAslFile = []
 116
 117 ## Trim preprocessed source code
 118 #
 119 # Remove extra content made by preprocessor. The preprocessor must enable the
 120 # line number generation option when preprocessing.
 121 #
 122 # @param  Source    File to be trimmed
 123 # @param  Target    File to store the trimmed content
 124 # @param  Convert   If True, convert standard HEX format to MASM format
 125 #
 126 def TrimPreprocessedFile(Source, Target, Convert):
 127     CreateDirectory(os.path.dirname(Target))
 128     try:
 129         f = open (Source, 'r')
 130     except:
 131         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 132
 133     # read whole file
 134     Lines = f.readlines()
 135     f.close()
 136
 137     PreprocessedFile = ""
 138     InjectedFile = ""
 139     LineIndexOfOriginalFile = None
 140     NewLines = []
 141     LineControlDirectiveFound = False
 142     for Index in range(len(Lines)):
 143         Line = Lines[Index]
 144         #
 145         # Find out the name of files injected by preprocessor from the lines
 146         # with Line Control directive
 147         #
 148         MatchList = gLineControlDirective.findall(Line)
 149         if MatchList != []:
 150             MatchList = MatchList[0]
 151             if len(MatchList) == 2:
 152                 LineNumber = int(MatchList[0], 0)
 153                 InjectedFile = MatchList[1]
 154                 # The first injetcted file must be the preprocessed file itself
 155                 if PreprocessedFile == "":
 156                     PreprocessedFile = InjectedFile
 157             LineControlDirectiveFound = True
 158             continue
 159         elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
 160             continue
 161
 162         if LineIndexOfOriginalFile == None:
 163             #
 164             # Any non-empty lines must be from original preprocessed file.
 165             # And this must be the first one.
 166             #
 167             LineIndexOfOriginalFile = Index
 168             EdkLogger.verbose("Found original file content starting from line %d"
 169                               % (LineIndexOfOriginalFile + 1))
 170
 171         # convert HEX number format if indicated
 172         if Convert:
 173             Line = gHexNumberPattern.sub(r"0\1h", Line)
 174
 175         if LineNumber != None:
 176             EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
 177             # in case preprocessor removed some lines, like blank or comment lines
 178             if LineNumber <= len(NewLines):
 179                 # possible?
 180                 NewLines[LineNumber - 1] = Line
 181             else:
 182                 if LineNumber > (len(NewLines) + 1):
 183                     for LineIndex in range(len(NewLines), LineNumber-1):
 184                         NewLines.append(os.linesep)
 185                 NewLines.append(Line)
 186             LineNumber = None
 187             EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
 188         else:
 189             NewLines.append(Line)
 190
 191     # in case there's no line directive or linemarker found
 192     if (not LineControlDirectiveFound) and NewLines == []:
 193         NewLines = Lines
 194
 195     # save to file
 196     try:
 197         f = open (Target, 'wb')
 198     except:
 199         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 200     f.writelines(NewLines)
 201     f.close()
 202
 203 ## Trim preprocessed VFR file
 204 #
 205 # Remove extra content made by preprocessor. The preprocessor doesn't need to
 206 # enable line number generation option when preprocessing.
 207 #
 208 # @param  Source    File to be trimmed
 209 # @param  Target    File to store the trimmed content
 210 #
 211 def TrimPreprocessedVfr(Source, Target):
 212     CreateDirectory(os.path.dirname(Target))
 213
 214     try:
 215         f = open (Source,'r')
 216     except:
 217         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 218     # read whole file
 219     Lines = f.readlines()
 220     f.close()
 221
 222     FoundTypedef = False
 223     Brace = 0
 224     TypedefStart = 0
 225     TypedefEnd = 0
 226     for Index in range(len(Lines)):
 227         Line = Lines[Index]
 228         # don't trim the lines from "formset" definition to the end of file
 229         if Line.strip() == 'formset':
 230             break
 231
 232         if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
 233             # empty the line number directive if it's not aomong "typedef struct"
 234             Lines[Index] = "\n"
 235             continue
 236
 237         if FoundTypedef == False and gTypedefPattern.search(Line) == None:
 238             # keep "#pragram pack" directive
 239             if gPragmaPattern.search(Line) == None:
 240                 Lines[Index] = "\n"
 241             continue
 242         elif FoundTypedef == False:
 243             # found "typedef struct", keept its position and set a flag
 244             FoundTypedef = True
 245             TypedefStart = Index
 246
 247         # match { and } to find the end of typedef definition
 248         if Line.find("{") >= 0:
 249             Brace += 1
 250         elif Line.find("}") >= 0:
 251             Brace -= 1
 252
 253         # "typedef struct" must end with a ";"
 254         if Brace == 0 and Line.find(";") >= 0:
 255             FoundTypedef = False
 256             TypedefEnd = Index
 257             # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
 258             if Line.strip("} ;\r\n") in ["GUID", "EFI_PLABEL", "PAL_CALL_RETURN"]:
 259                 for i in range(TypedefStart, TypedefEnd+1):
 260                     Lines[i] = "\n"
 261
 262     # save all lines trimmed
 263     try:
 264         f = open (Target,'w')
 265     except:
 266         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 267     f.writelines(Lines)
 268     f.close()
 269
 270 ## Read the content  ASL file, including ASL included, recursively
 271 #
 272 # @param  Source    File to be read
 273 # @param  Indent    Spaces before the Include() statement
 274 #
 275 def DoInclude(Source, Indent=''):
 276     NewFileContent = []
 277     # avoid A "include" B and B "include" A
 278     if Source in gIncludedAslFile:
 279         EdkLogger.warn("Trim", "Circular include",
 280                        ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), Source))
 281         return []
 282     gIncludedAslFile.append(Source)
 283
 284     try:
 285         F = open(Source,'r')
 286     except:
 287         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 288
 289     for Line in F:
 290         Result = gAslIncludePattern.findall(Line)
 291         if len(Result) == 0:
 292             NewFileContent.append("%s%s" % (Indent, Line))
 293             continue
 294         CurrentIndent = Indent + Result[0][0]
 295         IncludedFile = Result[0][1]
 296         NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent))
 297
 298     gIncludedAslFile.pop()
 299     F.close()
 300
 301     return NewFileContent
 302
 303
 304 ## Trim ASL file
 305 #
 306 # Replace ASL include statement with the content the included file
 307 #
 308 # @param  Source    File to be trimmed
 309 # @param  Target    File to store the trimmed content
 310 #
 311 def TrimAslFile(Source, Target):
 312     CreateDirectory(os.path.dirname(Target))
 313
 314     Cwd = os.getcwd()
 315     SourceDir = os.path.dirname(Source)
 316     if SourceDir == '':
 317         SourceDir = '.'
 318     os.chdir(SourceDir)
 319     Lines = DoInclude(Source)
 320     os.chdir(Cwd)
 321
 322     # save all lines trimmed
 323     try:
 324         f = open (Target,'w')
 325     except:
 326         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 327
 328     f.writelines(Lines)
 329     f.close()
 330
 331 ## Trim EDK source code file(s)
 332 #
 333 #
 334 # @param  Source    File or directory to be trimmed
 335 # @param  Target    File or directory to store the trimmed content
 336 #
 337 def TrimR8Sources(Source, Target):
 338     if os.path.isdir(Source):
 339         for CurrentDir, Dirs, Files in os.walk(Source):
 340             if '.svn' in Dirs:
 341                 Dirs.remove('.svn')
 342             elif "CVS" in Dirs:
 343                 Dirs.remove("CVS")
 344
 345             for FileName in Files:
 346                 Dummy, Ext = os.path.splitext(FileName)
 347                 if Ext.upper() not in ['.C', '.H']: continue
 348                 if Target == None or Target == '':
 349                     TrimR8SourceCode(
 350                         os.path.join(CurrentDir, FileName),
 351                         os.path.join(CurrentDir, FileName)
 352                         )
 353                 else:
 354                     TrimR8SourceCode(
 355                         os.path.join(CurrentDir, FileName),
 356                         os.path.join(Target, CurrentDir[len(Source)+1:], FileName)
 357                         )
 358     else:
 359         TrimR8SourceCode(Source, Target)
 360
 361 ## Trim one EDK source code file
 362 #
 363 # Do following replacement:
 364 #
 365 #   (**PeiServices\).PciCfg = <*>;
 366 #   =>  {
 367 #         STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
 368 #         (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
 369 #         &gEcpPeiPciCfgPpiGuid,
 370 #         <*>
 371 #       };
 372 #       (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
 373 #
 374 #   <*>Modify(<*>)
 375 #   =>  PeiLibPciCfgModify (<*>)
 376 #
 377 #   gRT->ReportStatusCode (<*>)
 378 #   => EfiLibReportStatusCode (<*>)
 379 #
 380 #   #include <LoadFile\.h>
 381 #   =>  #include <FvLoadFile.h>
 382 #
 383 #   CreateEvent (EFI_EVENT_SIGNAL_READY_TO_BOOT, <*>)
 384 #   => EfiCreateEventReadyToBoot (<*>)
 385 #
 386 #   CreateEvent (EFI_EVENT_SIGNAL_LEGACY_BOOT, <*>)
 387 #   =>  EfiCreateEventLegacyBoot (<*>)
 388 #
 389 # @param  Source    File to be trimmed
 390 # @param  Target    File to store the trimmed content
 391 #
 392 def TrimR8SourceCode(Source, Target):
 393     EdkLogger.verbose("\t%s -> %s" % (Source, Target))
 394     CreateDirectory(os.path.dirname(Target))
 395
 396     try:
 397         f = open (Source,'rb')
 398     except:
 399         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 400     # read whole file
 401     Lines = f.read()
 402     f.close()
 403
 404     NewLines = None
 405     for Re,Repl in gImportCodePatterns:
 406         if NewLines == None:
 407             NewLines = Re.sub(Repl, Lines)
 408         else:
 409             NewLines = Re.sub(Repl, NewLines)
 410
 411     # save all lines if trimmed
 412     if Source == Target and NewLines == Lines:
 413         return
 414
 415     try:
 416         f = open (Target,'wb')
 417     except:
 418         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 419     f.write(NewLines)
 420     f.close()
 421
 422
 423 ## Parse command line options
 424 #
 425 # Using standard Python module optparse to parse command line option of this tool.
 426 #
 427 # @retval Options   A optparse.Values object containing the parsed options
 428 # @retval InputFile Path of file to be trimmed
 429 #
 430 def Options():
 431     OptionList = [
 432         make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
 433                           help="The input file is preprocessed source code, including C or assembly code"),
 434         make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
 435                           help="The input file is preprocessed VFR file"),
 436         make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
 437                           help="The input file is ASL file"),
 438         make_option("-8", "--r8-source-code", dest="FileType", const="R8SourceCode", action="store_const",
 439                           help="The input file is source code for R8 to be trimmed for ECP"),
 440
 441         make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
 442                           help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
 443
 444         make_option("-o", "--output", dest="OutputFile",
 445                           help="File to store the trimmed content"),
 446         make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
 447                           help="Run verbosely"),
 448         make_option("-d", "--debug", dest="LogLevel", type="int",
 449                           help="Run with debug information"),
 450         make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
 451                           help="Run quietly"),
 452         make_option("-?", action="help", help="show this help message and exit"),
 453     ]
 454
 455     # use clearer usage to override default usage message
 456     UsageString = "%prog [-s|-r|-a] [-c] [-v|-d <debug_level>|-q] [-o <output_file>] <input_file>"
 457
 458     Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
 459     Parser.set_defaults(FileType="Vfr")
 460     Parser.set_defaults(ConvertHex=False)
 461     Parser.set_defaults(LogLevel=EdkLogger.INFO)
 462
 463     Options, Args = Parser.parse_args()
 464
 465     # error check
 466     if len(Args) == 0:
 467         EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
 468     if len(Args) > 1:
 469         EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
 470
 471     InputFile = Args[0]
 472     return Options, InputFile
 473
 474 ## Entrance method
 475 #
 476 # This method mainly dispatch specific methods per the command line options.
 477 # If no error found, return zero value so the caller of this tool can know
 478 # if it's executed successfully or not.
 479 #
 480 # @retval 0     Tool was successful
 481 # @retval 1     Tool failed
 482 #
 483 def Main():
 484     try:
 485         EdkLogger.Initialize()
 486         CommandOptions, InputFile = Options()
 487         if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
 488             EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
 489         else:
 490             EdkLogger.SetLevel(CommandOptions.LogLevel)
 491     except FatalError, X:
 492         return 1
 493
 494     try:
 495         if CommandOptions.FileType == "Vfr":
 496             if CommandOptions.OutputFile == None:
 497                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 498             TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
 499         elif CommandOptions.FileType == "Asl":
 500             if CommandOptions.OutputFile == None:
 501                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 502             TrimAslFile(InputFile, CommandOptions.OutputFile)
 503         elif CommandOptions.FileType == "R8SourceCode":
 504             TrimR8Sources(InputFile, CommandOptions.OutputFile)
 505         else :
 506             if CommandOptions.OutputFile == None:
 507                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 508             TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex)
 509     except FatalError, X:
 510         import platform
 511         import traceback
 512         if CommandOptions != None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
 513             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 514         return 1
 515     except:
 516         import traceback
 517         import platform
 518         EdkLogger.error(
 519                     "\nTrim",
 520                     CODE_ERROR,
 521                     "Unknown fatal error when trimming [%s]" % InputFile,
 522                     ExtraData="\n(Please send email to edk2-buildtools-devel@lists.sourceforge.net for help, attaching following call stack trace!)\n",
 523                     RaiseError=False
 524                     )
 525         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 526         return 1
 527
 528     return 0
 529
 530 if __name__ == '__main__':
 531     r = Main()
 532     ## 0-127 is a safe return range, and 1 is a standard default error
 533     if r < 0 or r > 127: r = 1
 534     sys.exit(r)
 535