BaseTools/Source/Python/Trim/Trim.py

   1 ## @file
   2 # Trim files preprocessed by compiler
   3 #
   4 # Copyright (c) 2007 - 2010, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import os
  18 import sys
  19 import re
  20
  21 from optparse import OptionParser
  22 from optparse import make_option
  23 from Common.BuildToolError import *
  24 from Common.Misc import *
  25
  26 import Common.EdkLogger as EdkLogger
  27
  28 # Version and Copyright
  29 __version_number__ = "0.10"
  30 __version__ = "%prog Version " + __version_number__
  31 __copyright__ = "Copyright (c) 2007-2010, Intel Corporation. All rights reserved."
  32
  33 ## Regular expression for matching Line Control directive like "#line xxx"
  34 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
  35 ## Regular expression for matching "typedef struct"
  36 gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)
  37 ## Regular expression for matching "#pragma pack"
  38 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
  39 ## Regular expression for matching HEX number
  40 gHexNumberPattern = re.compile("0[xX]([0-9a-fA-F]+)")
  41 ## Regular expression for matching "Include ()" in asl file
  42 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
  43 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
  44 gImportCodePatterns = [
  45     [
  46         re.compile('^(\s*)\(\*\*PeiServices\)\.PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
  47         '''\\1{
  48 \\1  STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
  49 \\1    (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
  50 \\1    &gEcpPeiPciCfgPpiGuid,
  51 \\1    \\2
  52 \\1  };
  53 \\1  (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
  54 \\1}'''
  55     ],
  56
  57     [
  58         re.compile('^(\s*)\(\*PeiServices\)->PciCfg\s*=\s*([^;\s]+);', re.MULTILINE),
  59         '''\\1{
  60 \\1  STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
  61 \\1    (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
  62 \\1    &gEcpPeiPciCfgPpiGuid,
  63 \\1    \\2
  64 \\1  };
  65 \\1  (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
  66 \\1}'''
  67     ],
  68
  69     [
  70         re.compile("(\s*).+->Modify[\s\n]*\(", re.MULTILINE),
  71         '\\1PeiLibPciCfgModify ('
  72     ],
  73
  74     [
  75         re.compile("(\W*)gRT->ReportStatusCode[\s\n]*\(", re.MULTILINE),
  76         '\\1EfiLibReportStatusCode ('
  77     ],
  78
  79     [
  80         re.compile('#include\s+EFI_GUID_DEFINITION\s*\(FirmwareFileSystem\)', re.MULTILINE),
  81         '#include EFI_GUID_DEFINITION (FirmwareFileSystem)\n#include EFI_GUID_DEFINITION (FirmwareFileSystem2)'
  82     ],
  83
  84     [
  85         re.compile('gEfiFirmwareFileSystemGuid', re.MULTILINE),
  86         'gEfiFirmwareFileSystem2Guid'
  87     ],
  88
  89     [
  90         re.compile('EFI_FVH_REVISION', re.MULTILINE),
  91         'EFI_FVH_PI_REVISION'
  92     ],
  93
  94     [
  95         re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_READY_TO_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
  96         '\\1EfiCreateEventReadyToBoot (\\2\\3;'
  97     ],
  98
  99     [
 100         re.compile("(\s*)\S*CreateEvent\s*\([\s\n]*EFI_EVENT_SIGNAL_LEGACY_BOOT[^,]*,((?:[^;]+\n)+)(\s*\));", re.MULTILINE),
 101         '\\1EfiCreateEventLegacyBoot (\\2\\3;'
 102     ],
 103 #    [
 104 #        re.compile("(\W)(PEI_PCI_CFG_PPI)(\W)", re.MULTILINE),
 105 #        '\\1ECP_\\2\\3'
 106 #    ]
 107 ]
 108
 109 ## file cache to avoid circular include in ASL file
 110 gIncludedAslFile = []
 111
 112 ## Trim preprocessed source code
 113 #
 114 # Remove extra content made by preprocessor. The preprocessor must enable the
 115 # line number generation option when preprocessing.
 116 #
 117 # @param  Source    File to be trimmed
 118 # @param  Target    File to store the trimmed content
 119 # @param  Convert   If True, convert standard HEX format to MASM format
 120 #
 121 def TrimPreprocessedFile(Source, Target, Convert):
 122     CreateDirectory(os.path.dirname(Target))
 123     try:
 124         f = open (Source, 'r')
 125     except:
 126         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 127
 128     # read whole file
 129     Lines = f.readlines()
 130     f.close()
 131
 132     PreprocessedFile = ""
 133     InjectedFile = ""
 134     LineIndexOfOriginalFile = None
 135     NewLines = []
 136     LineControlDirectiveFound = False
 137     for Index in range(len(Lines)):
 138         Line = Lines[Index]
 139         #
 140         # Find out the name of files injected by preprocessor from the lines
 141         # with Line Control directive
 142         #
 143         MatchList = gLineControlDirective.findall(Line)
 144         if MatchList != []:
 145             MatchList = MatchList[0]
 146             if len(MatchList) == 2:
 147                 LineNumber = int(MatchList[0], 0)
 148                 InjectedFile = MatchList[1]
 149                 # The first injetcted file must be the preprocessed file itself
 150                 if PreprocessedFile == "":
 151                     PreprocessedFile = InjectedFile
 152             LineControlDirectiveFound = True
 153             continue
 154         elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
 155             continue
 156
 157         if LineIndexOfOriginalFile == None:
 158             #
 159             # Any non-empty lines must be from original preprocessed file.
 160             # And this must be the first one.
 161             #
 162             LineIndexOfOriginalFile = Index
 163             EdkLogger.verbose("Found original file content starting from line %d"
 164                               % (LineIndexOfOriginalFile + 1))
 165
 166         # convert HEX number format if indicated
 167         if Convert:
 168             Line = gHexNumberPattern.sub(r"0\1h", Line)
 169
 170         if LineNumber != None:
 171             EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
 172             # in case preprocessor removed some lines, like blank or comment lines
 173             if LineNumber <= len(NewLines):
 174                 # possible?
 175                 NewLines[LineNumber - 1] = Line
 176             else:
 177                 if LineNumber > (len(NewLines) + 1):
 178                     for LineIndex in range(len(NewLines), LineNumber-1):
 179                         NewLines.append(os.linesep)
 180                 NewLines.append(Line)
 181             LineNumber = None
 182             EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
 183         else:
 184             NewLines.append(Line)
 185
 186     # in case there's no line directive or linemarker found
 187     if (not LineControlDirectiveFound) and NewLines == []:
 188         NewLines = Lines
 189
 190     # save to file
 191     try:
 192         f = open (Target, 'wb')
 193     except:
 194         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 195     f.writelines(NewLines)
 196     f.close()
 197
 198 ## Trim preprocessed VFR file
 199 #
 200 # Remove extra content made by preprocessor. The preprocessor doesn't need to
 201 # enable line number generation option when preprocessing.
 202 #
 203 # @param  Source    File to be trimmed
 204 # @param  Target    File to store the trimmed content
 205 #
 206 def TrimPreprocessedVfr(Source, Target):
 207     CreateDirectory(os.path.dirname(Target))
 208
 209     try:
 210         f = open (Source,'r')
 211     except:
 212         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 213     # read whole file
 214     Lines = f.readlines()
 215     f.close()
 216
 217     FoundTypedef = False
 218     Brace = 0
 219     TypedefStart = 0
 220     TypedefEnd = 0
 221     for Index in range(len(Lines)):
 222         Line = Lines[Index]
 223         # don't trim the lines from "formset" definition to the end of file
 224         if Line.strip() == 'formset':
 225             break
 226
 227         if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
 228             # empty the line number directive if it's not aomong "typedef struct"
 229             Lines[Index] = "\n"
 230             continue
 231
 232         if FoundTypedef == False and gTypedefPattern.search(Line) == None:
 233             # keep "#pragram pack" directive
 234             if gPragmaPattern.search(Line) == None:
 235                 Lines[Index] = "\n"
 236             continue
 237         elif FoundTypedef == False:
 238             # found "typedef struct", keept its position and set a flag
 239             FoundTypedef = True
 240             TypedefStart = Index
 241
 242         # match { and } to find the end of typedef definition
 243         if Line.find("{") >= 0:
 244             Brace += 1
 245         elif Line.find("}") >= 0:
 246             Brace -= 1
 247
 248         # "typedef struct" must end with a ";"
 249         if Brace == 0 and Line.find(";") >= 0:
 250             FoundTypedef = False
 251             TypedefEnd = Index
 252             # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
 253             if Line.strip("} ;\r\n") in ["GUID", "EFI_PLABEL", "PAL_CALL_RETURN"]:
 254                 for i in range(TypedefStart, TypedefEnd+1):
 255                     Lines[i] = "\n"
 256
 257     # save all lines trimmed
 258     try:
 259         f = open (Target,'w')
 260     except:
 261         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 262     f.writelines(Lines)
 263     f.close()
 264
 265 ## Read the content  ASL file, including ASL included, recursively
 266 #
 267 # @param  Source    File to be read
 268 # @param  Indent    Spaces before the Include() statement
 269 #
 270 def DoInclude(Source, Indent=''):
 271     NewFileContent = []
 272     # avoid A "include" B and B "include" A
 273     if Source in gIncludedAslFile:
 274         EdkLogger.warn("Trim", "Circular include",
 275                        ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), Source))
 276         return []
 277     gIncludedAslFile.append(Source)
 278
 279     try:
 280         F = open(Source,'r')
 281     except:
 282         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 283
 284     for Line in F:
 285         Result = gAslIncludePattern.findall(Line)
 286         if len(Result) == 0:
 287             NewFileContent.append("%s%s" % (Indent, Line))
 288             continue
 289         CurrentIndent = Indent + Result[0][0]
 290         IncludedFile = Result[0][1]
 291         NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent))
 292         NewFileContent.append("\n")
 293
 294     gIncludedAslFile.pop()
 295     F.close()
 296
 297     return NewFileContent
 298
 299
 300 ## Trim ASL file
 301 #
 302 # Replace ASL include statement with the content the included file
 303 #
 304 # @param  Source    File to be trimmed
 305 # @param  Target    File to store the trimmed content
 306 #
 307 def TrimAslFile(Source, Target):
 308     CreateDirectory(os.path.dirname(Target))
 309
 310     Cwd = os.getcwd()
 311     SourceDir = os.path.dirname(Source)
 312     if SourceDir == '':
 313         SourceDir = '.'
 314     os.chdir(SourceDir)
 315     Lines = DoInclude(Source)
 316     os.chdir(Cwd)
 317
 318     # save all lines trimmed
 319     try:
 320         f = open (Target,'w')
 321     except:
 322         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 323
 324     f.writelines(Lines)
 325     f.close()
 326
 327 ## Trim EDK source code file(s)
 328 #
 329 #
 330 # @param  Source    File or directory to be trimmed
 331 # @param  Target    File or directory to store the trimmed content
 332 #
 333 def TrimR8Sources(Source, Target):
 334     if os.path.isdir(Source):
 335         for CurrentDir, Dirs, Files in os.walk(Source):
 336             if '.svn' in Dirs:
 337                 Dirs.remove('.svn')
 338             elif "CVS" in Dirs:
 339                 Dirs.remove("CVS")
 340
 341             for FileName in Files:
 342                 Dummy, Ext = os.path.splitext(FileName)
 343                 if Ext.upper() not in ['.C', '.H']: continue
 344                 if Target == None or Target == '':
 345                     TrimR8SourceCode(
 346                         os.path.join(CurrentDir, FileName),
 347                         os.path.join(CurrentDir, FileName)
 348                         )
 349                 else:
 350                     TrimR8SourceCode(
 351                         os.path.join(CurrentDir, FileName),
 352                         os.path.join(Target, CurrentDir[len(Source)+1:], FileName)
 353                         )
 354     else:
 355         TrimR8SourceCode(Source, Target)
 356
 357 ## Trim one EDK source code file
 358 #
 359 # Do following replacement:
 360 #
 361 #   (**PeiServices\).PciCfg = <*>;
 362 #   =>  {
 363 #         STATIC EFI_PEI_PPI_DESCRIPTOR gEcpPeiPciCfgPpiList = {
 364 #         (EFI_PEI_PPI_DESCRIPTOR_PPI | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
 365 #         &gEcpPeiPciCfgPpiGuid,
 366 #         <*>
 367 #       };
 368 #       (**PeiServices).InstallPpi (PeiServices, &gEcpPeiPciCfgPpiList);
 369 #
 370 #   <*>Modify(<*>)
 371 #   =>  PeiLibPciCfgModify (<*>)
 372 #
 373 #   gRT->ReportStatusCode (<*>)
 374 #   => EfiLibReportStatusCode (<*>)
 375 #
 376 #   #include <LoadFile\.h>
 377 #   =>  #include <FvLoadFile.h>
 378 #
 379 #   CreateEvent (EFI_EVENT_SIGNAL_READY_TO_BOOT, <*>)
 380 #   => EfiCreateEventReadyToBoot (<*>)
 381 #
 382 #   CreateEvent (EFI_EVENT_SIGNAL_LEGACY_BOOT, <*>)
 383 #   =>  EfiCreateEventLegacyBoot (<*>)
 384 #
 385 # @param  Source    File to be trimmed
 386 # @param  Target    File to store the trimmed content
 387 #
 388 def TrimR8SourceCode(Source, Target):
 389     EdkLogger.verbose("\t%s -> %s" % (Source, Target))
 390     CreateDirectory(os.path.dirname(Target))
 391
 392     try:
 393         f = open (Source,'rb')
 394     except:
 395         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 396     # read whole file
 397     Lines = f.read()
 398     f.close()
 399
 400     NewLines = None
 401     for Re,Repl in gImportCodePatterns:
 402         if NewLines == None:
 403             NewLines = Re.sub(Repl, Lines)
 404         else:
 405             NewLines = Re.sub(Repl, NewLines)
 406
 407     # save all lines if trimmed
 408     if Source == Target and NewLines == Lines:
 409         return
 410
 411     try:
 412         f = open (Target,'wb')
 413     except:
 414         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 415     f.write(NewLines)
 416     f.close()
 417
 418
 419 ## Parse command line options
 420 #
 421 # Using standard Python module optparse to parse command line option of this tool.
 422 #
 423 # @retval Options   A optparse.Values object containing the parsed options
 424 # @retval InputFile Path of file to be trimmed
 425 #
 426 def Options():
 427     OptionList = [
 428         make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
 429                           help="The input file is preprocessed source code, including C or assembly code"),
 430         make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
 431                           help="The input file is preprocessed VFR file"),
 432         make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
 433                           help="The input file is ASL file"),
 434         make_option("-8", "--r8-source-code", dest="FileType", const="R8SourceCode", action="store_const",
 435                           help="The input file is source code for R8 to be trimmed for ECP"),
 436
 437         make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
 438                           help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
 439
 440         make_option("-o", "--output", dest="OutputFile",
 441                           help="File to store the trimmed content"),
 442         make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
 443                           help="Run verbosely"),
 444         make_option("-d", "--debug", dest="LogLevel", type="int",
 445                           help="Run with debug information"),
 446         make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
 447                           help="Run quietly"),
 448         make_option("-?", action="help", help="show this help message and exit"),
 449     ]
 450
 451     # use clearer usage to override default usage message
 452     UsageString = "%prog [-s|-r|-a] [-c] [-v|-d <debug_level>|-q] [-o <output_file>] <input_file>"
 453
 454     Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
 455     Parser.set_defaults(FileType="Vfr")
 456     Parser.set_defaults(ConvertHex=False)
 457     Parser.set_defaults(LogLevel=EdkLogger.INFO)
 458
 459     Options, Args = Parser.parse_args()
 460
 461     # error check
 462     if len(Args) == 0:
 463         EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
 464     if len(Args) > 1:
 465         EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
 466
 467     InputFile = Args[0]
 468     return Options, InputFile
 469
 470 ## Entrance method
 471 #
 472 # This method mainly dispatch specific methods per the command line options.
 473 # If no error found, return zero value so the caller of this tool can know
 474 # if it's executed successfully or not.
 475 #
 476 # @retval 0     Tool was successful
 477 # @retval 1     Tool failed
 478 #
 479 def Main():
 480     try:
 481         EdkLogger.Initialize()
 482         CommandOptions, InputFile = Options()
 483         if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
 484             EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
 485         else:
 486             EdkLogger.SetLevel(CommandOptions.LogLevel)
 487     except FatalError, X:
 488         return 1
 489
 490     try:
 491         if CommandOptions.FileType == "Vfr":
 492             if CommandOptions.OutputFile == None:
 493                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 494             TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
 495         elif CommandOptions.FileType == "Asl":
 496             if CommandOptions.OutputFile == None:
 497                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 498             TrimAslFile(InputFile, CommandOptions.OutputFile)
 499         elif CommandOptions.FileType == "R8SourceCode":
 500             TrimR8Sources(InputFile, CommandOptions.OutputFile)
 501         else :
 502             if CommandOptions.OutputFile == None:
 503                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 504             TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex)
 505     except FatalError, X:
 506         import platform
 507         import traceback
 508         if CommandOptions != None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
 509             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 510         return 1
 511     except:
 512         import traceback
 513         import platform
 514         EdkLogger.error(
 515                     "\nTrim",
 516                     CODE_ERROR,
 517                     "Unknown fatal error when trimming [%s]" % InputFile,
 518                     ExtraData="\n(Please send email to edk2-buildtools-devel@lists.sourceforge.net for help, attaching following call stack trace!)\n",
 519                     RaiseError=False
 520                     )
 521         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 522         return 1
 523
 524     return 0
 525
 526 if __name__ == '__main__':
 527     r = Main()
 528     ## 0-127 is a safe return range, and 1 is a standard default error
 529     if r < 0 or r > 127: r = 1
 530     sys.exit(r)
 531