BaseTools/Source/Python/Trim/Trim.py

   1 ## @file
   2 # Trim files preprocessed by compiler
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ##
   9 # Import Modules
  10 #
  11 import Common.LongFilePathOs as os
  12 import sys
  13 import re
  14 from io import BytesIO
  15 import codecs
  16 from optparse import OptionParser
  17 from optparse import make_option
  18 from Common.BuildToolError import *
  19 from Common.Misc import *
  20 from Common.DataType import *
  21 from Common.BuildVersion import gBUILD_VERSION
  22 import Common.EdkLogger as EdkLogger
  23 from Common.LongFilePathSupport import OpenLongFilePath as open
  24
  25 # Version and Copyright
  26 __version_number__ = ("0.10" + " " + gBUILD_VERSION)
  27 __version__ = "%prog Version " + __version_number__
  28 __copyright__ = "Copyright (c) 2007-2018, Intel Corporation. All rights reserved."
  29
  30 ## Regular expression for matching Line Control directive like "#line xxx"
  31 gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')
  32 ## Regular expression for matching "typedef struct"
  33 gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)
  34 ## Regular expression for matching "#pragma pack"
  35 gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)
  36 ## Regular expression for matching "typedef"
  37 gTypedef_SinglePattern = re.compile("^\s*typedef", re.MULTILINE)
  38 ## Regular expression for matching "typedef struct, typedef union, struct, union"
  39 gTypedef_MulPattern = re.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re.MULTILINE)
  40
  41 #
  42 # The following number pattern match will only match if following criteria is met:
  43 # There is leading non-(alphanumeric or _) character, and no following alphanumeric or _
  44 # as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match
  45 #
  46 ## Regular expression for matching HEX number
  47 gHexNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")
  48 ## Regular expression for matching decimal number with 'U' postfix
  49 gDecNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")
  50 ## Regular expression for matching constant with 'ULL' 'LL' postfix
  51 gLongNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")
  52
  53 ## Regular expression for matching "Include ()" in asl file
  54 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)
  55 ## Regular expression for matching C style #include "XXX.asl" in asl file
  56 gAslCIncludePattern = re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re.MULTILINE)
  57 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
  58
  59 ## file cache to avoid circular include in ASL file
  60 gIncludedAslFile = []
  61
  62 ## Trim preprocessed source code
  63 #
  64 # Remove extra content made by preprocessor. The preprocessor must enable the
  65 # line number generation option when preprocessing.
  66 #
  67 # @param  Source    File to be trimmed
  68 # @param  Target    File to store the trimmed content
  69 # @param  Convert   If True, convert standard HEX format to MASM format
  70 #
  71 def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):
  72     CreateDirectory(os.path.dirname(Target))
  73     try:
  74         with open(Source, "r") as File:
  75             Lines = File.readlines()
  76     except:
  77         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
  78
  79     PreprocessedFile = ""
  80     InjectedFile = ""
  81     LineIndexOfOriginalFile = None
  82     NewLines = []
  83     LineControlDirectiveFound = False
  84     for Index in range(len(Lines)):
  85         Line = Lines[Index]
  86         #
  87         # Find out the name of files injected by preprocessor from the lines
  88         # with Line Control directive
  89         #
  90         MatchList = gLineControlDirective.findall(Line)
  91         if MatchList != []:
  92             MatchList = MatchList[0]
  93             if len(MatchList) == 2:
  94                 LineNumber = int(MatchList[0], 0)
  95                 InjectedFile = MatchList[1]
  96                 InjectedFile = os.path.normpath(InjectedFile)
  97                 InjectedFile = os.path.normcase(InjectedFile)
  98                 # The first injected file must be the preprocessed file itself
  99                 if PreprocessedFile == "":
 100                     PreprocessedFile = InjectedFile
 101             LineControlDirectiveFound = True
 102             continue
 103         elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:
 104             continue
 105
 106         if LineIndexOfOriginalFile is None:
 107             #
 108             # Any non-empty lines must be from original preprocessed file.
 109             # And this must be the first one.
 110             #
 111             LineIndexOfOriginalFile = Index
 112             EdkLogger.verbose("Found original file content starting from line %d"
 113                               % (LineIndexOfOriginalFile + 1))
 114
 115         if TrimLong:
 116             Line = gLongNumberPattern.sub(r"\1", Line)
 117         # convert HEX number format if indicated
 118         if ConvertHex:
 119             Line = gHexNumberPattern.sub(r"0\2h", Line)
 120         else:
 121             Line = gHexNumberPattern.sub(r"\1\2", Line)
 122
 123         # convert Decimal number format
 124         Line = gDecNumberPattern.sub(r"\1", Line)
 125
 126         if LineNumber is not None:
 127             EdkLogger.verbose("Got line directive: line=%d" % LineNumber)
 128             # in case preprocessor removed some lines, like blank or comment lines
 129             if LineNumber <= len(NewLines):
 130                 # possible?
 131                 NewLines[LineNumber - 1] = Line
 132             else:
 133                 if LineNumber > (len(NewLines) + 1):
 134                     for LineIndex in range(len(NewLines), LineNumber-1):
 135                         NewLines.append(TAB_LINE_BREAK)
 136                 NewLines.append(Line)
 137             LineNumber = None
 138             EdkLogger.verbose("Now we have lines: %d" % len(NewLines))
 139         else:
 140             NewLines.append(Line)
 141
 142     # in case there's no line directive or linemarker found
 143     if (not LineControlDirectiveFound) and NewLines == []:
 144         MulPatternFlag = False
 145         SinglePatternFlag = False
 146         Brace = 0
 147         for Index in range(len(Lines)):
 148             Line = Lines[Index]
 149             if MulPatternFlag == False and gTypedef_MulPattern.search(Line) is None:
 150                 if SinglePatternFlag == False and gTypedef_SinglePattern.search(Line) is None:
 151                     # remove "#pragram pack" directive
 152                     if gPragmaPattern.search(Line) is None:
 153                         NewLines.append(Line)
 154                     continue
 155                 elif SinglePatternFlag == False:
 156                     SinglePatternFlag = True
 157                 if Line.find(";") >= 0:
 158                     SinglePatternFlag = False
 159             elif MulPatternFlag == False:
 160                 # found "typedef struct, typedef union, union, struct", keep its position and set a flag
 161                 MulPatternFlag = True
 162
 163             # match { and } to find the end of typedef definition
 164             if Line.find("{") >= 0:
 165                 Brace += 1
 166             elif Line.find("}") >= 0:
 167                 Brace -= 1
 168
 169             # "typedef struct, typedef union, union, struct" must end with a ";"
 170             if Brace == 0 and Line.find(";") >= 0:
 171                 MulPatternFlag = False
 172
 173     # save to file
 174     try:
 175         with open(Target, 'w') as File:
 176             File.writelines(NewLines)
 177     except:
 178         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 179
 180 ## Trim preprocessed VFR file
 181 #
 182 # Remove extra content made by preprocessor. The preprocessor doesn't need to
 183 # enable line number generation option when preprocessing.
 184 #
 185 # @param  Source    File to be trimmed
 186 # @param  Target    File to store the trimmed content
 187 #
 188 def TrimPreprocessedVfr(Source, Target):
 189     CreateDirectory(os.path.dirname(Target))
 190
 191     try:
 192         with open(Source, "r") as File:
 193             Lines = File.readlines()
 194     except:
 195         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 196     # read whole file
 197
 198     FoundTypedef = False
 199     Brace = 0
 200     TypedefStart = 0
 201     TypedefEnd = 0
 202     for Index in range(len(Lines)):
 203         Line = Lines[Index]
 204         # don't trim the lines from "formset" definition to the end of file
 205         if Line.strip() == 'formset':
 206             break
 207
 208         if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):
 209             # empty the line number directive if it's not aomong "typedef struct"
 210             Lines[Index] = "\n"
 211             continue
 212
 213         if FoundTypedef == False and gTypedefPattern.search(Line) is None:
 214             # keep "#pragram pack" directive
 215             if gPragmaPattern.search(Line) is None:
 216                 Lines[Index] = "\n"
 217             continue
 218         elif FoundTypedef == False:
 219             # found "typedef struct", keept its position and set a flag
 220             FoundTypedef = True
 221             TypedefStart = Index
 222
 223         # match { and } to find the end of typedef definition
 224         if Line.find("{") >= 0:
 225             Brace += 1
 226         elif Line.find("}") >= 0:
 227             Brace -= 1
 228
 229         # "typedef struct" must end with a ";"
 230         if Brace == 0 and Line.find(";") >= 0:
 231             FoundTypedef = False
 232             TypedefEnd = Index
 233             # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN
 234             if Line.strip("} ;\r\n") in [TAB_GUID, "EFI_PLABEL", "PAL_CALL_RETURN"]:
 235                 for i in range(TypedefStart, TypedefEnd+1):
 236                     Lines[i] = "\n"
 237
 238     # save all lines trimmed
 239     try:
 240         with open(Target, 'w') as File:
 241             File.writelines(Lines)
 242     except:
 243         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 244
 245 ## Read the content  ASL file, including ASL included, recursively
 246 #
 247 # @param  Source            File to be read
 248 # @param  Indent            Spaces before the Include() statement
 249 # @param  IncludePathList   The list of external include file
 250 # @param  LocalSearchPath   If LocalSearchPath is specified, this path will be searched
 251 #                           first for the included file; otherwise, only the path specified
 252 #                           in the IncludePathList will be searched.
 253 #
 254 def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
 255     NewFileContent = []
 256
 257     try:
 258         #
 259         # Search LocalSearchPath first if it is specified.
 260         #
 261         if LocalSearchPath:
 262             SearchPathList = [LocalSearchPath] + IncludePathList
 263         else:
 264             SearchPathList = IncludePathList
 265
 266         for IncludePath in SearchPathList:
 267             IncludeFile = os.path.join(IncludePath, Source)
 268             if os.path.isfile(IncludeFile):
 269                 try:
 270                     with open(IncludeFile, "r") as File:
 271                         F = File.readlines()
 272                 except:
 273                     with codecs.open(IncludeFile, "r", encoding='utf-8') as File:
 274                         F = File.readlines()
 275                 break
 276         else:
 277             EdkLogger.error("Trim", "Failed to find include file %s" % Source)
 278     except:
 279         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)
 280
 281
 282     # avoid A "include" B and B "include" A
 283     IncludeFile = os.path.abspath(os.path.normpath(IncludeFile))
 284     if IncludeFile in gIncludedAslFile:
 285         EdkLogger.warn("Trim", "Circular include",
 286                        ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), IncludeFile))
 287         return []
 288     gIncludedAslFile.append(IncludeFile)
 289
 290     for Line in F:
 291         LocalSearchPath = None
 292         Result = gAslIncludePattern.findall(Line)
 293         if len(Result) == 0:
 294             Result = gAslCIncludePattern.findall(Line)
 295             if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:
 296                 NewFileContent.append("%s%s" % (Indent, Line))
 297                 continue
 298             #
 299             # We should first search the local directory if current file are using pattern #include "XXX"
 300             #
 301             if Result[0][2] == '"':
 302                 LocalSearchPath = os.path.dirname(IncludeFile)
 303         CurrentIndent = Indent + Result[0][0]
 304         IncludedFile = Result[0][1]
 305         NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, IncludePathList, LocalSearchPath))
 306         NewFileContent.append("\n")
 307
 308     gIncludedAslFile.pop()
 309
 310     return NewFileContent
 311
 312
 313 ## Trim ASL file
 314 #
 315 # Replace ASL include statement with the content the included file
 316 #
 317 # @param  Source          File to be trimmed
 318 # @param  Target          File to store the trimmed content
 319 # @param  IncludePathFile The file to log the external include path
 320 #
 321 def TrimAslFile(Source, Target, IncludePathFile):
 322     CreateDirectory(os.path.dirname(Target))
 323
 324     SourceDir = os.path.dirname(Source)
 325     if SourceDir == '':
 326         SourceDir = '.'
 327
 328     #
 329     # Add source directory as the first search directory
 330     #
 331     IncludePathList = [SourceDir]
 332
 333     #
 334     # If additional include path file is specified, append them all
 335     # to the search directory list.
 336     #
 337     if IncludePathFile:
 338         try:
 339             LineNum = 0
 340             with open(IncludePathFile, 'r') as File:
 341                 FileLines = File.readlines()
 342             for Line in FileLines:
 343                 LineNum += 1
 344                 if Line.startswith("/I") or Line.startswith ("-I"):
 345                     IncludePathList.append(Line[2:].strip())
 346                 else:
 347                     EdkLogger.warn("Trim", "Invalid include line in include list file.", IncludePathFile, LineNum)
 348         except:
 349             EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=IncludePathFile)
 350
 351     Lines = DoInclude(Source, '', IncludePathList)
 352
 353     #
 354     # Undef MIN and MAX to avoid collision in ASL source code
 355     #
 356     Lines.insert(0, "#undef MIN\n#undef MAX\n")
 357
 358     # save all lines trimmed
 359     try:
 360         with open(Target, 'w') as File:
 361             File.writelines(Lines)
 362     except:
 363         EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)
 364
 365 def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):
 366     VfrNameList = []
 367     if os.path.isdir(DebugDir):
 368         for CurrentDir, Dirs, Files in os.walk(DebugDir):
 369             for FileName in Files:
 370                 Name, Ext = os.path.splitext(FileName)
 371                 if Ext == '.c' and Name != 'AutoGen':
 372                     VfrNameList.append (Name + 'Bin')
 373
 374     VfrNameList.append (ModuleName + 'Strings')
 375
 376     EfiFileName = os.path.join(DebugDir, ModuleName + '.efi')
 377     MapFileName = os.path.join(DebugDir, ModuleName + '.map')
 378     VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, VfrNameList)
 379
 380     if not VfrUniOffsetList:
 381         return
 382
 383     try:
 384         fInputfile = open(OutputFile, "wb+")
 385     except:
 386         EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)
 387
 388     # Use a instance of BytesIO to cache data
 389     fStringIO = BytesIO()
 390
 391     for Item in VfrUniOffsetList:
 392         if (Item[0].find("Strings") != -1):
 393             #
 394             # UNI offset in image.
 395             # GUID + Offset
 396             # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }
 397             #
 398             UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'
 399             fStringIO.write(UniGuid)
 400             UniValue = pack ('Q', int (Item[1], 16))
 401             fStringIO.write (UniValue)
 402         else:
 403             #
 404             # VFR binary offset in image.
 405             # GUID + Offset
 406             # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };
 407             #
 408             VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'
 409             fStringIO.write(VfrGuid)
 410             type (Item[1])
 411             VfrValue = pack ('Q', int (Item[1], 16))
 412             fStringIO.write (VfrValue)
 413
 414     #
 415     # write data into file.
 416     #
 417     try :
 418         fInputfile.write (fStringIO.getvalue())
 419     except:
 420         EdkLogger.error("Trim", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the file been locked or using by other applications." %OutputFile, None)
 421
 422     fStringIO.close ()
 423     fInputfile.close ()
 424
 425
 426 ## Parse command line options
 427 #
 428 # Using standard Python module optparse to parse command line option of this tool.
 429 #
 430 # @retval Options   A optparse.Values object containing the parsed options
 431 # @retval InputFile Path of file to be trimmed
 432 #
 433 def Options():
 434     OptionList = [
 435         make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",
 436                           help="The input file is preprocessed source code, including C or assembly code"),
 437         make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",
 438                           help="The input file is preprocessed VFR file"),
 439         make_option("--Vfr-Uni-Offset", dest="FileType", const="VfrOffsetBin", action="store_const",
 440                           help="The input file is EFI image"),
 441         make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",
 442                           help="The input file is ASL file"),
 443         make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",
 444                           help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),
 445
 446         make_option("-l", "--trim-long", dest="TrimLong", action="store_true",
 447                           help="Remove postfix of long number"),
 448         make_option("-i", "--include-path-file", dest="IncludePathFile",
 449                           help="The input file is include path list to search for ASL include file"),
 450         make_option("-o", "--output", dest="OutputFile",
 451                           help="File to store the trimmed content"),
 452         make_option("--ModuleName", dest="ModuleName", help="The module's BASE_NAME"),
 453         make_option("--DebugDir", dest="DebugDir",
 454                           help="Debug Output directory to store the output files"),
 455         make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,
 456                           help="Run verbosely"),
 457         make_option("-d", "--debug", dest="LogLevel", type="int",
 458                           help="Run with debug information"),
 459         make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,
 460                           help="Run quietly"),
 461         make_option("-?", action="help", help="show this help message and exit"),
 462     ]
 463
 464     # use clearer usage to override default usage message
 465     UsageString = "%prog [-s|-r|-a|--Vfr-Uni-Offset] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] [--ModuleName <ModuleName>] [--DebugDir <DebugDir>] [<input_file>]"
 466
 467     Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)
 468     Parser.set_defaults(FileType="Vfr")
 469     Parser.set_defaults(ConvertHex=False)
 470     Parser.set_defaults(LogLevel=EdkLogger.INFO)
 471
 472     Options, Args = Parser.parse_args()
 473
 474     # error check
 475     if Options.FileType == 'VfrOffsetBin':
 476         if len(Args) == 0:
 477             return Options, ''
 478         elif len(Args) > 1:
 479             EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
 480     if len(Args) == 0:
 481         EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())
 482     if len(Args) > 1:
 483         EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())
 484
 485     InputFile = Args[0]
 486     return Options, InputFile
 487
 488 ## Entrance method
 489 #
 490 # This method mainly dispatch specific methods per the command line options.
 491 # If no error found, return zero value so the caller of this tool can know
 492 # if it's executed successfully or not.
 493 #
 494 # @retval 0     Tool was successful
 495 # @retval 1     Tool failed
 496 #
 497 def Main():
 498     try:
 499         EdkLogger.Initialize()
 500         CommandOptions, InputFile = Options()
 501         if CommandOptions.LogLevel < EdkLogger.DEBUG_9:
 502             EdkLogger.SetLevel(CommandOptions.LogLevel + 1)
 503         else:
 504             EdkLogger.SetLevel(CommandOptions.LogLevel)
 505     except FatalError as X:
 506         return 1
 507
 508     try:
 509         if CommandOptions.FileType == "Vfr":
 510             if CommandOptions.OutputFile is None:
 511                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 512             TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)
 513         elif CommandOptions.FileType == "Asl":
 514             if CommandOptions.OutputFile is None:
 515                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 516             TrimAslFile(InputFile, CommandOptions.OutputFile, CommandOptions.IncludePathFile)
 517         elif CommandOptions.FileType == "VfrOffsetBin":
 518             GenerateVfrBinSec(CommandOptions.ModuleName, CommandOptions.DebugDir, CommandOptions.OutputFile)
 519         else :
 520             if CommandOptions.OutputFile is None:
 521                 CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'
 522             TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex, CommandOptions.TrimLong)
 523     except FatalError as X:
 524         import platform
 525         import traceback
 526         if CommandOptions is not None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:
 527             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 528         return 1
 529     except:
 530         import traceback
 531         import platform
 532         EdkLogger.error(
 533                     "\nTrim",
 534                     CODE_ERROR,
 535                     "Unknown fatal error when trimming [%s]" % InputFile,
 536                     ExtraData="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n",
 537                     RaiseError=False
 538                     )
 539         EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())
 540         return 1
 541
 542     return 0
 543
 544 if __name__ == '__main__':
 545     r = Main()
 546     ## 0-127 is a safe return range, and 1 is a standard default error
 547     if r < 0 or r > 127: r = 1
 548     sys.exit(r)
 549