[mirror_edk2.git] / BaseTools / Source / Python / Trim / Trim.py

## @file\r
# Trim files preprocessed by compiler\r
#\r
# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
#\r
\r
##\r
# Import Modules\r
#\r
import Common.LongFilePathOs as os\r
import sys\r
import re\r
from io import BytesIO\r
import codecs\r
from optparse import OptionParser\r
from optparse import make_option\r
from Common.BuildToolError import *\r
from Common.Misc import *\r
from Common.DataType import *\r
from Common.BuildVersion import gBUILD_VERSION\r
import Common.EdkLogger as EdkLogger\r
from Common.LongFilePathSupport import OpenLongFilePath as open\r
\r
# Version and Copyright\r
__version_number__ = ("0.10" + " " + gBUILD_VERSION)\r
__version__ = "%prog Version " + __version_number__\r
__copyright__ = "Copyright (c) 2007-2018, Intel Corporation. All rights reserved."\r
\r
## Regular expression for matching Line Control directive like "#line xxx"\r
gLineControlDirective = re.compile('^\s*#(?:line)?\s+([0-9]+)\s+"*([^"]*)"')\r
## Regular expression for matching "typedef struct"\r
gTypedefPattern = re.compile("^\s*typedef\s+struct(\s+\w+)?\s*[{]*$", re.MULTILINE)\r
## Regular expression for matching "#pragma pack"\r
gPragmaPattern = re.compile("^\s*#pragma\s+pack", re.MULTILINE)\r
## Regular expression for matching "typedef"\r
gTypedef_SinglePattern = re.compile("^\s*typedef", re.MULTILINE)\r
## Regular expression for matching "typedef struct, typedef union, struct, union"\r
gTypedef_MulPattern = re.compile("^\s*(typedef)?\s+(struct|union)(\s+\w+)?\s*[{]*$", re.MULTILINE)\r
\r
#\r
# The following number pattern match will only match if following criteria is met:\r
# There is leading non-(alphanumeric or _) character, and no following alphanumeric or _\r
# as the pattern is greedily match, so it is ok for the gDecNumberPattern or gHexNumberPattern to grab the maximum match\r
#\r
## Regular expression for matching HEX number\r
gHexNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX])([0-9a-fA-F]+)(U(?=$|[^a-zA-Z0-9_]))?")\r
## Regular expression for matching decimal number with 'U' postfix\r
gDecNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])([0-9]+)U(?=$|[^a-zA-Z0-9_])")\r
## Regular expression for matching constant with 'ULL' 'LL' postfix\r
gLongNumberPattern = re.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?LL(?=$|[^a-zA-Z0-9_])")\r
\r
## Regular expression for matching "Include ()" in asl file\r
gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", re.MULTILINE)\r
## Regular expression for matching C style #include "XXX.asl" in asl file\r
gAslCIncludePattern = re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re.MULTILINE)\r
## Patterns used to convert EDK conventions to EDK2 ECP conventions\r
\r
## file cache to avoid circular include in ASL file\r
gIncludedAslFile = []\r
\r
## Trim preprocessed source code\r
#\r
# Remove extra content made by preprocessor. The preprocessor must enable the\r
# line number generation option when preprocessing.\r
#\r
# @param  Source    File to be trimmed\r
# @param  Target    File to store the trimmed content\r
# @param  Convert   If True, convert standard HEX format to MASM format\r
#\r
def TrimPreprocessedFile(Source, Target, ConvertHex, TrimLong):\r
    CreateDirectory(os.path.dirname(Target))\r
    try:\r
        with open(Source, "r") as File:\r
            Lines = File.readlines()\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)\r
\r
    PreprocessedFile = ""\r
    InjectedFile = ""\r
    LineIndexOfOriginalFile = None\r
    NewLines = []\r
    LineControlDirectiveFound = False\r
    for Index in range(len(Lines)):\r
        Line = Lines[Index]\r
        #\r
        # Find out the name of files injected by preprocessor from the lines\r
        # with Line Control directive\r
        #\r
        MatchList = gLineControlDirective.findall(Line)\r
        if MatchList != []:\r
            MatchList = MatchList[0]\r
            if len(MatchList) == 2:\r
                LineNumber = int(MatchList[0], 0)\r
                InjectedFile = MatchList[1]\r
                InjectedFile = os.path.normpath(InjectedFile)\r
                InjectedFile = os.path.normcase(InjectedFile)\r
                # The first injected file must be the preprocessed file itself\r
                if PreprocessedFile == "":\r
                    PreprocessedFile = InjectedFile\r
            LineControlDirectiveFound = True\r
            continue\r
        elif PreprocessedFile == "" or InjectedFile != PreprocessedFile:\r
            continue\r
\r
        if LineIndexOfOriginalFile is None:\r
            #\r
            # Any non-empty lines must be from original preprocessed file.\r
            # And this must be the first one.\r
            #\r
            LineIndexOfOriginalFile = Index\r
            EdkLogger.verbose("Found original file content starting from line %d"\r
                              % (LineIndexOfOriginalFile + 1))\r
\r
        if TrimLong:\r
            Line = gLongNumberPattern.sub(r"\1", Line)\r
        # convert HEX number format if indicated\r
        if ConvertHex:\r
            Line = gHexNumberPattern.sub(r"0\2h", Line)\r
        else:\r
            Line = gHexNumberPattern.sub(r"\1\2", Line)\r
\r
        # convert Decimal number format\r
        Line = gDecNumberPattern.sub(r"\1", Line)\r
\r
        if LineNumber is not None:\r
            EdkLogger.verbose("Got line directive: line=%d" % LineNumber)\r
            # in case preprocessor removed some lines, like blank or comment lines\r
            if LineNumber <= len(NewLines):\r
                # possible?\r
                NewLines[LineNumber - 1] = Line\r
            else:\r
                if LineNumber > (len(NewLines) + 1):\r
                    for LineIndex in range(len(NewLines), LineNumber-1):\r
                        NewLines.append(TAB_LINE_BREAK)\r
                NewLines.append(Line)\r
            LineNumber = None\r
            EdkLogger.verbose("Now we have lines: %d" % len(NewLines))\r
        else:\r
            NewLines.append(Line)\r
\r
    # in case there's no line directive or linemarker found\r
    if (not LineControlDirectiveFound) and NewLines == []:\r
        MulPatternFlag = False\r
        SinglePatternFlag = False\r
        Brace = 0\r
        for Index in range(len(Lines)):\r
            Line = Lines[Index]\r
            if MulPatternFlag == False and gTypedef_MulPattern.search(Line) is None:\r
                if SinglePatternFlag == False and gTypedef_SinglePattern.search(Line) is None:\r
                    # remove "#pragram pack" directive\r
                    if gPragmaPattern.search(Line) is None:\r
                        NewLines.append(Line)\r
                    continue\r
                elif SinglePatternFlag == False:\r
                    SinglePatternFlag = True\r
                if Line.find(";") >= 0:\r
                    SinglePatternFlag = False\r
            elif MulPatternFlag == False:\r
                # found "typedef struct, typedef union, union, struct", keep its position and set a flag\r
                MulPatternFlag = True\r
\r
            # match { and } to find the end of typedef definition\r
            if Line.find("{") >= 0:\r
                Brace += 1\r
            elif Line.find("}") >= 0:\r
                Brace -= 1\r
\r
            # "typedef struct, typedef union, union, struct" must end with a ";"\r
            if Brace == 0 and Line.find(";") >= 0:\r
                MulPatternFlag = False\r
\r
    # save to file\r
    try:\r
        with open(Target, 'w') as File:\r
            File.writelines(NewLines)\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)\r
\r
## Trim preprocessed VFR file\r
#\r
# Remove extra content made by preprocessor. The preprocessor doesn't need to\r
# enable line number generation option when preprocessing.\r
#\r
# @param  Source    File to be trimmed\r
# @param  Target    File to store the trimmed content\r
#\r
def TrimPreprocessedVfr(Source, Target):\r
    CreateDirectory(os.path.dirname(Target))\r
\r
    try:\r
        with open(Source, "r") as File:\r
            Lines = File.readlines()\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)\r
    # read whole file\r
\r
    FoundTypedef = False\r
    Brace = 0\r
    TypedefStart = 0\r
    TypedefEnd = 0\r
    for Index in range(len(Lines)):\r
        Line = Lines[Index]\r
        # don't trim the lines from "formset" definition to the end of file\r
        if Line.strip() == 'formset':\r
            break\r
\r
        if FoundTypedef == False and (Line.find('#line') == 0 or Line.find('# ') == 0):\r
            # empty the line number directive if it's not aomong "typedef struct"\r
            Lines[Index] = "\n"\r
            continue\r
\r
        if FoundTypedef == False and gTypedefPattern.search(Line) is None:\r
            # keep "#pragram pack" directive\r
            if gPragmaPattern.search(Line) is None:\r
                Lines[Index] = "\n"\r
            continue\r
        elif FoundTypedef == False:\r
            # found "typedef struct", keept its position and set a flag\r
            FoundTypedef = True\r
            TypedefStart = Index\r
\r
        # match { and } to find the end of typedef definition\r
        if Line.find("{") >= 0:\r
            Brace += 1\r
        elif Line.find("}") >= 0:\r
            Brace -= 1\r
\r
        # "typedef struct" must end with a ";"\r
        if Brace == 0 and Line.find(";") >= 0:\r
            FoundTypedef = False\r
            TypedefEnd = Index\r
            # keep all "typedef struct" except to GUID, EFI_PLABEL and PAL_CALL_RETURN\r
            if Line.strip("} ;\r\n") in [TAB_GUID, "EFI_PLABEL", "PAL_CALL_RETURN"]:\r
                for i in range(TypedefStart, TypedefEnd+1):\r
                    Lines[i] = "\n"\r
\r
    # save all lines trimmed\r
    try:\r
        with open(Target, 'w') as File:\r
            File.writelines(Lines)\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)\r
\r
## Read the content  ASL file, including ASL included, recursively\r
#\r
# @param  Source            File to be read\r
# @param  Indent            Spaces before the Include() statement\r
# @param  IncludePathList   The list of external include file\r
# @param  LocalSearchPath   If LocalSearchPath is specified, this path will be searched\r
#                           first for the included file; otherwise, only the path specified\r
#                           in the IncludePathList will be searched.\r
#\r
def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):\r
    NewFileContent = []\r
\r
    try:\r
        #\r
        # Search LocalSearchPath first if it is specified.\r
        #\r
        if LocalSearchPath:\r
            SearchPathList = [LocalSearchPath] + IncludePathList\r
        else:\r
            SearchPathList = IncludePathList\r
\r
        for IncludePath in SearchPathList:\r
            IncludeFile = os.path.join(IncludePath, Source)\r
            if os.path.isfile(IncludeFile):\r
                try:\r
                    with open(IncludeFile, "r") as File:\r
                        F = File.readlines()\r
                except:\r
                    with codecs.open(IncludeFile, "r", encoding='utf-8') as File:\r
                        F = File.readlines()\r
                break\r
        else:\r
            EdkLogger.error("Trim", "Failed to find include file %s" % Source)\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Source)\r
\r
\r
    # avoid A "include" B and B "include" A\r
    IncludeFile = os.path.abspath(os.path.normpath(IncludeFile))\r
    if IncludeFile in gIncludedAslFile:\r
        EdkLogger.warn("Trim", "Circular include",\r
                       ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), IncludeFile))\r
        return []\r
    gIncludedAslFile.append(IncludeFile)\r
\r
    for Line in F:\r
        LocalSearchPath = None\r
        Result = gAslIncludePattern.findall(Line)\r
        if len(Result) == 0:\r
            Result = gAslCIncludePattern.findall(Line)\r
            if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:\r
                NewFileContent.append("%s%s" % (Indent, Line))\r
                continue\r
            #\r
            # We should first search the local directory if current file are using pattern #include "XXX"\r
            #\r
            if Result[0][2] == '"':\r
                LocalSearchPath = os.path.dirname(IncludeFile)\r
        CurrentIndent = Indent + Result[0][0]\r
        IncludedFile = Result[0][1]\r
        NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, IncludePathList, LocalSearchPath))\r
        NewFileContent.append("\n")\r
\r
    gIncludedAslFile.pop()\r
\r
    return NewFileContent\r
\r
\r
## Trim ASL file\r
#\r
# Replace ASL include statement with the content the included file\r
#\r
# @param  Source          File to be trimmed\r
# @param  Target          File to store the trimmed content\r
# @param  IncludePathFile The file to log the external include path\r
#\r
def TrimAslFile(Source, Target, IncludePathFile):\r
    CreateDirectory(os.path.dirname(Target))\r
\r
    SourceDir = os.path.dirname(Source)\r
    if SourceDir == '':\r
        SourceDir = '.'\r
\r
    #\r
    # Add source directory as the first search directory\r
    #\r
    IncludePathList = [SourceDir]\r
\r
    #\r
    # If additional include path file is specified, append them all\r
    # to the search directory list.\r
    #\r
    if IncludePathFile:\r
        try:\r
            LineNum = 0\r
            with open(IncludePathFile, 'r') as File:\r
                FileLines = File.readlines()\r
            for Line in FileLines:\r
                LineNum += 1\r
                if Line.startswith("/I") or Line.startswith ("-I"):\r
                    IncludePathList.append(Line[2:].strip())\r
                else:\r
                    EdkLogger.warn("Trim", "Invalid include line in include list file.", IncludePathFile, LineNum)\r
        except:\r
            EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=IncludePathFile)\r
\r
    Lines = DoInclude(Source, '', IncludePathList)\r
\r
    #\r
    # Undef MIN and MAX to avoid collision in ASL source code\r
    #\r
    Lines.insert(0, "#undef MIN\n#undef MAX\n")\r
\r
    # save all lines trimmed\r
    try:\r
        with open(Target, 'w') as File:\r
            File.writelines(Lines)\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, ExtraData=Target)\r
\r
def GenerateVfrBinSec(ModuleName, DebugDir, OutputFile):\r
    VfrNameList = []\r
    if os.path.isdir(DebugDir):\r
        for CurrentDir, Dirs, Files in os.walk(DebugDir):\r
            for FileName in Files:\r
                Name, Ext = os.path.splitext(FileName)\r
                if Ext == '.c' and Name != 'AutoGen':\r
                    VfrNameList.append (Name + 'Bin')\r
\r
    VfrNameList.append (ModuleName + 'Strings')\r
\r
    EfiFileName = os.path.join(DebugDir, ModuleName + '.efi')\r
    MapFileName = os.path.join(DebugDir, ModuleName + '.map')\r
    VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, VfrNameList)\r
\r
    if not VfrUniOffsetList:\r
        return\r
\r
    try:\r
        fInputfile = open(OutputFile, "wb+")\r
    except:\r
        EdkLogger.error("Trim", FILE_OPEN_FAILURE, "File open failed for %s" %OutputFile, None)\r
\r
    # Use a instance of BytesIO to cache data\r
    fStringIO = BytesIO()\r
\r
    for Item in VfrUniOffsetList:\r
        if (Item[0].find("Strings") != -1):\r
            #\r
            # UNI offset in image.\r
            # GUID + Offset\r
            # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }\r
            #\r
            UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'\r
            fStringIO.write(UniGuid)\r
            UniValue = pack ('Q', int (Item[1], 16))\r
            fStringIO.write (UniValue)\r
        else:\r
            #\r
            # VFR binary offset in image.\r
            # GUID + Offset\r
            # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };\r
            #\r
            VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'\r
            fStringIO.write(VfrGuid)\r
            type (Item[1])\r
            VfrValue = pack ('Q', int (Item[1], 16))\r
            fStringIO.write (VfrValue)\r
\r
    #\r
    # write data into file.\r
    #\r
    try :\r
        fInputfile.write (fStringIO.getvalue())\r
    except:\r
        EdkLogger.error("Trim", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the file been locked or using by other applications." %OutputFile, None)\r
\r
    fStringIO.close ()\r
    fInputfile.close ()\r
\r
\r
## Parse command line options\r
#\r
# Using standard Python module optparse to parse command line option of this tool.\r
#\r
# @retval Options   A optparse.Values object containing the parsed options\r
# @retval InputFile Path of file to be trimmed\r
#\r
def Options():\r
    OptionList = [\r
        make_option("-s", "--source-code", dest="FileType", const="SourceCode", action="store_const",\r
                          help="The input file is preprocessed source code, including C or assembly code"),\r
        make_option("-r", "--vfr-file", dest="FileType", const="Vfr", action="store_const",\r
                          help="The input file is preprocessed VFR file"),\r
        make_option("--Vfr-Uni-Offset", dest="FileType", const="VfrOffsetBin", action="store_const",\r
                          help="The input file is EFI image"),\r
        make_option("-a", "--asl-file", dest="FileType", const="Asl", action="store_const",\r
                          help="The input file is ASL file"),\r
        make_option("-c", "--convert-hex", dest="ConvertHex", action="store_true",\r
                          help="Convert standard hex format (0xabcd) to MASM format (abcdh)"),\r
\r
        make_option("-l", "--trim-long", dest="TrimLong", action="store_true",\r
                          help="Remove postfix of long number"),\r
        make_option("-i", "--include-path-file", dest="IncludePathFile",\r
                          help="The input file is include path list to search for ASL include file"),\r
        make_option("-o", "--output", dest="OutputFile",\r
                          help="File to store the trimmed content"),\r
        make_option("--ModuleName", dest="ModuleName", help="The module's BASE_NAME"),\r
        make_option("--DebugDir", dest="DebugDir",\r
                          help="Debug Output directory to store the output files"),\r
        make_option("-v", "--verbose", dest="LogLevel", action="store_const", const=EdkLogger.VERBOSE,\r
                          help="Run verbosely"),\r
        make_option("-d", "--debug", dest="LogLevel", type="int",\r
                          help="Run with debug information"),\r
        make_option("-q", "--quiet", dest="LogLevel", action="store_const", const=EdkLogger.QUIET,\r
                          help="Run quietly"),\r
        make_option("-?", action="help", help="show this help message and exit"),\r
    ]\r
\r
    # use clearer usage to override default usage message\r
    UsageString = "%prog [-s|-r|-a|--Vfr-Uni-Offset] [-c] [-v|-d <debug_level>|-q] [-i <include_path_file>] [-o <output_file>] [--ModuleName <ModuleName>] [--DebugDir <DebugDir>] [<input_file>]"\r
\r
    Parser = OptionParser(description=__copyright__, version=__version__, option_list=OptionList, usage=UsageString)\r
    Parser.set_defaults(FileType="Vfr")\r
    Parser.set_defaults(ConvertHex=False)\r
    Parser.set_defaults(LogLevel=EdkLogger.INFO)\r
\r
    Options, Args = Parser.parse_args()\r
\r
    # error check\r
    if Options.FileType == 'VfrOffsetBin':\r
        if len(Args) == 0:\r
            return Options, ''\r
        elif len(Args) > 1:\r
            EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())\r
    if len(Args) == 0:\r
        EdkLogger.error("Trim", OPTION_MISSING, ExtraData=Parser.get_usage())\r
    if len(Args) > 1:\r
        EdkLogger.error("Trim", OPTION_NOT_SUPPORTED, ExtraData=Parser.get_usage())\r
\r
    InputFile = Args[0]\r
    return Options, InputFile\r
\r
## Entrance method\r
#\r
# This method mainly dispatch specific methods per the command line options.\r
# If no error found, return zero value so the caller of this tool can know\r
# if it's executed successfully or not.\r
#\r
# @retval 0     Tool was successful\r
# @retval 1     Tool failed\r
#\r
def Main():\r
    try:\r
        EdkLogger.Initialize()\r
        CommandOptions, InputFile = Options()\r
        if CommandOptions.LogLevel < EdkLogger.DEBUG_9:\r
            EdkLogger.SetLevel(CommandOptions.LogLevel + 1)\r
        else:\r
            EdkLogger.SetLevel(CommandOptions.LogLevel)\r
    except FatalError as X:\r
        return 1\r
\r
    try:\r
        if CommandOptions.FileType == "Vfr":\r
            if CommandOptions.OutputFile is None:\r
                CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'\r
            TrimPreprocessedVfr(InputFile, CommandOptions.OutputFile)\r
        elif CommandOptions.FileType == "Asl":\r
            if CommandOptions.OutputFile is None:\r
                CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'\r
            TrimAslFile(InputFile, CommandOptions.OutputFile, CommandOptions.IncludePathFile)\r
        elif CommandOptions.FileType == "VfrOffsetBin":\r
            GenerateVfrBinSec(CommandOptions.ModuleName, CommandOptions.DebugDir, CommandOptions.OutputFile)\r
        else :\r
            if CommandOptions.OutputFile is None:\r
                CommandOptions.OutputFile = os.path.splitext(InputFile)[0] + '.iii'\r
            TrimPreprocessedFile(InputFile, CommandOptions.OutputFile, CommandOptions.ConvertHex, CommandOptions.TrimLong)\r
    except FatalError as X:\r
        import platform\r
        import traceback\r
        if CommandOptions is not None and CommandOptions.LogLevel <= EdkLogger.DEBUG_9:\r
            EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())\r
        return 1\r
    except:\r
        import traceback\r
        import platform\r
        EdkLogger.error(\r
                    "\nTrim",\r
                    CODE_ERROR,\r
                    "Unknown fatal error when trimming [%s]" % InputFile,\r
                    ExtraData="\n(Please send email to edk2-devel@lists.01.org for help, attaching following call stack trace!)\n",\r
                    RaiseError=False\r
                    )\r
        EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())\r
        return 1\r
\r
    return 0\r
\r
if __name__ == '__main__':\r
    r = Main()\r
    ## 0-127 is a safe return range, and 1 is a standard default error\r
    if r < 0 or r > 127: r = 1\r
    sys.exit(r)\r
\r