BaseTools:build failure in CLANGPDB tool chain

[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / IncludesAutoGen.py

## @file\r
# Build cache intermediate result and state\r
#\r
# Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.<BR>\r
# Copyright (c) 2020, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
#\r
from Common.caching import cached_property\r
import Common.EdkLogger as EdkLogger\r
import Common.LongFilePathOs as os\r
from Common.BuildToolError import *\r
from Common.Misc import SaveFileOnChange, PathClass\r
from Common.Misc import TemplateString\r
import sys\r
gIsFileMap = {}\r
\r
DEP_FILE_TAIL = "# Updated \n"\r
\r
class IncludesAutoGen():\r
    """ This class is to manage the dependent files witch are used in Makefile to support incremental build.\r
        1. C files:\r
            1. MSVS.\r
               cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures\r
               that messages and generate dependency files, .deps files.\r
            2. CLANG and GCC\r
               -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the\r
               .deps files.\r
        2. ASL files:\r
            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.\r
            2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file\r
            3. build tool updates the .deps file\r
        3. ASM files (.asm, .s or .nasm):\r
            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.\r
            2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file\r
            3. build tool updates the .deps file\r
    """\r
    def __init__(self, makefile_folder, ModuleAuto):\r
        self.d_folder = makefile_folder\r
        self.makefile_folder = makefile_folder\r
        self.module_autogen = ModuleAuto\r
        self.ToolChainFamily = ModuleAuto.ToolChainFamily\r
        self.workspace = ModuleAuto.WorkspaceDir\r
\r
    def CreateModuleDeps(self):\r
        SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)\r
\r
    def CreateDepsInclude(self):\r
        deps_file = {'deps_file':self.deps_files}\r
\r
        MakePath = self.module_autogen.BuildOption.get('MAKE', {}).get('PATH')\r
        if not MakePath:\r
            EdkLogger.error("build", PARAMETER_MISSING, Message="No Make path available.")\r
        elif "nmake" in MakePath:\r
            _INCLUDE_DEPS_TEMPLATE = TemplateString('''\r
        ${BEGIN}\r
        !IF EXIST(${deps_file})\r
        !INCLUDE ${deps_file}\r
        !ENDIF\r
        ${END}\r
               ''')\r
        else:\r
            _INCLUDE_DEPS_TEMPLATE = TemplateString('''\r
        ${BEGIN}\r
        -include ${deps_file}\r
        ${END}\r
               ''')\r
\r
        try:\r
            deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)\r
        except Exception as e:\r
            print(e)\r
        SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)\r
\r
    def CreateDepsTarget(self):\r
        SaveFileOnChange(os.path.join(self.makefile_folder,"deps_target"),"\n".join([item +":" for item in self.DepsCollection]),False)\r
\r
    @cached_property\r
    def deps_files(self):\r
        """ Get all .deps file under module build folder. """\r
        deps_files = []\r
        for root, _, files in os.walk(self.d_folder, topdown=False):\r
            for name in files:\r
                if not name.endswith(".deps"):\r
                    continue\r
                abspath = os.path.join(root, name)\r
                deps_files.append(abspath)\r
        return deps_files\r
\r
    @cached_property\r
    def DepsCollection(self):\r
        """ Collect all the dependency files list from all .deps files under a module's build folder """\r
        includes = set()\r
        targetname = [item[0].Name for item in self.TargetFileList.values()]\r
        for abspath in self.deps_files:\r
            try:\r
                with open(abspath,"r") as fd:\r
                    lines = fd.readlines()\r
\r
                firstlineitems = lines[0].split(": ")\r
                dependency_file = firstlineitems[1].strip(" \\\n")\r
                dependency_file = dependency_file.strip('''"''')\r
                if dependency_file:\r
                    if os.path.normpath(dependency_file +".deps") == abspath:\r
                        continue\r
                    filename = os.path.basename(dependency_file).strip()\r
                    if filename not in self.SourceFileList and filename not in targetname:\r
                        includes.add(dependency_file.strip())\r
\r
                for item in lines[1:]:\r
                    if item == DEP_FILE_TAIL:\r
                        continue\r
                    dependency_file = item.strip(" \\\n")\r
                    dependency_file = dependency_file.strip('''"''')\r
                    if dependency_file == '':\r
                        continue\r
                    if os.path.normpath(dependency_file +".deps") == abspath:\r
                        continue\r
                    filename = os.path.basename(dependency_file).strip()\r
                    if filename in self.SourceFileList:\r
                        continue\r
                    if filename in targetname:\r
                        continue\r
                    includes.add(dependency_file.strip())\r
            except Exception as e:\r
                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)\r
                continue\r
        rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))\r
        return rt\r
\r
    @cached_property\r
    def SourceFileList(self):\r
        """ Get a map of module's source files name to module's source files path """\r
        source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}\r
        middle_file = {}\r
        for afile in source:\r
            if afile.upper().endswith(".VFR"):\r
                middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})\r
            if afile.upper().endswith((".S","ASM")):\r
                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})\r
            if afile.upper().endswith(".ASL"):\r
                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})\r
        source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})\r
        source.update(middle_file)\r
        return source\r
\r
    @cached_property\r
    def HasNamesakeSourceFile(self):\r
        source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])\r
        rt = len(source_base_name) != len(self.module_autogen.SourceFileList)\r
        return rt\r
    @cached_property\r
    def CcPPCommandPathSet(self):\r
        rt = set()\r
        rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))\r
        rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))\r
        rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))\r
        rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))\r
        rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))\r
        rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))\r
        rt.discard(None)\r
        return rt\r
    @cached_property\r
    def TargetFileList(self):\r
        """ Get a map of module's target name to a tuple of module's targets path and whose input file path """\r
        targets = {}\r
        targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))\r
        for item in self.module_autogen.Targets.values():\r
            for block in item:\r
                targets[block.Target.Path] = (block.Target,block.Inputs[0])\r
        return targets\r
\r
    def GetRealTarget(self,source_file_abs):\r
        """ Get the final target file based on source file abspath """\r
        source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}\r
        source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}\r
        target_abs = source_target_map.get(source_file_abs)\r
        if target_abs is None:\r
            if source_file_abs.strip().endswith(".i"):\r
                sourcefilename = os.path.basename(source_file_abs.strip())\r
                for sourcefile in source_name_map:\r
                    if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:\r
                        target_abs = source_name_map[sourcefile]\r
                        break\r
                else:\r
                    target_abs = source_file_abs\r
            else:\r
                target_abs = source_file_abs\r
        return target_abs\r
\r
    def CreateDepsFileForMsvc(self, DepList):\r
        """ Generate dependency files, .deps file from /showIncludes output message """\r
        if not DepList:\r
            return\r
        ModuleDepDict = {}\r
        current_source = ""\r
        SourceFileAbsPathMap = self.SourceFileList\r
        for line in DepList:\r
            line = line.strip()\r
            if self.HasNamesakeSourceFile:\r
                for cc_cmd in self.CcPPCommandPathSet:\r
                    if cc_cmd in line:\r
                        if '''"'''+cc_cmd+'''"''' in line:\r
                            cc_options = line[len(cc_cmd)+2:].split()\r
                        else:\r
                            cc_options = line[len(cc_cmd):].split()\r
                        SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}\r
            if line in SourceFileAbsPathMap:\r
                current_source = line\r
                if current_source not in ModuleDepDict:\r
                    ModuleDepDict[SourceFileAbsPathMap[current_source]] = []\r
            elif "Note: including file:" ==  line.lstrip()[:21]:\r
                if not current_source:\r
                    EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)\r
                else:\r
                    ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())\r
\r
        for source_abs in ModuleDepDict:\r
            if ModuleDepDict[source_abs]:\r
                target_abs = self.GetRealTarget(source_abs)\r
                dep_file_name = os.path.basename(source_abs) + ".deps"\r
                SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)\r
\r
    def UpdateDepsFileforNonMsvc(self):\r
        """ Update .deps files.\r
            1. Update target path to absolute path.\r
            2. Update middle target to final target.\r
        """\r
\r
        for abspath in self.deps_files:\r
            if abspath.endswith(".trim.deps"):\r
                continue\r
            try:\r
                newcontent = []\r
                with open(abspath,"r") as fd:\r
                    lines = fd.readlines()\r
                if lines[-1] == DEP_FILE_TAIL:\r
                    continue\r
                firstlineitems = lines[0].strip().split(" ")\r
\r
                if len(firstlineitems) > 2:\r
                    sourceitem = firstlineitems[1]\r
                else:\r
                    sourceitem = lines[1].strip().split(" ")[0]\r
\r
                source_abs = self.SourceFileList.get(sourceitem,sourceitem)\r
                firstlineitems[0] = self.GetRealTarget(source_abs)\r
                p_target = firstlineitems\r
                if not p_target[0].strip().endswith(":"):\r
                    p_target[0] += ": "\r
\r
                if len(p_target) == 2:\r
                    p_target[0] += lines[1]\r
                    newcontent.append(p_target[0])\r
                    newcontent.extend(lines[2:])\r
                else:\r
                    line1 = " ".join(p_target).strip()\r
                    line1 += "\n"\r
                    newcontent.append(line1)\r
                    newcontent.extend(lines[1:])\r
\r
                newcontent.append("\n")\r
                newcontent.append(DEP_FILE_TAIL)\r
                with open(abspath,"w") as fw:\r
                    fw.write("".join(newcontent))\r
            except Exception as e:\r
                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)\r
                continue\r
\r
    def UpdateDepsFileforTrim(self):\r
        """ Update .deps file which generated by trim. """\r
\r
        for abspath in self.deps_files:\r
            if not abspath.endswith(".trim.deps"):\r
                continue\r
            try:\r
                newcontent = []\r
                with open(abspath,"r") as fd:\r
                    lines = fd.readlines()\r
                if lines[-1] == DEP_FILE_TAIL:\r
                    continue\r
\r
                source_abs = lines[0].strip().split(" ")[0]\r
                targetitem = self.GetRealTarget(source_abs.strip(" :"))\r
\r
                targetitem += ": "\r
                targetitem += lines[1]\r
                newcontent.append(targetitem)\r
                newcontent.extend(lines[2:])\r
                newcontent.append("\n")\r
                newcontent.append(DEP_FILE_TAIL)\r
                with open(abspath,"w") as fw:\r
                    fw.write("".join(newcontent))\r
            except Exception as e:\r
                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)\r
                continue\r