]> git.proxmox.com Git - mirror_edk2.git/commitdiff
BaseTools: Enhance Basetool for incremental build
authorBob Feng <bob.c.feng@intel.com>
Wed, 20 Nov 2019 02:58:30 +0000 (10:58 +0800)
committermergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tue, 10 Dec 2019 01:31:55 +0000 (01:31 +0000)
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Include dependency file in Makefile to enhance
incremental build

Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
BaseTools/Source/Python/AutoGen/GenMake.py
BaseTools/Source/Python/AutoGen/IncludesAutoGen.py [new file with mode: 0644]
BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
BaseTools/Source/Python/build/build.py

index 59a01a7f243ac0a1644ecee0d58eb42df4c6dbb6..fe94f9a4c232bb599a59563444c3985700c78ec6 100755 (executable)
@@ -185,6 +185,12 @@ class BuildFile(object):
         self._FileType = FileType\r
         FileContent = self._TEMPLATE_.Replace(self._TemplateDict)\r
         FileName = self._FILE_NAME_[FileType]\r
+        if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt")):\r
+            with open(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt"),"w+") as fd:\r
+                fd.write("")\r
+        if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "dependency")):\r
+            with open(os.path.join(self._AutoGenObject.MakeFileDir, "dependency"),"w+") as fd:\r
+                fd.write("")\r
         return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False)\r
 \r
     ## Return a list of directory creation command string\r
@@ -304,9 +310,6 @@ MAKE_FILE = ${makefile_path}
 ${BEGIN}${file_macro}\r
 ${END}\r
 \r
-COMMON_DEPS = ${BEGIN}${common_dependency_file} \\\r
-              ${END}\r
-\r
 #\r
 # Overridable Target Macro Definitions\r
 #\r
@@ -382,6 +385,8 @@ gen_fds:
 \t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds\r
 \t@cd $(MODULE_BUILD_DIR)\r
 \r
+${INCLUDETAG}\r
+\r
 #\r
 # Individual Object Build Targets\r
 #\r
@@ -515,9 +520,6 @@ cleanlib:
                     # Remove duplicated include path, if any\r
                     if Attr == "FLAGS":\r
                         Value = RemoveDupOption(Value, IncPrefix, MyAgo.IncludePathList)\r
-                        if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value:\r
-                            Value = Value.replace(' /MP', '')\r
-                            MyAgo.BuildOption[Tool][Attr] = Value\r
                         if Tool == "OPTROM" and PCI_COMPRESS_Flag:\r
                             ValueList = Value.split()\r
                             if ValueList:\r
@@ -540,7 +542,7 @@ cleanlib:
                 UnexpandMacro = []\r
                 NewStr = []\r
                 for Str in StrList:\r
-                    if '$' in Str:\r
+                    if '$' in Str or '-MMD' in Str or '-MF' in Str:\r
                         UnexpandMacro.append(Str)\r
                     else:\r
                         NewStr.append(Str)\r
@@ -590,10 +592,9 @@ cleanlib:
                                                 )\r
         FileMacroList.append(FileMacro)\r
         # Add support when compiling .nasm source files\r
-        for File in self.FileCache.keys():\r
-            if not str(File).endswith('.nasm'):\r
-                continue\r
-            IncludePathList = []\r
+        IncludePathList = []\r
+        asmsource = [item for item in MyAgo.SourceFileList if item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]\r
+        if asmsource:\r
             for P in  MyAgo.IncludePathList:\r
                 IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, self.Macros)\r
                 if IncludePath.endswith(os.sep):\r
@@ -606,7 +607,6 @@ cleanlib:
                     IncludePath = os.path.join(IncludePath, '')\r
                 IncludePathList.append(IncludePath)\r
             FileMacroList.append(self._FILE_MACRO_TEMPLATE.Replace({"macro_name": "NASM_INC", "source_file": IncludePathList}))\r
-            break\r
 \r
         # Generate macros used to represent files containing list of input files\r
         for ListFileMacro in self.ListFileMacros:\r
@@ -696,6 +696,7 @@ cleanlib:
             "file_macro"                : FileMacroList,\r
             "file_build_target"         : self.BuildTargetList,\r
             "backward_compatible_target": BcTargetList,\r
+            "INCLUDETAG"                   : self._INCLUDE_CMD_[self._FileType] + " " + os.path.join("$(MODULE_BUILD_DIR)","dependency")\r
         }\r
 \r
         return MakefileTemplateDict\r
@@ -903,16 +904,10 @@ cleanlib:
                 if Item in SourceFileList:\r
                     SourceFileList.remove(Item)\r
 \r
-        FileDependencyDict = self.GetFileDependency(\r
-                                    SourceFileList,\r
-                                    ForceIncludedFile,\r
-                                    self._AutoGenObject.IncludePathList + self._AutoGenObject.BuildOptionIncPathList\r
-                                    )\r
-\r
+        FileDependencyDict = {item:ForceIncludedFile for item in SourceFileList}\r
 \r
-        if FileDependencyDict:\r
-            for Dependency in FileDependencyDict.values():\r
-                self.DependencyHeaderFileSet.update(set(Dependency))\r
+        for Dependency in FileDependencyDict.values():\r
+            self.DependencyHeaderFileSet.update(set(Dependency))\r
 \r
         # Get a set of unique package includes from MetaFile\r
         parentMetaFileIncludes = set()\r
@@ -972,42 +967,16 @@ cleanlib:
                         ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path\r
                         )\r
 \r
-        DepSet = None\r
         for File,Dependency in FileDependencyDict.items():\r
             if not Dependency:\r
-                FileDependencyDict[File] = ['$(FORCE_REBUILD)']\r
                 continue\r
 \r
             self._AutoGenObject.AutoGenDepSet |= set(Dependency)\r
 \r
-            # skip non-C files\r
-            if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":\r
-                continue\r
-            elif DepSet is None:\r
-                DepSet = set(Dependency)\r
-            else:\r
-                DepSet &= set(Dependency)\r
-        # in case nothing in SourceFileList\r
-        if DepSet is None:\r
-            DepSet = set()\r
-        #\r
-        # Extract common files list in the dependency files\r
-        #\r
-        for File in DepSet:\r
-            self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))\r
-\r
         CmdSumDict = {}\r
         CmdTargetDict = {}\r
         CmdCppDict = {}\r
         DependencyDict = FileDependencyDict.copy()\r
-        for File in FileDependencyDict:\r
-            # skip non-C files\r
-            if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":\r
-                continue\r
-            NewDepSet = set(FileDependencyDict[File])\r
-            NewDepSet -= DepSet\r
-            FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)\r
-            DependencyDict[File] = list(NewDepSet)\r
 \r
         # Convert target description object to target string in makefile\r
         if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and TAB_C_CODE_FILE in self._AutoGenObject.Targets:\r
@@ -1080,17 +1049,13 @@ cleanlib:
                     else:\r
                         CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]\r
                     if CppPath.Path in DependencyDict:\r
-                        if '$(FORCE_REBUILD)' in DependencyDict[CppPath.Path]:\r
-                            if '$(FORCE_REBUILD)' not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):\r
-                                CmdCppDict[item.Target.SubDir].append('$(FORCE_REBUILD)')\r
-                        else:\r
-                            for Temp in DependencyDict[CppPath.Path]:\r
-                                try:\r
-                                    Path = self.PlaceMacro(Temp.Path, self.Macros)\r
-                                except:\r
-                                    continue\r
-                                if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):\r
-                                    CmdCppDict[item.Target.SubDir].append(Path)\r
+                        for Temp in DependencyDict[CppPath.Path]:\r
+                            try:\r
+                                Path = self.PlaceMacro(Temp.Path, self.Macros)\r
+                            except:\r
+                                continue\r
+                            if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):\r
+                                CmdCppDict[item.Target.SubDir].append(Path)\r
         if T.Commands:\r
             CommandList = T.Commands[:]\r
             for Item in CommandList[:]:\r
@@ -1109,7 +1074,7 @@ cleanlib:
                     CommandList.pop(Index)\r
                     if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])):\r
                         Cpplist = CmdCppDict[T.Target.SubDir]\r
-                        Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))\r
+                        Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))\r
                         T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])\r
                     else:\r
                         T.Commands.pop(Index)\r
diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
new file mode 100644 (file)
index 0000000..bb6e883
--- /dev/null
@@ -0,0 +1,284 @@
+## @file\r
+# Build cache intermediate result and state\r
+#\r
+# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>\r
+# SPDX-License-Identifier: BSD-2-Clause-Patent\r
+#\r
+from Common.caching import cached_property\r
+import Common.EdkLogger as EdkLogger\r
+import Common.LongFilePathOs as os\r
+from Common.BuildToolError import *\r
+from Common.Misc import SaveFileOnChange, PathClass\r
+from Common.Misc import TemplateString\r
+import sys\r
+gIsFileMap = {}\r
+if sys.platform == "win32":\r
+    _INCLUDE_DEPS_TEMPLATE = TemplateString('''\r
+${BEGIN}\r
+!IF EXIST(${deps_file})\r
+!INCLUDE ${deps_file}\r
+!ENDIF\r
+${END}\r
+        ''')\r
+else:\r
+    _INCLUDE_DEPS_TEMPLATE = TemplateString('''\r
+${BEGIN}\r
+-include ${deps_file}\r
+${END}\r
+        ''')\r
+\r
+DEP_FILE_TAIL = "# Updated \n"\r
+\r
+class IncludesAutoGen():\r
+    """ This class is to manage the dependent files witch are used in Makefile to support incremental build.\r
+        1. C files:\r
+            1. MSVS.\r
+               cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures\r
+               that messages and generate dependency files, .deps files.\r
+            2. CLANG and GCC\r
+               -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the\r
+               .deps files.\r
+        2. ASL files:\r
+            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.\r
+            2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file\r
+            3. build tool updates the .deps file\r
+        3. ASM files (.asm, .s or .nasm):\r
+            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.\r
+            2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file\r
+            3. build tool updates the .deps file\r
+    """\r
+    def __init__(self, makefile_folder, ModuleAuto):\r
+        self.d_folder = makefile_folder\r
+        self.makefile_folder = makefile_folder\r
+        self.module_autogen = ModuleAuto\r
+        self.ToolChainFamily = ModuleAuto.ToolChainFamily\r
+        self.workspace = ModuleAuto.WorkspaceDir\r
+\r
+    def CreateModuleDeps(self):\r
+        SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)\r
+\r
+    def CreateDepsInclude(self):\r
+        deps_file = {'deps_file':self.deps_files}\r
+        try:\r
+            deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)\r
+        except Exception as e:\r
+            print(e)\r
+        SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)\r
+\r
+    @cached_property\r
+    def deps_files(self):\r
+        """ Get all .deps file under module build folder. """\r
+        deps_files = []\r
+        for root, _, files in os.walk(self.d_folder, topdown=False):\r
+            for name in files:\r
+                if not name.endswith(".deps"):\r
+                    continue\r
+                abspath = os.path.join(root, name)\r
+                deps_files.append(abspath)\r
+        return deps_files\r
+\r
+    @cached_property\r
+    def DepsCollection(self):\r
+        """ Collect all the dependency files list from all .deps files under a module's build folder """\r
+        includes = set()\r
+        targetname = [item[0].Name for item in self.TargetFileList.values()]\r
+        for abspath in self.deps_files:\r
+            try:\r
+                with open(abspath,"r") as fd:\r
+                    lines = fd.readlines()\r
+\r
+                firstlineitems = lines[0].split(": ")\r
+                dependency_file = firstlineitems[1].strip(" \\\n")\r
+                dependency_file = dependency_file.strip('''"''')\r
+                if dependency_file:\r
+                    if os.path.normpath(dependency_file +".deps") == abspath:\r
+                        continue\r
+                    filename = os.path.basename(dependency_file).strip()\r
+                    if filename not in self.SourceFileList and filename not in targetname:\r
+                        includes.add(dependency_file.strip())\r
+\r
+                for item in lines[1:]:\r
+                    if item == DEP_FILE_TAIL:\r
+                        continue\r
+                    dependency_file = item.strip(" \\\n")\r
+                    dependency_file = dependency_file.strip('''"''')\r
+                    if os.path.normpath(dependency_file +".deps") == abspath:\r
+                        continue\r
+                    filename = os.path.basename(dependency_file).strip()\r
+                    if filename in self.SourceFileList:\r
+                        continue\r
+                    if filename in targetname:\r
+                        continue\r
+                    includes.add(dependency_file.strip())\r
+            except Exception as e:\r
+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)\r
+                continue\r
+        rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))\r
+        return rt\r
+\r
+    @cached_property\r
+    def SourceFileList(self):\r
+        """ Get a map of module's source files name to module's source files path """\r
+        source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}\r
+        middle_file = {}\r
+        for afile in source:\r
+            if afile.upper().endswith(".VFR"):\r
+                middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})\r
+            if afile.upper().endswith((".S","ASM")):\r
+                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})\r
+            if afile.upper().endswith(".ASL"):\r
+                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})\r
+        source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})\r
+        source.update(middle_file)\r
+        return source\r
+\r
+    @cached_property\r
+    def HasNamesakeSourceFile(self):\r
+        source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])\r
+        rt = len(source_base_name) != len(self.module_autogen.SourceFileList)\r
+        return rt\r
+    @cached_property\r
+    def CcPPCommandPathSet(self):\r
+        rt = set()\r
+        rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))\r
+        rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))\r
+        rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))\r
+        rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))\r
+        rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))\r
+        rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))\r
+        rt.discard(None)\r
+        return rt\r
+    @cached_property\r
+    def TargetFileList(self):\r
+        """ Get a map of module's target name to a tuple of module's targets path and whose input file path """\r
+        targets = {}\r
+        targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))\r
+        for item in self.module_autogen.Targets.values():\r
+            for block in item:\r
+                targets[block.Target.Path] = (block.Target,block.Inputs[0])\r
+        return targets\r
+\r
+    def GetRealTarget(self,source_file_abs):\r
+        """ Get the final target file based on source file abspath """\r
+        source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}\r
+        source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}\r
+        target_abs = source_target_map.get(source_file_abs)\r
+        if target_abs is None:\r
+            if source_file_abs.strip().endswith(".i"):\r
+                sourcefilename = os.path.basename(source_file_abs.strip())\r
+                for sourcefile in source_name_map:\r
+                    if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:\r
+                        target_abs = source_name_map[sourcefile]\r
+                        break\r
+                else:\r
+                    target_abs = source_file_abs\r
+            else:\r
+                target_abs = source_file_abs\r
+        return target_abs\r
+\r
+    def CreateDepsFileForMsvc(self, DepList):\r
+        """ Generate dependency files, .deps file from /showIncludes output message """\r
+        if not DepList:\r
+            return\r
+        ModuleDepDict = {}\r
+        current_source = ""\r
+        SourceFileAbsPathMap = self.SourceFileList\r
+        for line in DepList:\r
+            line = line.strip()\r
+            if self.HasNamesakeSourceFile:\r
+                for cc_cmd in self.CcPPCommandPathSet:\r
+                    if cc_cmd in line:\r
+                        if '''"'''+cc_cmd+'''"''' in line:\r
+                            cc_options = line[len(cc_cmd)+2:].split()\r
+                        else:\r
+                            cc_options = line[len(cc_cmd):].split()\r
+                        SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}\r
+            if line in SourceFileAbsPathMap:\r
+                current_source = line\r
+                if current_source not in ModuleDepDict:\r
+                    ModuleDepDict[SourceFileAbsPathMap[current_source]] = []\r
+            elif "Note: including file:" ==  line.lstrip()[:21]:\r
+                if not current_source:\r
+                    EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)\r
+                else:\r
+                    ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())\r
+\r
+        for source_abs in ModuleDepDict:\r
+            if ModuleDepDict[source_abs]:\r
+                target_abs = self.GetRealTarget(source_abs)\r
+                dep_file_name = os.path.basename(source_abs) + ".deps"\r
+                SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)\r
+\r
+    def UpdateDepsFileforNonMsvc(self):\r
+        """ Update .deps files.\r
+            1. Update target path to absolute path.\r
+            2. Update middle target to final target.\r
+        """\r
+\r
+        for abspath in self.deps_files:\r
+            if abspath.endswith(".trim.deps"):\r
+                continue\r
+            try:\r
+                newcontent = []\r
+                with open(abspath,"r") as fd:\r
+                    lines = fd.readlines()\r
+                if lines[-1] == DEP_FILE_TAIL:\r
+                    continue\r
+                firstlineitems = lines[0].strip().split(" ")\r
+\r
+                if len(firstlineitems) > 2:\r
+                    sourceitem = firstlineitems[1]\r
+                else:\r
+                    sourceitem = lines[1].strip().split(" ")[0]\r
+\r
+                source_abs = self.SourceFileList.get(sourceitem,sourceitem)\r
+                firstlineitems[0] = self.GetRealTarget(source_abs)\r
+                p_target = firstlineitems\r
+                if not p_target[0].strip().endswith(":"):\r
+                    p_target[0] += ": "\r
+\r
+                if len(p_target) == 2:\r
+                    p_target[0] += lines[1]\r
+                    newcontent.append(p_target[0])\r
+                    newcontent.extend(lines[2:])\r
+                else:\r
+                    line1 = " ".join(p_target).strip()\r
+                    line1 += "\n"\r
+                    newcontent.append(line1)\r
+                    newcontent.extend(lines[1:])\r
+\r
+                newcontent.append("\n")\r
+                newcontent.append(DEP_FILE_TAIL)\r
+                with open(abspath,"w") as fw:\r
+                    fw.write("".join(newcontent))\r
+            except Exception as e:\r
+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)\r
+                continue\r
+\r
+    def UpdateDepsFileforTrim(self):\r
+        """ Update .deps file which generated by trim. """\r
+\r
+        for abspath in self.deps_files:\r
+            if not abspath.endswith(".trim.deps"):\r
+                continue\r
+            try:\r
+                newcontent = []\r
+                with open(abspath,"r") as fd:\r
+                    lines = fd.readlines()\r
+                if lines[-1] == DEP_FILE_TAIL:\r
+                    continue\r
+\r
+                source_abs = lines[0].strip().split(" ")[0]\r
+                targetitem = self.GetRealTarget(source_abs.strip(" :"))\r
+\r
+                targetitem += ": "\r
+                targetitem += lines[1]\r
+                newcontent.append(targetitem)\r
+                newcontent.extend(lines[2:])\r
+                newcontent.append("\n")\r
+                newcontent.append(DEP_FILE_TAIL)\r
+                with open(abspath,"w") as fw:\r
+                    fw.write("".join(newcontent))\r
+            except Exception as e:\r
+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)\r
+                continue\r
index e6d6c43810ec5bda0a7df3fc584cf05546a1a272..1111d5de25acc35ba07c9f6232ffb241978caa34 100755 (executable)
@@ -1129,8 +1129,31 @@ class ModuleAutoGen(AutoGen):
             for Inc in IncludesList:\r
                 if Inc not in RetVal:\r
                     RetVal.append(str(Inc))\r
+        RetVal.extend(self.IncPathFromBuildOptions)\r
         return RetVal\r
 \r
+    @cached_property\r
+    def IncPathFromBuildOptions(self):\r
+        IncPathList = []\r
+        for tool in self.BuildOption:\r
+            if 'FLAGS' in self.BuildOption[tool]:\r
+                flags = self.BuildOption[tool]['FLAGS']\r
+                whitespace = False\r
+                for flag in flags.split(" "):\r
+                    flag = flag.strip()\r
+                    if flag.startswith(("/I","-I")):\r
+                        if len(flag)>2:\r
+                            if os.path.exists(flag[2:]):\r
+                                IncPathList.append(flag[2:])\r
+                        else:\r
+                            whitespace = True\r
+                            continue\r
+                    if whitespace and flag:\r
+                        if os.path.exists(flag):\r
+                            IncPathList.append(flag)\r
+                            whitespace = False\r
+        return IncPathList\r
+\r
     @cached_property\r
     def IncludePathLength(self):\r
         return sum(len(inc)+1 for inc in self.IncludePathList)\r
index 4b31356a4253f5ff4abb875417ffbc06a1517e41..6fcfc7ec7ab3be57cf64c43deaeaf59802d79746 100755 (executable)
@@ -57,7 +57,7 @@ from multiprocessing import Manager
 from AutoGen.DataPipe import MemoryDataPipe\r
 from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo\r
 from GenFds.FdfParser import FdfParser\r
-\r
+from AutoGen.IncludesAutoGen import IncludesAutoGen\r
 \r
 ## standard targets of build command\r
 gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']\r
@@ -175,18 +175,31 @@ def NormFile(FilePath, Workspace):
 # @param  To        The stream message put on\r
 # @param  ExitFlag  The flag used to indicate stopping reading\r
 #\r
-def ReadMessage(From, To, ExitFlag):\r
+def ReadMessage(From, To, ExitFlag,MemTo=None):\r
     while True:\r
         # read one line a time\r
         Line = From.readline()\r
         # empty string means "end"\r
         if Line is not None and Line != b"":\r
-            To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))\r
+            LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')\r
+            if MemTo is not None:\r
+                if "Note: including file:" ==  LineStr.lstrip()[:21]:\r
+                    MemTo.append(LineStr)\r
+                else:\r
+                    To(LineStr)\r
+                    MemTo.append(LineStr)\r
+            else:\r
+                To(LineStr)\r
         else:\r
             break\r
         if ExitFlag.isSet():\r
             break\r
 \r
+class MakeSubProc(Popen):\r
+    def __init__(self,*args, **argv):\r
+        super(MakeSubProc,self).__init__(*args, **argv)\r
+        self.ProcOut = []\r
+\r
 ## Launch an external program\r
 #\r
 # This method will call subprocess.Popen to execute an external program with\r
@@ -197,7 +210,7 @@ def ReadMessage(From, To, ExitFlag):
 # @param  Command               A list or string containing the call of the program\r
 # @param  WorkingDir            The directory in which the program will be running\r
 #\r
-def LaunchCommand(Command, WorkingDir):\r
+def LaunchCommand(Command, WorkingDir,ModuleAuto = None):\r
     BeginTime = time.time()\r
     # if working directory doesn't exist, Popen() will raise an exception\r
     if not os.path.isdir(WorkingDir):\r
@@ -216,19 +229,19 @@ def LaunchCommand(Command, WorkingDir):
     EndOfProcedure = None\r
     try:\r
         # launch the command\r
-        Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)\r
+        Proc = MakeSubProc(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)\r
 \r
         # launch two threads to read the STDOUT and STDERR\r
         EndOfProcedure = Event()\r
         EndOfProcedure.clear()\r
         if Proc.stdout:\r
-            StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))\r
+            StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))\r
             StdOutThread.setName("STDOUT-Redirector")\r
             StdOutThread.setDaemon(False)\r
             StdOutThread.start()\r
 \r
         if Proc.stderr:\r
-            StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))\r
+            StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure,Proc.ProcOut))\r
             StdErrThread.setName("STDERR-Redirector")\r
             StdErrThread.setDaemon(False)\r
             StdErrThread.start()\r
@@ -263,6 +276,15 @@ def LaunchCommand(Command, WorkingDir):
             EdkLogger.info(RespContent)\r
 \r
         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))\r
+    if ModuleAuto:\r
+        iau = IncludesAutoGen(WorkingDir,ModuleAuto)\r
+        if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:\r
+            iau.CreateDepsFileForMsvc(Proc.ProcOut)\r
+        else:\r
+            iau.UpdateDepsFileforNonMsvc()\r
+        iau.UpdateDepsFileforTrim()\r
+        iau.CreateModuleDeps()\r
+        iau.CreateDepsInclude()\r
     return "%dms" % (int(round((time.time() - BeginTime) * 1000)))\r
 \r
 ## The smallest unit that can be built in multi-thread build mode\r
@@ -608,7 +630,7 @@ class BuildTask:
     #\r
     def _CommandThread(self, Command, WorkingDir):\r
         try:\r
-            self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)\r
+            self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)\r
             self.CompleteFlag = True\r
 \r
             # Run hash operation post dependency, to account for libs\r
@@ -1276,19 +1298,32 @@ class Build():
 \r
         # build library\r
         if Target == 'libraries':\r
-            for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
+            DirList = []\r
+            for Lib in AutoGenObject.LibraryAutoGenList:\r
+                if not Lib.IsBinaryModule:\r
+                    DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))\r
+            for Lib, LibAutoGen in DirList:\r
                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']\r
-                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
+                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)\r
             return True\r
 \r
         # build module\r
         if Target == 'modules':\r
-            for Lib in AutoGenObject.LibraryBuildDirectoryList:\r
+            DirList = []\r
+            for Lib in AutoGenObject.LibraryAutoGenList:\r
+                if not Lib.IsBinaryModule:\r
+                    DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))\r
+            for Lib, LibAutoGen in DirList:\r
                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']\r
-                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
-            for Mod in AutoGenObject.ModuleBuildDirectoryList:\r
+                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)\r
+\r
+            DirList = []\r
+            for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:\r
+                if not ModuleAutoGen.IsBinaryModule:\r
+                    DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))\r
+            for Mod,ModAutoGen in DirList:\r
                 NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']\r
-                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)\r
+                LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)\r
             self.CreateAsBuiltInf()\r
             if GlobalData.gBinCacheDest:\r
                 self.UpdateBuildCache()\r