BaseTools: Fix nmake failure due to command-line length limitation

NMAKE is limited to command-line length of 4096 characters. Due to the
large number of /I directives specified on command line (one per include
directory), the path length of WORKSPACE is multiplied by the number of
/I directives and can exceed the limit.
This patch:
1. Add new build option -l, --cmd-len to set the maximum command line
length, default value is 4096.
2. Generate the response file only if the command line length exceed its
maximum characters (default is 4096) when build the module. Cover
PP_FLAGS, CC_FLAGS, VFRPP_FLAGS, APP_FLAGS, ASLPP_FLAGS, ASLCC_FLAGS and
ASM_FLAGS.
3. The content of the response file is combine from the FLAGS option and
INC option.
4. When build failure, it would print out the response file's file
location and its content.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Yonghong Zhu <yonghong.zhu@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index c7aa84fb342e02e47a5694a0758e35f58ff76754..4934c578fa598b9913911b70d6553e583418816d 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -2378,6 +2378,7 @@ class ModuleAutoGen(AutoGen):
         self._MakeFileDir     = None\r
 \r
         self._IncludePathList = None\r
+        self._IncludePathLength = 0\r
         self._AutoGenFileList = None\r
         self._UnicodeFileList = None\r
         self._SourceFileList  = None\r
@@ -3224,6 +3225,13 @@ class ModuleAutoGen(AutoGen):
                         self._IncludePathList.append(str(Inc))\r
         return self._IncludePathList\r
 \r
+    def _GetIncludePathLength(self):\r
+        self._IncludePathLength = 0\r
+        if self._IncludePathList:\r
+            for inc in self._IncludePathList:\r
+                self._IncludePathLength += len(' ' + inc)\r
+        return self._IncludePathLength\r
+\r
     ## Get HII EX PCDs which maybe used by VFR\r
     #\r
     #  efivarstore used by VFR may relate with HII EX PCDs\r
@@ -3816,6 +3824,7 @@ class ModuleAutoGen(AutoGen):
     CustomMakefile  = property(_GetCustomMakefile)\r
 \r
     IncludePathList = property(_GetIncludePathList)\r
+    IncludePathLength = property(_GetIncludePathLength)\r
     AutoGenFileList = property(_GetAutoGenFileList)\r
     UnicodeFileList = property(_GetUnicodeFileList)\r
     SourceFileList  = property(_GetSourceFileList)\r

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index ec24c70fd3f1850b19e987a1bfe03e638b783888..59ac2e8dda0f217168ce96798d0830fed6e53b43 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1,7 +1,7 @@
 ## @file\r
 # Create makefile for MS nmake and GNU make\r
 #\r
-# Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR>\r
+# Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.<BR>\r
 # This program and the accompanying materials\r
 # are licensed and made available under the terms and conditions of the BSD License\r
 # which accompanies this distribution.  The full text of the license may be found at\r
@@ -497,6 +497,22 @@ cleanlib:
                     ToolsDef.append("%s_%s = %s" % (Tool, Attr, Value))\r
             ToolsDef.append("")\r
 \r
+        # generate the Response file and Response flag\r
+        RespDict = self.CommandExceedLimit()\r
+        RespFileList = os.path.join(self._AutoGenObject.OutputDir, 'respfilelist.txt')\r
+        if RespDict:\r
+            RespFileListContent = ''\r
+            for Resp in RespDict.keys():\r
+                RespFile = os.path.join(self._AutoGenObject.OutputDir, str(Resp).lower() + '.txt')\r
+                SaveFileOnChange(RespFile, RespDict[Resp], False)\r
+                ToolsDef.append("%s = %s" % (Resp, '@' + RespFile))\r
+                RespFileListContent += '@' + RespFile + os.linesep\r
+                RespFileListContent += RespDict[Resp] + os.linesep\r
+            SaveFileOnChange(RespFileList, RespFileListContent, False)\r
+        else:\r
+            if os.path.exists(RespFileList):\r
+                os.remove(RespFileList)\r
+\r
         # convert source files and binary files to build targets\r
         self.ResultFileList = [str(T.Target) for T in self._AutoGenObject.CodaTargetList]\r
         if len(self.ResultFileList) == 0 and len(self._AutoGenObject.SourceFileList) <> 0:\r
@@ -620,6 +636,102 @@ cleanlib:
 \r
         return MakefileTemplateDict\r
 \r
+    def CommandExceedLimit(self):\r
+        FlagDict = {\r
+                    'CC'    :  { 'Macro' : '$(CC_FLAGS)',    'Value' : False},\r
+                    'PP'    :  { 'Macro' : '$(PP_FLAGS)',    'Value' : False},\r
+                    'APP'   :  { 'Macro' : '$(APP_FLAGS)',   'Value' : False},\r
+                    'ASLPP' :  { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False},\r
+                    'VFRPP' :  { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False},\r
+                    'ASM'   :  { 'Macro' : '$(ASM_FLAGS)',   'Value' : False},\r
+                    'ASLCC' :  { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False},\r
+                   }\r
+\r
+        RespDict = {}\r
+        FileTypeList = []\r
+        IncPrefix = self._INC_FLAG_[self._AutoGenObject.ToolChainFamily]\r
+\r
+        # base on the source files to decide the file type\r
+        for File in self._AutoGenObject.SourceFileList:\r
+            for type in self._AutoGenObject.FileTypes:\r
+                if File in self._AutoGenObject.FileTypes[type]:\r
+                    if type not in FileTypeList:\r
+                        FileTypeList.append(type)\r
+\r
+        # calculate the command-line length\r
+        if FileTypeList:\r
+            for type in FileTypeList:\r
+                BuildTargets = self._AutoGenObject.BuildRules[type].BuildTargets\r
+                for Target in BuildTargets:\r
+                    CommandList = BuildTargets[Target].Commands\r
+                    for SingleCommand in CommandList:\r
+                        Tool = ''\r
+                        SingleCommandLength = len(SingleCommand)\r
+                        SingleCommandList = SingleCommand.split()\r
+                        if len(SingleCommandList) > 0:\r
+                            for Flag in FlagDict.keys():\r
+                                if '$('+ Flag +')' in SingleCommandList[0]:\r
+                                    Tool = Flag\r
+                                    break\r
+                        if Tool:\r
+                            SingleCommandLength += len(self._AutoGenObject._BuildOption[Tool]['PATH'])\r
+                            for item in SingleCommandList[1:]:\r
+                                if FlagDict[Tool]['Macro'] in item:\r
+                                    Str = self._AutoGenObject._BuildOption[Tool]['FLAGS']\r
+                                    while(Str.find('$(') != -1):\r
+                                        for macro in self._AutoGenObject.Macros.keys():\r
+                                            MacroName = '$('+ macro + ')'\r
+                                            if (Str.find(MacroName) != -1):\r
+                                                Str = Str.replace(MacroName, self._AutoGenObject.Macros[macro])\r
+                                                break\r
+                                        else:\r
+                                            EdkLogger.error("build", AUTOGEN_ERROR, "Not supported macro is found in make command : %s" % Str, ExtraData="[%s]" % str(self._AutoGenObject))\r
+                                    SingleCommandLength += len(Str)\r
+                                elif '$(INC)' in item:\r
+                                    SingleCommandLength += self._AutoGenObject.IncludePathLength + len(IncPrefix) * len(self._AutoGenObject._IncludePathList)\r
+                                elif item.find('$(') != -1:\r
+                                    Str = item\r
+                                    for Option in self._AutoGenObject.BuildOption.keys():\r
+                                        for Attr in self._AutoGenObject.BuildOption[Option]:\r
+                                            if Str.find(Option + '_' + Attr) != -1:\r
+                                                Str = Str.replace('$(' + Option + '_' + Attr + ')', self._AutoGenObject.BuildOption[Option][Attr])\r
+                                    while(Str.find('$(') != -1):\r
+                                        for macro in self._AutoGenObject.Macros.keys():\r
+                                            MacroName = '$('+ macro + ')'\r
+                                            if (Str.find(MacroName) != -1):\r
+                                                Str = Str.replace(MacroName, self._AutoGenObject.Macros[macro])\r
+                                                break\r
+                                        else:\r
+                                            EdkLogger.error("build", AUTOGEN_ERROR, "Not supported macro is found in make command : %s" % Str, ExtraData="[%s]" % str(self._AutoGenObject))\r
+\r
+                                    SingleCommandLength += len(Str)\r
+\r
+                            if SingleCommandLength > GlobalData.gCommandMaxLength:\r
+                                FlagDict[Tool]['Value'] = True\r
+\r
+                # generate the response file content by combine the FLAGS and INC\r
+                for Flag in FlagDict.keys():\r
+                    if FlagDict[Flag]['Value']:\r
+                        Key = Flag + '_RESP'\r
+                        RespMacro = FlagDict[Flag]['Macro'].replace('FLAGS', 'RESP')\r
+                        Value = self._AutoGenObject.BuildOption[Flag]['FLAGS']\r
+                        for inc in self._AutoGenObject._IncludePathList:\r
+                            Value += ' ' + IncPrefix + inc\r
+                        while (Value.find('$(') != -1):\r
+                            for macro in self._AutoGenObject.Macros.keys():\r
+                                MacroName = '$('+ macro + ')'\r
+                                if (Value.find(MacroName) != -1):\r
+                                    Value = Value.replace(MacroName, self._AutoGenObject.Macros[macro])\r
+                                    break\r
+                            else:\r
+                                EdkLogger.error("build", AUTOGEN_ERROR, "Not supported macro is found in make command : %s" % Str, ExtraData="[%s]" % str(self._AutoGenObject))\r
+                        RespDict[Key] = Value\r
+                        for Target in BuildTargets:\r
+                            for i, SingleCommand in enumerate(BuildTargets[Target].Commands):\r
+                                if FlagDict[Flag]['Macro'] in SingleCommand:\r
+                                    BuildTargets[Target].Commands[i] = SingleCommand.replace('$(INC)','').replace(FlagDict[Flag]['Macro'], RespMacro)\r
+        return RespDict\r
+\r
     def ProcessBuildTargetList(self):\r
         #\r
         # Search dependency file list for each source file\r

diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py
index 4234bf5f955ff9154a6305e7f7e941ed77349a25..8f544bd1b2f5f59ceee02a4970bb644443b4ab1f 100644 (file)
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -34,7 +34,7 @@ gActivePlatform = None
 gCommandLineDefines = {}\r
 gEdkGlobal = {}\r
 gOverrideDir = {}\r
-\r
+gCommandMaxLength = 4096\r
 # for debug trace purpose when problem occurs\r
 gProcessingFile = ''\r
 gBuildingModule = ''\r

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 5619638bd846ff56427bffc83530a81adefe3db4..5c1fda1a937d6dc10ff9763989fa364ae20f5a91 100644 (file)
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -304,6 +304,14 @@ def LaunchCommand(Command, WorkingDir):
     if Proc.returncode != 0:\r
         if type(Command) != type(""):\r
             Command = " ".join(Command)\r
+        # print out the Response file and its content when make failure\r
+        RespFile = os.path.join(WorkingDir, 'OUTPUT', 'respfilelist.txt')\r
+        if os.path.isfile(RespFile):\r
+            f = open(RespFile)\r
+            RespContent = f.read()\r
+            f.close()\r
+            EdkLogger.info(RespContent)\r
+\r
         EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))\r
 \r
 ## The smallest unit that can be built in multi-thread build mode\r
@@ -775,6 +783,9 @@ class Build():
         self.UniFlag        = BuildOptions.Flag\r
         self.BuildModules = []\r
 \r
+        if BuildOptions.CommandLength:\r
+            GlobalData.gCommandMaxLength = BuildOptions.CommandLength\r
+\r
         # print dot character during doing some time-consuming work\r
         self.Progress = Utils.Progressor()\r
 \r
@@ -1931,6 +1942,7 @@ def MyOptionParser():
     Parser.add_option("--check-usage", action="store_true", dest="CheckUsage", default=False, help="Check usage content of entries listed in INF file.")\r
     Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files")\r
     Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: 'PcdName=Value' ")\r
+    Parser.add_option("-l", "--cmd-len", action="store", type="int", dest="CommandLength", help="Specify the maximum line length of build command. Default is 4096.")\r
 \r
     (Opt, Args) = Parser.parse_args()\r
     return (Opt, Args)\r