BaseTools: Fix some build and report file issue

1. increment build not skip make file when not change any file
2. report file generate abundant blank line
3. Build encounter Database is locked on some platform, using database
   auto commit
4. Fv BaseAddress must have if set

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yunhua Feng <yunhuax.feng@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index 2b3f93c3bccb3a1d48fd61b3cbe462e680708c1d..46e94d47e44b1b8d76eac4eafc8c20d3292e6b03 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -295,7 +295,7 @@ class WorkspaceAutoGen(AutoGen):
             SkippedArchList = set(self.ArchList).symmetric_difference(set(self.Platform.SupArchList))\r
             EdkLogger.verbose("\nArch [%s] is ignored because the platform supports [%s] only!"\r
                               % (" ".join(SkippedArchList), " ".join(self.Platform.SupArchList)))\r
-        self.ArchList = tuple(ArchList)\r
+        self.ArchList = tuple(sorted(ArchList))\r
 \r
         # Validate build target\r
         if self.BuildTarget not in self.Platform.BuildTargets:\r
@@ -616,17 +616,17 @@ class WorkspaceAutoGen(AutoGen):
         #\r
         content = 'gCommandLineDefines: '\r
         content += str(GlobalData.gCommandLineDefines)\r
-        content += os.linesep\r
+        content += "\n"\r
         content += 'BuildOptionPcd: '\r
         content += str(GlobalData.BuildOptionPcd)\r
-        content += os.linesep\r
+        content += "\n"\r
         content += 'Active Platform: '\r
         content += str(self.Platform)\r
-        content += os.linesep\r
+        content += "\n"\r
         if self.FdfFile:\r
             content += 'Flash Image Definition: '\r
             content += str(self.FdfFile)\r
-            content += os.linesep\r
+            content += "\n"\r
         SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False)\r
 \r
         #\r
@@ -636,7 +636,7 @@ class WorkspaceAutoGen(AutoGen):
         if Pa.PcdTokenNumber:\r
             if Pa.DynamicPcdList:\r
                 for Pcd in Pa.DynamicPcdList:\r
-                    PcdTokenNumber += os.linesep\r
+                    PcdTokenNumber += "\n"\r
                     PcdTokenNumber += str((Pcd.TokenCName, Pcd.TokenSpaceGuidCName))\r
                     PcdTokenNumber += ' : '\r
                     PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])\r
@@ -677,7 +677,7 @@ class WorkspaceAutoGen(AutoGen):
         if not os.path.exists(self.BuildDir):\r
             os.makedirs(self.BuildDir)\r
         with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:\r
-            for f in AllWorkSpaceMetaFiles:\r
+            for f in sorted(AllWorkSpaceMetaFiles):\r
                 print(f, file=file)\r
         return True\r
 \r
@@ -1598,6 +1598,9 @@ class PlatformAutoGen(AutoGen):
         self._DynamicPcdList.extend(list(UnicodePcdArray))\r
         self._DynamicPcdList.extend(list(HiiPcdArray))\r
         self._DynamicPcdList.extend(list(OtherPcdArray))\r
+        #python3.6 set is not ordered at all\r
+        self._DynamicPcdList = sorted(self._DynamicPcdList, key=lambda x:(x.TokenSpaceGuidCName, x.TokenCName))\r
+        self._NonDynamicPcdList = sorted(self._NonDynamicPcdList, key=lambda x: (x.TokenSpaceGuidCName, x.TokenCName))\r
         allskuset = [(SkuName, Sku.SkuId) for pcd in self._DynamicPcdList for (SkuName, Sku) in pcd.SkuInfoList.items()]\r
         for pcd in self._DynamicPcdList:\r
             if len(pcd.SkuInfoList) == 1:\r
@@ -2374,7 +2377,7 @@ class PlatformAutoGen(AutoGen):
                        list(PlatformModuleOptions.keys()) + list(ModuleTypeOptions.keys()) +\r
                        list(self.ToolDefinition.keys()))\r
         BuildOptions = defaultdict(lambda: defaultdict(str))\r
-        for Tool in AllTools:\r
+        for Tool in sorted(AllTools):\r
             for Options in [self.ToolDefinition, ModuleOptions, PlatformOptions, ModuleTypeOptions, PlatformModuleOptions]:\r
                 if Tool not in Options:\r
                     continue\r
@@ -3156,12 +3159,12 @@ class ModuleAutoGen(AutoGen):
     @cached_property\r
     def IntroTargetList(self):\r
         self.Targets\r
-        return self._IntroBuildTargetList\r
+        return sorted(self._IntroBuildTargetList, key=lambda x: str(x.Target))\r
 \r
     @cached_property\r
     def CodaTargetList(self):\r
         self.Targets\r
-        return self._FinalBuildTargetList\r
+        return sorted(self._FinalBuildTargetList, key=lambda x: str(x.Target))\r
 \r
     @cached_property\r
     def FileTypes(self):\r
@@ -3889,7 +3892,7 @@ class ModuleAutoGen(AutoGen):
             if os.path.exists (self.TimeStampPath):\r
                 os.remove (self.TimeStampPath)\r
             with open(self.TimeStampPath, 'w+') as file:\r
-                for f in FileSet:\r
+                for f in sorted(FileSet):\r
                     print(f, file=file)\r
 \r
         # Ignore generating makefile when it is a binary module\r

diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py
index a21880f31777ba5b739e0b989ea189f2ef7b33d4..4db5330e51cbd080fd7f2fbdff5b5b6910108d7b 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/GenC.py
+++ b/BaseTools/Source/Python/AutoGen/GenC.py
@@ -1011,7 +1011,10 @@ def CreateModulePcdCode(Info, AutoGenC, AutoGenH, Pcd):
                     Value = Value[:-1]\r
                 ValueNumber = int (Value, 0)\r
             except:\r
-                EdkLogger.error("build", AUTOGEN_ERROR,\r
+                try:\r
+                    ValueNumber = int(Value.lstrip('0'))\r
+                except:\r
+                    EdkLogger.error("build", AUTOGEN_ERROR,\r
                                 "PCD value is not valid dec or hex number for datum type [%s] of PCD %s.%s" % (Pcd.DatumType, Pcd.TokenSpaceGuidCName, TokenCName),\r
                                 ExtraData="[%s]" % str(Info))\r
             if ValueNumber < 0:\r

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index 0051aaf41fd88d432b3ed71fcead6a9bac327663..2f6fe06764773788e533cb87d021c5ef81755158 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -917,7 +917,7 @@ cleanlib:
         #\r
         # Extract common files list in the dependency files\r
         #\r
-        for File in DepSet:\r
+        for File in sorted(DepSet, key=lambda x: str(x)):\r
             self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))\r
 \r
         for File in FileDependencyDict:\r
@@ -926,11 +926,11 @@ cleanlib:
                 continue\r
             NewDepSet = set(FileDependencyDict[File])\r
             NewDepSet -= DepSet\r
-            FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)\r
+            FileDependencyDict[File] = ["$(COMMON_DEPS)"] + sorted(NewDepSet, key=lambda x: str(x))\r
 \r
         # Convert target description object to target string in makefile\r
         for Type in self._AutoGenObject.Targets:\r
-            for T in self._AutoGenObject.Targets[Type]:\r
+            for T in sorted(self._AutoGenObject.Targets[Type], key=lambda x: str(x)):\r
                 # Generate related macros if needed\r
                 if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros:\r
                     self.FileListMacros[T.FileListMacro] = []\r
@@ -1097,7 +1097,7 @@ cleanlib:
         DependencySet.update(ForceList)\r
         if File in DependencySet:\r
             DependencySet.remove(File)\r
-        DependencyList = list(DependencySet)  # remove duplicate ones\r
+        DependencyList = sorted(DependencySet, key=lambda x: str(x))  # remove duplicate ones\r
 \r
         return DependencyList\r
 \r

diff --git a/BaseTools/Source/Python/AutoGen/GenPcdDb.py b/BaseTools/Source/Python/AutoGen/GenPcdDb.py
index a546ecbbb7a499ca7c92060a5f993695ebdb12cf..68f92ef0637327d6d47515853d573a9a15a199a9 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/GenPcdDb.py
+++ b/BaseTools/Source/Python/AutoGen/GenPcdDb.py
@@ -1348,7 +1348,7 @@ def CreatePcdDatabasePhaseSpecificAutoGen (Platform, DynamicPcdList, Phase):
 \r
                 DbValueList.append(Sku.DefaultValue)\r
 \r
-        Pcd.TokenTypeList = list(set(Pcd.TokenTypeList))\r
+        Pcd.TokenTypeList = sorted(set(Pcd.TokenTypeList))\r
         if Pcd.DatumType == TAB_VOID:\r
             Dict['SIZE_TABLE_CNAME'].append(CName)\r
             Dict['SIZE_TABLE_GUID'].append(TokenSpaceGuid)\r
@@ -1449,7 +1449,7 @@ def CreatePcdDatabasePhaseSpecificAutoGen (Platform, DynamicPcdList, Phase):
             Dict['PCD_CNAME_LENGTH'][GeneratedTokenNumber] = len(CNameBinArray.split(","))\r
 \r
 \r
-        Pcd.TokenTypeList = list(set(Pcd.TokenTypeList))\r
+        Pcd.TokenTypeList = sorted(set(Pcd.TokenTypeList))\r
 \r
         # search the Offset and Table, used by LocalTokenNumberTableOffset\r
         if 'PCD_TYPE_HII' in Pcd.TokenTypeList:\r

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 71ab6da52989d25dd27abd0f2395faf5e01e67a6..370b4bbb49eaa647bff7d4f8e1cad3d05edf40ff 100644 (file)
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -1597,15 +1597,19 @@ def CheckPcdDatum(Type, Value):
             return False, "Invalid value [%s] of type [%s]; must be one of TRUE, True, true, 0x1, 0x01, 1"\\r
                           ", FALSE, False, false, 0x0, 0x00, 0" % (Value, Type)\r
     elif Type in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64]:\r
-        if Value and int(Value, 0) < 0:\r
-            return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)\r
         try:\r
-            Value = int(Value, 0)\r
-            if Value > MAX_VAL_TYPE[Type]:\r
-                return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)\r
+            Val = int(Value, 0)\r
         except:\r
-            return False, "Invalid value [%s] of type [%s];"\\r
-                          " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)\r
+            try:\r
+                Val = int(Value.lstrip('0'))\r
+            except:\r
+                return False, "Invalid value [%s] of type [%s];" \\r
+                              " must be a hexadecimal, decimal or octal in C language format." % (Value, Type)\r
+        if Val > MAX_VAL_TYPE[Type]:\r
+            return False, "Too large PCD value[%s] for datum type [%s]" % (Value, Type)\r
+        if Val < 0:\r
+            return False, "PCD can't be set to negative value[%s] for datum type [%s]" % (Value, Type)\r
+\r
     else:\r
         return True, "StructurePcd"\r
 \r

diff --git a/BaseTools/Source/Python/Common/StringUtils.py b/BaseTools/Source/Python/Common/StringUtils.py
index fe899b11d87f0e70f90707d21f7eb5de2e0209ac..794f4573fec2e5ea20db0d30f81b27e4466cd9ab 100644 (file)
--- a/BaseTools/Source/Python/Common/StringUtils.py
+++ b/BaseTools/Source/Python/Common/StringUtils.py
@@ -612,7 +612,7 @@ def PreCheck(FileName, FileContent, SupSectionTag):
         #\r
         # Regenerate FileContent\r
         #\r
-        NewFileContent = NewFileContent + Line + '\r\n'\r
+        NewFileContent = NewFileContent + Line + '\n'\r
 \r
     if IsFailed:\r
        EdkLogger.error("Parser", FORMAT_INVALID, Line=LineNo, File=FileName, RaiseError=EdkLogger.IsRaiseError)\r

diff --git a/BaseTools/Source/Python/GenFds/FvImageSection.py b/BaseTools/Source/Python/GenFds/FvImageSection.py
index 5cc5b969fd52c3add6d9bba7842932792cda8bf4..f15df235146ce7550a861407ea66094884ae94fb 100644 (file)
--- a/BaseTools/Source/Python/GenFds/FvImageSection.py
+++ b/BaseTools/Source/Python/GenFds/FvImageSection.py
@@ -102,6 +102,8 @@ class FvImageSection(FvImageSectionClassObject):
             Fv = GenFdsGlobalVariable.FdfParser.Profile.FvDict.get(self.FvName)\r
             if Fv is not None:\r
                 self.Fv = Fv\r
+                if not self.FvAddr and self.Fv.BaseAddress:\r
+                    self.FvAddr = self.Fv.BaseAddress\r
                 FvFileName = Fv.AddToBuffer(Buffer, self.FvAddr, MacroDict = Dict, Flag=IsMakefile)\r
                 if Fv.FvAlignment is not None:\r
                     if self.Alignment is None:\r

diff --git a/BaseTools/Source/Python/GenFds/GenFds.py b/BaseTools/Source/Python/GenFds/GenFds.py
index ed1bd33fdbb3246c23f221aee88d53d421906b42..32824ae80ec365e0a940fd411e12b6723ad2fe60 100644 (file)
--- a/BaseTools/Source/Python/GenFds/GenFds.py
+++ b/BaseTools/Source/Python/GenFds/GenFds.py
@@ -321,6 +321,8 @@ def main():
                             continue\r
                         for RegionData in RegionObj.RegionDataList:\r
                             if FvObj.UiFvName.upper() == RegionData.upper():\r
+                                if not FvObj.BaseAddress:\r
+                                    FvObj.BaseAddress = '0x%x' % (int(FdObj.BaseAddress, 0) + RegionObj.Offset)\r
                                 if FvObj.FvRegionInFD:\r
                                     if FvObj.FvRegionInFD != RegionObj.Size:\r
                                         EdkLogger.error("GenFds", FORMAT_INVALID, "The FV %s's region is specified in multiple FD with different value." %FvObj.UiFvName)\r
@@ -607,16 +609,21 @@ class GenFds :
         ModuleList = []\r
         FileGuidList = []\r
         GuidPattern = gGuidPattern\r
+        VariableGuidSet = set()\r
         for Arch in ArchList:\r
             PlatformDataBase = BuildDb.BuildObject[GenFdsGlobalVariable.ActivePlatform, Arch, GenFdsGlobalVariable.TargetName, GenFdsGlobalVariable.ToolChainTag]\r
             PkgList = GenFdsGlobalVariable.WorkSpace.GetPackageList(GenFdsGlobalVariable.ActivePlatform, Arch, GenFdsGlobalVariable.TargetName, GenFdsGlobalVariable.ToolChainTag)\r
             for P in PkgList:\r
                 PkgGuidDict.update(P.Guids)\r
-            for Name, Guid in PlatformDataBase.Pcds:\r
+            for Name, Guid in sorted(PlatformDataBase.Pcds):\r
                 Pcd = PlatformDataBase.Pcds[Name, Guid]\r
                 if Pcd.Type in [TAB_PCDS_DYNAMIC_HII, TAB_PCDS_DYNAMIC_EX_HII]:\r
                     for SkuId in Pcd.SkuInfoList:\r
                         Sku = Pcd.SkuInfoList[SkuId]\r
+                        if Sku.VariableGuid in VariableGuidSet:\r
+                            continue\r
+                        else:\r
+                            VariableGuidSet.add(Sku.VariableGuid)\r
                         if Sku.VariableGuid and Sku.VariableGuid in PkgGuidDict.keys():\r
                             GuidDict[Sku.VariableGuid] = PkgGuidDict[Sku.VariableGuid]\r
             for ModuleFile in PlatformDataBase.Modules:\r

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index c717401cab5813a2e773d13a1188e17b88282909..02aae3a67ba4c7e3f1710caafe7accf2eb321f24 100644 (file)
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -128,7 +128,7 @@ def GetDependencyList(FileStack, SearchPathList):
 \r
             if len(FileContent) == 0:\r
                 continue\r
-\r
+            IncludedFileList = []\r
             if FileContent[0] == 0xff or FileContent[0] == 0xfe:\r
                 FileContent = str(FileContent, "utf-16")\r
                 IncludedFileList = gIncludePattern.findall(FileContent)\r

diff --git a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
index 50ffe308aabb9ba6f9d08e35fa257ae35c8e74f4..fdf7738a3105be2afe769e925f68349778d5c6b1 100644 (file)
--- a/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
+++ b/BaseTools/Source/Python/Workspace/WorkspaceDatabase.py
@@ -164,7 +164,7 @@ class WorkspaceDatabase(object):
                 os.remove(DbPath)\r
 \r
         # create db with optimized parameters\r
-        self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')\r
+        self.Conn = sqlite3.connect(DbPath, isolation_level=None)\r
         self.Conn.execute("PRAGMA synchronous=OFF")\r
         self.Conn.execute("PRAGMA temp_store=MEMORY")\r
         self.Conn.execute("PRAGMA count_changes=OFF")\r

diff --git a/BaseTools/Source/Python/build/BuildReport.py b/BaseTools/Source/Python/build/BuildReport.py
index fd9294287fd1d8393d507e51a5fbb5cf9d48c984..06cf419931d116f7216ea55dfc92193a3eca143a 100644 (file)
--- a/BaseTools/Source/Python/build/BuildReport.py
+++ b/BaseTools/Source/Python/build/BuildReport.py
@@ -79,7 +79,7 @@ gGlueLibEntryPoint = re.compile(r"__EDKII_GLUE_MODULE_ENTRY_POINT__\s*=\s*(\w+)"
 gLineMaxLength = 120\r
 \r
 ## Tags for end of line in report\r
-gEndOfLine = "\r\n"\r
+gEndOfLine = "\n"\r
 \r
 ## Tags for section start, end and separator\r
 gSectionStart = ">" + "=" * (gLineMaxLength - 2) + "<"\r
@@ -1031,7 +1031,10 @@ class PcdReport(object):
 \r
 \r
                 if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES:\r
-                    PcdValueNumber = int(PcdValue.strip(), 0)\r
+                    try:\r
+                        PcdValueNumber = int(PcdValue.strip(), 0)\r
+                    except:\r
+                        PcdValueNumber = int(PcdValue.lstrip('0'))\r
                     if DecDefaultValue is None:\r
                         DecMatch = True\r
                     else:\r
@@ -1047,7 +1050,10 @@ class PcdReport(object):
                     if DscDefaultValue is None:\r
                         DscMatch = True\r
                     else:\r
-                        DscDefaultValueNumber = int(DscDefaultValue.strip(), 0)\r
+                        try:\r
+                            DscDefaultValueNumber = int(DscDefaultValue.strip(), 0)\r
+                        except:\r
+                            DscDefaultValueNumber = int(DscDefaultValue.lstrip('0'))\r
                         DscMatch = (DscDefaultValueNumber == PcdValueNumber)\r
                 else:\r
                     if DecDefaultValue is None:\r
@@ -1152,7 +1158,10 @@ class PcdReport(object):
                         for ModulePath in ModuleOverride:\r
                             ModuleDefault = ModuleOverride[ModulePath]\r
                             if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES:\r
-                                ModulePcdDefaultValueNumber = int(ModuleDefault.strip(), 0)\r
+                                try:\r
+                                    ModulePcdDefaultValueNumber = int(ModuleDefault.strip(), 0)\r
+                                except:\r
+                                    ModulePcdDefaultValueNumber = int(ModuleDefault.lstrip('0'))\r
                                 Match = (ModulePcdDefaultValueNumber == PcdValueNumber)\r
                                 if Pcd.DatumType == 'BOOLEAN':\r
                                     ModuleDefault = str(ModulePcdDefaultValueNumber)\r
@@ -1231,7 +1240,10 @@ class PcdReport(object):
                     if Value.startswith(('0x', '0X')):\r
                         Value = '{} ({:d})'.format(Value, int(Value, 0))\r
                     else:\r
-                        Value = "0x{:X} ({})".format(int(Value, 0), Value)\r
+                        try:\r
+                            Value = "0x{:X} ({})".format(int(Value, 0), Value)\r
+                        except:\r
+                            Value = "0x{:X} ({})".format(int(Value.lstrip('0')), Value)\r
                 FileWrite(File, '    %*s = %s' % (self.MaxLen + 19, 'DEC DEFAULT', Value))\r
             if IsStructure:\r
                 self.PrintStructureInfo(File, Pcd.DefaultValues)\r