BaseTools: Library hashing fix and optimization for --hash feature

[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / AutoGen.py

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index c174b5a0bb8c8c523fd0985a7b22548fd3aae1ab..a843f294b94a5e9d643147fc008ee42789583d4d 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -3935,6 +3935,7 @@ class ModuleAutoGen(AutoGen):
             f = open(HashFile, 'r')\r
             CacheHash = f.read()\r
             f.close()\r
             f = open(HashFile, 'r')\r
             CacheHash = f.read()\r
             f.close()\r

+            self.GenModuleHash()\r

             if GlobalData.gModuleHash[self.Arch][self.Name]:\r
                 if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]:\r
                     for root, dir, files in os.walk(FileDir):\r
             if GlobalData.gModuleHash[self.Arch][self.Name]:\r
                 if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]:\r
                     for root, dir, files in os.walk(FileDir):\r


@@ -4093,13 +4094,20 @@ class ModuleAutoGen(AutoGen):
         return RetVal\r
 \r
     def GenModuleHash(self):\r
         return RetVal\r
 \r
     def GenModuleHash(self):\r

+        # Initialize a dictionary for each arch type\r

         if self.Arch not in GlobalData.gModuleHash:\r
             GlobalData.gModuleHash[self.Arch] = {}\r
         if self.Arch not in GlobalData.gModuleHash:\r
             GlobalData.gModuleHash[self.Arch] = {}\r

-        if self.Name in GlobalData.gModuleHash[self.Arch] and GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():\r
-            return False\r
+\r
+        # Early exit if module or library has been hashed and is in memory\r
+        if self.Name in GlobalData.gModuleHash[self.Arch]:\r
+            return GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8')\r
+\r
+        # Initialze hash object\r

         m = hashlib.md5()\r
         m = hashlib.md5()\r

+\r

         # Add Platform level hash\r
         m.update(GlobalData.gPlatformHash.encode('utf-8'))\r
         # Add Platform level hash\r
         m.update(GlobalData.gPlatformHash.encode('utf-8'))\r

+\r

         # Add Package level hash\r
         if self.DependentPackageList:\r
             for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName):\r
         # Add Package level hash\r
         if self.DependentPackageList:\r
             for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName):\r


@@ -4118,6 +4126,7 @@ class ModuleAutoGen(AutoGen):
         Content = f.read()\r
         f.close()\r
         m.update(Content)\r
         Content = f.read()\r
         f.close()\r
         m.update(Content)\r

+\r

         # Add Module's source files\r
         if self.SourceFileList:\r
             for File in sorted(self.SourceFileList, key=lambda x: str(x)):\r
         # Add Module's source files\r
         if self.SourceFileList:\r
             for File in sorted(self.SourceFileList, key=lambda x: str(x)):\r


@@ -4126,27 +4135,45 @@ class ModuleAutoGen(AutoGen):
                 f.close()\r
                 m.update(Content)\r
 \r
                 f.close()\r
                 m.update(Content)\r
 \r

-        ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash")\r
-        if self.Name not in GlobalData.gModuleHash[self.Arch]:\r
-            GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()\r
-        if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():\r
-            return False\r
-        return SaveFileOnChange(ModuleHashFile, m.hexdigest(), False)\r
+        GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()\r
+\r
+        return GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8')\r

 \r
     ## Decide whether we can skip the ModuleAutoGen process\r
     def CanSkipbyHash(self):\r
 \r
     ## Decide whether we can skip the ModuleAutoGen process\r
     def CanSkipbyHash(self):\r

+        # Hashing feature is off\r
+        if not GlobalData.gUseHashCache:\r
+            return False\r
+\r
+        # Initialize a dictionary for each arch type\r
+        if self.Arch not in GlobalData.gBuildHashSkipTracking:\r
+            GlobalData.gBuildHashSkipTracking[self.Arch] = dict()\r
+\r

         # If library or Module is binary do not skip by hash\r
         if self.IsBinaryModule:\r
             return False\r
         # If library or Module is binary do not skip by hash\r
         if self.IsBinaryModule:\r
             return False\r

+\r

         # .inc is contains binary information so do not skip by hash as well\r
         for f_ext in self.SourceFileList:\r
             if '.inc' in str(f_ext):\r
                 return False\r
         # .inc is contains binary information so do not skip by hash as well\r
         for f_ext in self.SourceFileList:\r
             if '.inc' in str(f_ext):\r
                 return False\r

-        if GlobalData.gUseHashCache:\r
-            # If there is a valid hash or function generated a valid hash; function will return False\r
-            # and the statement below will return True\r
-            return not self.GenModuleHash()\r
-        return False\r
+\r
+        # Use Cache, if exists and if Module has a copy in cache\r
+        if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():\r
+            return True\r
+\r
+        # Early exit for libraries that haven't yet finished building\r
+        HashFile = path.join(self.BuildDir, self.Name + ".hash")\r
+        if self.IsLibrary and not os.path.exists(HashFile):\r
+            return False\r
+\r
+        # Return a Boolean based on if can skip by hash, either from memory or from IO.\r
+        if self.Name not in GlobalData.gBuildHashSkipTracking[self.Arch]:\r
+            # If hashes are the same, SaveFileOnChange() will return False.\r
+            GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] = not SaveFileOnChange(HashFile, self.GenModuleHash(), True)\r
+            return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name]\r
+        else:\r
+            return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name]\r

 \r
     ## Decide whether we can skip the ModuleAutoGen process\r
     #  If any source file is newer than the module than we cannot skip\r
 \r
     ## Decide whether we can skip the ModuleAutoGen process\r
     #  If any source file is newer than the module than we cannot skip\r