BaseTools: Library hashing fix and optimization for --hash feature

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1788

In V3: Must generate hashes before attempting to copy from cache for
hash verifcation
In V2: Build failure caused by passing incorrect boolean parameter to
SaveFileOnChange(). Fixed for patch instances.

Library hashing is now supported by the --hash feature. The --hash
feature implementation assumed that the hashing could be done in
place once per module, but that isn't true for libraries due to the
fact that they are built as dependencies. So on a clean build, we now
generate the .hash after the library dependencies are complete.
Added early escape as optimization, if hash already exists in memory.

Signed-off-by: Christian Rodriguez <christian.rodriguez@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>

diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index c174b5a0bb8c8c523fd0985a7b22548fd3aae1ab..a843f294b94a5e9d643147fc008ee42789583d4d 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -3935,6 +3935,7 @@ class ModuleAutoGen(AutoGen):
             f = open(HashFile, 'r')\r
             CacheHash = f.read()\r
             f.close()\r
+            self.GenModuleHash()\r
             if GlobalData.gModuleHash[self.Arch][self.Name]:\r
                 if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]:\r
                     for root, dir, files in os.walk(FileDir):\r
@@ -4093,13 +4094,20 @@ class ModuleAutoGen(AutoGen):
         return RetVal\r
 \r
     def GenModuleHash(self):\r
+        # Initialize a dictionary for each arch type\r
         if self.Arch not in GlobalData.gModuleHash:\r
             GlobalData.gModuleHash[self.Arch] = {}\r
-        if self.Name in GlobalData.gModuleHash[self.Arch] and GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():\r
-            return False\r
+\r
+        # Early exit if module or library has been hashed and is in memory\r
+        if self.Name in GlobalData.gModuleHash[self.Arch]:\r
+            return GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8')\r
+\r
+        # Initialze hash object\r
         m = hashlib.md5()\r
+\r
         # Add Platform level hash\r
         m.update(GlobalData.gPlatformHash.encode('utf-8'))\r
+\r
         # Add Package level hash\r
         if self.DependentPackageList:\r
             for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName):\r
@@ -4118,6 +4126,7 @@ class ModuleAutoGen(AutoGen):
         Content = f.read()\r
         f.close()\r
         m.update(Content)\r
+\r
         # Add Module's source files\r
         if self.SourceFileList:\r
             for File in sorted(self.SourceFileList, key=lambda x: str(x)):\r
@@ -4126,27 +4135,45 @@ class ModuleAutoGen(AutoGen):
                 f.close()\r
                 m.update(Content)\r
 \r
-        ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash")\r
-        if self.Name not in GlobalData.gModuleHash[self.Arch]:\r
-            GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()\r
-        if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():\r
-            return False\r
-        return SaveFileOnChange(ModuleHashFile, m.hexdigest(), False)\r
+        GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()\r
+\r
+        return GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8')\r
 \r
     ## Decide whether we can skip the ModuleAutoGen process\r
     def CanSkipbyHash(self):\r
+        # Hashing feature is off\r
+        if not GlobalData.gUseHashCache:\r
+            return False\r
+\r
+        # Initialize a dictionary for each arch type\r
+        if self.Arch not in GlobalData.gBuildHashSkipTracking:\r
+            GlobalData.gBuildHashSkipTracking[self.Arch] = dict()\r
+\r
         # If library or Module is binary do not skip by hash\r
         if self.IsBinaryModule:\r
             return False\r
+\r
         # .inc is contains binary information so do not skip by hash as well\r
         for f_ext in self.SourceFileList:\r
             if '.inc' in str(f_ext):\r
                 return False\r
-        if GlobalData.gUseHashCache:\r
-            # If there is a valid hash or function generated a valid hash; function will return False\r
-            # and the statement below will return True\r
-            return not self.GenModuleHash()\r
-        return False\r
+\r
+        # Use Cache, if exists and if Module has a copy in cache\r
+        if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():\r
+            return True\r
+\r
+        # Early exit for libraries that haven't yet finished building\r
+        HashFile = path.join(self.BuildDir, self.Name + ".hash")\r
+        if self.IsLibrary and not os.path.exists(HashFile):\r
+            return False\r
+\r
+        # Return a Boolean based on if can skip by hash, either from memory or from IO.\r
+        if self.Name not in GlobalData.gBuildHashSkipTracking[self.Arch]:\r
+            # If hashes are the same, SaveFileOnChange() will return False.\r
+            GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] = not SaveFileOnChange(HashFile, self.GenModuleHash(), True)\r
+            return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name]\r
+        else:\r
+            return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name]\r
 \r
     ## Decide whether we can skip the ModuleAutoGen process\r
     #  If any source file is newer than the module than we cannot skip\r

diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py
index 79f23c892d486e9ca1b2d208692cdcb8148819f8..95e28a988f1be13119064cf90177701e13f07d6f 100644 (file)
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -112,3 +112,9 @@ gSikpAutoGenCache = set()
 # Dictionary for tracking Module build status as success or failure\r
 # False -> Fail : True -> Success\r
 gModuleBuildTracking = dict()\r
+\r
+# Dictionary of booleans that dictate whether a module or\r
+# library can be skiped\r
+# Top Dict:     Key: Arch Type              Value: Dictionary\r
+# Second Dict:  Key: Module\Library Name    Value: True\False\r
+gBuildHashSkipTracking = dict()\r

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 7271570d2956db80e50e6667b1aa2f6d0d6baf07..027061191c5eee3aa06867d83008f58a3b00ead8 100644 (file)
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -593,7 +593,7 @@ class BuildTask:
     #\r
     def AddDependency(self, Dependency):\r
         for Dep in Dependency:\r
-            if not Dep.BuildObject.IsBinaryModule:\r
+            if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash():\r
                 self.DependencyList.append(BuildTask.New(Dep))    # BuildTask list\r
 \r
     ## The thread wrapper of LaunchCommand function\r
@@ -605,6 +605,11 @@ class BuildTask:
         try:\r
             self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)\r
             self.CompleteFlag = True\r
+\r
+            # Run hash operation post dependency, to account for libs\r
+            if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:\r
+                HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")\r
+                SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True)\r
         except:\r
             #\r
             # TRICK: hide the output of threads left running, so that the user can\r