BaseTools: Leverage compiler output to optimize binary cache

[mirror_edk2.git] / BaseTools / Source / Python / AutoGen / WorkspaceAutoGen.py

diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index ec0c25bd1487e7d426d6c4b9fca121137ed6ed4c..668126aaac426222016a6f8ed7023452a4f31df9 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -23,6 +23,7 @@ from Common.StringUtils import NormPath
 from Common.BuildToolError import *\r
 from Common.DataType import *\r
 from Common.Misc import *\r
+import json\r
 \r
 ## Regular expression for splitting Dependency Expression string into tokens\r
 gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")\r
@@ -127,7 +128,7 @@ class WorkspaceAutoGen(AutoGen):
 \r
         self.CreateBuildOptionsFile()\r
         self.CreatePcdTokenNumberFile()\r
-        self.CreateModuleHashInfo()\r
+        self.GeneratePlatformLevelHash()\r
 \r
     #\r
     # Merge Arch\r
@@ -528,12 +529,12 @@ class WorkspaceAutoGen(AutoGen):
                     PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])\r
         SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False)\r
 \r
-    def CreateModuleHashInfo(self):\r
+    def GeneratePlatformLevelHash(self):\r
         #\r
         # Get set of workspace metafiles\r
         #\r
         AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain)\r
-\r
+        AllWorkSpaceMetaFileList = sorted(AllWorkSpaceMetaFiles, key=lambda x: str(x))\r
         #\r
         # Retrieve latest modified time of all metafiles\r
         #\r
@@ -544,16 +545,35 @@ class WorkspaceAutoGen(AutoGen):
         self._SrcTimeStamp = SrcTimeStamp\r
 \r
         if GlobalData.gUseHashCache:\r
+            FileList = []\r
             m = hashlib.md5()\r
-            for files in AllWorkSpaceMetaFiles:\r
-                if files.endswith('.dec'):\r
+            for file in AllWorkSpaceMetaFileList:\r
+                if file.endswith('.dec'):\r
                     continue\r
-                f = open(files, 'rb')\r
+                f = open(file, 'rb')\r
                 Content = f.read()\r
                 f.close()\r
                 m.update(Content)\r
-            SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), False)\r
-            GlobalData.gPlatformHash = m.hexdigest()\r
+                FileList.append((str(file), hashlib.md5(Content).hexdigest()))\r
+\r
+            HashDir = path.join(self.BuildDir, "Hash_Platform")\r
+            HashFile = path.join(HashDir, 'Platform.hash.' + m.hexdigest())\r
+            SaveFileOnChange(HashFile, m.hexdigest(), False)\r
+            HashChainFile = path.join(HashDir, 'Platform.hashchain.' + m.hexdigest())\r
+            GlobalData.gPlatformHashFile = HashChainFile\r
+            try:\r
+                with open(HashChainFile, 'w') as f:\r
+                    json.dump(FileList, f, indent=2)\r
+            except:\r
+                EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)\r
+\r
+            if GlobalData.gBinCacheDest:\r
+                # Copy platform hash files to cache destination\r
+                FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, "Hash_Platform")\r
+                CacheFileDir = FileDir\r
+                CreateDirectory(CacheFileDir)\r
+                CopyFileOnChange(HashFile, CacheFileDir)\r
+                CopyFileOnChange(HashChainFile, CacheFileDir)\r
 \r
         #\r
         # Write metafile list to build directory\r
@@ -564,7 +584,7 @@ class WorkspaceAutoGen(AutoGen):
         if not os.path.exists(self.BuildDir):\r
             os.makedirs(self.BuildDir)\r
         with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:\r
-            for f in AllWorkSpaceMetaFiles:\r
+            for f in AllWorkSpaceMetaFileList:\r
                 print(f, file=file)\r
         return True\r
 \r
@@ -572,15 +592,16 @@ class WorkspaceAutoGen(AutoGen):
         if Pkg.PackageName in GlobalData.gPackageHash:\r
             return\r
 \r
-        PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)\r
+        PkgDir = os.path.join(self.BuildDir, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)\r
         CreateDirectory(PkgDir)\r
-        HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')\r
+        FileList = []\r
         m = hashlib.md5()\r
         # Get .dec file's hash value\r
         f = open(Pkg.MetaFile.Path, 'rb')\r
         Content = f.read()\r
         f.close()\r
         m.update(Content)\r
+        FileList.append((str(Pkg.MetaFile.Path), hashlib.md5(Content).hexdigest()))\r
         # Get include files hash value\r
         if Pkg.Includes:\r
             for inc in sorted(Pkg.Includes, key=lambda x: str(x)):\r
@@ -591,9 +612,28 @@ class WorkspaceAutoGen(AutoGen):
                         Content = f.read()\r
                         f.close()\r
                         m.update(Content)\r
-        SaveFileOnChange(HashFile, m.hexdigest(), False)\r
+                        FileList.append((str(File_Path), hashlib.md5(Content).hexdigest()))\r
         GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()\r
 \r
+        HashDir = PkgDir\r
+        HashFile = path.join(HashDir, Pkg.PackageName + '.hash.' + m.hexdigest())\r
+        SaveFileOnChange(HashFile, m.hexdigest(), False)\r
+        HashChainFile = path.join(HashDir, Pkg.PackageName + '.hashchain.' + m.hexdigest())\r
+        GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] = HashChainFile\r
+        try:\r
+            with open(HashChainFile, 'w') as f:\r
+                json.dump(FileList, f, indent=2)\r
+        except:\r
+            EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)\r
+\r
+        if GlobalData.gBinCacheDest:\r
+            # Copy Pkg hash files to cache destination dir\r
+            FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)\r
+            CacheFileDir = FileDir\r
+            CreateDirectory(CacheFileDir)\r
+            CopyFileOnChange(HashFile, CacheFileDir)\r
+            CopyFileOnChange(HashChainFile, CacheFileDir)\r
+\r
     def _GetMetaFiles(self, Target, Toolchain):\r
         AllWorkSpaceMetaFiles = set()\r
         #\r