X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FAutoGen.py;h=1c4c3954797cb2b5438b12ba9d71caa92f0f7cc7;hb=37de70b764200718cc39a21abc491c335e3da7b3;hp=fa6ec5608f84edcfdcdfa71a91a51f5b18829078;hpb=99adfe9f51cbe105ec12f0572571fc85762382fb;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index fa6ec5608f..1c4c395479 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -43,6 +43,7 @@ from Workspace.MetaFileCommentParser import UsageList from Common.MultipleWorkspace import MultipleWorkspace as mws import InfSectionParser import datetime +import hashlib ## Regular expression for splitting Dependency Expression string into tokens gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)") @@ -143,6 +144,8 @@ ${END} ${depexsection_item} +${userextension_tianocore_item} + ${tail_comments} [BuildOptions.${module_arch}] @@ -263,6 +266,10 @@ class WorkspaceAutoGen(AutoGen): self.FvTargetList = Fvs self.CapTargetList = Caps self.AutoGenObjectList = [] + self._BuildDir = None + self._FvDir = None + self._MakeFileDir = None + self._BuildCommand = None # there's many relative directory operations, so ... os.chdir(self.WorkspaceDir) @@ -642,6 +649,14 @@ class WorkspaceAutoGen(AutoGen): Pa.CollectFixedAtBuildPcds() self.AutoGenObjectList.append(Pa) + # + # Generate Package level hash value + # + GlobalData.gPackageHash[Arch] = {} + if GlobalData.gUseHashCache: + for Pkg in Pkgs: + self._GenPkgLevelHash(Pkg) + # # Check PCDs token value conflict in each DEC file. # @@ -655,19 +670,25 @@ class WorkspaceAutoGen(AutoGen): # if self.FdfFile: # self._CheckDuplicateInFV(Fdf) - self._BuildDir = None - self._FvDir = None - self._MakeFileDir = None - self._BuildCommand = None - # - # Create BuildOptions Macro & PCD metafile. + # Create BuildOptions Macro & PCD metafile, also add the Active Platform and FDF file. # content = 'gCommandLineDefines: ' content += str(GlobalData.gCommandLineDefines) content += os.linesep content += 'BuildOptionPcd: ' content += str(GlobalData.BuildOptionPcd) + content += os.linesep + content += 'Active Platform: ' + content += str(self.Platform) + content += os.linesep + if self.FdfFile: + content += 'Flash Image Definition: ' + content += str(self.FdfFile) + content += os.linesep + if GlobalData.gBinCacheDest: + content += 'Cache of .efi location: ' + content += str(GlobalData.gBinCacheDest) SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False) # @@ -697,6 +718,18 @@ class WorkspaceAutoGen(AutoGen): SrcTimeStamp = os.stat(f)[8] self._SrcTimeStamp = SrcTimeStamp + if GlobalData.gUseHashCache: + m = hashlib.md5() + for files in AllWorkSpaceMetaFiles: + if files.endswith('.dec'): + continue + f = open(files, 'r') + Content = f.read() + f.close() + m.update(Content) + SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), True) + GlobalData.gPlatformHash = m.hexdigest() + # # Write metafile list to build directory # @@ -710,6 +743,29 @@ class WorkspaceAutoGen(AutoGen): print >> file, f return True + def _GenPkgLevelHash(self, Pkg): + PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName) + CreateDirectory(PkgDir) + HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash') + m = hashlib.md5() + # Get .dec file's hash value + f = open(Pkg.MetaFile.Path, 'r') + Content = f.read() + f.close() + m.update(Content) + # Get include files hash value + if Pkg.Includes: + for inc in Pkg.Includes: + for Root, Dirs, Files in os.walk(str(inc)): + for File in Files: + File_Path = os.path.join(Root, File) + f = open(File_Path, 'r') + Content = f.read() + f.close() + m.update(Content) + SaveFileOnChange(HashFile, m.hexdigest(), True) + if Pkg.PackageName not in GlobalData.gPackageHash[Pkg.Arch]: + GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] = m.hexdigest() def _GetMetaFiles(self, Target, Toolchain, Arch): AllWorkSpaceMetaFiles = set() @@ -947,7 +1003,8 @@ class WorkspaceAutoGen(AutoGen): ## Return the directory to store all intermediate and final files built def _GetBuildDir(self): - return self.AutoGenObjectList[0].BuildDir + if self._BuildDir == None: + return self.AutoGenObjectList[0].BuildDir ## Return the build output directory platform specifies def _GetOutputDir(self): @@ -1250,12 +1307,15 @@ class PlatformAutoGen(AutoGen): # @param CreateModuleMakeFile Flag indicating if the makefile for # modules will be created as well # - def CreateMakeFile(self, CreateModuleMakeFile=False): + def CreateMakeFile(self, CreateModuleMakeFile=False, FfsCommand = {}): if CreateModuleMakeFile: for ModuleFile in self.Platform.Modules: Ma = ModuleAutoGen(self.Workspace, ModuleFile, self.BuildTarget, self.ToolChain, self.Arch, self.MetaFile) - Ma.CreateMakeFile(True) + if (ModuleFile.File, self.Arch) in FfsCommand: + Ma.CreateMakeFile(True, FfsCommand[ModuleFile.File, self.Arch]) + else: + Ma.CreateMakeFile(True) #Ma.CreateAsBuiltInf() # no need to create makefile for the platform more than once @@ -1741,6 +1801,7 @@ class PlatformAutoGen(AutoGen): self.OutputDir, self.BuildTarget + "_" + self.ToolChain, ) + GlobalData.gBuildDirectory = self._BuildDir return self._BuildDir ## Return directory of platform makefile @@ -2326,8 +2387,18 @@ class PlatformAutoGen(AutoGen): if Module in self.Platform.Modules: PlatformModule = self.Platform.Modules[str(Module)] for Key in PlatformModule.Pcds: + Flag = False if Key in Pcds: - self._OverridePcd(Pcds[Key], PlatformModule.Pcds[Key], Module) + ToPcd = Pcds[Key] + Flag = True + elif Key in GlobalData.MixedPcd: + for PcdItem in GlobalData.MixedPcd[Key]: + if PcdItem in Pcds: + ToPcd = Pcds[PcdItem] + Flag = True + break + if Flag: + self._OverridePcd(ToPcd, PlatformModule.Pcds[Key], Module) return Pcds.values() ## Resolve library names to library modules @@ -2676,6 +2747,7 @@ class ModuleAutoGen(AutoGen): self.BuildDatabase = self.Workspace.BuildDatabase self.BuildRuleOrder = None + self.BuildTime = 0 self._Module = None self._Name = None @@ -2691,6 +2763,7 @@ class ModuleAutoGen(AutoGen): self._BuildDir = None self._OutputDir = None + self._FfsOutputDir = None self._DebugDir = None self._MakeFileDir = None @@ -2807,6 +2880,7 @@ class ModuleAutoGen(AutoGen): self._Macro["PLATFORM_RELATIVE_DIR" ] = self.PlatformInfo.SourceDir self._Macro["PLATFORM_DIR" ] = mws.join(self.WorkspaceDir, self.PlatformInfo.SourceDir) self._Macro["PLATFORM_OUTPUT_DIR" ] = self.PlatformInfo.OutputDir + self._Macro["FFS_OUTPUT_DIR" ] = self.FfsOutputDir return self._Macro ## Return the module build data object @@ -2897,6 +2971,15 @@ class ModuleAutoGen(AutoGen): CreateDirectory(self._OutputDir) return self._OutputDir + ## Return the directory to store ffs file + def _GetFfsOutputDir(self): + if self._FfsOutputDir == None: + if GlobalData.gFdfParser != None: + self._FfsOutputDir = path.join(self.PlatformInfo.BuildDir, "FV", "Ffs", self.Guid + self.Name) + else: + self._FfsOutputDir = '' + return self._FfsOutputDir + ## Return the directory to store auto-gened source files of the mdoule def _GetDebugDir(self): if self._DebugDir == None: @@ -3081,6 +3164,30 @@ class ModuleAutoGen(AutoGen): self._DepexExpressionList[ModuleType] = DepexExpressionList return self._DepexExpressionList + # Get the tiano core user extension, it is contain dependent library. + # @retval: a list contain tiano core userextension. + # + def _GetTianoCoreUserExtensionList(self): + TianoCoreUserExtentionList = [] + for M in [self.Module] + self.DependentLibraryList: + Filename = M.MetaFile.Path + InfObj = InfSectionParser.InfSectionParser(Filename) + TianoCoreUserExtenList = InfObj.GetUserExtensionTianoCore() + for TianoCoreUserExtent in TianoCoreUserExtenList: + for Section in TianoCoreUserExtent.keys(): + ItemList = Section.split(TAB_SPLIT) + Arch = self.Arch + if len(ItemList) == 4: + Arch = ItemList[3] + if Arch.upper() == TAB_ARCH_COMMON or Arch.upper() == self.Arch.upper(): + TianoCoreList = [] + TianoCoreList.extend([TAB_SECTION_START + Section + TAB_SECTION_END]) + TianoCoreList.extend(TianoCoreUserExtent[Section][:]) + TianoCoreList.append('\n') + TianoCoreUserExtentionList.append(TianoCoreList) + + return TianoCoreUserExtentionList + ## Return the list of specification version required for the module # # @retval list The list of specification defined in module file @@ -3174,13 +3281,13 @@ class ModuleAutoGen(AutoGen): EdkLogger.debug(EdkLogger.DEBUG_9, "The toolchain [%s] for processing file [%s] is found, " "but [%s] is needed" % (F.TagName, str(F), self.ToolChain)) continue - # match tool chain family - if F.ToolChainFamily not in ("", "*", self.ToolChainFamily): + # match tool chain family or build rule family + if F.ToolChainFamily not in ("", "*", self.ToolChainFamily, self.BuildRuleFamily): EdkLogger.debug( EdkLogger.DEBUG_0, "The file [%s] must be built by tools of [%s], " \ - "but current toolchain family is [%s]" \ - % (str(F), F.ToolChainFamily, self.ToolChainFamily)) + "but current toolchain family is [%s], buildrule family is [%s]" \ + % (str(F), F.ToolChainFamily, self.ToolChainFamily, self.BuildRuleFamily)) continue # add the file path into search path list for file including @@ -3848,6 +3955,11 @@ class ModuleAutoGen(AutoGen): 'libraryclasses_item' : [] } + if 'MODULE_UNI_FILE' in MDefs: + UNIFile = os.path.join(self.MetaFile.Dir, MDefs['MODULE_UNI_FILE']) + if os.path.isfile(UNIFile): + shutil.copy2(UNIFile, self.OutputDir) + if self.AutoGenVersion > int(gInfSpecVersion, 0): AsBuiltInfDict['module_inf_version'] = '0x%08x' % self.AutoGenVersion else: @@ -3862,9 +3974,11 @@ class ModuleAutoGen(AutoGen): AsBuiltInfDict['module_pi_specification_version'] += [self.Specification['PI_SPECIFICATION_VERSION']] OutputDir = self.OutputDir.replace('\\', '/').strip('/') - + self.OutputFile = [] for Item in self.CodaTargetList: File = Item.Target.Path.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/') + if File not in self.OutputFile: + self.OutputFile.append(File) if Item.Target.Ext.lower() == '.aml': AsBuiltInfDict['binary_item'] += ['ASL|' + File] elif Item.Target.Ext.lower() == '.acpi': @@ -3874,6 +3988,8 @@ class ModuleAutoGen(AutoGen): else: AsBuiltInfDict['binary_item'] += ['BIN|' + File] if self.DepexGenerated: + if self.Name + '.depex' not in self.OutputFile: + self.OutputFile.append(self.Name + '.depex') if self.ModuleType in ['PEIM']: AsBuiltInfDict['binary_item'] += ['PEI_DEPEX|' + self.Name + '.depex'] if self.ModuleType in ['DXE_DRIVER', 'DXE_RUNTIME_DRIVER', 'DXE_SAL_DRIVER', 'UEFI_DRIVER']: @@ -3884,11 +4000,15 @@ class ModuleAutoGen(AutoGen): Bin = self._GenOffsetBin() if Bin: AsBuiltInfDict['binary_item'] += ['BIN|%s' % Bin] + if Bin not in self.OutputFile: + self.OutputFile.append(Bin) for Root, Dirs, Files in os.walk(OutputDir): for File in Files: if File.lower().endswith('.pdb'): AsBuiltInfDict['binary_item'] += ['DISPOSABLE|' + File] + if File not in self.OutputFile: + self.OutputFile.append(File) HeaderComments = self.Module.HeaderComments StartPos = 0 for Index in range(len(HeaderComments)): @@ -4047,6 +4167,16 @@ class ModuleAutoGen(AutoGen): for Library in self.LibraryAutoGenList: AsBuiltInfDict['libraryclasses_item'] += [Library.MetaFile.File.replace('\\', '/')] + # Generated UserExtensions TianoCore section. + # All tianocore user extensions are copied. + UserExtStr = '' + for TianoCore in self._GetTianoCoreUserExtensionList(): + UserExtStr += '\n'.join(TianoCore) + ExtensionFile = os.path.join(self.MetaFile.Dir, TianoCore[1]) + if os.path.isfile(ExtensionFile): + shutil.copy2(ExtensionFile, self.OutputDir) + AsBuiltInfDict['userextension_tianocore_item'] = UserExtStr + # Generated depex expression section in comments. AsBuiltInfDict['depexsection_item'] = '' DepexExpresion = self._GetDepexExpresionString() @@ -4059,26 +4189,68 @@ class ModuleAutoGen(AutoGen): SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False) self.IsAsBuiltInfCreated = True - + if GlobalData.gBinCacheDest: + self.CopyModuleToCache() + + def CopyModuleToCache(self): + FileDir = path.join(GlobalData.gBinCacheDest, self.Arch, self.SourceDir, self.MetaFile.BaseName) + CreateDirectory (FileDir) + HashFile = path.join(self.BuildDir, self.Name + '.hash') + ModuleFile = path.join(self.OutputDir, self.Name + '.inf') + if os.path.exists(HashFile): + shutil.copy2(HashFile, FileDir) + if os.path.exists(ModuleFile): + shutil.copy2(ModuleFile, FileDir) + if self.OutputFile: + for File in self.OutputFile: + if not os.path.isabs(File): + File = os.path.join(self.OutputDir, File) + if os.path.exists(File): + shutil.copy2(File, FileDir) + + def AttemptModuleCacheCopy(self): + if self.IsBinaryModule: + return False + FileDir = path.join(GlobalData.gBinCacheSource, self.Arch, self.SourceDir, self.MetaFile.BaseName) + HashFile = path.join(FileDir, self.Name + '.hash') + if os.path.exists(HashFile): + f = open(HashFile, 'r') + CacheHash = f.read() + f.close() + if GlobalData.gModuleHash[self.Arch][self.Name]: + if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]: + for root, dir, files in os.walk(FileDir): + for f in files: + if self.Name + '.hash' in f: + shutil.copy2(HashFile, self.BuildDir) + else: + File = path.join(root, f) + shutil.copy2(File, self.OutputDir) + if self.Name == "PcdPeim" or self.Name == "PcdDxe": + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + return True + return False + ## Create makefile for the module and its dependent libraries # # @param CreateLibraryMakeFile Flag indicating if or not the makefiles of # dependent libraries will be created # - def CreateMakeFile(self, CreateLibraryMakeFile=True): + def CreateMakeFile(self, CreateLibraryMakeFile=True, GenFfsList = []): # Ignore generating makefile when it is a binary module if self.IsBinaryModule: return if self.IsMakeFileCreated: return - if self.CanSkip(): - return - + self.GenFfsList = GenFfsList if not self.IsLibrary and CreateLibraryMakeFile: for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateMakeFile() + if self.CanSkip(): + return + if len(self.CustomMakefile) == 0: Makefile = GenMake.ModuleMakefile(self) else: @@ -4106,8 +4278,6 @@ class ModuleAutoGen(AutoGen): def CreateCodeFile(self, CreateLibraryCodeFile=True): if self.IsCodeFileCreated: return - if self.CanSkip(): - return # Need to generate PcdDatabase even PcdDriver is binarymodule if self.IsBinaryModule and self.PcdIsDriver != '': @@ -4122,6 +4292,9 @@ class ModuleAutoGen(AutoGen): for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateCodeFile() + if self.CanSkip(): + return + AutoGenList = [] IgoredAutoGenList = [] @@ -4187,8 +4360,54 @@ class ModuleAutoGen(AutoGen): self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE) return self._LibraryAutoGenList + def GenModuleHash(self): + if self.Arch not in GlobalData.gModuleHash: + GlobalData.gModuleHash[self.Arch] = {} + m = hashlib.md5() + # Add Platform level hash + m.update(GlobalData.gPlatformHash) + # Add Package level hash + if self.DependentPackageList: + for Pkg in self.DependentPackageList: + if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]: + m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName]) + + # Add Library hash + if self.LibraryAutoGenList: + for Lib in self.LibraryAutoGenList: + if Lib.Name not in GlobalData.gModuleHash[self.Arch]: + Lib.GenModuleHash() + m.update(GlobalData.gModuleHash[self.Arch][Lib.Name]) + + # Add Module self + f = open(str(self.MetaFile), 'r') + Content = f.read() + f.close() + m.update(Content) + # Add Module's source files + if self.SourceFileList: + for File in self.SourceFileList: + f = open(str(File), 'r') + Content = f.read() + f.close() + m.update(Content) + + ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash") + if self.Name not in GlobalData.gModuleHash[self.Arch]: + GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest() + if GlobalData.gBinCacheSource: + CacheValid = self.AttemptModuleCacheCopy() + if CacheValid: + return False + return SaveFileOnChange(ModuleHashFile, m.hexdigest(), True) + ## Decide whether we can skip the ModuleAutoGen process - # If any source file is newer than the modeule than we cannot skip + def CanSkipbyHash(self): + if GlobalData.gUseHashCache: + return not self.GenModuleHash() + + ## Decide whether we can skip the ModuleAutoGen process + # If any source file is newer than the module than we cannot skip # def CanSkip(self): if not os.path.exists(self.GetTimeStampPath()): @@ -4203,6 +4422,8 @@ class ModuleAutoGen(AutoGen): with open(self.GetTimeStampPath(),'r') as f: for source in f: source = source.rstrip('\n') + if not os.path.exists(source): + return False if source not in ModuleAutoGen.TimeDict : ModuleAutoGen.TimeDict[source] = os.stat(source)[8] if ModuleAutoGen.TimeDict[source] > DstTimeStamp: @@ -4250,6 +4471,7 @@ class ModuleAutoGen(AutoGen): IsBinaryModule = property(_IsBinaryModule) BuildDir = property(_GetBuildDir) OutputDir = property(_GetOutputDir) + FfsOutputDir = property(_GetFfsOutputDir) DebugDir = property(_GetDebugDir) MakeFileDir = property(_GetMakeFileDir) CustomMakefile = property(_GetCustomMakefile)