X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=BaseTools%2FSource%2FPython%2FAutoGen%2FAutoGen.py;h=14eb1384c6c738cd102b54d9e4aca029d4085210;hb=9c8c4478cfcacaf5fd60b75ff78d26732d93a5b8;hp=70c6c9118671c99bf63a7519000961f28eae4b07;hpb=c17956e0eedce299ac253ac40238ce90a5e623e0;p=mirror_edk2.git diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py index 70c6c91186..14eb1384c6 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -43,6 +43,7 @@ from Workspace.MetaFileCommentParser import UsageList from Common.MultipleWorkspace import MultipleWorkspace as mws import InfSectionParser import datetime +import hashlib ## Regular expression for splitting Dependency Expression string into tokens gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)") @@ -62,7 +63,10 @@ gMakeTypeMap = {"MSFT":"nmake", "GCC":"gmake"} ## Build rule configuration file -gDefaultBuildRuleFile = 'Conf/build_rule.txt' +gDefaultBuildRuleFile = 'build_rule.txt' + +## Tools definition configuration file +gDefaultToolsDefFile = 'tools_def.txt' ## Build rule default version AutoGenReqBuildRuleVerNum = "0.1" @@ -140,6 +144,8 @@ ${END} ${depexsection_item} +${userextension_tianocore_item} + ${tail_comments} [BuildOptions.${module_arch}] @@ -260,6 +266,10 @@ class WorkspaceAutoGen(AutoGen): self.FvTargetList = Fvs self.CapTargetList = Caps self.AutoGenObjectList = [] + self._BuildDir = None + self._FvDir = None + self._MakeFileDir = None + self._BuildCommand = None # there's many relative directory operations, so ... os.chdir(self.WorkspaceDir) @@ -413,7 +423,7 @@ class WorkspaceAutoGen(AutoGen): if HasTokenSpace: if (PcdItem.TokenCName, PcdItem.TokenSpaceGuidCName) == (TokenCName, TokenSpaceGuidCName): PcdDatumType = PcdItem.DatumType - NewValue = self._BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue) + NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue) FoundFlag = True else: if PcdItem.TokenCName == TokenCName: @@ -422,7 +432,7 @@ class WorkspaceAutoGen(AutoGen): TokenSpaceGuidCNameList.append(PcdItem.TokenSpaceGuidCName) PcdDatumType = PcdItem.DatumType TokenSpaceGuidCName = PcdItem.TokenSpaceGuidCName - NewValue = self._BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue) + NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue) FoundFlag = True else: EdkLogger.error( @@ -501,6 +511,22 @@ class WorkspaceAutoGen(AutoGen): SourcePcdDict['FixedAtBuild'].append((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName)) else: pass + # + # A PCD can only use one type for all source modules + # + for i in SourcePcdDict_Keys: + for j in SourcePcdDict_Keys: + if i != j: + IntersectionList = list(set(SourcePcdDict[i]).intersection(set(SourcePcdDict[j]))) + if len(IntersectionList) > 0: + EdkLogger.error( + 'build', + FORMAT_INVALID, + "Building modules from source INFs, following PCD use %s and %s access method. It must be corrected to use only one access method." % (i, j), + ExtraData="%s" % '\n\t'.join([str(P[1]+'.'+P[0]) for P in IntersectionList]) + ) + else: + pass # # intersection the BinaryPCD for Mixed PCD @@ -623,6 +649,14 @@ class WorkspaceAutoGen(AutoGen): Pa.CollectFixedAtBuildPcds() self.AutoGenObjectList.append(Pa) + # + # Generate Package level hash value + # + GlobalData.gPackageHash[Arch] = {} + if GlobalData.gUseHashCache: + for Pkg in Pkgs: + self._GenPkgLevelHash(Pkg) + # # Check PCDs token value conflict in each DEC file. # @@ -636,21 +670,40 @@ class WorkspaceAutoGen(AutoGen): # if self.FdfFile: # self._CheckDuplicateInFV(Fdf) - self._BuildDir = None - self._FvDir = None - self._MakeFileDir = None - self._BuildCommand = None - # - # Create BuildOptions Macro & PCD metafile. + # Create BuildOptions Macro & PCD metafile, also add the Active Platform and FDF file. # content = 'gCommandLineDefines: ' content += str(GlobalData.gCommandLineDefines) content += os.linesep content += 'BuildOptionPcd: ' content += str(GlobalData.BuildOptionPcd) + content += os.linesep + content += 'Active Platform: ' + content += str(self.Platform) + content += os.linesep + if self.FdfFile: + content += 'Flash Image Definition: ' + content += str(self.FdfFile) + content += os.linesep + if GlobalData.gBinCacheDest: + content += 'Cache of .efi location: ' + content += str(GlobalData.gBinCacheDest) SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False) + # + # Create PcdToken Number file for Dynamic/DynamicEx Pcd. + # + PcdTokenNumber = 'PcdTokenNumber: ' + if Pa.PcdTokenNumber: + if Pa.DynamicPcdList: + for Pcd in Pa.DynamicPcdList: + PcdTokenNumber += os.linesep + PcdTokenNumber += str((Pcd.TokenCName, Pcd.TokenSpaceGuidCName)) + PcdTokenNumber += ' : ' + PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName]) + SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False) + # # Get set of workspace metafiles # @@ -665,6 +718,18 @@ class WorkspaceAutoGen(AutoGen): SrcTimeStamp = os.stat(f)[8] self._SrcTimeStamp = SrcTimeStamp + if GlobalData.gUseHashCache: + m = hashlib.md5() + for files in AllWorkSpaceMetaFiles: + if files.endswith('.dec'): + continue + f = open(files, 'r') + Content = f.read() + f.close() + m.update(Content) + SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), True) + GlobalData.gPlatformHash = m.hexdigest() + # # Write metafile list to build directory # @@ -678,31 +743,29 @@ class WorkspaceAutoGen(AutoGen): print >> file, f return True - def _BuildOptionPcdValueFormat(self, TokenSpaceGuidCName, TokenCName, PcdDatumType, Value): - if PcdDatumType == 'VOID*': - if Value.startswith('L'): - if not Value[1]: - EdkLogger.error('build', OPTION_VALUE_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", B"{...}"') - Value = Value[0] + '"' + Value[1:] + '"' - elif Value.startswith('B'): - if not Value[1]: - EdkLogger.error('build', OPTION_VALUE_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", B"{...}"') - Value = Value[1:] - else: - if not Value[0]: - EdkLogger.error('build', OPTION_VALUE_INVALID, 'For Void* type PCD, when specify the Value in the command line, please use the following format: "string", L"string", B"{...}"') - Value = '"' + Value + '"' - - IsValid, Cause = CheckPcdDatum(PcdDatumType, Value) - if not IsValid: - EdkLogger.error('build', FORMAT_INVALID, Cause, ExtraData="%s.%s" % (TokenSpaceGuidCName, TokenCName)) - if PcdDatumType == 'BOOLEAN': - Value = Value.upper() - if Value == 'TRUE' or Value == '1': - Value = '1' - elif Value == 'FALSE' or Value == '0': - Value = '0' - return Value + def _GenPkgLevelHash(self, Pkg): + PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName) + CreateDirectory(PkgDir) + HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash') + m = hashlib.md5() + # Get .dec file's hash value + f = open(Pkg.MetaFile.Path, 'r') + Content = f.read() + f.close() + m.update(Content) + # Get include files hash value + if Pkg.Includes: + for inc in Pkg.Includes: + for Root, Dirs, Files in os.walk(str(inc)): + for File in Files: + File_Path = os.path.join(Root, File) + f = open(File_Path, 'r') + Content = f.read() + f.close() + m.update(Content) + SaveFileOnChange(HashFile, m.hexdigest(), True) + if Pkg.PackageName not in GlobalData.gPackageHash[Pkg.Arch]: + GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] = m.hexdigest() def _GetMetaFiles(self, Target, Toolchain, Arch): AllWorkSpaceMetaFiles = set() @@ -721,10 +784,19 @@ class WorkspaceAutoGen(AutoGen): AllWorkSpaceMetaFiles.add(self.MetaFile.Path) # + # add build_rule.txt & tools_def.txt + # + AllWorkSpaceMetaFiles.add(os.path.join(GlobalData.gConfDirectory, gDefaultBuildRuleFile)) + AllWorkSpaceMetaFiles.add(os.path.join(GlobalData.gConfDirectory, gDefaultToolsDefFile)) + # add BuildOption metafile # AllWorkSpaceMetaFiles.add(os.path.join(self.BuildDir, 'BuildOptions')) + # add PcdToken Number file for Dynamic/DynamicEx Pcd + # + AllWorkSpaceMetaFiles.add(os.path.join(self.BuildDir, 'PcdTokenNumber')) + for Arch in self.ArchList: Platform = self.BuildDatabase[self.MetaFile, Arch, Target, Toolchain] PGen = PlatformAutoGen(self, self.MetaFile, Target, Toolchain, Arch) @@ -931,7 +1003,8 @@ class WorkspaceAutoGen(AutoGen): ## Return the directory to store all intermediate and final files built def _GetBuildDir(self): - return self.AutoGenObjectList[0].BuildDir + if self._BuildDir == None: + return self.AutoGenObjectList[0].BuildDir ## Return the build output directory platform specifies def _GetOutputDir(self): @@ -2310,8 +2383,18 @@ class PlatformAutoGen(AutoGen): if Module in self.Platform.Modules: PlatformModule = self.Platform.Modules[str(Module)] for Key in PlatformModule.Pcds: + Flag = False if Key in Pcds: - self._OverridePcd(Pcds[Key], PlatformModule.Pcds[Key], Module) + ToPcd = Pcds[Key] + Flag = True + elif Key in GlobalData.MixedPcd: + for PcdItem in GlobalData.MixedPcd[Key]: + if PcdItem in Pcds: + ToPcd = Pcds[PcdItem] + Flag = True + break + if Flag: + self._OverridePcd(ToPcd, PlatformModule.Pcds[Key], Module) return Pcds.values() ## Resolve library names to library modules @@ -2660,6 +2743,7 @@ class ModuleAutoGen(AutoGen): self.BuildDatabase = self.Workspace.BuildDatabase self.BuildRuleOrder = None + self.BuildTime = 0 self._Module = None self._Name = None @@ -2735,10 +2819,7 @@ class ModuleAutoGen(AutoGen): if self._FixedAtBuildPcds: return self._FixedAtBuildPcds for Pcd in self.ModulePcdList: - if self.IsLibrary: - if not (Pcd.Pending == False and Pcd.Type == "FixedAtBuild"): - continue - elif Pcd.Type != "FixedAtBuild": + if Pcd.Type != "FixedAtBuild": continue if Pcd not in self._FixedAtBuildPcds: self._FixedAtBuildPcds.append(Pcd) @@ -3068,6 +3149,30 @@ class ModuleAutoGen(AutoGen): self._DepexExpressionList[ModuleType] = DepexExpressionList return self._DepexExpressionList + # Get the tiano core user extension, it is contain dependent library. + # @retval: a list contain tiano core userextension. + # + def _GetTianoCoreUserExtensionList(self): + TianoCoreUserExtentionList = [] + for M in [self.Module] + self.DependentLibraryList: + Filename = M.MetaFile.Path + InfObj = InfSectionParser.InfSectionParser(Filename) + TianoCoreUserExtenList = InfObj.GetUserExtensionTianoCore() + for TianoCoreUserExtent in TianoCoreUserExtenList: + for Section in TianoCoreUserExtent.keys(): + ItemList = Section.split(TAB_SPLIT) + Arch = self.Arch + if len(ItemList) == 4: + Arch = ItemList[3] + if Arch.upper() == TAB_ARCH_COMMON or Arch.upper() == self.Arch.upper(): + TianoCoreList = [] + TianoCoreList.extend([TAB_SECTION_START + Section + TAB_SECTION_END]) + TianoCoreList.extend(TianoCoreUserExtent[Section][:]) + TianoCoreList.append('\n') + TianoCoreUserExtentionList.append(TianoCoreList) + + return TianoCoreUserExtentionList + ## Return the list of specification version required for the module # # @retval list The list of specification defined in module file @@ -3835,6 +3940,11 @@ class ModuleAutoGen(AutoGen): 'libraryclasses_item' : [] } + if 'MODULE_UNI_FILE' in MDefs: + UNIFile = os.path.join(self.MetaFile.Dir, MDefs['MODULE_UNI_FILE']) + if os.path.isfile(UNIFile): + shutil.copy2(UNIFile, self.OutputDir) + if self.AutoGenVersion > int(gInfSpecVersion, 0): AsBuiltInfDict['module_inf_version'] = '0x%08x' % self.AutoGenVersion else: @@ -3849,9 +3959,11 @@ class ModuleAutoGen(AutoGen): AsBuiltInfDict['module_pi_specification_version'] += [self.Specification['PI_SPECIFICATION_VERSION']] OutputDir = self.OutputDir.replace('\\', '/').strip('/') - + self.OutputFile = [] for Item in self.CodaTargetList: File = Item.Target.Path.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/') + if File not in self.OutputFile: + self.OutputFile.append(File) if Item.Target.Ext.lower() == '.aml': AsBuiltInfDict['binary_item'] += ['ASL|' + File] elif Item.Target.Ext.lower() == '.acpi': @@ -3861,6 +3973,8 @@ class ModuleAutoGen(AutoGen): else: AsBuiltInfDict['binary_item'] += ['BIN|' + File] if self.DepexGenerated: + if self.Name + '.depex' not in self.OutputFile: + self.OutputFile.append(self.Name + '.depex') if self.ModuleType in ['PEIM']: AsBuiltInfDict['binary_item'] += ['PEI_DEPEX|' + self.Name + '.depex'] if self.ModuleType in ['DXE_DRIVER', 'DXE_RUNTIME_DRIVER', 'DXE_SAL_DRIVER', 'UEFI_DRIVER']: @@ -3871,11 +3985,15 @@ class ModuleAutoGen(AutoGen): Bin = self._GenOffsetBin() if Bin: AsBuiltInfDict['binary_item'] += ['BIN|%s' % Bin] + if Bin not in self.OutputFile: + self.OutputFile.append(Bin) for Root, Dirs, Files in os.walk(OutputDir): for File in Files: if File.lower().endswith('.pdb'): AsBuiltInfDict['binary_item'] += ['DISPOSABLE|' + File] + if File not in self.OutputFile: + self.OutputFile.append(File) HeaderComments = self.Module.HeaderComments StartPos = 0 for Index in range(len(HeaderComments)): @@ -3917,6 +4035,13 @@ class ModuleAutoGen(AutoGen): else: continue PcdValue = '' + if Pcd.DatumType == 'BOOLEAN': + BoolValue = Pcd.DefaultValue.upper() + if BoolValue == 'TRUE': + Pcd.DefaultValue = '1' + elif BoolValue == 'FALSE': + Pcd.DefaultValue = '0' + if Pcd.DatumType != 'VOID*': HexFormat = '0x%02x' if Pcd.DatumType == 'UINT16': @@ -4027,6 +4152,16 @@ class ModuleAutoGen(AutoGen): for Library in self.LibraryAutoGenList: AsBuiltInfDict['libraryclasses_item'] += [Library.MetaFile.File.replace('\\', '/')] + # Generated UserExtensions TianoCore section. + # All tianocore user extensions are copied. + UserExtStr = '' + for TianoCore in self._GetTianoCoreUserExtensionList(): + UserExtStr += '\n'.join(TianoCore) + ExtensionFile = os.path.join(self.MetaFile.Dir, TianoCore[1]) + if os.path.isfile(ExtensionFile): + shutil.copy2(ExtensionFile, self.OutputDir) + AsBuiltInfDict['userextension_tianocore_item'] = UserExtStr + # Generated depex expression section in comments. AsBuiltInfDict['depexsection_item'] = '' DepexExpresion = self._GetDepexExpresionString() @@ -4039,7 +4174,48 @@ class ModuleAutoGen(AutoGen): SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False) self.IsAsBuiltInfCreated = True - + if GlobalData.gBinCacheDest: + self.CopyModuleToCache() + + def CopyModuleToCache(self): + FileDir = path.join(GlobalData.gBinCacheDest, self.Arch, self.SourceDir, self.MetaFile.BaseName) + CreateDirectory (FileDir) + HashFile = path.join(self.BuildDir, self.Name + '.hash') + ModuleFile = path.join(self.OutputDir, self.Name + '.inf') + if os.path.exists(HashFile): + shutil.copy2(HashFile, FileDir) + if os.path.exists(ModuleFile): + shutil.copy2(ModuleFile, FileDir) + if self.OutputFile: + for File in self.OutputFile: + if not os.path.isabs(File): + File = os.path.join(self.OutputDir, File) + if os.path.exists(File): + shutil.copy2(File, FileDir) + + def AttemptModuleCacheCopy(self): + if self.IsBinaryModule: + return False + FileDir = path.join(GlobalData.gBinCacheSource, self.Arch, self.SourceDir, self.MetaFile.BaseName) + HashFile = path.join(FileDir, self.Name + '.hash') + if os.path.exists(HashFile): + f = open(HashFile, 'r') + CacheHash = f.read() + f.close() + if GlobalData.gModuleHash[self.Arch][self.Name]: + if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]: + for root, dir, files in os.walk(FileDir): + for f in files: + if self.Name + '.hash' in f: + shutil.copy2(HashFile, self.BuildDir) + else: + File = path.join(root, f) + shutil.copy2(File, self.OutputDir) + if self.Name == "PcdPeim" or self.Name == "PcdDxe": + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + return True + return False + ## Create makefile for the module and its dependent libraries # # @param CreateLibraryMakeFile Flag indicating if or not the makefiles of @@ -4167,8 +4343,54 @@ class ModuleAutoGen(AutoGen): self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE) return self._LibraryAutoGenList + def GenModuleHash(self): + if self.Arch not in GlobalData.gModuleHash: + GlobalData.gModuleHash[self.Arch] = {} + m = hashlib.md5() + # Add Platform level hash + m.update(GlobalData.gPlatformHash) + # Add Package level hash + if self.DependentPackageList: + for Pkg in self.DependentPackageList: + if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]: + m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName]) + + # Add Library hash + if self.LibraryAutoGenList: + for Lib in self.LibraryAutoGenList: + if Lib.Name not in GlobalData.gModuleHash[self.Arch]: + Lib.GenModuleHash() + m.update(GlobalData.gModuleHash[self.Arch][Lib.Name]) + + # Add Module self + f = open(str(self.MetaFile), 'r') + Content = f.read() + f.close() + m.update(Content) + # Add Module's source files + if self.SourceFileList: + for File in self.SourceFileList: + f = open(str(File), 'r') + Content = f.read() + f.close() + m.update(Content) + + ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash") + if self.Name not in GlobalData.gModuleHash[self.Arch]: + GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest() + if GlobalData.gBinCacheSource: + CacheValid = self.AttemptModuleCacheCopy() + if CacheValid: + return False + return SaveFileOnChange(ModuleHashFile, m.hexdigest(), True) + ## Decide whether we can skip the ModuleAutoGen process - # If any source file is newer than the modeule than we cannot skip + def CanSkipbyHash(self): + if GlobalData.gUseHashCache: + return not self.GenModuleHash() + + ## Decide whether we can skip the ModuleAutoGen process + # If any source file is newer than the module than we cannot skip # def CanSkip(self): if not os.path.exists(self.GetTimeStampPath()): @@ -4183,6 +4405,8 @@ class ModuleAutoGen(AutoGen): with open(self.GetTimeStampPath(),'r') as f: for source in f: source = source.rstrip('\n') + if not os.path.exists(source): + return False if source not in ModuleAutoGen.TimeDict : ModuleAutoGen.TimeDict[source] = os.stat(source)[8] if ModuleAutoGen.TimeDict[source] > DstTimeStamp: