BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py

   1 ## @file
   2 # Create makefile for MS nmake and GNU make
   3 #
   4 # Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ## Import Modules
   9 #
  10 from __future__ import print_function
  11 from __future__ import absolute_import
  12 import os.path as path
  13 import hashlib
  14 from collections import defaultdict
  15 from GenFds.FdfParser import FdfParser
  16 from Workspace.WorkspaceCommon import GetModuleLibInstances
  17 from AutoGen import GenMake
  18 from AutoGen.AutoGen import AutoGen
  19 from AutoGen.PlatformAutoGen import PlatformAutoGen
  20 from AutoGen.BuildEngine import gDefaultBuildRuleFile
  21 from Common.ToolDefClassObject import gDefaultToolsDefFile
  22 from Common.StringUtils import NormPath
  23 from Common.BuildToolError import *
  24 from Common.DataType import *
  25 from Common.Misc import *
  26 import json
  27
  28 ## Regular expression for splitting Dependency Expression string into tokens
  29 gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
  30
  31 ## Regular expression for match: PCD(xxxx.yyy)
  32 gPCDAsGuidPattern = re.compile(r"^PCD\(.+\..+\)$")
  33
  34 ## Workspace AutoGen class
  35 #
  36 #   This class is used mainly to control the whole platform build for different
  37 # architecture. This class will generate top level makefile.
  38 #
  39 class WorkspaceAutoGen(AutoGen):
  40     # call super().__init__ then call the worker function with different parameter count
  41     def __init__(self, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs):
  42         if not hasattr(self, "_Init"):
  43             self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs)
  44             self._Init = True
  45
  46     ## Initialize WorkspaceAutoGen
  47     #
  48     #   @param  WorkspaceDir            Root directory of workspace
  49     #   @param  ActivePlatform          Meta-file of active platform
  50     #   @param  Target                  Build target
  51     #   @param  Toolchain               Tool chain name
  52     #   @param  ArchList                List of architecture of current build
  53     #   @param  MetaFileDb              Database containing meta-files
  54     #   @param  BuildConfig             Configuration of build
  55     #   @param  ToolDefinition          Tool chain definitions
  56     #   @param  FlashDefinitionFile     File of flash definition
  57     #   @param  Fds                     FD list to be generated
  58     #   @param  Fvs                     FV list to be generated
  59     #   @param  Caps                    Capsule list to be generated
  60     #   @param  SkuId                   SKU id from command line
  61     #
  62     def _InitWorker(self, WorkspaceDir, ActivePlatform, Target, Toolchain, ArchList, MetaFileDb,
  63               BuildConfig, ToolDefinition, FlashDefinitionFile='', Fds=None, Fvs=None, Caps=None, SkuId='', UniFlag=None,
  64               Progress=None, BuildModule=None):
  65         self.BuildDatabase  = MetaFileDb
  66         self.MetaFile       = ActivePlatform
  67         self.WorkspaceDir   = WorkspaceDir
  68         self.Platform       = self.BuildDatabase[self.MetaFile, TAB_ARCH_COMMON, Target, Toolchain]
  69         GlobalData.gActivePlatform = self.Platform
  70         self.BuildTarget    = Target
  71         self.ToolChain      = Toolchain
  72         self.ArchList       = ArchList
  73         self.SkuId          = SkuId
  74         self.UniFlag        = UniFlag
  75
  76         self.TargetTxt      = BuildConfig
  77         self.ToolDef        = ToolDefinition
  78         self.FdfFile        = FlashDefinitionFile
  79         self.FdTargetList   = Fds if Fds else []
  80         self.FvTargetList   = Fvs if Fvs else []
  81         self.CapTargetList  = Caps if Caps else []
  82         self.AutoGenObjectList = []
  83         self._GuidDict = {}
  84
  85         # there's many relative directory operations, so ...
  86         os.chdir(self.WorkspaceDir)
  87
  88         self.MergeArch()
  89         self.ValidateBuildTarget()
  90
  91         EdkLogger.info("")
  92         if self.ArchList:
  93             EdkLogger.info('%-16s = %s' % ("Architecture(s)", ' '.join(self.ArchList)))
  94         EdkLogger.info('%-16s = %s' % ("Build target", self.BuildTarget))
  95         EdkLogger.info('%-16s = %s' % ("Toolchain", self.ToolChain))
  96
  97         EdkLogger.info('\n%-24s = %s' % ("Active Platform", self.Platform))
  98         if BuildModule:
  99             EdkLogger.info('%-24s = %s' % ("Active Module", BuildModule))
 100
 101         if self.FdfFile:
 102             EdkLogger.info('%-24s = %s' % ("Flash Image Definition", self.FdfFile))
 103
 104         EdkLogger.verbose("\nFLASH_DEFINITION = %s" % self.FdfFile)
 105
 106         if Progress:
 107             Progress.Start("\nProcessing meta-data")
 108         #
 109         # Mark now build in AutoGen Phase
 110         #
 111         GlobalData.gAutoGenPhase = True
 112         self.ProcessModuleFromPdf()
 113         self.ProcessPcdType()
 114         self.ProcessMixedPcd()
 115         self.VerifyPcdsFromFDF()
 116         self.CollectAllPcds()
 117         for Pa in self.AutoGenObjectList:
 118             Pa.FillData_LibConstPcd()
 119         self.GeneratePkgLevelHash()
 120         #
 121         # Check PCDs token value conflict in each DEC file.
 122         #
 123         self._CheckAllPcdsTokenValueConflict()
 124         #
 125         # Check PCD type and definition between DSC and DEC
 126         #
 127         self._CheckPcdDefineAndType()
 128
 129         self.CreateBuildOptionsFile()
 130         self.CreatePcdTokenNumberFile()
 131         self.GeneratePlatformLevelHash()
 132
 133     #
 134     # Merge Arch
 135     #
 136     def MergeArch(self):
 137         if not self.ArchList:
 138             ArchList = set(self.Platform.SupArchList)
 139         else:
 140             ArchList = set(self.ArchList) & set(self.Platform.SupArchList)
 141         if not ArchList:
 142             EdkLogger.error("build", PARAMETER_INVALID,
 143                             ExtraData = "Invalid ARCH specified. [Valid ARCH: %s]" % (" ".join(self.Platform.SupArchList)))
 144         elif self.ArchList and len(ArchList) != len(self.ArchList):
 145             SkippedArchList = set(self.ArchList).symmetric_difference(set(self.Platform.SupArchList))
 146             EdkLogger.verbose("\nArch [%s] is ignored because the platform supports [%s] only!"
 147                               % (" ".join(SkippedArchList), " ".join(self.Platform.SupArchList)))
 148         self.ArchList = tuple(ArchList)
 149
 150     # Validate build target
 151     def ValidateBuildTarget(self):
 152         if self.BuildTarget not in self.Platform.BuildTargets:
 153             EdkLogger.error("build", PARAMETER_INVALID,
 154                             ExtraData="Build target [%s] is not supported by the platform. [Valid target: %s]"
 155                                       % (self.BuildTarget, " ".join(self.Platform.BuildTargets)))
 156     @cached_property
 157     def FdfProfile(self):
 158         if not self.FdfFile:
 159             self.FdfFile = self.Platform.FlashDefinition
 160
 161         FdfProfile = None
 162         if self.FdfFile:
 163             Fdf = FdfParser(self.FdfFile.Path)
 164             Fdf.ParseFile()
 165             GlobalData.gFdfParser = Fdf
 166             if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
 167                 FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
 168                 for FdRegion in FdDict.RegionList:
 169                     if str(FdRegion.RegionType) == 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
 170                         if int(FdRegion.Offset) % 8 != 0:
 171                             EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
 172             FdfProfile = Fdf.Profile
 173         else:
 174             if self.FdTargetList:
 175                 EdkLogger.info("No flash definition file found. FD [%s] will be ignored." % " ".join(self.FdTargetList))
 176                 self.FdTargetList = []
 177             if self.FvTargetList:
 178                 EdkLogger.info("No flash definition file found. FV [%s] will be ignored." % " ".join(self.FvTargetList))
 179                 self.FvTargetList = []
 180             if self.CapTargetList:
 181                 EdkLogger.info("No flash definition file found. Capsule [%s] will be ignored." % " ".join(self.CapTargetList))
 182                 self.CapTargetList = []
 183
 184         return FdfProfile
 185
 186     def ProcessModuleFromPdf(self):
 187
 188         if self.FdfProfile:
 189             for fvname in self.FvTargetList:
 190                 if fvname.upper() not in self.FdfProfile.FvDict:
 191                     EdkLogger.error("build", OPTION_VALUE_INVALID,
 192                                     "No such an FV in FDF file: %s" % fvname)
 193
 194             # In DSC file may use FILE_GUID to override the module, then in the Platform.Modules use FILE_GUIDmodule.inf as key,
 195             # but the path (self.MetaFile.Path) is the real path
 196             for key in self.FdfProfile.InfDict:
 197                 if key == 'ArchTBD':
 198                     MetaFile_cache = defaultdict(set)
 199                     for Arch in self.ArchList:
 200                         Current_Platform_cache = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
 201                         for Pkey in Current_Platform_cache.Modules:
 202                             MetaFile_cache[Arch].add(Current_Platform_cache.Modules[Pkey].MetaFile)
 203                     for Inf in self.FdfProfile.InfDict[key]:
 204                         ModuleFile = PathClass(NormPath(Inf), GlobalData.gWorkspace, Arch)
 205                         for Arch in self.ArchList:
 206                             if ModuleFile in MetaFile_cache[Arch]:
 207                                 break
 208                         else:
 209                             ModuleData = self.BuildDatabase[ModuleFile, Arch, self.BuildTarget, self.ToolChain]
 210                             if not ModuleData.IsBinaryModule:
 211                                 EdkLogger.error('build', PARSER_ERROR, "Module %s NOT found in DSC file; Is it really a binary module?" % ModuleFile)
 212
 213                 else:
 214                     for Arch in self.ArchList:
 215                         if Arch == key:
 216                             Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
 217                             MetaFileList = set()
 218                             for Pkey in Platform.Modules:
 219                                 MetaFileList.add(Platform.Modules[Pkey].MetaFile)
 220                             for Inf in self.FdfProfile.InfDict[key]:
 221                                 ModuleFile = PathClass(NormPath(Inf), GlobalData.gWorkspace, Arch)
 222                                 if ModuleFile in MetaFileList:
 223                                     continue
 224                                 ModuleData = self.BuildDatabase[ModuleFile, Arch, self.BuildTarget, self.ToolChain]
 225                                 if not ModuleData.IsBinaryModule:
 226                                     EdkLogger.error('build', PARSER_ERROR, "Module %s NOT found in DSC file; Is it really a binary module?" % ModuleFile)
 227
 228
 229
 230     # parse FDF file to get PCDs in it, if any
 231     def VerifyPcdsFromFDF(self):
 232
 233         if self.FdfProfile:
 234             PcdSet = self.FdfProfile.PcdDict
 235             self.VerifyPcdDeclearation(PcdSet)
 236
 237     def ProcessPcdType(self):
 238         for Arch in self.ArchList:
 239             Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
 240             Platform.Pcds
 241             # generate the SourcePcdDict and BinaryPcdDict
 242             Libs = []
 243             for BuildData in list(self.BuildDatabase._CACHE_.values()):
 244                 if BuildData.Arch != Arch:
 245                     continue
 246                 if BuildData.MetaFile.Ext == '.inf' and str(BuildData) in Platform.Modules :
 247                     Libs.extend(GetModuleLibInstances(BuildData, Platform,
 248                                      self.BuildDatabase,
 249                                      Arch,
 250                                      self.BuildTarget,
 251                                      self.ToolChain,
 252                                      self.Platform.MetaFile,
 253                                      EdkLogger
 254                                      ))
 255             for BuildData in list(self.BuildDatabase._CACHE_.values()):
 256                 if BuildData.Arch != Arch:
 257                     continue
 258                 if BuildData.MetaFile.Ext == '.inf':
 259                     for key in BuildData.Pcds:
 260                         if BuildData.Pcds[key].Pending:
 261                             if key in Platform.Pcds:
 262                                 PcdInPlatform = Platform.Pcds[key]
 263                                 if PcdInPlatform.Type:
 264                                     BuildData.Pcds[key].Type = PcdInPlatform.Type
 265                                     BuildData.Pcds[key].Pending = False
 266
 267                             if BuildData.MetaFile in Platform.Modules:
 268                                 PlatformModule = Platform.Modules[str(BuildData.MetaFile)]
 269                                 if key in PlatformModule.Pcds:
 270                                     PcdInPlatform = PlatformModule.Pcds[key]
 271                                     if PcdInPlatform.Type:
 272                                         BuildData.Pcds[key].Type = PcdInPlatform.Type
 273                                         BuildData.Pcds[key].Pending = False
 274                             else:
 275                                 #Pcd used in Library, Pcd Type from reference module if Pcd Type is Pending
 276                                 if BuildData.Pcds[key].Pending:
 277                                     if bool(BuildData.LibraryClass):
 278                                         if BuildData in set(Libs):
 279                                             ReferenceModules = BuildData.ReferenceModules
 280                                             for ReferenceModule in ReferenceModules:
 281                                                 if ReferenceModule.MetaFile in Platform.Modules:
 282                                                     RefPlatformModule = Platform.Modules[str(ReferenceModule.MetaFile)]
 283                                                     if key in RefPlatformModule.Pcds:
 284                                                         PcdInReferenceModule = RefPlatformModule.Pcds[key]
 285                                                         if PcdInReferenceModule.Type:
 286                                                             BuildData.Pcds[key].Type = PcdInReferenceModule.Type
 287                                                             BuildData.Pcds[key].Pending = False
 288                                                             break
 289
 290     def ProcessMixedPcd(self):
 291         for Arch in self.ArchList:
 292             SourcePcdDict = {TAB_PCDS_DYNAMIC_EX:set(), TAB_PCDS_PATCHABLE_IN_MODULE:set(),TAB_PCDS_DYNAMIC:set(),TAB_PCDS_FIXED_AT_BUILD:set()}
 293             BinaryPcdDict = {TAB_PCDS_DYNAMIC_EX:set(), TAB_PCDS_PATCHABLE_IN_MODULE:set()}
 294             SourcePcdDict_Keys = SourcePcdDict.keys()
 295             BinaryPcdDict_Keys = BinaryPcdDict.keys()
 296
 297             # generate the SourcePcdDict and BinaryPcdDict
 298
 299             for BuildData in list(self.BuildDatabase._CACHE_.values()):
 300                 if BuildData.Arch != Arch:
 301                     continue
 302                 if BuildData.MetaFile.Ext == '.inf':
 303                     for key in BuildData.Pcds:
 304                         if TAB_PCDS_DYNAMIC_EX in BuildData.Pcds[key].Type:
 305                             if BuildData.IsBinaryModule:
 306                                 BinaryPcdDict[TAB_PCDS_DYNAMIC_EX].add((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName))
 307                             else:
 308                                 SourcePcdDict[TAB_PCDS_DYNAMIC_EX].add((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName))
 309
 310                         elif TAB_PCDS_PATCHABLE_IN_MODULE in BuildData.Pcds[key].Type:
 311                             if BuildData.MetaFile.Ext == '.inf':
 312                                 if BuildData.IsBinaryModule:
 313                                     BinaryPcdDict[TAB_PCDS_PATCHABLE_IN_MODULE].add((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName))
 314                                 else:
 315                                     SourcePcdDict[TAB_PCDS_PATCHABLE_IN_MODULE].add((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName))
 316
 317                         elif TAB_PCDS_DYNAMIC in BuildData.Pcds[key].Type:
 318                             SourcePcdDict[TAB_PCDS_DYNAMIC].add((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName))
 319                         elif TAB_PCDS_FIXED_AT_BUILD in BuildData.Pcds[key].Type:
 320                             SourcePcdDict[TAB_PCDS_FIXED_AT_BUILD].add((BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName))
 321
 322             #
 323             # A PCD can only use one type for all source modules
 324             #
 325             for i in SourcePcdDict_Keys:
 326                 for j in SourcePcdDict_Keys:
 327                     if i != j:
 328                         Intersections = SourcePcdDict[i].intersection(SourcePcdDict[j])
 329                         if len(Intersections) > 0:
 330                             EdkLogger.error(
 331                             'build',
 332                             FORMAT_INVALID,
 333                             "Building modules from source INFs, following PCD use %s and %s access method. It must be corrected to use only one access method." % (i, j),
 334                             ExtraData='\n\t'.join(str(P[1]+'.'+P[0]) for P in Intersections)
 335                             )
 336
 337             #
 338             # intersection the BinaryPCD for Mixed PCD
 339             #
 340             for i in BinaryPcdDict_Keys:
 341                 for j in BinaryPcdDict_Keys:
 342                     if i != j:
 343                         Intersections = BinaryPcdDict[i].intersection(BinaryPcdDict[j])
 344                         for item in Intersections:
 345                             NewPcd1 = (item[0] + '_' + i, item[1])
 346                             NewPcd2 = (item[0] + '_' + j, item[1])
 347                             if item not in GlobalData.MixedPcd:
 348                                 GlobalData.MixedPcd[item] = [NewPcd1, NewPcd2]
 349                             else:
 350                                 if NewPcd1 not in GlobalData.MixedPcd[item]:
 351                                     GlobalData.MixedPcd[item].append(NewPcd1)
 352                                 if NewPcd2 not in GlobalData.MixedPcd[item]:
 353                                     GlobalData.MixedPcd[item].append(NewPcd2)
 354
 355             #
 356             # intersection the SourcePCD and BinaryPCD for Mixed PCD
 357             #
 358             for i in SourcePcdDict_Keys:
 359                 for j in BinaryPcdDict_Keys:
 360                     if i != j:
 361                         Intersections = SourcePcdDict[i].intersection(BinaryPcdDict[j])
 362                         for item in Intersections:
 363                             NewPcd1 = (item[0] + '_' + i, item[1])
 364                             NewPcd2 = (item[0] + '_' + j, item[1])
 365                             if item not in GlobalData.MixedPcd:
 366                                 GlobalData.MixedPcd[item] = [NewPcd1, NewPcd2]
 367                             else:
 368                                 if NewPcd1 not in GlobalData.MixedPcd[item]:
 369                                     GlobalData.MixedPcd[item].append(NewPcd1)
 370                                 if NewPcd2 not in GlobalData.MixedPcd[item]:
 371                                     GlobalData.MixedPcd[item].append(NewPcd2)
 372
 373             BuildData = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
 374             for key in BuildData.Pcds:
 375                 for SinglePcd in GlobalData.MixedPcd:
 376                     if (BuildData.Pcds[key].TokenCName, BuildData.Pcds[key].TokenSpaceGuidCName) == SinglePcd:
 377                         for item in GlobalData.MixedPcd[SinglePcd]:
 378                             Pcd_Type = item[0].split('_')[-1]
 379                             if (Pcd_Type == BuildData.Pcds[key].Type) or (Pcd_Type == TAB_PCDS_DYNAMIC_EX and BuildData.Pcds[key].Type in PCD_DYNAMIC_EX_TYPE_SET) or \
 380                                (Pcd_Type == TAB_PCDS_DYNAMIC and BuildData.Pcds[key].Type in PCD_DYNAMIC_TYPE_SET):
 381                                 Value = BuildData.Pcds[key]
 382                                 Value.TokenCName = BuildData.Pcds[key].TokenCName + '_' + Pcd_Type
 383                                 if len(key) == 2:
 384                                     newkey = (Value.TokenCName, key[1])
 385                                 elif len(key) == 3:
 386                                     newkey = (Value.TokenCName, key[1], key[2])
 387                                 del BuildData.Pcds[key]
 388                                 BuildData.Pcds[newkey] = Value
 389                                 break
 390                         break
 391
 392         if self.FdfProfile:
 393             PcdSet = self.FdfProfile.PcdDict
 394             # handle the mixed pcd in FDF file
 395             for key in PcdSet:
 396                 if key in GlobalData.MixedPcd:
 397                     Value = PcdSet[key]
 398                     del PcdSet[key]
 399                     for item in GlobalData.MixedPcd[key]:
 400                         PcdSet[item] = Value
 401
 402     #Collect package set information from INF of FDF
 403     @cached_property
 404     def PkgSet(self):
 405         if not self.FdfFile:
 406             self.FdfFile = self.Platform.FlashDefinition
 407
 408         if self.FdfFile:
 409             ModuleList = self.FdfProfile.InfList
 410         else:
 411             ModuleList = []
 412         Pkgs = {}
 413         for Arch in self.ArchList:
 414             Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
 415             PkgSet = set()
 416             for mb in [self.BuildDatabase[m, Arch, self.BuildTarget, self.ToolChain] for m in Platform.Modules]:
 417                 PkgSet.update(mb.Packages)
 418             for Inf in ModuleList:
 419                 ModuleFile = PathClass(NormPath(Inf), GlobalData.gWorkspace, Arch)
 420                 if ModuleFile in Platform.Modules:
 421                     continue
 422                 ModuleData = self.BuildDatabase[ModuleFile, Arch, self.BuildTarget, self.ToolChain]
 423                 PkgSet.update(ModuleData.Packages)
 424             PkgSet.update(Platform.Packages)
 425             Pkgs[Arch] = list(PkgSet)
 426         return Pkgs
 427
 428     def VerifyPcdDeclearation(self,PcdSet):
 429         for Arch in self.ArchList:
 430             Platform = self.BuildDatabase[self.MetaFile, Arch, self.BuildTarget, self.ToolChain]
 431             Pkgs = self.PkgSet[Arch]
 432             DecPcds = set()
 433             DecPcdsKey = set()
 434             for Pkg in Pkgs:
 435                 for Pcd in Pkg.Pcds:
 436                     DecPcds.add((Pcd[0], Pcd[1]))
 437                     DecPcdsKey.add((Pcd[0], Pcd[1], Pcd[2]))
 438
 439             Platform.SkuName = self.SkuId
 440             for Name, Guid,Fileds in PcdSet:
 441                 if (Name, Guid) not in DecPcds:
 442                     EdkLogger.error(
 443                         'build',
 444                         PARSER_ERROR,
 445                         "PCD (%s.%s) used in FDF is not declared in DEC files." % (Guid, Name),
 446                         File = self.FdfProfile.PcdFileLineDict[Name, Guid, Fileds][0],
 447                         Line = self.FdfProfile.PcdFileLineDict[Name, Guid, Fileds][1]
 448                     )
 449                 else:
 450                     # Check whether Dynamic or DynamicEx PCD used in FDF file. If used, build break and give a error message.
 451                     if (Name, Guid, TAB_PCDS_FIXED_AT_BUILD) in DecPcdsKey \
 452                         or (Name, Guid, TAB_PCDS_PATCHABLE_IN_MODULE) in DecPcdsKey \
 453                         or (Name, Guid, TAB_PCDS_FEATURE_FLAG) in DecPcdsKey:
 454                         continue
 455                     elif (Name, Guid, TAB_PCDS_DYNAMIC) in DecPcdsKey or (Name, Guid, TAB_PCDS_DYNAMIC_EX) in DecPcdsKey:
 456                         EdkLogger.error(
 457                                 'build',
 458                                 PARSER_ERROR,
 459                                 "Using Dynamic or DynamicEx type of PCD [%s.%s] in FDF file is not allowed." % (Guid, Name),
 460                                 File = self.FdfProfile.PcdFileLineDict[Name, Guid, Fileds][0],
 461                                 Line = self.FdfProfile.PcdFileLineDict[Name, Guid, Fileds][1]
 462                         )
 463     def CollectAllPcds(self):
 464
 465         for Arch in self.ArchList:
 466             Pa = PlatformAutoGen(self, self.MetaFile, self.BuildTarget, self.ToolChain, Arch)
 467             #
 468             # Explicitly collect platform's dynamic PCDs
 469             #
 470             Pa.CollectPlatformDynamicPcds()
 471             Pa.CollectFixedAtBuildPcds()
 472             self.AutoGenObjectList.append(Pa)
 473         # We need to calculate the PcdTokenNumber after all Arch Pcds are collected.
 474         for Arch in self.ArchList:
 475             #Pcd TokenNumber
 476             Pa = PlatformAutoGen(self, self.MetaFile, self.BuildTarget, self.ToolChain, Arch)
 477             self.UpdateModuleDataPipe(Arch,  {"PCD_TNUM":Pa.PcdTokenNumber})
 478
 479     def UpdateModuleDataPipe(self,arch, attr_dict):
 480         for (Target, Toolchain, Arch, MetaFile) in AutoGen.Cache():
 481             if Arch != arch:
 482                 continue
 483             try:
 484                 AutoGen.Cache()[(Target, Toolchain, Arch, MetaFile)].DataPipe.DataContainer = attr_dict
 485             except Exception:
 486                 pass
 487     #
 488     # Generate Package level hash value
 489     #
 490     def GeneratePkgLevelHash(self):
 491         for Arch in self.ArchList:
 492             GlobalData.gPackageHash = {}
 493             if GlobalData.gUseHashCache:
 494                 for Pkg in self.PkgSet[Arch]:
 495                     self._GenPkgLevelHash(Pkg)
 496
 497
 498     def CreateBuildOptionsFile(self):
 499         #
 500         # Create BuildOptions Macro & PCD metafile, also add the Active Platform and FDF file.
 501         #
 502         content = 'gCommandLineDefines: '
 503         content += str(GlobalData.gCommandLineDefines)
 504         content += TAB_LINE_BREAK
 505         content += 'BuildOptionPcd: '
 506         content += str(GlobalData.BuildOptionPcd)
 507         content += TAB_LINE_BREAK
 508         content += 'Active Platform: '
 509         content += str(self.Platform)
 510         content += TAB_LINE_BREAK
 511         if self.FdfFile:
 512             content += 'Flash Image Definition: '
 513             content += str(self.FdfFile)
 514             content += TAB_LINE_BREAK
 515         SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False)
 516
 517     def CreatePcdTokenNumberFile(self):
 518         #
 519         # Create PcdToken Number file for Dynamic/DynamicEx Pcd.
 520         #
 521         PcdTokenNumber = 'PcdTokenNumber: '
 522         Pa = self.AutoGenObjectList[0]
 523         if Pa.PcdTokenNumber:
 524             if Pa.DynamicPcdList:
 525                 for Pcd in Pa.DynamicPcdList:
 526                     PcdTokenNumber += TAB_LINE_BREAK
 527                     PcdTokenNumber += str((Pcd.TokenCName, Pcd.TokenSpaceGuidCName))
 528                     PcdTokenNumber += ' : '
 529                     PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])
 530         SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False)
 531
 532     def GeneratePlatformLevelHash(self):
 533         #
 534         # Get set of workspace metafiles
 535         #
 536         AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain)
 537         AllWorkSpaceMetaFileList = sorted(AllWorkSpaceMetaFiles, key=lambda x: str(x))
 538         #
 539         # Retrieve latest modified time of all metafiles
 540         #
 541         SrcTimeStamp = 0
 542         for f in AllWorkSpaceMetaFiles:
 543             if os.stat(f)[8] > SrcTimeStamp:
 544                 SrcTimeStamp = os.stat(f)[8]
 545         self._SrcTimeStamp = SrcTimeStamp
 546
 547         if GlobalData.gUseHashCache:
 548             FileList = []
 549             m = hashlib.md5()
 550             for file in AllWorkSpaceMetaFileList:
 551                 if file.endswith('.dec'):
 552                     continue
 553                 f = open(file, 'rb')
 554                 Content = f.read()
 555                 f.close()
 556                 m.update(Content)
 557                 FileList.append((str(file), hashlib.md5(Content).hexdigest()))
 558
 559             HashDir = path.join(self.BuildDir, "Hash_Platform")
 560             HashFile = path.join(HashDir, 'Platform.hash.' + m.hexdigest())
 561             SaveFileOnChange(HashFile, m.hexdigest(), False)
 562             HashChainFile = path.join(HashDir, 'Platform.hashchain.' + m.hexdigest())
 563             GlobalData.gPlatformHashFile = HashChainFile
 564             try:
 565                 with open(HashChainFile, 'w') as f:
 566                     json.dump(FileList, f, indent=2)
 567             except:
 568                 EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
 569
 570             if GlobalData.gBinCacheDest:
 571                 # Copy platform hash files to cache destination
 572                 FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, "Hash_Platform")
 573                 CacheFileDir = FileDir
 574                 CreateDirectory(CacheFileDir)
 575                 CopyFileOnChange(HashFile, CacheFileDir)
 576                 CopyFileOnChange(HashChainFile, CacheFileDir)
 577
 578         #
 579         # Write metafile list to build directory
 580         #
 581         AutoGenFilePath = os.path.join(self.BuildDir, 'AutoGen')
 582         if os.path.exists (AutoGenFilePath):
 583             os.remove(AutoGenFilePath)
 584         if not os.path.exists(self.BuildDir):
 585             os.makedirs(self.BuildDir)
 586         with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:
 587             for f in AllWorkSpaceMetaFileList:
 588                 print(f, file=file)
 589         return True
 590
 591     def _GenPkgLevelHash(self, Pkg):
 592         if Pkg.PackageName in GlobalData.gPackageHash:
 593             return
 594
 595         PkgDir = os.path.join(self.BuildDir, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
 596         CreateDirectory(PkgDir)
 597         FileList = []
 598         m = hashlib.md5()
 599         # Get .dec file's hash value
 600         f = open(Pkg.MetaFile.Path, 'rb')
 601         Content = f.read()
 602         f.close()
 603         m.update(Content)
 604         FileList.append((str(Pkg.MetaFile.Path), hashlib.md5(Content).hexdigest()))
 605         # Get include files hash value
 606         if Pkg.Includes:
 607             for inc in sorted(Pkg.Includes, key=lambda x: str(x)):
 608                 for Root, Dirs, Files in os.walk(str(inc)):
 609                     for File in sorted(Files):
 610                         File_Path = os.path.join(Root, File)
 611                         f = open(File_Path, 'rb')
 612                         Content = f.read()
 613                         f.close()
 614                         m.update(Content)
 615                         FileList.append((str(File_Path), hashlib.md5(Content).hexdigest()))
 616         GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()
 617
 618         HashDir = PkgDir
 619         HashFile = path.join(HashDir, Pkg.PackageName + '.hash.' + m.hexdigest())
 620         SaveFileOnChange(HashFile, m.hexdigest(), False)
 621         HashChainFile = path.join(HashDir, Pkg.PackageName + '.hashchain.' + m.hexdigest())
 622         GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] = HashChainFile
 623         try:
 624             with open(HashChainFile, 'w') as f:
 625                 json.dump(FileList, f, indent=2)
 626         except:
 627             EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
 628
 629         if GlobalData.gBinCacheDest:
 630             # Copy Pkg hash files to cache destination dir
 631             FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
 632             CacheFileDir = FileDir
 633             CreateDirectory(CacheFileDir)
 634             CopyFileOnChange(HashFile, CacheFileDir)
 635             CopyFileOnChange(HashChainFile, CacheFileDir)
 636
 637     def _GetMetaFiles(self, Target, Toolchain):
 638         AllWorkSpaceMetaFiles = set()
 639         #
 640         # add fdf
 641         #
 642         if self.FdfFile:
 643             AllWorkSpaceMetaFiles.add (self.FdfFile.Path)
 644             for f in GlobalData.gFdfParser.GetAllIncludedFile():
 645                 AllWorkSpaceMetaFiles.add (f.FileName)
 646         #
 647         # add dsc
 648         #
 649         AllWorkSpaceMetaFiles.add(self.MetaFile.Path)
 650
 651         #
 652         # add build_rule.txt & tools_def.txt
 653         #
 654         AllWorkSpaceMetaFiles.add(os.path.join(GlobalData.gConfDirectory, gDefaultBuildRuleFile))
 655         AllWorkSpaceMetaFiles.add(os.path.join(GlobalData.gConfDirectory, gDefaultToolsDefFile))
 656
 657         # add BuildOption metafile
 658         #
 659         AllWorkSpaceMetaFiles.add(os.path.join(self.BuildDir, 'BuildOptions'))
 660
 661         # add PcdToken Number file for Dynamic/DynamicEx Pcd
 662         #
 663         AllWorkSpaceMetaFiles.add(os.path.join(self.BuildDir, 'PcdTokenNumber'))
 664
 665         for Pa in self.AutoGenObjectList:
 666             AllWorkSpaceMetaFiles.add(Pa.ToolDefinitionFile)
 667
 668         for Arch in self.ArchList:
 669             #
 670             # add dec
 671             #
 672             for Package in PlatformAutoGen(self, self.MetaFile, Target, Toolchain, Arch).PackageList:
 673                 AllWorkSpaceMetaFiles.add(Package.MetaFile.Path)
 674
 675             #
 676             # add included dsc
 677             #
 678             for filePath in self.BuildDatabase[self.MetaFile, Arch, Target, Toolchain]._RawData.IncludedFiles:
 679                 AllWorkSpaceMetaFiles.add(filePath.Path)
 680
 681         return AllWorkSpaceMetaFiles
 682
 683     def _CheckPcdDefineAndType(self):
 684         PcdTypeSet = {TAB_PCDS_FIXED_AT_BUILD,
 685             TAB_PCDS_PATCHABLE_IN_MODULE,
 686             TAB_PCDS_FEATURE_FLAG,
 687             TAB_PCDS_DYNAMIC,
 688             TAB_PCDS_DYNAMIC_EX}
 689
 690         # This dict store PCDs which are not used by any modules with specified arches
 691         UnusedPcd = OrderedDict()
 692         for Pa in self.AutoGenObjectList:
 693             # Key of DSC's Pcds dictionary is PcdCName, TokenSpaceGuid
 694             for Pcd in Pa.Platform.Pcds:
 695                 PcdType = Pa.Platform.Pcds[Pcd].Type
 696
 697                 # If no PCD type, this PCD comes from FDF
 698                 if not PcdType:
 699                     continue
 700
 701                 # Try to remove Hii and Vpd suffix
 702                 if PcdType.startswith(TAB_PCDS_DYNAMIC_EX):
 703                     PcdType = TAB_PCDS_DYNAMIC_EX
 704                 elif PcdType.startswith(TAB_PCDS_DYNAMIC):
 705                     PcdType = TAB_PCDS_DYNAMIC
 706
 707                 for Package in Pa.PackageList:
 708                     # Key of DEC's Pcds dictionary is PcdCName, TokenSpaceGuid, PcdType
 709                     if (Pcd[0], Pcd[1], PcdType) in Package.Pcds:
 710                         break
 711                     for Type in PcdTypeSet:
 712                         if (Pcd[0], Pcd[1], Type) in Package.Pcds:
 713                             EdkLogger.error(
 714                                 'build',
 715                                 FORMAT_INVALID,
 716                                 "Type [%s] of PCD [%s.%s] in DSC file doesn't match the type [%s] defined in DEC file." \
 717                                 % (Pa.Platform.Pcds[Pcd].Type, Pcd[1], Pcd[0], Type),
 718                                 ExtraData=None
 719                             )
 720                             return
 721                 else:
 722                     UnusedPcd.setdefault(Pcd, []).append(Pa.Arch)
 723
 724         for Pcd in UnusedPcd:
 725             EdkLogger.warn(
 726                 'build',
 727                 "The PCD was not specified by any INF module in the platform for the given architecture.\n"
 728                 "\tPCD: [%s.%s]\n\tPlatform: [%s]\n\tArch: %s"
 729                 % (Pcd[1], Pcd[0], os.path.basename(str(self.MetaFile)), str(UnusedPcd[Pcd])),
 730                 ExtraData=None
 731             )
 732
 733     def __repr__(self):
 734         return "%s [%s]" % (self.MetaFile, ", ".join(self.ArchList))
 735
 736     ## Return the directory to store FV files
 737     @cached_property
 738     def FvDir(self):
 739         return path.join(self.BuildDir, TAB_FV_DIRECTORY)
 740
 741     ## Return the directory to store all intermediate and final files built
 742     @cached_property
 743     def BuildDir(self):
 744         return self.AutoGenObjectList[0].BuildDir
 745
 746     ## Return the build output directory platform specifies
 747     @cached_property
 748     def OutputDir(self):
 749         return self.Platform.OutputDirectory
 750
 751     ## Return platform name
 752     @cached_property
 753     def Name(self):
 754         return self.Platform.PlatformName
 755
 756     ## Return meta-file GUID
 757     @cached_property
 758     def Guid(self):
 759         return self.Platform.Guid
 760
 761     ## Return platform version
 762     @cached_property
 763     def Version(self):
 764         return self.Platform.Version
 765
 766     ## Return paths of tools
 767     @cached_property
 768     def ToolDefinition(self):
 769         return self.AutoGenObjectList[0].ToolDefinition
 770
 771     ## Return directory of platform makefile
 772     #
 773     #   @retval     string  Makefile directory
 774     #
 775     @cached_property
 776     def MakeFileDir(self):
 777         return self.BuildDir
 778
 779     ## Return build command string
 780     #
 781     #   @retval     string  Build command string
 782     #
 783     @cached_property
 784     def BuildCommand(self):
 785         # BuildCommand should be all the same. So just get one from platform AutoGen
 786         return self.AutoGenObjectList[0].BuildCommand
 787
 788     ## Check the PCDs token value conflict in each DEC file.
 789     #
 790     # Will cause build break and raise error message while two PCDs conflict.
 791     #
 792     # @return  None
 793     #
 794     def _CheckAllPcdsTokenValueConflict(self):
 795         for Pa in self.AutoGenObjectList:
 796             for Package in Pa.PackageList:
 797                 PcdList = list(Package.Pcds.values())
 798                 PcdList.sort(key=lambda x: int(x.TokenValue, 0))
 799                 Count = 0
 800                 while (Count < len(PcdList) - 1) :
 801                     Item = PcdList[Count]
 802                     ItemNext = PcdList[Count + 1]
 803                     #
 804                     # Make sure in the same token space the TokenValue should be unique
 805                     #
 806                     if (int(Item.TokenValue, 0) == int(ItemNext.TokenValue, 0)):
 807                         SameTokenValuePcdList = []
 808                         SameTokenValuePcdList.append(Item)
 809                         SameTokenValuePcdList.append(ItemNext)
 810                         RemainPcdListLength = len(PcdList) - Count - 2
 811                         for ValueSameCount in range(RemainPcdListLength):
 812                             if int(PcdList[len(PcdList) - RemainPcdListLength + ValueSameCount].TokenValue, 0) == int(Item.TokenValue, 0):
 813                                 SameTokenValuePcdList.append(PcdList[len(PcdList) - RemainPcdListLength + ValueSameCount])
 814                             else:
 815                                 break;
 816                         #
 817                         # Sort same token value PCD list with TokenGuid and TokenCName
 818                         #
 819                         SameTokenValuePcdList.sort(key=lambda x: "%s.%s" % (x.TokenSpaceGuidCName, x.TokenCName))
 820                         SameTokenValuePcdListCount = 0
 821                         while (SameTokenValuePcdListCount < len(SameTokenValuePcdList) - 1):
 822                             Flag = False
 823                             TemListItem = SameTokenValuePcdList[SameTokenValuePcdListCount]
 824                             TemListItemNext = SameTokenValuePcdList[SameTokenValuePcdListCount + 1]
 825
 826                             if (TemListItem.TokenSpaceGuidCName == TemListItemNext.TokenSpaceGuidCName) and (TemListItem.TokenCName != TemListItemNext.TokenCName):
 827                                 for PcdItem in GlobalData.MixedPcd:
 828                                     if (TemListItem.TokenCName, TemListItem.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem] or \
 829                                         (TemListItemNext.TokenCName, TemListItemNext.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]:
 830                                         Flag = True
 831                                 if not Flag:
 832                                     EdkLogger.error(
 833                                                 'build',
 834                                                 FORMAT_INVALID,
 835                                                 "The TokenValue [%s] of PCD [%s.%s] is conflict with: [%s.%s] in %s"\
 836                                                 % (TemListItem.TokenValue, TemListItem.TokenSpaceGuidCName, TemListItem.TokenCName, TemListItemNext.TokenSpaceGuidCName, TemListItemNext.TokenCName, Package),
 837                                                 ExtraData=None
 838                                                 )
 839                             SameTokenValuePcdListCount += 1
 840                         Count += SameTokenValuePcdListCount
 841                     Count += 1
 842
 843                 PcdList = list(Package.Pcds.values())
 844                 PcdList.sort(key=lambda x: "%s.%s" % (x.TokenSpaceGuidCName, x.TokenCName))
 845                 Count = 0
 846                 while (Count < len(PcdList) - 1) :
 847                     Item = PcdList[Count]
 848                     ItemNext = PcdList[Count + 1]
 849                     #
 850                     # Check PCDs with same TokenSpaceGuidCName.TokenCName have same token value as well.
 851                     #
 852                     if (Item.TokenSpaceGuidCName == ItemNext.TokenSpaceGuidCName) and (Item.TokenCName == ItemNext.TokenCName) and (int(Item.TokenValue, 0) != int(ItemNext.TokenValue, 0)):
 853                         EdkLogger.error(
 854                                     'build',
 855                                     FORMAT_INVALID,
 856                                     "The TokenValue [%s] of PCD [%s.%s] in %s defined in two places should be same as well."\
 857                                     % (Item.TokenValue, Item.TokenSpaceGuidCName, Item.TokenCName, Package),
 858                                     ExtraData=None
 859                                     )
 860                     Count += 1
 861     ## Generate fds command
 862     @property
 863     def GenFdsCommand(self):
 864         return (GenMake.TopLevelMakefile(self)._TEMPLATE_.Replace(GenMake.TopLevelMakefile(self)._TemplateDict)).strip()
 865
 866     @property
 867     def GenFdsCommandDict(self):
 868         FdsCommandDict = {}
 869         LogLevel = EdkLogger.GetLevel()
 870         if LogLevel == EdkLogger.VERBOSE:
 871             FdsCommandDict["verbose"] = True
 872         elif LogLevel <= EdkLogger.DEBUG_9:
 873             FdsCommandDict["debug"] = LogLevel - 1
 874         elif LogLevel == EdkLogger.QUIET:
 875             FdsCommandDict["quiet"] = True
 876
 877         FdsCommandDict["GenfdsMultiThread"] = GlobalData.gEnableGenfdsMultiThread
 878         if GlobalData.gIgnoreSource:
 879             FdsCommandDict["IgnoreSources"] = True
 880
 881         FdsCommandDict["OptionPcd"] = []
 882         for pcd in GlobalData.BuildOptionPcd:
 883             if pcd[2]:
 884                 pcdname = '.'.join(pcd[0:3])
 885             else:
 886                 pcdname = '.'.join(pcd[0:2])
 887             if pcd[3].startswith('{'):
 888                 FdsCommandDict["OptionPcd"].append(pcdname + '=' + 'H' + '"' + pcd[3] + '"')
 889             else:
 890                 FdsCommandDict["OptionPcd"].append(pcdname + '=' + pcd[3])
 891
 892         MacroList = []
 893         # macros passed to GenFds
 894         MacroDict = {}
 895         MacroDict.update(GlobalData.gGlobalDefines)
 896         MacroDict.update(GlobalData.gCommandLineDefines)
 897         for MacroName in MacroDict:
 898             if MacroDict[MacroName] != "":
 899                 MacroList.append('"%s=%s"' % (MacroName, MacroDict[MacroName].replace('\\', '\\\\')))
 900             else:
 901                 MacroList.append('"%s"' % MacroName)
 902         FdsCommandDict["macro"] = MacroList
 903
 904         FdsCommandDict["fdf_file"] = [self.FdfFile]
 905         FdsCommandDict["build_target"] = self.BuildTarget
 906         FdsCommandDict["toolchain_tag"] = self.ToolChain
 907         FdsCommandDict["active_platform"] = str(self)
 908
 909         FdsCommandDict["conf_directory"] = GlobalData.gConfDirectory
 910         FdsCommandDict["build_architecture_list"] = ','.join(self.ArchList)
 911         FdsCommandDict["platform_build_directory"] = self.BuildDir
 912
 913         FdsCommandDict["fd"] = self.FdTargetList
 914         FdsCommandDict["fv"] = self.FvTargetList
 915         FdsCommandDict["cap"] = self.CapTargetList
 916         return FdsCommandDict
 917
 918     ## Create makefile for the platform and modules in it
 919     #
 920     #   @param      CreateDepsMakeFile      Flag indicating if the makefile for
 921     #                                       modules will be created as well
 922     #
 923     def CreateMakeFile(self, CreateDepsMakeFile=False):
 924         if not CreateDepsMakeFile:
 925             return
 926         for Pa in self.AutoGenObjectList:
 927             Pa.CreateMakeFile(CreateDepsMakeFile)
 928
 929     ## Create autogen code for platform and modules
 930     #
 931     #  Since there's no autogen code for platform, this method will do nothing
 932     #  if CreateModuleCodeFile is set to False.
 933     #
 934     #   @param      CreateDepsCodeFile      Flag indicating if creating module's
 935     #                                       autogen code file or not
 936     #
 937     def CreateCodeFile(self, CreateDepsCodeFile=False):
 938         if not CreateDepsCodeFile:
 939             return
 940         for Pa in self.AutoGenObjectList:
 941             Pa.CreateCodeFile(CreateDepsCodeFile)
 942
 943     ## Create AsBuilt INF file the platform
 944     #
 945     def CreateAsBuiltInf(self):
 946         return
 947