BaseTools/Source/Python/Common/ToolDefClassObject.py

   1 ## @file
   2 # This file is used to define each component of tools_def.txt file
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7
   8 ##
   9 # Import Modules
  10 #
  11 from __future__ import absolute_import
  12 import Common.LongFilePathOs as os
  13 import re
  14 from . import EdkLogger
  15
  16 from .BuildToolError import *
  17 from Common.TargetTxtClassObject import TargetTxtDict
  18 from Common.LongFilePathSupport import OpenLongFilePath as open
  19 from Common.Misc import PathClass
  20 from Common.StringUtils import NormPath
  21 import Common.GlobalData as GlobalData
  22 from Common import GlobalData
  23 from Common.MultipleWorkspace import MultipleWorkspace as mws
  24 from .DataType import TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG,\
  25                      TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE\
  26                      , TAB_TOD_DEFINES_FAMILY, TAB_TOD_DEFINES_BUILDRULEFAMILY,\
  27                      TAB_STAR, TAB_TAT_DEFINES_TOOL_CHAIN_CONF
  28
  29
  30 ##
  31 # Static variables used for pattern
  32 #
  33 gMacroRefPattern = re.compile('(DEF\([^\(\)]+\))')
  34 gEnvRefPattern = re.compile('(ENV\([^\(\)]+\))')
  35 gMacroDefPattern = re.compile("DEFINE\s+([^\s]+)")
  36 gDefaultToolsDefFile = "tools_def.txt"
  37
  38 ## ToolDefClassObject
  39 #
  40 # This class defined content used in file tools_def.txt
  41 #
  42 # @param object:               Inherited from object class
  43 # @param Filename:             Input value for full path of tools_def.txt
  44 #
  45 # @var ToolsDefTxtDictionary:  To store keys and values defined in target.txt
  46 # @var MacroDictionary:        To store keys and values defined in DEFINE statement
  47 #
  48 class ToolDefClassObject(object):
  49     def __init__(self, FileName=None):
  50         self.ToolsDefTxtDictionary = {}
  51         self.MacroDictionary = {}
  52         for Env in os.environ:
  53             self.MacroDictionary["ENV(%s)" % Env] = os.environ[Env]
  54
  55         if FileName is not None:
  56             self.LoadToolDefFile(FileName)
  57
  58     ## LoadToolDefFile
  59     #
  60     # Load target.txt file and parse it
  61     #
  62     # @param Filename:  Input value for full path of tools_def.txt
  63     #
  64     def LoadToolDefFile(self, FileName):
  65         # set multiple workspace
  66         PackagesPath = os.getenv("PACKAGES_PATH")
  67         mws.setWs(GlobalData.gWorkspace, PackagesPath)
  68
  69         self.ToolsDefTxtDatabase = {
  70             TAB_TOD_DEFINES_TARGET          :   [],
  71             TAB_TOD_DEFINES_TOOL_CHAIN_TAG  :   [],
  72             TAB_TOD_DEFINES_TARGET_ARCH     :   [],
  73             TAB_TOD_DEFINES_COMMAND_TYPE    :   []
  74         }
  75
  76         self.IncludeToolDefFile(FileName)
  77
  78         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET]))
  79         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]))
  80         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH]))
  81
  82         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE]))
  83
  84         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET].sort()
  85         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].sort()
  86         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH].sort()
  87         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE].sort()
  88
  89         KeyList = [TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG, TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE]
  90         for Index in range(3, -1, -1):
  91             # make a copy of the keys to enumerate over to prevent issues when
  92             # adding/removing items from the original dict.
  93             for Key in list(self.ToolsDefTxtDictionary.keys()):
  94                 List = Key.split('_')
  95                 if List[Index] == TAB_STAR:
  96                     for String in self.ToolsDefTxtDatabase[KeyList[Index]]:
  97                         List[Index] = String
  98                         NewKey = '%s_%s_%s_%s_%s' % tuple(List)
  99                         if NewKey not in self.ToolsDefTxtDictionary:
 100                             self.ToolsDefTxtDictionary[NewKey] = self.ToolsDefTxtDictionary[Key]
 101                     del self.ToolsDefTxtDictionary[Key]
 102                 elif List[Index] not in self.ToolsDefTxtDatabase[KeyList[Index]]:
 103                     del self.ToolsDefTxtDictionary[Key]
 104
 105
 106     ## IncludeToolDefFile
 107     #
 108     # Load target.txt file and parse it as if it's contents were inside the main file
 109     #
 110     # @param Filename:  Input value for full path of tools_def.txt
 111     #
 112     def IncludeToolDefFile(self, FileName):
 113         FileContent = []
 114         if os.path.isfile(FileName):
 115             try:
 116                 F = open(FileName, 'r')
 117                 FileContent = F.readlines()
 118             except:
 119                 EdkLogger.error("tools_def.txt parser", FILE_OPEN_FAILURE, ExtraData=FileName)
 120         else:
 121             EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=FileName)
 122
 123         for Index in range(len(FileContent)):
 124             Line = FileContent[Index].strip()
 125             if Line == "" or Line[0] == '#':
 126                 continue
 127
 128             if Line.startswith("!include"):
 129                 IncFile = Line[8:].strip()
 130                 Done, IncFile = self.ExpandMacros(IncFile)
 131                 if not Done:
 132                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 133                                     "Macro or Environment has not been defined",
 134                                 ExtraData=IncFile[4:-1], File=FileName, Line=Index+1)
 135                 IncFile = NormPath(IncFile)
 136
 137                 if not os.path.isabs(IncFile):
 138                     #
 139                     # try WORKSPACE
 140                     #
 141                     IncFileTmp = PathClass(IncFile, GlobalData.gWorkspace)
 142                     ErrorCode = IncFileTmp.Validate()[0]
 143                     if ErrorCode != 0:
 144                         #
 145                         # try PACKAGES_PATH
 146                         #
 147                         IncFileTmp = mws.join(GlobalData.gWorkspace, IncFile)
 148                         if not os.path.exists(IncFileTmp):
 149                             #
 150                             # try directory of current file
 151                             #
 152                             IncFileTmp = PathClass(IncFile, os.path.dirname(FileName))
 153                             ErrorCode = IncFileTmp.Validate()[0]
 154                             if ErrorCode != 0:
 155                                 EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=IncFile)
 156
 157                     if isinstance(IncFileTmp, PathClass):
 158                         IncFile = IncFileTmp.Path
 159                     else:
 160                         IncFile = IncFileTmp
 161
 162                 self.IncludeToolDefFile(IncFile)
 163                 continue
 164
 165             NameValuePair = Line.split("=", 1)
 166             if len(NameValuePair) != 2:
 167                 EdkLogger.warn("tools_def.txt parser", "Line %d: not correct assignment statement, skipped" % (Index + 1))
 168                 continue
 169
 170             Name = NameValuePair[0].strip()
 171             Value = NameValuePair[1].strip()
 172
 173             if Name == "IDENTIFIER":
 174                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found identifier statement, skipped: %s" % ((Index + 1), Value))
 175                 continue
 176
 177             MacroDefinition = gMacroDefPattern.findall(Name)
 178             if MacroDefinition != []:
 179                 Done, Value = self.ExpandMacros(Value)
 180                 if not Done:
 181                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 182                                     "Macro or Environment has not been defined",
 183                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 184
 185                 MacroName = MacroDefinition[0].strip()
 186                 self.MacroDictionary["DEF(%s)" % MacroName] = Value
 187                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found macro: %s = %s" % ((Index + 1), MacroName, Value))
 188                 continue
 189
 190             Done, Value = self.ExpandMacros(Value)
 191             if not Done:
 192                 EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 193                                 "Macro or Environment has not been defined",
 194                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 195
 196             List = Name.split('_')
 197             if len(List) != 5:
 198                 EdkLogger.verbose("Line %d: Not a valid name of definition: %s" % ((Index + 1), Name))
 199                 continue
 200             elif List[4] == TAB_STAR:
 201                 EdkLogger.verbose("Line %d: '*' is not allowed in last field: %s" % ((Index + 1), Name))
 202                 continue
 203             else:
 204                 self.ToolsDefTxtDictionary[Name] = Value
 205                 if List[0] != TAB_STAR:
 206                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] += [List[0]]
 207                 if List[1] != TAB_STAR:
 208                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] += [List[1]]
 209                 if List[2] != TAB_STAR:
 210                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] += [List[2]]
 211                 if List[3] != TAB_STAR:
 212                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] += [List[3]]
 213                 if List[4] == TAB_TOD_DEFINES_FAMILY and List[2] == TAB_STAR and List[3] == TAB_STAR:
 214                     if TAB_TOD_DEFINES_FAMILY not in self.ToolsDefTxtDatabase:
 215                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
 216                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 217                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
 218                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 219                     elif List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 220                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 221                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 222                     elif self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] != Value:
 223                         EdkLogger.verbose("Line %d: No override allowed for the family of a tool chain: %s" % ((Index + 1), Name))
 224                 if List[4] == TAB_TOD_DEFINES_BUILDRULEFAMILY and List[2] == TAB_STAR and List[3] == TAB_STAR:
 225                     if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolsDefTxtDatabase \
 226                        or List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 227                         EdkLogger.verbose("Line %d: The family is not specified, but BuildRuleFamily is specified for the tool chain: %s" % ((Index + 1), Name))
 228                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 229
 230     ## ExpandMacros
 231     #
 232     # Replace defined macros with real value
 233     #
 234     # @param Value:   The string with unreplaced macros
 235     #
 236     # @retval Value:  The string which has been replaced with real value
 237     #
 238     def ExpandMacros(self, Value):
 239         # os.environ contains all environment variables uppercase on Windows which cause the key in the self.MacroDictionary is uppercase, but Ref may not
 240         EnvReference = gEnvRefPattern.findall(Value)
 241         for Ref in EnvReference:
 242             if Ref not in self.MacroDictionary and Ref.upper() not in self.MacroDictionary:
 243                 Value = Value.replace(Ref, "")
 244             else:
 245                 if Ref in self.MacroDictionary:
 246                     Value = Value.replace(Ref, self.MacroDictionary[Ref])
 247                 else:
 248                     Value = Value.replace(Ref, self.MacroDictionary[Ref.upper()])
 249         MacroReference = gMacroRefPattern.findall(Value)
 250         for Ref in MacroReference:
 251             if Ref not in self.MacroDictionary:
 252                 return False, Ref
 253             Value = Value.replace(Ref, self.MacroDictionary[Ref])
 254
 255         return True, Value
 256
 257 ## ToolDefDict
 258 #
 259 # Load tools_def.txt in input Conf dir
 260 #
 261 # @param ConfDir:  Conf dir
 262 #
 263 # @retval ToolDef An instance of ToolDefClassObject() with loaded tools_def.txt
 264 #
 265 def ToolDefDict(ConfDir):
 266     Target = TargetTxtDict(ConfDir)
 267     ToolDef = ToolDefClassObject()
 268     if TAB_TAT_DEFINES_TOOL_CHAIN_CONF in Target.TargetTxtDictionary:
 269         ToolsDefFile = Target.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 270         if ToolsDefFile:
 271             ToolDef.LoadToolDefFile(os.path.normpath(ToolsDefFile))
 272         else:
 273             ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 274     else:
 275         ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 276     return ToolDef
 277
 278 ##
 279 #
 280 # This acts like the main() function for the script, unless it is 'import'ed into another
 281 # script.
 282 #
 283 if __name__ == '__main__':
 284     ToolDef = ToolDefDict(os.getenv("WORKSPACE"))
 285     pass