BaseTools/Source/Python/Common/ToolDefClassObject.py

   1 ## @file
   2 # This file is used to define each component of tools_def.txt file
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import Common.LongFilePathOs as os
  18 import re
  19 import EdkLogger
  20
  21 from BuildToolError import *
  22 from TargetTxtClassObject import *
  23 from Common.LongFilePathSupport import OpenLongFilePath as open
  24 from Common.Misc import PathClass
  25 from Common.StringUtils import NormPath
  26 import Common.GlobalData as GlobalData
  27 from Common import GlobalData
  28 from Common.MultipleWorkspace import MultipleWorkspace as mws
  29 from DataType import TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG,\
  30                      TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE\
  31                      , TAB_TOD_DEFINES_FAMILY, TAB_TOD_DEFINES_BUILDRULEFAMILY
  32
  33
  34 ##
  35 # Static variables used for pattern
  36 #
  37 gMacroRefPattern = re.compile('(DEF\([^\(\)]+\))')
  38 gEnvRefPattern = re.compile('(ENV\([^\(\)]+\))')
  39 gMacroDefPattern = re.compile("DEFINE\s+([^\s]+)")
  40 gDefaultToolsDefFile = "tools_def.txt"
  41
  42 ## ToolDefClassObject
  43 #
  44 # This class defined content used in file tools_def.txt
  45 #
  46 # @param object:               Inherited from object class
  47 # @param Filename:             Input value for full path of tools_def.txt
  48 #
  49 # @var ToolsDefTxtDictionary:  To store keys and values defined in target.txt
  50 # @var MacroDictionary:        To store keys and values defined in DEFINE statement
  51 #
  52 class ToolDefClassObject(object):
  53     def __init__(self, FileName=None):
  54         self.ToolsDefTxtDictionary = {}
  55         self.MacroDictionary = {}
  56         for Env in os.environ:
  57             self.MacroDictionary["ENV(%s)" % Env] = os.environ[Env]
  58
  59         if FileName is not None:
  60             self.LoadToolDefFile(FileName)
  61
  62     ## LoadToolDefFile
  63     #
  64     # Load target.txt file and parse it
  65     #
  66     # @param Filename:  Input value for full path of tools_def.txt
  67     #
  68     def LoadToolDefFile(self, FileName):
  69         # set multiple workspace
  70         PackagesPath = os.getenv("PACKAGES_PATH")
  71         mws.setWs(GlobalData.gWorkspace, PackagesPath)
  72
  73         self.ToolsDefTxtDatabase = {
  74             TAB_TOD_DEFINES_TARGET          :   [],
  75             TAB_TOD_DEFINES_TOOL_CHAIN_TAG  :   [],
  76             TAB_TOD_DEFINES_TARGET_ARCH     :   [],
  77             TAB_TOD_DEFINES_COMMAND_TYPE    :   []
  78         }
  79
  80         self.IncludeToolDefFile(FileName)
  81
  82         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET]))
  83         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]))
  84         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH]))
  85
  86         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE]))
  87
  88         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET].sort()
  89         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].sort()
  90         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH].sort()
  91         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE].sort()
  92
  93         KeyList = [TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG, TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE]
  94         for Index in range(3, -1, -1):
  95             # make a copy of the keys to enumerate over to prevent issues when
  96             # adding/removing items from the original dict.
  97             for Key in list(self.ToolsDefTxtDictionary.keys()):
  98                 List = Key.split('_')
  99                 if List[Index] == '*':
 100                     for String in self.ToolsDefTxtDatabase[KeyList[Index]]:
 101                         List[Index] = String
 102                         NewKey = '%s_%s_%s_%s_%s' % tuple(List)
 103                         if NewKey not in self.ToolsDefTxtDictionary:
 104                             self.ToolsDefTxtDictionary[NewKey] = self.ToolsDefTxtDictionary[Key]
 105                     del self.ToolsDefTxtDictionary[Key]
 106                 elif List[Index] not in self.ToolsDefTxtDatabase[KeyList[Index]]:
 107                     del self.ToolsDefTxtDictionary[Key]
 108
 109
 110     ## IncludeToolDefFile
 111     #
 112     # Load target.txt file and parse it as if it's contents were inside the main file
 113     #
 114     # @param Filename:  Input value for full path of tools_def.txt
 115     #
 116     def IncludeToolDefFile(self, FileName):
 117         FileContent = []
 118         if os.path.isfile(FileName):
 119             try:
 120                 F = open(FileName, 'r')
 121                 FileContent = F.readlines()
 122             except:
 123                 EdkLogger.error("tools_def.txt parser", FILE_OPEN_FAILURE, ExtraData=FileName)
 124         else:
 125             EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=FileName)
 126
 127         for Index in range(len(FileContent)):
 128             Line = FileContent[Index].strip()
 129             if Line == "" or Line[0] == '#':
 130                 continue
 131
 132             if Line.startswith("!include"):
 133                 IncFile = Line[8:].strip()
 134                 Done, IncFile = self.ExpandMacros(IncFile)
 135                 if not Done:
 136                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 137                                     "Macro or Environment has not been defined",
 138                                 ExtraData=IncFile[4:-1], File=FileName, Line=Index+1)
 139                 IncFile = NormPath(IncFile)
 140
 141                 if not os.path.isabs(IncFile):
 142                     #
 143                     # try WORKSPACE
 144                     #
 145                     IncFileTmp = PathClass(IncFile, GlobalData.gWorkspace)
 146                     ErrorCode = IncFileTmp.Validate()[0]
 147                     if ErrorCode != 0:
 148                         #
 149                         # try PACKAGES_PATH
 150                         #
 151                         IncFileTmp = mws.join(GlobalData.gWorkspace, IncFile)
 152                         if not os.path.exists(IncFileTmp):
 153                             #
 154                             # try directory of current file
 155                             #
 156                             IncFileTmp = PathClass(IncFile, os.path.dirname(FileName))
 157                             ErrorCode = IncFileTmp.Validate()[0]
 158                             if ErrorCode != 0:
 159                                 EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=IncFile)
 160
 161                     if isinstance(IncFileTmp, PathClass):
 162                         IncFile = IncFileTmp.Path
 163                     else:
 164                         IncFile = IncFileTmp
 165
 166                 self.IncludeToolDefFile(IncFile)
 167                 continue
 168
 169             NameValuePair = Line.split("=", 1)
 170             if len(NameValuePair) != 2:
 171                 EdkLogger.warn("tools_def.txt parser", "Line %d: not correct assignment statement, skipped" % (Index + 1))
 172                 continue
 173
 174             Name = NameValuePair[0].strip()
 175             Value = NameValuePair[1].strip()
 176
 177             if Name == "IDENTIFIER":
 178                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found identifier statement, skipped: %s" % ((Index + 1), Value))
 179                 continue
 180
 181             MacroDefinition = gMacroDefPattern.findall(Name)
 182             if MacroDefinition != []:
 183                 Done, Value = self.ExpandMacros(Value)
 184                 if not Done:
 185                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 186                                     "Macro or Environment has not been defined",
 187                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 188
 189                 MacroName = MacroDefinition[0].strip()
 190                 self.MacroDictionary["DEF(%s)" % MacroName] = Value
 191                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found macro: %s = %s" % ((Index + 1), MacroName, Value))
 192                 continue
 193
 194             Done, Value = self.ExpandMacros(Value)
 195             if not Done:
 196                 EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 197                                 "Macro or Environment has not been defined",
 198                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 199
 200             List = Name.split('_')
 201             if len(List) != 5:
 202                 EdkLogger.verbose("Line %d: Not a valid name of definition: %s" % ((Index + 1), Name))
 203                 continue
 204             elif List[4] == '*':
 205                 EdkLogger.verbose("Line %d: '*' is not allowed in last field: %s" % ((Index + 1), Name))
 206                 continue
 207             else:
 208                 self.ToolsDefTxtDictionary[Name] = Value
 209                 if List[0] != '*':
 210                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] += [List[0]]
 211                 if List[1] != '*':
 212                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] += [List[1]]
 213                 if List[2] != '*':
 214                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] += [List[2]]
 215                 if List[3] != '*':
 216                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] += [List[3]]
 217                 if List[4] == TAB_TOD_DEFINES_FAMILY and List[2] == '*' and List[3] == '*':
 218                     if TAB_TOD_DEFINES_FAMILY not in self.ToolsDefTxtDatabase:
 219                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
 220                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 221                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
 222                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 223                     elif List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 224                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 225                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 226                     elif self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] != Value:
 227                         EdkLogger.verbose("Line %d: No override allowed for the family of a tool chain: %s" % ((Index + 1), Name))
 228                 if List[4] == TAB_TOD_DEFINES_BUILDRULEFAMILY and List[2] == '*' and List[3] == '*':
 229                     if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolsDefTxtDatabase \
 230                        or List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 231                         EdkLogger.verbose("Line %d: The family is not specified, but BuildRuleFamily is specified for the tool chain: %s" % ((Index + 1), Name))
 232                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 233
 234     ## ExpandMacros
 235     #
 236     # Replace defined macros with real value
 237     #
 238     # @param Value:   The string with unreplaced macros
 239     #
 240     # @retval Value:  The string which has been replaced with real value
 241     #
 242     def ExpandMacros(self, Value):
 243         # os.environ contains all environment variables uppercase on Windows which cause the key in the self.MacroDictionary is uppercase, but Ref may not
 244         EnvReference = gEnvRefPattern.findall(Value)
 245         for Ref in EnvReference:
 246             if Ref not in self.MacroDictionary and Ref.upper() not in self.MacroDictionary:
 247                 Value = Value.replace(Ref, "")
 248             else:
 249                 if Ref in self.MacroDictionary:
 250                     Value = Value.replace(Ref, self.MacroDictionary[Ref])
 251                 else:
 252                     Value = Value.replace(Ref, self.MacroDictionary[Ref.upper()])
 253         MacroReference = gMacroRefPattern.findall(Value)
 254         for Ref in MacroReference:
 255             if Ref not in self.MacroDictionary:
 256                 return False, Ref
 257             Value = Value.replace(Ref, self.MacroDictionary[Ref])
 258
 259         return True, Value
 260
 261 ## ToolDefDict
 262 #
 263 # Load tools_def.txt in input Conf dir
 264 #
 265 # @param ConfDir:  Conf dir
 266 #
 267 # @retval ToolDef An instance of ToolDefClassObject() with loaded tools_def.txt
 268 #
 269 def ToolDefDict(ConfDir):
 270     Target = TargetTxtDict(ConfDir)
 271     ToolDef = ToolDefClassObject()
 272     if DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF in Target.TargetTxtDictionary:
 273         ToolsDefFile = Target.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 274         if ToolsDefFile:
 275             ToolDef.LoadToolDefFile(os.path.normpath(ToolsDefFile))
 276         else:
 277             ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 278     else:
 279         ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 280     return ToolDef
 281
 282 ##
 283 #
 284 # This acts like the main() function for the script, unless it is 'import'ed into another
 285 # script.
 286 #
 287 if __name__ == '__main__':
 288     ToolDef = ToolDefDict(os.getenv("WORKSPACE"))
 289     pass