BaseTools/Source/Python/Common/ToolDefClassObject.py

   1 ## @file
   2 # This file is used to define each component of tools_def.txt file
   3 #
   4 # Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import re
  20 from . import EdkLogger
  21
  22 from .BuildToolError import *
  23 from Common.TargetTxtClassObject import TargetTxtDict
  24 from Common.LongFilePathSupport import OpenLongFilePath as open
  25 from Common.Misc import PathClass
  26 from Common.StringUtils import NormPath
  27 import Common.GlobalData as GlobalData
  28 from Common import GlobalData
  29 from Common.MultipleWorkspace import MultipleWorkspace as mws
  30 from .DataType import TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG,\
  31                      TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE\
  32                      , TAB_TOD_DEFINES_FAMILY, TAB_TOD_DEFINES_BUILDRULEFAMILY,\
  33                      TAB_STAR, TAB_TAT_DEFINES_TOOL_CHAIN_CONF
  34
  35
  36 ##
  37 # Static variables used for pattern
  38 #
  39 gMacroRefPattern = re.compile('(DEF\([^\(\)]+\))')
  40 gEnvRefPattern = re.compile('(ENV\([^\(\)]+\))')
  41 gMacroDefPattern = re.compile("DEFINE\s+([^\s]+)")
  42 gDefaultToolsDefFile = "tools_def.txt"
  43
  44 ## ToolDefClassObject
  45 #
  46 # This class defined content used in file tools_def.txt
  47 #
  48 # @param object:               Inherited from object class
  49 # @param Filename:             Input value for full path of tools_def.txt
  50 #
  51 # @var ToolsDefTxtDictionary:  To store keys and values defined in target.txt
  52 # @var MacroDictionary:        To store keys and values defined in DEFINE statement
  53 #
  54 class ToolDefClassObject(object):
  55     def __init__(self, FileName=None):
  56         self.ToolsDefTxtDictionary = {}
  57         self.MacroDictionary = {}
  58         for Env in os.environ:
  59             self.MacroDictionary["ENV(%s)" % Env] = os.environ[Env]
  60
  61         if FileName is not None:
  62             self.LoadToolDefFile(FileName)
  63
  64     ## LoadToolDefFile
  65     #
  66     # Load target.txt file and parse it
  67     #
  68     # @param Filename:  Input value for full path of tools_def.txt
  69     #
  70     def LoadToolDefFile(self, FileName):
  71         # set multiple workspace
  72         PackagesPath = os.getenv("PACKAGES_PATH")
  73         mws.setWs(GlobalData.gWorkspace, PackagesPath)
  74
  75         self.ToolsDefTxtDatabase = {
  76             TAB_TOD_DEFINES_TARGET          :   [],
  77             TAB_TOD_DEFINES_TOOL_CHAIN_TAG  :   [],
  78             TAB_TOD_DEFINES_TARGET_ARCH     :   [],
  79             TAB_TOD_DEFINES_COMMAND_TYPE    :   []
  80         }
  81
  82         self.IncludeToolDefFile(FileName)
  83
  84         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET]))
  85         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]))
  86         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH]))
  87
  88         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE]))
  89
  90         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET].sort()
  91         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].sort()
  92         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH].sort()
  93         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE].sort()
  94
  95         KeyList = [TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG, TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE]
  96         for Index in range(3, -1, -1):
  97             # make a copy of the keys to enumerate over to prevent issues when
  98             # adding/removing items from the original dict.
  99             for Key in list(self.ToolsDefTxtDictionary.keys()):
 100                 List = Key.split('_')
 101                 if List[Index] == TAB_STAR:
 102                     for String in self.ToolsDefTxtDatabase[KeyList[Index]]:
 103                         List[Index] = String
 104                         NewKey = '%s_%s_%s_%s_%s' % tuple(List)
 105                         if NewKey not in self.ToolsDefTxtDictionary:
 106                             self.ToolsDefTxtDictionary[NewKey] = self.ToolsDefTxtDictionary[Key]
 107                     del self.ToolsDefTxtDictionary[Key]
 108                 elif List[Index] not in self.ToolsDefTxtDatabase[KeyList[Index]]:
 109                     del self.ToolsDefTxtDictionary[Key]
 110
 111
 112     ## IncludeToolDefFile
 113     #
 114     # Load target.txt file and parse it as if it's contents were inside the main file
 115     #
 116     # @param Filename:  Input value for full path of tools_def.txt
 117     #
 118     def IncludeToolDefFile(self, FileName):
 119         FileContent = []
 120         if os.path.isfile(FileName):
 121             try:
 122                 F = open(FileName, 'r')
 123                 FileContent = F.readlines()
 124             except:
 125                 EdkLogger.error("tools_def.txt parser", FILE_OPEN_FAILURE, ExtraData=FileName)
 126         else:
 127             EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=FileName)
 128
 129         for Index in range(len(FileContent)):
 130             Line = FileContent[Index].strip()
 131             if Line == "" or Line[0] == '#':
 132                 continue
 133
 134             if Line.startswith("!include"):
 135                 IncFile = Line[8:].strip()
 136                 Done, IncFile = self.ExpandMacros(IncFile)
 137                 if not Done:
 138                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 139                                     "Macro or Environment has not been defined",
 140                                 ExtraData=IncFile[4:-1], File=FileName, Line=Index+1)
 141                 IncFile = NormPath(IncFile)
 142
 143                 if not os.path.isabs(IncFile):
 144                     #
 145                     # try WORKSPACE
 146                     #
 147                     IncFileTmp = PathClass(IncFile, GlobalData.gWorkspace)
 148                     ErrorCode = IncFileTmp.Validate()[0]
 149                     if ErrorCode != 0:
 150                         #
 151                         # try PACKAGES_PATH
 152                         #
 153                         IncFileTmp = mws.join(GlobalData.gWorkspace, IncFile)
 154                         if not os.path.exists(IncFileTmp):
 155                             #
 156                             # try directory of current file
 157                             #
 158                             IncFileTmp = PathClass(IncFile, os.path.dirname(FileName))
 159                             ErrorCode = IncFileTmp.Validate()[0]
 160                             if ErrorCode != 0:
 161                                 EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=IncFile)
 162
 163                     if isinstance(IncFileTmp, PathClass):
 164                         IncFile = IncFileTmp.Path
 165                     else:
 166                         IncFile = IncFileTmp
 167
 168                 self.IncludeToolDefFile(IncFile)
 169                 continue
 170
 171             NameValuePair = Line.split("=", 1)
 172             if len(NameValuePair) != 2:
 173                 EdkLogger.warn("tools_def.txt parser", "Line %d: not correct assignment statement, skipped" % (Index + 1))
 174                 continue
 175
 176             Name = NameValuePair[0].strip()
 177             Value = NameValuePair[1].strip()
 178
 179             if Name == "IDENTIFIER":
 180                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found identifier statement, skipped: %s" % ((Index + 1), Value))
 181                 continue
 182
 183             MacroDefinition = gMacroDefPattern.findall(Name)
 184             if MacroDefinition != []:
 185                 Done, Value = self.ExpandMacros(Value)
 186                 if not Done:
 187                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 188                                     "Macro or Environment has not been defined",
 189                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 190
 191                 MacroName = MacroDefinition[0].strip()
 192                 self.MacroDictionary["DEF(%s)" % MacroName] = Value
 193                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found macro: %s = %s" % ((Index + 1), MacroName, Value))
 194                 continue
 195
 196             Done, Value = self.ExpandMacros(Value)
 197             if not Done:
 198                 EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 199                                 "Macro or Environment has not been defined",
 200                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 201
 202             List = Name.split('_')
 203             if len(List) != 5:
 204                 EdkLogger.verbose("Line %d: Not a valid name of definition: %s" % ((Index + 1), Name))
 205                 continue
 206             elif List[4] == TAB_STAR:
 207                 EdkLogger.verbose("Line %d: '*' is not allowed in last field: %s" % ((Index + 1), Name))
 208                 continue
 209             else:
 210                 self.ToolsDefTxtDictionary[Name] = Value
 211                 if List[0] != TAB_STAR:
 212                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] += [List[0]]
 213                 if List[1] != TAB_STAR:
 214                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] += [List[1]]
 215                 if List[2] != TAB_STAR:
 216                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] += [List[2]]
 217                 if List[3] != TAB_STAR:
 218                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] += [List[3]]
 219                 if List[4] == TAB_TOD_DEFINES_FAMILY and List[2] == TAB_STAR and List[3] == TAB_STAR:
 220                     if TAB_TOD_DEFINES_FAMILY not in self.ToolsDefTxtDatabase:
 221                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
 222                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 223                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
 224                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 225                     elif List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 226                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 227                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 228                     elif self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] != Value:
 229                         EdkLogger.verbose("Line %d: No override allowed for the family of a tool chain: %s" % ((Index + 1), Name))
 230                 if List[4] == TAB_TOD_DEFINES_BUILDRULEFAMILY and List[2] == TAB_STAR and List[3] == TAB_STAR:
 231                     if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolsDefTxtDatabase \
 232                        or List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 233                         EdkLogger.verbose("Line %d: The family is not specified, but BuildRuleFamily is specified for the tool chain: %s" % ((Index + 1), Name))
 234                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 235
 236     ## ExpandMacros
 237     #
 238     # Replace defined macros with real value
 239     #
 240     # @param Value:   The string with unreplaced macros
 241     #
 242     # @retval Value:  The string which has been replaced with real value
 243     #
 244     def ExpandMacros(self, Value):
 245         # os.environ contains all environment variables uppercase on Windows which cause the key in the self.MacroDictionary is uppercase, but Ref may not
 246         EnvReference = gEnvRefPattern.findall(Value)
 247         for Ref in EnvReference:
 248             if Ref not in self.MacroDictionary and Ref.upper() not in self.MacroDictionary:
 249                 Value = Value.replace(Ref, "")
 250             else:
 251                 if Ref in self.MacroDictionary:
 252                     Value = Value.replace(Ref, self.MacroDictionary[Ref])
 253                 else:
 254                     Value = Value.replace(Ref, self.MacroDictionary[Ref.upper()])
 255         MacroReference = gMacroRefPattern.findall(Value)
 256         for Ref in MacroReference:
 257             if Ref not in self.MacroDictionary:
 258                 return False, Ref
 259             Value = Value.replace(Ref, self.MacroDictionary[Ref])
 260
 261         return True, Value
 262
 263 ## ToolDefDict
 264 #
 265 # Load tools_def.txt in input Conf dir
 266 #
 267 # @param ConfDir:  Conf dir
 268 #
 269 # @retval ToolDef An instance of ToolDefClassObject() with loaded tools_def.txt
 270 #
 271 def ToolDefDict(ConfDir):
 272     Target = TargetTxtDict(ConfDir)
 273     ToolDef = ToolDefClassObject()
 274     if TAB_TAT_DEFINES_TOOL_CHAIN_CONF in Target.TargetTxtDictionary:
 275         ToolsDefFile = Target.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 276         if ToolsDefFile:
 277             ToolDef.LoadToolDefFile(os.path.normpath(ToolsDefFile))
 278         else:
 279             ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 280     else:
 281         ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 282     return ToolDef
 283
 284 ##
 285 #
 286 # This acts like the main() function for the script, unless it is 'import'ed into another
 287 # script.
 288 #
 289 if __name__ == '__main__':
 290     ToolDef = ToolDefDict(os.getenv("WORKSPACE"))
 291     pass