BaseTools/Source/Python/Common/ToolDefClassObject.py

   1 ## @file
   2 # This file is used to define each component of tools_def.txt file
   3 #
   4 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>
   5 # This program and the accompanying materials
   6 # are licensed and made available under the terms and conditions of the BSD License
   7 # which accompanies this distribution.  The full text of the license may be found at
   8 # http://opensource.org/licenses/bsd-license.php
   9 #
  10 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 from __future__ import absolute_import
  18 import Common.LongFilePathOs as os
  19 import re
  20 from . import EdkLogger
  21
  22 from .BuildToolError import *
  23 from .TargetTxtClassObject import *
  24 from Common.LongFilePathSupport import OpenLongFilePath as open
  25 from Common.Misc import PathClass
  26 from Common.StringUtils import NormPath
  27 import Common.GlobalData as GlobalData
  28 from Common import GlobalData
  29 from Common.MultipleWorkspace import MultipleWorkspace as mws
  30 from .DataType import TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG,\
  31                      TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE\
  32                      , TAB_TOD_DEFINES_FAMILY, TAB_TOD_DEFINES_BUILDRULEFAMILY
  33
  34
  35 ##
  36 # Static variables used for pattern
  37 #
  38 gMacroRefPattern = re.compile('(DEF\([^\(\)]+\))')
  39 gEnvRefPattern = re.compile('(ENV\([^\(\)]+\))')
  40 gMacroDefPattern = re.compile("DEFINE\s+([^\s]+)")
  41 gDefaultToolsDefFile = "tools_def.txt"
  42
  43 ## ToolDefClassObject
  44 #
  45 # This class defined content used in file tools_def.txt
  46 #
  47 # @param object:               Inherited from object class
  48 # @param Filename:             Input value for full path of tools_def.txt
  49 #
  50 # @var ToolsDefTxtDictionary:  To store keys and values defined in target.txt
  51 # @var MacroDictionary:        To store keys and values defined in DEFINE statement
  52 #
  53 class ToolDefClassObject(object):
  54     def __init__(self, FileName=None):
  55         self.ToolsDefTxtDictionary = {}
  56         self.MacroDictionary = {}
  57         for Env in os.environ:
  58             self.MacroDictionary["ENV(%s)" % Env] = os.environ[Env]
  59
  60         if FileName is not None:
  61             self.LoadToolDefFile(FileName)
  62
  63     ## LoadToolDefFile
  64     #
  65     # Load target.txt file and parse it
  66     #
  67     # @param Filename:  Input value for full path of tools_def.txt
  68     #
  69     def LoadToolDefFile(self, FileName):
  70         # set multiple workspace
  71         PackagesPath = os.getenv("PACKAGES_PATH")
  72         mws.setWs(GlobalData.gWorkspace, PackagesPath)
  73
  74         self.ToolsDefTxtDatabase = {
  75             TAB_TOD_DEFINES_TARGET          :   [],
  76             TAB_TOD_DEFINES_TOOL_CHAIN_TAG  :   [],
  77             TAB_TOD_DEFINES_TARGET_ARCH     :   [],
  78             TAB_TOD_DEFINES_COMMAND_TYPE    :   []
  79         }
  80
  81         self.IncludeToolDefFile(FileName)
  82
  83         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET]))
  84         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]))
  85         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH]))
  86
  87         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE]))
  88
  89         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET].sort()
  90         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].sort()
  91         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH].sort()
  92         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE].sort()
  93
  94         KeyList = [TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG, TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE]
  95         for Index in range(3, -1, -1):
  96             # make a copy of the keys to enumerate over to prevent issues when
  97             # adding/removing items from the original dict.
  98             for Key in list(self.ToolsDefTxtDictionary.keys()):
  99                 List = Key.split('_')
 100                 if List[Index] == '*':
 101                     for String in self.ToolsDefTxtDatabase[KeyList[Index]]:
 102                         List[Index] = String
 103                         NewKey = '%s_%s_%s_%s_%s' % tuple(List)
 104                         if NewKey not in self.ToolsDefTxtDictionary:
 105                             self.ToolsDefTxtDictionary[NewKey] = self.ToolsDefTxtDictionary[Key]
 106                     del self.ToolsDefTxtDictionary[Key]
 107                 elif List[Index] not in self.ToolsDefTxtDatabase[KeyList[Index]]:
 108                     del self.ToolsDefTxtDictionary[Key]
 109
 110
 111     ## IncludeToolDefFile
 112     #
 113     # Load target.txt file and parse it as if it's contents were inside the main file
 114     #
 115     # @param Filename:  Input value for full path of tools_def.txt
 116     #
 117     def IncludeToolDefFile(self, FileName):
 118         FileContent = []
 119         if os.path.isfile(FileName):
 120             try:
 121                 F = open(FileName, 'r')
 122                 FileContent = F.readlines()
 123             except:
 124                 EdkLogger.error("tools_def.txt parser", FILE_OPEN_FAILURE, ExtraData=FileName)
 125         else:
 126             EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=FileName)
 127
 128         for Index in range(len(FileContent)):
 129             Line = FileContent[Index].strip()
 130             if Line == "" or Line[0] == '#':
 131                 continue
 132
 133             if Line.startswith("!include"):
 134                 IncFile = Line[8:].strip()
 135                 Done, IncFile = self.ExpandMacros(IncFile)
 136                 if not Done:
 137                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 138                                     "Macro or Environment has not been defined",
 139                                 ExtraData=IncFile[4:-1], File=FileName, Line=Index+1)
 140                 IncFile = NormPath(IncFile)
 141
 142                 if not os.path.isabs(IncFile):
 143                     #
 144                     # try WORKSPACE
 145                     #
 146                     IncFileTmp = PathClass(IncFile, GlobalData.gWorkspace)
 147                     ErrorCode = IncFileTmp.Validate()[0]
 148                     if ErrorCode != 0:
 149                         #
 150                         # try PACKAGES_PATH
 151                         #
 152                         IncFileTmp = mws.join(GlobalData.gWorkspace, IncFile)
 153                         if not os.path.exists(IncFileTmp):
 154                             #
 155                             # try directory of current file
 156                             #
 157                             IncFileTmp = PathClass(IncFile, os.path.dirname(FileName))
 158                             ErrorCode = IncFileTmp.Validate()[0]
 159                             if ErrorCode != 0:
 160                                 EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=IncFile)
 161
 162                     if isinstance(IncFileTmp, PathClass):
 163                         IncFile = IncFileTmp.Path
 164                     else:
 165                         IncFile = IncFileTmp
 166
 167                 self.IncludeToolDefFile(IncFile)
 168                 continue
 169
 170             NameValuePair = Line.split("=", 1)
 171             if len(NameValuePair) != 2:
 172                 EdkLogger.warn("tools_def.txt parser", "Line %d: not correct assignment statement, skipped" % (Index + 1))
 173                 continue
 174
 175             Name = NameValuePair[0].strip()
 176             Value = NameValuePair[1].strip()
 177
 178             if Name == "IDENTIFIER":
 179                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found identifier statement, skipped: %s" % ((Index + 1), Value))
 180                 continue
 181
 182             MacroDefinition = gMacroDefPattern.findall(Name)
 183             if MacroDefinition != []:
 184                 Done, Value = self.ExpandMacros(Value)
 185                 if not Done:
 186                     EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 187                                     "Macro or Environment has not been defined",
 188                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 189
 190                 MacroName = MacroDefinition[0].strip()
 191                 self.MacroDictionary["DEF(%s)" % MacroName] = Value
 192                 EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found macro: %s = %s" % ((Index + 1), MacroName, Value))
 193                 continue
 194
 195             Done, Value = self.ExpandMacros(Value)
 196             if not Done:
 197                 EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
 198                                 "Macro or Environment has not been defined",
 199                                 ExtraData=Value[4:-1], File=FileName, Line=Index+1)
 200
 201             List = Name.split('_')
 202             if len(List) != 5:
 203                 EdkLogger.verbose("Line %d: Not a valid name of definition: %s" % ((Index + 1), Name))
 204                 continue
 205             elif List[4] == '*':
 206                 EdkLogger.verbose("Line %d: '*' is not allowed in last field: %s" % ((Index + 1), Name))
 207                 continue
 208             else:
 209                 self.ToolsDefTxtDictionary[Name] = Value
 210                 if List[0] != '*':
 211                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] += [List[0]]
 212                 if List[1] != '*':
 213                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] += [List[1]]
 214                 if List[2] != '*':
 215                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] += [List[2]]
 216                 if List[3] != '*':
 217                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] += [List[3]]
 218                 if List[4] == TAB_TOD_DEFINES_FAMILY and List[2] == '*' and List[3] == '*':
 219                     if TAB_TOD_DEFINES_FAMILY not in self.ToolsDefTxtDatabase:
 220                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
 221                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 222                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
 223                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 224                     elif List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 225                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
 226                         self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 227                     elif self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] != Value:
 228                         EdkLogger.verbose("Line %d: No override allowed for the family of a tool chain: %s" % ((Index + 1), Name))
 229                 if List[4] == TAB_TOD_DEFINES_BUILDRULEFAMILY and List[2] == '*' and List[3] == '*':
 230                     if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolsDefTxtDatabase \
 231                        or List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
 232                         EdkLogger.verbose("Line %d: The family is not specified, but BuildRuleFamily is specified for the tool chain: %s" % ((Index + 1), Name))
 233                     self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
 234
 235     ## ExpandMacros
 236     #
 237     # Replace defined macros with real value
 238     #
 239     # @param Value:   The string with unreplaced macros
 240     #
 241     # @retval Value:  The string which has been replaced with real value
 242     #
 243     def ExpandMacros(self, Value):
 244         # os.environ contains all environment variables uppercase on Windows which cause the key in the self.MacroDictionary is uppercase, but Ref may not
 245         EnvReference = gEnvRefPattern.findall(Value)
 246         for Ref in EnvReference:
 247             if Ref not in self.MacroDictionary and Ref.upper() not in self.MacroDictionary:
 248                 Value = Value.replace(Ref, "")
 249             else:
 250                 if Ref in self.MacroDictionary:
 251                     Value = Value.replace(Ref, self.MacroDictionary[Ref])
 252                 else:
 253                     Value = Value.replace(Ref, self.MacroDictionary[Ref.upper()])
 254         MacroReference = gMacroRefPattern.findall(Value)
 255         for Ref in MacroReference:
 256             if Ref not in self.MacroDictionary:
 257                 return False, Ref
 258             Value = Value.replace(Ref, self.MacroDictionary[Ref])
 259
 260         return True, Value
 261
 262 ## ToolDefDict
 263 #
 264 # Load tools_def.txt in input Conf dir
 265 #
 266 # @param ConfDir:  Conf dir
 267 #
 268 # @retval ToolDef An instance of ToolDefClassObject() with loaded tools_def.txt
 269 #
 270 def ToolDefDict(ConfDir):
 271     Target = TargetTxtDict(ConfDir)
 272     ToolDef = ToolDefClassObject()
 273     if DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF in Target.TargetTxtDictionary:
 274         ToolsDefFile = Target.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
 275         if ToolsDefFile:
 276             ToolDef.LoadToolDefFile(os.path.normpath(ToolsDefFile))
 277         else:
 278             ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 279     else:
 280         ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
 281     return ToolDef
 282
 283 ##
 284 #
 285 # This acts like the main() function for the script, unless it is 'import'ed into another
 286 # script.
 287 #
 288 if __name__ == '__main__':
 289     ToolDef = ToolDefDict(os.getenv("WORKSPACE"))
 290     pass