BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

   1 ## @file
   2 # Build cache intermediate result and state
   3 #
   4 # Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.<BR>
   5 # Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
   6 # SPDX-License-Identifier: BSD-2-Clause-Patent
   7 #
   8 from Common.caching import cached_property
   9 import Common.EdkLogger as EdkLogger
  10 import Common.LongFilePathOs as os
  11 from Common.BuildToolError import *
  12 from Common.Misc import SaveFileOnChange, PathClass
  13 from Common.Misc import TemplateString
  14 import sys
  15 gIsFileMap = {}
  16
  17 DEP_FILE_TAIL = "# Updated \n"
  18
  19 class IncludesAutoGen():
  20     """ This class is to manage the dependent files witch are used in Makefile to support incremental build.
  21         1. C files:
  22             1. MSVS.
  23                cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures
  24                that messages and generate dependency files, .deps files.
  25             2. CLANG and GCC
  26                -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the
  27                .deps files.
  28         2. ASL files:
  29             1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
  30             2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file
  31             3. build tool updates the .deps file
  32         3. ASM files (.asm, .s or .nasm):
  33             1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
  34             2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file
  35             3. build tool updates the .deps file
  36     """
  37     def __init__(self, makefile_folder, ModuleAuto):
  38         self.d_folder = makefile_folder
  39         self.makefile_folder = makefile_folder
  40         self.module_autogen = ModuleAuto
  41         self.ToolChainFamily = ModuleAuto.ToolChainFamily
  42         self.workspace = ModuleAuto.WorkspaceDir
  43
  44     def CreateModuleDeps(self):
  45         SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)
  46
  47     def CreateDepsInclude(self):
  48         deps_file = {'deps_file':self.deps_files}
  49
  50         MakePath = self.module_autogen.BuildOption.get('MAKE', {}).get('PATH')
  51         if not MakePath:
  52             EdkLogger.error("build", PARAMETER_MISSING, Message="No Make path available.")
  53         elif "nmake" in MakePath:
  54             _INCLUDE_DEPS_TEMPLATE = TemplateString('''
  55         ${BEGIN}
  56         !IF EXIST(${deps_file})
  57         !INCLUDE ${deps_file}
  58         !ENDIF
  59         ${END}
  60                ''')
  61         else:
  62             _INCLUDE_DEPS_TEMPLATE = TemplateString('''
  63         ${BEGIN}
  64         -include ${deps_file}
  65         ${END}
  66                ''')
  67
  68         try:
  69             deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
  70         except Exception as e:
  71             print(e)
  72         SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
  73
  74     def CreateDepsTarget(self):
  75         SaveFileOnChange(os.path.join(self.makefile_folder,"deps_target"),"\n".join([item +":" for item in self.DepsCollection]),False)
  76
  77     @cached_property
  78     def deps_files(self):
  79         """ Get all .deps file under module build folder. """
  80         deps_files = []
  81         for root, _, files in os.walk(self.d_folder, topdown=False):
  82             for name in files:
  83                 if not name.endswith(".deps"):
  84                     continue
  85                 abspath = os.path.join(root, name)
  86                 deps_files.append(abspath)
  87         return deps_files
  88
  89     @cached_property
  90     def DepsCollection(self):
  91         """ Collect all the dependency files list from all .deps files under a module's build folder """
  92         includes = set()
  93         targetname = [item[0].Name for item in self.TargetFileList.values()]
  94         for abspath in self.deps_files:
  95             try:
  96                 with open(abspath,"r") as fd:
  97                     lines = fd.readlines()
  98
  99                 firstlineitems = lines[0].split(": ")
 100                 dependency_file = firstlineitems[1].strip(" \\\n")
 101                 dependency_file = dependency_file.strip('''"''')
 102                 if dependency_file:
 103                     if os.path.normpath(dependency_file +".deps") == abspath:
 104                         continue
 105                     filename = os.path.basename(dependency_file).strip()
 106                     if filename not in self.SourceFileList and filename not in targetname:
 107                         includes.add(dependency_file.strip())
 108
 109                 for item in lines[1:]:
 110                     if item == DEP_FILE_TAIL:
 111                         continue
 112                     dependency_file = item.strip(" \\\n")
 113                     dependency_file = dependency_file.strip('''"''')
 114                     if os.path.normpath(dependency_file +".deps") == abspath:
 115                         continue
 116                     filename = os.path.basename(dependency_file).strip()
 117                     if filename in self.SourceFileList:
 118                         continue
 119                     if filename in targetname:
 120                         continue
 121                     includes.add(dependency_file.strip())
 122             except Exception as e:
 123                 EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
 124                 continue
 125         rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))
 126         return rt
 127
 128     @cached_property
 129     def SourceFileList(self):
 130         """ Get a map of module's source files name to module's source files path """
 131         source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}
 132         middle_file = {}
 133         for afile in source:
 134             if afile.upper().endswith(".VFR"):
 135                 middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})
 136             if afile.upper().endswith((".S","ASM")):
 137                 middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
 138             if afile.upper().endswith(".ASL"):
 139                 middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
 140         source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})
 141         source.update(middle_file)
 142         return source
 143
 144     @cached_property
 145     def HasNamesakeSourceFile(self):
 146         source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])
 147         rt = len(source_base_name) != len(self.module_autogen.SourceFileList)
 148         return rt
 149     @cached_property
 150     def CcPPCommandPathSet(self):
 151         rt = set()
 152         rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))
 153         rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))
 154         rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))
 155         rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))
 156         rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))
 157         rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))
 158         rt.discard(None)
 159         return rt
 160     @cached_property
 161     def TargetFileList(self):
 162         """ Get a map of module's target name to a tuple of module's targets path and whose input file path """
 163         targets = {}
 164         targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))
 165         for item in self.module_autogen.Targets.values():
 166             for block in item:
 167                 targets[block.Target.Path] = (block.Target,block.Inputs[0])
 168         return targets
 169
 170     def GetRealTarget(self,source_file_abs):
 171         """ Get the final target file based on source file abspath """
 172         source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}
 173         source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}
 174         target_abs = source_target_map.get(source_file_abs)
 175         if target_abs is None:
 176             if source_file_abs.strip().endswith(".i"):
 177                 sourcefilename = os.path.basename(source_file_abs.strip())
 178                 for sourcefile in source_name_map:
 179                     if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:
 180                         target_abs = source_name_map[sourcefile]
 181                         break
 182                 else:
 183                     target_abs = source_file_abs
 184             else:
 185                 target_abs = source_file_abs
 186         return target_abs
 187
 188     def CreateDepsFileForMsvc(self, DepList):
 189         """ Generate dependency files, .deps file from /showIncludes output message """
 190         if not DepList:
 191             return
 192         ModuleDepDict = {}
 193         current_source = ""
 194         SourceFileAbsPathMap = self.SourceFileList
 195         for line in DepList:
 196             line = line.strip()
 197             if self.HasNamesakeSourceFile:
 198                 for cc_cmd in self.CcPPCommandPathSet:
 199                     if cc_cmd in line:
 200                         if '''"'''+cc_cmd+'''"''' in line:
 201                             cc_options = line[len(cc_cmd)+2:].split()
 202                         else:
 203                             cc_options = line[len(cc_cmd):].split()
 204                         SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
 205             if line in SourceFileAbsPathMap:
 206                 current_source = line
 207                 if current_source not in ModuleDepDict:
 208                     ModuleDepDict[SourceFileAbsPathMap[current_source]] = []
 209             elif "Note: including file:" ==  line.lstrip()[:21]:
 210                 if not current_source:
 211                     EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)
 212                 else:
 213                     ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())
 214
 215         for source_abs in ModuleDepDict:
 216             if ModuleDepDict[source_abs]:
 217                 target_abs = self.GetRealTarget(source_abs)
 218                 dep_file_name = os.path.basename(source_abs) + ".deps"
 219                 SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)
 220
 221     def UpdateDepsFileforNonMsvc(self):
 222         """ Update .deps files.
 223             1. Update target path to absolute path.
 224             2. Update middle target to final target.
 225         """
 226
 227         for abspath in self.deps_files:
 228             if abspath.endswith(".trim.deps"):
 229                 continue
 230             try:
 231                 newcontent = []
 232                 with open(abspath,"r") as fd:
 233                     lines = fd.readlines()
 234                 if lines[-1] == DEP_FILE_TAIL:
 235                     continue
 236                 firstlineitems = lines[0].strip().split(" ")
 237
 238                 if len(firstlineitems) > 2:
 239                     sourceitem = firstlineitems[1]
 240                 else:
 241                     sourceitem = lines[1].strip().split(" ")[0]
 242
 243                 source_abs = self.SourceFileList.get(sourceitem,sourceitem)
 244                 firstlineitems[0] = self.GetRealTarget(source_abs)
 245                 p_target = firstlineitems
 246                 if not p_target[0].strip().endswith(":"):
 247                     p_target[0] += ": "
 248
 249                 if len(p_target) == 2:
 250                     p_target[0] += lines[1]
 251                     newcontent.append(p_target[0])
 252                     newcontent.extend(lines[2:])
 253                 else:
 254                     line1 = " ".join(p_target).strip()
 255                     line1 += "\n"
 256                     newcontent.append(line1)
 257                     newcontent.extend(lines[1:])
 258
 259                 newcontent.append("\n")
 260                 newcontent.append(DEP_FILE_TAIL)
 261                 with open(abspath,"w") as fw:
 262                     fw.write("".join(newcontent))
 263             except Exception as e:
 264                 EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
 265                 continue
 266
 267     def UpdateDepsFileforTrim(self):
 268         """ Update .deps file which generated by trim. """
 269
 270         for abspath in self.deps_files:
 271             if not abspath.endswith(".trim.deps"):
 272                 continue
 273             try:
 274                 newcontent = []
 275                 with open(abspath,"r") as fd:
 276                     lines = fd.readlines()
 277                 if lines[-1] == DEP_FILE_TAIL:
 278                     continue
 279
 280                 source_abs = lines[0].strip().split(" ")[0]
 281                 targetitem = self.GetRealTarget(source_abs.strip(" :"))
 282
 283                 targetitem += ": "
 284                 targetitem += lines[1]
 285                 newcontent.append(targetitem)
 286                 newcontent.extend(lines[2:])
 287                 newcontent.append("\n")
 288                 newcontent.append(DEP_FILE_TAIL)
 289                 with open(abspath,"w") as fw:
 290                     fw.write("".join(newcontent))
 291             except Exception as e:
 292                 EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
 293                 continue