BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

   1 ## @file
   2 # Build cache intermediate result and state
   3 #
   4 # Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.<BR>
   5 # SPDX-License-Identifier: BSD-2-Clause-Patent
   6 #
   7 from Common.caching import cached_property
   8 import Common.EdkLogger as EdkLogger
   9 import Common.LongFilePathOs as os
  10 from Common.BuildToolError import *
  11 from Common.Misc import SaveFileOnChange, PathClass
  12 from Common.Misc import TemplateString
  13 import sys
  14 gIsFileMap = {}
  15 if sys.platform == "win32":
  16     _INCLUDE_DEPS_TEMPLATE = TemplateString('''
  17 ${BEGIN}
  18 !IF EXIST(${deps_file})
  19 !INCLUDE ${deps_file}
  20 !ENDIF
  21 ${END}
  22         ''')
  23 else:
  24     _INCLUDE_DEPS_TEMPLATE = TemplateString('''
  25 ${BEGIN}
  26 -include ${deps_file}
  27 ${END}
  28         ''')
  29
  30 DEP_FILE_TAIL = "# Updated \n"
  31
  32 class IncludesAutoGen():
  33     """ This class is to manage the dependent files witch are used in Makefile to support incremental build.
  34         1. C files:
  35             1. MSVS.
  36                cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures
  37                that messages and generate dependency files, .deps files.
  38             2. CLANG and GCC
  39                -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the
  40                .deps files.
  41         2. ASL files:
  42             1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
  43             2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file
  44             3. build tool updates the .deps file
  45         3. ASM files (.asm, .s or .nasm):
  46             1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
  47             2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file
  48             3. build tool updates the .deps file
  49     """
  50     def __init__(self, makefile_folder, ModuleAuto):
  51         self.d_folder = makefile_folder
  52         self.makefile_folder = makefile_folder
  53         self.module_autogen = ModuleAuto
  54         self.ToolChainFamily = ModuleAuto.ToolChainFamily
  55         self.workspace = ModuleAuto.WorkspaceDir
  56
  57     def CreateModuleDeps(self):
  58         SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)
  59
  60     def CreateDepsInclude(self):
  61         deps_file = {'deps_file':self.deps_files}
  62         try:
  63             deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
  64         except Exception as e:
  65             print(e)
  66         SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
  67
  68     def CreateDepsTarget(self):
  69         SaveFileOnChange(os.path.join(self.makefile_folder,"deps_target"),"\n".join([item +":" for item in self.DepsCollection]),False)
  70
  71     @cached_property
  72     def deps_files(self):
  73         """ Get all .deps file under module build folder. """
  74         deps_files = []
  75         for root, _, files in os.walk(self.d_folder, topdown=False):
  76             for name in files:
  77                 if not name.endswith(".deps"):
  78                     continue
  79                 abspath = os.path.join(root, name)
  80                 deps_files.append(abspath)
  81         return deps_files
  82
  83     @cached_property
  84     def DepsCollection(self):
  85         """ Collect all the dependency files list from all .deps files under a module's build folder """
  86         includes = set()
  87         targetname = [item[0].Name for item in self.TargetFileList.values()]
  88         for abspath in self.deps_files:
  89             try:
  90                 with open(abspath,"r") as fd:
  91                     lines = fd.readlines()
  92
  93                 firstlineitems = lines[0].split(": ")
  94                 dependency_file = firstlineitems[1].strip(" \\\n")
  95                 dependency_file = dependency_file.strip('''"''')
  96                 if dependency_file:
  97                     if os.path.normpath(dependency_file +".deps") == abspath:
  98                         continue
  99                     filename = os.path.basename(dependency_file).strip()
 100                     if filename not in self.SourceFileList and filename not in targetname:
 101                         includes.add(dependency_file.strip())
 102
 103                 for item in lines[1:]:
 104                     if item == DEP_FILE_TAIL:
 105                         continue
 106                     dependency_file = item.strip(" \\\n")
 107                     dependency_file = dependency_file.strip('''"''')
 108                     if os.path.normpath(dependency_file +".deps") == abspath:
 109                         continue
 110                     filename = os.path.basename(dependency_file).strip()
 111                     if filename in self.SourceFileList:
 112                         continue
 113                     if filename in targetname:
 114                         continue
 115                     includes.add(dependency_file.strip())
 116             except Exception as e:
 117                 EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
 118                 continue
 119         rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))
 120         return rt
 121
 122     @cached_property
 123     def SourceFileList(self):
 124         """ Get a map of module's source files name to module's source files path """
 125         source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}
 126         middle_file = {}
 127         for afile in source:
 128             if afile.upper().endswith(".VFR"):
 129                 middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})
 130             if afile.upper().endswith((".S","ASM")):
 131                 middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
 132             if afile.upper().endswith(".ASL"):
 133                 middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
 134         source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})
 135         source.update(middle_file)
 136         return source
 137
 138     @cached_property
 139     def HasNamesakeSourceFile(self):
 140         source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])
 141         rt = len(source_base_name) != len(self.module_autogen.SourceFileList)
 142         return rt
 143     @cached_property
 144     def CcPPCommandPathSet(self):
 145         rt = set()
 146         rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))
 147         rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))
 148         rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))
 149         rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))
 150         rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))
 151         rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))
 152         rt.discard(None)
 153         return rt
 154     @cached_property
 155     def TargetFileList(self):
 156         """ Get a map of module's target name to a tuple of module's targets path and whose input file path """
 157         targets = {}
 158         targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))
 159         for item in self.module_autogen.Targets.values():
 160             for block in item:
 161                 targets[block.Target.Path] = (block.Target,block.Inputs[0])
 162         return targets
 163
 164     def GetRealTarget(self,source_file_abs):
 165         """ Get the final target file based on source file abspath """
 166         source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}
 167         source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}
 168         target_abs = source_target_map.get(source_file_abs)
 169         if target_abs is None:
 170             if source_file_abs.strip().endswith(".i"):
 171                 sourcefilename = os.path.basename(source_file_abs.strip())
 172                 for sourcefile in source_name_map:
 173                     if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:
 174                         target_abs = source_name_map[sourcefile]
 175                         break
 176                 else:
 177                     target_abs = source_file_abs
 178             else:
 179                 target_abs = source_file_abs
 180         return target_abs
 181
 182     def CreateDepsFileForMsvc(self, DepList):
 183         """ Generate dependency files, .deps file from /showIncludes output message """
 184         if not DepList:
 185             return
 186         ModuleDepDict = {}
 187         current_source = ""
 188         SourceFileAbsPathMap = self.SourceFileList
 189         for line in DepList:
 190             line = line.strip()
 191             if self.HasNamesakeSourceFile:
 192                 for cc_cmd in self.CcPPCommandPathSet:
 193                     if cc_cmd in line:
 194                         if '''"'''+cc_cmd+'''"''' in line:
 195                             cc_options = line[len(cc_cmd)+2:].split()
 196                         else:
 197                             cc_options = line[len(cc_cmd):].split()
 198                         SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
 199             if line in SourceFileAbsPathMap:
 200                 current_source = line
 201                 if current_source not in ModuleDepDict:
 202                     ModuleDepDict[SourceFileAbsPathMap[current_source]] = []
 203             elif "Note: including file:" ==  line.lstrip()[:21]:
 204                 if not current_source:
 205                     EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)
 206                 else:
 207                     ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())
 208
 209         for source_abs in ModuleDepDict:
 210             if ModuleDepDict[source_abs]:
 211                 target_abs = self.GetRealTarget(source_abs)
 212                 dep_file_name = os.path.basename(source_abs) + ".deps"
 213                 SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)
 214
 215     def UpdateDepsFileforNonMsvc(self):
 216         """ Update .deps files.
 217             1. Update target path to absolute path.
 218             2. Update middle target to final target.
 219         """
 220
 221         for abspath in self.deps_files:
 222             if abspath.endswith(".trim.deps"):
 223                 continue
 224             try:
 225                 newcontent = []
 226                 with open(abspath,"r") as fd:
 227                     lines = fd.readlines()
 228                 if lines[-1] == DEP_FILE_TAIL:
 229                     continue
 230                 firstlineitems = lines[0].strip().split(" ")
 231
 232                 if len(firstlineitems) > 2:
 233                     sourceitem = firstlineitems[1]
 234                 else:
 235                     sourceitem = lines[1].strip().split(" ")[0]
 236
 237                 source_abs = self.SourceFileList.get(sourceitem,sourceitem)
 238                 firstlineitems[0] = self.GetRealTarget(source_abs)
 239                 p_target = firstlineitems
 240                 if not p_target[0].strip().endswith(":"):
 241                     p_target[0] += ": "
 242
 243                 if len(p_target) == 2:
 244                     p_target[0] += lines[1]
 245                     newcontent.append(p_target[0])
 246                     newcontent.extend(lines[2:])
 247                 else:
 248                     line1 = " ".join(p_target).strip()
 249                     line1 += "\n"
 250                     newcontent.append(line1)
 251                     newcontent.extend(lines[1:])
 252
 253                 newcontent.append("\n")
 254                 newcontent.append(DEP_FILE_TAIL)
 255                 with open(abspath,"w") as fw:
 256                     fw.write("".join(newcontent))
 257             except Exception as e:
 258                 EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
 259                 continue
 260
 261     def UpdateDepsFileforTrim(self):
 262         """ Update .deps file which generated by trim. """
 263
 264         for abspath in self.deps_files:
 265             if not abspath.endswith(".trim.deps"):
 266                 continue
 267             try:
 268                 newcontent = []
 269                 with open(abspath,"r") as fd:
 270                     lines = fd.readlines()
 271                 if lines[-1] == DEP_FILE_TAIL:
 272                     continue
 273
 274                 source_abs = lines[0].strip().split(" ")[0]
 275                 targetitem = self.GetRealTarget(source_abs.strip(" :"))
 276
 277                 targetitem += ": "
 278                 targetitem += lines[1]
 279                 newcontent.append(targetitem)
 280                 newcontent.extend(lines[2:])
 281                 newcontent.append("\n")
 282                 newcontent.append(DEP_FILE_TAIL)
 283                 with open(abspath,"w") as fw:
 284                     fw.write("".join(newcontent))
 285             except Exception as e:
 286                 EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
 287                 continue