.pytool/CISettings.py

   1 # @file
   2 #
   3 # Copyright (c) Microsoft Corporation.
   4 # Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>
   5 # Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>
   6 # SPDX-License-Identifier: BSD-2-Clause-Patent
   7 ##
   8 import os
   9 import logging
  10 from edk2toolext.environment import shell_environment
  11 from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager
  12 from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule
  13 from edk2toolext.invocables.edk2_update import UpdateSettingsManager
  14 from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager
  15 from edk2toollib.utility_functions import GetHostInfo
  16
  17
  18 class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):
  19
  20     def __init__(self):
  21         self.ActualPackages = []
  22         self.ActualTargets = []
  23         self.ActualArchitectures = []
  24         self.ActualToolChainTag = ""
  25         self.UseBuiltInBaseTools = None
  26         self.ActualScopes = None
  27
  28     # ####################################################################################### #
  29     #                             Extra CmdLine configuration                                 #
  30     # ####################################################################################### #
  31
  32     def AddCommandLineOptions(self, parserObj):
  33         group = parserObj.add_mutually_exclusive_group()
  34         group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")
  35         group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")
  36
  37     def RetrieveCommandLineOptions(self, args):
  38         super().RetrieveCommandLineOptions(args)
  39         if args.force_piptools:
  40             self.UseBuiltInBaseTools = True
  41         if args.no_piptools:
  42             self.UseBuiltInBaseTools = False
  43
  44     # ####################################################################################### #
  45     #                        Default Support for this Ci Build                                #
  46     # ####################################################################################### #
  47
  48     def GetPackagesSupported(self):
  49         ''' return iterable of edk2 packages supported by this build.
  50         These should be edk2 workspace relative paths '''
  51
  52         return ("ArmVirtPkg",
  53                 "DynamicTablesPkg",
  54                 "EmulatorPkg",
  55                 "MdePkg",
  56                 "MdeModulePkg",
  57                 "NetworkPkg",
  58                 "PcAtChipsetPkg",
  59                 "SecurityPkg",
  60                 "UefiCpuPkg",
  61                 "FmpDevicePkg",
  62                 "ShellPkg",
  63                 "StandaloneMmPkg",
  64                 "FatPkg",
  65                 "CryptoPkg",
  66                 "UnitTestFrameworkPkg",
  67                 "OvmfPkg",
  68                 "RedfishPkg"
  69                 )
  70
  71     def GetArchitecturesSupported(self):
  72         ''' return iterable of edk2 architectures supported by this build '''
  73         return (
  74                 "IA32",
  75                 "X64",
  76                 "ARM",
  77                 "AARCH64",
  78                 "RISCV64")
  79
  80     def GetTargetsSupported(self):
  81         ''' return iterable of edk2 target tags supported by this build '''
  82         return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")
  83
  84     # ####################################################################################### #
  85     #                     Verify and Save requested Ci Build Config                           #
  86     # ####################################################################################### #
  87
  88     def SetPackages(self, list_of_requested_packages):
  89         ''' Confirm the requested package list is valid and configure SettingsManager
  90         to build the requested packages.
  91
  92         Raise UnsupportedException if a requested_package is not supported
  93         '''
  94         unsupported = set(list_of_requested_packages) - \
  95             set(self.GetPackagesSupported())
  96         if(len(unsupported) > 0):
  97             logging.critical(
  98                 "Unsupported Package Requested: " + " ".join(unsupported))
  99             raise Exception("Unsupported Package Requested: " +
 100                             " ".join(unsupported))
 101         self.ActualPackages = list_of_requested_packages
 102
 103     def SetArchitectures(self, list_of_requested_architectures):
 104         ''' Confirm the requests architecture list is valid and configure SettingsManager
 105         to run only the requested architectures.
 106
 107         Raise Exception if a list_of_requested_architectures is not supported
 108         '''
 109         unsupported = set(list_of_requested_architectures) - \
 110             set(self.GetArchitecturesSupported())
 111         if(len(unsupported) > 0):
 112             logging.critical(
 113                 "Unsupported Architecture Requested: " + " ".join(unsupported))
 114             raise Exception(
 115                 "Unsupported Architecture Requested: " + " ".join(unsupported))
 116         self.ActualArchitectures = list_of_requested_architectures
 117
 118     def SetTargets(self, list_of_requested_target):
 119         ''' Confirm the request target list is valid and configure SettingsManager
 120         to run only the requested targets.
 121
 122         Raise UnsupportedException if a requested_target is not supported
 123         '''
 124         unsupported = set(list_of_requested_target) - \
 125             set(self.GetTargetsSupported())
 126         if(len(unsupported) > 0):
 127             logging.critical(
 128                 "Unsupported Targets Requested: " + " ".join(unsupported))
 129             raise Exception("Unsupported Targets Requested: " +
 130                             " ".join(unsupported))
 131         self.ActualTargets = list_of_requested_target
 132
 133     # ####################################################################################### #
 134     #                         Actual Configuration for Ci Build                               #
 135     # ####################################################################################### #
 136
 137     def GetActiveScopes(self):
 138         ''' return tuple containing scopes that should be active for this process '''
 139         if self.ActualScopes is None:
 140             scopes = ("cibuild", "edk2-build", "host-based-test")
 141
 142             self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
 143
 144             is_linux = GetHostInfo().os.upper() == "LINUX"
 145
 146             if self.UseBuiltInBaseTools is None:
 147                 is_linux = GetHostInfo().os.upper() == "LINUX"
 148                 # try and import the pip module for basetools
 149                 try:
 150                     import edk2basetools
 151                     self.UseBuiltInBaseTools = True
 152                 except ImportError:
 153                     self.UseBuiltInBaseTools = False
 154                     pass
 155
 156             if self.UseBuiltInBaseTools == True:
 157                 scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
 158                 logging.warning("Using Pip Tools based BaseTools")
 159             else:
 160                 logging.warning("Falling back to using in-tree BaseTools")
 161
 162             if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):
 163                 if "AARCH64" in self.ActualArchitectures:
 164                     scopes += ("gcc_aarch64_linux",)
 165                 if "ARM" in self.ActualArchitectures:
 166                     scopes += ("gcc_arm_linux",)
 167                 if "RISCV64" in self.ActualArchitectures:
 168                     scopes += ("gcc_riscv64_unknown",)
 169             self.ActualScopes = scopes
 170         return self.ActualScopes
 171
 172     def GetRequiredSubmodules(self):
 173         ''' return iterable containing RequiredSubmodule objects.
 174         If no RequiredSubmodules return an empty iterable
 175         '''
 176         rs = []
 177         rs.append(RequiredSubmodule(
 178             "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))
 179         rs.append(RequiredSubmodule(
 180             "CryptoPkg/Library/OpensslLib/openssl", False))
 181         rs.append(RequiredSubmodule(
 182             "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))
 183         rs.append(RequiredSubmodule(
 184             "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))
 185         rs.append(RequiredSubmodule(
 186             "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))
 187         rs.append(RequiredSubmodule(
 188             "BaseTools/Source/C/BrotliCompress/brotli", False))
 189         rs.append(RequiredSubmodule(
 190             "RedfishPkg/Library/JsonLib/jansson", False))
 191         return rs
 192
 193     def GetName(self):
 194         return "Edk2"
 195
 196     def GetDependencies(self):
 197         return [
 198         ]
 199
 200     def GetPackagesPath(self):
 201         return ()
 202
 203     def GetWorkspaceRoot(self):
 204         ''' get WorkspacePath '''
 205         return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 206
 207     def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:
 208         ''' Filter potential packages to test based on changed files. '''
 209         build_these_packages = []
 210         possible_packages = potentialPackagesList.copy()
 211         for f in changedFilesList:
 212             # split each part of path for comparison later
 213             nodes = f.split("/")
 214
 215             # python file change in .pytool folder causes building all
 216             if f.endswith(".py") and ".pytool" in nodes:
 217                 build_these_packages = possible_packages
 218                 break
 219
 220             # BaseTools files that might change the build
 221             if "BaseTools" in nodes:
 222                 if os.path.splitext(f) not in [".txt", ".md"]:
 223                     build_these_packages = possible_packages
 224                     break
 225         return build_these_packages