[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmPlatformPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmbeddedPkg",\r
                "EmulatorPkg",\r
                "IntelFsp2Pkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "PrmPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg",\r
                "UefiPayloadPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
	25	self.UseBuiltInBaseTools = None\r
	26	self.ActualScopes = None\r
	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
	36	\r
	37	def RetrieveCommandLineOptions(self, args):\r
	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
	52	return ("ArmPkg",\r
	53	"ArmPlatformPkg",\r
	54	"ArmVirtPkg",\r
	55	"DynamicTablesPkg",\r
	56	"EmbeddedPkg",\r
	57	"EmulatorPkg",\r
	58	"IntelFsp2Pkg",\r
	59	"MdePkg",\r
	60	"MdeModulePkg",\r
	61	"NetworkPkg",\r
	62	"PcAtChipsetPkg",\r
	63	"SecurityPkg",\r
	64	"UefiCpuPkg",\r
	65	"FmpDevicePkg",\r
	66	"ShellPkg",\r
	67	"StandaloneMmPkg",\r
	68	"FatPkg",\r
	69	"CryptoPkg",\r
	70	"PrmPkg",\r
	71	"UnitTestFrameworkPkg",\r
	72	"OvmfPkg",\r
	73	"RedfishPkg",\r
	74	"UefiPayloadPkg"\r
	75	)\r
	76	\r
	77	def GetArchitecturesSupported(self):\r
	78	''' return iterable of edk2 architectures supported by this build '''\r
	79	return (\r
	80	"IA32",\r
	81	"X64",\r
	82	"ARM",\r
	83	"AARCH64",\r
	84	"RISCV64")\r
	85	\r
	86	def GetTargetsSupported(self):\r
	87	''' return iterable of edk2 target tags supported by this build '''\r
	88	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	89	\r
	90	# ####################################################################################### #\r
	91	# Verify and Save requested Ci Build Config #\r
	92	# ####################################################################################### #\r
	93	\r
	94	def SetPackages(self, list_of_requested_packages):\r
	95	''' Confirm the requested package list is valid and configure SettingsManager\r
	96	to build the requested packages.\r
	97	\r
	98	Raise UnsupportedException if a requested_package is not supported\r
	99	'''\r
	100	unsupported = set(list_of_requested_packages) - \\r
	101	set(self.GetPackagesSupported())\r
	102	if(len(unsupported) > 0):\r
	103	logging.critical(\r
	104	"Unsupported Package Requested: " + " ".join(unsupported))\r
	105	raise Exception("Unsupported Package Requested: " +\r
	106	" ".join(unsupported))\r
	107	self.ActualPackages = list_of_requested_packages\r
	108	\r
	109	def SetArchitectures(self, list_of_requested_architectures):\r
	110	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	111	to run only the requested architectures.\r
	112	\r
	113	Raise Exception if a list_of_requested_architectures is not supported\r
	114	'''\r
	115	unsupported = set(list_of_requested_architectures) - \\r
	116	set(self.GetArchitecturesSupported())\r
	117	if(len(unsupported) > 0):\r
	118	logging.critical(\r
	119	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	120	raise Exception(\r
	121	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	122	self.ActualArchitectures = list_of_requested_architectures\r
	123	\r
	124	def SetTargets(self, list_of_requested_target):\r
	125	''' Confirm the request target list is valid and configure SettingsManager\r
	126	to run only the requested targets.\r
	127	\r
	128	Raise UnsupportedException if a requested_target is not supported\r
	129	'''\r
	130	unsupported = set(list_of_requested_target) - \\r
	131	set(self.GetTargetsSupported())\r
	132	if(len(unsupported) > 0):\r
	133	logging.critical(\r
	134	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	135	raise Exception("Unsupported Targets Requested: " +\r
	136	" ".join(unsupported))\r
	137	self.ActualTargets = list_of_requested_target\r
	138	\r
	139	# ####################################################################################### #\r
	140	# Actual Configuration for Ci Build #\r
	141	# ####################################################################################### #\r
	142	\r
	143	def GetActiveScopes(self):\r
	144	''' return tuple containing scopes that should be active for this process '''\r
	145	if self.ActualScopes is None:\r
	146	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	147	\r
	148	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	149	\r
	150	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	151	\r
	152	if self.UseBuiltInBaseTools is None:\r
	153	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	154	# try and import the pip module for basetools\r
	155	try:\r
	156	import edk2basetools\r
	157	self.UseBuiltInBaseTools = True\r
	158	except ImportError:\r
	159	self.UseBuiltInBaseTools = False\r
	160	pass\r
	161	\r
	162	if self.UseBuiltInBaseTools == True:\r
	163	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	164	logging.warning("Using Pip Tools based BaseTools")\r
	165	else:\r
	166	logging.warning("Falling back to using in-tree BaseTools")\r
	167	\r
	168	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	169	if "AARCH64" in self.ActualArchitectures:\r
	170	scopes += ("gcc_aarch64_linux",)\r
	171	if "ARM" in self.ActualArchitectures:\r
	172	scopes += ("gcc_arm_linux",)\r
	173	if "RISCV64" in self.ActualArchitectures:\r
	174	scopes += ("gcc_riscv64_unknown",)\r
	175	self.ActualScopes = scopes\r
	176	return self.ActualScopes\r
	177	\r
	178	def GetRequiredSubmodules(self):\r
	179	''' return iterable containing RequiredSubmodule objects.\r
	180	If no RequiredSubmodules return an empty iterable\r
	181	'''\r
	182	rs = []\r
	183	rs.append(RequiredSubmodule(\r
	184	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	185	rs.append(RequiredSubmodule(\r
	186	"CryptoPkg/Library/OpensslLib/openssl", False))\r
	187	rs.append(RequiredSubmodule(\r
	188	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
	189	rs.append(RequiredSubmodule(\r
	190	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
	191	rs.append(RequiredSubmodule(\r
	192	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
	193	rs.append(RequiredSubmodule(\r
	194	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
	195	rs.append(RequiredSubmodule(\r
	196	"RedfishPkg/Library/JsonLib/jansson", False))\r
	197	return rs\r
	198	\r
	199	def GetName(self):\r
	200	return "Edk2"\r
	201	\r
	202	def GetDependencies(self):\r
	203	return [\r
	204	]\r
	205	\r
	206	def GetPackagesPath(self):\r
	207	return ()\r
	208	\r
	209	def GetWorkspaceRoot(self):\r
	210	''' get WorkspacePath '''\r
	211	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	212	\r
	213	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	214	''' Filter potential packages to test based on changed files. '''\r
	215	build_these_packages = []\r
	216	possible_packages = potentialPackagesList.copy()\r
	217	for f in changedFilesList:\r
	218	# split each part of path for comparison later\r
	219	nodes = f.split("/")\r
	220	\r
	221	# python file change in .pytool folder causes building all\r
	222	if f.endswith(".py") and ".pytool" in nodes:\r
	223	build_these_packages = possible_packages\r
	224	break\r
	225	\r
	226	# BaseTools files that might change the build\r
	227	if "BaseTools" in nodes:\r
	228	if os.path.splitext(f) not in [".txt", ".md"]:\r
	229	build_these_packages = possible_packages\r
	230	break\r
	231	return build_these_packages\r