[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmPlatformPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmbeddedPkg",\r
                "EmulatorPkg",\r
                "IntelFsp2Pkg",\r
                "IntelFsp2WrapperPkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "SignedCapsulePkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "PrmPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg",\r
                "SourceLevelDebugPkg",\r
                "UefiPayloadPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64",\r
                "LOONGARCH64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
                if "LOONGARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_loongarch64_unknown_linux",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
	25	self.UseBuiltInBaseTools = None\r
	26	self.ActualScopes = None\r
	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
	36	\r
	37	def RetrieveCommandLineOptions(self, args):\r
	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
	52	return ("ArmPkg",\r
	53	"ArmPlatformPkg",\r
	54	"ArmVirtPkg",\r
	55	"DynamicTablesPkg",\r
	56	"EmbeddedPkg",\r
	57	"EmulatorPkg",\r
	58	"IntelFsp2Pkg",\r
	59	"IntelFsp2WrapperPkg",\r
	60	"MdePkg",\r
	61	"MdeModulePkg",\r
	62	"NetworkPkg",\r
	63	"PcAtChipsetPkg",\r
	64	"SecurityPkg",\r
	65	"UefiCpuPkg",\r
	66	"FmpDevicePkg",\r
	67	"ShellPkg",\r
	68	"SignedCapsulePkg",\r
	69	"StandaloneMmPkg",\r
	70	"FatPkg",\r
	71	"CryptoPkg",\r
	72	"PrmPkg",\r
	73	"UnitTestFrameworkPkg",\r
	74	"OvmfPkg",\r
	75	"RedfishPkg",\r
	76	"SourceLevelDebugPkg",\r
	77	"UefiPayloadPkg"\r
	78	)\r
	79	\r
	80	def GetArchitecturesSupported(self):\r
	81	''' return iterable of edk2 architectures supported by this build '''\r
	82	return (\r
	83	"IA32",\r
	84	"X64",\r
	85	"ARM",\r
	86	"AARCH64",\r
	87	"RISCV64",\r
	88	"LOONGARCH64")\r
	89	\r
	90	def GetTargetsSupported(self):\r
	91	''' return iterable of edk2 target tags supported by this build '''\r
	92	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	93	\r
	94	# ####################################################################################### #\r
	95	# Verify and Save requested Ci Build Config #\r
	96	# ####################################################################################### #\r
	97	\r
	98	def SetPackages(self, list_of_requested_packages):\r
	99	''' Confirm the requested package list is valid and configure SettingsManager\r
	100	to build the requested packages.\r
	101	\r
	102	Raise UnsupportedException if a requested_package is not supported\r
	103	'''\r
	104	unsupported = set(list_of_requested_packages) - \\r
	105	set(self.GetPackagesSupported())\r
	106	if(len(unsupported) > 0):\r
	107	logging.critical(\r
	108	"Unsupported Package Requested: " + " ".join(unsupported))\r
	109	raise Exception("Unsupported Package Requested: " +\r
	110	" ".join(unsupported))\r
	111	self.ActualPackages = list_of_requested_packages\r
	112	\r
	113	def SetArchitectures(self, list_of_requested_architectures):\r
	114	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	115	to run only the requested architectures.\r
	116	\r
	117	Raise Exception if a list_of_requested_architectures is not supported\r
	118	'''\r
	119	unsupported = set(list_of_requested_architectures) - \\r
	120	set(self.GetArchitecturesSupported())\r
	121	if(len(unsupported) > 0):\r
	122	logging.critical(\r
	123	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	124	raise Exception(\r
	125	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	126	self.ActualArchitectures = list_of_requested_architectures\r
	127	\r
	128	def SetTargets(self, list_of_requested_target):\r
	129	''' Confirm the request target list is valid and configure SettingsManager\r
	130	to run only the requested targets.\r
	131	\r
	132	Raise UnsupportedException if a requested_target is not supported\r
	133	'''\r
	134	unsupported = set(list_of_requested_target) - \\r
	135	set(self.GetTargetsSupported())\r
	136	if(len(unsupported) > 0):\r
	137	logging.critical(\r
	138	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	139	raise Exception("Unsupported Targets Requested: " +\r
	140	" ".join(unsupported))\r
	141	self.ActualTargets = list_of_requested_target\r
	142	\r
	143	# ####################################################################################### #\r
	144	# Actual Configuration for Ci Build #\r
	145	# ####################################################################################### #\r
	146	\r
	147	def GetActiveScopes(self):\r
	148	''' return tuple containing scopes that should be active for this process '''\r
	149	if self.ActualScopes is None:\r
	150	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	151	\r
	152	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	153	\r
	154	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	155	\r
	156	if self.UseBuiltInBaseTools is None:\r
	157	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	158	# try and import the pip module for basetools\r
	159	try:\r
	160	import edk2basetools\r
	161	self.UseBuiltInBaseTools = True\r
	162	except ImportError:\r
	163	self.UseBuiltInBaseTools = False\r
	164	pass\r
	165	\r
	166	if self.UseBuiltInBaseTools == True:\r
	167	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	168	logging.warning("Using Pip Tools based BaseTools")\r
	169	else:\r
	170	logging.warning("Falling back to using in-tree BaseTools")\r
	171	\r
	172	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	173	if "AARCH64" in self.ActualArchitectures:\r
	174	scopes += ("gcc_aarch64_linux",)\r
	175	if "ARM" in self.ActualArchitectures:\r
	176	scopes += ("gcc_arm_linux",)\r
	177	if "RISCV64" in self.ActualArchitectures:\r
	178	scopes += ("gcc_riscv64_unknown",)\r
	179	if "LOONGARCH64" in self.ActualArchitectures:\r
	180	scopes += ("gcc_loongarch64_unknown_linux",)\r
	181	self.ActualScopes = scopes\r
	182	return self.ActualScopes\r
	183	\r
	184	def GetRequiredSubmodules(self):\r
	185	''' return iterable containing RequiredSubmodule objects.\r
	186	If no RequiredSubmodules return an empty iterable\r
	187	'''\r
	188	rs = []\r
	189	rs.append(RequiredSubmodule(\r
	190	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	191	rs.append(RequiredSubmodule(\r
	192	"CryptoPkg/Library/OpensslLib/openssl", False))\r
	193	rs.append(RequiredSubmodule(\r
	194	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
	195	rs.append(RequiredSubmodule(\r
	196	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
	197	rs.append(RequiredSubmodule(\r
	198	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
	199	rs.append(RequiredSubmodule(\r
	200	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
	201	rs.append(RequiredSubmodule(\r
	202	"RedfishPkg/Library/JsonLib/jansson", False))\r
	203	return rs\r
	204	\r
	205	def GetName(self):\r
	206	return "Edk2"\r
	207	\r
	208	def GetDependencies(self):\r
	209	return [\r
	210	]\r
	211	\r
	212	def GetPackagesPath(self):\r
	213	return ()\r
	214	\r
	215	def GetWorkspaceRoot(self):\r
	216	''' get WorkspacePath '''\r
	217	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	218	\r
	219	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	220	''' Filter potential packages to test based on changed files. '''\r
	221	build_these_packages = []\r
	222	possible_packages = potentialPackagesList.copy()\r
	223	for f in changedFilesList:\r
	224	# split each part of path for comparison later\r
	225	nodes = f.split("/")\r
	226	\r
	227	# python file change in .pytool folder causes building all\r
	228	if f.endswith(".py") and ".pytool" in nodes:\r
	229	build_these_packages = possible_packages\r
	230	break\r
	231	\r
	232	# BaseTools files that might change the build\r
	233	if "BaseTools" in nodes:\r
	234	if os.path.splitext(f) not in [".txt", ".md"]:\r
	235	build_these_packages = possible_packages\r
	236	break\r
	237	return build_these_packages\r