[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmPlatformPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmbeddedPkg",\r
                "EmulatorPkg",\r
                "IntelFsp2Pkg",\r
                "IntelFsp2WrapperPkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "SignedCapsulePkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "PrmPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg",\r
                "SourceLevelDebugPkg",\r
                "UefiPayloadPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
4eb2baba SB	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
e6956d00	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
d8d1f666	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
4eb2baba SB	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
1534b622 MC	25	self.UseBuiltInBaseTools = None\r
1534b622 MC	26	self.ActualScopes = None\r
4eb2baba SB	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
1534b622 MC	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
4eb2baba SB	36	\r
4eb2baba SB	37	def RetrieveCommandLineOptions(self, args):\r
1534b622 MC	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
4eb2baba SB	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
2942cb58	52	return ("ArmPkg",\r
a4cf1959	53	"ArmPlatformPkg",\r
2942cb58	54	"ArmVirtPkg",\r
82c65f14	55	"DynamicTablesPkg",\r
43c11115	56	"EmbeddedPkg",\r
0358c0bf	57	"EmulatorPkg",\r
28b16c01	58	"IntelFsp2Pkg",\r
9ecab62d	59	"IntelFsp2WrapperPkg",\r
0358c0bf	60	"MdePkg",\r
4eb2baba SB	61	"MdeModulePkg",\r
	62	"NetworkPkg",\r
	63	"PcAtChipsetPkg",\r
	64	"SecurityPkg",\r
	65	"UefiCpuPkg",\r
	66	"FmpDevicePkg",\r
	67	"ShellPkg",\r
b3d379d1	68	"SignedCapsulePkg",\r
d8d1f666	69	"StandaloneMmPkg",\r
4eb2baba	70	"FatPkg",\r
61364ab9	71	"CryptoPkg",\r
f3c11224	72	"PrmPkg",\r
0358c0bf	73	"UnitTestFrameworkPkg",\r
c4add64f	74	"OvmfPkg",\r
e3ee8c8d	75	"RedfishPkg",\r
cf01fdd5	76	"SourceLevelDebugPkg",\r
e3ee8c8d	77	"UefiPayloadPkg"\r
4eb2baba SB	78	)\r
	79	\r
	80	def GetArchitecturesSupported(self):\r
	81	''' return iterable of edk2 architectures supported by this build '''\r
e6956d00 AC	82	return (\r
e6956d00 AC	83	"IA32",\r
4eb2baba SB	84	"X64",\r
4eb2baba SB	85	"ARM",\r
e6956d00 AC	86	"AARCH64",\r
e6956d00 AC	87	"RISCV64")\r
4eb2baba SB	88	\r
	89	def GetTargetsSupported(self):\r
	90	''' return iterable of edk2 target tags supported by this build '''\r
	91	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	92	\r
	93	# ####################################################################################### #\r
	94	# Verify and Save requested Ci Build Config #\r
	95	# ####################################################################################### #\r
	96	\r
	97	def SetPackages(self, list_of_requested_packages):\r
	98	''' Confirm the requested package list is valid and configure SettingsManager\r
	99	to build the requested packages.\r
	100	\r
	101	Raise UnsupportedException if a requested_package is not supported\r
	102	'''\r
	103	unsupported = set(list_of_requested_packages) - \\r
	104	set(self.GetPackagesSupported())\r
	105	if(len(unsupported) > 0):\r
	106	logging.critical(\r
	107	"Unsupported Package Requested: " + " ".join(unsupported))\r
	108	raise Exception("Unsupported Package Requested: " +\r
	109	" ".join(unsupported))\r
	110	self.ActualPackages = list_of_requested_packages\r
	111	\r
	112	def SetArchitectures(self, list_of_requested_architectures):\r
	113	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	114	to run only the requested architectures.\r
	115	\r
	116	Raise Exception if a list_of_requested_architectures is not supported\r
	117	'''\r
	118	unsupported = set(list_of_requested_architectures) - \\r
	119	set(self.GetArchitecturesSupported())\r
	120	if(len(unsupported) > 0):\r
	121	logging.critical(\r
	122	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	123	raise Exception(\r
	124	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	125	self.ActualArchitectures = list_of_requested_architectures\r
	126	\r
	127	def SetTargets(self, list_of_requested_target):\r
	128	''' Confirm the request target list is valid and configure SettingsManager\r
	129	to run only the requested targets.\r
	130	\r
	131	Raise UnsupportedException if a requested_target is not supported\r
	132	'''\r
	133	unsupported = set(list_of_requested_target) - \\r
	134	set(self.GetTargetsSupported())\r
	135	if(len(unsupported) > 0):\r
	136	logging.critical(\r
	137	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	138	raise Exception("Unsupported Targets Requested: " +\r
	139	" ".join(unsupported))\r
	140	self.ActualTargets = list_of_requested_target\r
	141	\r
	142	# ####################################################################################### #\r
	143	# Actual Configuration for Ci Build #\r
	144	# ####################################################################################### #\r
	145	\r
	146	def GetActiveScopes(self):\r
	147	''' return tuple containing scopes that should be active for this process '''\r
1534b622 MC	148	if self.ActualScopes is None:\r
	149	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	150	\r
	151	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	152	\r
	153	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	154	\r
	155	if self.UseBuiltInBaseTools is None:\r
	156	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	157	# try and import the pip module for basetools\r
	158	try:\r
	159	import edk2basetools\r
	160	self.UseBuiltInBaseTools = True\r
	161	except ImportError:\r
	162	self.UseBuiltInBaseTools = False\r
	163	pass\r
	164	\r
	165	if self.UseBuiltInBaseTools == True:\r
	166	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	167	logging.warning("Using Pip Tools based BaseTools")\r
	168	else:\r
	169	logging.warning("Falling back to using in-tree BaseTools")\r
	170	\r
	171	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	172	if "AARCH64" in self.ActualArchitectures:\r
	173	scopes += ("gcc_aarch64_linux",)\r
	174	if "ARM" in self.ActualArchitectures:\r
	175	scopes += ("gcc_arm_linux",)\r
	176	if "RISCV64" in self.ActualArchitectures:\r
	177	scopes += ("gcc_riscv64_unknown",)\r
	178	self.ActualScopes = scopes\r
	179	return self.ActualScopes\r
4eb2baba SB	180	\r
	181	def GetRequiredSubmodules(self):\r
	182	''' return iterable containing RequiredSubmodule objects.\r
	183	If no RequiredSubmodules return an empty iterable\r
	184	'''\r
61364ab9	185	rs = []\r
4eb2baba SB	186	rs.append(RequiredSubmodule(\r
	187	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	188	rs.append(RequiredSubmodule(\r
	189	"CryptoPkg/Library/OpensslLib/openssl", False))\r
61364ab9 MK	190	rs.append(RequiredSubmodule(\r
61364ab9 MK	191	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
48b6c60c SZ	192	rs.append(RequiredSubmodule(\r
48b6c60c SZ	193	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
58802e02 SZ	194	rs.append(RequiredSubmodule(\r
58802e02 SZ	195	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
06033f5a SZ	196	rs.append(RequiredSubmodule(\r
06033f5a SZ	197	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
acabf1b0 AC	198	rs.append(RequiredSubmodule(\r
acabf1b0 AC	199	"RedfishPkg/Library/JsonLib/jansson", False))\r
4eb2baba SB	200	return rs\r
	201	\r
	202	def GetName(self):\r
	203	return "Edk2"\r
	204	\r
	205	def GetDependencies(self):\r
61364ab9 MK	206	return [\r
61364ab9 MK	207	]\r
4eb2baba SB	208	\r
	209	def GetPackagesPath(self):\r
	210	return ()\r
	211	\r
	212	def GetWorkspaceRoot(self):\r
	213	''' get WorkspacePath '''\r
	214	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	215	\r
	216	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	217	''' Filter potential packages to test based on changed files. '''\r
61364ab9 MK	218	build_these_packages = []\r
61364ab9 MK	219	possible_packages = potentialPackagesList.copy()\r
4eb2baba	220	for f in changedFilesList:\r
61364ab9 MK	221	# split each part of path for comparison later\r
61364ab9 MK	222	nodes = f.split("/")\r
4eb2baba SB	223	\r
	224	# python file change in .pytool folder causes building all\r
	225	if f.endswith(".py") and ".pytool" in nodes:\r
	226	build_these_packages = possible_packages\r
	227	break\r
	228	\r
	229	# BaseTools files that might change the build\r
	230	if "BaseTools" in nodes:\r
	231	if os.path.splitext(f) not in [".txt", ".md"]:\r
	232	build_these_packages = possible_packages\r
	233	break\r
	234	return build_these_packages\r