[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmPlatformPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmbeddedPkg",\r
                "EmulatorPkg",\r
                "IntelFsp2Pkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "PrmPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg",\r
                "UefiPayloadPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
4eb2baba SB	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
e6956d00	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
d8d1f666	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
4eb2baba SB	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
1534b622 MC	25	self.UseBuiltInBaseTools = None\r
1534b622 MC	26	self.ActualScopes = None\r
4eb2baba SB	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
1534b622 MC	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
4eb2baba SB	36	\r
4eb2baba SB	37	def RetrieveCommandLineOptions(self, args):\r
1534b622 MC	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
4eb2baba SB	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
2942cb58	52	return ("ArmPkg",\r
a4cf1959	53	"ArmPlatformPkg",\r
2942cb58	54	"ArmVirtPkg",\r
82c65f14	55	"DynamicTablesPkg",\r
43c11115	56	"EmbeddedPkg",\r
0358c0bf	57	"EmulatorPkg",\r
28b16c01	58	"IntelFsp2Pkg",\r
0358c0bf	59	"MdePkg",\r
4eb2baba SB	60	"MdeModulePkg",\r
	61	"NetworkPkg",\r
	62	"PcAtChipsetPkg",\r
	63	"SecurityPkg",\r
	64	"UefiCpuPkg",\r
	65	"FmpDevicePkg",\r
	66	"ShellPkg",\r
d8d1f666	67	"StandaloneMmPkg",\r
4eb2baba	68	"FatPkg",\r
61364ab9	69	"CryptoPkg",\r
f3c11224	70	"PrmPkg",\r
0358c0bf	71	"UnitTestFrameworkPkg",\r
c4add64f	72	"OvmfPkg",\r
e3ee8c8d	73	"RedfishPkg",\r
e3ee8c8d	74	"UefiPayloadPkg"\r
4eb2baba SB	75	)\r
	76	\r
	77	def GetArchitecturesSupported(self):\r
	78	''' return iterable of edk2 architectures supported by this build '''\r
e6956d00 AC	79	return (\r
e6956d00 AC	80	"IA32",\r
4eb2baba SB	81	"X64",\r
4eb2baba SB	82	"ARM",\r
e6956d00 AC	83	"AARCH64",\r
e6956d00 AC	84	"RISCV64")\r
4eb2baba SB	85	\r
	86	def GetTargetsSupported(self):\r
	87	''' return iterable of edk2 target tags supported by this build '''\r
	88	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	89	\r
	90	# ####################################################################################### #\r
	91	# Verify and Save requested Ci Build Config #\r
	92	# ####################################################################################### #\r
	93	\r
	94	def SetPackages(self, list_of_requested_packages):\r
	95	''' Confirm the requested package list is valid and configure SettingsManager\r
	96	to build the requested packages.\r
	97	\r
	98	Raise UnsupportedException if a requested_package is not supported\r
	99	'''\r
	100	unsupported = set(list_of_requested_packages) - \\r
	101	set(self.GetPackagesSupported())\r
	102	if(len(unsupported) > 0):\r
	103	logging.critical(\r
	104	"Unsupported Package Requested: " + " ".join(unsupported))\r
	105	raise Exception("Unsupported Package Requested: " +\r
	106	" ".join(unsupported))\r
	107	self.ActualPackages = list_of_requested_packages\r
	108	\r
	109	def SetArchitectures(self, list_of_requested_architectures):\r
	110	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	111	to run only the requested architectures.\r
	112	\r
	113	Raise Exception if a list_of_requested_architectures is not supported\r
	114	'''\r
	115	unsupported = set(list_of_requested_architectures) - \\r
	116	set(self.GetArchitecturesSupported())\r
	117	if(len(unsupported) > 0):\r
	118	logging.critical(\r
	119	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	120	raise Exception(\r
	121	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	122	self.ActualArchitectures = list_of_requested_architectures\r
	123	\r
	124	def SetTargets(self, list_of_requested_target):\r
	125	''' Confirm the request target list is valid and configure SettingsManager\r
	126	to run only the requested targets.\r
	127	\r
	128	Raise UnsupportedException if a requested_target is not supported\r
	129	'''\r
	130	unsupported = set(list_of_requested_target) - \\r
	131	set(self.GetTargetsSupported())\r
	132	if(len(unsupported) > 0):\r
	133	logging.critical(\r
	134	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	135	raise Exception("Unsupported Targets Requested: " +\r
	136	" ".join(unsupported))\r
	137	self.ActualTargets = list_of_requested_target\r
	138	\r
	139	# ####################################################################################### #\r
	140	# Actual Configuration for Ci Build #\r
	141	# ####################################################################################### #\r
	142	\r
	143	def GetActiveScopes(self):\r
	144	''' return tuple containing scopes that should be active for this process '''\r
1534b622 MC	145	if self.ActualScopes is None:\r
	146	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	147	\r
	148	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	149	\r
	150	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	151	\r
	152	if self.UseBuiltInBaseTools is None:\r
	153	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	154	# try and import the pip module for basetools\r
	155	try:\r
	156	import edk2basetools\r
	157	self.UseBuiltInBaseTools = True\r
	158	except ImportError:\r
	159	self.UseBuiltInBaseTools = False\r
	160	pass\r
	161	\r
	162	if self.UseBuiltInBaseTools == True:\r
	163	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	164	logging.warning("Using Pip Tools based BaseTools")\r
	165	else:\r
	166	logging.warning("Falling back to using in-tree BaseTools")\r
	167	\r
	168	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	169	if "AARCH64" in self.ActualArchitectures:\r
	170	scopes += ("gcc_aarch64_linux",)\r
	171	if "ARM" in self.ActualArchitectures:\r
	172	scopes += ("gcc_arm_linux",)\r
	173	if "RISCV64" in self.ActualArchitectures:\r
	174	scopes += ("gcc_riscv64_unknown",)\r
	175	self.ActualScopes = scopes\r
	176	return self.ActualScopes\r
4eb2baba SB	177	\r
	178	def GetRequiredSubmodules(self):\r
	179	''' return iterable containing RequiredSubmodule objects.\r
	180	If no RequiredSubmodules return an empty iterable\r
	181	'''\r
61364ab9	182	rs = []\r
4eb2baba SB	183	rs.append(RequiredSubmodule(\r
	184	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	185	rs.append(RequiredSubmodule(\r
	186	"CryptoPkg/Library/OpensslLib/openssl", False))\r
61364ab9 MK	187	rs.append(RequiredSubmodule(\r
61364ab9 MK	188	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
48b6c60c SZ	189	rs.append(RequiredSubmodule(\r
48b6c60c SZ	190	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
58802e02 SZ	191	rs.append(RequiredSubmodule(\r
58802e02 SZ	192	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
06033f5a SZ	193	rs.append(RequiredSubmodule(\r
06033f5a SZ	194	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
acabf1b0 AC	195	rs.append(RequiredSubmodule(\r
acabf1b0 AC	196	"RedfishPkg/Library/JsonLib/jansson", False))\r
4eb2baba SB	197	return rs\r
	198	\r
	199	def GetName(self):\r
	200	return "Edk2"\r
	201	\r
	202	def GetDependencies(self):\r
61364ab9 MK	203	return [\r
61364ab9 MK	204	]\r
4eb2baba SB	205	\r
	206	def GetPackagesPath(self):\r
	207	return ()\r
	208	\r
	209	def GetWorkspaceRoot(self):\r
	210	''' get WorkspacePath '''\r
	211	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	212	\r
	213	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	214	''' Filter potential packages to test based on changed files. '''\r
61364ab9 MK	215	build_these_packages = []\r
61364ab9 MK	216	possible_packages = potentialPackagesList.copy()\r
4eb2baba	217	for f in changedFilesList:\r
61364ab9 MK	218	# split each part of path for comparison later\r
61364ab9 MK	219	nodes = f.split("/")\r
4eb2baba SB	220	\r
	221	# python file change in .pytool folder causes building all\r
	222	if f.endswith(".py") and ".pytool" in nodes:\r
	223	build_these_packages = possible_packages\r
	224	break\r
	225	\r
	226	# BaseTools files that might change the build\r
	227	if "BaseTools" in nodes:\r
	228	if os.path.splitext(f) not in [".txt", ".md"]:\r
	229	build_these_packages = possible_packages\r
	230	break\r
	231	return build_these_packages\r