[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmPlatformPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmulatorPkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "PrmPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg",\r
                "UefiPayloadPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
4eb2baba SB	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
e6956d00	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
d8d1f666	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
4eb2baba SB	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
1534b622 MC	25	self.UseBuiltInBaseTools = None\r
1534b622 MC	26	self.ActualScopes = None\r
4eb2baba SB	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
1534b622 MC	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
4eb2baba SB	36	\r
4eb2baba SB	37	def RetrieveCommandLineOptions(self, args):\r
1534b622 MC	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
4eb2baba SB	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
2942cb58	52	return ("ArmPkg",\r
a4cf1959	53	"ArmPlatformPkg",\r
2942cb58	54	"ArmVirtPkg",\r
82c65f14	55	"DynamicTablesPkg",\r
0358c0bf SB	56	"EmulatorPkg",\r
0358c0bf SB	57	"MdePkg",\r
4eb2baba SB	58	"MdeModulePkg",\r
	59	"NetworkPkg",\r
	60	"PcAtChipsetPkg",\r
	61	"SecurityPkg",\r
	62	"UefiCpuPkg",\r
	63	"FmpDevicePkg",\r
	64	"ShellPkg",\r
d8d1f666	65	"StandaloneMmPkg",\r
4eb2baba	66	"FatPkg",\r
61364ab9	67	"CryptoPkg",\r
f3c11224	68	"PrmPkg",\r
0358c0bf	69	"UnitTestFrameworkPkg",\r
c4add64f	70	"OvmfPkg",\r
e3ee8c8d	71	"RedfishPkg",\r
e3ee8c8d	72	"UefiPayloadPkg"\r
4eb2baba SB	73	)\r
	74	\r
	75	def GetArchitecturesSupported(self):\r
	76	''' return iterable of edk2 architectures supported by this build '''\r
e6956d00 AC	77	return (\r
e6956d00 AC	78	"IA32",\r
4eb2baba SB	79	"X64",\r
4eb2baba SB	80	"ARM",\r
e6956d00 AC	81	"AARCH64",\r
e6956d00 AC	82	"RISCV64")\r
4eb2baba SB	83	\r
	84	def GetTargetsSupported(self):\r
	85	''' return iterable of edk2 target tags supported by this build '''\r
	86	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	87	\r
	88	# ####################################################################################### #\r
	89	# Verify and Save requested Ci Build Config #\r
	90	# ####################################################################################### #\r
	91	\r
	92	def SetPackages(self, list_of_requested_packages):\r
	93	''' Confirm the requested package list is valid and configure SettingsManager\r
	94	to build the requested packages.\r
	95	\r
	96	Raise UnsupportedException if a requested_package is not supported\r
	97	'''\r
	98	unsupported = set(list_of_requested_packages) - \\r
	99	set(self.GetPackagesSupported())\r
	100	if(len(unsupported) > 0):\r
	101	logging.critical(\r
	102	"Unsupported Package Requested: " + " ".join(unsupported))\r
	103	raise Exception("Unsupported Package Requested: " +\r
	104	" ".join(unsupported))\r
	105	self.ActualPackages = list_of_requested_packages\r
	106	\r
	107	def SetArchitectures(self, list_of_requested_architectures):\r
	108	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	109	to run only the requested architectures.\r
	110	\r
	111	Raise Exception if a list_of_requested_architectures is not supported\r
	112	'''\r
	113	unsupported = set(list_of_requested_architectures) - \\r
	114	set(self.GetArchitecturesSupported())\r
	115	if(len(unsupported) > 0):\r
	116	logging.critical(\r
	117	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	118	raise Exception(\r
	119	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	120	self.ActualArchitectures = list_of_requested_architectures\r
	121	\r
	122	def SetTargets(self, list_of_requested_target):\r
	123	''' Confirm the request target list is valid and configure SettingsManager\r
	124	to run only the requested targets.\r
	125	\r
	126	Raise UnsupportedException if a requested_target is not supported\r
	127	'''\r
	128	unsupported = set(list_of_requested_target) - \\r
	129	set(self.GetTargetsSupported())\r
	130	if(len(unsupported) > 0):\r
	131	logging.critical(\r
	132	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	133	raise Exception("Unsupported Targets Requested: " +\r
	134	" ".join(unsupported))\r
	135	self.ActualTargets = list_of_requested_target\r
	136	\r
	137	# ####################################################################################### #\r
	138	# Actual Configuration for Ci Build #\r
	139	# ####################################################################################### #\r
	140	\r
	141	def GetActiveScopes(self):\r
	142	''' return tuple containing scopes that should be active for this process '''\r
1534b622 MC	143	if self.ActualScopes is None:\r
	144	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	145	\r
	146	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	147	\r
	148	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	149	\r
	150	if self.UseBuiltInBaseTools is None:\r
	151	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	152	# try and import the pip module for basetools\r
	153	try:\r
	154	import edk2basetools\r
	155	self.UseBuiltInBaseTools = True\r
	156	except ImportError:\r
	157	self.UseBuiltInBaseTools = False\r
	158	pass\r
	159	\r
	160	if self.UseBuiltInBaseTools == True:\r
	161	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	162	logging.warning("Using Pip Tools based BaseTools")\r
	163	else:\r
	164	logging.warning("Falling back to using in-tree BaseTools")\r
	165	\r
	166	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	167	if "AARCH64" in self.ActualArchitectures:\r
	168	scopes += ("gcc_aarch64_linux",)\r
	169	if "ARM" in self.ActualArchitectures:\r
	170	scopes += ("gcc_arm_linux",)\r
	171	if "RISCV64" in self.ActualArchitectures:\r
	172	scopes += ("gcc_riscv64_unknown",)\r
	173	self.ActualScopes = scopes\r
	174	return self.ActualScopes\r
4eb2baba SB	175	\r
	176	def GetRequiredSubmodules(self):\r
	177	''' return iterable containing RequiredSubmodule objects.\r
	178	If no RequiredSubmodules return an empty iterable\r
	179	'''\r
61364ab9	180	rs = []\r
4eb2baba SB	181	rs.append(RequiredSubmodule(\r
	182	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	183	rs.append(RequiredSubmodule(\r
	184	"CryptoPkg/Library/OpensslLib/openssl", False))\r
61364ab9 MK	185	rs.append(RequiredSubmodule(\r
61364ab9 MK	186	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
48b6c60c SZ	187	rs.append(RequiredSubmodule(\r
48b6c60c SZ	188	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
58802e02 SZ	189	rs.append(RequiredSubmodule(\r
58802e02 SZ	190	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
06033f5a SZ	191	rs.append(RequiredSubmodule(\r
06033f5a SZ	192	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
acabf1b0 AC	193	rs.append(RequiredSubmodule(\r
acabf1b0 AC	194	"RedfishPkg/Library/JsonLib/jansson", False))\r
4eb2baba SB	195	return rs\r
	196	\r
	197	def GetName(self):\r
	198	return "Edk2"\r
	199	\r
	200	def GetDependencies(self):\r
61364ab9 MK	201	return [\r
61364ab9 MK	202	]\r
4eb2baba SB	203	\r
	204	def GetPackagesPath(self):\r
	205	return ()\r
	206	\r
	207	def GetWorkspaceRoot(self):\r
	208	''' get WorkspacePath '''\r
	209	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	210	\r
	211	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	212	''' Filter potential packages to test based on changed files. '''\r
61364ab9 MK	213	build_these_packages = []\r
61364ab9 MK	214	possible_packages = potentialPackagesList.copy()\r
4eb2baba	215	for f in changedFilesList:\r
61364ab9 MK	216	# split each part of path for comparison later\r
61364ab9 MK	217	nodes = f.split("/")\r
4eb2baba SB	218	\r
	219	# python file change in .pytool folder causes building all\r
	220	if f.endswith(".py") and ".pytool" in nodes:\r
	221	build_these_packages = possible_packages\r
	222	break\r
	223	\r
	224	# BaseTools files that might change the build\r
	225	if "BaseTools" in nodes:\r
	226	if os.path.splitext(f) not in [".txt", ".md"]:\r
	227	build_these_packages = possible_packages\r
	228	break\r
	229	return build_these_packages\r