[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmulatorPkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
4eb2baba SB	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
e6956d00	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
d8d1f666	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
4eb2baba SB	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
1534b622 MC	25	self.UseBuiltInBaseTools = None\r
1534b622 MC	26	self.ActualScopes = None\r
4eb2baba SB	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
1534b622 MC	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
4eb2baba SB	36	\r
4eb2baba SB	37	def RetrieveCommandLineOptions(self, args):\r
1534b622 MC	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
4eb2baba SB	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
0358c0bf	52	return ("ArmVirtPkg",\r
82c65f14	53	"DynamicTablesPkg",\r
0358c0bf SB	54	"EmulatorPkg",\r
0358c0bf SB	55	"MdePkg",\r
4eb2baba SB	56	"MdeModulePkg",\r
	57	"NetworkPkg",\r
	58	"PcAtChipsetPkg",\r
	59	"SecurityPkg",\r
	60	"UefiCpuPkg",\r
	61	"FmpDevicePkg",\r
	62	"ShellPkg",\r
d8d1f666	63	"StandaloneMmPkg",\r
4eb2baba	64	"FatPkg",\r
61364ab9	65	"CryptoPkg",\r
0358c0bf	66	"UnitTestFrameworkPkg",\r
c4add64f AC	67	"OvmfPkg",\r
c4add64f AC	68	"RedfishPkg"\r
4eb2baba SB	69	)\r
	70	\r
	71	def GetArchitecturesSupported(self):\r
	72	''' return iterable of edk2 architectures supported by this build '''\r
e6956d00 AC	73	return (\r
e6956d00 AC	74	"IA32",\r
4eb2baba SB	75	"X64",\r
4eb2baba SB	76	"ARM",\r
e6956d00 AC	77	"AARCH64",\r
e6956d00 AC	78	"RISCV64")\r
4eb2baba SB	79	\r
	80	def GetTargetsSupported(self):\r
	81	''' return iterable of edk2 target tags supported by this build '''\r
	82	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	83	\r
	84	# ####################################################################################### #\r
	85	# Verify and Save requested Ci Build Config #\r
	86	# ####################################################################################### #\r
	87	\r
	88	def SetPackages(self, list_of_requested_packages):\r
	89	''' Confirm the requested package list is valid and configure SettingsManager\r
	90	to build the requested packages.\r
	91	\r
	92	Raise UnsupportedException if a requested_package is not supported\r
	93	'''\r
	94	unsupported = set(list_of_requested_packages) - \\r
	95	set(self.GetPackagesSupported())\r
	96	if(len(unsupported) > 0):\r
	97	logging.critical(\r
	98	"Unsupported Package Requested: " + " ".join(unsupported))\r
	99	raise Exception("Unsupported Package Requested: " +\r
	100	" ".join(unsupported))\r
	101	self.ActualPackages = list_of_requested_packages\r
	102	\r
	103	def SetArchitectures(self, list_of_requested_architectures):\r
	104	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	105	to run only the requested architectures.\r
	106	\r
	107	Raise Exception if a list_of_requested_architectures is not supported\r
	108	'''\r
	109	unsupported = set(list_of_requested_architectures) - \\r
	110	set(self.GetArchitecturesSupported())\r
	111	if(len(unsupported) > 0):\r
	112	logging.critical(\r
	113	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	114	raise Exception(\r
	115	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	116	self.ActualArchitectures = list_of_requested_architectures\r
	117	\r
	118	def SetTargets(self, list_of_requested_target):\r
	119	''' Confirm the request target list is valid and configure SettingsManager\r
	120	to run only the requested targets.\r
	121	\r
	122	Raise UnsupportedException if a requested_target is not supported\r
	123	'''\r
	124	unsupported = set(list_of_requested_target) - \\r
	125	set(self.GetTargetsSupported())\r
	126	if(len(unsupported) > 0):\r
	127	logging.critical(\r
	128	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	129	raise Exception("Unsupported Targets Requested: " +\r
	130	" ".join(unsupported))\r
	131	self.ActualTargets = list_of_requested_target\r
	132	\r
	133	# ####################################################################################### #\r
	134	# Actual Configuration for Ci Build #\r
	135	# ####################################################################################### #\r
	136	\r
	137	def GetActiveScopes(self):\r
	138	''' return tuple containing scopes that should be active for this process '''\r
1534b622 MC	139	if self.ActualScopes is None:\r
	140	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	141	\r
	142	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	143	\r
	144	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	145	\r
	146	if self.UseBuiltInBaseTools is None:\r
	147	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	148	# try and import the pip module for basetools\r
	149	try:\r
	150	import edk2basetools\r
	151	self.UseBuiltInBaseTools = True\r
	152	except ImportError:\r
	153	self.UseBuiltInBaseTools = False\r
	154	pass\r
	155	\r
	156	if self.UseBuiltInBaseTools == True:\r
	157	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	158	logging.warning("Using Pip Tools based BaseTools")\r
	159	else:\r
	160	logging.warning("Falling back to using in-tree BaseTools")\r
	161	\r
	162	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	163	if "AARCH64" in self.ActualArchitectures:\r
	164	scopes += ("gcc_aarch64_linux",)\r
	165	if "ARM" in self.ActualArchitectures:\r
	166	scopes += ("gcc_arm_linux",)\r
	167	if "RISCV64" in self.ActualArchitectures:\r
	168	scopes += ("gcc_riscv64_unknown",)\r
	169	self.ActualScopes = scopes\r
	170	return self.ActualScopes\r
4eb2baba SB	171	\r
	172	def GetRequiredSubmodules(self):\r
	173	''' return iterable containing RequiredSubmodule objects.\r
	174	If no RequiredSubmodules return an empty iterable\r
	175	'''\r
61364ab9	176	rs = []\r
4eb2baba SB	177	rs.append(RequiredSubmodule(\r
	178	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	179	rs.append(RequiredSubmodule(\r
	180	"CryptoPkg/Library/OpensslLib/openssl", False))\r
61364ab9 MK	181	rs.append(RequiredSubmodule(\r
61364ab9 MK	182	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
48b6c60c SZ	183	rs.append(RequiredSubmodule(\r
48b6c60c SZ	184	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
58802e02 SZ	185	rs.append(RequiredSubmodule(\r
58802e02 SZ	186	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
06033f5a SZ	187	rs.append(RequiredSubmodule(\r
06033f5a SZ	188	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
acabf1b0 AC	189	rs.append(RequiredSubmodule(\r
acabf1b0 AC	190	"RedfishPkg/Library/JsonLib/jansson", False))\r
4eb2baba SB	191	return rs\r
	192	\r
	193	def GetName(self):\r
	194	return "Edk2"\r
	195	\r
	196	def GetDependencies(self):\r
61364ab9 MK	197	return [\r
61364ab9 MK	198	]\r
4eb2baba SB	199	\r
	200	def GetPackagesPath(self):\r
	201	return ()\r
	202	\r
	203	def GetWorkspaceRoot(self):\r
	204	''' get WorkspacePath '''\r
	205	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	206	\r
	207	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	208	''' Filter potential packages to test based on changed files. '''\r
61364ab9 MK	209	build_these_packages = []\r
61364ab9 MK	210	possible_packages = potentialPackagesList.copy()\r
4eb2baba	211	for f in changedFilesList:\r
61364ab9 MK	212	# split each part of path for comparison later\r
61364ab9 MK	213	nodes = f.split("/")\r
4eb2baba SB	214	\r
	215	# python file change in .pytool folder causes building all\r
	216	if f.endswith(".py") and ".pytool" in nodes:\r
	217	build_these_packages = possible_packages\r
	218	break\r
	219	\r
	220	# BaseTools files that might change the build\r
	221	if "BaseTools" in nodes:\r
	222	if os.path.splitext(f) not in [".txt", ".md"]:\r
	223	build_these_packages = possible_packages\r
	224	break\r
	225	return build_these_packages\r