[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmulatorPkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
4eb2baba SB	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
e6956d00	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
d8d1f666	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
4eb2baba SB	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
1534b622 MC	25	self.UseBuiltInBaseTools = None\r
1534b622 MC	26	self.ActualScopes = None\r
4eb2baba SB	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
1534b622 MC	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
4eb2baba SB	36	\r
4eb2baba SB	37	def RetrieveCommandLineOptions(self, args):\r
1534b622 MC	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
4eb2baba SB	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
2942cb58 PG	52	return ("ArmPkg",\r
2942cb58 PG	53	"ArmVirtPkg",\r
82c65f14	54	"DynamicTablesPkg",\r
0358c0bf SB	55	"EmulatorPkg",\r
0358c0bf SB	56	"MdePkg",\r
4eb2baba SB	57	"MdeModulePkg",\r
	58	"NetworkPkg",\r
	59	"PcAtChipsetPkg",\r
	60	"SecurityPkg",\r
	61	"UefiCpuPkg",\r
	62	"FmpDevicePkg",\r
	63	"ShellPkg",\r
d8d1f666	64	"StandaloneMmPkg",\r
4eb2baba	65	"FatPkg",\r
61364ab9	66	"CryptoPkg",\r
0358c0bf	67	"UnitTestFrameworkPkg",\r
c4add64f AC	68	"OvmfPkg",\r
c4add64f AC	69	"RedfishPkg"\r
4eb2baba SB	70	)\r
	71	\r
	72	def GetArchitecturesSupported(self):\r
	73	''' return iterable of edk2 architectures supported by this build '''\r
e6956d00 AC	74	return (\r
e6956d00 AC	75	"IA32",\r
4eb2baba SB	76	"X64",\r
4eb2baba SB	77	"ARM",\r
e6956d00 AC	78	"AARCH64",\r
e6956d00 AC	79	"RISCV64")\r
4eb2baba SB	80	\r
	81	def GetTargetsSupported(self):\r
	82	''' return iterable of edk2 target tags supported by this build '''\r
	83	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	84	\r
	85	# ####################################################################################### #\r
	86	# Verify and Save requested Ci Build Config #\r
	87	# ####################################################################################### #\r
	88	\r
	89	def SetPackages(self, list_of_requested_packages):\r
	90	''' Confirm the requested package list is valid and configure SettingsManager\r
	91	to build the requested packages.\r
	92	\r
	93	Raise UnsupportedException if a requested_package is not supported\r
	94	'''\r
	95	unsupported = set(list_of_requested_packages) - \\r
	96	set(self.GetPackagesSupported())\r
	97	if(len(unsupported) > 0):\r
	98	logging.critical(\r
	99	"Unsupported Package Requested: " + " ".join(unsupported))\r
	100	raise Exception("Unsupported Package Requested: " +\r
	101	" ".join(unsupported))\r
	102	self.ActualPackages = list_of_requested_packages\r
	103	\r
	104	def SetArchitectures(self, list_of_requested_architectures):\r
	105	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	106	to run only the requested architectures.\r
	107	\r
	108	Raise Exception if a list_of_requested_architectures is not supported\r
	109	'''\r
	110	unsupported = set(list_of_requested_architectures) - \\r
	111	set(self.GetArchitecturesSupported())\r
	112	if(len(unsupported) > 0):\r
	113	logging.critical(\r
	114	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	115	raise Exception(\r
	116	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	117	self.ActualArchitectures = list_of_requested_architectures\r
	118	\r
	119	def SetTargets(self, list_of_requested_target):\r
	120	''' Confirm the request target list is valid and configure SettingsManager\r
	121	to run only the requested targets.\r
	122	\r
	123	Raise UnsupportedException if a requested_target is not supported\r
	124	'''\r
	125	unsupported = set(list_of_requested_target) - \\r
	126	set(self.GetTargetsSupported())\r
	127	if(len(unsupported) > 0):\r
	128	logging.critical(\r
	129	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	130	raise Exception("Unsupported Targets Requested: " +\r
	131	" ".join(unsupported))\r
	132	self.ActualTargets = list_of_requested_target\r
	133	\r
	134	# ####################################################################################### #\r
	135	# Actual Configuration for Ci Build #\r
	136	# ####################################################################################### #\r
	137	\r
	138	def GetActiveScopes(self):\r
	139	''' return tuple containing scopes that should be active for this process '''\r
1534b622 MC	140	if self.ActualScopes is None:\r
	141	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	142	\r
	143	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	144	\r
	145	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	146	\r
	147	if self.UseBuiltInBaseTools is None:\r
	148	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	149	# try and import the pip module for basetools\r
	150	try:\r
	151	import edk2basetools\r
	152	self.UseBuiltInBaseTools = True\r
	153	except ImportError:\r
	154	self.UseBuiltInBaseTools = False\r
	155	pass\r
	156	\r
	157	if self.UseBuiltInBaseTools == True:\r
	158	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	159	logging.warning("Using Pip Tools based BaseTools")\r
	160	else:\r
	161	logging.warning("Falling back to using in-tree BaseTools")\r
	162	\r
	163	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	164	if "AARCH64" in self.ActualArchitectures:\r
	165	scopes += ("gcc_aarch64_linux",)\r
	166	if "ARM" in self.ActualArchitectures:\r
	167	scopes += ("gcc_arm_linux",)\r
	168	if "RISCV64" in self.ActualArchitectures:\r
	169	scopes += ("gcc_riscv64_unknown",)\r
	170	self.ActualScopes = scopes\r
	171	return self.ActualScopes\r
4eb2baba SB	172	\r
	173	def GetRequiredSubmodules(self):\r
	174	''' return iterable containing RequiredSubmodule objects.\r
	175	If no RequiredSubmodules return an empty iterable\r
	176	'''\r
61364ab9	177	rs = []\r
4eb2baba SB	178	rs.append(RequiredSubmodule(\r
	179	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	180	rs.append(RequiredSubmodule(\r
	181	"CryptoPkg/Library/OpensslLib/openssl", False))\r
61364ab9 MK	182	rs.append(RequiredSubmodule(\r
61364ab9 MK	183	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
48b6c60c SZ	184	rs.append(RequiredSubmodule(\r
48b6c60c SZ	185	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
58802e02 SZ	186	rs.append(RequiredSubmodule(\r
58802e02 SZ	187	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
06033f5a SZ	188	rs.append(RequiredSubmodule(\r
06033f5a SZ	189	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
acabf1b0 AC	190	rs.append(RequiredSubmodule(\r
acabf1b0 AC	191	"RedfishPkg/Library/JsonLib/jansson", False))\r
4eb2baba SB	192	return rs\r
	193	\r
	194	def GetName(self):\r
	195	return "Edk2"\r
	196	\r
	197	def GetDependencies(self):\r
61364ab9 MK	198	return [\r
61364ab9 MK	199	]\r
4eb2baba SB	200	\r
	201	def GetPackagesPath(self):\r
	202	return ()\r
	203	\r
	204	def GetWorkspaceRoot(self):\r
	205	''' get WorkspacePath '''\r
	206	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	207	\r
	208	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	209	''' Filter potential packages to test based on changed files. '''\r
61364ab9 MK	210	build_these_packages = []\r
61364ab9 MK	211	possible_packages = potentialPackagesList.copy()\r
4eb2baba	212	for f in changedFilesList:\r
61364ab9 MK	213	# split each part of path for comparison later\r
61364ab9 MK	214	nodes = f.split("/")\r
4eb2baba SB	215	\r
	216	# python file change in .pytool folder causes building all\r
	217	if f.endswith(".py") and ".pytool" in nodes:\r
	218	build_these_packages = possible_packages\r
	219	break\r
	220	\r
	221	# BaseTools files that might change the build\r
	222	if "BaseTools" in nodes:\r
	223	if os.path.splitext(f) not in [".txt", ".md"]:\r
	224	build_these_packages = possible_packages\r
	225	break\r
	226	return build_these_packages\r