[mirror_edk2.git] / .pytool / CISettings.py

# @file\r
#\r
# Copyright (c) Microsoft Corporation.\r
# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import os\r
import logging\r
from edk2toolext.environment import shell_environment\r
from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
from edk2toollib.utility_functions import GetHostInfo\r
\r
\r
class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
\r
    def __init__(self):\r
        self.ActualPackages = []\r
        self.ActualTargets = []\r
        self.ActualArchitectures = []\r
        self.ActualToolChainTag = ""\r
        self.UseBuiltInBaseTools = None\r
        self.ActualScopes = None\r
\r
    # ####################################################################################### #\r
    #                             Extra CmdLine configuration                                 #\r
    # ####################################################################################### #\r
\r
    def AddCommandLineOptions(self, parserObj):\r
        group = parserObj.add_mutually_exclusive_group()\r
        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
\r
    def RetrieveCommandLineOptions(self, args):\r
        super().RetrieveCommandLineOptions(args)\r
        if args.force_piptools:\r
            self.UseBuiltInBaseTools = True\r
        if args.no_piptools:\r
            self.UseBuiltInBaseTools = False\r
\r
    # ####################################################################################### #\r
    #                        Default Support for this Ci Build                                #\r
    # ####################################################################################### #\r
\r
    def GetPackagesSupported(self):\r
        ''' return iterable of edk2 packages supported by this build.\r
        These should be edk2 workspace relative paths '''\r
\r
        return ("ArmPkg",\r
                "ArmPlatformPkg",\r
                "ArmVirtPkg",\r
                "DynamicTablesPkg",\r
                "EmulatorPkg",\r
                "MdePkg",\r
                "MdeModulePkg",\r
                "NetworkPkg",\r
                "PcAtChipsetPkg",\r
                "SecurityPkg",\r
                "UefiCpuPkg",\r
                "FmpDevicePkg",\r
                "ShellPkg",\r
                "StandaloneMmPkg",\r
                "FatPkg",\r
                "CryptoPkg",\r
                "UnitTestFrameworkPkg",\r
                "OvmfPkg",\r
                "RedfishPkg"\r
                )\r
\r
    def GetArchitecturesSupported(self):\r
        ''' return iterable of edk2 architectures supported by this build '''\r
        return (\r
                "IA32",\r
                "X64",\r
                "ARM",\r
                "AARCH64",\r
                "RISCV64")\r
\r
    def GetTargetsSupported(self):\r
        ''' return iterable of edk2 target tags supported by this build '''\r
        return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
\r
    # ####################################################################################### #\r
    #                     Verify and Save requested Ci Build Config                           #\r
    # ####################################################################################### #\r
\r
    def SetPackages(self, list_of_requested_packages):\r
        ''' Confirm the requested package list is valid and configure SettingsManager\r
        to build the requested packages.\r
\r
        Raise UnsupportedException if a requested_package is not supported\r
        '''\r
        unsupported = set(list_of_requested_packages) - \\r
            set(self.GetPackagesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Package Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Package Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualPackages = list_of_requested_packages\r
\r
    def SetArchitectures(self, list_of_requested_architectures):\r
        ''' Confirm the requests architecture list is valid and configure SettingsManager\r
        to run only the requested architectures.\r
\r
        Raise Exception if a list_of_requested_architectures is not supported\r
        '''\r
        unsupported = set(list_of_requested_architectures) - \\r
            set(self.GetArchitecturesSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
            raise Exception(\r
                "Unsupported Architecture Requested: " + " ".join(unsupported))\r
        self.ActualArchitectures = list_of_requested_architectures\r
\r
    def SetTargets(self, list_of_requested_target):\r
        ''' Confirm the request target list is valid and configure SettingsManager\r
        to run only the requested targets.\r
\r
        Raise UnsupportedException if a requested_target is not supported\r
        '''\r
        unsupported = set(list_of_requested_target) - \\r
            set(self.GetTargetsSupported())\r
        if(len(unsupported) > 0):\r
            logging.critical(\r
                "Unsupported Targets Requested: " + " ".join(unsupported))\r
            raise Exception("Unsupported Targets Requested: " +\r
                            " ".join(unsupported))\r
        self.ActualTargets = list_of_requested_target\r
\r
    # ####################################################################################### #\r
    #                         Actual Configuration for Ci Build                               #\r
    # ####################################################################################### #\r
\r
    def GetActiveScopes(self):\r
        ''' return tuple containing scopes that should be active for this process '''\r
        if self.ActualScopes is None:\r
            scopes = ("cibuild", "edk2-build", "host-based-test")\r
\r
            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
\r
            is_linux = GetHostInfo().os.upper() == "LINUX"\r
\r
            if self.UseBuiltInBaseTools is None:\r
                is_linux = GetHostInfo().os.upper() == "LINUX"\r
                # try and import the pip module for basetools\r
                try:\r
                    import edk2basetools\r
                    self.UseBuiltInBaseTools = True\r
                except ImportError:\r
                    self.UseBuiltInBaseTools = False\r
                    pass\r
\r
            if self.UseBuiltInBaseTools == True:\r
                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
                logging.warning("Using Pip Tools based BaseTools")\r
            else:\r
                logging.warning("Falling back to using in-tree BaseTools")\r
\r
            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
                if "AARCH64" in self.ActualArchitectures:\r
                    scopes += ("gcc_aarch64_linux",)\r
                if "ARM" in self.ActualArchitectures:\r
                    scopes += ("gcc_arm_linux",)\r
                if "RISCV64" in self.ActualArchitectures:\r
                    scopes += ("gcc_riscv64_unknown",)\r
            self.ActualScopes = scopes\r
        return self.ActualScopes\r
\r
    def GetRequiredSubmodules(self):\r
        ''' return iterable containing RequiredSubmodule objects.\r
        If no RequiredSubmodules return an empty iterable\r
        '''\r
        rs = []\r
        rs.append(RequiredSubmodule(\r
            "ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
        rs.append(RequiredSubmodule(\r
            "CryptoPkg/Library/OpensslLib/openssl", False))\r
        rs.append(RequiredSubmodule(\r
            "UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
        rs.append(RequiredSubmodule(\r
            "MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "BaseTools/Source/C/BrotliCompress/brotli", False))\r
        rs.append(RequiredSubmodule(\r
            "RedfishPkg/Library/JsonLib/jansson", False))\r
        return rs\r
\r
    def GetName(self):\r
        return "Edk2"\r
\r
    def GetDependencies(self):\r
        return [\r
        ]\r
\r
    def GetPackagesPath(self):\r
        return ()\r
\r
    def GetWorkspaceRoot(self):\r
        ''' get WorkspacePath '''\r
        return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
\r
    def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
        ''' Filter potential packages to test based on changed files. '''\r
        build_these_packages = []\r
        possible_packages = potentialPackagesList.copy()\r
        for f in changedFilesList:\r
            # split each part of path for comparison later\r
            nodes = f.split("/")\r
\r
            # python file change in .pytool folder causes building all\r
            if f.endswith(".py") and ".pytool" in nodes:\r
                build_these_packages = possible_packages\r
                break\r
\r
            # BaseTools files that might change the build\r
            if "BaseTools" in nodes:\r
                if os.path.splitext(f) not in [".txt", ".md"]:\r
                    build_these_packages = possible_packages\r
                    break\r
        return build_these_packages\r
Commit	Line	Data
4eb2baba SB	1	# @file\r
	2	#\r
	3	# Copyright (c) Microsoft Corporation.\r
e6956d00	4	# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.<BR>\r
d8d1f666	5	# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>\r
4eb2baba SB	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import os\r
	9	import logging\r
	10	from edk2toolext.environment import shell_environment\r
	11	from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager\r
	12	from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule\r
	13	from edk2toolext.invocables.edk2_update import UpdateSettingsManager\r
	14	from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager\r
	15	from edk2toollib.utility_functions import GetHostInfo\r
	16	\r
	17	\r
	18	class Settings(CiBuildSettingsManager, UpdateSettingsManager, SetupSettingsManager, PrEvalSettingsManager):\r
	19	\r
	20	def __init__(self):\r
	21	self.ActualPackages = []\r
	22	self.ActualTargets = []\r
	23	self.ActualArchitectures = []\r
	24	self.ActualToolChainTag = ""\r
1534b622 MC	25	self.UseBuiltInBaseTools = None\r
1534b622 MC	26	self.ActualScopes = None\r
4eb2baba SB	27	\r
	28	# ####################################################################################### #\r
	29	# Extra CmdLine configuration #\r
	30	# ####################################################################################### #\r
	31	\r
	32	def AddCommandLineOptions(self, parserObj):\r
1534b622 MC	33	group = parserObj.add_mutually_exclusive_group()\r
	34	group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")\r
	35	group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")\r
4eb2baba SB	36	\r
4eb2baba SB	37	def RetrieveCommandLineOptions(self, args):\r
1534b622 MC	38	super().RetrieveCommandLineOptions(args)\r
	39	if args.force_piptools:\r
	40	self.UseBuiltInBaseTools = True\r
	41	if args.no_piptools:\r
	42	self.UseBuiltInBaseTools = False\r
4eb2baba SB	43	\r
	44	# ####################################################################################### #\r
	45	# Default Support for this Ci Build #\r
	46	# ####################################################################################### #\r
	47	\r
	48	def GetPackagesSupported(self):\r
	49	''' return iterable of edk2 packages supported by this build.\r
	50	These should be edk2 workspace relative paths '''\r
	51	\r
2942cb58	52	return ("ArmPkg",\r
a4cf1959	53	"ArmPlatformPkg",\r
2942cb58	54	"ArmVirtPkg",\r
82c65f14	55	"DynamicTablesPkg",\r
0358c0bf SB	56	"EmulatorPkg",\r
0358c0bf SB	57	"MdePkg",\r
4eb2baba SB	58	"MdeModulePkg",\r
	59	"NetworkPkg",\r
	60	"PcAtChipsetPkg",\r
	61	"SecurityPkg",\r
	62	"UefiCpuPkg",\r
	63	"FmpDevicePkg",\r
	64	"ShellPkg",\r
d8d1f666	65	"StandaloneMmPkg",\r
4eb2baba	66	"FatPkg",\r
61364ab9	67	"CryptoPkg",\r
0358c0bf	68	"UnitTestFrameworkPkg",\r
c4add64f AC	69	"OvmfPkg",\r
c4add64f AC	70	"RedfishPkg"\r
4eb2baba SB	71	)\r
	72	\r
	73	def GetArchitecturesSupported(self):\r
	74	''' return iterable of edk2 architectures supported by this build '''\r
e6956d00 AC	75	return (\r
e6956d00 AC	76	"IA32",\r
4eb2baba SB	77	"X64",\r
4eb2baba SB	78	"ARM",\r
e6956d00 AC	79	"AARCH64",\r
e6956d00 AC	80	"RISCV64")\r
4eb2baba SB	81	\r
	82	def GetTargetsSupported(self):\r
	83	''' return iterable of edk2 target tags supported by this build '''\r
	84	return ("DEBUG", "RELEASE", "NO-TARGET", "NOOPT")\r
	85	\r
	86	# ####################################################################################### #\r
	87	# Verify and Save requested Ci Build Config #\r
	88	# ####################################################################################### #\r
	89	\r
	90	def SetPackages(self, list_of_requested_packages):\r
	91	''' Confirm the requested package list is valid and configure SettingsManager\r
	92	to build the requested packages.\r
	93	\r
	94	Raise UnsupportedException if a requested_package is not supported\r
	95	'''\r
	96	unsupported = set(list_of_requested_packages) - \\r
	97	set(self.GetPackagesSupported())\r
	98	if(len(unsupported) > 0):\r
	99	logging.critical(\r
	100	"Unsupported Package Requested: " + " ".join(unsupported))\r
	101	raise Exception("Unsupported Package Requested: " +\r
	102	" ".join(unsupported))\r
	103	self.ActualPackages = list_of_requested_packages\r
	104	\r
	105	def SetArchitectures(self, list_of_requested_architectures):\r
	106	''' Confirm the requests architecture list is valid and configure SettingsManager\r
	107	to run only the requested architectures.\r
	108	\r
	109	Raise Exception if a list_of_requested_architectures is not supported\r
	110	'''\r
	111	unsupported = set(list_of_requested_architectures) - \\r
	112	set(self.GetArchitecturesSupported())\r
	113	if(len(unsupported) > 0):\r
	114	logging.critical(\r
	115	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	116	raise Exception(\r
	117	"Unsupported Architecture Requested: " + " ".join(unsupported))\r
	118	self.ActualArchitectures = list_of_requested_architectures\r
	119	\r
	120	def SetTargets(self, list_of_requested_target):\r
	121	''' Confirm the request target list is valid and configure SettingsManager\r
	122	to run only the requested targets.\r
	123	\r
	124	Raise UnsupportedException if a requested_target is not supported\r
	125	'''\r
	126	unsupported = set(list_of_requested_target) - \\r
	127	set(self.GetTargetsSupported())\r
	128	if(len(unsupported) > 0):\r
	129	logging.critical(\r
	130	"Unsupported Targets Requested: " + " ".join(unsupported))\r
	131	raise Exception("Unsupported Targets Requested: " +\r
	132	" ".join(unsupported))\r
	133	self.ActualTargets = list_of_requested_target\r
	134	\r
	135	# ####################################################################################### #\r
	136	# Actual Configuration for Ci Build #\r
	137	# ####################################################################################### #\r
	138	\r
	139	def GetActiveScopes(self):\r
	140	''' return tuple containing scopes that should be active for this process '''\r
1534b622 MC	141	if self.ActualScopes is None:\r
	142	scopes = ("cibuild", "edk2-build", "host-based-test")\r
	143	\r
	144	self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")\r
	145	\r
	146	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	147	\r
	148	if self.UseBuiltInBaseTools is None:\r
	149	is_linux = GetHostInfo().os.upper() == "LINUX"\r
	150	# try and import the pip module for basetools\r
	151	try:\r
	152	import edk2basetools\r
	153	self.UseBuiltInBaseTools = True\r
	154	except ImportError:\r
	155	self.UseBuiltInBaseTools = False\r
	156	pass\r
	157	\r
	158	if self.UseBuiltInBaseTools == True:\r
	159	scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)\r
	160	logging.warning("Using Pip Tools based BaseTools")\r
	161	else:\r
	162	logging.warning("Falling back to using in-tree BaseTools")\r
	163	\r
	164	if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):\r
	165	if "AARCH64" in self.ActualArchitectures:\r
	166	scopes += ("gcc_aarch64_linux",)\r
	167	if "ARM" in self.ActualArchitectures:\r
	168	scopes += ("gcc_arm_linux",)\r
	169	if "RISCV64" in self.ActualArchitectures:\r
	170	scopes += ("gcc_riscv64_unknown",)\r
	171	self.ActualScopes = scopes\r
	172	return self.ActualScopes\r
4eb2baba SB	173	\r
	174	def GetRequiredSubmodules(self):\r
	175	''' return iterable containing RequiredSubmodule objects.\r
	176	If no RequiredSubmodules return an empty iterable\r
	177	'''\r
61364ab9	178	rs = []\r
4eb2baba SB	179	rs.append(RequiredSubmodule(\r
	180	"ArmPkg/Library/ArmSoftFloatLib/berkeley-softfloat-3", False))\r
	181	rs.append(RequiredSubmodule(\r
	182	"CryptoPkg/Library/OpensslLib/openssl", False))\r
61364ab9 MK	183	rs.append(RequiredSubmodule(\r
61364ab9 MK	184	"UnitTestFrameworkPkg/Library/CmockaLib/cmocka", False))\r
48b6c60c SZ	185	rs.append(RequiredSubmodule(\r
48b6c60c SZ	186	"MdeModulePkg/Universal/RegularExpressionDxe/oniguruma", False))\r
58802e02 SZ	187	rs.append(RequiredSubmodule(\r
58802e02 SZ	188	"MdeModulePkg/Library/BrotliCustomDecompressLib/brotli", False))\r
06033f5a SZ	189	rs.append(RequiredSubmodule(\r
06033f5a SZ	190	"BaseTools/Source/C/BrotliCompress/brotli", False))\r
acabf1b0 AC	191	rs.append(RequiredSubmodule(\r
acabf1b0 AC	192	"RedfishPkg/Library/JsonLib/jansson", False))\r
4eb2baba SB	193	return rs\r
	194	\r
	195	def GetName(self):\r
	196	return "Edk2"\r
	197	\r
	198	def GetDependencies(self):\r
61364ab9 MK	199	return [\r
61364ab9 MK	200	]\r
4eb2baba SB	201	\r
	202	def GetPackagesPath(self):\r
	203	return ()\r
	204	\r
	205	def GetWorkspaceRoot(self):\r
	206	''' get WorkspacePath '''\r
	207	return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\r
	208	\r
	209	def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list:\r
	210	''' Filter potential packages to test based on changed files. '''\r
61364ab9 MK	211	build_these_packages = []\r
61364ab9 MK	212	possible_packages = potentialPackagesList.copy()\r
4eb2baba	213	for f in changedFilesList:\r
61364ab9 MK	214	# split each part of path for comparison later\r
61364ab9 MK	215	nodes = f.split("/")\r
4eb2baba SB	216	\r
	217	# python file change in .pytool folder causes building all\r
	218	if f.endswith(".py") and ".pytool" in nodes:\r
	219	build_these_packages = possible_packages\r
	220	break\r
	221	\r
	222	# BaseTools files that might change the build\r
	223	if "BaseTools" in nodes:\r
	224	if os.path.splitext(f) not in [".txt", ".md"]:\r
	225	build_these_packages = possible_packages\r
	226	break\r
	227	return build_these_packages\r