[ceph.git] / ceph / src / pybind / mgr / cephadm / services / nfs.py

import errno
import logging
import os
import subprocess
import tempfile
from typing import Dict, Tuple, Any, List, cast, Optional

from mgr_module import HandleCommandResult
from mgr_module import NFS_POOL_NAME as POOL_NAME

from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec

from orchestrator import DaemonDescription

from cephadm.services.cephadmservice import AuthEntity, CephadmDaemonDeploySpec, CephService

logger = logging.getLogger(__name__)


class NFSService(CephService):
    TYPE = 'nfs'

    def ranked(self) -> bool:
        return True

    def fence(self, daemon_id: str) -> None:
        logger.info(f'Fencing old nfs.{daemon_id}')
        ret, out, err = self.mgr.mon_command({
            'prefix': 'auth rm',
            'entity': f'client.nfs.{daemon_id}',
        })

        # TODO: block/fence this entity (in case it is still running somewhere)

    def fence_old_ranks(self,
                        spec: ServiceSpec,
                        rank_map: Dict[int, Dict[int, Optional[str]]],
                        num_ranks: int) -> None:
        for rank, m in list(rank_map.items()):
            if rank >= num_ranks:
                for daemon_id in m.values():
                    if daemon_id is not None:
                        self.fence(daemon_id)
                del rank_map[rank]
                nodeid = f'{spec.service_name()}.{rank}'
                self.mgr.log.info(f'Removing {nodeid} from the ganesha grace table')
                self.run_grace_tool(cast(NFSServiceSpec, spec), 'remove', nodeid)
                self.mgr.spec_store.save_rank_map(spec.service_name(), rank_map)
            else:
                max_gen = max(m.keys())
                for gen, daemon_id in list(m.items()):
                    if gen < max_gen:
                        if daemon_id is not None:
                            self.fence(daemon_id)
                        del rank_map[rank][gen]
                        self.mgr.spec_store.save_rank_map(spec.service_name(), rank_map)

    def config(self, spec: NFSServiceSpec) -> None:  # type: ignore
        from nfs.cluster import create_ganesha_pool

        assert self.TYPE == spec.service_type
        create_ganesha_pool(self.mgr)

    def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec:
        assert self.TYPE == daemon_spec.daemon_type
        daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
        return daemon_spec

    def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
        assert self.TYPE == daemon_spec.daemon_type

        daemon_type = daemon_spec.daemon_type
        daemon_id = daemon_spec.daemon_id
        host = daemon_spec.host
        spec = cast(NFSServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec)

        deps: List[str] = []

        nodeid = f'{daemon_spec.service_name}.{daemon_spec.rank}'

        # create the RADOS recovery pool keyring
        rados_user = f'{daemon_type}.{daemon_id}'
        rados_keyring = self.create_keyring(daemon_spec)

        # ensure rank is known to ganesha
        self.mgr.log.info(f'Ensuring {nodeid} is in the ganesha grace table')
        self.run_grace_tool(spec, 'add', nodeid)

        # create the rados config object
        self.create_rados_config_obj(spec)

        # create the RGW keyring
        rgw_user = f'{rados_user}-rgw'
        rgw_keyring = self.create_rgw_keyring(daemon_spec)

        # generate the ganesha config
        def get_ganesha_conf() -> str:
            context = {
                "user": rados_user,
                "nodeid": nodeid,
                "pool": POOL_NAME,
                "namespace": spec.service_id,
                "rgw_user": rgw_user,
                "url": f'rados://{POOL_NAME}/{spec.service_id}/{spec.rados_config_name()}',
                # fall back to default NFS port if not present in daemon_spec
                "port": daemon_spec.ports[0] if daemon_spec.ports else 2049,
                "bind_addr": daemon_spec.ip if daemon_spec.ip else '',
            }
            return self.mgr.template.render('services/nfs/ganesha.conf.j2', context)

        # generate the cephadm config json
        def get_cephadm_config() -> Dict[str, Any]:
            config: Dict[str, Any] = {}
            config['pool'] = POOL_NAME
            config['namespace'] = spec.service_id
            config['userid'] = rados_user
            config['extra_args'] = ['-N', 'NIV_EVENT']
            config['files'] = {
                'ganesha.conf': get_ganesha_conf(),
            }
            config.update(
                self.get_config_and_keyring(
                    daemon_type, daemon_id,
                    keyring=rados_keyring,
                    host=host
                )
            )
            config['rgw'] = {
                'cluster': 'ceph',
                'user': rgw_user,
                'keyring': rgw_keyring,
            }
            logger.debug('Generated cephadm config-json: %s' % config)
            return config

        return get_cephadm_config(), deps

    def create_rados_config_obj(self,
                                spec: NFSServiceSpec,
                                clobber: bool = False) -> None:
        objname = spec.rados_config_name()
        cmd = [
            'rados',
            '-n', f"mgr.{self.mgr.get_mgr_id()}",
            '-k', str(self.mgr.get_ceph_option('keyring')),
            '-p', POOL_NAME,
            '--namespace', cast(str, spec.service_id),
        ]
        result = subprocess.run(
            cmd + ['get', objname, '-'],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            timeout=10)
        if not result.returncode and not clobber:
            logger.info('Rados config object exists: %s' % objname)
        else:
            logger.info('Creating rados config object: %s' % objname)
            result = subprocess.run(
                cmd + ['put', objname, '-'],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                timeout=10)
            if result.returncode:
                self.mgr.log.warning(
                    f'Unable to create rados config object {objname}: {result.stderr.decode("utf-8")}'
                )
                raise RuntimeError(result.stderr.decode("utf-8"))

    def create_keyring(self, daemon_spec: CephadmDaemonDeploySpec) -> str:
        daemon_id = daemon_spec.daemon_id
        spec = cast(NFSServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
        entity: AuthEntity = self.get_auth_entity(daemon_id)

        osd_caps = 'allow rw pool=%s namespace=%s' % (POOL_NAME, spec.service_id)

        logger.info('Creating key for %s' % entity)
        keyring = self.get_keyring_with_caps(entity,
                                             ['mon', 'allow r',
                                              'osd', osd_caps])

        return keyring

    def create_rgw_keyring(self, daemon_spec: CephadmDaemonDeploySpec) -> str:
        daemon_id = daemon_spec.daemon_id
        entity: AuthEntity = self.get_auth_entity(f'{daemon_id}-rgw')

        logger.info('Creating key for %s' % entity)
        keyring = self.get_keyring_with_caps(entity,
                                             ['mon', 'allow r',
                                              'osd', 'allow rwx tag rgw *=*'])

        return keyring

    def run_grace_tool(self,
                       spec: NFSServiceSpec,
                       action: str,
                       nodeid: str) -> None:
        # write a temp keyring and referencing config file.  this is a kludge
        # because the ganesha-grace-tool can only authenticate as a client (and
        # not a mgr).  Also, it doesn't allow you to pass a keyring location via
        # the command line, nor does it parse the CEPH_ARGS env var.
        tmp_id = f'mgr.nfs.grace.{spec.service_name()}'
        entity = AuthEntity(f'client.{tmp_id}')
        keyring = self.get_keyring_with_caps(
            entity,
            ['mon', 'allow r', 'osd', f'allow rwx pool {POOL_NAME}']
        )
        tmp_keyring = tempfile.NamedTemporaryFile(mode='w', prefix='mgr-grace-keyring')
        os.fchmod(tmp_keyring.fileno(), 0o600)
        tmp_keyring.write(keyring)
        tmp_keyring.flush()
        tmp_conf = tempfile.NamedTemporaryFile(mode='w', prefix='mgr-grace-conf')
        tmp_conf.write(self.mgr.get_minimal_ceph_conf())
        tmp_conf.write(f'\tkeyring = {tmp_keyring.name}\n')
        tmp_conf.flush()
        try:
            cmd: List[str] = [
                'ganesha-rados-grace',
                '--cephconf', tmp_conf.name,
                '--userid', tmp_id,
                '--pool', POOL_NAME,
                '--ns', cast(str, spec.service_id),
                action, nodeid,
            ]
            self.mgr.log.debug(cmd)
            result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    timeout=10)
            if result.returncode:
                self.mgr.log.warning(
                    f'ganesha-rados-grace tool failed: {result.stderr.decode("utf-8")}'
                )
                raise RuntimeError(f'grace tool failed: {result.stderr.decode("utf-8")}')

        finally:
            self.mgr.check_mon_command({
                'prefix': 'auth rm',
                'entity': entity,
            })

    def remove_rgw_keyring(self, daemon: DaemonDescription) -> None:
        assert daemon.daemon_id is not None
        daemon_id: str = daemon.daemon_id
        entity: AuthEntity = self.get_auth_entity(f'{daemon_id}-rgw')

        logger.info(f'Removing key for {entity}')
        self.mgr.check_mon_command({
            'prefix': 'auth rm',
            'entity': entity,
        })

    def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None:
        super().post_remove(daemon, is_failed_deploy=is_failed_deploy)
        self.remove_rgw_keyring(daemon)

    def ok_to_stop(self,
                   daemon_ids: List[str],
                   force: bool = False,
                   known: Optional[List[str]] = None) -> HandleCommandResult:
        # if only 1 nfs, alert user (this is not passable with --force)
        warn, warn_message = self._enough_daemons_to_stop(self.TYPE, daemon_ids, 'NFS', 1, True)
        if warn:
            return HandleCommandResult(-errno.EBUSY, '', warn_message)

        # if reached here, there is > 1 nfs daemon.
        if force:
            return HandleCommandResult(0, warn_message, '')

        # if reached here, > 1 nfs daemon and no force flag.
        # Provide warning
        warn_message = "WARNING: Removing NFS daemons can cause clients to lose connectivity. "
        return HandleCommandResult(-errno.EBUSY, '', warn_message)

    def purge(self, service_name: str) -> None:
        if service_name not in self.mgr.spec_store:
            return
        spec = cast(NFSServiceSpec, self.mgr.spec_store[service_name].spec)

        logger.info(f'Removing grace file for {service_name}')
        cmd = [
            'rados',
            '-n', f"mgr.{self.mgr.get_mgr_id()}",
            '-k', str(self.mgr.get_ceph_option('keyring')),
            '-p', POOL_NAME,
            '--namespace', cast(str, spec.service_id),
            'rm', 'grace',
        ]
        subprocess.run(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            timeout=10
        )
Commit	Line	Data
f67539c2	1	import errno
801d1391	2	import logging
b3b6e05e TL	3	import os
	4	import subprocess
	5	import tempfile
f67539c2 TL	6	from typing import Dict, Tuple, Any, List, cast, Optional
	7
	8	from mgr_module import HandleCommandResult
a4b75251	9	from mgr_module import NFS_POOL_NAME as POOL_NAME
801d1391	10
b3b6e05e	11	from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec
801d1391	12
f67539c2	13	from orchestrator import DaemonDescription
801d1391	14
f67539c2	15	from cephadm.services.cephadmservice import AuthEntity, CephadmDaemonDeploySpec, CephService
801d1391 TL	16
	17	logger = logging.getLogger(__name__)
	18
e306af50	19
f91f0fd5	20	class NFSService(CephService):
f6b5b4d7 TL	21	TYPE = 'nfs'
f6b5b4d7 TL	22
b3b6e05e TL	23	def ranked(self) -> bool:
	24	return True
	25
	26	def fence(self, daemon_id: str) -> None:
	27	logger.info(f'Fencing old nfs.{daemon_id}')
	28	ret, out, err = self.mgr.mon_command({
	29	'prefix': 'auth rm',
	30	'entity': f'client.nfs.{daemon_id}',
	31	})
	32
	33	# TODO: block/fence this entity (in case it is still running somewhere)
	34
	35	def fence_old_ranks(self,
	36	spec: ServiceSpec,
	37	rank_map: Dict[int, Dict[int, Optional[str]]],
	38	num_ranks: int) -> None:
	39	for rank, m in list(rank_map.items()):
	40	if rank >= num_ranks:
	41	for daemon_id in m.values():
	42	if daemon_id is not None:
	43	self.fence(daemon_id)
	44	del rank_map[rank]
	45	nodeid = f'{spec.service_name()}.{rank}'
	46	self.mgr.log.info(f'Removing {nodeid} from the ganesha grace table')
	47	self.run_grace_tool(cast(NFSServiceSpec, spec), 'remove', nodeid)
	48	self.mgr.spec_store.save_rank_map(spec.service_name(), rank_map)
	49	else:
	50	max_gen = max(m.keys())
	51	for gen, daemon_id in list(m.items()):
	52	if gen < max_gen:
	53	if daemon_id is not None:
	54	self.fence(daemon_id)
	55	del rank_map[rank][gen]
	56	self.mgr.spec_store.save_rank_map(spec.service_name(), rank_map)
	57
522d829b	58	def config(self, spec: NFSServiceSpec) -> None: # type: ignore
b3b6e05e TL	59	from nfs.cluster import create_ganesha_pool
b3b6e05e TL	60
f6b5b4d7	61	assert self.TYPE == spec.service_type
a4b75251	62	create_ganesha_pool(self.mgr)
f6b5b4d7	63
f67539c2	64	def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec:
f6b5b4d7	65	assert self.TYPE == daemon_spec.daemon_type
f67539c2	66	daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec)
f91f0fd5	67	return daemon_spec
801d1391	68
f67539c2	69	def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
f91f0fd5	70	assert self.TYPE == daemon_spec.daemon_type
801d1391	71
f91f0fd5 TL	72	daemon_type = daemon_spec.daemon_type
	73	daemon_id = daemon_spec.daemon_id
	74	host = daemon_spec.host
f67539c2	75	spec = cast(NFSServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
801d1391	76
f91f0fd5	77	deps: List[str] = []
801d1391	78
b3b6e05e TL	79	nodeid = f'{daemon_spec.service_name}.{daemon_spec.rank}'
b3b6e05e TL	80
f91f0fd5 TL	81	# create the RADOS recovery pool keyring
	82	rados_user = f'{daemon_type}.{daemon_id}'
	83	rados_keyring = self.create_keyring(daemon_spec)
801d1391	84
b3b6e05e TL	85	# ensure rank is known to ganesha
	86	self.mgr.log.info(f'Ensuring {nodeid} is in the ganesha grace table')
	87	self.run_grace_tool(spec, 'add', nodeid)
	88
f91f0fd5 TL	89	# create the rados config object
	90	self.create_rados_config_obj(spec)
	91
	92	# create the RGW keyring
	93	rgw_user = f'{rados_user}-rgw'
	94	rgw_keyring = self.create_rgw_keyring(daemon_spec)
	95
	96	# generate the ganesha config
	97	def get_ganesha_conf() -> str:
b3b6e05e TL	98	context = {
	99	"user": rados_user,
	100	"nodeid": nodeid,
a4b75251 TL	101	"pool": POOL_NAME,
a4b75251 TL	102	"namespace": spec.service_id,
b3b6e05e	103	"rgw_user": rgw_user,
a4b75251	104	"url": f'rados://{POOL_NAME}/{spec.service_id}/{spec.rados_config_name()}',
b3b6e05e TL	105	# fall back to default NFS port if not present in daemon_spec
	106	"port": daemon_spec.ports[0] if daemon_spec.ports else 2049,
	107	"bind_addr": daemon_spec.ip if daemon_spec.ip else '',
	108	}
f91f0fd5 TL	109	return self.mgr.template.render('services/nfs/ganesha.conf.j2', context)
	110
	111	# generate the cephadm config json
	112	def get_cephadm_config() -> Dict[str, Any]:
	113	config: Dict[str, Any] = {}
a4b75251 TL	114	config['pool'] = POOL_NAME
a4b75251 TL	115	config['namespace'] = spec.service_id
f91f0fd5 TL	116	config['userid'] = rados_user
	117	config['extra_args'] = ['-N', 'NIV_EVENT']
	118	config['files'] = {
	119	'ganesha.conf': get_ganesha_conf(),
	120	}
	121	config.update(
	122	self.get_config_and_keyring(
	123	daemon_type, daemon_id,
	124	keyring=rados_keyring,
	125	host=host
	126	)
	127	)
	128	config['rgw'] = {
	129	'cluster': 'ceph',
	130	'user': rgw_user,
	131	'keyring': rgw_keyring,
	132	}
	133	logger.debug('Generated cephadm config-json: %s' % config)
	134	return config
	135
	136	return get_cephadm_config(), deps
	137
	138	def create_rados_config_obj(self,
	139	spec: NFSServiceSpec,
	140	clobber: bool = False) -> None:
b3b6e05e TL	141	objname = spec.rados_config_name()
	142	cmd = [
	143	'rados',
	144	'-n', f"mgr.{self.mgr.get_mgr_id()}",
	145	'-k', str(self.mgr.get_ceph_option('keyring')),
a4b75251 TL	146	'-p', POOL_NAME,
a4b75251 TL	147	'--namespace', cast(str, spec.service_id),
b3b6e05e	148	]
b3b6e05e TL	149	result = subprocess.run(
	150	cmd + ['get', objname, '-'],
	151	stdout=subprocess.PIPE, stderr=subprocess.PIPE,
	152	timeout=10)
	153	if not result.returncode and not clobber:
	154	logger.info('Rados config object exists: %s' % objname)
	155	else:
	156	logger.info('Creating rados config object: %s' % objname)
	157	result = subprocess.run(
	158	cmd + ['put', objname, '-'],
	159	stdout=subprocess.PIPE, stderr=subprocess.PIPE,
	160	timeout=10)
	161	if result.returncode:
	162	self.mgr.log.warning(
	163	f'Unable to create rados config object {objname}: {result.stderr.decode("utf-8")}'
	164	)
	165	raise RuntimeError(result.stderr.decode("utf-8"))
801d1391	166
f67539c2	167	def create_keyring(self, daemon_spec: CephadmDaemonDeploySpec) -> str:
f91f0fd5	168	daemon_id = daemon_spec.daemon_id
f67539c2	169	spec = cast(NFSServiceSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
f91f0fd5 TL	170	entity: AuthEntity = self.get_auth_entity(daemon_id)
f91f0fd5 TL	171
a4b75251	172	osd_caps = 'allow rw pool=%s namespace=%s' % (POOL_NAME, spec.service_id)
f91f0fd5	173
f67539c2 TL	174	logger.info('Creating key for %s' % entity)
	175	keyring = self.get_keyring_with_caps(entity,
	176	['mon', 'allow r',
	177	'osd', osd_caps])
f91f0fd5 TL	178
	179	return keyring
	180
f67539c2	181	def create_rgw_keyring(self, daemon_spec: CephadmDaemonDeploySpec) -> str:
f91f0fd5 TL	182	daemon_id = daemon_spec.daemon_id
	183	entity: AuthEntity = self.get_auth_entity(f'{daemon_id}-rgw')
	184
f67539c2 TL	185	logger.info('Creating key for %s' % entity)
	186	keyring = self.get_keyring_with_caps(entity,
	187	['mon', 'allow r',
	188	'osd', 'allow rwx tag rgw ='])
f91f0fd5 TL	189
	190	return keyring
	191
b3b6e05e TL	192	def run_grace_tool(self,
	193	spec: NFSServiceSpec,
	194	action: str,
	195	nodeid: str) -> None:
	196	# write a temp keyring and referencing config file. this is a kludge
	197	# because the ganesha-grace-tool can only authenticate as a client (and
	198	# not a mgr). Also, it doesn't allow you to pass a keyring location via
	199	# the command line, nor does it parse the CEPH_ARGS env var.
	200	tmp_id = f'mgr.nfs.grace.{spec.service_name()}'
	201	entity = AuthEntity(f'client.{tmp_id}')
	202	keyring = self.get_keyring_with_caps(
	203	entity,
a4b75251	204	['mon', 'allow r', 'osd', f'allow rwx pool {POOL_NAME}']
b3b6e05e TL	205	)
	206	tmp_keyring = tempfile.NamedTemporaryFile(mode='w', prefix='mgr-grace-keyring')
	207	os.fchmod(tmp_keyring.fileno(), 0o600)
	208	tmp_keyring.write(keyring)
	209	tmp_keyring.flush()
	210	tmp_conf = tempfile.NamedTemporaryFile(mode='w', prefix='mgr-grace-conf')
	211	tmp_conf.write(self.mgr.get_minimal_ceph_conf())
	212	tmp_conf.write(f'\tkeyring = {tmp_keyring.name}\n')
	213	tmp_conf.flush()
	214	try:
	215	cmd: List[str] = [
	216	'ganesha-rados-grace',
	217	'--cephconf', tmp_conf.name,
	218	'--userid', tmp_id,
a4b75251 TL	219	'--pool', POOL_NAME,
	220	'--ns', cast(str, spec.service_id),
	221	action, nodeid,
b3b6e05e	222	]
b3b6e05e TL	223	self.mgr.log.debug(cmd)
	224	result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
	225	timeout=10)
	226	if result.returncode:
	227	self.mgr.log.warning(
	228	f'ganesha-rados-grace tool failed: {result.stderr.decode("utf-8")}'
	229	)
	230	raise RuntimeError(f'grace tool failed: {result.stderr.decode("utf-8")}')
	231
	232	finally:
	233	self.mgr.check_mon_command({
	234	'prefix': 'auth rm',
	235	'entity': entity,
	236	})
	237
f91f0fd5	238	def remove_rgw_keyring(self, daemon: DaemonDescription) -> None:
f67539c2	239	assert daemon.daemon_id is not None
f91f0fd5 TL	240	daemon_id: str = daemon.daemon_id
	241	entity: AuthEntity = self.get_auth_entity(f'{daemon_id}-rgw')
	242
f67539c2	243	logger.info(f'Removing key for {entity}')
b3b6e05e	244	self.mgr.check_mon_command({
f91f0fd5 TL	245	'prefix': 'auth rm',
	246	'entity': entity,
	247	})
	248
a4b75251 TL	249	def post_remove(self, daemon: DaemonDescription, is_failed_deploy: bool) -> None:
a4b75251 TL	250	super().post_remove(daemon, is_failed_deploy=is_failed_deploy)
f91f0fd5	251	self.remove_rgw_keyring(daemon)
f67539c2 TL	252
	253	def ok_to_stop(self,
	254	daemon_ids: List[str],
	255	force: bool = False,
	256	known: Optional[List[str]] = None) -> HandleCommandResult:
	257	# if only 1 nfs, alert user (this is not passable with --force)
	258	warn, warn_message = self._enough_daemons_to_stop(self.TYPE, daemon_ids, 'NFS', 1, True)
	259	if warn:
	260	return HandleCommandResult(-errno.EBUSY, '', warn_message)
	261
	262	# if reached here, there is > 1 nfs daemon.
	263	if force:
	264	return HandleCommandResult(0, warn_message, '')
	265
	266	# if reached here, > 1 nfs daemon and no force flag.
	267	# Provide warning
	268	warn_message = "WARNING: Removing NFS daemons can cause clients to lose connectivity. "
	269	return HandleCommandResult(-errno.EBUSY, '', warn_message)
b3b6e05e TL	270
	271	def purge(self, service_name: str) -> None:
	272	if service_name not in self.mgr.spec_store:
	273	return
	274	spec = cast(NFSServiceSpec, self.mgr.spec_store[service_name].spec)
	275
	276	logger.info(f'Removing grace file for {service_name}')
	277	cmd = [
	278	'rados',
	279	'-n', f"mgr.{self.mgr.get_mgr_id()}",
	280	'-k', str(self.mgr.get_ceph_option('keyring')),
a4b75251 TL	281	'-p', POOL_NAME,
	282	'--namespace', cast(str, spec.service_id),
	283	'rm', 'grace',
b3b6e05e	284	]
b3b6e05e TL	285	subprocess.run(
	286	cmd,
	287	stdout=subprocess.PIPE,
	288	stderr=subprocess.PIPE,
	289	timeout=10
	290	)