[ceph.git] / ceph / src / cephadm / box / util.py

import json
import os
import subprocess
import sys
import copy
from abc import ABCMeta, abstractmethod
from enum import Enum
from typing import Any, Callable, Dict, List

class Colors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKCYAN = '\033[96m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

class Config:
    args = {
        'fsid': '00000000-0000-0000-0000-0000deadbeef',
        'config_folder': '/etc/ceph/',
        'config': '/etc/ceph/ceph.conf',
        'keyring': '/etc/ceph/ceph.keyring',
        'loop_img': 'loop-images/loop.img',
        'engine': 'podman',
        'docker_yaml': 'docker-compose-docker.yml',
        'docker_v1_yaml': 'docker-compose.cgroup1.yml',
        'podman_yaml': 'docker-compose-podman.yml',
        'loop_img_dir': 'loop-images',
    }

    @staticmethod
    def set(key, value):
        Config.args[key] = value

    @staticmethod
    def get(key):
        if key in Config.args:
            return Config.args[key]
        return None

    @staticmethod
    def add_args(args: Dict[str, str]) -> None:
        Config.args.update(args)

class Target:
    def __init__(self, argv, subparsers):
        self.argv = argv
        self.parser = subparsers.add_parser(
            self.__class__.__name__.lower(), help=self.__class__._help
        )

    def set_args(self):
        """
        adding the required arguments of the target should go here, example:
        self.parser.add_argument(..)
        """
        raise NotImplementedError()

    def main(self):
        """
        A target will be setup by first calling this main function
        where the parser is initialized.
        """
        args = self.parser.parse_args(self.argv)
        Config.add_args(vars(args))
        function = getattr(self, args.action)
        function()


def ensure_outside_container(func) -> Callable:
    def wrapper(*args, **kwargs):
        if not inside_container():
            return func(*args, **kwargs)
        else:
            raise RuntimeError('This command should be ran outside a container')

    return wrapper


def ensure_inside_container(func) -> bool:
    def wrapper(*args, **kwargs):
        if inside_container():
            return func(*args, **kwargs)
        else:
            raise RuntimeError('This command should be ran inside a container')

    return wrapper


def colored(msg, color: Colors):
    return color + msg + Colors.ENDC

class BoxType(str, Enum):
  SEED = 'seed'
  HOST = 'host'

class HostContainer:
    def __init__(self, _name, _type) -> None:
        self._name: str = _name
        self._type: BoxType = _type

    @property
    def name(self) -> str:
        return self._name

    @property
    def type(self) -> BoxType:
        return self._type
    def __str__(self) -> str:
        return f'{self.name} {self.type}'

def run_shell_command(command: str, expect_error=False, verbose=True, expect_exit_code=0) -> str:
    if Config.get('verbose'):
        print(f'{colored("Running command", Colors.HEADER)}: {colored(command, Colors.OKBLUE)}')

    process = subprocess.Popen(
        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )

    out = ''
    err = ''
    # let's read when output comes so it is in real time
    while True:
        # TODO: improve performance of this part, I think this part is a problem
        pout = process.stdout.read(1).decode('latin1')
        if pout == '' and process.poll() is not None:
            break
        if pout:
            if Config.get('verbose') and verbose:
                sys.stdout.write(pout)
                sys.stdout.flush()
            out += pout

    process.wait()

    err += process.stderr.read().decode('latin1').strip()
    out = out.strip()

    if process.returncode != 0 and not expect_error and process.returncode != expect_exit_code:
        err = colored(err, Colors.FAIL);
        
        raise RuntimeError(f'Failed command: {command}\n{err}\nexit code: {process.returncode}')
        sys.exit(1)
    return out


def run_dc_shell_commands(commands: str, container: HostContainer, expect_error=False) -> str:
    for command in commands.split('\n'):
        command = command.strip()
        if not command:
            continue
        run_dc_shell_command(command.strip(), container, expect_error=expect_error)

def run_shell_commands(commands: str, expect_error=False) -> str:
    for command in commands.split('\n'):
        command = command.strip()
        if not command:
            continue
        run_shell_command(command, expect_error=expect_error)

@ensure_inside_container
def run_cephadm_shell_command(command: str, expect_error=False) -> str:
    config = Config.get('config')
    keyring = Config.get('keyring')
    fsid = Config.get('fsid')

    with_cephadm_image = 'CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:main'
    out = run_shell_command(
        f'{with_cephadm_image} cephadm --verbose shell --fsid {fsid} --config {config} --keyring {keyring} -- {command}',
        expect_error,
    )
    return out


def run_dc_shell_command(
        command: str, container: HostContainer, expect_error=False
) -> str:
    out = get_container_engine().run_exec(container, command, expect_error=expect_error)
    return out

def inside_container() -> bool:
    return os.path.exists('/.box_container')

def get_container_id(container_name: str):
    return run_shell_command(f"{engine()} ps | \grep " + container_name + " | awk '{ print $1 }'")

def engine():
    return Config.get('engine')

def engine_compose():
    return f'{engine()}-compose'

def get_seed_name():
    if engine() == 'docker':
        return 'seed'
    elif engine() == 'podman':
        return 'box_hosts_0'
    else:
        print(f'unkown engine {engine()}')
        sys.exit(1)


@ensure_outside_container
def get_boxes_container_info(with_seed: bool = False) -> Dict[str, Any]:
    # NOTE: this could be cached
    ips_query = engine() + " inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} %tab% {{.Name}} %tab% {{.Config.Hostname}}' $("+ engine() + " ps -aq) --format json"
    containers = json.loads(run_shell_command(ips_query, verbose=False))
    # FIXME: if things get more complex a class representing a container info might be useful,
    # for now representing data this way is faster.
    info = {'size': 0, 'ips': [], 'container_names': [], 'hostnames': []}
    for container in containers:
        # Most commands use hosts only
        name = container['Name']
        if name.startswith('box_hosts'):
            if not with_seed and name == get_seed_name():
                continue
            info['size'] += 1
            print(container['NetworkSettings'])
            if 'Networks' in container['NetworkSettings']:
                info['ips'].append(container['NetworkSettings']['Networks']['box_network']['IPAddress'])
            else:
                info['ips'].append('n/a')
            info['container_names'].append(name)
            info['hostnames'].append(container['Config']['Hostname'])
    return info


def get_orch_hosts():
    if inside_container():
        orch_host_ls_out = run_cephadm_shell_command('ceph orch host ls --format json')
    else:
        orch_host_ls_out = run_dc_shell_command(f'cephadm shell --keyring /etc/ceph/ceph.keyring --config /etc/ceph/ceph.conf -- ceph orch host ls --format json', 
                                                get_container_engine().get_seed())
        sp = orch_host_ls_out.split('\n')
        orch_host_ls_out  = sp[len(sp) - 1]
    hosts = json.loads(orch_host_ls_out)
    return hosts


class ContainerEngine(metaclass=ABCMeta):
    @property
    @abstractmethod
    def command(self) -> str: pass

    @property
    @abstractmethod
    def seed_name(self) -> str: pass

    @property
    @abstractmethod
    def dockerfile(self) -> str: pass

    @property
    def host_name_prefix(self) -> str: 
        return 'box_hosts_'

    @abstractmethod
    def up(self, hosts: int): pass

    def run_exec(self, container: HostContainer, command: str, expect_error: bool = False):
        return run_shell_command(' '.join([self.command, 'exec', container.name, command]), 
                                 expect_error=expect_error) 

    def run(self, engine_command: str, expect_error: bool = False):
        return run_shell_command(' '.join([self.command, engine_command]), expect_error=expect_error) 

    def get_containers(self) -> List[HostContainer]:
        ps_out = json.loads(run_shell_command('podman ps --format json'))
        containers = [] 
        for container in ps_out:
            if not container['Names']:
                raise RuntimeError(f'Container {container} missing name')
            name = container['Names'][0]
            if name == self.seed_name:
                containers.append(HostContainer(name, BoxType.SEED))
            elif name.startswith(self.host_name_prefix):
                containers.append(HostContainer(name, BoxType.HOST))
        return containers

    def get_seed(self) -> HostContainer:
        for container in self.get_containers():
            if container.type == BoxType.SEED:
                return container
        raise RuntimeError('Missing seed container')

    def get_container(self, container_name: str):
        containers = self.get_containers()
        for container in containers:
            if container.name == container_name:
                return container
        return None


    def restart(self):
        pass


class DockerEngine(ContainerEngine):
    command = 'docker'
    seed_name = 'seed'
    dockerfile = 'DockerfileDocker'

    def restart(self):
        run_shell_command('systemctl restart docker')

    def up(self, hosts: int):
        dcflags = f'-f {Config.get("docker_yaml")}'
        if not os.path.exists('/sys/fs/cgroup/cgroup.controllers'):
            dcflags += f' -f {Config.get("docker_v1_yaml")}'
        run_shell_command(f'{engine_compose()} {dcflags} up --scale hosts={hosts} -d')

class PodmanEngine(ContainerEngine):
    command = 'podman'
    seed_name = 'box_hosts_0'
    dockerfile = 'DockerfilePodman'

    CAPS = [
            "SYS_ADMIN",
            "NET_ADMIN",
            "SYS_TIME",
            "SYS_RAWIO",
            "MKNOD",
            "NET_RAW",
            "SETUID",
            "SETGID",
            "CHOWN",
            "SYS_PTRACE",
            "SYS_TTY_CONFIG",
            "CAP_AUDIT_WRITE",
            "CAP_AUDIT_CONTROL",
            ]

    VOLUMES = [
                '../../../:/ceph:z',
                '../:/cephadm:z',
                '/run/udev:/run/udev',
                '/sys/dev/block:/sys/dev/block',
                '/sys/fs/cgroup:/sys/fs/cgroup:ro',
                '/dev/fuse:/dev/fuse',
                '/dev/disk:/dev/disk',
                '/sys/devices/virtual/block:/sys/devices/virtual/block',
                '/sys/block:/dev/block',
                '/dev/mapper:/dev/mapper',
                '/dev/mapper/control:/dev/mapper/control',
            ]

    TMPFS = ['/run', '/tmp']

    # FIXME: right now we are assuming every service will be exposed through the seed, but this is far
    # from the truth. Services can be deployed on different hosts so we need a system to manage this.
    SEED_PORTS = [
            8443, # dashboard
            3000, # grafana
            9093, # alertmanager
            9095  # prometheus
            ]


    def setup_podman_env(self, hosts: int = 1, osd_devs={}):
        network_name = 'box_network'
        networks = run_shell_command('podman network ls')
        if network_name not in networks:
            run_shell_command(f'podman network create -d bridge {network_name}')

        args = [
                '--group-add', 'keep-groups', 
                '--device', '/dev/fuse' ,
                '-it' ,
                '-d',
                '-e', 'CEPH_BRANCH=main',
                '--stop-signal', 'RTMIN+3'
                ]

        for cap in self.CAPS:
            args.append('--cap-add')
            args.append(cap)

        for volume in self.VOLUMES:
            args.append('-v')
            args.append(volume)

        for tmp in self.TMPFS:
            args.append('--tmpfs')
            args.append(tmp)


        for osd_dev in osd_devs.values():
            device = osd_dev["device"]
            args.append('--device')
            args.append(f'{device}:{device}')


        for host in range(hosts+1): # 0 will be the seed
            options = copy.copy(args)
            options.append('--name')
            options.append(f'box_hosts_{host}')
            options.append('--network')
            options.append(f'{network_name}')
            if host == 0:
                for port in self.SEED_PORTS:
                    options.append('-p')
                    options.append(f'{port}:{port}')

            options.append('cephadm-box')
            options = ' '.join(options)

            run_shell_command(f'podman run {options}')

    def up(self, hosts: int):
        import osd
        self.setup_podman_env(hosts=hosts, osd_devs=osd.load_osd_devices())

def get_container_engine() -> ContainerEngine:
    if engine() == 'docker':
        return DockerEngine()
    else:
        return PodmanEngine()
Commit	Line	Data
33c7a0ef	1	import json
20effc67 TL	2	import os
	3	import subprocess
	4	import sys
1e59de90 TL	5	import copy
	6	from abc import ABCMeta, abstractmethod
	7	from enum import Enum
	8	from typing import Any, Callable, Dict, List
20effc67	9
1e59de90 TL	10	class Colors:
	11	HEADER = '\033[95m'
	12	OKBLUE = '\033[94m'
	13	OKCYAN = '\033[96m'
	14	OKGREEN = '\033[92m'
	15	WARNING = '\033[93m'
	16	FAIL = '\033[91m'
	17	ENDC = '\033[0m'
	18	BOLD = '\033[1m'
	19	UNDERLINE = '\033[4m'
20effc67 TL	20
	21	class Config:
	22	args = {
	23	'fsid': '00000000-0000-0000-0000-0000deadbeef',
	24	'config_folder': '/etc/ceph/',
	25	'config': '/etc/ceph/ceph.conf',
	26	'keyring': '/etc/ceph/ceph.keyring',
	27	'loop_img': 'loop-images/loop.img',
1e59de90 TL	28	'engine': 'podman',
	29	'docker_yaml': 'docker-compose-docker.yml',
	30	'docker_v1_yaml': 'docker-compose.cgroup1.yml',
	31	'podman_yaml': 'docker-compose-podman.yml',
	32	'loop_img_dir': 'loop-images',
20effc67	33	}
33c7a0ef	34
20effc67 TL	35	@staticmethod
	36	def set(key, value):
	37	Config.args[key] = value
	38
	39	@staticmethod
	40	def get(key):
	41	if key in Config.args:
	42	return Config.args[key]
	43	return None
	44
	45	@staticmethod
33c7a0ef	46	def add_args(args: Dict[str, str]) -> None:
20effc67 TL	47	Config.args.update(args)
	48
	49	class Target:
	50	def __init__(self, argv, subparsers):
	51	self.argv = argv
33c7a0ef TL	52	self.parser = subparsers.add_parser(
	53	self.__class__.__name__.lower(), help=self.__class__._help
	54	)
20effc67 TL	55
	56	def set_args(self):
	57	"""
	58	adding the required arguments of the target should go here, example:
	59	self.parser.add_argument(..)
	60	"""
	61	raise NotImplementedError()
	62
	63	def main(self):
	64	"""
	65	A target will be setup by first calling this main function
	66	where the parser is initialized.
	67	"""
	68	args = self.parser.parse_args(self.argv)
	69	Config.add_args(vars(args))
	70	function = getattr(self, args.action)
	71	function()
	72
33c7a0ef TL	73
33c7a0ef TL	74	def ensure_outside_container(func) -> Callable:
20effc67 TL	75	def wrapper(args, *kwargs):
	76	if not inside_container():
	77	return func(args, *kwargs)
	78	else:
	79	raise RuntimeError('This command should be ran outside a container')
33c7a0ef	80
20effc67	81	return wrapper
33c7a0ef TL	82
33c7a0ef TL	83
20effc67 TL	84	def ensure_inside_container(func) -> bool:
	85	def wrapper(args, *kwargs):
	86	if inside_container():
	87	return func(args, *kwargs)
	88	else:
	89	raise RuntimeError('This command should be ran inside a container')
33c7a0ef	90
20effc67 TL	91	return wrapper
	92
	93
1e59de90 TL	94	def colored(msg, color: Colors):
	95	return color + msg + Colors.ENDC
	96
	97	class BoxType(str, Enum):
	98	SEED = 'seed'
	99	HOST = 'host'
	100
	101	class HostContainer:
	102	def __init__(self, _name, _type) -> None:
	103	self._name: str = _name
	104	self._type: BoxType = _type
	105
	106	@property
	107	def name(self) -> str:
	108	return self._name
	109
	110	@property
	111	def type(self) -> BoxType:
	112	return self._type
	113	def __str__(self) -> str:
	114	return f'{self.name} {self.type}'
	115
	116	def run_shell_command(command: str, expect_error=False, verbose=True, expect_exit_code=0) -> str:
20effc67	117	if Config.get('verbose'):
1e59de90 TL	118	print(f'{colored("Running command", Colors.HEADER)}: {colored(command, Colors.OKBLUE)}')
1e59de90 TL	119
33c7a0ef TL	120	process = subprocess.Popen(
	121	command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
	122	)
20effc67 TL	123
20effc67 TL	124	out = ''
1e59de90	125	err = ''
20effc67 TL	126	# let's read when output comes so it is in real time
	127	while True:
	128	# TODO: improve performance of this part, I think this part is a problem
33c7a0ef	129	pout = process.stdout.read(1).decode('latin1')
20effc67 TL	130	if pout == '' and process.poll() is not None:
	131	break
	132	if pout:
1e59de90	133	if Config.get('verbose') and verbose:
20effc67 TL	134	sys.stdout.write(pout)
	135	sys.stdout.flush()
	136	out += pout
1e59de90	137
20effc67 TL	138	process.wait()
20effc67 TL	139
1e59de90	140	err += process.stderr.read().decode('latin1').strip()
20effc67 TL	141	out = out.strip()
20effc67 TL	142
1e59de90 TL	143	if process.returncode != 0 and not expect_error and process.returncode != expect_exit_code:
	144	err = colored(err, Colors.FAIL);
	145
	146	raise RuntimeError(f'Failed command: {command}\n{err}\nexit code: {process.returncode}')
20effc67 TL	147	sys.exit(1)
	148	return out
	149
33c7a0ef	150
1e59de90 TL	151	def run_dc_shell_commands(commands: str, container: HostContainer, expect_error=False) -> str:
	152	for command in commands.split('\n'):
	153	command = command.strip()
	154	if not command:
	155	continue
	156	run_dc_shell_command(command.strip(), container, expect_error=expect_error)
	157
	158	def run_shell_commands(commands: str, expect_error=False) -> str:
	159	for command in commands.split('\n'):
	160	command = command.strip()
	161	if not command:
	162	continue
	163	run_shell_command(command, expect_error=expect_error)
	164
20effc67 TL	165	@ensure_inside_container
	166	def run_cephadm_shell_command(command: str, expect_error=False) -> str:
	167	config = Config.get('config')
	168	keyring = Config.get('keyring')
1e59de90	169	fsid = Config.get('fsid')
20effc67	170
1e59de90	171	with_cephadm_image = 'CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:main'
33c7a0ef	172	out = run_shell_command(
1e59de90	173	f'{with_cephadm_image} cephadm --verbose shell --fsid {fsid} --config {config} --keyring {keyring} -- {command}',
33c7a0ef TL	174	expect_error,
33c7a0ef TL	175	)
20effc67 TL	176	return out
20effc67 TL	177
33c7a0ef TL	178
33c7a0ef TL	179	def run_dc_shell_command(
1e59de90	180	command: str, container: HostContainer, expect_error=False
33c7a0ef	181	) -> str:
1e59de90	182	out = get_container_engine().run_exec(container, command, expect_error=expect_error)
20effc67 TL	183	return out
	184
	185	def inside_container() -> bool:
1e59de90 TL	186	return os.path.exists('/.box_container')
	187
	188	def get_container_id(container_name: str):
	189	return run_shell_command(f"{engine()} ps \| \grep " + container_name + " \| awk '{ print $1 }'")
	190
	191	def engine():
	192	return Config.get('engine')
	193
	194	def engine_compose():
	195	return f'{engine()}-compose'
	196
	197	def get_seed_name():
	198	if engine() == 'docker':
	199	return 'seed'
	200	elif engine() == 'podman':
	201	return 'box_hosts_0'
	202	else:
	203	print(f'unkown engine {engine()}')
	204	sys.exit(1)
20effc67	205
20effc67	206
33c7a0ef TL	207	@ensure_outside_container
	208	def get_boxes_container_info(with_seed: bool = False) -> Dict[str, Any]:
	209	# NOTE: this could be cached
1e59de90 TL	210	ips_query = engine() + " inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} %tab% {{.Name}} %tab% {{.Config.Hostname}}' $("+ engine() + " ps -aq) --format json"
1e59de90 TL	211	containers = json.loads(run_shell_command(ips_query, verbose=False))
33c7a0ef TL	212	# FIXME: if things get more complex a class representing a container info might be useful,
	213	# for now representing data this way is faster.
	214	info = {'size': 0, 'ips': [], 'container_names': [], 'hostnames': []}
1e59de90	215	for container in containers:
33c7a0ef	216	# Most commands use hosts only
1e59de90 TL	217	name = container['Name']
	218	if name.startswith('box_hosts'):
	219	if not with_seed and name == get_seed_name():
	220	continue
33c7a0ef	221	info['size'] += 1
1e59de90 TL	222	print(container['NetworkSettings'])
	223	if 'Networks' in container['NetworkSettings']:
	224	info['ips'].append(container['NetworkSettings']['Networks']['box_network']['IPAddress'])
	225	else:
	226	info['ips'].append('n/a')
	227	info['container_names'].append(name)
	228	info['hostnames'].append(container['Config']['Hostname'])
33c7a0ef TL	229	return info
	230
	231
	232	def get_orch_hosts():
1e59de90 TL	233	if inside_container():
	234	orch_host_ls_out = run_cephadm_shell_command('ceph orch host ls --format json')
	235	else:
	236	orch_host_ls_out = run_dc_shell_command(f'cephadm shell --keyring /etc/ceph/ceph.keyring --config /etc/ceph/ceph.conf -- ceph orch host ls --format json',
	237	get_container_engine().get_seed())
	238	sp = orch_host_ls_out.split('\n')
	239	orch_host_ls_out = sp[len(sp) - 1]
33c7a0ef TL	240	hosts = json.loads(orch_host_ls_out)
33c7a0ef TL	241	return hosts
1e59de90 TL	242
	243
	244	class ContainerEngine(metaclass=ABCMeta):
	245	@property
	246	@abstractmethod
	247	def command(self) -> str: pass
	248
	249	@property
	250	@abstractmethod
	251	def seed_name(self) -> str: pass
	252
	253	@property
	254	@abstractmethod
	255	def dockerfile(self) -> str: pass
	256
	257	@property
	258	def host_name_prefix(self) -> str:
	259	return 'box_hosts_'
	260
	261	@abstractmethod
	262	def up(self, hosts: int): pass
	263
	264	def run_exec(self, container: HostContainer, command: str, expect_error: bool = False):
	265	return run_shell_command(' '.join([self.command, 'exec', container.name, command]),
	266	expect_error=expect_error)
	267
	268	def run(self, engine_command: str, expect_error: bool = False):
	269	return run_shell_command(' '.join([self.command, engine_command]), expect_error=expect_error)
	270
	271	def get_containers(self) -> List[HostContainer]:
	272	ps_out = json.loads(run_shell_command('podman ps --format json'))
	273	containers = []
	274	for container in ps_out:
	275	if not container['Names']:
	276	raise RuntimeError(f'Container {container} missing name')
	277	name = container['Names'][0]
	278	if name == self.seed_name:
	279	containers.append(HostContainer(name, BoxType.SEED))
	280	elif name.startswith(self.host_name_prefix):
	281	containers.append(HostContainer(name, BoxType.HOST))
	282	return containers
	283
	284	def get_seed(self) -> HostContainer:
	285	for container in self.get_containers():
	286	if container.type == BoxType.SEED:
	287	return container
	288	raise RuntimeError('Missing seed container')
	289
	290	def get_container(self, container_name: str):
	291	containers = self.get_containers()
	292	for container in containers:
	293	if container.name == container_name:
	294	return container
	295	return None
	296
	297
	298	def restart(self):
	299	pass
	300
	301
	302	class DockerEngine(ContainerEngine):
	303	command = 'docker'
	304	seed_name = 'seed'
	305	dockerfile = 'DockerfileDocker'
306
307	def restart(self):
308	run_shell_command('systemctl restart docker')
309
310	def up(self, hosts: int):
311	dcflags = f'-f {Config.get("docker_yaml")}'
312	if not os.path.exists('/sys/fs/cgroup/cgroup.controllers'):
313	dcflags += f' -f {Config.get("docker_v1_yaml")}'
314	run_shell_command(f'{engine_compose()} {dcflags} up --scale hosts={hosts} -d')
315
316	class PodmanEngine(ContainerEngine):
317	command = 'podman'
318	seed_name = 'box_hosts_0'
319	dockerfile = 'DockerfilePodman'
320
321	CAPS = [
322	"SYS_ADMIN",
323	"NET_ADMIN",
324	"SYS_TIME",
325	"SYS_RAWIO",
326	"MKNOD",
327	"NET_RAW",
328	"SETUID",
329	"SETGID",
330	"CHOWN",
331	"SYS_PTRACE",
332	"SYS_TTY_CONFIG",
333	"CAP_AUDIT_WRITE",
334	"CAP_AUDIT_CONTROL",
335	]
336
337	VOLUMES = [
338	'../../../:/ceph:z',
339	'../:/cephadm:z',
340	'/run/udev:/run/udev',
341	'/sys/dev/block:/sys/dev/block',
342	'/sys/fs/cgroup:/sys/fs/cgroup:ro',
343	'/dev/fuse:/dev/fuse',
344	'/dev/disk:/dev/disk',
345	'/sys/devices/virtual/block:/sys/devices/virtual/block',
346	'/sys/block:/dev/block',
347	'/dev/mapper:/dev/mapper',
348	'/dev/mapper/control:/dev/mapper/control',
349	]
350
351	TMPFS = ['/run', '/tmp']
352
353	# FIXME: right now we are assuming every service will be exposed through the seed, but this is far
354	# from the truth. Services can be deployed on different hosts so we need a system to manage this.
355	SEED_PORTS = [
356	8443, # dashboard
357	3000, # grafana
358	9093, # alertmanager
359	9095 # prometheus
360	]
361
362
363	def setup_podman_env(self, hosts: int = 1, osd_devs={}):
364	network_name = 'box_network'
365	networks = run_shell_command('podman network ls')
366	if network_name not in networks:
367	run_shell_command(f'podman network create -d bridge {network_name}')
368
369	args = [
370	'--group-add', 'keep-groups',
371	'--device', '/dev/fuse' ,
372	'-it' ,
373	'-d',
374	'-e', 'CEPH_BRANCH=main',
375	'--stop-signal', 'RTMIN+3'
376	]
377
378	for cap in self.CAPS:
379	args.append('--cap-add')
380	args.append(cap)
381
382	for volume in self.VOLUMES:
383	args.append('-v')
384	args.append(volume)
385
386	for tmp in self.TMPFS:
387	args.append('--tmpfs')
388	args.append(tmp)
389
390
391	for osd_dev in osd_devs.values():
392	device = osd_dev["device"]
393	args.append('--device')
394	args.append(f'{device}:{device}')
395
396
397	for host in range(hosts+1): # 0 will be the seed
398	options = copy.copy(args)
399	options.append('--name')
400	options.append(f'box_hosts_{host}')
401	options.append('--network')
402	options.append(f'{network_name}')
403	if host == 0:
404	for port in self.SEED_PORTS:
405	options.append('-p')
406	options.append(f'{port}:{port}')
407
408	options.append('cephadm-box')
409	options = ' '.join(options)
410
411	run_shell_command(f'podman run {options}')
412
413	def up(self, hosts: int):
414	import osd
415	self.setup_podman_env(hosts=hosts, osd_devs=osd.load_osd_devices())
416
417	def get_container_engine() -> ContainerEngine:
418	if engine() == 'docker':
419	return DockerEngine()
420	else:
421	return PodmanEngine()