ceph/src/cephadm/box/box.py

   1 #!/bin/python3
   2 import argparse
   3 import os
   4 import stat
   5 import sys
   6
   7 import host
   8 import osd
   9 from util import (
  10     Config,
  11     Target,
  12     ensure_inside_container,
  13     ensure_outside_container,
  14     get_boxes_container_info,
  15     run_cephadm_shell_command,
  16     run_dc_shell_command,
  17     run_shell_command,
  18 )
  19
  20 CEPH_IMAGE = 'quay.ceph.io/ceph-ci/ceph:master'
  21 BOX_IMAGE = 'cephadm-box:latest'
  22
  23 # NOTE: this image tar is a trickeroo so cephadm won't pull the image everytime
  24 # we deploy a cluster. Keep in mind that you'll be responsible of pulling the
  25 # image yourself with `box cluster setup`
  26 CEPH_IMAGE_TAR = 'docker/ceph/image/quay.ceph.image.tar'
  27
  28
  29 def remove_ceph_image_tar():
  30     if os.path.exists(CEPH_IMAGE_TAR):
  31         os.remove(CEPH_IMAGE_TAR)
  32
  33
  34 def cleanup_box() -> None:
  35     osd.cleanup()
  36     remove_ceph_image_tar()
  37
  38
  39 def image_exists(image_name: str):
  40     # extract_tag
  41     assert image_name.find(':')
  42     image_name, tag = image_name.split(':')
  43     images = run_shell_command('docker image ls').split('\n')
  44     IMAGE_NAME = 0
  45     TAG = 1
  46     for image in images:
  47         image = image.split()
  48         print(image)
  49         print(image_name, tag)
  50         if image[IMAGE_NAME] == image_name and image[TAG] == tag:
  51             return True
  52     return False
  53
  54
  55 def get_ceph_image():
  56     print('Getting ceph image')
  57     run_shell_command(f'docker pull {CEPH_IMAGE}')
  58     # update
  59     run_shell_command(f'docker build -t {CEPH_IMAGE} docker/ceph')
  60     if not os.path.exists('docker/ceph/image'):
  61         os.mkdir('docker/ceph/image')
  62
  63     remove_ceph_image_tar()
  64
  65     run_shell_command(f'docker save {CEPH_IMAGE} -o {CEPH_IMAGE_TAR}')
  66     print('Ceph image added')
  67
  68
  69 def get_box_image():
  70     print('Getting box image')
  71     run_shell_command('docker build -t cephadm-box -f Dockerfile .')
  72     print('Box image added')
  73
  74
  75 class Cluster(Target):
  76     _help = 'Manage docker cephadm boxes'
  77     actions = ['bootstrap', 'start', 'down', 'list', 'sh', 'setup', 'cleanup']
  78
  79     def set_args(self):
  80         self.parser.add_argument(
  81             'action', choices=Cluster.actions, help='Action to perform on the box'
  82         )
  83         self.parser.add_argument('--osds', type=int, default=3, help='Number of osds')
  84
  85         self.parser.add_argument('--hosts', type=int, default=2, help='Number of hosts')
  86         self.parser.add_argument('--skip-deploy-osds', action='store_true', help='skip deploy osd')
  87         self.parser.add_argument('--skip-create-loop', action='store_true', help='skip create loopback device')
  88         self.parser.add_argument('--skip-monitoring-stack', action='store_true', help='skip monitoring stack')
  89         self.parser.add_argument('--skip-dashboard', action='store_true', help='skip dashboard')
  90         self.parser.add_argument('--expanded', action='store_true', help='deploy 3 hosts and 3 osds')
  91
  92     @ensure_outside_container
  93     def setup(self):
  94         get_ceph_image()
  95         get_box_image()
  96
  97     @ensure_outside_container
  98     def cleanup(self):
  99         cleanup_box()
 100
 101     @ensure_inside_container
 102     def bootstrap(self):
 103         print('Running bootstrap on seed')
 104         cephadm_path = os.environ.get('CEPHADM_PATH')
 105         os.symlink('/cephadm/cephadm', cephadm_path)
 106         run_shell_command(
 107             'systemctl restart docker'
 108         )  # restart to ensure docker is using daemon.json
 109
 110         st = os.stat(cephadm_path)
 111         os.chmod(cephadm_path, st.st_mode | stat.S_IEXEC)
 112
 113         run_shell_command('docker load < /cephadm/box/docker/ceph/image/quay.ceph.image.tar')
 114         # cephadm guid error because it sometimes tries to use quay.ceph.io/ceph-ci/ceph:<none>
 115         # instead of master's tag
 116         run_shell_command('export CEPH_SOURCE_FOLDER=/ceph')
 117         run_shell_command('export CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master')
 118         run_shell_command(
 119             'echo "export CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master" >> ~/.bashrc'
 120         )
 121
 122         extra_args = []
 123
 124         extra_args.append('--skip-pull')
 125
 126         # cephadm prints in warning, let's redirect it to the output so shell_command doesn't
 127         # complain
 128         extra_args.append('2>&0')
 129
 130         extra_args = ' '.join(extra_args)
 131         skip_monitoring_stack = (
 132             '--skip-monitoring-stack' if Config.get('skip-monitoring-stack') else ''
 133         )
 134         skip_dashboard = '--skip-dashboard' if Config.get('skip-dashboard') else ''
 135
 136         fsid = Config.get('fsid')
 137         config_folder = Config.get('config_folder')
 138         config = Config.get('config')
 139         keyring = Config.get('keyring')
 140         if not os.path.exists(config_folder):
 141             os.mkdir(config_folder)
 142
 143         cephadm_bootstrap_command = (
 144             '$CEPHADM_PATH --verbose bootstrap '
 145             '--mon-ip "$(hostname -i)" '
 146             '--allow-fqdn-hostname '
 147             '--initial-dashboard-password admin '
 148             '--dashboard-password-noupdate '
 149             '--shared_ceph_folder /ceph '
 150             '--allow-overwrite '
 151             f'--output-config {config} '
 152             f'--output-keyring {keyring} '
 153             f'--output-config {config} '
 154             f'--fsid "{fsid}" '
 155             '--log-to-file '
 156             f'{skip_dashboard} '
 157             f'{skip_monitoring_stack} '
 158             f'{extra_args} '
 159         )
 160
 161         print('Running cephadm bootstrap...')
 162         run_shell_command(cephadm_bootstrap_command)
 163         print('Cephadm bootstrap complete')
 164
 165         run_shell_command('sudo vgchange --refresh')
 166         run_shell_command('cephadm ls')
 167         run_shell_command('ln -s /ceph/src/cephadm/box/box.py /usr/bin/box')
 168
 169         # NOTE: sometimes cephadm in the box takes a while to update the containers
 170         # running in the cluster and it cannot deploy the osds. In this case
 171         # run: box -v osd deploy --vg vg1 to deploy osds again.
 172         run_cephadm_shell_command('ceph -s')
 173         print('Bootstrap completed!')
 174
 175     @ensure_outside_container
 176     def start(self):
 177         osds = Config.get('osds')
 178         hosts = Config.get('hosts')
 179
 180         # ensure boxes don't exist
 181         run_shell_command('docker-compose down')
 182
 183         print('Checking docker images')
 184         if not image_exists(CEPH_IMAGE):
 185             get_ceph_image()
 186         if not image_exists(BOX_IMAGE):
 187             get_box_image()
 188
 189         if not Config.get('skip_create_loop'):
 190             print('Adding logical volumes (block devices) in loopback device...')
 191             osd.create_loopback_devices(osds)
 192             print(f'Added {osds} logical volumes in a loopback device')
 193
 194         print('Starting containers')
 195
 196         dcflags = '-f docker-compose.yml'
 197         if not os.path.exists('/sys/fs/cgroup/cgroup.controllers'):
 198             dcflags += ' -f docker-compose.cgroup1.yml'
 199         run_shell_command(f'docker-compose {dcflags} up --scale hosts={hosts} -d')
 200
 201         run_shell_command('sudo sysctl net.ipv4.conf.all.forwarding=1')
 202         run_shell_command('sudo iptables -P FORWARD ACCEPT')
 203
 204         print('Seting up host ssh servers')
 205         for h in range(hosts):
 206             host._setup_ssh(h + 1)
 207
 208         verbose = '-v' if Config.get('verbose') else ''
 209         skip_deploy = '--skip-deploy-osds' if Config.get('skip-deploy-osds') else ''
 210         skip_monitoring_stack = (
 211             '--skip-monitoring-stack' if Config.get('skip-monitoring-stack') else ''
 212         )
 213         skip_dashboard = '--skip-dashboard' if Config.get('skip-dashboard') else ''
 214         box_bootstrap_command = (
 215             f'/cephadm/box/box.py {verbose} cluster bootstrap '
 216             f'--osds {osds} '
 217             f'--hosts {hosts} '
 218             f'{skip_deploy} '
 219             f'{skip_dashboard} '
 220             f'{skip_monitoring_stack} '
 221         )
 222         run_dc_shell_command(box_bootstrap_command, 1, 'seed')
 223
 224         info = get_boxes_container_info()
 225         ips = info['ips']
 226         hostnames = info['hostnames']
 227         print(ips)
 228         host._copy_cluster_ssh_key(ips)
 229
 230         expanded = Config.get('expanded')
 231         if expanded:
 232             host._add_hosts(ips, hostnames)
 233
 234         if expanded and not Config.get('skip-deploy-osds'):
 235             print('Deploying osds... This could take up to minutes')
 236             osd.deploy_osds_in_vg('vg1')
 237             print('Osds deployed')
 238
 239         print('Bootstrap finished successfully')
 240
 241     @ensure_outside_container
 242     def down(self):
 243         run_shell_command('docker-compose down')
 244         cleanup_box()
 245         print('Successfully killed all boxes')
 246
 247     @ensure_outside_container
 248     def list(self):
 249         info = get_boxes_container_info(with_seed=True)
 250         for i in range(info['size']):
 251             ip = info['ips'][i]
 252             name = info['container_names'][i]
 253             hostname = info['hostnames'][i]
 254             print(f'{name} \t{ip} \t{hostname}')
 255
 256     @ensure_outside_container
 257     def sh(self):
 258         # we need verbose to see the prompt after running shell command
 259         Config.set('verbose', True)
 260         print('Seed bash')
 261         run_shell_command('docker-compose exec seed bash')
 262
 263
 264 targets = {
 265     'cluster': Cluster,
 266     'osd': osd.Osd,
 267     'host': host.Host,
 268 }
 269
 270
 271 def main():
 272     parser = argparse.ArgumentParser()
 273     parser.add_argument(
 274         '-v', action='store_true', dest='verbose', help='be more verbose'
 275     )
 276
 277     subparsers = parser.add_subparsers()
 278     target_instances = {}
 279     for name, target in targets.items():
 280         target_instances[name] = target(None, subparsers)
 281
 282     for count, arg in enumerate(sys.argv, 1):
 283         if arg in targets:
 284             instance = target_instances[arg]
 285             if hasattr(instance, 'main'):
 286                 instance.argv = sys.argv[count:]
 287                 instance.set_args()
 288                 args = parser.parse_args()
 289                 Config.add_args(vars(args))
 290                 instance.main()
 291                 sys.exit(0)
 292
 293     parser.print_help()
 294
 295
 296 if __name__ == '__main__':
 297     main()