ceph/src/cephadm/box/osd.py

   1 import json
   2 import os
   3 from typing import Dict
   4
   5 from util import (
   6     Config,
   7     Target,
   8     ensure_inside_container,
   9     ensure_outside_container,
  10     get_orch_hosts,
  11     inside_container,
  12     run_cephadm_shell_command,
  13     run_dc_shell_command,
  14     run_shell_command,
  15 )
  16
  17
  18 def remove_loop_img() -> None:
  19     loop_image = Config.get('loop_img')
  20     if os.path.exists(loop_image):
  21         os.remove(loop_image)
  22
  23
  24 @ensure_outside_container
  25 def create_loopback_devices(osds: int) -> None:
  26     assert osds
  27     size = (5 * osds) + 1
  28     print(f'Using {size}GB of data to store osds')
  29     avail_loop = run_shell_command('sudo losetup -f')
  30
  31     # create loop if we cannot find it
  32     if not os.path.exists(avail_loop):
  33         num_loops = int(run_shell_command("lsmod | grep loop | awk '{print $3}'"))
  34         num_loops += 1
  35         run_shell_command(f'mknod {avail_loop} b 7 {num_loops}')
  36
  37     if os.path.ismount(avail_loop):
  38         os.umount(avail_loop)
  39
  40     loop_devices = json.loads(run_shell_command('losetup -l -J', expect_error=True))
  41     for dev in loop_devices['loopdevices']:
  42         if dev['name'] == avail_loop:
  43             run_shell_command(f'sudo losetup -d {avail_loop}')
  44
  45     if not os.path.exists('./loop-images'):
  46         os.mkdir('loop-images')
  47
  48     remove_loop_img()
  49
  50     loop_image = Config.get('loop_img')
  51     run_shell_command(f'sudo dd if=/dev/zero of={loop_image} bs=1 count=0 seek={size}G')
  52     run_shell_command(f'sudo losetup {avail_loop} {loop_image}')
  53
  54     # cleanup last call
  55     cleanup()
  56
  57     run_shell_command(f'sudo pvcreate {avail_loop} ')
  58     run_shell_command(f'sudo vgcreate vg1 {avail_loop}')
  59
  60     p = int(100 / osds)
  61     for i in range(osds):
  62         run_shell_command('sudo vgchange --refresh')
  63         run_shell_command(f'sudo lvcreate -l {p}%VG --name lv{i} vg1')
  64
  65
  66 def get_lvm_osd_data(data: str) -> Dict[str, str]:
  67     osd_lvm_info = run_cephadm_shell_command(f'ceph-volume lvm list {data}')
  68     osd_data = {}
  69     for line in osd_lvm_info.split('\n'):
  70         line = line.strip()
  71         if not line:
  72             continue
  73         line = line.split()
  74         if line[0].startswith('===') or line[0].startswith('[block]'):
  75             continue
  76         # "block device" key -> "block_device"
  77         key = '_'.join(line[:-1])
  78         osd_data[key] = line[-1]
  79     return osd_data
  80
  81
  82 @ensure_inside_container
  83 def deploy_osd(data: str, hostname: str) -> bool:
  84     out = run_cephadm_shell_command(f'ceph orch daemon add osd "{hostname}:{data}"')
  85     return 'Created osd(s)' in out
  86
  87
  88 def cleanup() -> None:
  89     vg = 'vg1'
  90     pvs = json.loads(run_shell_command('sudo pvs --reportformat json'))
  91     for pv in pvs['report'][0]['pv']:
  92         if pv['vg_name'] == vg:
  93             device = pv['pv_name']
  94             run_shell_command(f'sudo vgremove -f --yes {vg}')
  95             run_shell_command(f'sudo losetup -d {device}')
  96             run_shell_command(f'sudo wipefs -af {device}')
  97             # FIX: this can fail with excluded filter
  98             run_shell_command(f'sudo pvremove -f --yes {device}', expect_error=True)
  99             break
 100
 101     remove_loop_img()
 102
 103
 104 def deploy_osds_in_vg(vg: str):
 105     """
 106     rotate host will deploy each osd in a different host
 107
 108     deploying osds will not succeed with starting services so this
 109     makes another process to run on the background
 110     """
 111     if inside_container():
 112         lvs = json.loads(run_shell_command('lvs --reportformat json'))
 113         # distribute osds per host
 114         hosts = get_orch_hosts()
 115         host_index = 0
 116         for lv in lvs['report'][0]['lv']:
 117             if lv['vg_name'] == vg:
 118                 deployed = False
 119                 while not deployed:
 120                     deployed = deploy_osd(
 121                         f'{vg}/{lv["lv_name"]}', hosts[host_index]['hostname']
 122                     )
 123                 host_index = (host_index + 1) % len(hosts)
 124     else:
 125         verbose = '-v' if Config.get('verbose') else ''
 126         print('Redirecting deploy osd in vg to inside container')
 127         run_dc_shell_command(
 128             f'/cephadm/box/box.py {verbose} osd deploy --vg {vg}', 1, 'seed'
 129         )
 130
 131
 132 class Osd(Target):
 133     _help = """
 134     Deploy osds and create needed block devices with loopback devices:
 135     Actions:
 136     - deploy: Deploy an osd given a block device
 137     - create_loop: Create needed loopback devices and block devices in logical volumes
 138     for a number of osds.
 139     """
 140     actions = ['deploy', 'create_loop']
 141
 142     def set_args(self):
 143         self.parser.add_argument('action', choices=Osd.actions)
 144         self.parser.add_argument('--data', type=str, help='path to a block device')
 145         self.parser.add_argument('--hostname', type=str, help='host to deploy osd')
 146         self.parser.add_argument('--osds', type=int, default=0, help='number of osds')
 147         self.parser.add_argument(
 148             '--vg', type=str, help='Deploy with all lv from virtual group'
 149         )
 150
 151     def deploy(self):
 152         data = Config.get('data')
 153         hostname = Config.get('hostname')
 154         vg = Config.get('vg')
 155         if not hostname:
 156             # assume this host
 157             hostname = run_shell_command('hostname')
 158         if vg:
 159             deploy_osds_in_vg(vg)
 160         else:
 161             deploy_osd(data, hostname)
 162
 163     @ensure_outside_container
 164     def create_loop(self):
 165         osds = Config.get('osds')
 166         create_loopback_devices(osds)
 167         print('Successfully added logical volumes in loopback devices')