ceph/qa/tasks/vip.py

   1 import contextlib
   2 import ipaddress
   3 import logging
   4 import re
   5
   6 from teuthology.config import config as teuth_config
   7
   8 log = logging.getLogger(__name__)
   9
  10
  11 def subst_vip(ctx, cmd):
  12     p = re.compile(r'({{VIP(\d+)}})')
  13     for m in p.findall(cmd):
  14         n = int(m[1])
  15         if n >= len(ctx.vip["vips"]):
  16             log.warning(f'no VIP{n} (we have {len(ctx.vip["vips"])})')
  17         else:
  18             cmd = cmd.replace(m[0], str(ctx.vip["vips"][n]))
  19
  20     if '{{VIPPREFIXLEN}}' in cmd:
  21         cmd = cmd.replace('{{VIPPREFIXLEN}}', str(ctx.vip["vnet"].prefixlen))
  22
  23     if '{{VIPSUBNET}}' in cmd:
  24         cmd = cmd.replace('{{VIPSUBNET}}', str(ctx.vip["vnet"].network_address))
  25
  26     return cmd
  27
  28
  29 def echo(ctx, config):
  30     """
  31     This is mostly for debugging
  32     """
  33     for remote in ctx.cluster.remotes.keys():
  34         log.info(subst_vip(ctx, config))
  35
  36
  37 def map_vips(mip, count):
  38     for mapping in teuth_config.get('vip', []):
  39         mnet = ipaddress.ip_network(mapping['machine_subnet'])
  40         vnet = ipaddress.ip_network(mapping['virtual_subnet'])
  41         if vnet.prefixlen >= mnet.prefixlen:
  42             log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix")
  43             return None
  44         if mip in mnet:
  45             pos = list(mnet.hosts()).index(mip)
  46             log.info(f"{mip} in {mnet}, pos {pos}")
  47             r = []
  48             for sub in vnet.subnets(new_prefix=mnet.prefixlen):
  49                 r += [list(sub.hosts())[pos]]
  50                 count -= 1
  51                 if count == 0:
  52                     break
  53             return vnet, r
  54     return None
  55
  56
  57 @contextlib.contextmanager
  58 def task(ctx, config):
  59     """
  60     Set up a virtual network and allocate virtual IP(s) for each machine.
  61
  62     The strategy here is to set up a private virtual subnet that is larger than
  63     the subnet the machine(s) exist in, and allocate virtual IPs from that pool.
  64
  65     - The teuthology.yaml must include a section like::
  66
  67         vip:
  68           - machine_subnet: 172.21.0.0/20
  69             virtual_subnet: 10.0.0.0/16
  70
  71       At least one item's machine_subnet should map the subnet the test machine's
  72       primary IP lives in (the one DNS resolves to).  The virtual_subnet must have a
  73       shorter prefix (i.e., larger than the machine_subnet).  If there are multiple
  74       machine_subnets, they cannot map into the same virtual_subnet.
  75
  76     - Each machine gets an IP in the virtual_subset statically configured by the vip
  77       task. This lets all test machines reach each other and (most importantly) any
  78       virtual IPs.
  79
  80     - 1 or more virtual IPs are then mapped for the task.  These IPs are chosen based
  81       on one of the remotes.  This uses a lot of network space but it avoids any
  82       conflicts between tests.
  83
  84     To use a virtual IP, the {{VIP0}}, {{VIP1}}, etc. substitutions can be used.
  85
  86     {{VIPSUBNET}} is the virtual_subnet address (10.0.0.0 in the example).
  87
  88     {{VIPPREFIXLEN}} is the virtual_subnet prefix (16 in the example.
  89
  90     These substitutions work for vip.echo, and (at the time of writing) cephadm.apply
  91     and cephadm.shell.
  92     """
  93     if config is None:
  94         config = {}
  95     count = config.get('count', 1)
  96
  97     ctx.vip_static = {}
  98     ctx.vip = {}
  99
 100     log.info("Allocating static IPs for each host...")
 101     for remote in ctx.cluster.remotes.keys():
 102         ip = remote.ssh.get_transport().getpeername()[0]
 103         log.info(f'peername {ip}')
 104         mip = ipaddress.ip_address(ip)
 105         vnet, vips = map_vips(mip, count + 1)
 106         static = vips.pop(0)
 107         log.info(f"{remote.hostname} static {static}, vnet {vnet}")
 108
 109         if not ctx.vip:
 110             # do this only once (use the first remote we see), since we only need 1
 111             # set of virtual IPs, regardless of how many remotes we have.
 112             log.info("VIPs are {map(str, vips)}")
 113             ctx.vip = {
 114                 'vnet': vnet,
 115                 'vips': vips,
 116             }
 117         else:
 118             # all remotes must be in the same virtual network...
 119             assert vnet == ctx.vip['vnet']
 120
 121         # pick interface
 122         p = re.compile(r'^(\S+) dev (\S+) (.*)scope link (.*)src (\S+)')
 123         iface = None
 124         for line in remote.sh(['sudo', 'ip','route','ls']).splitlines():
 125             m = p.findall(line)
 126             if not m:
 127                 continue
 128             route_iface = m[0][1]
 129             route_ip = m[0][4]
 130             if route_ip == ip:
 131                 iface = route_iface
 132                 break
 133
 134         if not iface:
 135             log.error(f"Unable to find {remote.hostname} interface for {ip}")
 136             continue
 137
 138         # configure
 139         log.info(f"Configuring {static} on {remote.hostname} iface {iface}...")
 140         remote.sh(['sudo',
 141                    'ip', 'addr', 'add',
 142                    str(static) + '/' + str(vnet.prefixlen),
 143                    'dev', iface])
 144
 145         ctx.vip_static[remote] = {
 146             "iface": iface,
 147             "static": static,
 148         }
 149
 150     try:
 151         yield
 152
 153     finally:
 154         for remote, m in ctx.vip_static.items():
 155             log.info(f"Removing {m['static']} (and any VIPs) on {remote.hostname} iface {m['iface']}...")
 156             remote.sh(['sudo',
 157                        'ip', 'addr', 'del',
 158                        str(m['static']) + '/' + str(ctx.vip['vnet'].prefixlen),
 159                        'dev', m['iface']])
 160
 161             for vip in ctx.vip['vips']:
 162                 remote.sh(
 163                     [
 164                         'sudo',
 165                         'ip', 'addr', 'del',
 166                         str(vip) + '/' + str(ctx.vip['vnet'].prefixlen),
 167                         'dev', m['iface']
 168                     ],
 169                     check_status=False,
 170                 )
 171