ceph/qa/tasks/kubeadm.py

   1 """
   2 Kubernetes cluster task, deployed via kubeadm
   3 """
   4 import argparse
   5 import contextlib
   6 import ipaddress
   7 import json
   8 import logging
   9 import random
  10 import yaml
  11 from io import BytesIO
  12
  13 from teuthology import misc as teuthology
  14 from teuthology import contextutil
  15 from teuthology.config import config as teuth_config
  16 from teuthology.orchestra import run
  17
  18 log = logging.getLogger(__name__)
  19
  20
  21 def _kubectl(ctx, config, args, **kwargs):
  22     cluster_name = config['cluster']
  23     ctx.kubeadm[cluster_name].bootstrap_remote.run(
  24         args=['kubectl'] + args,
  25         **kwargs,
  26     )
  27
  28
  29 def kubectl(ctx, config):
  30     if isinstance(config, str):
  31         config = [config]
  32     assert isinstance(config, list)
  33     for c in config:
  34         if isinstance(c, str):
  35             _kubectl(ctx, config, c.split(' '))
  36         else:
  37             _kubectl(ctx, config, c)
  38
  39
  40 @contextlib.contextmanager
  41 def preflight(ctx, config):
  42     run.wait(
  43         ctx.cluster.run(
  44             args=[
  45                 'sudo', 'modprobe', 'br_netfilter',
  46                 run.Raw('&&'),
  47                 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-ip6tables=1',
  48                 run.Raw('&&'),
  49                 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-iptables=1',
  50                 run.Raw('&&'),
  51                 'sudo', 'sysctl', 'net.ipv4.ip_forward=1',
  52                 run.Raw('&&'),
  53                 'sudo', 'swapoff', '-a',
  54             ],
  55             wait=False,
  56         )
  57     )
  58
  59     # set docker cgroup driver = systemd
  60     #  see https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
  61     #  see https://github.com/kubernetes/kubeadm/issues/2066
  62     for remote in ctx.cluster.remotes.keys():
  63         try:
  64             orig = remote.read_file('/etc/docker/daemon.json', sudo=True)
  65             j = json.loads(orig)
  66         except Exception as e:
  67             log.info(f'Failed to pull old daemon.json: {e}')
  68             j = {}
  69         j["exec-opts"] = ["native.cgroupdriver=systemd"]
  70         j["log-driver"] = "json-file"
  71         j["log-opts"] = {"max-size": "100m"}
  72         j["storage-driver"] = "overlay2"
  73         remote.write_file('/etc/docker/daemon.json', json.dumps(j), sudo=True)
  74     run.wait(
  75         ctx.cluster.run(
  76             args=[
  77                 'sudo', 'systemctl', 'restart', 'docker',
  78                 run.Raw('||'),
  79                 'true',
  80             ],
  81             wait=False,
  82         )
  83     )
  84     yield
  85
  86
  87 @contextlib.contextmanager
  88 def kubeadm_install(ctx, config):
  89     version = config.get('version', '1.21')
  90
  91     os_type = teuthology.get_distro(ctx)
  92     os_version = teuthology.get_distro_version(ctx)
  93
  94     try:
  95         if os_type in ['centos', 'rhel']:
  96             os = f"CentOS_{os_version.split('.')[0]}"
  97             log.info('Installing cri-o')
  98             run.wait(
  99                 ctx.cluster.run(
 100                     args=[
 101                         'sudo',
 102                         'curl', '-L', '-o',
 103                         '/etc/yum.repos.d/devel:kubic:libcontainers:stable.repo',
 104                         f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{os}/devel:kubic:libcontainers:stable.repo',
 105                         run.Raw('&&'),
 106                         'sudo',
 107                         'curl', '-L', '-o',
 108                         f'/etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:{version}.repo',
 109                         f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{version}/{os}/devel:kubic:libcontainers:stable:cri-o:{version}.repo',
 110                         run.Raw('&&'),
 111                         'sudo', 'dnf', 'install', '-y', 'cri-o',
 112                     ],
 113                     wait=False,
 114                 )
 115             )
 116
 117             log.info('Installing kube{adm,ctl,let}')
 118             repo = """[kubernetes]
 119 name=Kubernetes
 120 baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch
 121 enabled=1
 122 gpgcheck=1
 123 repo_gpgcheck=1
 124 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 125 """
 126             for remote in ctx.cluster.remotes.keys():
 127                 remote.write_file(
 128                     '/etc/yum.repos.d/kubernetes.repo',
 129                     repo,
 130                     sudo=True,
 131                 )
 132             run.wait(
 133                 ctx.cluster.run(
 134                     args=[
 135                         'sudo', 'dnf', 'install', '-y',
 136                         'kubelet', 'kubeadm', 'kubectl',
 137                         'iproute-tc', 'bridge-utils',
 138                     ],
 139                     wait=False,
 140                 )
 141             )
 142
 143             # fix cni config
 144             for remote in ctx.cluster.remotes.keys():
 145                 conf = """# from https://github.com/cri-o/cri-o/blob/master/tutorials/kubernetes.md#flannel-network
 146 {
 147     "name": "crio",
 148     "type": "flannel"
 149 }
 150 """
 151                 remote.write_file('/etc/cni/net.d/10-crio-flannel.conf', conf, sudo=True)
 152                 remote.run(args=[
 153                     'sudo', 'rm', '-f',
 154                     '/etc/cni/net.d/87-podman-bridge.conflist',
 155                     '/etc/cni/net.d/100-crio-bridge.conf',
 156                 ])
 157
 158             # start crio
 159             run.wait(
 160                 ctx.cluster.run(
 161                     args=[
 162                         'sudo', 'systemctl', 'daemon-reload',
 163                         run.Raw('&&'),
 164                         'sudo', 'systemctl', 'enable', 'crio', '--now',
 165                     ],
 166                     wait=False,
 167                 )
 168             )
 169
 170         elif os_type == 'ubuntu':
 171             os = f"xUbuntu_{os_version}"
 172             log.info('Installing kube{adm,ctl,let}')
 173             run.wait(
 174                 ctx.cluster.run(
 175                     args=[
 176                         'sudo', 'apt', 'update',
 177                         run.Raw('&&'),
 178                         'sudo', 'apt', 'install', '-y',
 179                         'apt-transport-https', 'ca-certificates', 'curl',
 180                         run.Raw('&&'),
 181                         'sudo', 'curl', '-fsSLo',
 182                         '/usr/share/keyrings/kubernetes-archive-keyring.gpg',
 183                         'https://packages.cloud.google.com/apt/doc/apt-key.gpg',
 184                         run.Raw('&&'),
 185                         'echo', 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main',
 186                         run.Raw('|'),
 187                         'sudo', 'tee', '/etc/apt/sources.list.d/kubernetes.list',
 188                         run.Raw('&&'),
 189                         'sudo', 'apt', 'update',
 190                         run.Raw('&&'),
 191                         'sudo', 'apt', 'install', '-y',
 192                         'kubelet', 'kubeadm', 'kubectl',
 193                         'bridge-utils',
 194                     ],
 195                     wait=False,
 196                 )
 197             )
 198
 199         else:
 200             raise RuntimeError(f'unsupported distro {os_type} for cri-o')
 201
 202         run.wait(
 203             ctx.cluster.run(
 204                 args=[
 205                     'sudo', 'systemctl', 'enable', '--now', 'kubelet',
 206                     run.Raw('&&'),
 207                     'sudo', 'kubeadm', 'config', 'images', 'pull',
 208                 ],
 209                 wait=False,
 210             )
 211         )
 212
 213         yield
 214
 215     finally:
 216         if config.get('uninstall', True):
 217             log.info('Uninstalling kube{adm,let,ctl}')
 218             if os_type in ['centos', 'rhel']:
 219                 run.wait(
 220                     ctx.cluster.run(
 221                         args=[
 222                             'sudo', 'rm', '-f',
 223                             '/etc/yum.repos.d/kubernetes.repo',
 224                             run.Raw('&&'),
 225                             'sudo', 'dnf', 'remove', '-y',
 226                             'kubeadm', 'kubelet', 'kubectl', 'cri-o',
 227                         ],
 228                         wait=False
 229                     )
 230                 )
 231             elif os_type == 'ubuntu' and False:
 232                 run.wait(
 233                     ctx.cluster.run(
 234                         args=[
 235                             'sudo', 'rm', '-f',
 236                             '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list',
 237                             f'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{version}.list',
 238                             '/etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg',
 239                             run.Raw('&&'),
 240                             'sudo', 'apt', 'remove', '-y',
 241                             'kkubeadm', 'kubelet', 'kubectl', 'cri-o', 'cri-o-runc',
 242                         ],
 243                         wait=False,
 244                     )
 245                 )
 246
 247
 248 @contextlib.contextmanager
 249 def kubeadm_init_join(ctx, config):
 250     cluster_name = config['cluster']
 251
 252     bootstrap_remote = None
 253     remotes = {}      # remote -> ip
 254     for remote, roles in ctx.cluster.remotes.items():
 255         for role in roles:
 256             if role.startswith('host.'):
 257                 if not bootstrap_remote:
 258                     bootstrap_remote = remote
 259                 if remote not in remotes:
 260                     remotes[remote] = remote.ssh.get_transport().getpeername()[0]
 261     if not bootstrap_remote:
 262         raise RuntimeError('must define at least one host.something role')
 263     ctx.kubeadm[cluster_name].bootstrap_remote = bootstrap_remote
 264     ctx.kubeadm[cluster_name].remotes = remotes
 265     ctx.kubeadm[cluster_name].token = 'abcdef.' + ''.join([
 266         random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(16)
 267     ])
 268     log.info(f'Token: {ctx.kubeadm[cluster_name].token}')
 269     log.info(f'Remotes: {ctx.kubeadm[cluster_name].remotes}')
 270
 271     try:
 272         # init
 273         cmd = [
 274             'sudo', 'kubeadm', 'init',
 275             '--node-name', ctx.kubeadm[cluster_name].bootstrap_remote.shortname,
 276             '--token', ctx.kubeadm[cluster_name].token,
 277             '--pod-network-cidr', str(ctx.kubeadm[cluster_name].pod_subnet),
 278         ]
 279         bootstrap_remote.run(args=cmd)
 280
 281         # join additional nodes
 282         joins = []
 283         for remote, ip in ctx.kubeadm[cluster_name].remotes.items():
 284             if remote == bootstrap_remote:
 285                 continue
 286             cmd = [
 287                 'sudo', 'kubeadm', 'join',
 288                 ctx.kubeadm[cluster_name].remotes[ctx.kubeadm[cluster_name].bootstrap_remote] + ':6443',
 289                 '--node-name', remote.shortname,
 290                 '--token', ctx.kubeadm[cluster_name].token,
 291                 '--discovery-token-unsafe-skip-ca-verification',
 292             ]
 293             joins.append(remote.run(args=cmd, wait=False))
 294         run.wait(joins)
 295         yield
 296
 297     except Exception as e:
 298         log.exception(e)
 299         raise
 300
 301     finally:
 302         log.info('Cleaning up node')
 303         run.wait(
 304             ctx.cluster.run(
 305                 args=['sudo', 'kubeadm', 'reset', 'cleanup-node', '-f'],
 306                 wait=False,
 307             )
 308         )
 309
 310
 311 @contextlib.contextmanager
 312 def kubectl_config(ctx, config):
 313     cluster_name = config['cluster']
 314     bootstrap_remote = ctx.kubeadm[cluster_name].bootstrap_remote
 315
 316     ctx.kubeadm[cluster_name].admin_conf = \
 317         bootstrap_remote.read_file('/etc/kubernetes/admin.conf', sudo=True)
 318
 319     log.info('Setting up kubectl')
 320     try:
 321         ctx.cluster.run(args=[
 322             'mkdir', '-p', '.kube',
 323             run.Raw('&&'),
 324             'sudo', 'mkdir', '-p', '/root/.kube',
 325         ])
 326         for remote in ctx.kubeadm[cluster_name].remotes.keys():
 327             remote.write_file('.kube/config', ctx.kubeadm[cluster_name].admin_conf)
 328             remote.sudo_write_file('/root/.kube/config',
 329                                    ctx.kubeadm[cluster_name].admin_conf)
 330         yield
 331
 332     except Exception as e:
 333         log.exception(e)
 334         raise
 335
 336     finally:
 337         log.info('Deconfiguring kubectl')
 338         ctx.cluster.run(args=[
 339             'rm', '-rf', '.kube',
 340             run.Raw('&&'),
 341             'sudo', 'rm', '-rf', '/root/.kube',
 342         ])
 343
 344
 345 def map_vnet(mip):
 346     for mapping in teuth_config.get('vnet', []):
 347         mnet = ipaddress.ip_network(mapping['machine_subnet'])
 348         vnet = ipaddress.ip_network(mapping['virtual_subnet'])
 349         if vnet.prefixlen >= mnet.prefixlen:
 350             log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix")
 351             return None
 352         if mip in mnet:
 353             pos = list(mnet.hosts()).index(mip)
 354             log.info(f"{mip} is in {mnet} at pos {pos}")
 355             sub = list(vnet.subnets(32 - mnet.prefixlen))[pos]
 356             return sub
 357     return None
 358
 359
 360 @contextlib.contextmanager
 361 def allocate_pod_subnet(ctx, config):
 362     """
 363     Allocate a private subnet that will not collide with other test machines/clusters
 364     """
 365     cluster_name = config['cluster']
 366     assert cluster_name == 'kubeadm', 'multiple subnets not yet implemented'
 367
 368     log.info('Identifying pod subnet')
 369     remote = list(ctx.cluster.remotes.keys())[0]
 370     ip = remote.ssh.get_transport().getpeername()[0]
 371     mip = ipaddress.ip_address(ip)
 372     vnet = map_vnet(mip)
 373     assert vnet
 374     log.info(f'Pod subnet: {vnet}')
 375     ctx.kubeadm[cluster_name].pod_subnet = vnet
 376     yield
 377
 378
 379 @contextlib.contextmanager
 380 def pod_network(ctx, config):
 381     cluster_name = config['cluster']
 382     pnet = config.get('pod_network', 'calico')
 383     if pnet == 'flannel':
 384         r = ctx.kubeadm[cluster_name].bootstrap_remote.run(
 385             args=[
 386                 'curl',
 387                 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml',
 388             ],
 389             stdout=BytesIO(),
 390         )
 391         assert r.exitstatus == 0
 392         flannel = list(yaml.load_all(r.stdout.getvalue(), Loader=yaml.FullLoader))
 393         for o in flannel:
 394             if o.get('data', {}).get('net-conf.json'):
 395                 log.info(f'Updating {o}')
 396                 o['data']['net-conf.json'] = o['data']['net-conf.json'].replace(
 397                     '10.244.0.0/16',
 398                     str(ctx.kubeadm[cluster_name].pod_subnet)
 399                 )
 400                 log.info(f'Now {o}')
 401         flannel_yaml = yaml.dump_all(flannel)
 402         log.debug(f'Flannel:\n{flannel_yaml}')
 403         _kubectl(ctx, config, ['apply', '-f', '-'], stdin=flannel_yaml)
 404
 405     elif pnet == 'calico':
 406         _kubectl(ctx, config, [
 407             'create', '-f',
 408             'https://docs.projectcalico.org/manifests/tigera-operator.yaml'
 409         ])
 410         cr = {
 411             'apiVersion': 'operator.tigera.io/v1',
 412             'kind': 'Installation',
 413             'metadata': {'name': 'default'},
 414             'spec': {
 415                 'calicoNetwork': {
 416                     'ipPools': [
 417                         {
 418                             'blockSize': 26,
 419                             'cidr': str(ctx.kubeadm[cluster_name].pod_subnet),
 420                             'encapsulation': 'IPIPCrossSubnet',
 421                             'natOutgoing': 'Enabled',
 422                             'nodeSelector': 'all()',
 423                         }
 424                     ]
 425                 }
 426             }
 427         }
 428         _kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump(cr))
 429
 430     else:
 431         raise RuntimeError(f'unrecognized pod_network {pnet}')
 432
 433     try:
 434         yield
 435
 436     finally:
 437         if pnet == 'flannel':
 438             _kubectl(ctx, config, [
 439                 'delete', '-f',
 440                 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml',
 441             ])
 442
 443         elif pnet == 'calico':
 444             _kubectl(ctx, config, ['delete', 'installation', 'default'])
 445             _kubectl(ctx, config, [
 446                 'delete', '-f',
 447                 'https://docs.projectcalico.org/manifests/tigera-operator.yaml'
 448             ])
 449
 450
 451 @contextlib.contextmanager
 452 def setup_pvs(ctx, config):
 453     """
 454     Create PVs for all scratch LVs and set up a trivial provisioner
 455     """
 456     log.info('Scanning for scratch devices')
 457     crs = []
 458     for remote in ctx.cluster.remotes.keys():
 459         ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines()
 460         log.info(f'Scratch devices on {remote.shortname}: {ls}')
 461         for dev in ls:
 462             devname = dev.split('/')[-1].replace("_", "-")
 463             crs.append({
 464                 'apiVersion': 'v1',
 465                 'kind': 'PersistentVolume',
 466                 'metadata': {'name': f'{remote.shortname}-{devname}'},
 467                 'spec': {
 468                     'volumeMode': 'Block',
 469                     'accessModes': ['ReadWriteOnce'],
 470                     'capacity': {'storage': '100Gi'},  # doesn't matter?
 471                     'persistentVolumeReclaimPolicy': 'Retain',
 472                     'storageClassName': 'scratch',
 473                     'local': {'path': dev},
 474                     'nodeAffinity': {
 475                         'required': {
 476                             'nodeSelectorTerms': [
 477                                 {
 478                                     'matchExpressions': [
 479                                         {
 480                                             'key': 'kubernetes.io/hostname',
 481                                             'operator': 'In',
 482                                             'values': [remote.shortname]
 483                                         }
 484                                     ]
 485                                 }
 486                             ]
 487                         }
 488                     }
 489                 }
 490             })
 491             # overwriting first few MB is enough to make k8s happy
 492             remote.run(args=[
 493                 'sudo', 'dd', 'if=/dev/zero', f'of={dev}', 'bs=1M', 'count=10'
 494             ])
 495     crs.append({
 496         'kind': 'StorageClass',
 497         'apiVersion': 'storage.k8s.io/v1',
 498         'metadata': {'name': 'scratch'},
 499         'provisioner': 'kubernetes.io/no-provisioner',
 500         'volumeBindingMode': 'WaitForFirstConsumer',
 501     })
 502     y = yaml.dump_all(crs)
 503     log.info('Creating PVs + StorageClass')
 504     log.debug(y)
 505     _kubectl(ctx, config, ['create', '-f', '-'], stdin=y)
 506
 507     yield
 508
 509
 510 @contextlib.contextmanager
 511 def final(ctx, config):
 512     cluster_name = config['cluster']
 513
 514     # remove master node taint
 515     _kubectl(ctx, config, [
 516         'taint', 'node',
 517         ctx.kubeadm[cluster_name].bootstrap_remote.shortname,
 518         'node-role.kubernetes.io/master-',
 519         run.Raw('||'),
 520         'true',
 521     ])
 522
 523     yield
 524
 525
 526 @contextlib.contextmanager
 527 def task(ctx, config):
 528     if not config:
 529         config = {}
 530     assert isinstance(config, dict), \
 531         "task only supports a dictionary for configuration"
 532
 533     log.info('Kubeadm start')
 534
 535     overrides = ctx.config.get('overrides', {})
 536     teuthology.deep_merge(config, overrides.get('kubeadm', {}))
 537     log.info('Config: ' + str(config))
 538
 539     # set up cluster context
 540     if not hasattr(ctx, 'kubeadm'):
 541         ctx.kubeadm = {}
 542     if 'cluster' not in config:
 543         config['cluster'] = 'kubeadm'
 544     cluster_name = config['cluster']
 545     if cluster_name not in ctx.kubeadm:
 546         ctx.kubeadm[cluster_name] = argparse.Namespace()
 547
 548     with contextutil.nested(
 549             lambda: preflight(ctx, config),
 550             lambda: allocate_pod_subnet(ctx, config),
 551             lambda: kubeadm_install(ctx, config),
 552             lambda: kubeadm_init_join(ctx, config),
 553             lambda: kubectl_config(ctx, config),
 554             lambda: pod_network(ctx, config),
 555             lambda: setup_pvs(ctx, config),
 556             lambda: final(ctx, config),
 557     ):
 558         try:
 559             log.info('Kubeadm complete, yielding')
 560             yield
 561
 562         finally:
 563             log.info('Tearing down kubeadm')