]>
Commit | Line | Data |
---|---|---|
b3b6e05e TL |
1 | """ |
2 | Kubernetes cluster task, deployed via kubeadm | |
3 | """ | |
4 | import argparse | |
5 | import contextlib | |
6 | import ipaddress | |
20effc67 | 7 | import json |
b3b6e05e TL |
8 | import logging |
9 | import random | |
10 | import yaml | |
11 | from io import BytesIO | |
12 | ||
13 | from teuthology import misc as teuthology | |
14 | from teuthology import contextutil | |
15 | from teuthology.config import config as teuth_config | |
16 | from teuthology.orchestra import run | |
17 | ||
18 | log = logging.getLogger(__name__) | |
19 | ||
20 | ||
21 | def _kubectl(ctx, config, args, **kwargs): | |
22 | cluster_name = config['cluster'] | |
23 | ctx.kubeadm[cluster_name].bootstrap_remote.run( | |
24 | args=['kubectl'] + args, | |
25 | **kwargs, | |
26 | ) | |
27 | ||
28 | ||
29 | def kubectl(ctx, config): | |
30 | if isinstance(config, str): | |
31 | config = [config] | |
32 | assert isinstance(config, list) | |
33 | for c in config: | |
34 | if isinstance(c, str): | |
35 | _kubectl(ctx, config, c.split(' ')) | |
36 | else: | |
37 | _kubectl(ctx, config, c) | |
38 | ||
39 | ||
40 | @contextlib.contextmanager | |
41 | def preflight(ctx, config): | |
42 | run.wait( | |
43 | ctx.cluster.run( | |
44 | args=[ | |
45 | 'sudo', 'modprobe', 'br_netfilter', | |
46 | run.Raw('&&'), | |
47 | 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-ip6tables=1', | |
48 | run.Raw('&&'), | |
49 | 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-iptables=1', | |
50 | run.Raw('&&'), | |
51 | 'sudo', 'sysctl', 'net.ipv4.ip_forward=1', | |
52 | run.Raw('&&'), | |
53 | 'sudo', 'swapoff', '-a', | |
54 | ], | |
55 | wait=False, | |
56 | ) | |
57 | ) | |
a4b75251 TL |
58 | |
59 | # set docker cgroup driver = systemd | |
60 | # see https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker | |
61 | # see https://github.com/kubernetes/kubeadm/issues/2066 | |
a4b75251 | 62 | for remote in ctx.cluster.remotes.keys(): |
20effc67 TL |
63 | try: |
64 | orig = remote.read_file('/etc/docker/daemon.json', sudo=True) | |
65 | j = json.loads(orig) | |
66 | except Exception as e: | |
67 | log.info(f'Failed to pull old daemon.json: {e}') | |
68 | j = {} | |
69 | j["exec-opts"] = ["native.cgroupdriver=systemd"] | |
70 | j["log-driver"] = "json-file" | |
71 | j["log-opts"] = {"max-size": "100m"} | |
72 | j["storage-driver"] = "overlay2" | |
73 | remote.write_file('/etc/docker/daemon.json', json.dumps(j), sudo=True) | |
a4b75251 TL |
74 | run.wait( |
75 | ctx.cluster.run( | |
76 | args=[ | |
77 | 'sudo', 'systemctl', 'restart', 'docker', | |
78 | run.Raw('||'), | |
79 | 'true', | |
80 | ], | |
81 | wait=False, | |
82 | ) | |
83 | ) | |
b3b6e05e TL |
84 | yield |
85 | ||
86 | ||
87 | @contextlib.contextmanager | |
88 | def kubeadm_install(ctx, config): | |
89 | version = config.get('version', '1.21') | |
90 | ||
91 | os_type = teuthology.get_distro(ctx) | |
92 | os_version = teuthology.get_distro_version(ctx) | |
93 | ||
94 | try: | |
95 | if os_type in ['centos', 'rhel']: | |
96 | os = f"CentOS_{os_version.split('.')[0]}" | |
97 | log.info('Installing cri-o') | |
98 | run.wait( | |
99 | ctx.cluster.run( | |
100 | args=[ | |
101 | 'sudo', | |
102 | 'curl', '-L', '-o', | |
103 | '/etc/yum.repos.d/devel:kubic:libcontainers:stable.repo', | |
104 | f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{os}/devel:kubic:libcontainers:stable.repo', | |
105 | run.Raw('&&'), | |
106 | 'sudo', | |
107 | 'curl', '-L', '-o', | |
108 | f'/etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:{version}.repo', | |
109 | f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{version}/{os}/devel:kubic:libcontainers:stable:cri-o:{version}.repo', | |
110 | run.Raw('&&'), | |
111 | 'sudo', 'dnf', 'install', '-y', 'cri-o', | |
112 | ], | |
113 | wait=False, | |
114 | ) | |
115 | ) | |
116 | ||
117 | log.info('Installing kube{adm,ctl,let}') | |
118 | repo = """[kubernetes] | |
119 | name=Kubernetes | |
120 | baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch | |
121 | enabled=1 | |
122 | gpgcheck=1 | |
123 | repo_gpgcheck=1 | |
124 | gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg | |
125 | """ | |
126 | for remote in ctx.cluster.remotes.keys(): | |
127 | remote.write_file( | |
128 | '/etc/yum.repos.d/kubernetes.repo', | |
129 | repo, | |
130 | sudo=True, | |
131 | ) | |
132 | run.wait( | |
133 | ctx.cluster.run( | |
134 | args=[ | |
135 | 'sudo', 'dnf', 'install', '-y', | |
136 | 'kubelet', 'kubeadm', 'kubectl', | |
137 | 'iproute-tc', 'bridge-utils', | |
138 | ], | |
139 | wait=False, | |
140 | ) | |
141 | ) | |
142 | ||
143 | # fix cni config | |
144 | for remote in ctx.cluster.remotes.keys(): | |
145 | conf = """# from https://github.com/cri-o/cri-o/blob/master/tutorials/kubernetes.md#flannel-network | |
146 | { | |
147 | "name": "crio", | |
148 | "type": "flannel" | |
149 | } | |
150 | """ | |
151 | remote.write_file('/etc/cni/net.d/10-crio-flannel.conf', conf, sudo=True) | |
152 | remote.run(args=[ | |
153 | 'sudo', 'rm', '-f', | |
154 | '/etc/cni/net.d/87-podman-bridge.conflist', | |
155 | '/etc/cni/net.d/100-crio-bridge.conf', | |
156 | ]) | |
157 | ||
158 | # start crio | |
159 | run.wait( | |
160 | ctx.cluster.run( | |
161 | args=[ | |
162 | 'sudo', 'systemctl', 'daemon-reload', | |
163 | run.Raw('&&'), | |
164 | 'sudo', 'systemctl', 'enable', 'crio', '--now', | |
165 | ], | |
166 | wait=False, | |
167 | ) | |
168 | ) | |
169 | ||
170 | elif os_type == 'ubuntu': | |
171 | os = f"xUbuntu_{os_version}" | |
172 | log.info('Installing kube{adm,ctl,let}') | |
173 | run.wait( | |
174 | ctx.cluster.run( | |
175 | args=[ | |
176 | 'sudo', 'apt', 'update', | |
177 | run.Raw('&&'), | |
178 | 'sudo', 'apt', 'install', '-y', | |
179 | 'apt-transport-https', 'ca-certificates', 'curl', | |
180 | run.Raw('&&'), | |
181 | 'sudo', 'curl', '-fsSLo', | |
182 | '/usr/share/keyrings/kubernetes-archive-keyring.gpg', | |
183 | 'https://packages.cloud.google.com/apt/doc/apt-key.gpg', | |
184 | run.Raw('&&'), | |
185 | 'echo', 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main', | |
186 | run.Raw('|'), | |
187 | 'sudo', 'tee', '/etc/apt/sources.list.d/kubernetes.list', | |
188 | run.Raw('&&'), | |
189 | 'sudo', 'apt', 'update', | |
190 | run.Raw('&&'), | |
191 | 'sudo', 'apt', 'install', '-y', | |
192 | 'kubelet', 'kubeadm', 'kubectl', | |
193 | 'bridge-utils', | |
194 | ], | |
195 | wait=False, | |
196 | ) | |
197 | ) | |
198 | ||
199 | else: | |
200 | raise RuntimeError(f'unsupported distro {os_type} for cri-o') | |
201 | ||
202 | run.wait( | |
203 | ctx.cluster.run( | |
204 | args=[ | |
205 | 'sudo', 'systemctl', 'enable', '--now', 'kubelet', | |
206 | run.Raw('&&'), | |
207 | 'sudo', 'kubeadm', 'config', 'images', 'pull', | |
208 | ], | |
209 | wait=False, | |
210 | ) | |
211 | ) | |
212 | ||
213 | yield | |
214 | ||
215 | finally: | |
216 | if config.get('uninstall', True): | |
217 | log.info('Uninstalling kube{adm,let,ctl}') | |
218 | if os_type in ['centos', 'rhel']: | |
219 | run.wait( | |
220 | ctx.cluster.run( | |
221 | args=[ | |
222 | 'sudo', 'rm', '-f', | |
223 | '/etc/yum.repos.d/kubernetes.repo', | |
224 | run.Raw('&&'), | |
225 | 'sudo', 'dnf', 'remove', '-y', | |
226 | 'kubeadm', 'kubelet', 'kubectl', 'cri-o', | |
227 | ], | |
228 | wait=False | |
229 | ) | |
230 | ) | |
231 | elif os_type == 'ubuntu' and False: | |
232 | run.wait( | |
233 | ctx.cluster.run( | |
234 | args=[ | |
235 | 'sudo', 'rm', '-f', | |
236 | '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list', | |
237 | f'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{version}.list', | |
238 | '/etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg', | |
239 | run.Raw('&&'), | |
240 | 'sudo', 'apt', 'remove', '-y', | |
241 | 'kkubeadm', 'kubelet', 'kubectl', 'cri-o', 'cri-o-runc', | |
242 | ], | |
243 | wait=False, | |
244 | ) | |
245 | ) | |
246 | ||
247 | ||
248 | @contextlib.contextmanager | |
249 | def kubeadm_init_join(ctx, config): | |
250 | cluster_name = config['cluster'] | |
251 | ||
252 | bootstrap_remote = None | |
253 | remotes = {} # remote -> ip | |
254 | for remote, roles in ctx.cluster.remotes.items(): | |
255 | for role in roles: | |
256 | if role.startswith('host.'): | |
257 | if not bootstrap_remote: | |
258 | bootstrap_remote = remote | |
259 | if remote not in remotes: | |
260 | remotes[remote] = remote.ssh.get_transport().getpeername()[0] | |
261 | if not bootstrap_remote: | |
262 | raise RuntimeError('must define at least one host.something role') | |
263 | ctx.kubeadm[cluster_name].bootstrap_remote = bootstrap_remote | |
264 | ctx.kubeadm[cluster_name].remotes = remotes | |
265 | ctx.kubeadm[cluster_name].token = 'abcdef.' + ''.join([ | |
266 | random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(16) | |
267 | ]) | |
268 | log.info(f'Token: {ctx.kubeadm[cluster_name].token}') | |
269 | log.info(f'Remotes: {ctx.kubeadm[cluster_name].remotes}') | |
270 | ||
271 | try: | |
272 | # init | |
273 | cmd = [ | |
274 | 'sudo', 'kubeadm', 'init', | |
275 | '--node-name', ctx.kubeadm[cluster_name].bootstrap_remote.shortname, | |
276 | '--token', ctx.kubeadm[cluster_name].token, | |
277 | '--pod-network-cidr', str(ctx.kubeadm[cluster_name].pod_subnet), | |
278 | ] | |
279 | bootstrap_remote.run(args=cmd) | |
280 | ||
281 | # join additional nodes | |
282 | joins = [] | |
283 | for remote, ip in ctx.kubeadm[cluster_name].remotes.items(): | |
284 | if remote == bootstrap_remote: | |
285 | continue | |
286 | cmd = [ | |
287 | 'sudo', 'kubeadm', 'join', | |
288 | ctx.kubeadm[cluster_name].remotes[ctx.kubeadm[cluster_name].bootstrap_remote] + ':6443', | |
289 | '--node-name', remote.shortname, | |
290 | '--token', ctx.kubeadm[cluster_name].token, | |
291 | '--discovery-token-unsafe-skip-ca-verification', | |
292 | ] | |
293 | joins.append(remote.run(args=cmd, wait=False)) | |
294 | run.wait(joins) | |
295 | yield | |
296 | ||
297 | except Exception as e: | |
298 | log.exception(e) | |
299 | raise | |
300 | ||
301 | finally: | |
302 | log.info('Cleaning up node') | |
303 | run.wait( | |
304 | ctx.cluster.run( | |
305 | args=['sudo', 'kubeadm', 'reset', 'cleanup-node', '-f'], | |
306 | wait=False, | |
307 | ) | |
308 | ) | |
309 | ||
310 | ||
311 | @contextlib.contextmanager | |
312 | def kubectl_config(ctx, config): | |
313 | cluster_name = config['cluster'] | |
314 | bootstrap_remote = ctx.kubeadm[cluster_name].bootstrap_remote | |
315 | ||
316 | ctx.kubeadm[cluster_name].admin_conf = \ | |
317 | bootstrap_remote.read_file('/etc/kubernetes/admin.conf', sudo=True) | |
318 | ||
319 | log.info('Setting up kubectl') | |
320 | try: | |
321 | ctx.cluster.run(args=[ | |
322 | 'mkdir', '-p', '.kube', | |
323 | run.Raw('&&'), | |
324 | 'sudo', 'mkdir', '-p', '/root/.kube', | |
325 | ]) | |
326 | for remote in ctx.kubeadm[cluster_name].remotes.keys(): | |
327 | remote.write_file('.kube/config', ctx.kubeadm[cluster_name].admin_conf) | |
328 | remote.sudo_write_file('/root/.kube/config', | |
329 | ctx.kubeadm[cluster_name].admin_conf) | |
330 | yield | |
331 | ||
332 | except Exception as e: | |
333 | log.exception(e) | |
334 | raise | |
335 | ||
336 | finally: | |
337 | log.info('Deconfiguring kubectl') | |
338 | ctx.cluster.run(args=[ | |
339 | 'rm', '-rf', '.kube', | |
340 | run.Raw('&&'), | |
341 | 'sudo', 'rm', '-rf', '/root/.kube', | |
342 | ]) | |
343 | ||
344 | ||
345 | def map_vnet(mip): | |
346 | for mapping in teuth_config.get('vnet', []): | |
347 | mnet = ipaddress.ip_network(mapping['machine_subnet']) | |
348 | vnet = ipaddress.ip_network(mapping['virtual_subnet']) | |
349 | if vnet.prefixlen >= mnet.prefixlen: | |
350 | log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix") | |
351 | return None | |
352 | if mip in mnet: | |
353 | pos = list(mnet.hosts()).index(mip) | |
354 | log.info(f"{mip} is in {mnet} at pos {pos}") | |
355 | sub = list(vnet.subnets(32 - mnet.prefixlen))[pos] | |
356 | return sub | |
357 | return None | |
358 | ||
359 | ||
360 | @contextlib.contextmanager | |
361 | def allocate_pod_subnet(ctx, config): | |
362 | """ | |
363 | Allocate a private subnet that will not collide with other test machines/clusters | |
364 | """ | |
365 | cluster_name = config['cluster'] | |
366 | assert cluster_name == 'kubeadm', 'multiple subnets not yet implemented' | |
367 | ||
368 | log.info('Identifying pod subnet') | |
369 | remote = list(ctx.cluster.remotes.keys())[0] | |
370 | ip = remote.ssh.get_transport().getpeername()[0] | |
371 | mip = ipaddress.ip_address(ip) | |
372 | vnet = map_vnet(mip) | |
373 | assert vnet | |
374 | log.info(f'Pod subnet: {vnet}') | |
375 | ctx.kubeadm[cluster_name].pod_subnet = vnet | |
376 | yield | |
377 | ||
378 | ||
379 | @contextlib.contextmanager | |
380 | def pod_network(ctx, config): | |
381 | cluster_name = config['cluster'] | |
382 | pnet = config.get('pod_network', 'calico') | |
383 | if pnet == 'flannel': | |
384 | r = ctx.kubeadm[cluster_name].bootstrap_remote.run( | |
385 | args=[ | |
386 | 'curl', | |
387 | 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml', | |
388 | ], | |
389 | stdout=BytesIO(), | |
390 | ) | |
391 | assert r.exitstatus == 0 | |
392 | flannel = list(yaml.load_all(r.stdout.getvalue(), Loader=yaml.FullLoader)) | |
393 | for o in flannel: | |
394 | if o.get('data', {}).get('net-conf.json'): | |
395 | log.info(f'Updating {o}') | |
396 | o['data']['net-conf.json'] = o['data']['net-conf.json'].replace( | |
397 | '10.244.0.0/16', | |
398 | str(ctx.kubeadm[cluster_name].pod_subnet) | |
399 | ) | |
400 | log.info(f'Now {o}') | |
401 | flannel_yaml = yaml.dump_all(flannel) | |
402 | log.debug(f'Flannel:\n{flannel_yaml}') | |
403 | _kubectl(ctx, config, ['apply', '-f', '-'], stdin=flannel_yaml) | |
404 | ||
405 | elif pnet == 'calico': | |
406 | _kubectl(ctx, config, [ | |
407 | 'apply', '-f', | |
408 | 'https://docs.projectcalico.org/manifests/tigera-operator.yaml' | |
409 | ]) | |
410 | cr = { | |
411 | 'apiVersion': 'operator.tigera.io/v1', | |
412 | 'kind': 'Installation', | |
413 | 'metadata': {'name': 'default'}, | |
414 | 'spec': { | |
415 | 'calicoNetwork': { | |
416 | 'ipPools': [ | |
417 | { | |
418 | 'blockSize': 26, | |
419 | 'cidr': str(ctx.kubeadm[cluster_name].pod_subnet), | |
20effc67 | 420 | 'encapsulation': 'IPIPCrossSubnet', |
b3b6e05e TL |
421 | 'natOutgoing': 'Enabled', |
422 | 'nodeSelector': 'all()', | |
423 | } | |
424 | ] | |
425 | } | |
426 | } | |
427 | } | |
428 | _kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump(cr)) | |
429 | ||
430 | else: | |
431 | raise RuntimeError(f'unrecognized pod_network {pnet}') | |
432 | ||
433 | try: | |
434 | yield | |
435 | ||
436 | finally: | |
437 | if pnet == 'flannel': | |
438 | _kubectl(ctx, config, [ | |
439 | 'delete', '-f', | |
440 | 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml', | |
441 | ]) | |
442 | ||
443 | elif pnet == 'calico': | |
444 | _kubectl(ctx, config, ['delete', 'installation', 'default']) | |
445 | _kubectl(ctx, config, [ | |
446 | 'delete', '-f', | |
447 | 'https://docs.projectcalico.org/manifests/tigera-operator.yaml' | |
448 | ]) | |
449 | ||
450 | ||
451 | @contextlib.contextmanager | |
452 | def setup_pvs(ctx, config): | |
453 | """ | |
454 | Create PVs for all scratch LVs and set up a trivial provisioner | |
455 | """ | |
456 | log.info('Scanning for scratch devices') | |
457 | crs = [] | |
458 | for remote in ctx.cluster.remotes.keys(): | |
459 | ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines() | |
460 | log.info(f'Scratch devices on {remote.shortname}: {ls}') | |
461 | for dev in ls: | |
462 | devname = dev.split('/')[-1].replace("_", "-") | |
463 | crs.append({ | |
464 | 'apiVersion': 'v1', | |
465 | 'kind': 'PersistentVolume', | |
466 | 'metadata': {'name': f'{remote.shortname}-{devname}'}, | |
467 | 'spec': { | |
468 | 'volumeMode': 'Block', | |
469 | 'accessModes': ['ReadWriteOnce'], | |
470 | 'capacity': {'storage': '100Gi'}, # doesn't matter? | |
20effc67 | 471 | 'persistentVolumeReclaimPolicy': 'Retain', |
b3b6e05e TL |
472 | 'storageClassName': 'scratch', |
473 | 'local': {'path': dev}, | |
474 | 'nodeAffinity': { | |
475 | 'required': { | |
476 | 'nodeSelectorTerms': [ | |
477 | { | |
478 | 'matchExpressions': [ | |
479 | { | |
480 | 'key': 'kubernetes.io/hostname', | |
481 | 'operator': 'In', | |
482 | 'values': [remote.shortname] | |
483 | } | |
484 | ] | |
485 | } | |
486 | ] | |
487 | } | |
488 | } | |
489 | } | |
490 | }) | |
491 | # overwriting first few MB is enough to make k8s happy | |
492 | remote.run(args=[ | |
493 | 'sudo', 'dd', 'if=/dev/zero', f'of={dev}', 'bs=1M', 'count=10' | |
494 | ]) | |
495 | crs.append({ | |
496 | 'kind': 'StorageClass', | |
497 | 'apiVersion': 'storage.k8s.io/v1', | |
498 | 'metadata': {'name': 'scratch'}, | |
499 | 'provisioner': 'kubernetes.io/no-provisioner', | |
500 | 'volumeBindingMode': 'WaitForFirstConsumer', | |
501 | }) | |
502 | y = yaml.dump_all(crs) | |
503 | log.info('Creating PVs + StorageClass') | |
504 | log.debug(y) | |
505 | _kubectl(ctx, config, ['create', '-f', '-'], stdin=y) | |
506 | ||
507 | yield | |
508 | ||
509 | ||
510 | @contextlib.contextmanager | |
511 | def final(ctx, config): | |
512 | cluster_name = config['cluster'] | |
513 | ||
514 | # remove master node taint | |
515 | _kubectl(ctx, config, [ | |
516 | 'taint', 'node', | |
517 | ctx.kubeadm[cluster_name].bootstrap_remote.shortname, | |
518 | 'node-role.kubernetes.io/master-', | |
519 | run.Raw('||'), | |
520 | 'true', | |
521 | ]) | |
522 | ||
523 | yield | |
524 | ||
525 | ||
526 | @contextlib.contextmanager | |
527 | def task(ctx, config): | |
528 | if not config: | |
529 | config = {} | |
530 | assert isinstance(config, dict), \ | |
531 | "task only supports a dictionary for configuration" | |
532 | ||
533 | log.info('Kubeadm start') | |
534 | ||
535 | overrides = ctx.config.get('overrides', {}) | |
536 | teuthology.deep_merge(config, overrides.get('kubeadm', {})) | |
537 | log.info('Config: ' + str(config)) | |
538 | ||
539 | # set up cluster context | |
540 | if not hasattr(ctx, 'kubeadm'): | |
541 | ctx.kubeadm = {} | |
542 | if 'cluster' not in config: | |
543 | config['cluster'] = 'kubeadm' | |
544 | cluster_name = config['cluster'] | |
545 | if cluster_name not in ctx.kubeadm: | |
546 | ctx.kubeadm[cluster_name] = argparse.Namespace() | |
547 | ||
548 | with contextutil.nested( | |
549 | lambda: preflight(ctx, config), | |
550 | lambda: allocate_pod_subnet(ctx, config), | |
551 | lambda: kubeadm_install(ctx, config), | |
552 | lambda: kubeadm_init_join(ctx, config), | |
553 | lambda: kubectl_config(ctx, config), | |
554 | lambda: pod_network(ctx, config), | |
555 | lambda: setup_pvs(ctx, config), | |
556 | lambda: final(ctx, config), | |
557 | ): | |
558 | try: | |
559 | log.info('Kubeadm complete, yielding') | |
560 | yield | |
561 | ||
562 | finally: | |
563 | log.info('Tearing down kubeadm') |