]>
Commit | Line | Data |
---|---|---|
f67539c2 TL |
1 | """ |
2 | Deploy and configure PyKMIP for Teuthology | |
3 | """ | |
4 | import argparse | |
5 | import contextlib | |
6 | import logging | |
7 | import time | |
8 | import tempfile | |
9 | import json | |
10 | import os | |
11 | from io import BytesIO | |
12 | from teuthology.orchestra.daemon import DaemonGroup | |
13 | from teuthology.orchestra.remote import Remote | |
14 | ||
15 | import pprint | |
16 | ||
17 | from teuthology import misc as teuthology | |
18 | from teuthology import contextutil | |
19 | from teuthology.orchestra import run | |
20 | from teuthology.packaging import install_package | |
21 | from teuthology.packaging import remove_package | |
22 | from teuthology.exceptions import ConfigError | |
23 | from tasks.util import get_remote_for_role | |
24 | ||
25 | log = logging.getLogger(__name__) | |
26 | ||
27 | ||
28 | def get_pykmip_dir(ctx): | |
29 | return '{tdir}/pykmip'.format(tdir=teuthology.get_testdir(ctx)) | |
30 | ||
31 | def run_in_pykmip_dir(ctx, client, args, **kwargs): | |
32 | (remote,) = [client] if isinstance(client,Remote) else ctx.cluster.only(client).remotes.keys() | |
33 | return remote.run( | |
34 | args=['cd', get_pykmip_dir(ctx), run.Raw('&&'), ] + args, | |
35 | **kwargs | |
36 | ) | |
37 | ||
38 | def run_in_pykmip_venv(ctx, client, args, **kwargs): | |
39 | return run_in_pykmip_dir(ctx, client, | |
40 | args = ['.', '.pykmipenv/bin/activate', | |
41 | run.Raw('&&') | |
42 | ] + args, **kwargs) | |
43 | ||
44 | @contextlib.contextmanager | |
45 | def download(ctx, config): | |
46 | """ | |
47 | Download PyKMIP from github. | |
48 | Remove downloaded file upon exit. | |
49 | ||
50 | The context passed in should be identical to the context | |
51 | passed in to the main task. | |
52 | """ | |
53 | assert isinstance(config, dict) | |
54 | log.info('Downloading pykmip...') | |
55 | pykmipdir = get_pykmip_dir(ctx) | |
56 | ||
57 | for (client, cconf) in config.items(): | |
58 | branch = cconf.get('force-branch', 'master') | |
59 | repo = cconf.get('force-repo', 'https://github.com/OpenKMIP/PyKMIP') | |
60 | sha1 = cconf.get('sha1') | |
61 | log.info("Using branch '%s' for pykmip", branch) | |
62 | log.info('sha1=%s', sha1) | |
63 | ||
64 | ctx.cluster.only(client).run( | |
65 | args=[ | |
66 | 'git', 'clone', '-b', branch, repo, | |
67 | pykmipdir, | |
68 | ], | |
69 | ) | |
70 | if sha1 is not None: | |
71 | run_in_pykmip_dir(ctx, client, [ | |
72 | 'git', 'reset', '--hard', sha1, | |
73 | ], | |
74 | ) | |
75 | try: | |
76 | yield | |
77 | finally: | |
78 | log.info('Removing pykmip...') | |
79 | for client in config: | |
80 | ctx.cluster.only(client).run( | |
81 | args=[ 'rm', '-rf', pykmipdir ], | |
82 | ) | |
83 | ||
84 | _bindep_txt = """# should be part of PyKMIP | |
85 | libffi-dev [platform:dpkg] | |
86 | libffi-devel [platform:rpm] | |
87 | libssl-dev [platform:dpkg] | |
88 | openssl-devel [platform:redhat] | |
89 | libopenssl-devel [platform:suse] | |
90 | libsqlite3-dev [platform:dpkg] | |
91 | sqlite-devel [platform:rpm] | |
92 | python-dev [platform:dpkg] | |
93 | python-devel [(platform:redhat platform:base-py2)] | |
94 | python3-dev [platform:dpkg] | |
95 | python3-devel [(platform:redhat platform:base-py3) platform:suse] | |
96 | python3 [platform:suse] | |
97 | """ | |
98 | ||
99 | @contextlib.contextmanager | |
100 | def install_packages(ctx, config): | |
101 | """ | |
102 | Download the packaged dependencies of PyKMIP. | |
103 | Remove install packages upon exit. | |
104 | ||
105 | The context passed in should be identical to the context | |
106 | passed in to the main task. | |
107 | """ | |
108 | assert isinstance(config, dict) | |
109 | log.info('Installing system dependenies for PyKMIP...') | |
110 | ||
111 | packages = {} | |
112 | for (client, _) in config.items(): | |
113 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
114 | # use bindep to read which dependencies we need from temp/bindep.txt | |
115 | fd, local_temp_path = tempfile.mkstemp(suffix='.txt', | |
116 | prefix='bindep-') | |
117 | os.write(fd, _bindep_txt.encode()) | |
118 | os.close(fd) | |
119 | fd, remote_temp_path = tempfile.mkstemp(suffix='.txt', | |
120 | prefix='bindep-') | |
121 | os.close(fd) | |
122 | remote.put_file(local_temp_path, remote_temp_path) | |
123 | os.remove(local_temp_path) | |
124 | run_in_pykmip_venv(ctx, remote, ['pip', 'install', 'bindep']) | |
125 | r = run_in_pykmip_venv(ctx, remote, | |
126 | ['bindep', '--brief', '--file', remote_temp_path], | |
127 | stdout=BytesIO(), | |
128 | check_status=False) # returns 1 on success? | |
129 | packages[client] = r.stdout.getvalue().decode().splitlines() | |
130 | for dep in packages[client]: | |
131 | install_package(dep, remote) | |
132 | try: | |
133 | yield | |
134 | finally: | |
135 | log.info('Removing system dependencies of PyKMIP...') | |
136 | ||
137 | for (client, _) in config.items(): | |
138 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
139 | for dep in packages[client]: | |
140 | remove_package(dep, remote) | |
141 | ||
142 | @contextlib.contextmanager | |
143 | def setup_venv(ctx, config): | |
144 | """ | |
145 | Setup the virtualenv for PyKMIP using pip. | |
146 | """ | |
147 | assert isinstance(config, dict) | |
148 | log.info('Setting up virtualenv for pykmip...') | |
149 | for (client, _) in config.items(): | |
a4b75251 | 150 | run_in_pykmip_dir(ctx, client, ['python3', '-m', 'venv', '.pykmipenv']) |
20effc67 | 151 | run_in_pykmip_venv(ctx, client, ['pip', 'install', '--upgrade', 'pip']) |
f67539c2 TL |
152 | run_in_pykmip_venv(ctx, client, ['pip', 'install', 'pytz', '-e', get_pykmip_dir(ctx)]) |
153 | yield | |
154 | ||
155 | def assign_ports(ctx, config, initial_port): | |
156 | """ | |
157 | Assign port numbers starting from @initial_port | |
158 | """ | |
159 | port = initial_port | |
160 | role_endpoints = {} | |
161 | for remote, roles_for_host in ctx.cluster.remotes.items(): | |
162 | for role in roles_for_host: | |
163 | if role in config: | |
164 | r = get_remote_for_role(ctx, role) | |
165 | role_endpoints[role] = r.ip_address, port, r.hostname | |
166 | port += 1 | |
167 | ||
168 | return role_endpoints | |
169 | ||
170 | def copy_policy_json(ctx, cclient, cconfig): | |
171 | run_in_pykmip_dir(ctx, cclient, | |
172 | ['cp', | |
173 | get_pykmip_dir(ctx)+'/examples/policy.json', | |
174 | get_pykmip_dir(ctx)]) | |
175 | ||
176 | _pykmip_configuration = """# configuration for pykmip | |
177 | [server] | |
178 | hostname={ipaddr} | |
179 | port={port} | |
180 | certificate_path={servercert} | |
181 | key_path={serverkey} | |
182 | ca_path={clientca} | |
183 | auth_suite=TLS1.2 | |
184 | policy_path={confdir} | |
185 | enable_tls_client_auth=False | |
186 | tls_cipher_suites= | |
187 | TLS_RSA_WITH_AES_128_CBC_SHA256 | |
188 | TLS_RSA_WITH_AES_256_CBC_SHA256 | |
189 | TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 | |
190 | logging_level=DEBUG | |
191 | database_path={confdir}/pykmip.sqlite | |
192 | [client] | |
193 | host={hostname} | |
194 | port=5696 | |
195 | certfile={clientcert} | |
196 | keyfile={clientkey} | |
197 | ca_certs={clientca} | |
198 | ssl_version=PROTOCOL_TLSv1_2 | |
199 | """ | |
200 | ||
201 | def create_pykmip_conf(ctx, cclient, cconfig): | |
202 | log.info('#0 cclient={} cconfig={}'.format(pprint.pformat(cclient),pprint.pformat(cconfig))) | |
203 | (remote,) = ctx.cluster.only(cclient).remotes.keys() | |
204 | pykmip_ipaddr, pykmip_port, pykmip_hostname = ctx.pykmip.endpoints[cclient] | |
205 | log.info('#1 ip,p,h {} {} {}'.format(pykmip_ipaddr, pykmip_port, pykmip_hostname)) | |
206 | clientca = cconfig.get('clientca', None) | |
207 | log.info('#2 clientca {}'.format(clientca)) | |
208 | serverkey = None | |
209 | servercert = cconfig.get('servercert', None) | |
210 | log.info('#3 servercert {}'.format(servercert)) | |
211 | servercert = ctx.ssl_certificates.get(servercert) | |
212 | log.info('#4 servercert {}'.format(servercert)) | |
213 | clientkey = None | |
214 | clientcert = cconfig.get('clientcert', None) | |
215 | log.info('#3 clientcert {}'.format(clientcert)) | |
216 | clientcert = ctx.ssl_certificates.get(clientcert) | |
217 | log.info('#4 clientcert {}'.format(clientcert)) | |
218 | clientca = ctx.ssl_certificates.get(clientca) | |
219 | log.info('#5 clientca {}'.format(clientca)) | |
220 | if servercert != None: | |
221 | serverkey = servercert.key | |
222 | servercert = servercert.certificate | |
223 | log.info('#6 serverkey {} servercert {}'.format(serverkey, servercert)) | |
224 | if clientcert != None: | |
225 | clientkey = clientcert.key | |
226 | clientcert = clientcert.certificate | |
227 | log.info('#6 clientkey {} clientcert {}'.format(clientkey, clientcert)) | |
228 | if clientca != None: | |
229 | clientca = clientca.certificate | |
230 | log.info('#7 clientca {}'.format(clientca)) | |
231 | if servercert == None or clientca == None or serverkey == None: | |
232 | log.info('#8 clientca {} serverkey {} servercert {}'.format(clientca, serverkey, servercert)) | |
233 | raise ConfigError('pykmip: Missing/bad servercert or clientca') | |
234 | pykmipdir = get_pykmip_dir(ctx) | |
235 | kmip_conf = _pykmip_configuration.format( | |
236 | ipaddr=pykmip_ipaddr, | |
237 | port=pykmip_port, | |
238 | confdir=pykmipdir, | |
239 | hostname=pykmip_hostname, | |
240 | clientca=clientca, | |
241 | clientkey=clientkey, | |
242 | clientcert=clientcert, | |
243 | serverkey=serverkey, | |
244 | servercert=servercert | |
245 | ) | |
246 | fd, local_temp_path = tempfile.mkstemp(suffix='.conf', | |
247 | prefix='pykmip') | |
248 | os.write(fd, kmip_conf.encode()) | |
249 | os.close(fd) | |
250 | remote.put_file(local_temp_path, pykmipdir+'/pykmip.conf') | |
251 | os.remove(local_temp_path) | |
252 | ||
253 | @contextlib.contextmanager | |
254 | def configure_pykmip(ctx, config): | |
255 | """ | |
256 | Configure pykmip paste-api and pykmip-api. | |
257 | """ | |
258 | assert isinstance(config, dict) | |
259 | (cclient, cconfig) = next(iter(config.items())) | |
260 | ||
261 | copy_policy_json(ctx, cclient, cconfig) | |
262 | create_pykmip_conf(ctx, cclient, cconfig) | |
263 | try: | |
264 | yield | |
265 | finally: | |
266 | pass | |
267 | ||
268 | def has_ceph_task(tasks): | |
269 | for task in tasks: | |
270 | for name, conf in task.items(): | |
271 | if name == 'ceph': | |
272 | return True | |
273 | return False | |
274 | ||
275 | @contextlib.contextmanager | |
276 | def run_pykmip(ctx, config): | |
277 | assert isinstance(config, dict) | |
278 | if hasattr(ctx, 'daemons'): | |
279 | pass | |
280 | elif has_ceph_task(ctx.config['tasks']): | |
281 | log.info('Delay start pykmip so ceph can do once-only daemon logic') | |
282 | try: | |
283 | yield | |
284 | finally: | |
285 | pass | |
286 | else: | |
287 | ctx.daemons = DaemonGroup() | |
288 | log.info('Running pykmip...') | |
289 | ||
290 | pykmipdir = get_pykmip_dir(ctx) | |
291 | ||
292 | for (client, _) in config.items(): | |
293 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
294 | cluster_name, _, client_id = teuthology.split_role(client) | |
295 | ||
296 | # start the public endpoint | |
297 | client_public_with_id = 'pykmip.public' + '.' + client_id | |
298 | ||
299 | run_cmd = 'cd ' + pykmipdir + ' && ' + \ | |
300 | '. .pykmipenv/bin/activate && ' + \ | |
301 | 'HOME={}'.format(pykmipdir) + ' && ' + \ | |
302 | 'exec pykmip-server -f pykmip.conf -l ' + \ | |
303 | pykmipdir + '/pykmip.log & { read; kill %1; }' | |
304 | ||
305 | ctx.daemons.add_daemon( | |
306 | remote, 'pykmip', client_public_with_id, | |
307 | cluster=cluster_name, | |
308 | args=['bash', '-c', run_cmd], | |
309 | logger=log.getChild(client), | |
310 | stdin=run.PIPE, | |
311 | cwd=pykmipdir, | |
312 | wait=False, | |
313 | check_status=False, | |
314 | ) | |
315 | ||
316 | # sleep driven synchronization | |
317 | time.sleep(10) | |
318 | try: | |
319 | yield | |
320 | finally: | |
321 | log.info('Stopping PyKMIP instance') | |
322 | ctx.daemons.get_daemon('pykmip', client_public_with_id, | |
323 | cluster_name).stop() | |
324 | ||
325 | make_keys_template = """ | |
326 | from kmip.pie import client | |
327 | from kmip import enums | |
328 | import ssl | |
329 | import sys | |
330 | import json | |
331 | from io import BytesIO | |
332 | ||
333 | c = client.ProxyKmipClient(config_file="{replace-with-config-file-path}") | |
334 | ||
335 | rl=[] | |
336 | for kwargs in {replace-with-secrets}: | |
337 | with c: | |
338 | key_id = c.create( | |
339 | enums.CryptographicAlgorithm.AES, | |
340 | 256, | |
341 | operation_policy_name='default', | |
342 | cryptographic_usage_mask=[ | |
343 | enums.CryptographicUsageMask.ENCRYPT, | |
344 | enums.CryptographicUsageMask.DECRYPT | |
345 | ], | |
346 | **kwargs | |
347 | ) | |
348 | c.activate(key_id) | |
349 | attrs = c.get_attributes(uid=key_id) | |
350 | r = {} | |
351 | for a in attrs[1]: | |
352 | r[str(a.attribute_name)] = str(a.attribute_value) | |
353 | rl.append(r) | |
354 | print(json.dumps(rl)) | |
355 | """ | |
356 | ||
357 | @contextlib.contextmanager | |
358 | def create_secrets(ctx, config): | |
359 | """ | |
360 | Create and activate any requested keys in kmip | |
361 | """ | |
362 | assert isinstance(config, dict) | |
363 | ||
364 | pykmipdir = get_pykmip_dir(ctx) | |
365 | pykmip_conf_path = pykmipdir + '/pykmip.conf' | |
366 | my_output = BytesIO() | |
367 | for (client,cconf) in config.items(): | |
368 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
369 | secrets=cconf.get('secrets') | |
370 | if secrets: | |
371 | secrets_json = json.dumps(cconf['secrets']) | |
372 | make_keys = make_keys_template \ | |
373 | .replace("{replace-with-secrets}",secrets_json) \ | |
374 | .replace("{replace-with-config-file-path}",pykmip_conf_path) | |
375 | my_output.truncate() | |
376 | remote.run(args=[run.Raw('. cephtest/pykmip/.pykmipenv/bin/activate;' \ | |
377 | + 'python')], stdin=make_keys, stdout = my_output) | |
378 | ctx.pykmip.keys[client] = json.loads(my_output.getvalue().decode()) | |
379 | try: | |
380 | yield | |
381 | finally: | |
382 | pass | |
383 | ||
384 | @contextlib.contextmanager | |
385 | def task(ctx, config): | |
386 | """ | |
387 | Deploy and configure PyKMIP | |
388 | ||
389 | Example of configuration: | |
390 | ||
391 | tasks: | |
392 | - install: | |
393 | - ceph: | |
394 | conf: | |
395 | client: | |
396 | rgw crypt s3 kms backend: kmip | |
397 | rgw crypt kmip ca path: /home/ubuntu/cephtest/ca/kmiproot.crt | |
398 | rgw crypt kmip client cert: /home/ubuntu/cephtest/ca/kmip-client.crt | |
399 | rgw crypt kmip client key: /home/ubuntu/cephtest/ca/kmip-client.key | |
400 | rgw crypt kmip kms key template: pykmip-$keyid | |
401 | - openssl_keys: | |
402 | kmiproot: | |
403 | client: client.0 | |
404 | cn: kmiproot | |
405 | key-type: rsa:4096 | |
406 | - openssl_keys: | |
407 | kmip-server: | |
408 | client: client.0 | |
409 | ca: kmiproot | |
410 | kmip-client: | |
411 | client: client.0 | |
412 | ca: kmiproot | |
413 | cn: rgw-client | |
414 | - pykmip: | |
415 | client.0: | |
416 | force-branch: master | |
417 | clientca: kmiproot | |
418 | servercert: kmip-server | |
419 | clientcert: kmip-client | |
420 | secrets: | |
421 | - name: pykmip-key-1 | |
422 | - name: pykmip-key-2 | |
423 | - rgw: | |
424 | client.0: | |
425 | use-pykmip-role: client.0 | |
426 | - s3tests: | |
427 | client.0: | |
428 | force-branch: master | |
429 | """ | |
430 | assert config is None or isinstance(config, list) \ | |
431 | or isinstance(config, dict), \ | |
432 | "task pykmip only supports a list or dictionary for configuration" | |
433 | all_clients = ['client.{id}'.format(id=id_) | |
434 | for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] | |
435 | if config is None: | |
436 | config = all_clients | |
437 | if isinstance(config, list): | |
438 | config = dict.fromkeys(config) | |
439 | ||
440 | overrides = ctx.config.get('overrides', {}) | |
441 | # merge each client section, not the top level. | |
442 | for client in config.keys(): | |
443 | if not config[client]: | |
444 | config[client] = {} | |
445 | teuthology.deep_merge(config[client], overrides.get('pykmip', {})) | |
446 | ||
447 | log.debug('PyKMIP config is %s', config) | |
448 | ||
449 | if not hasattr(ctx, 'ssl_certificates'): | |
450 | raise ConfigError('pykmip must run after the openssl_keys task') | |
451 | ||
452 | ||
453 | ctx.pykmip = argparse.Namespace() | |
454 | ctx.pykmip.endpoints = assign_ports(ctx, config, 5696) | |
455 | ctx.pykmip.keys = {} | |
456 | ||
457 | with contextutil.nested( | |
458 | lambda: download(ctx=ctx, config=config), | |
459 | lambda: setup_venv(ctx=ctx, config=config), | |
460 | lambda: install_packages(ctx=ctx, config=config), | |
461 | lambda: configure_pykmip(ctx=ctx, config=config), | |
462 | lambda: run_pykmip(ctx=ctx, config=config), | |
463 | lambda: create_secrets(ctx=ctx, config=config), | |
464 | ): | |
465 | yield |