ceph/qa/tasks/pykmip.py

   1 """
   2 Deploy and configure PyKMIP for Teuthology
   3 """
   4 import argparse
   5 import contextlib
   6 import logging
   7 import time
   8 import tempfile
   9 import json
  10 import os
  11 from io import BytesIO
  12 from teuthology.orchestra.daemon import DaemonGroup
  13 from teuthology.orchestra.remote import Remote
  14
  15 import pprint
  16
  17 from teuthology import misc as teuthology
  18 from teuthology import contextutil
  19 from teuthology.orchestra import run
  20 from teuthology.packaging import install_package
  21 from teuthology.packaging import remove_package
  22 from teuthology.exceptions import ConfigError
  23 from tasks.util import get_remote_for_role
  24
  25 log = logging.getLogger(__name__)
  26
  27
  28 def get_pykmip_dir(ctx):
  29     return '{tdir}/pykmip'.format(tdir=teuthology.get_testdir(ctx))
  30
  31 def run_in_pykmip_dir(ctx, client, args, **kwargs):
  32     (remote,) = [client] if isinstance(client,Remote) else ctx.cluster.only(client).remotes.keys()
  33     return remote.run(
  34         args=['cd', get_pykmip_dir(ctx), run.Raw('&&'), ] + args,
  35         **kwargs
  36     )
  37
  38 def run_in_pykmip_venv(ctx, client, args, **kwargs):
  39     return run_in_pykmip_dir(ctx, client,
  40         args = ['.', '.pykmipenv/bin/activate',
  41                          run.Raw('&&')
  42                         ] + args, **kwargs)
  43
  44 @contextlib.contextmanager
  45 def download(ctx, config):
  46     """
  47     Download PyKMIP from github.
  48     Remove downloaded file upon exit.
  49
  50     The context passed in should be identical to the context
  51     passed in to the main task.
  52     """
  53     assert isinstance(config, dict)
  54     log.info('Downloading pykmip...')
  55     pykmipdir = get_pykmip_dir(ctx)
  56
  57     for (client, cconf) in config.items():
  58         branch = cconf.get('force-branch', 'master')
  59         repo = cconf.get('force-repo', 'https://github.com/OpenKMIP/PyKMIP')
  60         sha1 = cconf.get('sha1')
  61         log.info("Using branch '%s' for pykmip", branch)
  62         log.info('sha1=%s', sha1)
  63
  64         ctx.cluster.only(client).run(
  65             args=[
  66                 'git', 'clone', '-b', branch, repo,
  67                 pykmipdir,
  68                 ],
  69             )
  70         if sha1 is not None:
  71             run_in_pykmip_dir(ctx, client, [
  72                     'git', 'reset', '--hard', sha1,
  73                 ],
  74             )
  75     try:
  76         yield
  77     finally:
  78         log.info('Removing pykmip...')
  79         for client in config:
  80             ctx.cluster.only(client).run(
  81                 args=[ 'rm', '-rf', pykmipdir ],
  82             )
  83
  84 _bindep_txt = """# should be part of PyKMIP
  85 libffi-dev [platform:dpkg]
  86 libffi-devel [platform:rpm]
  87 libssl-dev [platform:dpkg]
  88 openssl-devel [platform:redhat]
  89 libopenssl-devel [platform:suse]
  90 libsqlite3-dev [platform:dpkg]
  91 sqlite-devel [platform:rpm]
  92 python-dev [platform:dpkg]
  93 python-devel [(platform:redhat platform:base-py2)]
  94 python3-dev [platform:dpkg]
  95 python3-devel [(platform:redhat platform:base-py3) platform:suse]
  96 python3 [platform:suse]
  97 """
  98
  99 @contextlib.contextmanager
 100 def install_packages(ctx, config):
 101     """
 102     Download the packaged dependencies of PyKMIP.
 103     Remove install packages upon exit.
 104
 105     The context passed in should be identical to the context
 106     passed in to the main task.
 107     """
 108     assert isinstance(config, dict)
 109     log.info('Installing system dependenies for PyKMIP...')
 110
 111     packages = {}
 112     for (client, _) in config.items():
 113         (remote,) = ctx.cluster.only(client).remotes.keys()
 114         # use bindep to read which dependencies we need from temp/bindep.txt
 115         fd, local_temp_path = tempfile.mkstemp(suffix='.txt',
 116                                                prefix='bindep-')
 117         os.write(fd, _bindep_txt.encode())
 118         os.close(fd)
 119         fd, remote_temp_path = tempfile.mkstemp(suffix='.txt',
 120                                                prefix='bindep-')
 121         os.close(fd)
 122         remote.put_file(local_temp_path, remote_temp_path)
 123         os.remove(local_temp_path)
 124         run_in_pykmip_venv(ctx, remote, ['pip', 'install', 'bindep'])
 125         r = run_in_pykmip_venv(ctx, remote,
 126                 ['bindep', '--brief', '--file', remote_temp_path],
 127                 stdout=BytesIO(),
 128                 check_status=False) # returns 1 on success?
 129         packages[client] = r.stdout.getvalue().decode().splitlines()
 130         for dep in packages[client]:
 131             install_package(dep, remote)
 132     try:
 133         yield
 134     finally:
 135         log.info('Removing system dependencies of PyKMIP...')
 136
 137         for (client, _) in config.items():
 138             (remote,) = ctx.cluster.only(client).remotes.keys()
 139             for dep in packages[client]:
 140                 remove_package(dep, remote)
 141
 142 @contextlib.contextmanager
 143 def setup_venv(ctx, config):
 144     """
 145     Setup the virtualenv for PyKMIP using pip.
 146     """
 147     assert isinstance(config, dict)
 148     log.info('Setting up virtualenv for pykmip...')
 149     for (client, _) in config.items():
 150         run_in_pykmip_dir(ctx, client, ['python3', '-m', 'venv', '.pykmipenv'])
 151         run_in_pykmip_venv(ctx, client, ['pip', 'install', '--upgrade', 'pip'])
 152         run_in_pykmip_venv(ctx, client, ['pip', 'install', 'pytz', '-e', get_pykmip_dir(ctx)])
 153     yield
 154
 155 def assign_ports(ctx, config, initial_port):
 156     """
 157     Assign port numbers starting from @initial_port
 158     """
 159     port = initial_port
 160     role_endpoints = {}
 161     for remote, roles_for_host in ctx.cluster.remotes.items():
 162         for role in roles_for_host:
 163             if role in config:
 164                 r = get_remote_for_role(ctx, role)
 165                 role_endpoints[role] = r.ip_address, port, r.hostname
 166                 port += 1
 167
 168     return role_endpoints
 169
 170 def copy_policy_json(ctx, cclient, cconfig):
 171     run_in_pykmip_dir(ctx, cclient,
 172                         ['cp',
 173                          get_pykmip_dir(ctx)+'/examples/policy.json',
 174                          get_pykmip_dir(ctx)])
 175
 176 _pykmip_configuration = """# configuration for pykmip
 177 [server]
 178 hostname={ipaddr}
 179 port={port}
 180 certificate_path={servercert}
 181 key_path={serverkey}
 182 ca_path={clientca}
 183 auth_suite=TLS1.2
 184 policy_path={confdir}
 185 enable_tls_client_auth=False
 186 tls_cipher_suites=
 187     TLS_RSA_WITH_AES_128_CBC_SHA256
 188     TLS_RSA_WITH_AES_256_CBC_SHA256
 189     TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384
 190 logging_level=DEBUG
 191 database_path={confdir}/pykmip.sqlite
 192 [client]
 193 host={hostname}
 194 port=5696
 195 certfile={clientcert}
 196 keyfile={clientkey}
 197 ca_certs={clientca}
 198 ssl_version=PROTOCOL_TLSv1_2
 199 """
 200
 201 def create_pykmip_conf(ctx, cclient, cconfig):
 202     log.info('#0 cclient={} cconfig={}'.format(pprint.pformat(cclient),pprint.pformat(cconfig)))
 203     (remote,) = ctx.cluster.only(cclient).remotes.keys()
 204     pykmip_ipaddr, pykmip_port, pykmip_hostname = ctx.pykmip.endpoints[cclient]
 205     log.info('#1 ip,p,h {} {} {}'.format(pykmip_ipaddr, pykmip_port, pykmip_hostname))
 206     clientca = cconfig.get('clientca', None)
 207     log.info('#2 clientca {}'.format(clientca))
 208     serverkey = None
 209     servercert = cconfig.get('servercert', None)
 210     log.info('#3 servercert {}'.format(servercert))
 211     servercert = ctx.ssl_certificates.get(servercert)
 212     log.info('#4 servercert {}'.format(servercert))
 213     clientkey = None
 214     clientcert = cconfig.get('clientcert', None)
 215     log.info('#3 clientcert {}'.format(clientcert))
 216     clientcert = ctx.ssl_certificates.get(clientcert)
 217     log.info('#4 clientcert {}'.format(clientcert))
 218     clientca = ctx.ssl_certificates.get(clientca)
 219     log.info('#5 clientca {}'.format(clientca))
 220     if servercert != None:
 221       serverkey = servercert.key
 222       servercert = servercert.certificate
 223       log.info('#6 serverkey {} servercert {}'.format(serverkey, servercert))
 224     if clientcert != None:
 225       clientkey = clientcert.key
 226       clientcert = clientcert.certificate
 227       log.info('#6 clientkey {} clientcert {}'.format(clientkey, clientcert))
 228     if clientca != None:
 229       clientca = clientca.certificate
 230       log.info('#7 clientca {}'.format(clientca))
 231     if servercert == None or clientca == None or serverkey == None:
 232       log.info('#8 clientca {} serverkey {} servercert {}'.format(clientca, serverkey, servercert))
 233       raise ConfigError('pykmip: Missing/bad servercert or clientca')
 234     pykmipdir = get_pykmip_dir(ctx)
 235     kmip_conf = _pykmip_configuration.format(
 236         ipaddr=pykmip_ipaddr,
 237         port=pykmip_port,
 238         confdir=pykmipdir,
 239         hostname=pykmip_hostname,
 240         clientca=clientca,
 241         clientkey=clientkey,
 242         clientcert=clientcert,
 243         serverkey=serverkey,
 244         servercert=servercert
 245     )
 246     fd, local_temp_path = tempfile.mkstemp(suffix='.conf',
 247                                            prefix='pykmip')
 248     os.write(fd, kmip_conf.encode())
 249     os.close(fd)
 250     remote.put_file(local_temp_path, pykmipdir+'/pykmip.conf')
 251     os.remove(local_temp_path)
 252
 253 @contextlib.contextmanager
 254 def configure_pykmip(ctx, config):
 255     """
 256     Configure pykmip paste-api and pykmip-api.
 257     """
 258     assert isinstance(config, dict)
 259     (cclient, cconfig) = next(iter(config.items()))
 260
 261     copy_policy_json(ctx, cclient, cconfig)
 262     create_pykmip_conf(ctx, cclient, cconfig)
 263     try:
 264         yield
 265     finally:
 266         pass
 267
 268 def has_ceph_task(tasks):
 269     for task in tasks:
 270         for name, conf in task.items():
 271             if name == 'ceph':
 272                 return True
 273     return False
 274
 275 @contextlib.contextmanager
 276 def run_pykmip(ctx, config):
 277     assert isinstance(config, dict)
 278     if hasattr(ctx, 'daemons'):
 279         pass
 280     elif has_ceph_task(ctx.config['tasks']):
 281         log.info('Delay start pykmip so ceph can do once-only daemon logic')
 282         try:
 283             yield
 284         finally:
 285             pass
 286     else:
 287         ctx.daemons = DaemonGroup()
 288     log.info('Running pykmip...')
 289
 290     pykmipdir = get_pykmip_dir(ctx)
 291
 292     for (client, _) in config.items():
 293         (remote,) = ctx.cluster.only(client).remotes.keys()
 294         cluster_name, _, client_id = teuthology.split_role(client)
 295
 296         # start the public endpoint
 297         client_public_with_id = 'pykmip.public' + '.' + client_id
 298
 299         run_cmd = 'cd ' + pykmipdir + ' && ' + \
 300                   '. .pykmipenv/bin/activate && ' + \
 301                   'HOME={}'.format(pykmipdir) + ' && ' + \
 302                   'exec pykmip-server -f pykmip.conf -l ' + \
 303                   pykmipdir + '/pykmip.log & { read; kill %1; }'
 304
 305         ctx.daemons.add_daemon(
 306             remote, 'pykmip', client_public_with_id,
 307             cluster=cluster_name,
 308             args=['bash', '-c', run_cmd],
 309             logger=log.getChild(client),
 310             stdin=run.PIPE,
 311             cwd=pykmipdir,
 312             wait=False,
 313             check_status=False,
 314         )
 315
 316         # sleep driven synchronization
 317         time.sleep(10)
 318     try:
 319         yield
 320     finally:
 321         log.info('Stopping PyKMIP instance')
 322         ctx.daemons.get_daemon('pykmip', client_public_with_id,
 323                                cluster_name).stop()
 324
 325 make_keys_template = """
 326 from kmip.pie import client
 327 from kmip import enums
 328 import ssl
 329 import sys
 330 import json
 331 from io import BytesIO
 332
 333 c = client.ProxyKmipClient(config_file="{replace-with-config-file-path}")
 334
 335 rl=[]
 336 for kwargs in {replace-with-secrets}:
 337  with c:
 338   key_id = c.create(
 339    enums.CryptographicAlgorithm.AES,
 340    256,
 341    operation_policy_name='default',
 342    cryptographic_usage_mask=[
 343     enums.CryptographicUsageMask.ENCRYPT,
 344     enums.CryptographicUsageMask.DECRYPT
 345    ],
 346    **kwargs
 347   )
 348   c.activate(key_id)
 349   attrs = c.get_attributes(uid=key_id)
 350   r = {}
 351   for a in attrs[1]:
 352    r[str(a.attribute_name)] = str(a.attribute_value)
 353   rl.append(r)
 354 print(json.dumps(rl))
 355 """
 356
 357 @contextlib.contextmanager
 358 def create_secrets(ctx, config):
 359     """
 360     Create and activate any requested keys in kmip
 361     """
 362     assert isinstance(config, dict)
 363
 364     pykmipdir = get_pykmip_dir(ctx)
 365     pykmip_conf_path = pykmipdir + '/pykmip.conf'
 366     my_output = BytesIO()
 367     for (client,cconf) in config.items():
 368         (remote,) = ctx.cluster.only(client).remotes.keys()
 369         secrets=cconf.get('secrets')
 370         if secrets:
 371             secrets_json = json.dumps(cconf['secrets'])
 372             make_keys = make_keys_template \
 373                 .replace("{replace-with-secrets}",secrets_json) \
 374                 .replace("{replace-with-config-file-path}",pykmip_conf_path)
 375             my_output.truncate()
 376             remote.run(args=[run.Raw('. cephtest/pykmip/.pykmipenv/bin/activate;' \
 377                 + 'python')], stdin=make_keys, stdout = my_output)
 378             ctx.pykmip.keys[client] = json.loads(my_output.getvalue().decode())
 379     try:
 380         yield
 381     finally:
 382         pass
 383
 384 @contextlib.contextmanager
 385 def task(ctx, config):
 386     """
 387     Deploy and configure PyKMIP
 388
 389     Example of configuration:
 390
 391     tasks:
 392     - install:
 393     - ceph:
 394        conf:
 395         client:
 396          rgw crypt s3 kms backend: kmip
 397          rgw crypt kmip ca path: /home/ubuntu/cephtest/ca/kmiproot.crt
 398          rgw crypt kmip client cert: /home/ubuntu/cephtest/ca/kmip-client.crt
 399          rgw crypt kmip client key: /home/ubuntu/cephtest/ca/kmip-client.key
 400          rgw crypt kmip kms key template: pykmip-$keyid
 401     - openssl_keys:
 402        kmiproot:
 403          client: client.0
 404          cn: kmiproot
 405          key-type: rsa:4096
 406     - openssl_keys:
 407        kmip-server:
 408          client: client.0
 409          ca: kmiproot
 410        kmip-client:
 411          client: client.0
 412          ca: kmiproot
 413          cn: rgw-client
 414     - pykmip:
 415         client.0:
 416           force-branch: master
 417           clientca: kmiproot
 418           servercert: kmip-server
 419           clientcert: kmip-client
 420           secrets:
 421           - name: pykmip-key-1
 422           - name: pykmip-key-2
 423     - rgw:
 424         client.0:
 425           use-pykmip-role: client.0
 426     - s3tests:
 427         client.0:
 428           force-branch: master
 429     """
 430     assert config is None or isinstance(config, list) \
 431         or isinstance(config, dict), \
 432         "task pykmip only supports a list or dictionary for configuration"
 433     all_clients = ['client.{id}'.format(id=id_)
 434                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 435     if config is None:
 436         config = all_clients
 437     if isinstance(config, list):
 438         config = dict.fromkeys(config)
 439
 440     overrides = ctx.config.get('overrides', {})
 441     # merge each client section, not the top level.
 442     for client in config.keys():
 443         if not config[client]:
 444             config[client] = {}
 445         teuthology.deep_merge(config[client], overrides.get('pykmip', {}))
 446
 447     log.debug('PyKMIP config is %s', config)
 448
 449     if not hasattr(ctx, 'ssl_certificates'):
 450         raise ConfigError('pykmip must run after the openssl_keys task')
 451
 452
 453     ctx.pykmip = argparse.Namespace()
 454     ctx.pykmip.endpoints = assign_ports(ctx, config, 5696)
 455     ctx.pykmip.keys = {}
 456
 457     with contextutil.nested(
 458         lambda: download(ctx=ctx, config=config),
 459         lambda: setup_venv(ctx=ctx, config=config),
 460         lambda: install_packages(ctx=ctx, config=config),
 461         lambda: configure_pykmip(ctx=ctx, config=config),
 462         lambda: run_pykmip(ctx=ctx, config=config),
 463         lambda: create_secrets(ctx=ctx, config=config),
 464         ):
 465         yield