ceph/qa/tasks/devstack.py

   1 #!/usr/bin/env python
   2 import contextlib
   3 import logging
   4 import textwrap
   5 import time
   6 from configparser import ConfigParser
   7 from io import BytesIO, StringIO
   8
   9 from teuthology.orchestra import run
  10 from teuthology import misc
  11 from teuthology.contextutil import nested
  12
  13 log = logging.getLogger(__name__)
  14
  15 DEVSTACK_GIT_REPO = 'https://github.com/openstack-dev/devstack.git'
  16 DS_STABLE_BRANCHES = ("havana", "grizzly")
  17
  18 is_devstack_node = lambda role: role.startswith('devstack')
  19 is_osd_node = lambda role: role.startswith('osd')
  20
  21
  22 @contextlib.contextmanager
  23 def task(ctx, config):
  24     if config is None:
  25         config = {}
  26     if not isinstance(config, dict):
  27         raise TypeError("config must be a dict")
  28     with nested(lambda: install(ctx=ctx, config=config),
  29                 lambda: smoke(ctx=ctx, config=config),
  30                 ):
  31         yield
  32
  33
  34 @contextlib.contextmanager
  35 def install(ctx, config):
  36     """
  37     Install OpenStack DevStack and configure it to use a Ceph cluster for
  38     Glance and Cinder.
  39
  40     Requires one node with a role 'devstack'
  41
  42     Since devstack runs rampant on the system it's used on, typically you will
  43     want to reprovision that machine after using devstack on it.
  44
  45     Also, the default 2GB of RAM that is given to vps nodes is insufficient. I
  46     recommend 4GB. Downburst can be instructed to give 4GB to a vps node by
  47     adding this to the yaml:
  48
  49     downburst:
  50         ram: 4G
  51
  52     This was created using documentation found here:
  53         https://github.com/openstack-dev/devstack/blob/master/README.md
  54         http://docs.ceph.com/en/latest/rbd/rbd-openstack/
  55     """
  56     if config is None:
  57         config = {}
  58     if not isinstance(config, dict):
  59         raise TypeError("config must be a dict")
  60
  61     devstack_node = next(iter(ctx.cluster.only(is_devstack_node).remotes.keys()))
  62     an_osd_node = next(iter(ctx.cluster.only(is_osd_node).remotes.keys()))
  63
  64     devstack_branch = config.get("branch", "master")
  65     install_devstack(devstack_node, devstack_branch)
  66     try:
  67         configure_devstack_and_ceph(ctx, config, devstack_node, an_osd_node)
  68         yield
  69     finally:
  70         pass
  71
  72
  73 def install_devstack(devstack_node, branch="master"):
  74     log.info("Cloning DevStack repo...")
  75
  76     args = ['git', 'clone', DEVSTACK_GIT_REPO]
  77     devstack_node.run(args=args)
  78
  79     if branch != "master":
  80         if branch in DS_STABLE_BRANCHES and not branch.startswith("stable"):
  81             branch = "stable/" + branch
  82         log.info("Checking out {branch} branch...".format(branch=branch))
  83         cmd = "cd devstack && git checkout " + branch
  84         devstack_node.run(args=cmd)
  85
  86     log.info("Installing DevStack...")
  87     args = ['cd', 'devstack', run.Raw('&&'), './stack.sh']
  88     devstack_node.run(args=args)
  89
  90
  91 def configure_devstack_and_ceph(ctx, config, devstack_node, ceph_node):
  92     pool_size = config.get('pool_size', '128')
  93     create_pools(ceph_node, pool_size)
  94     distribute_ceph_conf(devstack_node, ceph_node)
  95     # This is where we would install python-ceph and ceph-common but it appears
  96     # the ceph task does that for us.
  97     generate_ceph_keys(ceph_node)
  98     distribute_ceph_keys(devstack_node, ceph_node)
  99     secret_uuid = set_libvirt_secret(devstack_node, ceph_node)
 100     update_devstack_config_files(devstack_node, secret_uuid)
 101     set_apache_servername(devstack_node)
 102     # Rebooting is the most-often-used method of restarting devstack services
 103     misc.reboot(devstack_node)
 104     start_devstack(devstack_node)
 105     restart_apache(devstack_node)
 106
 107
 108 def create_pools(ceph_node, pool_size):
 109     log.info("Creating pools on Ceph cluster...")
 110
 111     for pool_name in ['volumes', 'images', 'backups']:
 112         args = ['sudo', 'ceph', 'osd', 'pool', 'create', pool_name, pool_size]
 113         ceph_node.run(args=args)
 114
 115
 116 def distribute_ceph_conf(devstack_node, ceph_node):
 117     log.info("Copying ceph.conf to DevStack node...")
 118
 119     ceph_conf_path = '/etc/ceph/ceph.conf'
 120     ceph_conf = ceph_node.read_file(ceph_conf_path, sudo=True)
 121     devstack_node.write_file(ceph_conf_path, ceph_conf, sudo=True)
 122
 123
 124 def generate_ceph_keys(ceph_node):
 125     log.info("Generating Ceph keys...")
 126
 127     ceph_auth_cmds = [
 128         ['sudo', 'ceph', 'auth', 'get-or-create', 'client.cinder', 'mon',
 129             'allow r', 'osd', 'allow class-read object_prefix rbd_children, allow rwx pool=volumes, allow rx pool=images'],  # noqa
 130         ['sudo', 'ceph', 'auth', 'get-or-create', 'client.glance', 'mon',
 131             'allow r', 'osd', 'allow class-read object_prefix rbd_children, allow rwx pool=images'],  # noqa
 132         ['sudo', 'ceph', 'auth', 'get-or-create', 'client.cinder-backup', 'mon',
 133             'allow r', 'osd', 'allow class-read object_prefix rbd_children, allow rwx pool=backups'],  # noqa
 134     ]
 135     for cmd in ceph_auth_cmds:
 136         ceph_node.run(args=cmd)
 137
 138
 139 def distribute_ceph_keys(devstack_node, ceph_node):
 140     log.info("Copying Ceph keys to DevStack node...")
 141
 142     def copy_key(from_remote, key_name, to_remote, dest_path, owner):
 143         key_stringio = BytesIO()
 144         from_remote.run(
 145             args=['sudo', 'ceph', 'auth', 'get-or-create', key_name],
 146             stdout=key_stringio)
 147         key_stringio.seek(0)
 148         to_remote.write_file(dest_path, key_stringio, owner=owner, sudo=True)
 149     keys = [
 150         dict(name='client.glance',
 151              path='/etc/ceph/ceph.client.glance.keyring',
 152              # devstack appears to just want root:root
 153              #owner='glance:glance',
 154              ),
 155         dict(name='client.cinder',
 156              path='/etc/ceph/ceph.client.cinder.keyring',
 157              # devstack appears to just want root:root
 158              #owner='cinder:cinder',
 159              ),
 160         dict(name='client.cinder-backup',
 161              path='/etc/ceph/ceph.client.cinder-backup.keyring',
 162              # devstack appears to just want root:root
 163              #owner='cinder:cinder',
 164              ),
 165     ]
 166     for key_dict in keys:
 167         copy_key(ceph_node, key_dict['name'], devstack_node,
 168                  key_dict['path'], key_dict.get('owner'))
 169
 170
 171 def set_libvirt_secret(devstack_node, ceph_node):
 172     log.info("Setting libvirt secret...")
 173
 174     cinder_key = ceph_node.sh('sudo ceph auth get-key client.cinder').strip()
 175     uuid = devstack_node.sh('uuidgen').strip()
 176
 177     secret_path = '/tmp/secret.xml'
 178     secret_template = textwrap.dedent("""
 179     <secret ephemeral='no' private='no'>
 180         <uuid>{uuid}</uuid>
 181         <usage type='ceph'>
 182             <name>client.cinder secret</name>
 183         </usage>
 184     </secret>""")
 185     secret_data = secret_template.format(uuid=uuid)
 186     devstack_node.write_file(secret_path, secret_data)
 187     devstack_node.run(args=['sudo', 'virsh', 'secret-define', '--file',
 188                             secret_path])
 189     devstack_node.run(args=['sudo', 'virsh', 'secret-set-value', '--secret',
 190                             uuid, '--base64', cinder_key])
 191     return uuid
 192
 193
 194 def update_devstack_config_files(devstack_node, secret_uuid):
 195     log.info("Updating DevStack config files to use Ceph...")
 196
 197     def backup_config(node, file_name, backup_ext='.orig.teuth'):
 198         node.run(args=['cp', '-f', file_name, file_name + backup_ext])
 199
 200     def update_config(config_name, config_stream, update_dict,
 201                       section='DEFAULT'):
 202         parser = ConfigParser()
 203         parser.read_file(config_stream)
 204         for (key, value) in update_dict.items():
 205             parser.set(section, key, value)
 206         out_stream = StringIO()
 207         parser.write(out_stream)
 208         out_stream.seek(0)
 209         return out_stream
 210
 211     updates = [
 212         dict(name='/etc/glance/glance-api.conf', options=dict(
 213             default_store='rbd',
 214             rbd_store_user='glance',
 215             rbd_store_pool='images',
 216             show_image_direct_url='True',)),
 217         dict(name='/etc/cinder/cinder.conf', options=dict(
 218             volume_driver='cinder.volume.drivers.rbd.RBDDriver',
 219             rbd_pool='volumes',
 220             rbd_ceph_conf='/etc/ceph/ceph.conf',
 221             rbd_flatten_volume_from_snapshot='false',
 222             rbd_max_clone_depth='5',
 223             glance_api_version='2',
 224             rbd_user='cinder',
 225             rbd_secret_uuid=secret_uuid,
 226             backup_driver='cinder.backup.drivers.ceph',
 227             backup_ceph_conf='/etc/ceph/ceph.conf',
 228             backup_ceph_user='cinder-backup',
 229             backup_ceph_chunk_size='134217728',
 230             backup_ceph_pool='backups',
 231             backup_ceph_stripe_unit='0',
 232             backup_ceph_stripe_count='0',
 233             restore_discard_excess_bytes='true',
 234             )),
 235         dict(name='/etc/nova/nova.conf', options=dict(
 236             libvirt_images_type='rbd',
 237             libvirt_images_rbd_pool='volumes',
 238             libvirt_images_rbd_ceph_conf='/etc/ceph/ceph.conf',
 239             rbd_user='cinder',
 240             rbd_secret_uuid=secret_uuid,
 241             libvirt_inject_password='false',
 242             libvirt_inject_key='false',
 243             libvirt_inject_partition='-2',
 244             )),
 245     ]
 246
 247     for update in updates:
 248         file_name = update['name']
 249         options = update['options']
 250         config_data = devstack_node.read_file(file_name, sudo=True)
 251         config_stream = StringIO(config_data)
 252         backup_config(devstack_node, file_name)
 253         new_config_stream = update_config(file_name, config_stream, options)
 254         devstack_node.write_file(file_name, new_config_stream, sudo=True)
 255
 256
 257 def set_apache_servername(node):
 258     # Apache complains: "Could not reliably determine the server's fully
 259     # qualified domain name, using 127.0.0.1 for ServerName"
 260     # So, let's make sure it knows its name.
 261     log.info("Setting Apache ServerName...")
 262
 263     hostname = node.hostname
 264     config_file = '/etc/apache2/conf.d/servername'
 265     config_data = "ServerName {name}".format(name=hostname)
 266     node.write_file(config_file, config_data, sudo=True)
 267
 268
 269 def start_devstack(devstack_node):
 270     log.info("Patching devstack start script...")
 271     # This causes screen to start headless - otherwise rejoin-stack.sh fails
 272     # because there is no terminal attached.
 273     cmd = "cd devstack && sed -ie 's/screen -c/screen -dm -c/' rejoin-stack.sh"
 274     devstack_node.run(args=cmd)
 275
 276     log.info("Starting devstack...")
 277     cmd = "cd devstack && ./rejoin-stack.sh"
 278     devstack_node.run(args=cmd)
 279
 280     # This was added because I was getting timeouts on Cinder requests - which
 281     # were trying to access Keystone on port 5000. A more robust way to handle
 282     # this would be to introduce a wait-loop on devstack_node that checks to
 283     # see if a service is listening on port 5000.
 284     log.info("Waiting 30s for devstack to start...")
 285     time.sleep(30)
 286
 287
 288 def restart_apache(node):
 289     node.run(args=['sudo', '/etc/init.d/apache2', 'restart'], wait=True)
 290
 291
 292 @contextlib.contextmanager
 293 def exercise(ctx, config):
 294     log.info("Running devstack exercises...")
 295
 296     if config is None:
 297         config = {}
 298     if not isinstance(config, dict):
 299         raise TypeError("config must be a dict")
 300
 301     devstack_node = next(iter(ctx.cluster.only(is_devstack_node).remotes.keys()))
 302
 303     # TODO: save the log *and* preserve failures
 304     #devstack_archive_dir = create_devstack_archive(ctx, devstack_node)
 305
 306     try:
 307         #cmd = "cd devstack && ./exercise.sh 2>&1 | tee {dir}/exercise.log".format(  # noqa
 308         #    dir=devstack_archive_dir)
 309         cmd = "cd devstack && ./exercise.sh"
 310         devstack_node.run(args=cmd, wait=True)
 311         yield
 312     finally:
 313         pass
 314
 315
 316 def create_devstack_archive(ctx, devstack_node):
 317     test_dir = misc.get_testdir(ctx)
 318     devstack_archive_dir = "{test_dir}/archive/devstack".format(
 319         test_dir=test_dir)
 320     devstack_node.run(args="mkdir -p " + devstack_archive_dir)
 321     return devstack_archive_dir
 322
 323
 324 @contextlib.contextmanager
 325 def smoke(ctx, config):
 326     log.info("Running a basic smoketest...")
 327
 328     devstack_node = next(iter(ctx.cluster.only(is_devstack_node).remotes.keys()))
 329     an_osd_node = next(iter(ctx.cluster.only(is_osd_node).remotes.keys()))
 330
 331     try:
 332         create_volume(devstack_node, an_osd_node, 'smoke0', 1)
 333         yield
 334     finally:
 335         pass
 336
 337
 338 def create_volume(devstack_node, ceph_node, vol_name, size):
 339     """
 340     :param size: The size of the volume, in GB
 341     """
 342     size = str(size)
 343     log.info("Creating a {size}GB volume named {name}...".format(
 344         name=vol_name,
 345         size=size))
 346     args = ['source', 'devstack/openrc', run.Raw('&&'), 'cinder', 'create',
 347             '--display-name', vol_name, size]
 348     cinder_create = devstack_node.sh(args, wait=True)
 349     vol_info = parse_os_table(cinder_create)
 350     log.debug("Volume info: %s", str(vol_info))
 351
 352     try:
 353         rbd_output = ceph_node.sh("rbd --id cinder ls -l volumes", wait=True)
 354     except run.CommandFailedError:
 355         log.debug("Original rbd call failed; retrying without '--id cinder'")
 356         rbd_output = ceph_node.sh("rbd ls -l volumes", wait=True)
 357
 358     assert vol_info['id'] in rbd_output, \
 359         "Volume not found on Ceph cluster"
 360     assert vol_info['size'] == size, \
 361         "Volume size on Ceph cluster is different than specified"
 362     return vol_info['id']
 363
 364
 365 def parse_os_table(table_str):
 366     out_dict = dict()
 367     for line in table_str.split('\n'):
 368         if line.startswith('|'):
 369             items = line.split()
 370             out_dict[items[1]] = items[3]
 371     return out_dict