ceph/qa/tasks/calamari_setup.py

   1 """
   2 Calamari setup task
   3 """
   4 import contextlib
   5 import logging
   6 import os
   7 import requests
   8 import shutil
   9 import webbrowser
  10
  11 from cStringIO import StringIO
  12 from teuthology.orchestra import run
  13 from teuthology import contextutil
  14 from teuthology import misc
  15
  16 log = logging.getLogger(__name__)
  17
  18
  19 DEFAULTS = {
  20     'version': 'v0.80.9',
  21     'test_image': None,
  22     'start_browser': False,
  23     'email': 'x@y.com',
  24     'no_epel': True,
  25     'calamari_user': 'admin',
  26     'calamari_password': 'admin',
  27 }
  28
  29
  30 @contextlib.contextmanager
  31 def task(ctx, config):
  32     """
  33     Do the setup of a calamari server.
  34
  35     - calamari_setup:
  36         version: 'v80.1'
  37         test_image: <path to tarball or iso>
  38
  39     Options are (see DEFAULTS above):
  40
  41     version -- ceph version we are testing against
  42     test_image -- Can be an HTTP URL, in which case fetch from this
  43                   http path; can also be local path
  44     start_browser -- If True, start a browser.  To be used by runs that will
  45                      bring up a browser quickly for human use.  Set to False
  46                      for overnight suites that are testing for problems in
  47                      the installation itself
  48     email -- email address for the user
  49     no_epel -- indicates if we should remove epel files prior to yum
  50                installations.
  51     calamari_user -- user name to log into gui
  52     calamari_password -- calamari user password
  53     """
  54     local_config = DEFAULTS
  55     local_config.update(config)
  56     config = local_config
  57     cal_svr = None
  58     for remote_, roles in ctx.cluster.remotes.items():
  59         if 'client.0' in roles:
  60             cal_svr = remote_
  61             break
  62     if not cal_svr:
  63         raise RuntimeError('client.0 not found in roles')
  64     with contextutil.nested(
  65         lambda: adjust_yum_repos(ctx, cal_svr, config['no_epel']),
  66         lambda: calamari_install(config, cal_svr),
  67         lambda: ceph_install(ctx, cal_svr),
  68         # do it again because ceph-deploy installed epel for centos
  69         lambda: remove_epel(ctx, config['no_epel']),
  70         lambda: calamari_connect(ctx, cal_svr),
  71         lambda: browser(config['start_browser'], cal_svr.hostname),
  72     ):
  73         yield
  74
  75
  76 @contextlib.contextmanager
  77 def adjust_yum_repos(ctx, cal_svr, no_epel):
  78     """
  79     For each remote machine, fix the repos if yum is used.
  80     """
  81     ice_distro = str(cal_svr.os)
  82     if ice_distro.startswith('rhel') or ice_distro.startswith('centos'):
  83         if no_epel:
  84             for remote in ctx.cluster.remotes:
  85                 fix_yum_repos(remote, ice_distro)
  86     try:
  87         yield
  88     finally:
  89         if ice_distro.startswith('rhel') or ice_distro.startswith('centos'):
  90             if no_epel:
  91                 for remote in ctx.cluster.remotes:
  92                     restore_yum_repos(remote)
  93
  94
  95 def restore_yum_repos(remote):
  96     """
  97     Copy the old saved repo back in.
  98     """
  99     if remote.run(args=['sudo', 'rm', '-rf', '/etc/yum.repos.d']).exitstatus:
 100         return False
 101     if remote.run(args=['sudo', 'mv', '/etc/yum.repos.d.old',
 102                         '/etc/yum.repos.d']).exitstatus:
 103         return False
 104
 105
 106 def fix_yum_repos(remote, distro):
 107     """
 108     For yum calamari installations, the repos.d directory should only
 109     contain a repo file named rhel<version-number>.repo
 110     """
 111     if distro.startswith('centos'):
 112         # hack alert: detour: install lttng for ceph
 113         # this works because epel is preinstalled on the vpms
 114         # this is not a generic solution
 115         # this is here solely to test the one-off 1.3.0 release for centos6
 116         remote.run(args="sudo yum -y install lttng-tools")
 117         cmds = [
 118             'sudo mkdir /etc/yum.repos.d.old'.split(),
 119             ['sudo', 'cp', run.Raw('/etc/yum.repos.d/*'),
 120              '/etc/yum.repos.d.old'],
 121             ['sudo', 'rm', run.Raw('/etc/yum.repos.d/epel*')],
 122         ]
 123         for cmd in cmds:
 124             if remote.run(args=cmd).exitstatus:
 125                 return False
 126     else:
 127         cmds = [
 128             'sudo mv /etc/yum.repos.d /etc/yum.repos.d.old'.split(),
 129             'sudo mkdir /etc/yum.repos.d'.split(),
 130         ]
 131         for cmd in cmds:
 132             if remote.run(args=cmd).exitstatus:
 133                 return False
 134
 135         # map "distroversion" from Remote.os to a tuple of
 136         # (repo title, repo name descriptor, apt-mirror repo path chunk)
 137         yum_repo_params = {
 138             'rhel 6.4': ('rhel6-server', 'RHEL', 'rhel6repo-server'),
 139             'rhel 6.5': ('rhel6-server', 'RHEL', 'rhel6repo-server'),
 140             'rhel 7.0': ('rhel7-server', 'RHEL', 'rhel7repo/server'),
 141         }
 142         repotitle, reponame, path = yum_repo_params[distro]
 143         repopath = '/etc/yum.repos.d/%s.repo' % repotitle
 144         # TO DO:  Make this data configurable too
 145         repo_contents = '\n'.join(
 146             ('[%s]' % repotitle,
 147              'name=%s $releasever - $basearch' % reponame,
 148              'baseurl=http://apt-mirror.front.sepia.ceph.com/' + path,
 149              'gpgcheck=0',
 150              'enabled=1')
 151         )
 152         misc.sudo_write_file(remote, repopath, repo_contents)
 153     cmds = [
 154         'sudo yum clean all'.split(),
 155         'sudo yum makecache'.split(),
 156     ]
 157     for cmd in cmds:
 158         if remote.run(args=cmd).exitstatus:
 159             return False
 160     return True
 161
 162
 163 @contextlib.contextmanager
 164 def remove_epel(ctx, no_epel):
 165     """
 166     just remove epel.  No undo; assumed that it's used after
 167     adjust_yum_repos, and relies on its state-save/restore.
 168     """
 169     if no_epel:
 170         for remote in ctx.cluster.remotes:
 171             if remote.os.name.startswith('centos'):
 172                 remote.run(args=[
 173                     'sudo', 'rm', '-f', run.Raw('/etc/yum.repos.d/epel*')
 174                 ])
 175     try:
 176         yield
 177     finally:
 178         pass
 179
 180
 181 def get_iceball_with_http(url, destdir):
 182     '''
 183     Copy iceball with http to destdir.  Try both .tar.gz and .iso.
 184     '''
 185     # stream=True means we don't download until copyfileobj below,
 186     # and don't need a temp file
 187     r = requests.get(url, stream=True)
 188     if not r.ok:
 189         raise RuntimeError("Failed to download %s", str(url))
 190     filename = os.path.join(destdir, url.split('/')[-1])
 191     with open(filename, 'w') as f:
 192         shutil.copyfileobj(r.raw, f)
 193     log.info('saved %s as %s' % (url, filename))
 194     return filename
 195
 196
 197 @contextlib.contextmanager
 198 def calamari_install(config, cal_svr):
 199     """
 200     Install calamari
 201
 202     The steps here are:
 203         -- Get the iceball, locally or from http
 204         -- Copy the iceball to the calamari server, and untar/mount it.
 205         -- Run ice-setup on the calamari server.
 206         -- Run calamari-ctl initialize.
 207     """
 208     client_id = str(cal_svr)
 209     at_loc = client_id.find('@')
 210     if at_loc > 0:
 211         client_id = client_id[at_loc + 1:]
 212
 213     test_image = config['test_image']
 214
 215     if not test_image:
 216         raise RuntimeError('Must supply test image')
 217     log.info('calamari test image: %s' % test_image)
 218     delete_iceball = False
 219
 220     if test_image.startswith('http'):
 221         iceball_file = get_iceball_with_http(test_image, '/tmp')
 222         delete_iceball = True
 223     else:
 224         iceball_file = test_image
 225
 226     remote_iceball_file = os.path.join('/tmp', os.path.split(iceball_file)[1])
 227     cal_svr.put_file(iceball_file, remote_iceball_file)
 228     if iceball_file.endswith('.tar.gz'):   # XXX specify tar/iso in config?
 229         icetype = 'tarball'
 230     elif iceball_file.endswith('.iso'):
 231         icetype = 'iso'
 232     else:
 233         raise RuntimeError('Can''t handle iceball {0}'.format(iceball_file))
 234
 235     if icetype == 'tarball':
 236         ret = cal_svr.run(args=['gunzip', run.Raw('<'), remote_iceball_file,
 237                           run.Raw('|'), 'tar', 'xvf', run.Raw('-')])
 238         if ret.exitstatus:
 239             raise RuntimeError('remote iceball untar failed')
 240     elif icetype == 'iso':
 241         mountpoint = '/mnt/'   # XXX create?
 242         ret = cal_svr.run(
 243             args=['sudo', 'mount', '-o', 'loop', '-r',
 244                   remote_iceball_file, mountpoint]
 245         )
 246
 247     # install ice_setup package
 248     args = {
 249         'deb': 'sudo dpkg -i /mnt/ice-setup*deb',
 250         'rpm': 'sudo yum -y localinstall /mnt/ice_setup*rpm'
 251     }.get(cal_svr.system_type, None)
 252     if not args:
 253         raise RuntimeError('{0}: unknown system type'.format(cal_svr))
 254     ret = cal_svr.run(args=args)
 255     if ret.exitstatus:
 256         raise RuntimeError('ice_setup package install failed')
 257
 258     # Run ice_setup
 259     icesetdata = 'yes\n\n%s\nhttp\n' % client_id
 260     ice_in = StringIO(icesetdata)
 261     ice_out = StringIO()
 262     if icetype == 'tarball':
 263         args = 'sudo python ice_setup.py'
 264     else:
 265         args = 'sudo ice_setup -d /mnt'
 266     ret = cal_svr.run(args=args, stdin=ice_in, stdout=ice_out)
 267     log.debug(ice_out.getvalue())
 268     if ret.exitstatus:
 269         raise RuntimeError('ice_setup failed')
 270
 271     # Run calamari-ctl initialize.
 272     icesetdata = '%s\n%s\n%s\n%s\n' % (
 273         config['calamari_user'],
 274         config['email'],
 275         config['calamari_password'],
 276         config['calamari_password'],
 277     )
 278     ice_in = StringIO(icesetdata)
 279     ret = cal_svr.run(args=['sudo', 'calamari-ctl', 'initialize'],
 280                       stdin=ice_in, stdout=ice_out)
 281     log.debug(ice_out.getvalue())
 282     if ret.exitstatus:
 283         raise RuntimeError('calamari-ctl initialize failed')
 284     try:
 285         yield
 286     finally:
 287         log.info('Cleaning up after Calamari installation')
 288         if icetype == 'iso':
 289             cal_svr.run(args=['sudo', 'umount', mountpoint])
 290         if delete_iceball:
 291             os.unlink(iceball_file)
 292
 293
 294 @contextlib.contextmanager
 295 def ceph_install(ctx, cal_svr):
 296     """
 297     Install ceph if ceph was not previously installed by teuthology.  This
 298     code tests the case where calamari is installed on a brand new system.
 299     """
 300     loc_inst = False
 301     if 'install' not in [x.keys()[0] for x in ctx.config['tasks']]:
 302         loc_inst = True
 303         ret = deploy_ceph(ctx, cal_svr)
 304         if ret:
 305             raise RuntimeError('ceph installs failed')
 306     try:
 307         yield
 308     finally:
 309         if loc_inst:
 310             if not undeploy_ceph(ctx, cal_svr):
 311                 log.error('Cleanup of Ceph installed by Calamari-setup failed')
 312
 313
 314 def deploy_ceph(ctx, cal_svr):
 315     """
 316     Perform the ceph-deploy actions needed to bring up a Ceph cluster.  This
 317     test is needed to check the ceph-deploy that comes with the calamari
 318     package.
 319     """
 320     osd_to_name = {}
 321     all_machines = set()
 322     all_mons = set()
 323     all_osds = set()
 324
 325     # collect which remotes are osds and which are mons
 326     for remote in ctx.cluster.remotes:
 327         all_machines.add(remote.shortname)
 328         roles = ctx.cluster.remotes[remote]
 329         for role in roles:
 330             daemon_type, number = role.split('.')
 331             if daemon_type == 'osd':
 332                 all_osds.add(remote.shortname)
 333                 osd_to_name[number] = remote.shortname
 334             if daemon_type == 'mon':
 335                 all_mons.add(remote.shortname)
 336
 337     # figure out whether we're in "1.3+" mode: prior to 1.3, there was
 338     # only one Ceph repo, and it was all installed on every Ceph host.
 339     # with 1.3, we've split that into MON and OSD repos (in order to
 340     # be able to separately track subscriptions per-node).  This
 341     # requires new switches to ceph-deploy to select which locally-served
 342     # repo is connected to which cluster host.
 343     #
 344     # (TODO: A further issue is that the installation/setup may not have
 345     # created local repos at all, but that is the subject of a future
 346     # change.)
 347
 348     r = cal_svr.run(args='/usr/bin/test -d /mnt/MON', check_status=False)
 349     use_install_repo = (r.returncode == 0)
 350
 351     # pre-1.3:
 352     # ceph-deploy new <all_mons>
 353     # ceph-deploy install <all_machines>
 354     # ceph-deploy mon create-initial
 355     #
 356     # 1.3 and later:
 357     # ceph-deploy new <all_mons>
 358     # ceph-deploy install --repo --release=ceph-mon <all_mons>
 359     # ceph-deploy install <all_mons>
 360     # ceph-deploy install --repo --release=ceph-osd <all_osds>
 361     # ceph-deploy install <all_osds>
 362     # ceph-deploy mon create-initial
 363     #
 364     # one might think the install <all_mons> and install <all_osds>
 365     # commands would need --mon and --osd, but #12147 has not yet
 366     # made it into RHCS 1.3.0; since the package split also hasn't
 367     # landed, we can avoid using the flag and avoid the bug.
 368
 369     cmds = ['ceph-deploy new ' + ' '.join(all_mons)]
 370
 371     if use_install_repo:
 372         cmds.append('ceph-deploy repo ceph-mon ' +
 373                     ' '.join(all_mons))
 374         cmds.append('ceph-deploy install --no-adjust-repos --mon ' +
 375                     ' '.join(all_mons))
 376         cmds.append('ceph-deploy repo ceph-osd ' +
 377                     ' '.join(all_osds))
 378         cmds.append('ceph-deploy install --no-adjust-repos --osd ' +
 379                     ' '.join(all_osds))
 380         # We tell users to use `hostname` in our docs. Do the same here.
 381         cmds.append('ceph-deploy install --no-adjust-repos --cli `hostname`')
 382     else:
 383         cmds.append('ceph-deploy install ' + ' '.join(all_machines))
 384
 385     cmds.append('ceph-deploy mon create-initial')
 386
 387     for cmd in cmds:
 388         cal_svr.run(args=cmd).exitstatus
 389
 390     disk_labels = '_dcba'
 391     # NEEDS WORK assumes disks start with vd (need to check this somewhere)
 392     for cmd_pts in [['disk', 'zap'], ['osd', 'prepare'], ['osd', 'activate']]:
 393         mach_osd_cnt = {}
 394         for osdn in osd_to_name:
 395             osd_mac = osd_to_name[osdn]
 396             mach_osd_cnt[osd_mac] = mach_osd_cnt.get(osd_mac, 0) + 1
 397             arg_list = ['ceph-deploy']
 398             arg_list.extend(cmd_pts)
 399             disk_id = '%s:vd%s' % (osd_to_name[osdn],
 400                                    disk_labels[mach_osd_cnt[osd_mac]])
 401             if 'activate' in cmd_pts:
 402                 disk_id += '1'
 403             arg_list.append(disk_id)
 404             cal_svr.run(args=arg_list).exitstatus
 405
 406
 407 def undeploy_ceph(ctx, cal_svr):
 408     """
 409     Cleanup deployment of ceph.
 410     """
 411     all_machines = []
 412     ret = True
 413     for remote in ctx.cluster.remotes:
 414         roles = ctx.cluster.remotes[remote]
 415         if (
 416             not any('osd' in role for role in roles) and
 417             not any('mon' in role for role in roles)
 418         ):
 419             continue
 420         ret &= remote.run(
 421             args=['sudo', 'stop', 'ceph-all', run.Raw('||'),
 422                   'sudo', 'service', 'ceph', 'stop']
 423         ).exitstatus
 424         all_machines.append(remote.shortname)
 425     all_machines = set(all_machines)
 426     cmd1 = ['ceph-deploy', 'uninstall']
 427     cmd1.extend(all_machines)
 428     ret &= cal_svr.run(args=cmd1).exitstatus
 429     cmd2 = ['ceph-deploy', 'purge']
 430     cmd2.extend(all_machines)
 431     ret &= cal_svr.run(args=cmd2).exitstatus
 432     for remote in ctx.cluster.remotes:
 433         ret &= remote.run(args=['sudo', 'rm', '-rf',
 434                                 '.ssh/known_hosts']).exitstatus
 435     return ret
 436
 437
 438 @contextlib.contextmanager
 439 def calamari_connect(ctx, cal_svr):
 440     """
 441     Connect calamari to the ceph nodes.
 442     """
 443     connects = ['ceph-deploy', 'calamari', 'connect']
 444     for machine_info in ctx.cluster.remotes:
 445         if 'client.0' not in ctx.cluster.remotes[machine_info]:
 446             connects.append(machine_info.shortname)
 447     ret = cal_svr.run(args=connects)
 448     if ret.exitstatus:
 449         raise RuntimeError('calamari connect failed')
 450     try:
 451         yield
 452     finally:
 453         log.info('Calamari test terminating')
 454
 455
 456 @contextlib.contextmanager
 457 def browser(start_browser, web_page):
 458     """
 459     Bring up a browser, if wanted.
 460     """
 461     if start_browser:
 462         webbrowser.open('http://%s' % web_page)
 463     try:
 464         yield
 465     finally:
 466         if start_browser:
 467             log.info('Web browser support terminating')