ceph/qa/tasks/restart.py

   1 """
   2 Daemon restart
   3 """
   4 import logging
   5 import pipes
   6
   7 from teuthology import misc as teuthology
   8 from teuthology.orchestra import run as tor
   9
  10 from teuthology.orchestra import run
  11 log = logging.getLogger(__name__)
  12
  13 def restart_daemon(ctx, config, role, id_, *args):
  14     """
  15     Handle restart (including the execution of the command parameters passed)
  16     """
  17     log.info('Restarting {r}.{i} daemon...'.format(r=role, i=id_))
  18     daemon = ctx.daemons.get_daemon(role, id_)
  19     log.debug('Waiting for exit of {r}.{i} daemon...'.format(r=role, i=id_))
  20     try:
  21         daemon.wait_for_exit()
  22     except tor.CommandFailedError as e:
  23         log.debug('Command Failed: {e}'.format(e=e))
  24     if len(args) > 0:
  25         confargs = ['--{k}={v}'.format(k=k, v=v) for k,v in zip(args[0::2], args[1::2])]
  26         log.debug('Doing restart of {r}.{i} daemon with args: {a}...'.format(r=role, i=id_, a=confargs))
  27         daemon.restart_with_args(confargs)
  28     else:
  29         log.debug('Doing restart of {r}.{i} daemon...'.format(r=role, i=id_))
  30         daemon.restart()
  31
  32 def get_tests(ctx, config, role, remote, testdir):
  33     """Download restart tests"""
  34     srcdir = '{tdir}/restart.{role}'.format(tdir=testdir, role=role)
  35
  36     refspec = config.get('branch')
  37     if refspec is None:
  38         refspec = config.get('sha1')
  39     if refspec is None:
  40         refspec = config.get('tag')
  41     if refspec is None:
  42         refspec = 'HEAD'
  43     log.info('Pulling restart qa/workunits from ref %s', refspec)
  44
  45     remote.run(
  46         logger=log.getChild(role),
  47         args=[
  48             'mkdir', '--', srcdir,
  49             run.Raw('&&'),
  50             'git',
  51             'archive',
  52             '--remote=git://git.ceph.com/ceph.git',
  53             '%s:qa/workunits' % refspec,
  54             run.Raw('|'),
  55             'tar',
  56             '-C', srcdir,
  57             '-x',
  58             '-f-',
  59             run.Raw('&&'),
  60             'cd', '--', srcdir,
  61             run.Raw('&&'),
  62             'if', 'test', '-e', 'Makefile', run.Raw(';'), 'then', 'make', run.Raw(';'), 'fi',
  63             run.Raw('&&'),
  64             'find', '-executable', '-type', 'f', '-printf', r'%P\0',
  65             run.Raw('>{tdir}/restarts.list'.format(tdir=testdir)),
  66             ],
  67         )
  68     restarts = sorted(remote.read_file(f'{testdir}/restarts.list').decode().split('\0'))
  69     return (srcdir, restarts)
  70
  71 def task(ctx, config):
  72     """
  73     Execute commands and allow daemon restart with config options.
  74     Each process executed can output to stdout restart commands of the form:
  75         restart <role> <id> <conf_key1> <conf_value1> <conf_key2> <conf_value2>
  76     This will restart the daemon <role>.<id> with the specified config values once
  77     by modifying the conf file with those values, and then replacing the old conf file
  78     once the daemon is restarted.
  79     This task does not kill a running daemon, it assumes the daemon will abort on an
  80     assert specified in the config.
  81
  82         tasks:
  83         - install:
  84         - ceph:
  85         - restart:
  86             exec:
  87               client.0:
  88                 - test_backtraces.py
  89
  90     """
  91     assert isinstance(config, dict), "task kill got invalid config"
  92
  93     testdir = teuthology.get_testdir(ctx)
  94
  95     try:
  96         assert 'exec' in config, "config requires exec key with <role>: <command> entries"
  97         for role, task in config['exec'].items():
  98             log.info('restart for role {r}'.format(r=role))
  99             (remote,) = ctx.cluster.only(role).remotes.keys()
 100             srcdir, restarts = get_tests(ctx, config, role, remote, testdir)
 101             log.info('Running command on role %s host %s', role, remote.name)
 102             spec = '{spec}'.format(spec=task[0])
 103             log.info('Restarts list: %s', restarts)
 104             log.info('Spec is %s', spec)
 105             to_run = [w for w in restarts if w == task or w.find(spec) != -1]
 106             log.info('To run: %s', to_run)
 107             for c in to_run:
 108                 log.info('Running restart script %s...', c)
 109                 args = [
 110                     run.Raw('TESTDIR="{tdir}"'.format(tdir=testdir)),
 111                     ]
 112                 env = config.get('env')
 113                 if env is not None:
 114                     for var, val in env.items():
 115                         quoted_val = pipes.quote(val)
 116                         env_arg = '{var}={val}'.format(var=var, val=quoted_val)
 117                         args.append(run.Raw(env_arg))
 118                 args.extend([
 119                             'adjust-ulimits',
 120                             'ceph-coverage',
 121                             '{tdir}/archive/coverage'.format(tdir=testdir),
 122                             '{srcdir}/{c}'.format(
 123                                 srcdir=srcdir,
 124                                 c=c,
 125                                 ),
 126                             ])
 127                 proc = remote.run(
 128                     args=args,
 129                     stdout=tor.PIPE,
 130                     stdin=tor.PIPE,
 131                     stderr=log,
 132                     wait=False,
 133                     )
 134                 log.info('waiting for a command from script')
 135                 while True:
 136                     l = proc.stdout.readline()
 137                     if not l or l == '':
 138                         break
 139                     log.debug('script command: {c}'.format(c=l))
 140                     ll = l.strip()
 141                     cmd = ll.split(' ')
 142                     if cmd[0] == "done":
 143                         break
 144                     assert cmd[0] == 'restart', "script sent invalid command request to kill task"
 145                     # cmd should be: restart <role> <id> <conf_key1> <conf_value1> <conf_key2> <conf_value2>
 146                     # or to clear, just: restart <role> <id>
 147                     restart_daemon(ctx, config, cmd[1], cmd[2], *cmd[3:])
 148                     proc.stdin.writelines(['restarted\n'])
 149                     proc.stdin.flush()
 150                 try:
 151                     proc.wait()
 152                 except tor.CommandFailedError:
 153                     raise Exception('restart task got non-zero exit status from script: {s}'.format(s=c))
 154     finally:
 155         log.info('Finishing %s on %s...', task, role)
 156         remote.run(
 157             logger=log.getChild(role),
 158             args=[
 159                 'rm', '-rf', '--', '{tdir}/restarts.list'.format(tdir=testdir), srcdir,
 160                 ],
 161             )