ceph/qa/workunits/restart/test-backtraces.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import print_function
   4
   5 import subprocess
   6 import json
   7 import os
   8 import time
   9 import sys
  10
  11 if sys.version_info[0] == 2:
  12     from cStringIO import StringIO
  13
  14     range = xrange
  15
  16 elif sys.version_info[0] == 3:
  17     from io import StringIO
  18
  19     range = range
  20
  21 import rados as rados
  22 import cephfs as cephfs
  23
  24 prefix='testbt'
  25
  26 def get_name(b, i, j):
  27     c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j)
  28     return c, b + '/' + c
  29
  30 def mkdir(ceph, d):
  31     print("mkdir {d}".format(d=d), file=sys.stderr)
  32     ceph.mkdir(d, 0o755)
  33     return ceph.stat(d)['st_ino']
  34
  35 def create(ceph, f):
  36     print("creating {f}".format(f=f), file=sys.stderr)
  37     fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644)
  38     ceph.close(fd)
  39     return ceph.stat(f)['st_ino']
  40
  41 def set_mds_config_param(ceph, param):
  42     with open('/dev/null', 'rb') as devnull:
  43         confarg = ''
  44         if conf != '':
  45             confarg = '-c {c}'.format(c=conf)
  46         r = subprocess.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull)
  47         if r != 0:
  48             raise Exception
  49
  50 import ConfigParser
  51 import contextlib
  52
  53 class _TrimIndentFile(object):
  54     def __init__(self, fp):
  55         self.fp = fp
  56
  57     def readline(self):
  58         line = self.fp.readline()
  59         return line.lstrip(' \t')
  60
  61 def _optionxform(s):
  62     s = s.replace('_', ' ')
  63     s = '_'.join(s.split())
  64     return s
  65
  66 def conf_set_kill_mds(location, killnum):
  67     print('setting mds kill config option for {l}.{k}'.format(l=location, k=killnum), file=sys.stderr)
  68     print("restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum))
  69     sys.stdout.flush()
  70     for l in sys.stdin.readline():
  71         if l == 'restarted':
  72             break
  73
  74 def flush(ceph, testnum):
  75     print('flushing {t}'.format(t=testnum), file=sys.stderr)
  76     set_mds_config_param(ceph, '--mds_log_max_segments 1')
  77
  78     for i in range(1, 500):
  79         f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i)
  80         print('flushing with create {f}'.format(f=f), file=sys.stderr)
  81         fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644)
  82         ceph.close(fd)
  83         ceph.unlink(f)
  84
  85     print('flush doing shutdown', file=sys.stderr)
  86     ceph.shutdown()
  87     print('flush reinitializing ceph', file=sys.stderr)
  88     ceph = cephfs.LibCephFS(conffile=conf)
  89     print('flush doing mount', file=sys.stderr)
  90     ceph.mount()
  91     return ceph
  92
  93 def kill_mds(ceph, location, killnum):
  94     print('killing mds: {l}.{k}'.format(l=location, k=killnum), file=sys.stderr)
  95     set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum))
  96
  97 def wait_for_mds(ceph):
  98     # wait for restart
  99     while True:
 100         confarg = ''
 101         if conf != '':
 102             confarg = '-c {c}'.format(c=conf)
 103         r = subprocess.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True).decode()
 104         if r.find('a=up:active'):
 105             break
 106         time.sleep(1)
 107
 108 def decode(value):
 109
 110     tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid())
 111     with open(tmpfile, 'w+') as f:
 112       f.write(value)
 113
 114     p = subprocess.Popen(
 115         [
 116             'ceph-dencoder',
 117             'import',
 118             tmpfile,
 119             'type',
 120             'inode_backtrace_t',
 121             'decode',
 122             'dump_json',
 123         ],
 124         stdin=subprocess.PIPE,
 125         stdout=subprocess.PIPE,
 126       )
 127     (stdout, _) = p.communicate(input=value)
 128     p.stdin.close()
 129     if p.returncode != 0:
 130         raise Exception
 131     os.remove(tmpfile)
 132     return json.loads(stdout)
 133
 134 class VerifyFailure(Exception):
 135     pass
 136
 137 def verify(rados_ioctx, ino, values, pool):
 138     print('getting parent attr for ino: %lx.00000000' % ino, file=sys.stderr)
 139     savede = None
 140     for i in range(1, 20):
 141         try:
 142             savede = None
 143             binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent')
 144         except rados.ObjectNotFound as e:
 145             # wait for a bit to let segments get flushed out
 146             savede = e
 147             time.sleep(10)
 148     if savede:
 149         raise savede
 150
 151     bt = decode(binbt)
 152
 153     if bt['ino'] != ino:
 154         raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
 155                     bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values))
 156     ind = 0
 157     for (n, i) in values:
 158         if bt['ancestors'][ind]['dirino'] != i:
 159             raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
 160                     b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values))
 161         if bt['ancestors'][ind]['dname'] != n:
 162             raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
 163                     b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values))
 164         ind += 1
 165
 166     if bt['pool'] != pool:
 167         raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
 168                     btp=bt['pool'], p=pool, bt=bt, i=ino, v=values))
 169
 170 def make_abc(ceph, rooti, i):
 171     expected_bt = []
 172     c, d = get_name("/", i, 0)
 173     expected_bt = [(c, rooti)] + expected_bt
 174     di = mkdir(ceph, d)
 175     c, d = get_name(d, i, 1)
 176     expected_bt = [(c, di)] + expected_bt
 177     di = mkdir(ceph, d)
 178     c, f = get_name(d, i, 2)
 179     fi = create(ceph, f)
 180     expected_bt = [(c, di)] + expected_bt
 181     return fi, expected_bt
 182
 183 test = -1
 184 if len(sys.argv) > 1:
 185     test = int(sys.argv[1])
 186
 187 conf = ''
 188 if len(sys.argv) > 2:
 189     conf = sys.argv[2]
 190
 191 radosobj = rados.Rados(conffile=conf)
 192 radosobj.connect()
 193 ioctx = radosobj.open_ioctx('data')
 194
 195 ceph = cephfs.LibCephFS(conffile=conf)
 196 ceph.mount()
 197
 198 rooti = ceph.stat('/')['st_ino']
 199
 200 test = -1
 201 if len(sys.argv) > 1:
 202     test = int(sys.argv[1])
 203
 204 conf = '/etc/ceph/ceph.conf'
 205 if len(sys.argv) > 2:
 206     conf = sys.argv[2]
 207
 208 # create /a/b/c
 209 # flush
 210 # verify
 211
 212 i = 0
 213 if test < 0 or test == i:
 214   print('Running test %d: basic verify' % i, file=sys.stderr)
 215   ino, expected_bt = make_abc(ceph, rooti, i)
 216   ceph = flush(ceph, i)
 217   verify(ioctx, ino, expected_bt, 0)
 218
 219 i += 1
 220
 221 # kill-mds-at-openc-1
 222 # create /a/b/c
 223 # restart-mds
 224 # flush
 225 # verify
 226
 227 if test < 0 or test == i:
 228   print('Running test %d: kill openc' % i, file=sys.stderr)
 229   print("restart mds a")
 230   sys.stdout.flush()
 231   kill_mds(ceph, 'openc', 1)
 232   ino, expected_bt = make_abc(ceph, rooti, i)
 233   ceph = flush(ceph, i)
 234   verify(ioctx, ino, expected_bt, 0)
 235
 236 i += 1
 237
 238 # kill-mds-at-openc-1
 239 # create /a/b/c
 240 # restart-mds with kill-mds-at-replay-1
 241 # restart-mds
 242 # flush
 243 # verify
 244 if test < 0 or test == i:
 245   print('Running test %d: kill openc/replay' % i, file=sys.stderr)
 246   # these are reversed because we want to prepare the config
 247   conf_set_kill_mds('journal_replay', 1)
 248   kill_mds(ceph, 'openc', 1)
 249   print("restart mds a")
 250   sys.stdout.flush()
 251   ino, expected_bt = make_abc(ceph, rooti, i)
 252   ceph = flush(ceph, i)
 253   verify(ioctx, ino, expected_bt, 0)
 254
 255 i += 1
 256
 257 ioctx.close()
 258 radosobj.shutdown()
 259 ceph.shutdown()
 260
 261 print("done")
 262 sys.stdout.flush()