]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/usr/bin/env python |
2 | ||
3 | from __future__ import print_function | |
4 | ||
5 | import subprocess | |
6 | import json | |
7 | import os | |
8 | import time | |
9 | import sys | |
10 | ||
11 | if sys.version_info[0] == 2: | |
12 | from cStringIO import StringIO | |
13 | ||
14 | range = xrange | |
15 | ||
16 | elif sys.version_info[0] == 3: | |
17 | from io import StringIO | |
18 | ||
19 | range = range | |
20 | ||
21 | import rados as rados | |
22 | import cephfs as cephfs | |
23 | ||
24 | prefix='testbt' | |
25 | ||
26 | def get_name(b, i, j): | |
27 | c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j) | |
28 | return c, b + '/' + c | |
29 | ||
30 | def mkdir(ceph, d): | |
31 | print("mkdir {d}".format(d=d), file=sys.stderr) | |
32 | ceph.mkdir(d, 0o755) | |
33 | return ceph.stat(d)['st_ino'] | |
34 | ||
35 | def create(ceph, f): | |
36 | print("creating {f}".format(f=f), file=sys.stderr) | |
37 | fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644) | |
38 | ceph.close(fd) | |
39 | return ceph.stat(f)['st_ino'] | |
40 | ||
41 | def set_mds_config_param(ceph, param): | |
42 | with open('/dev/null', 'rb') as devnull: | |
43 | confarg = '' | |
44 | if conf != '': | |
45 | confarg = '-c {c}'.format(c=conf) | |
46 | r = subprocess.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull) | |
47 | if r != 0: | |
48 | raise Exception | |
49 | ||
50 | import ConfigParser | |
51 | import contextlib | |
52 | ||
53 | class _TrimIndentFile(object): | |
54 | def __init__(self, fp): | |
55 | self.fp = fp | |
56 | ||
57 | def readline(self): | |
58 | line = self.fp.readline() | |
59 | return line.lstrip(' \t') | |
60 | ||
61 | def _optionxform(s): | |
62 | s = s.replace('_', ' ') | |
63 | s = '_'.join(s.split()) | |
64 | return s | |
65 | ||
66 | def conf_set_kill_mds(location, killnum): | |
67 | print('setting mds kill config option for {l}.{k}'.format(l=location, k=killnum), file=sys.stderr) | |
68 | print("restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum)) | |
69 | sys.stdout.flush() | |
70 | for l in sys.stdin.readline(): | |
71 | if l == 'restarted': | |
72 | break | |
73 | ||
74 | def flush(ceph, testnum): | |
75 | print('flushing {t}'.format(t=testnum), file=sys.stderr) | |
76 | set_mds_config_param(ceph, '--mds_log_max_segments 1') | |
77 | ||
78 | for i in range(1, 500): | |
79 | f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i) | |
80 | print('flushing with create {f}'.format(f=f), file=sys.stderr) | |
81 | fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644) | |
82 | ceph.close(fd) | |
83 | ceph.unlink(f) | |
84 | ||
85 | print('flush doing shutdown', file=sys.stderr) | |
86 | ceph.shutdown() | |
87 | print('flush reinitializing ceph', file=sys.stderr) | |
88 | ceph = cephfs.LibCephFS(conffile=conf) | |
89 | print('flush doing mount', file=sys.stderr) | |
90 | ceph.mount() | |
91 | return ceph | |
92 | ||
93 | def kill_mds(ceph, location, killnum): | |
94 | print('killing mds: {l}.{k}'.format(l=location, k=killnum), file=sys.stderr) | |
95 | set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum)) | |
96 | ||
97 | def wait_for_mds(ceph): | |
98 | # wait for restart | |
99 | while True: | |
100 | confarg = '' | |
101 | if conf != '': | |
102 | confarg = '-c {c}'.format(c=conf) | |
103 | r = subprocess.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True).decode() | |
104 | if r.find('a=up:active'): | |
105 | break | |
106 | time.sleep(1) | |
107 | ||
108 | def decode(value): | |
109 | ||
110 | tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid()) | |
111 | with open(tmpfile, 'w+') as f: | |
112 | f.write(value) | |
113 | ||
114 | p = subprocess.Popen( | |
115 | [ | |
116 | 'ceph-dencoder', | |
117 | 'import', | |
118 | tmpfile, | |
119 | 'type', | |
120 | 'inode_backtrace_t', | |
121 | 'decode', | |
122 | 'dump_json', | |
123 | ], | |
124 | stdin=subprocess.PIPE, | |
125 | stdout=subprocess.PIPE, | |
126 | ) | |
127 | (stdout, _) = p.communicate(input=value) | |
128 | p.stdin.close() | |
129 | if p.returncode != 0: | |
130 | raise Exception | |
131 | os.remove(tmpfile) | |
132 | return json.loads(stdout) | |
133 | ||
134 | class VerifyFailure(Exception): | |
135 | pass | |
136 | ||
137 | def verify(rados_ioctx, ino, values, pool): | |
138 | print('getting parent attr for ino: %lx.00000000' % ino, file=sys.stderr) | |
139 | savede = None | |
140 | for i in range(1, 20): | |
141 | try: | |
142 | savede = None | |
143 | binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent') | |
144 | except rados.ObjectNotFound as e: | |
145 | # wait for a bit to let segments get flushed out | |
146 | savede = e | |
147 | time.sleep(10) | |
148 | if savede: | |
149 | raise savede | |
150 | ||
151 | bt = decode(binbt) | |
152 | ||
153 | if bt['ino'] != ino: | |
154 | raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
155 | bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values)) | |
156 | ind = 0 | |
157 | for (n, i) in values: | |
158 | if bt['ancestors'][ind]['dirino'] != i: | |
159 | raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
160 | b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values)) | |
161 | if bt['ancestors'][ind]['dname'] != n: | |
162 | raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
163 | b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values)) | |
164 | ind += 1 | |
165 | ||
166 | if bt['pool'] != pool: | |
167 | raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
168 | btp=bt['pool'], p=pool, bt=bt, i=ino, v=values)) | |
169 | ||
170 | def make_abc(ceph, rooti, i): | |
171 | expected_bt = [] | |
172 | c, d = get_name("/", i, 0) | |
173 | expected_bt = [(c, rooti)] + expected_bt | |
174 | di = mkdir(ceph, d) | |
175 | c, d = get_name(d, i, 1) | |
176 | expected_bt = [(c, di)] + expected_bt | |
177 | di = mkdir(ceph, d) | |
178 | c, f = get_name(d, i, 2) | |
179 | fi = create(ceph, f) | |
180 | expected_bt = [(c, di)] + expected_bt | |
181 | return fi, expected_bt | |
182 | ||
183 | test = -1 | |
184 | if len(sys.argv) > 1: | |
185 | test = int(sys.argv[1]) | |
186 | ||
187 | conf = '' | |
188 | if len(sys.argv) > 2: | |
189 | conf = sys.argv[2] | |
190 | ||
191 | radosobj = rados.Rados(conffile=conf) | |
192 | radosobj.connect() | |
193 | ioctx = radosobj.open_ioctx('data') | |
194 | ||
195 | ceph = cephfs.LibCephFS(conffile=conf) | |
196 | ceph.mount() | |
197 | ||
198 | rooti = ceph.stat('/')['st_ino'] | |
199 | ||
200 | test = -1 | |
201 | if len(sys.argv) > 1: | |
202 | test = int(sys.argv[1]) | |
203 | ||
204 | conf = '/etc/ceph/ceph.conf' | |
205 | if len(sys.argv) > 2: | |
206 | conf = sys.argv[2] | |
207 | ||
208 | # create /a/b/c | |
209 | # flush | |
210 | # verify | |
211 | ||
212 | i = 0 | |
213 | if test < 0 or test == i: | |
214 | print('Running test %d: basic verify' % i, file=sys.stderr) | |
215 | ino, expected_bt = make_abc(ceph, rooti, i) | |
216 | ceph = flush(ceph, i) | |
217 | verify(ioctx, ino, expected_bt, 0) | |
218 | ||
219 | i += 1 | |
220 | ||
221 | # kill-mds-at-openc-1 | |
222 | # create /a/b/c | |
223 | # restart-mds | |
224 | # flush | |
225 | # verify | |
226 | ||
227 | if test < 0 or test == i: | |
228 | print('Running test %d: kill openc' % i, file=sys.stderr) | |
229 | print("restart mds a") | |
230 | sys.stdout.flush() | |
231 | kill_mds(ceph, 'openc', 1) | |
232 | ino, expected_bt = make_abc(ceph, rooti, i) | |
233 | ceph = flush(ceph, i) | |
234 | verify(ioctx, ino, expected_bt, 0) | |
235 | ||
236 | i += 1 | |
237 | ||
238 | # kill-mds-at-openc-1 | |
239 | # create /a/b/c | |
240 | # restart-mds with kill-mds-at-replay-1 | |
241 | # restart-mds | |
242 | # flush | |
243 | # verify | |
244 | if test < 0 or test == i: | |
245 | print('Running test %d: kill openc/replay' % i, file=sys.stderr) | |
246 | # these are reversed because we want to prepare the config | |
247 | conf_set_kill_mds('journal_replay', 1) | |
248 | kill_mds(ceph, 'openc', 1) | |
249 | print("restart mds a") | |
250 | sys.stdout.flush() | |
251 | ino, expected_bt = make_abc(ceph, rooti, i) | |
252 | ceph = flush(ceph, i) | |
253 | verify(ioctx, ino, expected_bt, 0) | |
254 | ||
255 | i += 1 | |
256 | ||
257 | ioctx.close() | |
258 | radosobj.shutdown() | |
259 | ceph.shutdown() | |
260 | ||
261 | print("done") | |
262 | sys.stdout.flush() |