]>
Commit | Line | Data |
---|---|---|
9f95a23c | 1 | #!/usr/bin/env python3 |
7c673cae FG |
2 | |
3 | from __future__ import print_function | |
4 | ||
5 | import subprocess | |
6 | import json | |
7 | import os | |
8 | import time | |
9 | import sys | |
10 | ||
7c673cae FG |
11 | import rados as rados |
12 | import cephfs as cephfs | |
13 | ||
14 | prefix='testbt' | |
15 | ||
16 | def get_name(b, i, j): | |
17 | c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j) | |
18 | return c, b + '/' + c | |
19 | ||
20 | def mkdir(ceph, d): | |
21 | print("mkdir {d}".format(d=d), file=sys.stderr) | |
22 | ceph.mkdir(d, 0o755) | |
23 | return ceph.stat(d)['st_ino'] | |
24 | ||
25 | def create(ceph, f): | |
26 | print("creating {f}".format(f=f), file=sys.stderr) | |
27 | fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644) | |
28 | ceph.close(fd) | |
29 | return ceph.stat(f)['st_ino'] | |
30 | ||
31 | def set_mds_config_param(ceph, param): | |
32 | with open('/dev/null', 'rb') as devnull: | |
33 | confarg = '' | |
34 | if conf != '': | |
35 | confarg = '-c {c}'.format(c=conf) | |
36 | r = subprocess.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull) | |
37 | if r != 0: | |
38 | raise Exception | |
39 | ||
7c673cae FG |
40 | |
41 | class _TrimIndentFile(object): | |
42 | def __init__(self, fp): | |
43 | self.fp = fp | |
44 | ||
45 | def readline(self): | |
46 | line = self.fp.readline() | |
47 | return line.lstrip(' \t') | |
48 | ||
49 | def _optionxform(s): | |
50 | s = s.replace('_', ' ') | |
51 | s = '_'.join(s.split()) | |
52 | return s | |
53 | ||
54 | def conf_set_kill_mds(location, killnum): | |
55 | print('setting mds kill config option for {l}.{k}'.format(l=location, k=killnum), file=sys.stderr) | |
56 | print("restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum)) | |
57 | sys.stdout.flush() | |
58 | for l in sys.stdin.readline(): | |
59 | if l == 'restarted': | |
60 | break | |
61 | ||
62 | def flush(ceph, testnum): | |
63 | print('flushing {t}'.format(t=testnum), file=sys.stderr) | |
64 | set_mds_config_param(ceph, '--mds_log_max_segments 1') | |
65 | ||
66 | for i in range(1, 500): | |
67 | f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i) | |
68 | print('flushing with create {f}'.format(f=f), file=sys.stderr) | |
69 | fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644) | |
70 | ceph.close(fd) | |
71 | ceph.unlink(f) | |
72 | ||
73 | print('flush doing shutdown', file=sys.stderr) | |
74 | ceph.shutdown() | |
75 | print('flush reinitializing ceph', file=sys.stderr) | |
76 | ceph = cephfs.LibCephFS(conffile=conf) | |
77 | print('flush doing mount', file=sys.stderr) | |
78 | ceph.mount() | |
79 | return ceph | |
80 | ||
81 | def kill_mds(ceph, location, killnum): | |
82 | print('killing mds: {l}.{k}'.format(l=location, k=killnum), file=sys.stderr) | |
83 | set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum)) | |
84 | ||
85 | def wait_for_mds(ceph): | |
86 | # wait for restart | |
87 | while True: | |
88 | confarg = '' | |
89 | if conf != '': | |
90 | confarg = '-c {c}'.format(c=conf) | |
91 | r = subprocess.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True).decode() | |
92 | if r.find('a=up:active'): | |
93 | break | |
94 | time.sleep(1) | |
95 | ||
96 | def decode(value): | |
97 | ||
98 | tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid()) | |
99 | with open(tmpfile, 'w+') as f: | |
100 | f.write(value) | |
101 | ||
102 | p = subprocess.Popen( | |
103 | [ | |
104 | 'ceph-dencoder', | |
105 | 'import', | |
106 | tmpfile, | |
107 | 'type', | |
108 | 'inode_backtrace_t', | |
109 | 'decode', | |
110 | 'dump_json', | |
111 | ], | |
112 | stdin=subprocess.PIPE, | |
113 | stdout=subprocess.PIPE, | |
114 | ) | |
115 | (stdout, _) = p.communicate(input=value) | |
116 | p.stdin.close() | |
117 | if p.returncode != 0: | |
118 | raise Exception | |
119 | os.remove(tmpfile) | |
120 | return json.loads(stdout) | |
121 | ||
122 | class VerifyFailure(Exception): | |
123 | pass | |
124 | ||
125 | def verify(rados_ioctx, ino, values, pool): | |
126 | print('getting parent attr for ino: %lx.00000000' % ino, file=sys.stderr) | |
127 | savede = None | |
128 | for i in range(1, 20): | |
129 | try: | |
130 | savede = None | |
131 | binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent') | |
132 | except rados.ObjectNotFound as e: | |
133 | # wait for a bit to let segments get flushed out | |
134 | savede = e | |
135 | time.sleep(10) | |
136 | if savede: | |
137 | raise savede | |
138 | ||
139 | bt = decode(binbt) | |
140 | ||
9f95a23c | 141 | ind = 0 |
7c673cae FG |
142 | if bt['ino'] != ino: |
143 | raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
144 | bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values)) | |
7c673cae FG |
145 | for (n, i) in values: |
146 | if bt['ancestors'][ind]['dirino'] != i: | |
147 | raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
148 | b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values)) | |
149 | if bt['ancestors'][ind]['dname'] != n: | |
150 | raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
151 | b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values)) | |
152 | ind += 1 | |
153 | ||
154 | if bt['pool'] != pool: | |
155 | raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( | |
156 | btp=bt['pool'], p=pool, bt=bt, i=ino, v=values)) | |
157 | ||
158 | def make_abc(ceph, rooti, i): | |
159 | expected_bt = [] | |
160 | c, d = get_name("/", i, 0) | |
161 | expected_bt = [(c, rooti)] + expected_bt | |
162 | di = mkdir(ceph, d) | |
163 | c, d = get_name(d, i, 1) | |
164 | expected_bt = [(c, di)] + expected_bt | |
165 | di = mkdir(ceph, d) | |
166 | c, f = get_name(d, i, 2) | |
167 | fi = create(ceph, f) | |
168 | expected_bt = [(c, di)] + expected_bt | |
169 | return fi, expected_bt | |
170 | ||
171 | test = -1 | |
172 | if len(sys.argv) > 1: | |
173 | test = int(sys.argv[1]) | |
174 | ||
175 | conf = '' | |
176 | if len(sys.argv) > 2: | |
177 | conf = sys.argv[2] | |
178 | ||
179 | radosobj = rados.Rados(conffile=conf) | |
180 | radosobj.connect() | |
181 | ioctx = radosobj.open_ioctx('data') | |
182 | ||
183 | ceph = cephfs.LibCephFS(conffile=conf) | |
184 | ceph.mount() | |
185 | ||
186 | rooti = ceph.stat('/')['st_ino'] | |
187 | ||
188 | test = -1 | |
189 | if len(sys.argv) > 1: | |
190 | test = int(sys.argv[1]) | |
191 | ||
192 | conf = '/etc/ceph/ceph.conf' | |
193 | if len(sys.argv) > 2: | |
194 | conf = sys.argv[2] | |
195 | ||
196 | # create /a/b/c | |
197 | # flush | |
198 | # verify | |
199 | ||
200 | i = 0 | |
201 | if test < 0 or test == i: | |
202 | print('Running test %d: basic verify' % i, file=sys.stderr) | |
203 | ino, expected_bt = make_abc(ceph, rooti, i) | |
204 | ceph = flush(ceph, i) | |
205 | verify(ioctx, ino, expected_bt, 0) | |
206 | ||
207 | i += 1 | |
208 | ||
209 | # kill-mds-at-openc-1 | |
210 | # create /a/b/c | |
211 | # restart-mds | |
212 | # flush | |
213 | # verify | |
214 | ||
215 | if test < 0 or test == i: | |
216 | print('Running test %d: kill openc' % i, file=sys.stderr) | |
217 | print("restart mds a") | |
218 | sys.stdout.flush() | |
219 | kill_mds(ceph, 'openc', 1) | |
220 | ino, expected_bt = make_abc(ceph, rooti, i) | |
221 | ceph = flush(ceph, i) | |
222 | verify(ioctx, ino, expected_bt, 0) | |
223 | ||
224 | i += 1 | |
225 | ||
226 | # kill-mds-at-openc-1 | |
227 | # create /a/b/c | |
228 | # restart-mds with kill-mds-at-replay-1 | |
229 | # restart-mds | |
230 | # flush | |
231 | # verify | |
232 | if test < 0 or test == i: | |
233 | print('Running test %d: kill openc/replay' % i, file=sys.stderr) | |
234 | # these are reversed because we want to prepare the config | |
235 | conf_set_kill_mds('journal_replay', 1) | |
236 | kill_mds(ceph, 'openc', 1) | |
237 | print("restart mds a") | |
238 | sys.stdout.flush() | |
239 | ino, expected_bt = make_abc(ceph, rooti, i) | |
240 | ceph = flush(ceph, i) | |
241 | verify(ioctx, ino, expected_bt, 0) | |
242 | ||
243 | i += 1 | |
244 | ||
245 | ioctx.close() | |
246 | radosobj.shutdown() | |
247 | ceph.shutdown() | |
248 | ||
249 | print("done") | |
250 | sys.stdout.flush() |