]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | from StringIO import StringIO |
2 | import json | |
3 | import logging | |
eafe8130 | 4 | import time |
7c673cae FG |
5 | from textwrap import dedent |
6 | from teuthology.orchestra.run import CommandFailedError | |
7 | from teuthology import misc | |
8 | ||
9 | from teuthology.orchestra import remote as orchestra_remote | |
10 | from teuthology.orchestra import run | |
11 | from teuthology.contextutil import MaxWhileTries | |
12 | from .mount import CephFSMount | |
13 | ||
14 | log = logging.getLogger(__name__) | |
15 | ||
16 | ||
17 | UMOUNT_TIMEOUT = 300 | |
18 | ||
19 | ||
20 | class KernelMount(CephFSMount): | |
eafe8130 | 21 | def __init__(self, ctx, test_dir, client_id, client_remote, |
7c673cae | 22 | ipmi_user, ipmi_password, ipmi_domain): |
11fdf7f2 | 23 | super(KernelMount, self).__init__(ctx, test_dir, client_id, client_remote) |
7c673cae FG |
24 | |
25 | self.mounted = False | |
26 | self.ipmi_user = ipmi_user | |
27 | self.ipmi_password = ipmi_password | |
28 | self.ipmi_domain = ipmi_domain | |
29 | ||
7c673cae | 30 | def mount(self, mount_path=None, mount_fs_name=None): |
11fdf7f2 TL |
31 | self.setupfs(name=mount_fs_name) |
32 | ||
7c673cae FG |
33 | log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format( |
34 | id=self.client_id, remote=self.client_remote, mnt=self.mountpoint)) | |
35 | ||
7c673cae FG |
36 | self.client_remote.run( |
37 | args=[ | |
38 | 'mkdir', | |
39 | '--', | |
40 | self.mountpoint, | |
41 | ], | |
f64942e4 | 42 | timeout=(5*60), |
7c673cae FG |
43 | ) |
44 | ||
45 | if mount_path is None: | |
46 | mount_path = "/" | |
47 | ||
eafe8130 TL |
48 | opts = 'name={id},norequire_active_mds,conf={conf}'.format(id=self.client_id, |
49 | conf=self.config_path) | |
7c673cae FG |
50 | |
51 | if mount_fs_name is not None: | |
52 | opts += ",mds_namespace={0}".format(mount_fs_name) | |
53 | ||
54 | self.client_remote.run( | |
55 | args=[ | |
56 | 'sudo', | |
57 | 'adjust-ulimits', | |
58 | 'ceph-coverage', | |
59 | '{tdir}/archive/coverage'.format(tdir=self.test_dir), | |
eafe8130 TL |
60 | '/bin/mount', |
61 | '-t', | |
62 | 'ceph', | |
63 | ':{mount_path}'.format(mount_path=mount_path), | |
7c673cae FG |
64 | self.mountpoint, |
65 | '-v', | |
66 | '-o', | |
67 | opts | |
68 | ], | |
f64942e4 | 69 | timeout=(30*60), |
7c673cae FG |
70 | ) |
71 | ||
72 | self.client_remote.run( | |
f64942e4 | 73 | args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(5*60)) |
7c673cae FG |
74 | |
75 | self.mounted = True | |
76 | ||
77 | def umount(self, force=False): | |
78 | log.debug('Unmounting client client.{id}...'.format(id=self.client_id)) | |
79 | ||
80 | cmd=['sudo', 'umount', self.mountpoint] | |
81 | if force: | |
82 | cmd.append('-f') | |
83 | ||
c07f9fc5 | 84 | try: |
f64942e4 | 85 | self.client_remote.run(args=cmd, timeout=(15*60)) |
c07f9fc5 FG |
86 | except Exception as e: |
87 | self.client_remote.run(args=[ | |
88 | 'sudo', | |
89 | run.Raw('PATH=/usr/sbin:$PATH'), | |
90 | 'lsof', | |
91 | run.Raw(';'), | |
92 | 'ps', 'auxf', | |
f64942e4 | 93 | ], timeout=(15*60)) |
c07f9fc5 | 94 | raise e |
7c673cae FG |
95 | |
96 | rproc = self.client_remote.run( | |
97 | args=[ | |
98 | 'rmdir', | |
99 | '--', | |
100 | self.mountpoint, | |
101 | ], | |
102 | wait=False | |
103 | ) | |
104 | run.wait([rproc], UMOUNT_TIMEOUT) | |
105 | self.mounted = False | |
106 | ||
107 | def cleanup(self): | |
108 | pass | |
109 | ||
28e407b8 | 110 | def umount_wait(self, force=False, require_clean=False, timeout=900): |
7c673cae FG |
111 | """ |
112 | Unlike the fuse client, the kernel client's umount is immediate | |
113 | """ | |
114 | if not self.is_mounted(): | |
115 | return | |
116 | ||
117 | try: | |
118 | self.umount(force) | |
119 | except (CommandFailedError, MaxWhileTries): | |
120 | if not force: | |
121 | raise | |
122 | ||
123 | self.kill() | |
124 | self.kill_cleanup() | |
125 | ||
126 | self.mounted = False | |
127 | ||
128 | def is_mounted(self): | |
129 | return self.mounted | |
130 | ||
131 | def wait_until_mounted(self): | |
132 | """ | |
133 | Unlike the fuse client, the kernel client is up and running as soon | |
134 | as the initial mount() function returns. | |
135 | """ | |
136 | assert self.mounted | |
137 | ||
138 | def teardown(self): | |
139 | super(KernelMount, self).teardown() | |
140 | if self.mounted: | |
141 | self.umount() | |
142 | ||
143 | def kill(self): | |
144 | """ | |
145 | The Ceph kernel client doesn't have a mechanism to kill itself (doing | |
146 | that in side the kernel would be weird anyway), so we reboot the whole node | |
147 | to get the same effect. | |
148 | ||
149 | We use IPMI to reboot, because we don't want the client to send any | |
150 | releases of capabilities. | |
151 | """ | |
152 | ||
153 | con = orchestra_remote.getRemoteConsole(self.client_remote.hostname, | |
154 | self.ipmi_user, | |
155 | self.ipmi_password, | |
156 | self.ipmi_domain) | |
eafe8130 | 157 | con.hard_reset(wait_for_login=False) |
7c673cae FG |
158 | |
159 | self.mounted = False | |
160 | ||
161 | def kill_cleanup(self): | |
162 | assert not self.mounted | |
163 | ||
eafe8130 TL |
164 | # We need to do a sleep here because we don't know how long it will |
165 | # take for a hard_reset to be effected. | |
166 | time.sleep(30) | |
167 | ||
168 | try: | |
169 | # Wait for node to come back up after reboot | |
170 | misc.reconnect(None, 300, [self.client_remote]) | |
171 | except: | |
172 | # attempt to get some useful debug output: | |
173 | con = orchestra_remote.getRemoteConsole(self.client_remote.hostname, | |
174 | self.ipmi_user, | |
175 | self.ipmi_password, | |
176 | self.ipmi_domain) | |
177 | con.check_status(timeout=60) | |
178 | raise | |
7c673cae | 179 | |
eafe8130 TL |
180 | # Remove mount directory |
181 | self.client_remote.run(args=['uptime'], timeout=10) | |
7c673cae FG |
182 | |
183 | # Remove mount directory | |
184 | self.client_remote.run( | |
185 | args=[ | |
186 | 'rmdir', | |
187 | '--', | |
188 | self.mountpoint, | |
189 | ], | |
f64942e4 | 190 | timeout=(5*60), |
7c673cae FG |
191 | ) |
192 | ||
193 | def _find_debug_dir(self): | |
194 | """ | |
195 | Find the debugfs folder for this mount | |
196 | """ | |
197 | pyscript = dedent(""" | |
198 | import glob | |
199 | import os | |
200 | import json | |
201 | ||
202 | def get_id_to_dir(): | |
203 | result = {} | |
204 | for dir in glob.glob("/sys/kernel/debug/ceph/*"): | |
205 | mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines() | |
206 | client_id = mds_sessions_lines[1].split()[1].strip('"') | |
207 | ||
208 | result[client_id] = dir | |
209 | return result | |
210 | ||
211 | print json.dumps(get_id_to_dir()) | |
212 | """) | |
213 | ||
214 | p = self.client_remote.run(args=[ | |
215 | 'sudo', 'python', '-c', pyscript | |
f64942e4 | 216 | ], stdout=StringIO(), timeout=(5*60)) |
7c673cae FG |
217 | client_id_to_dir = json.loads(p.stdout.getvalue()) |
218 | ||
219 | try: | |
220 | return client_id_to_dir[self.client_id] | |
221 | except KeyError: | |
222 | log.error("Client id '{0}' debug dir not found (clients seen were: {1})".format( | |
223 | self.client_id, ",".join(client_id_to_dir.keys()) | |
224 | )) | |
225 | raise | |
226 | ||
227 | def _read_debug_file(self, filename): | |
228 | debug_dir = self._find_debug_dir() | |
229 | ||
230 | pyscript = dedent(""" | |
231 | import os | |
232 | ||
233 | print open(os.path.join("{debug_dir}", "{filename}")).read() | |
234 | """).format(debug_dir=debug_dir, filename=filename) | |
235 | ||
236 | p = self.client_remote.run(args=[ | |
237 | 'sudo', 'python', '-c', pyscript | |
f64942e4 | 238 | ], stdout=StringIO(), timeout=(5*60)) |
7c673cae FG |
239 | return p.stdout.getvalue() |
240 | ||
241 | def get_global_id(self): | |
242 | """ | |
243 | Look up the CephFS client ID for this mount, using debugfs. | |
244 | """ | |
245 | ||
246 | assert self.mounted | |
247 | ||
248 | mds_sessions = self._read_debug_file("mds_sessions") | |
249 | lines = mds_sessions.split("\n") | |
250 | return int(lines[0].split()[1]) | |
251 | ||
252 | def get_osd_epoch(self): | |
253 | """ | |
254 | Return 2-tuple of osd_epoch, osd_epoch_barrier | |
255 | """ | |
256 | osd_map = self._read_debug_file("osdmap") | |
257 | lines = osd_map.split("\n") | |
258 | first_line_tokens = lines[0].split() | |
259 | epoch, barrier = int(first_line_tokens[1]), int(first_line_tokens[3]) | |
260 | ||
261 | return epoch, barrier |