]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/cephfs/mount.py
bcc9aefd89543420a2571260d87c6f59f4c89dac
1 from contextlib
import contextmanager
6 from textwrap
import dedent
8 from StringIO
import StringIO
9 from teuthology
.orchestra
import run
10 from teuthology
.orchestra
.run
import CommandFailedError
, ConnectionLostError
11 from tasks
.cephfs
.filesystem
import Filesystem
13 log
= logging
.getLogger(__name__
)
16 class CephFSMount(object):
17 def __init__(self
, ctx
, test_dir
, client_id
, client_remote
):
19 :param test_dir: Global teuthology test dir
20 :param client_id: Client ID, the 'foo' in client.foo
21 :param client_remote: Remote instance for the host where client will run
25 self
.test_dir
= test_dir
26 self
.client_id
= client_id
27 self
.client_remote
= client_remote
28 self
.mountpoint_dir_name
= 'mnt.{id}'.format(id=self
.client_id
)
31 self
.test_files
= ['a', 'b', 'c']
33 self
.background_procs
= []
38 self
.test_dir
, '{dir_name}'.format(dir_name
=self
.mountpoint_dir_name
))
41 raise NotImplementedError()
43 def setupfs(self
, name
=None):
44 if name
is None and self
.fs
is not None:
45 # Previous mount existed, reuse the old name
47 self
.fs
= Filesystem(self
.ctx
, name
=name
)
48 log
.info('Wait for MDS to reach steady state...')
49 self
.fs
.wait_for_daemons()
50 log
.info('Ready to start {}...'.format(type(self
).__name
__))
52 def mount(self
, mount_path
=None, mount_fs_name
=None):
53 raise NotImplementedError()
56 raise NotImplementedError()
58 def umount_wait(self
, force
=False, require_clean
=False):
61 :param force: Expect that the mount will not shutdown cleanly: kill
63 :param require_clean: Wait for the Ceph client associated with the
64 mount (e.g. ceph-fuse) to terminate, and
65 raise if it doesn't do so cleanly.
68 raise NotImplementedError()
70 def kill_cleanup(self
):
71 raise NotImplementedError()
74 raise NotImplementedError()
77 raise NotImplementedError()
79 def wait_until_mounted(self
):
80 raise NotImplementedError()
82 def get_keyring_path(self
):
83 return '/etc/ceph/ceph.client.{id}.keyring'.format(id=self
.client_id
)
86 def config_path(self
):
88 Path to ceph.conf: override this if you're not a normal systemwide ceph install
91 return "/etc/ceph/ceph.conf"
96 A context manager, from an initially unmounted state, to mount
97 this, yield, and then unmount and clean up.
100 self
.wait_until_mounted()
106 def is_blacklisted(self
):
107 addr
= self
.get_global_addr()
108 blacklist
= json
.loads(self
.fs
.mon_manager
.raw_cluster_cmd("osd", "blacklist", "ls", "--format=json"))
110 if addr
== b
["addr"]:
114 def create_files(self
):
115 assert(self
.is_mounted())
117 for suffix
in self
.test_files
:
118 log
.info("Creating file {0}".format(suffix
))
119 self
.client_remote
.run(args
=[
120 'sudo', 'touch', os
.path
.join(self
.mountpoint
, suffix
)
123 def check_files(self
):
124 assert(self
.is_mounted())
126 for suffix
in self
.test_files
:
127 log
.info("Checking file {0}".format(suffix
))
128 r
= self
.client_remote
.run(args
=[
129 'sudo', 'ls', os
.path
.join(self
.mountpoint
, suffix
)
130 ], check_status
=False)
131 if r
.exitstatus
!= 0:
132 raise RuntimeError("Expected file {0} not found".format(suffix
))
134 def create_destroy(self
):
135 assert(self
.is_mounted())
137 filename
= "{0} {1}".format(datetime
.datetime
.now(), self
.client_id
)
138 log
.debug("Creating test file {0}".format(filename
))
139 self
.client_remote
.run(args
=[
140 'sudo', 'touch', os
.path
.join(self
.mountpoint
, filename
)
142 log
.debug("Deleting test file {0}".format(filename
))
143 self
.client_remote
.run(args
=[
144 'sudo', 'rm', '-f', os
.path
.join(self
.mountpoint
, filename
)
147 def _run_python(self
, pyscript
, py_version
='python'):
148 return self
.client_remote
.run(
149 args
=['sudo', 'adjust-ulimits', 'daemon-helper', 'kill',
150 py_version
, '-c', pyscript
], wait
=False, stdin
=run
.PIPE
,
153 def run_python(self
, pyscript
, py_version
='python'):
154 p
= self
._run
_python
(pyscript
, py_version
)
156 return p
.stdout
.getvalue().strip()
158 def run_shell(self
, args
, wait
=True):
159 args
= ["cd", self
.mountpoint
, run
.Raw('&&'), "sudo"] + args
160 return self
.client_remote
.run(args
=args
, stdout
=StringIO(),
161 stderr
=StringIO(), wait
=wait
)
163 def open_no_data(self
, basename
):
165 A pure metadata operation
167 assert(self
.is_mounted())
169 path
= os
.path
.join(self
.mountpoint
, basename
)
171 p
= self
._run
_python
(dedent(
173 f = open("{path}", 'w')
174 """.format(path
=path
)
178 def open_background(self
, basename
="background_file"):
180 Open a file for writing, then block such that the client
181 will hold a capability.
183 Don't return until the remote process has got as far as opening
184 the file, then return the RemoteProcess instance.
186 assert(self
.is_mounted())
188 path
= os
.path
.join(self
.mountpoint
, basename
)
190 pyscript
= dedent("""
193 f = open("{path}", 'w')
199 """).format(path
=path
)
201 rproc
= self
._run
_python
(pyscript
)
202 self
.background_procs
.append(rproc
)
204 # This wait would not be sufficient if the file had already
205 # existed, but it's simple and in practice users of open_background
206 # are not using it on existing files.
207 self
.wait_for_visible(basename
)
211 def wait_for_visible(self
, basename
="background_file", timeout
=30):
214 r
= self
.client_remote
.run(args
=[
215 'sudo', 'ls', os
.path
.join(self
.mountpoint
, basename
)
216 ], check_status
=False)
217 if r
.exitstatus
== 0:
218 log
.debug("File {0} became visible from {1} after {2}s".format(
219 basename
, self
.client_id
, i
))
225 raise RuntimeError("Timed out after {0}s waiting for {1} to become visible from {2}".format(
226 i
, basename
, self
.client_id
))
228 def lock_background(self
, basename
="background_file", do_flock
=True):
230 Open and lock a files for writing, hold the lock in a background process
232 assert(self
.is_mounted())
234 path
= os
.path
.join(self
.mountpoint
, basename
)
241 script_builder
+= """
242 f1 = open("{path}-1", 'w')
243 fcntl.flock(f1, fcntl.LOCK_EX | fcntl.LOCK_NB)"""
244 script_builder
+= """
245 f2 = open("{path}-2", 'w')
246 lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
247 fcntl.fcntl(f2, fcntl.F_SETLK, lockdata)
252 pyscript
= dedent(script_builder
).format(path
=path
)
254 log
.info("lock_background file {0}".format(basename
))
255 rproc
= self
._run
_python
(pyscript
)
256 self
.background_procs
.append(rproc
)
259 def lock_and_release(self
, basename
="background_file"):
260 assert(self
.is_mounted())
262 path
= os
.path
.join(self
.mountpoint
, basename
)
268 f1 = open("{path}-1", 'w')
269 fcntl.flock(f1, fcntl.LOCK_EX)
270 f2 = open("{path}-2", 'w')
271 lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
272 fcntl.fcntl(f2, fcntl.F_SETLK, lockdata)
274 pyscript
= dedent(script
).format(path
=path
)
276 log
.info("lock_and_release file {0}".format(basename
))
277 return self
._run
_python
(pyscript
)
279 def check_filelock(self
, basename
="background_file", do_flock
=True):
280 assert(self
.is_mounted())
282 path
= os
.path
.join(self
.mountpoint
, basename
)
289 script_builder
+= """
290 f1 = open("{path}-1", 'r')
292 fcntl.flock(f1, fcntl.LOCK_EX | fcntl.LOCK_NB)
294 if e.errno == errno.EAGAIN:
297 raise RuntimeError("flock on file {path}-1 not found")"""
298 script_builder
+= """
299 f2 = open("{path}-2", 'r')
301 lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
302 fcntl.fcntl(f2, fcntl.F_SETLK, lockdata)
304 if e.errno == errno.EAGAIN:
307 raise RuntimeError("posix lock on file {path}-2 not found")
309 pyscript
= dedent(script_builder
).format(path
=path
)
311 log
.info("check lock on file {0}".format(basename
))
312 self
.client_remote
.run(args
=[
313 'sudo', 'python', '-c', pyscript
316 def write_background(self
, basename
="background_file", loop
=False):
318 Open a file for writing, complete as soon as you can
322 assert(self
.is_mounted())
324 path
= os
.path
.join(self
.mountpoint
, basename
)
326 pyscript
= dedent("""
330 fd = os.open("{path}", os.O_RDWR | os.O_CREAT, 0644)
333 os.write(fd, 'content')
340 """).format(path
=path
, loop
=str(loop
))
342 rproc
= self
._run
_python
(pyscript
)
343 self
.background_procs
.append(rproc
)
346 def write_n_mb(self
, filename
, n_mb
, seek
=0, wait
=True):
348 Write the requested number of megabytes to a file
350 assert(self
.is_mounted())
352 return self
.run_shell(["dd", "if=/dev/urandom", "of={0}".format(filename
),
353 "bs=1M", "conv=fdatasync",
354 "count={0}".format(n_mb
),
355 "seek={0}".format(seek
)
358 def write_test_pattern(self
, filename
, size
):
359 log
.info("Writing {0} bytes to {1}".format(size
, filename
))
360 return self
.run_python(dedent("""
364 for i in range(0, {size}):
365 val = zlib.crc32("%s" % i) & 7
369 path
=os
.path
.join(self
.mountpoint
, filename
),
373 def validate_test_pattern(self
, filename
, size
):
374 log
.info("Validating {0} bytes from {1}".format(size
, filename
))
375 return self
.run_python(dedent("""
381 if len(bytes) != {size}:
382 raise RuntimeError("Bad length {{0}} vs. expected {{1}}".format(
385 for i, b in enumerate(bytes):
386 val = zlib.crc32("%s" % i) & 7
388 raise RuntimeError("Bad data at offset {{0}}".format(i))
390 path
=os
.path
.join(self
.mountpoint
, filename
),
394 def open_n_background(self
, fs_path
, count
):
396 Open N files for writing, hold them open in a background process
398 :param fs_path: Path relative to CephFS root, e.g. "foo/bar"
399 :return: a RemoteProcess
401 assert(self
.is_mounted())
403 abs_path
= os
.path
.join(self
.mountpoint
, fs_path
)
405 pyscript
= dedent("""
411 abs_path = "{abs_path}"
413 if not os.path.exists(os.path.dirname(abs_path)):
414 os.makedirs(os.path.dirname(abs_path))
417 for i in range(0, n):
418 fname = "{{0}}_{{1}}".format(abs_path, i)
419 handles.append(open(fname, 'w'))
423 """).format(abs_path
=abs_path
, count
=count
)
425 rproc
= self
._run
_python
(pyscript
)
426 self
.background_procs
.append(rproc
)
429 def create_n_files(self
, fs_path
, count
, sync
=False):
430 assert(self
.is_mounted())
432 abs_path
= os
.path
.join(self
.mountpoint
, fs_path
)
434 pyscript
= dedent("""
440 abs_path = "{abs_path}"
442 if not os.path.exists(os.path.dirname(abs_path)):
443 os.makedirs(os.path.dirname(abs_path))
445 for i in range(0, n):
446 fname = "{{0}}_{{1}}".format(abs_path, i)
453 """).format(abs_path
=abs_path
, count
=count
, sync
=str(sync
))
455 self
.run_python(pyscript
)
458 for p
in self
.background_procs
:
459 log
.info("Terminating background process")
460 self
._kill
_background
(p
)
462 self
.background_procs
= []
464 def _kill_background(self
, p
):
469 except (CommandFailedError
, ConnectionLostError
):
472 def kill_background(self
, p
):
474 For a process that was returned by one of the _background member functions,
477 self
._kill
_background
(p
)
478 self
.background_procs
.remove(p
)
480 def get_global_id(self
):
481 raise NotImplementedError()
483 def get_global_inst(self
):
484 raise NotImplementedError()
486 def get_global_addr(self
):
487 raise NotImplementedError()
489 def get_osd_epoch(self
):
490 raise NotImplementedError()
492 def stat(self
, fs_path
, wait
=True):
494 stat a file, and return the result as a dictionary like this:
496 "st_ctime": 1414161137.0,
497 "st_mtime": 1414161137.0,
505 "st_atime": 1431520593.0
508 Raises exception on absent file.
510 abs_path
= os
.path
.join(self
.mountpoint
, fs_path
)
512 pyscript
= dedent("""
519 s = os.stat("{path}")
523 attrs = ["st_mode", "st_ino", "st_dev", "st_nlink", "st_uid", "st_gid", "st_size", "st_atime", "st_mtime", "st_ctime"]
525 dict([(a, getattr(s, a)) for a in attrs]),
527 """).format(path
=abs_path
)
528 proc
= self
._run
_python
(pyscript
)
531 return json
.loads(proc
.stdout
.getvalue().strip())
535 def touch(self
, fs_path
):
537 Create a dentry if it doesn't already exist. This python
538 implementation exists because the usual command line tool doesn't
539 pass through error codes like EIO.
544 abs_path
= os
.path
.join(self
.mountpoint
, fs_path
)
545 pyscript
= dedent("""
550 f = open("{path}", "w")
554 """).format(path
=abs_path
)
555 proc
= self
._run
_python
(pyscript
)
558 def path_to_ino(self
, fs_path
, follow_symlinks
=True):
559 abs_path
= os
.path
.join(self
.mountpoint
, fs_path
)
562 pyscript
= dedent("""
566 print os.stat("{path}").st_ino
567 """).format(path
=abs_path
)
569 pyscript
= dedent("""
573 print os.lstat("{path}").st_ino
574 """).format(path
=abs_path
)
576 proc
= self
._run
_python
(pyscript
)
578 return int(proc
.stdout
.getvalue().strip())
580 def path_to_nlink(self
, fs_path
):
581 abs_path
= os
.path
.join(self
.mountpoint
, fs_path
)
583 pyscript
= dedent("""
587 print os.stat("{path}").st_nlink
588 """).format(path
=abs_path
)
590 proc
= self
._run
_python
(pyscript
)
592 return int(proc
.stdout
.getvalue().strip())
594 def ls(self
, path
=None):
596 Wrap ls: return a list of strings
602 ls_text
= self
.run_shell(cmd
).stdout
.getvalue().strip()
605 return ls_text
.split("\n")
607 # Special case because otherwise split on empty string
608 # gives you [''] instead of []
611 def setfattr(self
, path
, key
, val
):
615 :param path: relative to mount point
616 :param key: xattr name
617 :param val: xattr value
620 self
.run_shell(["setfattr", "-n", key
, "-v", val
, path
])
622 def getfattr(self
, path
, attr
):
624 Wrap getfattr: return the values of a named xattr on one file, or
625 None if the attribute is not found.
629 p
= self
.run_shell(["getfattr", "--only-values", "-n", attr
, path
], wait
=False)
632 except CommandFailedError
as e
:
633 if e
.exitstatus
== 1 and "No such attribute" in p
.stderr
.getvalue():
638 return p
.stdout
.getvalue()
642 Wrap df: return a dict of usage fields in bytes
645 p
= self
.run_shell(["df", "-B1", "."])
646 lines
= p
.stdout
.getvalue().strip().split("\n")
647 fs
, total
, used
, avail
= lines
[1].split()[:4]
653 "available": int(avail
)