1 # cython: embedsignature=True
3 This module is a thin wrapper around librados.
5 Error codes from librados are turned into exceptions that subclass
6 :class:`Error`. Almost all methods may raise :class:`Error(the base class of all rados exceptions), :class:`PermissionError`
7 (the base class of all rados exceptions), :class:`PermissionError`
8 and :class:`IOError`, in addition to those documented for the
11 # Copyright 2011 Josh Durgin
12 # Copyright 2011, Hannu Valtonen <hannu.valtonen@ormod.com>
13 # Copyright 2015 Hector Martin <marcan@marcan.st>
14 # Copyright 2016 Mehdi Abaakouk <sileht@redhat.com>
16 from cpython cimport PyObject, ref
17 from cpython.pycapsule cimport *
18 from libc cimport errno
19 from libc.stdint cimport *
20 from libc.stdlib cimport malloc, realloc, free
26 from collections import Callable
27 from datetime import datetime
28 from functools import partial, wraps
29 from itertools import chain
31 # Are we running Python 2.x
32 if sys.version_info[0] < 3:
38 cdef extern from "Python.h":
39 # These are in cpython/string.pxd, but use "object" types instead of
40 # PyObject*, which invokes assumptions in cpython that we need to
41 # legitimately break to implement zero-copy string buffers in Ioctx.read().
42 # This is valid use of the Python API and documented as a special case.
43 PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL
44 char* PyBytes_AsString(PyObject *string) except NULL
45 int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1
46 void PyEval_InitThreads()
49 cdef extern from "time.h":
50 ctypedef long int time_t
51 ctypedef long int suseconds_t
54 cdef extern from "sys/time.h":
60 cdef extern from "rados/rados_types.h" nogil:
61 cdef char* _LIBRADOS_ALL_NSPACES "LIBRADOS_ALL_NSPACES"
64 cdef extern from "rados/librados.h" nogil:
66 _LIBRADOS_OP_FLAG_EXCL "LIBRADOS_OP_FLAG_EXCL"
67 _LIBRADOS_OP_FLAG_FAILOK "LIBRADOS_OP_FLAG_FAILOK"
68 _LIBRADOS_OP_FLAG_FADVISE_RANDOM "LIBRADOS_OP_FLAG_FADVISE_RANDOM"
69 _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL "LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL"
70 _LIBRADOS_OP_FLAG_FADVISE_WILLNEED "LIBRADOS_OP_FLAG_FADVISE_WILLNEED"
71 _LIBRADOS_OP_FLAG_FADVISE_DONTNEED "LIBRADOS_OP_FLAG_FADVISE_DONTNEED"
72 _LIBRADOS_OP_FLAG_FADVISE_NOCACHE "LIBRADOS_OP_FLAG_FADVISE_NOCACHE"
76 _LIBRADOS_OPERATION_NOFLAG "LIBRADOS_OPERATION_NOFLAG"
77 _LIBRADOS_OPERATION_BALANCE_READS "LIBRADOS_OPERATION_BALANCE_READS"
78 _LIBRADOS_OPERATION_LOCALIZE_READS "LIBRADOS_OPERATION_LOCALIZE_READS"
79 _LIBRADOS_OPERATION_ORDER_READS_WRITES "LIBRADOS_OPERATION_ORDER_READS_WRITES"
80 _LIBRADOS_OPERATION_IGNORE_CACHE "LIBRADOS_OPERATION_IGNORE_CACHE"
81 _LIBRADOS_OPERATION_SKIPRWLOCKS "LIBRADOS_OPERATION_SKIPRWLOCKS"
82 _LIBRADOS_OPERATION_IGNORE_OVERLAY "LIBRADOS_OPERATION_IGNORE_OVERLAY"
83 _LIBRADOS_CREATE_EXCLUSIVE "LIBRADOS_CREATE_EXCLUSIVE"
84 _LIBRADOS_CREATE_IDEMPOTENT "LIBRADOS_CREATE_IDEMPOTENT"
86 cdef uint64_t _LIBRADOS_SNAP_HEAD "LIBRADOS_SNAP_HEAD"
88 ctypedef void* rados_t
89 ctypedef void* rados_config_t
90 ctypedef void* rados_ioctx_t
91 ctypedef void* rados_xattrs_iter_t
92 ctypedef void* rados_omap_iter_t
93 ctypedef void* rados_list_ctx_t
94 ctypedef uint64_t rados_snap_t
95 ctypedef void *rados_write_op_t
96 ctypedef void *rados_read_op_t
97 ctypedef void *rados_completion_t
98 ctypedef void (*rados_callback_t)(rados_completion_t cb, void *arg)
99 ctypedef void (*rados_log_callback_t)(void *arg, const char *line, const char *who,
100 uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg)
101 ctypedef void (*rados_log_callback2_t)(void *arg, const char *line, const char *channel, const char *who, const char *name,
102 uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg)
105 cdef struct rados_cluster_stat_t:
111 cdef struct rados_pool_stat_t:
115 uint64_t num_object_clones
116 uint64_t num_object_copies
117 uint64_t num_objects_missing_on_primary
118 uint64_t num_objects_unfound
119 uint64_t num_objects_degraded
125 void rados_buffer_free(char *buf)
127 void rados_version(int *major, int *minor, int *extra)
128 int rados_create2(rados_t *pcluster, const char *const clustername,
129 const char * const name, uint64_t flags)
130 int rados_create_with_context(rados_t *cluster, rados_config_t cct)
131 int rados_connect(rados_t cluster)
132 void rados_shutdown(rados_t cluster)
133 int rados_conf_read_file(rados_t cluster, const char *path)
134 int rados_conf_parse_argv_remainder(rados_t cluster, int argc, const char **argv, const char **remargv)
135 int rados_conf_parse_env(rados_t cluster, const char *var)
136 int rados_conf_set(rados_t cluster, char *option, const char *value)
137 int rados_conf_get(rados_t cluster, char *option, char *buf, size_t len)
139 int rados_ioctx_pool_stat(rados_ioctx_t io, rados_pool_stat_t *stats)
140 int64_t rados_pool_lookup(rados_t cluster, const char *pool_name)
141 int rados_pool_reverse_lookup(rados_t cluster, int64_t id, char *buf, size_t maxlen)
142 int rados_pool_create(rados_t cluster, const char *pool_name)
143 int rados_pool_create_with_auid(rados_t cluster, const char *pool_name, uint64_t auid)
144 int rados_pool_create_with_crush_rule(rados_t cluster, const char *pool_name, uint8_t crush_rule_num)
145 int rados_pool_create_with_all(rados_t cluster, const char *pool_name, uint64_t auid, uint8_t crush_rule_num)
146 int rados_pool_get_base_tier(rados_t cluster, int64_t pool, int64_t *base_tier)
147 int rados_pool_list(rados_t cluster, char *buf, size_t len)
148 int rados_pool_delete(rados_t cluster, const char *pool_name)
149 int rados_inconsistent_pg_list(rados_t cluster, int64_t pool, char *buf, size_t len)
151 int rados_cluster_stat(rados_t cluster, rados_cluster_stat_t *result)
152 int rados_cluster_fsid(rados_t cluster, char *buf, size_t len)
153 int rados_blacklist_add(rados_t cluster, char *client_address, uint32_t expire_seconds)
154 int rados_application_enable(rados_ioctx_t io, const char *app_name,
156 int rados_application_list(rados_ioctx_t io, char *values,
158 int rados_application_metadata_get(rados_ioctx_t io, const char *app_name,
159 const char *key, char *value,
161 int rados_application_metadata_set(rados_ioctx_t io, const char *app_name,
162 const char *key, const char *value)
163 int rados_application_metadata_remove(rados_ioctx_t io,
164 const char *app_name, const char *key)
165 int rados_application_metadata_list(rados_ioctx_t io,
166 const char *app_name, char *keys,
167 size_t *key_len, char *values,
169 int rados_ping_monitor(rados_t cluster, const char *mon_id, char **outstr, size_t *outstrlen)
170 int rados_mon_command(rados_t cluster, const char **cmd, size_t cmdlen,
171 const char *inbuf, size_t inbuflen,
172 char **outbuf, size_t *outbuflen,
173 char **outs, size_t *outslen)
174 int rados_mgr_command(rados_t cluster, const char **cmd, size_t cmdlen,
175 const char *inbuf, size_t inbuflen,
176 char **outbuf, size_t *outbuflen,
177 char **outs, size_t *outslen)
178 int rados_mon_command_target(rados_t cluster, const char *name, const char **cmd, size_t cmdlen,
179 const char *inbuf, size_t inbuflen,
180 char **outbuf, size_t *outbuflen,
181 char **outs, size_t *outslen)
182 int rados_osd_command(rados_t cluster, int osdid, const char **cmd, size_t cmdlen,
183 const char *inbuf, size_t inbuflen,
184 char **outbuf, size_t *outbuflen,
185 char **outs, size_t *outslen)
186 int rados_pg_command(rados_t cluster, const char *pgstr, const char **cmd, size_t cmdlen,
187 const char *inbuf, size_t inbuflen,
188 char **outbuf, size_t *outbuflen,
189 char **outs, size_t *outslen)
190 int rados_monitor_log(rados_t cluster, const char *level, rados_log_callback_t cb, void *arg)
191 int rados_monitor_log2(rados_t cluster, const char *level, rados_log_callback2_t cb, void *arg)
193 int rados_wait_for_latest_osdmap(rados_t cluster)
195 int rados_ioctx_create(rados_t cluster, const char *pool_name, rados_ioctx_t *ioctx)
196 void rados_ioctx_destroy(rados_ioctx_t io)
197 int rados_ioctx_pool_set_auid(rados_ioctx_t io, uint64_t auid)
198 void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key)
199 void rados_ioctx_set_namespace(rados_ioctx_t io, const char * nspace)
201 uint64_t rados_get_last_version(rados_ioctx_t io)
202 int rados_stat(rados_ioctx_t io, const char *o, uint64_t *psize, time_t *pmtime)
203 int rados_write(rados_ioctx_t io, const char *oid, const char *buf, size_t len, uint64_t off)
204 int rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len)
205 int rados_append(rados_ioctx_t io, const char *oid, const char *buf, size_t len)
206 int rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off)
207 int rados_remove(rados_ioctx_t io, const char *oid)
208 int rados_trunc(rados_ioctx_t io, const char *oid, uint64_t size)
209 int rados_getxattr(rados_ioctx_t io, const char *o, const char *name, char *buf, size_t len)
210 int rados_setxattr(rados_ioctx_t io, const char *o, const char *name, const char *buf, size_t len)
211 int rados_rmxattr(rados_ioctx_t io, const char *o, const char *name)
212 int rados_getxattrs(rados_ioctx_t io, const char *oid, rados_xattrs_iter_t *iter)
213 int rados_getxattrs_next(rados_xattrs_iter_t iter, const char **name, const char **val, size_t *len)
214 void rados_getxattrs_end(rados_xattrs_iter_t iter)
216 int rados_nobjects_list_open(rados_ioctx_t io, rados_list_ctx_t *ctx)
217 int rados_nobjects_list_next(rados_list_ctx_t ctx, const char **entry, const char **key, const char **nspace)
218 void rados_nobjects_list_close(rados_list_ctx_t ctx)
220 int rados_ioctx_snap_rollback(rados_ioctx_t io, const char * oid, const char * snapname)
221 int rados_ioctx_snap_create(rados_ioctx_t io, const char * snapname)
222 int rados_ioctx_snap_remove(rados_ioctx_t io, const char * snapname)
223 int rados_ioctx_snap_lookup(rados_ioctx_t io, const char * name, rados_snap_t * id)
224 int rados_ioctx_snap_get_name(rados_ioctx_t io, rados_snap_t id, char * name, int maxlen)
225 void rados_ioctx_snap_set_read(rados_ioctx_t io, rados_snap_t snap)
226 int rados_ioctx_snap_list(rados_ioctx_t io, rados_snap_t * snaps, int maxlen)
227 int rados_ioctx_snap_get_stamp(rados_ioctx_t io, rados_snap_t id, time_t * t)
229 int rados_lock_exclusive(rados_ioctx_t io, const char * oid, const char * name,
230 const char * cookie, const char * desc,
231 timeval * duration, uint8_t flags)
232 int rados_lock_shared(rados_ioctx_t io, const char * o, const char * name,
233 const char * cookie, const char * tag, const char * desc,
234 timeval * duration, uint8_t flags)
235 int rados_unlock(rados_ioctx_t io, const char * o, const char * name, const char * cookie)
237 rados_write_op_t rados_create_write_op()
238 void rados_release_write_op(rados_write_op_t write_op)
240 rados_read_op_t rados_create_read_op()
241 void rados_release_read_op(rados_read_op_t read_op)
243 int rados_aio_create_completion(void * cb_arg, rados_callback_t cb_complete, rados_callback_t cb_safe, rados_completion_t * pc)
244 void rados_aio_release(rados_completion_t c)
245 int rados_aio_stat(rados_ioctx_t io, const char *oid, rados_completion_t completion, uint64_t *psize, time_t *pmtime)
246 int rados_aio_write(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len, uint64_t off)
247 int rados_aio_append(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len)
248 int rados_aio_write_full(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len)
249 int rados_aio_remove(rados_ioctx_t io, const char * oid, rados_completion_t completion)
250 int rados_aio_read(rados_ioctx_t io, const char * oid, rados_completion_t completion, char * buf, size_t len, uint64_t off)
251 int rados_aio_flush(rados_ioctx_t io)
253 int rados_aio_get_return_value(rados_completion_t c)
254 int rados_aio_wait_for_complete_and_cb(rados_completion_t c)
255 int rados_aio_wait_for_safe_and_cb(rados_completion_t c)
256 int rados_aio_wait_for_complete(rados_completion_t c)
257 int rados_aio_wait_for_safe(rados_completion_t c)
258 int rados_aio_is_complete(rados_completion_t c)
259 int rados_aio_is_safe(rados_completion_t c)
261 int rados_exec(rados_ioctx_t io, const char * oid, const char * cls, const char * method,
262 const char * in_buf, size_t in_len, char * buf, size_t out_len)
263 int rados_aio_exec(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * cls, const char * method,
264 const char * in_buf, size_t in_len, char * buf, size_t out_len)
266 int rados_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, const char * oid, time_t * mtime, int flags)
267 int rados_aio_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, time_t *mtime, int flags)
268 void rados_write_op_omap_set(rados_write_op_t write_op, const char * const* keys, const char * const* vals, const size_t * lens, size_t num)
269 void rados_write_op_omap_rm_keys(rados_write_op_t write_op, const char * const* keys, size_t keys_len)
270 void rados_write_op_omap_clear(rados_write_op_t write_op)
271 void rados_write_op_set_flags(rados_write_op_t write_op, int flags)
273 void rados_write_op_create(rados_write_op_t write_op, int exclusive, const char *category)
274 void rados_write_op_append(rados_write_op_t write_op, const char *buffer, size_t len)
275 void rados_write_op_write_full(rados_write_op_t write_op, const char *buffer, size_t len)
276 void rados_write_op_write(rados_write_op_t write_op, const char *buffer, size_t len, uint64_t offset)
277 void rados_write_op_remove(rados_write_op_t write_op)
278 void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset)
279 void rados_write_op_zero(rados_write_op_t write_op, uint64_t offset, uint64_t len)
281 void rados_read_op_omap_get_vals2(rados_read_op_t read_op, const char * start_after, const char * filter_prefix, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval)
282 void rados_read_op_omap_get_keys2(rados_read_op_t read_op, const char * start_after, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval)
283 void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op, const char * const* keys, size_t keys_len, rados_omap_iter_t * iter, int * prval)
284 int rados_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, const char * oid, int flags)
285 int rados_aio_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, int flags)
286 void rados_read_op_set_flags(rados_read_op_t read_op, int flags)
287 int rados_omap_get_next(rados_omap_iter_t iter, const char * const* key, const char * const* val, size_t * len)
288 void rados_omap_get_end(rados_omap_iter_t iter)
291 LIBRADOS_OP_FLAG_EXCL = _LIBRADOS_OP_FLAG_EXCL
292 LIBRADOS_OP_FLAG_FAILOK = _LIBRADOS_OP_FLAG_FAILOK
293 LIBRADOS_OP_FLAG_FADVISE_RANDOM = _LIBRADOS_OP_FLAG_FADVISE_RANDOM
294 LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL = _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL
295 LIBRADOS_OP_FLAG_FADVISE_WILLNEED = _LIBRADOS_OP_FLAG_FADVISE_WILLNEED
296 LIBRADOS_OP_FLAG_FADVISE_DONTNEED = _LIBRADOS_OP_FLAG_FADVISE_DONTNEED
297 LIBRADOS_OP_FLAG_FADVISE_NOCACHE = _LIBRADOS_OP_FLAG_FADVISE_NOCACHE
299 LIBRADOS_SNAP_HEAD = _LIBRADOS_SNAP_HEAD
301 LIBRADOS_OPERATION_NOFLAG = _LIBRADOS_OPERATION_NOFLAG
302 LIBRADOS_OPERATION_BALANCE_READS = _LIBRADOS_OPERATION_BALANCE_READS
303 LIBRADOS_OPERATION_LOCALIZE_READS = _LIBRADOS_OPERATION_LOCALIZE_READS
304 LIBRADOS_OPERATION_ORDER_READS_WRITES = _LIBRADOS_OPERATION_ORDER_READS_WRITES
305 LIBRADOS_OPERATION_IGNORE_CACHE = _LIBRADOS_OPERATION_IGNORE_CACHE
306 LIBRADOS_OPERATION_SKIPRWLOCKS = _LIBRADOS_OPERATION_SKIPRWLOCKS
307 LIBRADOS_OPERATION_IGNORE_OVERLAY = _LIBRADOS_OPERATION_IGNORE_OVERLAY
309 LIBRADOS_ALL_NSPACES = _LIBRADOS_ALL_NSPACES.decode('utf-8')
311 LIBRADOS_CREATE_EXCLUSIVE = _LIBRADOS_CREATE_EXCLUSIVE
312 LIBRADOS_CREATE_IDEMPOTENT = _LIBRADOS_CREATE_IDEMPOTENT
314 ANONYMOUS_AUID = 0xffffffffffffffff
318 class Error(Exception):
319 """ `Error` class, derived from `Exception` """
323 class InvalidArgumentError(Error):
327 class OSError(Error):
328 """ `OSError` class, derived from `Error` """
329 def __init__(self, message, errno=None):
330 super(OSError, self).__init__(message)
334 msg = super(OSError, self).__str__()
335 if self.errno is None:
337 return '[errno {0}] {1}'.format(self.errno, msg)
339 def __reduce__(self):
340 return (self.__class__, (self.message, self.errno))
342 class InterruptedOrTimeoutError(OSError):
343 """ `InterruptedOrTimeoutError` class, derived from `OSError` """
347 class PermissionError(OSError):
348 """ `PermissionError` class, derived from `OSError` """
352 class PermissionDeniedError(OSError):
353 """ deal with EACCES related. """
357 class ObjectNotFound(OSError):
358 """ `ObjectNotFound` class, derived from `OSError` """
362 class NoData(OSError):
363 """ `NoData` class, derived from `OSError` """
367 class ObjectExists(OSError):
368 """ `ObjectExists` class, derived from `OSError` """
372 class ObjectBusy(OSError):
373 """ `ObjectBusy` class, derived from `IOError` """
377 class IOError(OSError):
378 """ `ObjectBusy` class, derived from `OSError` """
382 class NoSpace(OSError):
383 """ `NoSpace` class, derived from `OSError` """
387 class RadosStateError(Error):
388 """ `RadosStateError` class, derived from `Error` """
392 class IoctxStateError(Error):
393 """ `IoctxStateError` class, derived from `Error` """
397 class ObjectStateError(Error):
398 """ `ObjectStateError` class, derived from `Error` """
402 class LogicError(Error):
403 """ `` class, derived from `Error` """
407 class TimedOut(OSError):
408 """ `TimedOut` class, derived from `OSError` """
412 IF UNAME_SYSNAME == "FreeBSD":
413 cdef errno_to_exception = {
414 errno.EPERM : PermissionError,
415 errno.ENOENT : ObjectNotFound,
417 errno.ENOSPC : NoSpace,
418 errno.EEXIST : ObjectExists,
419 errno.EBUSY : ObjectBusy,
420 errno.ENOATTR : NoData,
421 errno.EINTR : InterruptedOrTimeoutError,
422 errno.ETIMEDOUT : TimedOut,
423 errno.EACCES : PermissionDeniedError,
424 errno.EINVAL : InvalidArgumentError,
427 cdef errno_to_exception = {
428 errno.EPERM : PermissionError,
429 errno.ENOENT : ObjectNotFound,
431 errno.ENOSPC : NoSpace,
432 errno.EEXIST : ObjectExists,
433 errno.EBUSY : ObjectBusy,
434 errno.ENODATA : NoData,
435 errno.EINTR : InterruptedOrTimeoutError,
436 errno.ETIMEDOUT : TimedOut,
437 errno.EACCES : PermissionDeniedError,
438 errno.EINVAL : InvalidArgumentError,
442 cdef make_ex(ret, msg):
444 Translate a librados return code into an exception.
446 :param ret: the return code
448 :param msg: the error message to use
450 :returns: a subclass of :class:`Error`
453 if ret in errno_to_exception:
454 return errno_to_exception[ret](msg, errno=ret)
456 return OSError(msg, errno=ret)
459 # helper to specify an optional argument, where in addition to `cls`, `None`
465 # validate argument types of an instance method
466 # kwargs is an un-ordered dict, so use args instead
467 def requires(*types):
468 def is_type_of(v, t):
472 return isinstance(v, t)
474 def check_type(val, arg_name, arg_type):
475 if isinstance(arg_type, tuple):
476 if any(is_type_of(val, t) for t in arg_type):
478 type_names = ' or '.join('None' if t is None else t.__name__
480 raise TypeError('%s must be %s' % (arg_name, type_names))
482 if is_type_of(val, arg_type):
484 assert(arg_type is not None)
485 raise TypeError('%s must be %s' % (arg_name, arg_type.__name__))
488 # FIXME(sileht): this stop with
489 # AttributeError: 'method_descriptor' object has no attribute '__module__'
491 def validate_func(*args, **kwargs):
492 # ignore the `self` arg
493 pos_args = zip(args[1:], types)
494 named_args = ((kwargs[name], (name, spec)) for name, spec in types
496 for arg_val, (arg_name, arg_type) in chain(pos_args, named_args):
497 check_type(arg_val, arg_name, arg_type)
498 return f(*args, **kwargs)
503 def cstr(val, name, encoding="utf-8", opt=False):
505 Create a byte string from a Python string
507 :param basestring val: Python string
508 :param str name: Name of the string parameter, for exceptions
509 :param str encoding: Encoding to use
510 :param bool opt: If True, None is allowed
512 :raises: :class:`InvalidArgument`
514 if opt and val is None:
516 if isinstance(val, bytes):
518 elif isinstance(val, unicode):
519 return val.encode(encoding)
521 raise TypeError('%s must be a string' % name)
524 def cstr_list(list_str, name, encoding="utf-8"):
525 return [cstr(s, name) for s in list_str]
528 def decode_cstr(val, encoding="utf-8"):
530 Decode a byte string into a Python string.
532 :param bytes val: byte string
533 :rtype: unicode or None
538 return val.decode(encoding)
541 cdef char* opt_str(s) except? NULL:
547 cdef void* realloc_chk(void* ptr, size_t size) except NULL:
548 cdef void *ret = realloc(ptr, size)
550 raise MemoryError("realloc failed")
554 cdef size_t * to_csize_t_array(list_int):
555 cdef size_t *ret = <size_t *>malloc(len(list_int) * sizeof(size_t))
557 raise MemoryError("malloc failed")
558 for i in xrange(len(list_int)):
559 ret[i] = <size_t>list_int[i]
563 cdef char ** to_bytes_array(list_bytes):
564 cdef char **ret = <char **>malloc(len(list_bytes) * sizeof(char *))
566 raise MemoryError("malloc failed")
567 for i in xrange(len(list_bytes)):
568 ret[i] = <char *>list_bytes[i]
573 cdef int __monitor_callback(void *arg, const char *line, const char *who,
574 uint64_t sec, uint64_t nsec, uint64_t seq,
575 const char *level, const char *msg) with gil:
576 cdef object cb_info = <object>arg
577 cb_info[0](cb_info[1], line, who, sec, nsec, seq, level, msg)
580 cdef int __monitor_callback2(void *arg, const char *line, const char *channel,
583 uint64_t sec, uint64_t nsec, uint64_t seq,
584 const char *level, const char *msg) with gil:
585 cdef object cb_info = <object>arg
586 cb_info[0](cb_info[1], line, channel, name, who, sec, nsec, seq, level, msg)
590 class Version(object):
591 """ Version information """
592 def __init__(self, major, minor, extra):
598 return "%d.%d.%d" % (self.major, self.minor, self.extra)
601 cdef class Rados(object):
602 """This class wraps librados functions"""
603 # NOTE(sileht): attributes declared in .pyd
605 def __init__(self, *args, **kwargs):
607 self.__setup(*args, **kwargs)
609 @requires(('rados_id', opt(str_type)), ('name', opt(str_type)), ('clustername', opt(str_type)),
610 ('conffile', opt(str_type)))
611 def __setup(self, rados_id=None, name=None, clustername=None,
612 conf_defaults=None, conffile=None, conf=None, flags=0,
614 self.monitor_callback = None
615 self.monitor_callback2 = None
616 self.parsed_args = []
617 self.conf_defaults = conf_defaults
618 self.conffile = conffile
619 self.rados_id = rados_id
621 if rados_id and name:
622 raise Error("Rados(): can't supply both rados_id and name")
624 name = 'client.' + rados_id
626 name = 'client.admin'
627 if clustername is None:
630 name = cstr(name, 'name')
631 clustername = cstr(clustername, 'clustername')
634 char *_clustername = clustername
639 # Unpack void* (aka rados_config_t) from capsule
640 rados_config = <rados_config_t> PyCapsule_GetPointer(context, NULL)
642 ret = rados_create_with_context(&self.cluster, rados_config)
645 ret = rados_create2(&self.cluster, _clustername, _name, _flags)
647 raise Error("rados_initialize failed with error code: %d" % ret)
649 self.state = "configuring"
650 # order is important: conf_defaults, then conffile, then conf
652 for key, value in conf_defaults.items():
653 self.conf_set(key, value)
654 if conffile is not None:
655 # read the default conf file when '' is given
658 self.conf_read_file(conffile)
660 for key, value in conf.items():
661 self.conf_set(key, value)
663 def require_state(self, *args):
665 Checks if the Rados object is in a special state
667 :raises: RadosStateError
669 if self.state in args:
671 raise RadosStateError("You cannot perform that operation on a \
672 Rados object in state %s." % self.state)
676 Disconnects from the cluster. Call this explicitly when a
677 Rados.connect()ed object is no longer used.
679 if self.state != "shutdown":
681 rados_shutdown(self.cluster)
682 self.state = "shutdown"
688 def __exit__(self, type_, value, traceback):
694 Get the version number of the ``librados`` C library.
696 :returns: a tuple of ``(major, minor, extra)`` components of the
703 rados_version(&major, &minor, &extra)
704 return Version(major, minor, extra)
706 @requires(('path', opt(str_type)))
707 def conf_read_file(self, path=None):
709 Configure the cluster handle using a Ceph config file.
711 :param path: path to the config file
714 self.require_state("configuring", "connected")
715 path = cstr(path, 'path', opt=True)
717 char *_path = opt_str(path)
719 ret = rados_conf_read_file(self.cluster, _path)
721 raise make_ex(ret, "error calling conf_read_file")
723 def conf_parse_argv(self, args):
725 Parse known arguments from args, and remove; returned
726 args contain only those unknown to ceph
728 self.require_state("configuring", "connected")
732 cargs = cstr_list(args, 'args')
734 int _argc = len(args)
735 char **_argv = to_bytes_array(cargs)
736 char **_remargv = NULL
739 _remargv = <char **>malloc(_argc * sizeof(char *))
741 ret = rados_conf_parse_argv_remainder(self.cluster, _argc,
743 <const char**>_remargv)
745 raise make_ex(ret, "error calling conf_parse_argv_remainder")
747 # _remargv was allocated with fixed argc; collapse return
748 # list to eliminate any missing args
749 retargs = [decode_cstr(a) for a in _remargv[:_argc]
751 self.parsed_args = args
757 def conf_parse_env(self, var='CEPH_ARGS'):
759 Parse known arguments from an environment variable, normally
762 self.require_state("configuring", "connected")
766 var = cstr(var, 'var')
770 ret = rados_conf_parse_env(self.cluster, _var)
772 raise make_ex(ret, "error calling conf_parse_env")
774 @requires(('option', str_type))
775 def conf_get(self, option):
777 Get the value of a configuration option
779 :param option: which option to read
782 :returns: str - value of the option or None
783 :raises: :class:`TypeError`
785 self.require_state("configuring", "connected")
786 option = cstr(option, 'option')
788 char *_option = option
794 ret_buf = <char *>realloc_chk(ret_buf, length)
796 ret = rados_conf_get(self.cluster, _option, ret_buf, length)
798 return decode_cstr(ret_buf)
799 elif ret == -errno.ENAMETOOLONG:
801 elif ret == -errno.ENOENT:
804 raise make_ex(ret, "error calling conf_get")
808 @requires(('option', str_type), ('val', str_type))
809 def conf_set(self, option, val):
811 Set the value of a configuration option
813 :param option: which option to set
815 :param option: value of the option
818 :raises: :class:`TypeError`, :class:`ObjectNotFound`
820 self.require_state("configuring", "connected")
821 option = cstr(option, 'option')
822 val = cstr(val, 'val')
824 char *_option = option
828 ret = rados_conf_set(self.cluster, _option, _val)
830 raise make_ex(ret, "error calling conf_set")
832 def ping_monitor(self, mon_id):
834 Ping a monitor to assess liveness
836 May be used as a simply way to assess liveness, or to obtain
837 information about the monitor in a simple way even in the
840 :param mon_id: the ID portion of the monitor's name (i.e., mon.<ID>)
842 :returns: the string reply from the monitor
845 self.require_state("configuring", "connected")
847 mon_id = cstr(mon_id, 'mon_id')
849 char *_mon_id = mon_id
854 ret = rados_ping_monitor(self.cluster, _mon_id, &outstr, &outstrlen)
857 raise make_ex(ret, "error calling ping_monitor")
860 my_outstr = outstr[:outstrlen]
861 rados_buffer_free(outstr)
862 return decode_cstr(my_outstr)
864 def connect(self, timeout=0):
866 Connect to the cluster. Use shutdown() to release resources.
868 self.require_state("configuring")
869 # NOTE(sileht): timeout was supported by old python API,
870 # but this is not something available in C API, so ignore
871 # for now and remove it later
873 ret = rados_connect(self.cluster)
875 raise make_ex(ret, "error connecting to the cluster")
876 self.state = "connected"
878 def get_cluster_stats(self):
880 Read usage info about the cluster
882 This tells you total space, space used, space available, and number
883 of objects. These are not updated immediately when data is written,
884 they are eventually consistent.
886 :returns: dict - contains the following keys:
888 - ``kb`` (int) - total space
890 - ``kb_used`` (int) - space used
892 - ``kb_avail`` (int) - free space available
894 - ``num_objects`` (int) - number of objects
898 rados_cluster_stat_t stats
901 ret = rados_cluster_stat(self.cluster, &stats)
905 ret, "Rados.get_cluster_stats(%s): get_stats failed" % self.rados_id)
906 return {'kb': stats.kb,
907 'kb_used': stats.kb_used,
908 'kb_avail': stats.kb_avail,
909 'num_objects': stats.num_objects}
911 @requires(('pool_name', str_type))
912 def pool_exists(self, pool_name):
914 Checks if a given pool exists.
916 :param pool_name: name of the pool to check
919 :raises: :class:`TypeError`, :class:`Error`
920 :returns: bool - whether the pool exists, false otherwise.
922 self.require_state("connected")
924 pool_name = cstr(pool_name, 'pool_name')
926 char *_pool_name = pool_name
929 ret = rados_pool_lookup(self.cluster, _pool_name)
932 elif ret == -errno.ENOENT:
935 raise make_ex(ret, "error looking up pool '%s'" % pool_name)
937 @requires(('pool_name', str_type))
938 def pool_lookup(self, pool_name):
940 Returns a pool's ID based on its name.
942 :param pool_name: name of the pool to look up
945 :raises: :class:`TypeError`, :class:`Error`
946 :returns: int - pool ID, or None if it doesn't exist
948 self.require_state("connected")
949 pool_name = cstr(pool_name, 'pool_name')
951 char *_pool_name = pool_name
954 ret = rados_pool_lookup(self.cluster, _pool_name)
957 elif ret == -errno.ENOENT:
960 raise make_ex(ret, "error looking up pool '%s'" % pool_name)
962 @requires(('pool_id', int))
963 def pool_reverse_lookup(self, pool_id):
965 Returns a pool's name based on its ID.
967 :param pool_id: ID of the pool to look up
970 :raises: :class:`TypeError`, :class:`Error`
971 :returns: string - pool name, or None if it doesn't exist
973 self.require_state("connected")
975 int64_t _pool_id = pool_id
981 name = <char *>realloc_chk(name, size)
983 ret = rados_pool_reverse_lookup(self.cluster, _pool_id, name, size)
986 elif ret != -errno.ERANGE and size <= 4096:
988 elif ret == -errno.ENOENT:
991 raise make_ex(ret, "error reverse looking up pool '%s'" % pool_id)
993 return decode_cstr(name)
998 @requires(('pool_name', str_type), ('auid', opt(int)), ('crush_rule', opt(int)))
999 def create_pool(self, pool_name, auid=None, crush_rule=None):
1002 - with default settings: if auid=None and crush_rule=None
1003 - owned by a specific auid: auid given and crush_rule=None
1004 - with a specific CRUSH rule: if auid=None and crush_rule given
1005 - with a specific CRUSH rule and auid: if auid and crush_rule given
1007 :param pool_name: name of the pool to create
1008 :type pool_name: str
1009 :param auid: the id of the owner of the new pool
1011 :param crush_rule: rule to use for placement in the new pool
1012 :type crush_rule: int
1014 :raises: :class:`TypeError`, :class:`Error`
1016 self.require_state("connected")
1018 pool_name = cstr(pool_name, 'pool_name')
1020 char *_pool_name = pool_name
1024 if auid is None and crush_rule is None:
1026 ret = rados_pool_create(self.cluster, _pool_name)
1028 _crush_rule = crush_rule
1030 ret = rados_pool_create_with_crush_rule(self.cluster, _pool_name, _crush_rule)
1031 elif crush_rule is None:
1034 ret = rados_pool_create_with_auid(self.cluster, _pool_name, _auid)
1037 _crush_rule = crush_rule
1039 ret = rados_pool_create_with_all(self.cluster, _pool_name, _auid, _crush_rule)
1041 raise make_ex(ret, "error creating pool '%s'" % pool_name)
1043 @requires(('pool_id', int))
1044 def get_pool_base_tier(self, pool_id):
1048 :returns: base pool, or pool_id if tiering is not configured for the pool
1050 self.require_state("connected")
1052 int64_t base_tier = 0
1053 int64_t _pool_id = pool_id
1056 ret = rados_pool_get_base_tier(self.cluster, _pool_id, &base_tier)
1058 raise make_ex(ret, "get_pool_base_tier(%d)" % pool_id)
1059 return int(base_tier)
1061 @requires(('pool_name', str_type))
1062 def delete_pool(self, pool_name):
1064 Delete a pool and all data inside it.
1066 The pool is removed from the cluster immediately,
1067 but the actual data is deleted in the background.
1069 :param pool_name: name of the pool to delete
1070 :type pool_name: str
1072 :raises: :class:`TypeError`, :class:`Error`
1074 self.require_state("connected")
1076 pool_name = cstr(pool_name, 'pool_name')
1078 char *_pool_name = pool_name
1081 ret = rados_pool_delete(self.cluster, _pool_name)
1083 raise make_ex(ret, "error deleting pool '%s'" % pool_name)
1085 @requires(('pool_id', int))
1086 def get_inconsistent_pgs(self, pool_id):
1088 List inconsistent placement groups in the given pool
1090 :param pool_id: ID of the pool in which PGs are listed
1092 :returns: list - inconsistent placement groups
1094 self.require_state("connected")
1096 int64_t pool = pool_id
1102 pgs = <char *>realloc_chk(pgs, size);
1104 ret = rados_inconsistent_pg_list(self.cluster, pool,
1111 raise make_ex(ret, "error calling inconsistent_pg_list")
1112 return [pg for pg in decode_cstr(pgs[:ret]).split('\0') if pg]
1116 def list_pools(self):
1118 Gets a list of pool names.
1120 :returns: list - of pool names.
1122 self.require_state("connected")
1125 char *c_names = NULL
1129 c_names = <char *>realloc_chk(c_names, size)
1131 ret = rados_pool_list(self.cluster, c_names, size)
1136 return [name for name in decode_cstr(c_names[:ret]).split('\0')
1143 Get the fsid of the cluster as a hexadecimal string.
1145 :raises: :class:`Error`
1146 :returns: str - cluster fsid
1148 self.require_state("connected")
1152 PyObject* ret_s = NULL
1154 ret_s = PyBytes_FromStringAndSize(NULL, buf_len)
1156 ret_buf = PyBytes_AsString(ret_s)
1158 ret = rados_cluster_fsid(self.cluster, ret_buf, buf_len)
1160 raise make_ex(ret, "error getting cluster fsid")
1161 if ret != <int>buf_len:
1162 _PyBytes_Resize(&ret_s, ret)
1163 return <object>ret_s
1165 # We DECREF unconditionally: the cast to object above will have
1166 # INCREFed if necessary. This also takes care of exceptions,
1167 # including if _PyString_Resize fails (that will free the string
1168 # itself and set ret_s to NULL, hence XDECREF).
1169 ref.Py_XDECREF(ret_s)
1171 @requires(('ioctx_name', str_type))
1172 def open_ioctx(self, ioctx_name):
1174 Create an io context
1176 The io context allows you to perform operations within a particular
1179 :param ioctx_name: name of the pool
1180 :type ioctx_name: str
1182 :raises: :class:`TypeError`, :class:`Error`
1183 :returns: Ioctx - Rados Ioctx object
1185 self.require_state("connected")
1186 ioctx_name = cstr(ioctx_name, 'ioctx_name')
1189 char *_ioctx_name = ioctx_name
1191 ret = rados_ioctx_create(self.cluster, _ioctx_name, &ioctx)
1193 raise make_ex(ret, "error opening pool '%s'" % ioctx_name)
1194 io = Ioctx(ioctx_name)
1198 def mon_command(self, cmd, inbuf, timeout=0, target=None):
1200 mon_command[_target](cmd, inbuf, outbuf, outbuflen, outs, outslen)
1201 returns (int ret, string outbuf, string outs)
1203 # NOTE(sileht): timeout is ignored because C API doesn't provide
1204 # timeout argument, but we keep it for backward compat with old python binding
1206 self.require_state("connected")
1207 cmd = cstr_list(cmd, 'c')
1209 if isinstance(target, int):
1210 # NOTE(sileht): looks weird but test_monmap_dump pass int
1211 target = str(target)
1213 target = cstr(target, 'target', opt=True)
1214 inbuf = cstr(inbuf, 'inbuf')
1217 char *_target = opt_str(target)
1218 char **_cmd = to_bytes_array(cmd)
1219 size_t _cmdlen = len(cmd)
1221 char *_inbuf = inbuf
1222 size_t _inbuf_len = len(inbuf)
1232 ret = rados_mon_command_target(self.cluster, _target,
1233 <const char **>_cmd, _cmdlen,
1234 <const char*>_inbuf, _inbuf_len,
1235 &_outbuf, &_outbuf_len,
1239 ret = rados_mon_command(self.cluster,
1240 <const char **>_cmd, _cmdlen,
1241 <const char*>_inbuf, _inbuf_len,
1242 &_outbuf, &_outbuf_len,
1245 my_outs = decode_cstr(_outs[:_outs_len])
1246 my_outbuf = _outbuf[:_outbuf_len]
1248 rados_buffer_free(_outs)
1250 rados_buffer_free(_outbuf)
1251 return (ret, my_outbuf, my_outs)
1255 def osd_command(self, osdid, cmd, inbuf, timeout=0):
1257 osd_command(osdid, cmd, inbuf, outbuf, outbuflen, outs, outslen)
1258 returns (int ret, string outbuf, string outs)
1260 # NOTE(sileht): timeout is ignored because C API doesn't provide
1261 # timeout argument, but we keep it for backward compat with old python binding
1262 self.require_state("connected")
1264 cmd = cstr_list(cmd, 'cmd')
1265 inbuf = cstr(inbuf, 'inbuf')
1269 char **_cmd = to_bytes_array(cmd)
1270 size_t _cmdlen = len(cmd)
1272 char *_inbuf = inbuf
1273 size_t _inbuf_len = len(inbuf)
1282 ret = rados_osd_command(self.cluster, _osdid,
1283 <const char **>_cmd, _cmdlen,
1284 <const char*>_inbuf, _inbuf_len,
1285 &_outbuf, &_outbuf_len,
1288 my_outs = decode_cstr(_outs[:_outs_len])
1289 my_outbuf = _outbuf[:_outbuf_len]
1291 rados_buffer_free(_outs)
1293 rados_buffer_free(_outbuf)
1294 return (ret, my_outbuf, my_outs)
1298 def mgr_command(self, cmd, inbuf, timeout=0):
1300 returns (int ret, string outbuf, string outs)
1302 # NOTE(sileht): timeout is ignored because C API doesn't provide
1303 # timeout argument, but we keep it for backward compat with old python binding
1304 self.require_state("connected")
1306 cmd = cstr_list(cmd, 'cmd')
1307 inbuf = cstr(inbuf, 'inbuf')
1310 char **_cmd = to_bytes_array(cmd)
1311 size_t _cmdlen = len(cmd)
1313 char *_inbuf = inbuf
1314 size_t _inbuf_len = len(inbuf)
1323 ret = rados_mgr_command(self.cluster,
1324 <const char **>_cmd, _cmdlen,
1325 <const char*>_inbuf, _inbuf_len,
1326 &_outbuf, &_outbuf_len,
1329 my_outs = decode_cstr(_outs[:_outs_len])
1330 my_outbuf = _outbuf[:_outbuf_len]
1332 rados_buffer_free(_outs)
1334 rados_buffer_free(_outbuf)
1335 return (ret, my_outbuf, my_outs)
1339 def pg_command(self, pgid, cmd, inbuf, timeout=0):
1341 pg_command(pgid, cmd, inbuf, outbuf, outbuflen, outs, outslen)
1342 returns (int ret, string outbuf, string outs)
1344 # NOTE(sileht): timeout is ignored because C API doesn't provide
1345 # timeout argument, but we keep it for backward compat with old python binding
1346 self.require_state("connected")
1348 pgid = cstr(pgid, 'pgid')
1349 cmd = cstr_list(cmd, 'cmd')
1350 inbuf = cstr(inbuf, 'inbuf')
1354 char **_cmd = to_bytes_array(cmd)
1355 size_t _cmdlen = len(cmd)
1357 char *_inbuf = inbuf
1358 size_t _inbuf_len = len(inbuf)
1367 ret = rados_pg_command(self.cluster, _pgid,
1368 <const char **>_cmd, _cmdlen,
1369 <const char *>_inbuf, _inbuf_len,
1370 &_outbuf, &_outbuf_len,
1373 my_outs = decode_cstr(_outs[:_outs_len])
1374 my_outbuf = _outbuf[:_outbuf_len]
1376 rados_buffer_free(_outs)
1378 rados_buffer_free(_outbuf)
1379 return (ret, my_outbuf, my_outs)
1383 def wait_for_latest_osdmap(self):
1384 self.require_state("connected")
1386 ret = rados_wait_for_latest_osdmap(self.cluster)
1389 def blacklist_add(self, client_address, expire_seconds=0):
1391 Blacklist a client from the OSDs
1393 :param client_address: client address
1394 :type client_address: str
1395 :param expire_seconds: number of seconds to blacklist
1396 :type expire_seconds: int
1398 :raises: :class:`Error`
1400 self.require_state("connected")
1401 client_address = cstr(client_address, 'client_address')
1403 uint32_t _expire_seconds = expire_seconds
1404 char *_client_address = client_address
1407 ret = rados_blacklist_add(self.cluster, _client_address, _expire_seconds)
1409 raise make_ex(ret, "error blacklisting client '%s'" % client_address)
1411 def monitor_log(self, level, callback, arg):
1412 if level not in MONITOR_LEVELS:
1413 raise LogicError("invalid monitor level " + level)
1414 if callback is not None and not callable(callback):
1415 raise LogicError("callback must be a callable function or None")
1417 level = cstr(level, 'level')
1418 cdef char *_level = level
1420 if callback is None:
1422 r = rados_monitor_log(self.cluster, <const char*>_level, NULL, NULL)
1423 self.monitor_callback = None
1424 self.monitor_callback2 = None
1427 cb = (callback, arg)
1428 cdef PyObject* _arg = <PyObject*>cb
1430 r = rados_monitor_log(self.cluster, <const char*>_level,
1431 <rados_log_callback_t>&__monitor_callback, _arg)
1434 raise make_ex(r, 'error calling rados_monitor_log')
1435 # NOTE(sileht): Prevents the callback method from being garbage collected
1436 self.monitor_callback = cb
1437 self.monitor_callback2 = None
1439 def monitor_log2(self, level, callback, arg):
1440 if level not in MONITOR_LEVELS:
1441 raise LogicError("invalid monitor level " + level)
1442 if callback is not None and not callable(callback):
1443 raise LogicError("callback must be a callable function or None")
1445 level = cstr(level, 'level')
1446 cdef char *_level = level
1448 if callback is None:
1450 r = rados_monitor_log2(self.cluster, <const char*>_level, NULL, NULL)
1451 self.monitor_callback = None
1452 self.monitor_callback2 = None
1455 cb = (callback, arg)
1456 cdef PyObject* _arg = <PyObject*>cb
1458 r = rados_monitor_log2(self.cluster, <const char*>_level,
1459 <rados_log_callback2_t>&__monitor_callback2, _arg)
1462 raise make_ex(r, 'error calling rados_monitor_log')
1463 # NOTE(sileht): Prevents the callback method from being garbage collected
1464 self.monitor_callback = None
1465 self.monitor_callback2 = cb
1468 cdef class OmapIterator(object):
1471 cdef public Ioctx ioctx
1472 cdef rados_omap_iter_t ctx
1474 def __cinit__(self, Ioctx ioctx):
1482 Get the next key-value pair in the object
1483 :returns: next rados.OmapItem
1491 ret = rados_omap_get_next(self.ctx, &key_, &val_, &len_)
1494 raise make_ex(ret, "error iterating over the omap")
1496 raise StopIteration()
1497 key = decode_cstr(key_)
1503 def __dealloc__(self):
1505 rados_omap_get_end(self.ctx)
1508 cdef class ObjectIterator(object):
1509 """rados.Ioctx Object iterator"""
1511 cdef rados_list_ctx_t ctx
1513 cdef public object ioctx
1515 def __cinit__(self, Ioctx ioctx):
1519 ret = rados_nobjects_list_open(ioctx.io, &self.ctx)
1521 raise make_ex(ret, "error iterating over the objects in ioctx '%s'"
1529 Get the next object name and locator in the pool
1531 :raises: StopIteration
1532 :returns: next rados.Ioctx Object
1535 const char *key_ = NULL
1536 const char *locator_ = NULL
1537 const char *nspace_ = NULL
1540 ret = rados_nobjects_list_next(self.ctx, &key_, &locator_, &nspace_)
1543 raise StopIteration()
1545 key = decode_cstr(key_)
1546 locator = decode_cstr(locator_) if locator_ != NULL else None
1547 nspace = decode_cstr(nspace_) if nspace_ != NULL else None
1548 return Object(self.ioctx, key, locator, nspace)
1550 def __dealloc__(self):
1552 rados_nobjects_list_close(self.ctx)
1555 cdef class XattrIterator(object):
1556 """Extended attribute iterator"""
1558 cdef rados_xattrs_iter_t it
1561 cdef public Ioctx ioctx
1562 cdef public object oid
1564 def __cinit__(self, Ioctx ioctx, oid):
1566 self.oid = cstr(oid, 'oid')
1567 self._oid = self.oid
1570 ret = rados_getxattrs(ioctx.io, self._oid, &self.it)
1572 raise make_ex(ret, "Failed to get rados xattrs for object %r" % oid)
1579 Get the next xattr on the object
1581 :raises: StopIteration
1582 :returns: pair - of name and value of the next Xattr
1585 const char *name_ = NULL
1586 const char *val_ = NULL
1590 ret = rados_getxattrs_next(self.it, &name_, &val_, &len_)
1592 raise make_ex(ret, "error iterating over the extended attributes \
1593 in '%s'" % self.oid)
1595 raise StopIteration()
1596 name = decode_cstr(name_)
1600 def __dealloc__(self):
1602 rados_getxattrs_end(self.it)
1605 cdef class SnapIterator(object):
1606 """Snapshot iterator"""
1608 cdef public Ioctx ioctx
1610 cdef rados_snap_t *snaps
1614 def __cinit__(self, Ioctx ioctx):
1616 # We don't know how big a buffer we need until we've called the
1617 # function. So use the exponential doubling strategy.
1618 cdef int num_snaps = 10
1620 self.snaps = <rados_snap_t*>realloc_chk(self.snaps,
1622 sizeof(rados_snap_t))
1625 ret = rados_ioctx_snap_list(ioctx.io, self.snaps, num_snaps)
1629 elif ret != -errno.ERANGE:
1630 raise make_ex(ret, "error calling rados_snap_list for \
1631 ioctx '%s'" % self.ioctx.name)
1632 num_snaps = num_snaps * 2
1640 Get the next Snapshot
1642 :raises: :class:`Error`, StopIteration
1643 :returns: Snap - next snapshot
1645 if self.cur_snap >= self.max_snap:
1649 rados_snap_t snap_id = self.snaps[self.cur_snap]
1655 name = <char *>realloc_chk(name, name_len)
1657 ret = rados_ioctx_snap_get_name(self.ioctx.io, snap_id, name, name_len)
1660 elif ret != -errno.ERANGE:
1661 raise make_ex(ret, "rados_snap_get_name error")
1663 name_len = name_len * 2
1665 snap = Snap(self.ioctx, decode_cstr(name[:name_len]).rstrip('\0'), snap_id)
1666 self.cur_snap = self.cur_snap + 1
1672 cdef class Snap(object):
1673 """Snapshot object"""
1674 cdef public Ioctx ioctx
1675 cdef public object name
1677 # NOTE(sileht): old API was storing the ctypes object
1678 # instead of the value ....
1679 cdef public rados_snap_t snap_id
1681 def __cinit__(self, Ioctx ioctx, object name, rados_snap_t snap_id):
1684 self.snap_id = snap_id
1687 return "rados.Snap(ioctx=%s,name=%s,snap_id=%d)" \
1688 % (str(self.ioctx), self.name, self.snap_id)
1690 def get_timestamp(self):
1692 Find when a snapshot in the current pool occurred
1694 :raises: :class:`Error`
1695 :returns: datetime - the data and time the snapshot was created
1697 cdef time_t snap_time
1700 ret = rados_ioctx_snap_get_stamp(self.ioctx.io, self.snap_id, &snap_time)
1702 raise make_ex(ret, "rados_ioctx_snap_get_stamp error")
1703 return datetime.fromtimestamp(snap_time)
1706 cdef class Completion(object):
1707 """completion object"""
1715 rados_callback_t complete_cb
1716 rados_callback_t safe_cb
1717 rados_completion_t rados_comp
1720 def __cinit__(self, Ioctx ioctx, object oncomplete, object onsafe):
1721 self.oncomplete = oncomplete
1722 self.onsafe = onsafe
1727 Is an asynchronous operation safe?
1729 This does not imply that the safe callback has finished.
1731 :returns: True if the operation is safe
1734 ret = rados_aio_is_safe(self.rados_comp)
1737 def is_complete(self):
1739 Has an asynchronous operation completed?
1741 This does not imply that the safe callback has finished.
1743 :returns: True if the operation is completed
1746 ret = rados_aio_is_complete(self.rados_comp)
1749 def wait_for_safe(self):
1751 Wait for an asynchronous operation to be marked safe
1753 This does not imply that the safe callback has finished.
1756 rados_aio_wait_for_safe(self.rados_comp)
1758 def wait_for_complete(self):
1760 Wait for an asynchronous operation to complete
1762 This does not imply that the complete callback has finished.
1765 rados_aio_wait_for_complete(self.rados_comp)
1767 def wait_for_safe_and_cb(self):
1769 Wait for an asynchronous operation to be marked safe and for
1770 the safe callback to have returned
1773 rados_aio_wait_for_safe_and_cb(self.rados_comp)
1775 def wait_for_complete_and_cb(self):
1777 Wait for an asynchronous operation to complete and for the
1778 complete callback to have returned
1780 :returns: whether the operation is completed
1783 ret = rados_aio_wait_for_complete_and_cb(self.rados_comp)
1786 def get_return_value(self):
1788 Get the return value of an asychronous operation
1790 The return value is set when the operation is complete or safe,
1791 whichever comes first.
1793 :returns: int - return value of the operation
1796 ret = rados_aio_get_return_value(self.rados_comp)
1799 def __dealloc__(self):
1801 Release a completion
1803 Call this when you no longer need the completion. It may not be
1804 freed immediately if the operation is not acked and committed.
1806 ref.Py_XDECREF(self.buf)
1808 if self.rados_comp != NULL:
1810 rados_aio_release(self.rados_comp)
1811 self.rados_comp = NULL
1813 def _complete(self):
1814 self.oncomplete(self)
1815 with self.ioctx.lock:
1817 self.ioctx.complete_completions.remove(self)
1821 with self.ioctx.lock:
1823 self.ioctx.safe_completions.remove(self)
1826 with self.ioctx.lock:
1828 self.ioctx.complete_completions.remove(self)
1830 self.ioctx.safe_completions.remove(self)
1833 class OpCtx(object):
1834 def __enter__(self):
1835 return self.create()
1837 def __exit__(self, type, msg, traceback):
1841 cdef class WriteOp(object):
1842 cdef rados_write_op_t write_op
1846 self.write_op = rados_create_write_op()
1851 rados_release_write_op(self.write_op)
1853 @requires(('exclusive', opt(int)))
1854 def new(self, exclusive=None):
1860 int _exclusive = exclusive
1863 rados_write_op_create(self.write_op, _exclusive, NULL)
1871 rados_write_op_remove(self.write_op)
1873 @requires(('flags', int))
1874 def set_flags(self, flags=LIBRADOS_OPERATION_NOFLAG):
1876 Set flags for the last operation added to this write_op.
1877 :para flags: flags to apply to the last operation
1885 rados_write_op_set_flags(self.write_op, _flags)
1887 @requires(('to_write', bytes))
1888 def append(self, to_write):
1890 Append data to an object synchronously
1891 :param to_write: data to write
1892 :type to_write: bytes
1896 char *_to_write = to_write
1897 size_t length = len(to_write)
1900 rados_write_op_append(self.write_op, _to_write, length)
1902 @requires(('to_write', bytes))
1903 def write_full(self, to_write):
1905 Write whole object, atomically replacing it.
1906 :param to_write: data to write
1907 :type to_write: bytes
1911 char *_to_write = to_write
1912 size_t length = len(to_write)
1915 rados_write_op_write_full(self.write_op, _to_write, length)
1917 @requires(('to_write', bytes), ('offset', int))
1918 def write(self, to_write, offset=0):
1921 :param to_write: data to write
1922 :type to_write: bytes
1923 :param offset: byte offset in the object to begin writing at
1928 char *_to_write = to_write
1929 size_t length = len(to_write)
1930 uint64_t _offset = offset
1933 rados_write_op_write(self.write_op, _to_write, length, _offset)
1935 @requires(('offset', int), ('length', int))
1936 def zero(self, offset, length):
1938 Zero part of an object.
1939 :param offset: byte offset in the object to begin writing at
1941 :param offset: number of zero to write
1946 size_t _length = length
1947 uint64_t _offset = offset
1950 rados_write_op_zero(self.write_op, _length, _offset)
1952 @requires(('offset', int))
1953 def truncate(self, offset):
1956 :param offset: byte offset in the object to begin truncating at
1961 uint64_t _offset = offset
1964 rados_write_op_truncate(self.write_op, _offset)
1967 class WriteOpCtx(WriteOp, OpCtx):
1968 """write operation context manager"""
1971 cdef class ReadOp(object):
1972 cdef rados_read_op_t read_op
1976 self.read_op = rados_create_read_op()
1981 rados_release_read_op(self.read_op)
1983 @requires(('flags', int))
1984 def set_flags(self, flags=LIBRADOS_OPERATION_NOFLAG):
1986 Set flags for the last operation added to this read_op.
1987 :para flags: flags to apply to the last operation
1995 rados_read_op_set_flags(self.read_op, _flags)
1998 class ReadOpCtx(ReadOp, OpCtx):
1999 """read operation context manager"""
2002 cdef int __aio_safe_cb(rados_completion_t completion, void *args) with gil:
2004 Callback to onsafe() for asynchronous operations
2006 cdef object cb = <object>args
2011 cdef int __aio_complete_cb(rados_completion_t completion, void *args) with gil:
2013 Callback to oncomplete() for asynchronous operations
2015 cdef object cb = <object>args
2020 cdef class Ioctx(object):
2021 """rados.Ioctx object"""
2022 # NOTE(sileht): attributes declared in .pyd
2024 def __init__(self, name):
2028 self.locator_key = ""
2030 self.lock = threading.Lock()
2031 self.safe_completions = []
2032 self.complete_completions = []
2034 def __enter__(self):
2037 def __exit__(self, type_, value, traceback):
2041 def __dealloc__(self):
2044 def __track_completion(self, completion_obj):
2045 if completion_obj.oncomplete:
2047 self.complete_completions.append(completion_obj)
2048 if completion_obj.onsafe:
2050 self.safe_completions.append(completion_obj)
2052 def __get_completion(self, oncomplete, onsafe):
2054 Constructs a completion to use with asynchronous operations
2056 :param oncomplete: what to do when the write is safe and complete in memory
2058 :type oncomplete: completion
2059 :param onsafe: what to do when the write is safe and complete on storage
2061 :type onsafe: completion
2063 :raises: :class:`Error`
2064 :returns: completion object
2067 completion_obj = Completion(self, oncomplete, onsafe)
2070 rados_callback_t complete_cb = NULL
2071 rados_callback_t safe_cb = NULL
2072 rados_completion_t completion
2073 PyObject* p_completion_obj= <PyObject*>completion_obj
2076 complete_cb = <rados_callback_t>&__aio_complete_cb
2078 safe_cb = <rados_callback_t>&__aio_safe_cb
2081 ret = rados_aio_create_completion(p_completion_obj, complete_cb, safe_cb,
2084 raise make_ex(ret, "error getting a completion")
2086 completion_obj.rados_comp = completion
2087 return completion_obj
2089 @requires(('object_name', str_type), ('oncomplete', opt(Callable)))
2090 def aio_stat(self, object_name, oncomplete):
2092 Asynchronously get object stats (size/mtime)
2094 oncomplete will be called with the returned size and mtime
2095 as well as the completion:
2097 oncomplete(completion, size, mtime)
2099 :param object_name: the name of the object to get stats from
2100 :type object_name: str
2101 :param oncomplete: what to do when the stat is complete
2102 :type oncomplete: completion
2104 :raises: :class:`Error`
2105 :returns: completion object
2108 object_name = cstr(object_name, 'object_name')
2111 Completion completion
2112 char *_object_name = object_name
2116 def oncomplete_(completion_v):
2117 cdef Completion _completion_v = completion_v
2118 return_value = _completion_v.get_return_value()
2119 if return_value >= 0:
2120 return oncomplete(_completion_v, psize, time.localtime(pmtime))
2122 return oncomplete(_completion_v, None, None)
2124 completion = self.__get_completion(oncomplete_, None)
2125 self.__track_completion(completion)
2127 ret = rados_aio_stat(self.io, _object_name, completion.rados_comp,
2131 completion._cleanup()
2132 raise make_ex(ret, "error stating %s" % object_name)
2135 @requires(('object_name', str_type), ('to_write', bytes), ('offset', int),
2136 ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)))
2137 def aio_write(self, object_name, to_write, offset=0,
2138 oncomplete=None, onsafe=None):
2140 Write data to an object asynchronously
2142 Queues the write and returns.
2144 :param object_name: name of the object
2145 :type object_name: str
2146 :param to_write: data to write
2147 :type to_write: bytes
2148 :param offset: byte offset in the object to begin writing at
2150 :param oncomplete: what to do when the write is safe and complete in memory
2152 :type oncomplete: completion
2153 :param onsafe: what to do when the write is safe and complete on storage
2155 :type onsafe: completion
2157 :raises: :class:`Error`
2158 :returns: completion object
2161 object_name = cstr(object_name, 'object_name')
2164 Completion completion
2165 char* _object_name = object_name
2166 char* _to_write = to_write
2167 size_t size = len(to_write)
2168 uint64_t _offset = offset
2170 completion = self.__get_completion(oncomplete, onsafe)
2171 self.__track_completion(completion)
2173 ret = rados_aio_write(self.io, _object_name, completion.rados_comp,
2174 _to_write, size, _offset)
2176 completion._cleanup()
2177 raise make_ex(ret, "error writing object %s" % object_name)
2180 @requires(('object_name', str_type), ('to_write', bytes), ('oncomplete', opt(Callable)),
2181 ('onsafe', opt(Callable)))
2182 def aio_write_full(self, object_name, to_write,
2183 oncomplete=None, onsafe=None):
2185 Asychronously write an entire object
2187 The object is filled with the provided data. If the object exists,
2188 it is atomically truncated and then written.
2189 Queues the write and returns.
2191 :param object_name: name of the object
2192 :type object_name: str
2193 :param to_write: data to write
2195 :param oncomplete: what to do when the write is safe and complete in memory
2197 :type oncomplete: completion
2198 :param onsafe: what to do when the write is safe and complete on storage
2200 :type onsafe: completion
2202 :raises: :class:`Error`
2203 :returns: completion object
2206 object_name = cstr(object_name, 'object_name')
2209 Completion completion
2210 char* _object_name = object_name
2211 char* _to_write = to_write
2212 size_t size = len(to_write)
2214 completion = self.__get_completion(oncomplete, onsafe)
2215 self.__track_completion(completion)
2217 ret = rados_aio_write_full(self.io, _object_name,
2218 completion.rados_comp,
2221 completion._cleanup()
2222 raise make_ex(ret, "error writing object %s" % object_name)
2225 @requires(('object_name', str_type), ('to_append', bytes), ('oncomplete', opt(Callable)),
2226 ('onsafe', opt(Callable)))
2227 def aio_append(self, object_name, to_append, oncomplete=None, onsafe=None):
2229 Asychronously append data to an object
2231 Queues the write and returns.
2233 :param object_name: name of the object
2234 :type object_name: str
2235 :param to_append: data to append
2236 :type to_append: str
2237 :param offset: byte offset in the object to begin writing at
2239 :param oncomplete: what to do when the write is safe and complete in memory
2241 :type oncomplete: completion
2242 :param onsafe: what to do when the write is safe and complete on storage
2244 :type onsafe: completion
2246 :raises: :class:`Error`
2247 :returns: completion object
2249 object_name = cstr(object_name, 'object_name')
2252 Completion completion
2253 char* _object_name = object_name
2254 char* _to_append = to_append
2255 size_t size = len(to_append)
2257 completion = self.__get_completion(oncomplete, onsafe)
2258 self.__track_completion(completion)
2260 ret = rados_aio_append(self.io, _object_name,
2261 completion.rados_comp,
2264 completion._cleanup()
2265 raise make_ex(ret, "error appending object %s" % object_name)
2268 def aio_flush(self):
2270 Block until all pending writes in an io context are safe
2272 :raises: :class:`Error`
2275 ret = rados_aio_flush(self.io)
2277 raise make_ex(ret, "error flushing")
2279 @requires(('object_name', str_type), ('length', int), ('offset', int),
2280 ('oncomplete', opt(Callable)))
2281 def aio_read(self, object_name, length, offset, oncomplete):
2283 Asychronously read data from an object
2285 oncomplete will be called with the returned read value as
2286 well as the completion:
2288 oncomplete(completion, data_read)
2290 :param object_name: name of the object to read from
2291 :type object_name: str
2292 :param length: the number of bytes to read
2294 :param offset: byte offset in the object to begin reading from
2296 :param oncomplete: what to do when the read is complete
2297 :type oncomplete: completion
2299 :raises: :class:`Error`
2300 :returns: completion object
2303 object_name = cstr(object_name, 'object_name')
2306 Completion completion
2307 char* _object_name = object_name
2308 uint64_t _offset = offset
2311 size_t _length = length
2313 def oncomplete_(completion_v):
2314 cdef Completion _completion_v = completion_v
2315 return_value = _completion_v.get_return_value()
2316 if return_value > 0 and return_value != length:
2317 _PyBytes_Resize(&_completion_v.buf, return_value)
2318 return oncomplete(_completion_v, <object>_completion_v.buf if return_value >= 0 else None)
2320 completion = self.__get_completion(oncomplete_, None)
2321 completion.buf = PyBytes_FromStringAndSize(NULL, length)
2322 ret_buf = PyBytes_AsString(completion.buf)
2323 self.__track_completion(completion)
2325 ret = rados_aio_read(self.io, _object_name, completion.rados_comp,
2326 ret_buf, _length, _offset)
2328 completion._cleanup()
2329 raise make_ex(ret, "error reading %s" % object_name)
2332 @requires(('object_name', str_type), ('cls', str_type), ('method', str_type),
2333 ('data', bytes), ('length', int),
2334 ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)))
2335 def aio_execute(self, object_name, cls, method, data,
2336 length=8192, oncomplete=None, onsafe=None):
2338 Asynchronously execute an OSD class method on an object.
2340 oncomplete and onsafe will be called with the data returned from
2341 the plugin as well as the completion:
2343 oncomplete(completion, data)
2344 onsafe(completion, data)
2346 :param object_name: name of the object
2347 :type object_name: str
2348 :param cls: name of the object class
2350 :param method: name of the method
2352 :param data: input data
2354 :param length: size of output buffer in bytes (default=8192)
2356 :param oncomplete: what to do when the execution is complete
2357 :type oncomplete: completion
2358 :param onsafe: what to do when the execution is safe and complete
2359 :type onsafe: completion
2361 :raises: :class:`Error`
2362 :returns: completion object
2365 object_name = cstr(object_name, 'object_name')
2366 cls = cstr(cls, 'cls')
2367 method = cstr(method, 'method')
2369 Completion completion
2370 char *_object_name = object_name
2372 char *_method = method
2374 size_t _data_len = len(data)
2377 size_t _length = length
2379 def oncomplete_(completion_v):
2380 cdef Completion _completion_v = completion_v
2381 return_value = _completion_v.get_return_value()
2382 if return_value > 0 and return_value != length:
2383 _PyBytes_Resize(&_completion_v.buf, return_value)
2384 return oncomplete(_completion_v, <object>_completion_v.buf if return_value >= 0 else None)
2386 def onsafe_(completion_v):
2387 cdef Completion _completion_v = completion_v
2388 return_value = _completion_v.get_return_value()
2389 return onsafe(_completion_v, <object>_completion_v.buf if return_value >= 0 else None)
2391 completion = self.__get_completion(oncomplete_ if oncomplete else None, onsafe_ if onsafe else None)
2392 completion.buf = PyBytes_FromStringAndSize(NULL, length)
2393 ret_buf = PyBytes_AsString(completion.buf)
2394 self.__track_completion(completion)
2396 ret = rados_aio_exec(self.io, _object_name, completion.rados_comp,
2397 _cls, _method, _data, _data_len, ret_buf, _length)
2399 completion._cleanup()
2400 raise make_ex(ret, "error executing %s::%s on %s" % (cls, method, object_name))
2403 @requires(('object_name', str_type), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)))
2404 def aio_remove(self, object_name, oncomplete=None, onsafe=None):
2406 Asychronously remove an object
2408 :param object_name: name of the object to remove
2409 :type object_name: str
2410 :param oncomplete: what to do when the remove is safe and complete in memory
2412 :type oncomplete: completion
2413 :param onsafe: what to do when the remove is safe and complete on storage
2415 :type onsafe: completion
2417 :raises: :class:`Error`
2418 :returns: completion object
2420 object_name = cstr(object_name, 'object_name')
2423 Completion completion
2424 char* _object_name = object_name
2426 completion = self.__get_completion(oncomplete, onsafe)
2427 self.__track_completion(completion)
2429 ret = rados_aio_remove(self.io, _object_name,
2430 completion.rados_comp)
2432 completion._cleanup()
2433 raise make_ex(ret, "error removing %s" % object_name)
2436 def require_ioctx_open(self):
2438 Checks if the rados.Ioctx object state is 'open'
2440 :raises: IoctxStateError
2442 if self.state != "open":
2443 raise IoctxStateError("The pool is %s" % self.state)
2445 def change_auid(self, auid):
2447 Attempt to change an io context's associated auid "owner."
2449 Requires that you have write permission on both the current and new
2452 :raises: :class:`Error`
2454 self.require_ioctx_open()
2457 uint64_t _auid = auid
2460 ret = rados_ioctx_pool_set_auid(self.io, _auid)
2462 raise make_ex(ret, "error changing auid of '%s' to %d"
2463 % (self.name, auid))
2465 @requires(('loc_key', str_type))
2466 def set_locator_key(self, loc_key):
2468 Set the key for mapping objects to pgs within an io context.
2470 The key is used instead of the object name to determine which
2471 placement groups an object is put in. This affects all subsequent
2472 operations of the io context - until a different locator key is
2473 set, all objects in this io context will be placed in the same pg.
2475 :param loc_key: the key to use as the object locator, or NULL to discard
2476 any previously set key
2479 :raises: :class:`TypeError`
2481 self.require_ioctx_open()
2482 cloc_key = cstr(loc_key, 'loc_key')
2483 cdef char *_loc_key = cloc_key
2485 rados_ioctx_locator_set_key(self.io, _loc_key)
2486 self.locator_key = loc_key
2488 def get_locator_key(self):
2490 Get the locator_key of context
2492 :returns: locator_key
2494 return self.locator_key
2496 @requires(('snap_id', long))
2497 def set_read(self, snap_id):
2499 Set the snapshot for reading objects.
2501 To stop to read from snapshot, use set_read(LIBRADOS_SNAP_HEAD)
2503 :param snap_id: the snapshot Id
2506 :raises: :class:`TypeError`
2508 self.require_ioctx_open()
2509 cdef rados_snap_t _snap_id = snap_id
2511 rados_ioctx_snap_set_read(self.io, _snap_id)
2513 @requires(('nspace', str_type))
2514 def set_namespace(self, nspace):
2516 Set the namespace for objects within an io context.
2518 The namespace in addition to the object name fully identifies
2519 an object. This affects all subsequent operations of the io context
2520 - until a different namespace is set, all objects in this io context
2521 will be placed in the same namespace.
2523 :param nspace: the namespace to use, or None/"" for the default namespace
2526 :raises: :class:`TypeError`
2528 self.require_ioctx_open()
2531 cnspace = cstr(nspace, 'nspace')
2532 cdef char *_nspace = cnspace
2534 rados_ioctx_set_namespace(self.io, _nspace)
2535 self.nspace = nspace
2537 def get_namespace(self):
2539 Get the namespace of context
2547 Close a rados.Ioctx object.
2549 This just tells librados that you no longer need to use the io context.
2550 It may not be freed immediately if there are pending asynchronous
2551 requests on it, but you should not use an io context again after
2552 calling this function on it.
2554 if self.state == "open":
2555 self.require_ioctx_open()
2557 rados_ioctx_destroy(self.io)
2558 self.state = "closed"
2561 @requires(('key', str_type), ('data', bytes))
2562 def write(self, key, data, offset=0):
2564 Write data to an object synchronously
2566 :param key: name of the object
2568 :param data: data to write
2570 :param offset: byte offset in the object to begin writing at
2573 :raises: :class:`TypeError`
2574 :raises: :class:`LogicError`
2575 :returns: int - 0 on success
2577 self.require_ioctx_open()
2579 key = cstr(key, 'key')
2583 size_t length = len(data)
2584 uint64_t _offset = offset
2587 ret = rados_write(self.io, _key, _data, length, _offset)
2591 raise make_ex(ret, "Ioctx.write(%s): failed to write %s"
2594 raise LogicError("Ioctx.write(%s): rados_write \
2595 returned %d, but should return zero on success." % (self.name, ret))
2597 @requires(('key', str_type), ('data', bytes))
2598 def write_full(self, key, data):
2600 Write an entire object synchronously.
2602 The object is filled with the provided data. If the object exists,
2603 it is atomically truncated and then written.
2605 :param key: name of the object
2607 :param data: data to write
2610 :raises: :class:`TypeError`
2611 :raises: :class:`Error`
2612 :returns: int - 0 on success
2614 self.require_ioctx_open()
2615 key = cstr(key, 'key')
2619 size_t length = len(data)
2622 ret = rados_write_full(self.io, _key, _data, length)
2626 raise make_ex(ret, "Ioctx.write_full(%s): failed to write %s"
2629 raise LogicError("Ioctx.write_full(%s): rados_write_full \
2630 returned %d, but should return zero on success." % (self.name, ret))
2632 @requires(('key', str_type), ('data', bytes))
2633 def append(self, key, data):
2635 Append data to an object synchronously
2637 :param key: name of the object
2639 :param data: data to write
2642 :raises: :class:`TypeError`
2643 :raises: :class:`LogicError`
2644 :returns: int - 0 on success
2646 self.require_ioctx_open()
2647 key = cstr(key, 'key')
2651 size_t length = len(data)
2654 ret = rados_append(self.io, _key, _data, length)
2658 raise make_ex(ret, "Ioctx.append(%s): failed to append %s"
2661 raise LogicError("Ioctx.append(%s): rados_append \
2662 returned %d, but should return zero on success." % (self.name, ret))
2664 @requires(('key', str_type))
2665 def read(self, key, length=8192, offset=0):
2667 Read data from an object synchronously
2669 :param key: name of the object
2671 :param length: the number of bytes to read (default=8192)
2673 :param offset: byte offset in the object to begin reading at
2676 :raises: :class:`TypeError`
2677 :raises: :class:`Error`
2678 :returns: str - data read from object
2680 self.require_ioctx_open()
2681 key = cstr(key, 'key')
2685 uint64_t _offset = offset
2686 size_t _length = length
2687 PyObject* ret_s = NULL
2689 ret_s = PyBytes_FromStringAndSize(NULL, length)
2691 ret_buf = PyBytes_AsString(ret_s)
2693 ret = rados_read(self.io, _key, ret_buf, _length, _offset)
2695 raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key))
2698 _PyBytes_Resize(&ret_s, ret)
2700 return <object>ret_s
2702 # We DECREF unconditionally: the cast to object above will have
2703 # INCREFed if necessary. This also takes care of exceptions,
2704 # including if _PyString_Resize fails (that will free the string
2705 # itself and set ret_s to NULL, hence XDECREF).
2706 ref.Py_XDECREF(ret_s)
2708 @requires(('key', str_type), ('cls', str_type), ('method', str_type), ('data', bytes))
2709 def execute(self, key, cls, method, data, length=8192):
2711 Execute an OSD class method on an object.
2713 :param key: name of the object
2715 :param cls: name of the object class
2717 :param method: name of the method
2719 :param data: input data
2721 :param length: size of output buffer in bytes (default=8192)
2724 :raises: :class:`TypeError`
2725 :raises: :class:`Error`
2726 :returns: (ret, method output)
2728 self.require_ioctx_open()
2730 key = cstr(key, 'key')
2731 cls = cstr(cls, 'cls')
2732 method = cstr(method, 'method')
2736 char *_method = method
2738 size_t _data_len = len(data)
2741 size_t _length = length
2742 PyObject* ret_s = NULL
2744 ret_s = PyBytes_FromStringAndSize(NULL, length)
2746 ret_buf = PyBytes_AsString(ret_s)
2748 ret = rados_exec(self.io, _key, _cls, _method, _data,
2749 _data_len, ret_buf, _length)
2751 raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key))
2754 _PyBytes_Resize(&ret_s, ret)
2756 return ret, <object>ret_s
2758 # We DECREF unconditionally: the cast to object above will have
2759 # INCREFed if necessary. This also takes care of exceptions,
2760 # including if _PyString_Resize fails (that will free the string
2761 # itself and set ret_s to NULL, hence XDECREF).
2762 ref.Py_XDECREF(ret_s)
2764 def get_stats(self):
2766 Get pool usage statistics
2768 :returns: dict - contains the following keys:
2770 - ``num_bytes`` (int) - size of pool in bytes
2772 - ``num_kb`` (int) - size of pool in kbytes
2774 - ``num_objects`` (int) - number of objects in the pool
2776 - ``num_object_clones`` (int) - number of object clones
2778 - ``num_object_copies`` (int) - number of object copies
2780 - ``num_objects_missing_on_primary`` (int) - number of objets
2783 - ``num_objects_unfound`` (int) - number of unfound objects
2785 - ``num_objects_degraded`` (int) - number of degraded objects
2787 - ``num_rd`` (int) - bytes read
2789 - ``num_rd_kb`` (int) - kbytes read
2791 - ``num_wr`` (int) - bytes written
2793 - ``num_wr_kb`` (int) - kbytes written
2795 self.require_ioctx_open()
2796 cdef rados_pool_stat_t stats
2798 ret = rados_ioctx_pool_stat(self.io, &stats)
2800 raise make_ex(ret, "Ioctx.get_stats(%s): get_stats failed" % self.name)
2801 return {'num_bytes': stats.num_bytes,
2802 'num_kb': stats.num_kb,
2803 'num_objects': stats.num_objects,
2804 'num_object_clones': stats.num_object_clones,
2805 'num_object_copies': stats.num_object_copies,
2806 "num_objects_missing_on_primary": stats.num_objects_missing_on_primary,
2807 "num_objects_unfound": stats.num_objects_unfound,
2808 "num_objects_degraded": stats.num_objects_degraded,
2809 "num_rd": stats.num_rd,
2810 "num_rd_kb": stats.num_rd_kb,
2811 "num_wr": stats.num_wr,
2812 "num_wr_kb": stats.num_wr_kb}
2814 @requires(('key', str_type))
2815 def remove_object(self, key):
2819 This does not delete any snapshots of the object.
2821 :param key: the name of the object to delete
2824 :raises: :class:`TypeError`
2825 :raises: :class:`Error`
2826 :returns: bool - True on success
2828 self.require_ioctx_open()
2829 key = cstr(key, 'key')
2834 ret = rados_remove(self.io, _key)
2836 raise make_ex(ret, "Failed to remove '%s'" % key)
2839 @requires(('key', str_type))
2840 def trunc(self, key, size):
2844 If this enlarges the object, the new area is logically filled with
2845 zeroes. If this shrinks the object, the excess data is removed.
2847 :param key: the name of the object to resize
2849 :param size: the new size of the object in bytes
2852 :raises: :class:`TypeError`
2853 :raises: :class:`Error`
2854 :returns: int - 0 on success, otherwise raises error
2857 self.require_ioctx_open()
2858 key = cstr(key, 'key')
2861 uint64_t _size = size
2864 ret = rados_trunc(self.io, _key, _size)
2866 raise make_ex(ret, "Ioctx.trunc(%s): failed to truncate %s" % (self.name, key))
2869 @requires(('key', str_type))
2870 def stat(self, key):
2872 Get object stats (size/mtime)
2874 :param key: the name of the object to get stats from
2877 :raises: :class:`TypeError`
2878 :raises: :class:`Error`
2879 :returns: (size,timestamp)
2881 self.require_ioctx_open()
2883 key = cstr(key, 'key')
2890 ret = rados_stat(self.io, _key, &psize, &pmtime)
2892 raise make_ex(ret, "Failed to stat %r" % key)
2893 return psize, time.localtime(pmtime)
2895 @requires(('key', str_type), ('xattr_name', str_type))
2896 def get_xattr(self, key, xattr_name):
2898 Get the value of an extended attribute on an object.
2900 :param key: the name of the object to get xattr from
2902 :param xattr_name: which extended attribute to read
2903 :type xattr_name: str
2905 :raises: :class:`TypeError`
2906 :raises: :class:`Error`
2907 :returns: str - value of the xattr
2909 self.require_ioctx_open()
2911 key = cstr(key, 'key')
2912 xattr_name = cstr(xattr_name, 'xattr_name')
2915 char *_xattr_name = xattr_name
2916 size_t ret_length = 4096
2917 char *ret_buf = NULL
2920 while ret_length < 4096 * 1024 * 1024:
2921 ret_buf = <char *>realloc_chk(ret_buf, ret_length)
2923 ret = rados_getxattr(self.io, _key, _xattr_name, ret_buf, ret_length)
2924 if ret == -errno.ERANGE:
2927 raise make_ex(ret, "Failed to get xattr %r" % xattr_name)
2930 return ret_buf[:ret]
2934 @requires(('oid', str_type))
2935 def get_xattrs(self, oid):
2937 Start iterating over xattrs on an object.
2939 :param oid: the name of the object to get xattrs from
2942 :raises: :class:`TypeError`
2943 :raises: :class:`Error`
2944 :returns: XattrIterator
2946 self.require_ioctx_open()
2947 return XattrIterator(self, oid)
2949 @requires(('key', str_type), ('xattr_name', str_type), ('xattr_value', bytes))
2950 def set_xattr(self, key, xattr_name, xattr_value):
2952 Set an extended attribute on an object.
2954 :param key: the name of the object to set xattr to
2956 :param xattr_name: which extended attribute to set
2957 :type xattr_name: str
2958 :param xattr_value: the value of the extended attribute
2959 :type xattr_value: bytes
2961 :raises: :class:`TypeError`
2962 :raises: :class:`Error`
2963 :returns: bool - True on success, otherwise raise an error
2965 self.require_ioctx_open()
2967 key = cstr(key, 'key')
2968 xattr_name = cstr(xattr_name, 'xattr_name')
2971 char *_xattr_name = xattr_name
2972 char *_xattr_value = xattr_value
2973 size_t _xattr_value_len = len(xattr_value)
2976 ret = rados_setxattr(self.io, _key, _xattr_name,
2977 _xattr_value, _xattr_value_len)
2979 raise make_ex(ret, "Failed to set xattr %r" % xattr_name)
2982 @requires(('key', str_type), ('xattr_name', str_type))
2983 def rm_xattr(self, key, xattr_name):
2985 Removes an extended attribute on from an object.
2987 :param key: the name of the object to remove xattr from
2989 :param xattr_name: which extended attribute to remove
2990 :type xattr_name: str
2992 :raises: :class:`TypeError`
2993 :raises: :class:`Error`
2994 :returns: bool - True on success, otherwise raise an error
2996 self.require_ioctx_open()
2998 key = cstr(key, 'key')
2999 xattr_name = cstr(xattr_name, 'xattr_name')
3002 char *_xattr_name = xattr_name
3005 ret = rados_rmxattr(self.io, _key, _xattr_name)
3007 raise make_ex(ret, "Failed to delete key %r xattr %r" %
3011 def list_objects(self):
3013 Get ObjectIterator on rados.Ioctx object.
3015 :returns: ObjectIterator
3017 self.require_ioctx_open()
3018 return ObjectIterator(self)
3020 def list_snaps(self):
3022 Get SnapIterator on rados.Ioctx object.
3024 :returns: SnapIterator
3026 self.require_ioctx_open()
3027 return SnapIterator(self)
3029 @requires(('snap_name', str_type))
3030 def create_snap(self, snap_name):
3032 Create a pool-wide snapshot
3034 :param snap_name: the name of the snapshot
3035 :type snap_name: str
3037 :raises: :class:`TypeError`
3038 :raises: :class:`Error`
3040 self.require_ioctx_open()
3041 snap_name = cstr(snap_name, 'snap_name')
3042 cdef char *_snap_name = snap_name
3045 ret = rados_ioctx_snap_create(self.io, _snap_name)
3047 raise make_ex(ret, "Failed to create snap %s" % snap_name)
3049 @requires(('snap_name', str_type))
3050 def remove_snap(self, snap_name):
3052 Removes a pool-wide snapshot
3054 :param snap_name: the name of the snapshot
3055 :type snap_name: str
3057 :raises: :class:`TypeError`
3058 :raises: :class:`Error`
3060 self.require_ioctx_open()
3061 snap_name = cstr(snap_name, 'snap_name')
3062 cdef char *_snap_name = snap_name
3065 ret = rados_ioctx_snap_remove(self.io, _snap_name)
3067 raise make_ex(ret, "Failed to remove snap %s" % snap_name)
3069 @requires(('snap_name', str_type))
3070 def lookup_snap(self, snap_name):
3072 Get the id of a pool snapshot
3074 :param snap_name: the name of the snapshot to lookop
3075 :type snap_name: str
3077 :raises: :class:`TypeError`
3078 :raises: :class:`Error`
3079 :returns: Snap - on success
3081 self.require_ioctx_open()
3082 csnap_name = cstr(snap_name, 'snap_name')
3084 char *_snap_name = csnap_name
3085 rados_snap_t snap_id
3088 ret = rados_ioctx_snap_lookup(self.io, _snap_name, &snap_id)
3090 raise make_ex(ret, "Failed to lookup snap %s" % snap_name)
3091 return Snap(self, snap_name, int(snap_id))
3093 @requires(('oid', str_type), ('snap_name', str_type))
3094 def snap_rollback(self, oid, snap_name):
3096 Rollback an object to a snapshot
3098 :param oid: the name of the object
3100 :param snap_name: the name of the snapshot
3101 :type snap_name: str
3103 :raises: :class:`TypeError`
3104 :raises: :class:`Error`
3106 self.require_ioctx_open()
3107 oid = cstr(oid, 'oid')
3108 snap_name = cstr(snap_name, 'snap_name')
3110 char *_snap_name = snap_name
3114 ret = rados_ioctx_snap_rollback(self.io, _oid, _snap_name)
3116 raise make_ex(ret, "Failed to rollback %s" % oid)
3118 def get_last_version(self):
3120 Return the version of the last object read or written to.
3122 This exposes the internal version number of the last object read or
3123 written via this io context
3125 :returns: version of the last object used
3127 self.require_ioctx_open()
3129 ret = rados_get_last_version(self.io)
3132 def create_write_op(self):
3134 create write operation object.
3135 need call release_write_op after use
3137 return WriteOp().create()
3139 def create_read_op(self):
3141 create read operation object.
3142 need call release_read_op after use
3144 return ReadOp().create()
3146 def release_write_op(self, write_op):
3148 release memory alloc by create_write_op
3152 def release_read_op(self, read_op):
3154 release memory alloc by create_read_op
3155 :para read_op: read_op object
3160 @requires(('write_op', WriteOp), ('keys', tuple), ('values', tuple))
3161 def set_omap(self, write_op, keys, values):
3163 set keys values to write_op
3164 :para write_op: write_operation object
3165 :type write_op: WriteOp
3166 :para keys: a tuple of keys
3168 :para values: a tuple of values
3172 if len(keys) != len(values):
3173 raise Error("Rados(): keys and values must have the same number of items")
3175 keys = cstr_list(keys, 'keys')
3177 WriteOp _write_op = write_op
3178 size_t key_num = len(keys)
3179 char **_keys = to_bytes_array(keys)
3180 char **_values = to_bytes_array(values)
3181 size_t *_lens = to_csize_t_array([len(v) for v in values])
3185 rados_write_op_omap_set(_write_op.write_op,
3186 <const char**>_keys,
3187 <const char**>_values,
3188 <const size_t*>_lens, key_num)
3194 @requires(('write_op', WriteOp), ('oid', str_type), ('mtime', opt(int)), ('flags', opt(int)))
3195 def operate_write_op(self, write_op, oid, mtime=0, flags=LIBRADOS_OPERATION_NOFLAG):
3197 excute the real write operation
3198 :para write_op: write operation object
3199 :type write_op: WriteOp
3200 :para oid: object name
3202 :para mtime: the time to set the mtime to, 0 for the current time
3204 :para flags: flags to apply to the entire operation
3208 oid = cstr(oid, 'oid')
3210 WriteOp _write_op = write_op
3212 time_t _mtime = mtime
3216 ret = rados_write_op_operate(_write_op.write_op, self.io, _oid, &_mtime, _flags)
3218 raise make_ex(ret, "Failed to operate write op for oid %s" % oid)
3220 @requires(('write_op', WriteOp), ('oid', str_type), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)), ('mtime', opt(int)), ('flags', opt(int)))
3221 def operate_aio_write_op(self, write_op, oid, oncomplete=None, onsafe=None, mtime=0, flags=LIBRADOS_OPERATION_NOFLAG):
3223 excute the real write operation asynchronously
3224 :para write_op: write operation object
3225 :type write_op: WriteOp
3226 :para oid: object name
3228 :param oncomplete: what to do when the remove is safe and complete in memory
3230 :type oncomplete: completion
3231 :param onsafe: what to do when the remove is safe and complete on storage
3233 :type onsafe: completion
3234 :para mtime: the time to set the mtime to, 0 for the current time
3236 :para flags: flags to apply to the entire operation
3239 :raises: :class:`Error`
3240 :returns: completion object
3243 oid = cstr(oid, 'oid')
3245 WriteOp _write_op = write_op
3247 Completion completion
3248 time_t _mtime = mtime
3251 completion = self.__get_completion(oncomplete, onsafe)
3252 self.__track_completion(completion)
3255 ret = rados_aio_write_op_operate(_write_op.write_op, self.io, completion.rados_comp, _oid,
3258 completion._cleanup()
3259 raise make_ex(ret, "Failed to operate aio write op for oid %s" % oid)
3262 @requires(('read_op', ReadOp), ('oid', str_type), ('flag', opt(int)))
3263 def operate_read_op(self, read_op, oid, flag=LIBRADOS_OPERATION_NOFLAG):
3265 excute the real read operation
3266 :para read_op: read operation object
3267 :type read_op: ReadOp
3268 :para oid: object name
3270 :para flag: flags to apply to the entire operation
3273 oid = cstr(oid, 'oid')
3275 ReadOp _read_op = read_op
3280 ret = rados_read_op_operate(_read_op.read_op, self.io, _oid, _flag)
3282 raise make_ex(ret, "Failed to operate read op for oid %s" % oid)
3284 @requires(('read_op', ReadOp), ('oid', str_type), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)), ('flag', opt(int)))
3285 def operate_aio_read_op(self, read_op, oid, oncomplete=None, onsafe=None, flag=LIBRADOS_OPERATION_NOFLAG):
3287 excute the real read operation
3288 :para read_op: read operation object
3289 :type read_op: ReadOp
3290 :para oid: object name
3292 :param oncomplete: what to do when the remove is safe and complete in memory
3294 :type oncomplete: completion
3295 :param onsafe: what to do when the remove is safe and complete on storage
3297 :type onsafe: completion
3298 :para flag: flags to apply to the entire operation
3301 oid = cstr(oid, 'oid')
3303 ReadOp _read_op = read_op
3305 Completion completion
3308 completion = self.__get_completion(oncomplete, onsafe)
3309 self.__track_completion(completion)
3312 ret = rados_aio_read_op_operate(_read_op.read_op, self.io, completion.rados_comp, _oid, _flag)
3314 completion._cleanup()
3315 raise make_ex(ret, "Failed to operate aio read op for oid %s" % oid)
3318 @requires(('read_op', ReadOp), ('start_after', str_type), ('filter_prefix', str_type), ('max_return', int))
3319 def get_omap_vals(self, read_op, start_after, filter_prefix, max_return):
3322 :para read_op: read operation object
3323 :type read_op: ReadOp
3324 :para start_after: list keys starting after start_after
3325 :type start_after: str
3326 :para filter_prefix: list only keys beginning with filter_prefix
3327 :type filter_prefix: str
3328 :para max_return: list no more than max_return key/value pairs
3329 :type max_return: int
3330 :returns: an iterator over the requested omap values, return value from this action
3333 start_after = cstr(start_after, 'start_after') if start_after else None
3334 filter_prefix = cstr(filter_prefix, 'filter_prefix') if filter_prefix else None
3336 char *_start_after = opt_str(start_after)
3337 char *_filter_prefix = opt_str(filter_prefix)
3338 ReadOp _read_op = read_op
3339 rados_omap_iter_t iter_addr = NULL
3340 int _max_return = max_return
3344 rados_read_op_omap_get_vals2(_read_op.read_op, _start_after, _filter_prefix,
3345 _max_return, &iter_addr, NULL, &prval)
3346 it = OmapIterator(self)
3348 return it, int(prval)
3350 @requires(('read_op', ReadOp), ('start_after', str_type), ('max_return', int))
3351 def get_omap_keys(self, read_op, start_after, max_return):
3354 :para read_op: read operation object
3355 :type read_op: ReadOp
3356 :para start_after: list keys starting after start_after
3357 :type start_after: str
3358 :para max_return: list no more than max_return key/value pairs
3359 :type max_return: int
3360 :returns: an iterator over the requested omap values, return value from this action
3362 start_after = cstr(start_after, 'start_after') if start_after else None
3364 char *_start_after = opt_str(start_after)
3365 ReadOp _read_op = read_op
3366 rados_omap_iter_t iter_addr = NULL
3367 int _max_return = max_return
3371 rados_read_op_omap_get_keys2(_read_op.read_op, _start_after,
3372 _max_return, &iter_addr, NULL, &prval)
3373 it = OmapIterator(self)
3375 return it, int(prval)
3377 @requires(('read_op', ReadOp), ('keys', tuple))
3378 def get_omap_vals_by_keys(self, read_op, keys):
3380 get the omap values by keys
3381 :para read_op: read operation object
3382 :type read_op: ReadOp
3383 :para keys: input key tuple
3385 :returns: an iterator over the requested omap values, return value from this action
3387 keys = cstr_list(keys, 'keys')
3389 ReadOp _read_op = read_op
3390 rados_omap_iter_t iter_addr
3391 char **_keys = to_bytes_array(keys)
3392 size_t key_num = len(keys)
3397 rados_read_op_omap_get_vals_by_keys(_read_op.read_op,
3398 <const char**>_keys,
3399 key_num, &iter_addr, &prval)
3400 it = OmapIterator(self)
3402 return it, int(prval)
3406 @requires(('write_op', WriteOp), ('keys', tuple))
3407 def remove_omap_keys(self, write_op, keys):
3409 remove omap keys specifiled
3410 :para write_op: write operation object
3411 :type write_op: WriteOp
3412 :para keys: input key tuple
3416 keys = cstr_list(keys, 'keys')
3418 WriteOp _write_op = write_op
3419 size_t key_num = len(keys)
3420 char **_keys = to_bytes_array(keys)
3424 rados_write_op_omap_rm_keys(_write_op.write_op, <const char**>_keys, key_num)
3428 @requires(('write_op', WriteOp))
3429 def clear_omap(self, write_op):
3431 Remove all key/value pairs from an object
3432 :para write_op: write operation object
3433 :type write_op: WriteOp
3437 WriteOp _write_op = write_op
3440 rados_write_op_omap_clear(_write_op.write_op)
3442 @requires(('key', str_type), ('name', str_type), ('cookie', str_type), ('desc', str_type),
3443 ('duration', opt(int)), ('flags', int))
3444 def lock_exclusive(self, key, name, cookie, desc="", duration=None, flags=0):
3447 Take an exclusive lock on an object
3449 :param key: name of the object
3451 :param name: name of the lock
3453 :param cookie: cookie of the lock
3455 :param desc: description of the lock
3457 :param duration: duration of the lock in seconds
3462 :raises: :class:`TypeError`
3463 :raises: :class:`Error`
3465 self.require_ioctx_open()
3467 key = cstr(key, 'key')
3468 name = cstr(name, 'name')
3469 cookie = cstr(cookie, 'cookie')
3470 desc = cstr(desc, 'desc')
3475 char* _cookie = cookie
3477 uint8_t _flags = flags
3480 if duration is None:
3482 ret = rados_lock_exclusive(self.io, _key, _name, _cookie, _desc,
3485 _duration.tv_sec = duration
3487 ret = rados_lock_exclusive(self.io, _key, _name, _cookie, _desc,
3491 raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key))
3493 @requires(('key', str_type), ('name', str_type), ('cookie', str_type), ('tag', str_type),
3494 ('desc', str_type), ('duration', opt(int)), ('flags', int))
3495 def lock_shared(self, key, name, cookie, tag, desc="", duration=None, flags=0):
3498 Take a shared lock on an object
3500 :param key: name of the object
3502 :param name: name of the lock
3504 :param cookie: cookie of the lock
3506 :param tag: tag of the lock
3508 :param desc: description of the lock
3510 :param duration: duration of the lock in seconds
3515 :raises: :class:`TypeError`
3516 :raises: :class:`Error`
3518 self.require_ioctx_open()
3520 key = cstr(key, 'key')
3521 tag = cstr(tag, 'tag')
3522 name = cstr(name, 'name')
3523 cookie = cstr(cookie, 'cookie')
3524 desc = cstr(desc, 'desc')
3530 char* _cookie = cookie
3532 uint8_t _flags = flags
3535 if duration is None:
3537 ret = rados_lock_shared(self.io, _key, _name, _cookie, _tag, _desc,
3540 _duration.tv_sec = duration
3542 ret = rados_lock_shared(self.io, _key, _name, _cookie, _tag, _desc,
3545 raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key))
3547 @requires(('key', str_type), ('name', str_type), ('cookie', str_type))
3548 def unlock(self, key, name, cookie):
3551 Release a shared or exclusive lock on an object
3553 :param key: name of the object
3555 :param name: name of the lock
3557 :param cookie: cookie of the lock
3560 :raises: :class:`TypeError`
3561 :raises: :class:`Error`
3563 self.require_ioctx_open()
3565 key = cstr(key, 'key')
3566 name = cstr(name, 'name')
3567 cookie = cstr(cookie, 'cookie')
3572 char* _cookie = cookie
3575 ret = rados_unlock(self.io, _key, _name, _cookie)
3577 raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key))
3579 def application_enable(self, app_name, force=False):
3581 Enable an application on an OSD pool
3583 :param app_name: application name
3585 :param force: False if only a single app should exist per pool
3586 :type expire_seconds: boool
3588 :raises: :class:`Error`
3590 app_name = cstr(app_name, 'app_name')
3592 char *_app_name = app_name
3593 int _force = (1 if force else 0)
3596 ret = rados_application_enable(self.io, _app_name, _force)
3598 raise make_ex(ret, "error enabling application")
3600 def application_list(self):
3602 Returns a list of enabled applications
3604 :returns: list of app name string
3612 apps = <char *>realloc_chk(apps, length)
3614 ret = rados_application_list(self.io, apps, &length)
3616 return [decode_cstr(app) for app in
3617 apps[:length].split(b'\0') if app]
3618 elif ret == -errno.ENOENT:
3620 elif ret == -errno.ERANGE:
3623 raise make_ex(ret, "error listing applications")
3627 def application_metadata_set(self, app_name, key, value):
3629 Sets application metadata on an OSD pool
3631 :param app_name: application name
3633 :param key: metadata key
3635 :param value: metadata value
3638 :raises: :class:`Error`
3640 app_name = cstr(app_name, 'app_name')
3641 key = cstr(key, 'key')
3642 value = cstr(value, 'value')
3644 char *_app_name = app_name
3646 char *_value = value
3649 ret = rados_application_metadata_set(self.io, _app_name, _key,
3652 raise make_ex(ret, "error setting application metadata")
3654 def application_metadata_remove(self, app_name, key):
3656 Remove application metadata from an OSD pool
3658 :param app_name: application name
3660 :param key: metadata key
3663 :raises: :class:`Error`
3665 app_name = cstr(app_name, 'app_name')
3666 key = cstr(key, 'key')
3668 char *_app_name = app_name
3672 ret = rados_application_metadata_remove(self.io, _app_name, _key)
3674 raise make_ex(ret, "error removing application metadata")
3676 def application_metadata_list(self, app_name):
3678 Returns a list of enabled applications
3680 :param app_name: application name
3682 :returns: list of key/value tuples
3684 app_name = cstr(app_name, 'app_name')
3686 char *_app_name = app_name
3687 size_t key_length = 128
3688 size_t val_length = 128
3694 c_keys = <char *>realloc_chk(c_keys, key_length)
3695 c_vals = <char *>realloc_chk(c_vals, val_length)
3697 ret = rados_application_metadata_list(self.io, _app_name,
3698 c_keys, &key_length,
3699 c_vals, &val_length)
3701 keys = [decode_cstr(key) for key in
3702 c_keys[:key_length].split(b'\0') if key]
3703 vals = [decode_cstr(val) for val in
3704 c_vals[:val_length].split(b'\0') if val]
3705 return zip(keys, vals)
3706 elif ret == -errno.ERANGE:
3709 raise make_ex(ret, "error listing application metadata")
3715 def set_object_locator(func):
3716 def retfunc(self, *args, **kwargs):
3717 if self.locator_key is not None:
3718 old_locator = self.ioctx.get_locator_key()
3719 self.ioctx.set_locator_key(self.locator_key)
3720 retval = func(self, *args, **kwargs)
3721 self.ioctx.set_locator_key(old_locator)
3724 return func(self, *args, **kwargs)
3728 def set_object_namespace(func):
3729 def retfunc(self, *args, **kwargs):
3730 if self.nspace is None:
3731 raise LogicError("Namespace not set properly in context")
3732 old_nspace = self.ioctx.get_namespace()
3733 self.ioctx.set_namespace(self.nspace)
3734 retval = func(self, *args, **kwargs)
3735 self.ioctx.set_namespace(old_nspace)
3740 class Object(object):
3741 """Rados object wrapper, makes the object look like a file"""
3742 def __init__(self, ioctx, key, locator_key=None, nspace=None):
3746 self.state = "exists"
3747 self.locator_key = locator_key
3748 self.nspace = "" if nspace is None else nspace
3751 return "rados.Object(ioctx=%s,key=%s,nspace=%s,locator=%s)" % \
3752 (str(self.ioctx), self.key, "--default--"
3753 if self.nspace is "" else self.nspace, self.locator_key)
3755 def require_object_exists(self):
3756 if self.state != "exists":
3757 raise ObjectStateError("The object is %s" % self.state)
3760 @set_object_namespace
3761 def read(self, length=1024 * 1024):
3762 self.require_object_exists()
3763 ret = self.ioctx.read(self.key, length, self.offset)
3764 self.offset += len(ret)
3768 @set_object_namespace
3769 def write(self, string_to_write):
3770 self.require_object_exists()
3771 ret = self.ioctx.write(self.key, string_to_write, self.offset)
3773 self.offset += len(string_to_write)
3777 @set_object_namespace
3779 self.require_object_exists()
3780 self.ioctx.remove_object(self.key)
3781 self.state = "removed"
3784 @set_object_namespace
3786 self.require_object_exists()
3787 return self.ioctx.stat(self.key)
3789 def seek(self, position):
3790 self.require_object_exists()
3791 self.offset = position
3794 @set_object_namespace
3795 def get_xattr(self, xattr_name):
3796 self.require_object_exists()
3797 return self.ioctx.get_xattr(self.key, xattr_name)
3800 @set_object_namespace
3801 def get_xattrs(self):
3802 self.require_object_exists()
3803 return self.ioctx.get_xattrs(self.key)
3806 @set_object_namespace
3807 def set_xattr(self, xattr_name, xattr_value):
3808 self.require_object_exists()
3809 return self.ioctx.set_xattr(self.key, xattr_name, xattr_value)
3812 @set_object_namespace
3813 def rm_xattr(self, xattr_name):
3814 self.require_object_exists()
3815 return self.ioctx.rm_xattr(self.key, xattr_name)
3826 class MonitorLog(object):
3827 # NOTE(sileht): Keep this class for backward compat
3828 # method moved to Rados.monitor_log()
3830 For watching cluster log messages. Instantiate an object and keep
3831 it around while callback is periodically called. Construct with
3832 'level' to monitor 'level' messages (one of MONITOR_LEVELS).
3833 arg will be passed to the callback.
3835 callback will be called with:
3836 arg (given to __init__)
3837 line (the full line, including timestamp, who, level, msg)
3838 who (which entity issued the log message)
3839 timestamp_sec (sec of a struct timespec)
3840 timestamp_nsec (sec of a struct timespec)
3841 seq (sequence number)
3842 level (string representing the level of the log message)
3843 msg (the message itself)
3844 callback's return value is ignored
3846 def __init__(self, cluster, level, callback, arg):
3848 self.callback = callback
3850 self.cluster = cluster
3851 self.cluster.monitor_log(level, callback, arg)