1 # cython: embedsignature=True
3 This module is a thin wrapper around librados.
5 Error codes from librados are turned into exceptions that subclass
6 :class:`Error`. Almost all methods may raise :class:`Error(the base class of all rados exceptions), :class:`PermissionError`
7 (the base class of all rados exceptions), :class:`PermissionError`
8 and :class:`IOError`, in addition to those documented for the
11 # Copyright 2011 Josh Durgin
12 # Copyright 2011, Hannu Valtonen <hannu.valtonen@ormod.com>
13 # Copyright 2015 Hector Martin <marcan@marcan.st>
14 # Copyright 2016 Mehdi Abaakouk <sileht@redhat.com>
16 from cpython cimport PyObject, ref
17 from cpython.pycapsule cimport *
18 from libc cimport errno
19 from libc.stdint cimport *
20 from libc.stdlib cimport malloc, realloc, free
27 from collections.abc import Callable
29 from collections import Callable
30 from datetime import datetime
31 from functools import partial, wraps
32 from itertools import chain
34 # Are we running Python 2.x
35 if sys.version_info[0] < 3:
41 cdef extern from "Python.h":
42 # These are in cpython/string.pxd, but use "object" types instead of
43 # PyObject*, which invokes assumptions in cpython that we need to
44 # legitimately break to implement zero-copy string buffers in Ioctx.read().
45 # This is valid use of the Python API and documented as a special case.
46 PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL
47 char* PyBytes_AsString(PyObject *string) except NULL
48 int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1
49 void PyEval_InitThreads()
52 cdef extern from "time.h":
53 ctypedef long int time_t
54 ctypedef long int suseconds_t
57 cdef extern from "sys/time.h":
63 cdef extern from "rados/rados_types.h" nogil:
64 cdef char* _LIBRADOS_ALL_NSPACES "LIBRADOS_ALL_NSPACES"
67 cdef extern from "rados/librados.h" nogil:
69 _LIBRADOS_OP_FLAG_EXCL "LIBRADOS_OP_FLAG_EXCL"
70 _LIBRADOS_OP_FLAG_FAILOK "LIBRADOS_OP_FLAG_FAILOK"
71 _LIBRADOS_OP_FLAG_FADVISE_RANDOM "LIBRADOS_OP_FLAG_FADVISE_RANDOM"
72 _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL "LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL"
73 _LIBRADOS_OP_FLAG_FADVISE_WILLNEED "LIBRADOS_OP_FLAG_FADVISE_WILLNEED"
74 _LIBRADOS_OP_FLAG_FADVISE_DONTNEED "LIBRADOS_OP_FLAG_FADVISE_DONTNEED"
75 _LIBRADOS_OP_FLAG_FADVISE_NOCACHE "LIBRADOS_OP_FLAG_FADVISE_NOCACHE"
79 _LIBRADOS_OPERATION_NOFLAG "LIBRADOS_OPERATION_NOFLAG"
80 _LIBRADOS_OPERATION_BALANCE_READS "LIBRADOS_OPERATION_BALANCE_READS"
81 _LIBRADOS_OPERATION_LOCALIZE_READS "LIBRADOS_OPERATION_LOCALIZE_READS"
82 _LIBRADOS_OPERATION_ORDER_READS_WRITES "LIBRADOS_OPERATION_ORDER_READS_WRITES"
83 _LIBRADOS_OPERATION_IGNORE_CACHE "LIBRADOS_OPERATION_IGNORE_CACHE"
84 _LIBRADOS_OPERATION_SKIPRWLOCKS "LIBRADOS_OPERATION_SKIPRWLOCKS"
85 _LIBRADOS_OPERATION_IGNORE_OVERLAY "LIBRADOS_OPERATION_IGNORE_OVERLAY"
86 _LIBRADOS_CREATE_EXCLUSIVE "LIBRADOS_CREATE_EXCLUSIVE"
87 _LIBRADOS_CREATE_IDEMPOTENT "LIBRADOS_CREATE_IDEMPOTENT"
89 cdef uint64_t _LIBRADOS_SNAP_HEAD "LIBRADOS_SNAP_HEAD"
91 ctypedef void* rados_xattrs_iter_t
92 ctypedef void* rados_omap_iter_t
93 ctypedef void* rados_list_ctx_t
94 ctypedef uint64_t rados_snap_t
95 ctypedef void *rados_write_op_t
96 ctypedef void *rados_read_op_t
97 ctypedef void *rados_completion_t
98 ctypedef void (*rados_callback_t)(rados_completion_t cb, void *arg)
99 ctypedef void (*rados_log_callback_t)(void *arg, const char *line, const char *who,
100 uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg)
101 ctypedef void (*rados_log_callback2_t)(void *arg, const char *line, const char *channel, const char *who, const char *name,
102 uint64_t sec, uint64_t nsec, uint64_t seq, const char *level, const char *msg)
105 cdef struct rados_cluster_stat_t:
111 cdef struct rados_pool_stat_t:
115 uint64_t num_object_clones
116 uint64_t num_object_copies
117 uint64_t num_objects_missing_on_primary
118 uint64_t num_objects_unfound
119 uint64_t num_objects_degraded
125 void rados_buffer_free(char *buf)
127 void rados_version(int *major, int *minor, int *extra)
128 int rados_create2(rados_t *pcluster, const char *const clustername,
129 const char * const name, uint64_t flags)
130 int rados_create_with_context(rados_t *cluster, rados_config_t cct)
131 int rados_connect(rados_t cluster)
132 void rados_shutdown(rados_t cluster)
133 uint64_t rados_get_instance_id(rados_t cluster)
134 int rados_conf_read_file(rados_t cluster, const char *path)
135 int rados_conf_parse_argv_remainder(rados_t cluster, int argc, const char **argv, const char **remargv)
136 int rados_conf_parse_env(rados_t cluster, const char *var)
137 int rados_conf_set(rados_t cluster, char *option, const char *value)
138 int rados_conf_get(rados_t cluster, char *option, char *buf, size_t len)
140 int rados_ioctx_pool_stat(rados_ioctx_t io, rados_pool_stat_t *stats)
141 int64_t rados_pool_lookup(rados_t cluster, const char *pool_name)
142 int rados_pool_reverse_lookup(rados_t cluster, int64_t id, char *buf, size_t maxlen)
143 int rados_pool_create(rados_t cluster, const char *pool_name)
144 int rados_pool_create_with_crush_rule(rados_t cluster, const char *pool_name, uint8_t crush_rule_num)
145 int rados_pool_get_base_tier(rados_t cluster, int64_t pool, int64_t *base_tier)
146 int rados_pool_list(rados_t cluster, char *buf, size_t len)
147 int rados_pool_delete(rados_t cluster, const char *pool_name)
148 int rados_inconsistent_pg_list(rados_t cluster, int64_t pool, char *buf, size_t len)
150 int rados_cluster_stat(rados_t cluster, rados_cluster_stat_t *result)
151 int rados_cluster_fsid(rados_t cluster, char *buf, size_t len)
152 int rados_blacklist_add(rados_t cluster, char *client_address, uint32_t expire_seconds)
153 int rados_application_enable(rados_ioctx_t io, const char *app_name,
155 void rados_set_osdmap_full_try(rados_ioctx_t io)
156 void rados_unset_osdmap_full_try(rados_ioctx_t io)
157 int rados_application_list(rados_ioctx_t io, char *values,
159 int rados_application_metadata_get(rados_ioctx_t io, const char *app_name,
160 const char *key, char *value,
162 int rados_application_metadata_set(rados_ioctx_t io, const char *app_name,
163 const char *key, const char *value)
164 int rados_application_metadata_remove(rados_ioctx_t io,
165 const char *app_name, const char *key)
166 int rados_application_metadata_list(rados_ioctx_t io,
167 const char *app_name, char *keys,
168 size_t *key_len, char *values,
170 int rados_ping_monitor(rados_t cluster, const char *mon_id, char **outstr, size_t *outstrlen)
171 int rados_mon_command(rados_t cluster, const char **cmd, size_t cmdlen,
172 const char *inbuf, size_t inbuflen,
173 char **outbuf, size_t *outbuflen,
174 char **outs, size_t *outslen)
175 int rados_mgr_command(rados_t cluster, const char **cmd, size_t cmdlen,
176 const char *inbuf, size_t inbuflen,
177 char **outbuf, size_t *outbuflen,
178 char **outs, size_t *outslen)
179 int rados_mon_command_target(rados_t cluster, const char *name, const char **cmd, size_t cmdlen,
180 const char *inbuf, size_t inbuflen,
181 char **outbuf, size_t *outbuflen,
182 char **outs, size_t *outslen)
183 int rados_osd_command(rados_t cluster, int osdid, const char **cmd, size_t cmdlen,
184 const char *inbuf, size_t inbuflen,
185 char **outbuf, size_t *outbuflen,
186 char **outs, size_t *outslen)
187 int rados_pg_command(rados_t cluster, const char *pgstr, const char **cmd, size_t cmdlen,
188 const char *inbuf, size_t inbuflen,
189 char **outbuf, size_t *outbuflen,
190 char **outs, size_t *outslen)
191 int rados_monitor_log(rados_t cluster, const char *level, rados_log_callback_t cb, void *arg)
192 int rados_monitor_log2(rados_t cluster, const char *level, rados_log_callback2_t cb, void *arg)
194 int rados_wait_for_latest_osdmap(rados_t cluster)
196 int rados_service_register(rados_t cluster, const char *service, const char *daemon, const char *metadata_dict)
197 int rados_service_update_status(rados_t cluster, const char *status_dict)
199 int rados_ioctx_create(rados_t cluster, const char *pool_name, rados_ioctx_t *ioctx)
200 int rados_ioctx_create2(rados_t cluster, int64_t pool_id, rados_ioctx_t *ioctx)
201 void rados_ioctx_destroy(rados_ioctx_t io)
202 void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key)
203 void rados_ioctx_set_namespace(rados_ioctx_t io, const char * nspace)
205 uint64_t rados_get_last_version(rados_ioctx_t io)
206 int rados_stat(rados_ioctx_t io, const char *o, uint64_t *psize, time_t *pmtime)
207 int rados_write(rados_ioctx_t io, const char *oid, const char *buf, size_t len, uint64_t off)
208 int rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len)
209 int rados_append(rados_ioctx_t io, const char *oid, const char *buf, size_t len)
210 int rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off)
211 int rados_remove(rados_ioctx_t io, const char *oid)
212 int rados_trunc(rados_ioctx_t io, const char *oid, uint64_t size)
213 int rados_getxattr(rados_ioctx_t io, const char *o, const char *name, char *buf, size_t len)
214 int rados_setxattr(rados_ioctx_t io, const char *o, const char *name, const char *buf, size_t len)
215 int rados_rmxattr(rados_ioctx_t io, const char *o, const char *name)
216 int rados_getxattrs(rados_ioctx_t io, const char *oid, rados_xattrs_iter_t *iter)
217 int rados_getxattrs_next(rados_xattrs_iter_t iter, const char **name, const char **val, size_t *len)
218 void rados_getxattrs_end(rados_xattrs_iter_t iter)
220 int rados_nobjects_list_open(rados_ioctx_t io, rados_list_ctx_t *ctx)
221 int rados_nobjects_list_next(rados_list_ctx_t ctx, const char **entry, const char **key, const char **nspace)
222 void rados_nobjects_list_close(rados_list_ctx_t ctx)
224 int rados_ioctx_pool_requires_alignment2(rados_ioctx_t io, int * requires)
225 int rados_ioctx_pool_required_alignment2(rados_ioctx_t io, uint64_t * alignment)
227 int rados_ioctx_snap_rollback(rados_ioctx_t io, const char * oid, const char * snapname)
228 int rados_ioctx_snap_create(rados_ioctx_t io, const char * snapname)
229 int rados_ioctx_snap_remove(rados_ioctx_t io, const char * snapname)
230 int rados_ioctx_snap_lookup(rados_ioctx_t io, const char * name, rados_snap_t * id)
231 int rados_ioctx_snap_get_name(rados_ioctx_t io, rados_snap_t id, char * name, int maxlen)
232 void rados_ioctx_snap_set_read(rados_ioctx_t io, rados_snap_t snap)
233 int rados_ioctx_snap_list(rados_ioctx_t io, rados_snap_t * snaps, int maxlen)
234 int rados_ioctx_snap_get_stamp(rados_ioctx_t io, rados_snap_t id, time_t * t)
236 int rados_ioctx_selfmanaged_snap_create(rados_ioctx_t io,
237 rados_snap_t *snapid)
238 int rados_ioctx_selfmanaged_snap_remove(rados_ioctx_t io,
240 int rados_ioctx_selfmanaged_snap_set_write_ctx(rados_ioctx_t io,
241 rados_snap_t snap_seq,
244 int rados_ioctx_selfmanaged_snap_rollback(rados_ioctx_t io, const char *oid,
247 int rados_lock_exclusive(rados_ioctx_t io, const char * oid, const char * name,
248 const char * cookie, const char * desc,
249 timeval * duration, uint8_t flags)
250 int rados_lock_shared(rados_ioctx_t io, const char * o, const char * name,
251 const char * cookie, const char * tag, const char * desc,
252 timeval * duration, uint8_t flags)
253 int rados_unlock(rados_ioctx_t io, const char * o, const char * name, const char * cookie)
255 rados_write_op_t rados_create_write_op()
256 void rados_release_write_op(rados_write_op_t write_op)
258 rados_read_op_t rados_create_read_op()
259 void rados_release_read_op(rados_read_op_t read_op)
261 int rados_aio_create_completion(void * cb_arg, rados_callback_t cb_complete, rados_callback_t cb_safe, rados_completion_t * pc)
262 void rados_aio_release(rados_completion_t c)
263 int rados_aio_stat(rados_ioctx_t io, const char *oid, rados_completion_t completion, uint64_t *psize, time_t *pmtime)
264 int rados_aio_write(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len, uint64_t off)
265 int rados_aio_append(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len)
266 int rados_aio_write_full(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * buf, size_t len)
267 int rados_aio_remove(rados_ioctx_t io, const char * oid, rados_completion_t completion)
268 int rados_aio_read(rados_ioctx_t io, const char * oid, rados_completion_t completion, char * buf, size_t len, uint64_t off)
269 int rados_aio_flush(rados_ioctx_t io)
271 int rados_aio_get_return_value(rados_completion_t c)
272 int rados_aio_wait_for_complete_and_cb(rados_completion_t c)
273 int rados_aio_wait_for_safe_and_cb(rados_completion_t c)
274 int rados_aio_wait_for_complete(rados_completion_t c)
275 int rados_aio_wait_for_safe(rados_completion_t c)
276 int rados_aio_is_complete(rados_completion_t c)
277 int rados_aio_is_safe(rados_completion_t c)
279 int rados_exec(rados_ioctx_t io, const char * oid, const char * cls, const char * method,
280 const char * in_buf, size_t in_len, char * buf, size_t out_len)
281 int rados_aio_exec(rados_ioctx_t io, const char * oid, rados_completion_t completion, const char * cls, const char * method,
282 const char * in_buf, size_t in_len, char * buf, size_t out_len)
284 int rados_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, const char * oid, time_t * mtime, int flags)
285 int rados_aio_write_op_operate(rados_write_op_t write_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, time_t *mtime, int flags)
286 void rados_write_op_omap_set(rados_write_op_t write_op, const char * const* keys, const char * const* vals, const size_t * lens, size_t num)
287 void rados_write_op_omap_rm_keys(rados_write_op_t write_op, const char * const* keys, size_t keys_len)
288 void rados_write_op_omap_clear(rados_write_op_t write_op)
289 void rados_write_op_set_flags(rados_write_op_t write_op, int flags)
291 void rados_write_op_create(rados_write_op_t write_op, int exclusive, const char *category)
292 void rados_write_op_append(rados_write_op_t write_op, const char *buffer, size_t len)
293 void rados_write_op_write_full(rados_write_op_t write_op, const char *buffer, size_t len)
294 void rados_write_op_assert_version(rados_write_op_t write_op, uint64_t ver)
295 void rados_write_op_write(rados_write_op_t write_op, const char *buffer, size_t len, uint64_t offset)
296 void rados_write_op_remove(rados_write_op_t write_op)
297 void rados_write_op_truncate(rados_write_op_t write_op, uint64_t offset)
298 void rados_write_op_zero(rados_write_op_t write_op, uint64_t offset, uint64_t len)
300 void rados_read_op_omap_get_vals2(rados_read_op_t read_op, const char * start_after, const char * filter_prefix, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval)
301 void rados_read_op_omap_get_keys2(rados_read_op_t read_op, const char * start_after, uint64_t max_return, rados_omap_iter_t * iter, unsigned char *pmore, int * prval)
302 void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op, const char * const* keys, size_t keys_len, rados_omap_iter_t * iter, int * prval)
303 int rados_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, const char * oid, int flags)
304 int rados_aio_read_op_operate(rados_read_op_t read_op, rados_ioctx_t io, rados_completion_t completion, const char *oid, int flags)
305 void rados_read_op_set_flags(rados_read_op_t read_op, int flags)
306 int rados_omap_get_next(rados_omap_iter_t iter, const char * const* key, const char * const* val, size_t * len)
307 void rados_omap_get_end(rados_omap_iter_t iter)
308 int rados_notify2(rados_ioctx_t io, const char * o, const char *buf, int buf_len, uint64_t timeout_ms, char **reply_buffer, size_t *reply_buffer_len)
311 LIBRADOS_OP_FLAG_EXCL = _LIBRADOS_OP_FLAG_EXCL
312 LIBRADOS_OP_FLAG_FAILOK = _LIBRADOS_OP_FLAG_FAILOK
313 LIBRADOS_OP_FLAG_FADVISE_RANDOM = _LIBRADOS_OP_FLAG_FADVISE_RANDOM
314 LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL = _LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL
315 LIBRADOS_OP_FLAG_FADVISE_WILLNEED = _LIBRADOS_OP_FLAG_FADVISE_WILLNEED
316 LIBRADOS_OP_FLAG_FADVISE_DONTNEED = _LIBRADOS_OP_FLAG_FADVISE_DONTNEED
317 LIBRADOS_OP_FLAG_FADVISE_NOCACHE = _LIBRADOS_OP_FLAG_FADVISE_NOCACHE
319 LIBRADOS_SNAP_HEAD = _LIBRADOS_SNAP_HEAD
321 LIBRADOS_OPERATION_NOFLAG = _LIBRADOS_OPERATION_NOFLAG
322 LIBRADOS_OPERATION_BALANCE_READS = _LIBRADOS_OPERATION_BALANCE_READS
323 LIBRADOS_OPERATION_LOCALIZE_READS = _LIBRADOS_OPERATION_LOCALIZE_READS
324 LIBRADOS_OPERATION_ORDER_READS_WRITES = _LIBRADOS_OPERATION_ORDER_READS_WRITES
325 LIBRADOS_OPERATION_IGNORE_CACHE = _LIBRADOS_OPERATION_IGNORE_CACHE
326 LIBRADOS_OPERATION_SKIPRWLOCKS = _LIBRADOS_OPERATION_SKIPRWLOCKS
327 LIBRADOS_OPERATION_IGNORE_OVERLAY = _LIBRADOS_OPERATION_IGNORE_OVERLAY
329 LIBRADOS_ALL_NSPACES = _LIBRADOS_ALL_NSPACES.decode('utf-8')
331 LIBRADOS_CREATE_EXCLUSIVE = _LIBRADOS_CREATE_EXCLUSIVE
332 LIBRADOS_CREATE_IDEMPOTENT = _LIBRADOS_CREATE_IDEMPOTENT
334 ANONYMOUS_AUID = 0xffffffffffffffff
338 class Error(Exception):
339 """ `Error` class, derived from `Exception` """
340 def __init__(self, message, errno=None):
341 super(Exception, self).__init__(message)
345 msg = super(Exception, self).__str__()
346 if self.errno is None:
348 return '[errno {0}] {1}'.format(self.errno, msg)
350 def __reduce__(self):
351 return (self.__class__, (self.message, self.errno))
353 class InvalidArgumentError(Error):
356 class OSError(Error):
357 """ `OSError` class, derived from `Error` """
360 class InterruptedOrTimeoutError(OSError):
361 """ `InterruptedOrTimeoutError` class, derived from `OSError` """
365 class PermissionError(OSError):
366 """ `PermissionError` class, derived from `OSError` """
370 class PermissionDeniedError(OSError):
371 """ deal with EACCES related. """
375 class ObjectNotFound(OSError):
376 """ `ObjectNotFound` class, derived from `OSError` """
380 class NoData(OSError):
381 """ `NoData` class, derived from `OSError` """
385 class ObjectExists(OSError):
386 """ `ObjectExists` class, derived from `OSError` """
390 class ObjectBusy(OSError):
391 """ `ObjectBusy` class, derived from `IOError` """
395 class IOError(OSError):
396 """ `ObjectBusy` class, derived from `OSError` """
400 class NoSpace(OSError):
401 """ `NoSpace` class, derived from `OSError` """
405 class RadosStateError(Error):
406 """ `RadosStateError` class, derived from `Error` """
410 class IoctxStateError(Error):
411 """ `IoctxStateError` class, derived from `Error` """
415 class ObjectStateError(Error):
416 """ `ObjectStateError` class, derived from `Error` """
420 class LogicError(Error):
421 """ `` class, derived from `Error` """
425 class TimedOut(OSError):
426 """ `TimedOut` class, derived from `OSError` """
430 IF UNAME_SYSNAME == "FreeBSD":
431 cdef errno_to_exception = {
432 errno.EPERM : PermissionError,
433 errno.ENOENT : ObjectNotFound,
435 errno.ENOSPC : NoSpace,
436 errno.EEXIST : ObjectExists,
437 errno.EBUSY : ObjectBusy,
438 errno.ENOATTR : NoData,
439 errno.EINTR : InterruptedOrTimeoutError,
440 errno.ETIMEDOUT : TimedOut,
441 errno.EACCES : PermissionDeniedError,
442 errno.EINVAL : InvalidArgumentError,
445 cdef errno_to_exception = {
446 errno.EPERM : PermissionError,
447 errno.ENOENT : ObjectNotFound,
449 errno.ENOSPC : NoSpace,
450 errno.EEXIST : ObjectExists,
451 errno.EBUSY : ObjectBusy,
452 errno.ENODATA : NoData,
453 errno.EINTR : InterruptedOrTimeoutError,
454 errno.ETIMEDOUT : TimedOut,
455 errno.EACCES : PermissionDeniedError,
456 errno.EINVAL : InvalidArgumentError,
460 cdef make_ex(ret, msg):
462 Translate a librados return code into an exception.
464 :param ret: the return code
466 :param msg: the error message to use
468 :returns: a subclass of :class:`Error`
471 if ret in errno_to_exception:
472 return errno_to_exception[ret](msg, errno=ret)
474 return OSError(msg, errno=ret)
477 # helper to specify an optional argument, where in addition to `cls`, `None`
483 # validate argument types of an instance method
484 # kwargs is an un-ordered dict, so use args instead
485 def requires(*types):
486 def is_type_of(v, t):
490 return isinstance(v, t)
492 def check_type(val, arg_name, arg_type):
493 if isinstance(arg_type, tuple):
494 if any(is_type_of(val, t) for t in arg_type):
496 type_names = ' or '.join('None' if t is None else t.__name__
498 raise TypeError('%s must be %s' % (arg_name, type_names))
500 if is_type_of(val, arg_type):
502 assert(arg_type is not None)
503 raise TypeError('%s must be %s' % (arg_name, arg_type.__name__))
506 # FIXME(sileht): this stop with
507 # AttributeError: 'method_descriptor' object has no attribute '__module__'
509 def validate_func(*args, **kwargs):
510 # ignore the `self` arg
511 pos_args = zip(args[1:], types)
512 named_args = ((kwargs[name], (name, spec)) for name, spec in types
514 for arg_val, (arg_name, arg_type) in chain(pos_args, named_args):
515 check_type(arg_val, arg_name, arg_type)
516 return f(*args, **kwargs)
521 def cstr(val, name, encoding="utf-8", opt=False):
523 Create a byte string from a Python string
525 :param basestring val: Python string
526 :param str name: Name of the string parameter, for exceptions
527 :param str encoding: Encoding to use
528 :param bool opt: If True, None is allowed
530 :raises: :class:`InvalidArgument`
532 if opt and val is None:
534 if isinstance(val, bytes):
536 elif isinstance(val, unicode):
537 return val.encode(encoding)
539 raise TypeError('%s must be a string' % name)
542 def cstr_list(list_str, name, encoding="utf-8"):
543 return [cstr(s, name) for s in list_str]
546 def decode_cstr(val, encoding="utf-8"):
548 Decode a byte string into a Python string.
550 :param bytes val: byte string
551 :rtype: unicode or None
556 return val.decode(encoding)
559 cdef char* opt_str(s) except? NULL:
565 cdef void* realloc_chk(void* ptr, size_t size) except NULL:
566 cdef void *ret = realloc(ptr, size)
568 raise MemoryError("realloc failed")
572 cdef size_t * to_csize_t_array(list_int):
573 cdef size_t *ret = <size_t *>malloc(len(list_int) * sizeof(size_t))
575 raise MemoryError("malloc failed")
576 for i in xrange(len(list_int)):
577 ret[i] = <size_t>list_int[i]
581 cdef char ** to_bytes_array(list_bytes):
582 cdef char **ret = <char **>malloc(len(list_bytes) * sizeof(char *))
584 raise MemoryError("malloc failed")
585 for i in xrange(len(list_bytes)):
586 ret[i] = <char *>list_bytes[i]
591 cdef int __monitor_callback(void *arg, const char *line, const char *who,
592 uint64_t sec, uint64_t nsec, uint64_t seq,
593 const char *level, const char *msg) with gil:
594 cdef object cb_info = <object>arg
595 cb_info[0](cb_info[1], line, who, sec, nsec, seq, level, msg)
598 cdef int __monitor_callback2(void *arg, const char *line, const char *channel,
601 uint64_t sec, uint64_t nsec, uint64_t seq,
602 const char *level, const char *msg) with gil:
603 cdef object cb_info = <object>arg
604 cb_info[0](cb_info[1], line, channel, name, who, sec, nsec, seq, level, msg)
608 class Version(object):
609 """ Version information """
610 def __init__(self, major, minor, extra):
616 return "%d.%d.%d" % (self.major, self.minor, self.extra)
619 cdef class Rados(object):
620 """This class wraps librados functions"""
621 # NOTE(sileht): attributes declared in .pyd
623 def __init__(self, *args, **kwargs):
625 self.__setup(*args, **kwargs)
627 @requires(('rados_id', opt(str_type)), ('name', opt(str_type)), ('clustername', opt(str_type)),
628 ('conffile', opt(str_type)))
629 def __setup(self, rados_id=None, name=None, clustername=None,
630 conf_defaults=None, conffile=None, conf=None, flags=0,
632 self.monitor_callback = None
633 self.monitor_callback2 = None
634 self.parsed_args = []
635 self.conf_defaults = conf_defaults
636 self.conffile = conffile
637 self.rados_id = rados_id
639 if rados_id and name:
640 raise Error("Rados(): can't supply both rados_id and name")
642 name = 'client.' + rados_id
644 name = 'client.admin'
645 if clustername is None:
648 name = cstr(name, 'name')
649 clustername = cstr(clustername, 'clustername')
652 char *_clustername = clustername
657 # Unpack void* (aka rados_config_t) from capsule
658 rados_config = <rados_config_t> PyCapsule_GetPointer(context, NULL)
660 ret = rados_create_with_context(&self.cluster, rados_config)
663 ret = rados_create2(&self.cluster, _clustername, _name, _flags)
665 raise Error("rados_initialize failed with error code: %d" % ret)
667 self.state = "configuring"
668 # order is important: conf_defaults, then conffile, then conf
670 for key, value in conf_defaults.items():
671 self.conf_set(key, value)
672 if conffile is not None:
673 # read the default conf file when '' is given
676 self.conf_read_file(conffile)
678 for key, value in conf.items():
679 self.conf_set(key, value)
681 def require_state(self, *args):
683 Checks if the Rados object is in a special state
685 :raises: :class:`RadosStateError`
687 if self.state in args:
689 raise RadosStateError("You cannot perform that operation on a \
690 Rados object in state %s." % self.state)
694 Disconnects from the cluster. Call this explicitly when a
695 Rados.connect()ed object is no longer used.
697 if self.state != "shutdown":
699 rados_shutdown(self.cluster)
700 self.state = "shutdown"
706 def __exit__(self, type_, value, traceback):
712 Get the version number of the ``librados`` C library.
714 :returns: a tuple of ``(major, minor, extra)`` components of the
721 rados_version(&major, &minor, &extra)
722 return Version(major, minor, extra)
724 @requires(('path', opt(str_type)))
725 def conf_read_file(self, path=None):
727 Configure the cluster handle using a Ceph config file.
729 :param path: path to the config file
732 self.require_state("configuring", "connected")
733 path = cstr(path, 'path', opt=True)
735 char *_path = opt_str(path)
737 ret = rados_conf_read_file(self.cluster, _path)
739 raise make_ex(ret, "error calling conf_read_file")
741 def conf_parse_argv(self, args):
743 Parse known arguments from args, and remove; returned
744 args contain only those unknown to ceph
746 self.require_state("configuring", "connected")
750 cargs = cstr_list(args, 'args')
752 int _argc = len(args)
753 char **_argv = to_bytes_array(cargs)
754 char **_remargv = NULL
757 _remargv = <char **>malloc(_argc * sizeof(char *))
759 ret = rados_conf_parse_argv_remainder(self.cluster, _argc,
761 <const char**>_remargv)
763 raise make_ex(ret, "error calling conf_parse_argv_remainder")
765 # _remargv was allocated with fixed argc; collapse return
766 # list to eliminate any missing args
767 retargs = [decode_cstr(a) for a in _remargv[:_argc]
769 self.parsed_args = args
775 def conf_parse_env(self, var='CEPH_ARGS'):
777 Parse known arguments from an environment variable, normally
780 self.require_state("configuring", "connected")
784 var = cstr(var, 'var')
788 ret = rados_conf_parse_env(self.cluster, _var)
790 raise make_ex(ret, "error calling conf_parse_env")
792 @requires(('option', str_type))
793 def conf_get(self, option):
795 Get the value of a configuration option
797 :param option: which option to read
800 :returns: str - value of the option or None
801 :raises: :class:`TypeError`
803 self.require_state("configuring", "connected")
804 option = cstr(option, 'option')
806 char *_option = option
812 ret_buf = <char *>realloc_chk(ret_buf, length)
814 ret = rados_conf_get(self.cluster, _option, ret_buf, length)
816 return decode_cstr(ret_buf)
817 elif ret == -errno.ENAMETOOLONG:
819 elif ret == -errno.ENOENT:
822 raise make_ex(ret, "error calling conf_get")
826 @requires(('option', str_type), ('val', str_type))
827 def conf_set(self, option, val):
829 Set the value of a configuration option
831 :param option: which option to set
833 :param option: value of the option
836 :raises: :class:`TypeError`, :class:`ObjectNotFound`
838 self.require_state("configuring", "connected")
839 option = cstr(option, 'option')
840 val = cstr(val, 'val')
842 char *_option = option
846 ret = rados_conf_set(self.cluster, _option, _val)
848 raise make_ex(ret, "error calling conf_set")
850 def ping_monitor(self, mon_id):
852 Ping a monitor to assess liveness
854 May be used as a simply way to assess liveness, or to obtain
855 information about the monitor in a simple way even in the
858 :param mon_id: the ID portion of the monitor's name (i.e., mon.<ID>)
860 :returns: the string reply from the monitor
863 self.require_state("configuring", "connected")
865 mon_id = cstr(mon_id, 'mon_id')
867 char *_mon_id = mon_id
872 ret = rados_ping_monitor(self.cluster, _mon_id, &outstr, &outstrlen)
875 raise make_ex(ret, "error calling ping_monitor")
878 my_outstr = outstr[:outstrlen]
879 rados_buffer_free(outstr)
880 return decode_cstr(my_outstr)
882 def connect(self, timeout=0):
884 Connect to the cluster. Use shutdown() to release resources.
886 self.require_state("configuring")
887 # NOTE(sileht): timeout was supported by old python API,
888 # but this is not something available in C API, so ignore
889 # for now and remove it later
891 ret = rados_connect(self.cluster)
893 raise make_ex(ret, "error connecting to the cluster")
894 self.state = "connected"
896 def get_instance_id(self):
898 Get a global id for current instance
900 self.require_state("connected")
902 ret = rados_get_instance_id(self.cluster)
905 def get_cluster_stats(self):
907 Read usage info about the cluster
909 This tells you total space, space used, space available, and number
910 of objects. These are not updated immediately when data is written,
911 they are eventually consistent.
913 :returns: dict - contains the following keys:
915 - ``kb`` (int) - total space
917 - ``kb_used`` (int) - space used
919 - ``kb_avail`` (int) - free space available
921 - ``num_objects`` (int) - number of objects
925 rados_cluster_stat_t stats
928 ret = rados_cluster_stat(self.cluster, &stats)
932 ret, "Rados.get_cluster_stats(%s): get_stats failed" % self.rados_id)
933 return {'kb': stats.kb,
934 'kb_used': stats.kb_used,
935 'kb_avail': stats.kb_avail,
936 'num_objects': stats.num_objects}
938 @requires(('pool_name', str_type))
939 def pool_exists(self, pool_name):
941 Checks if a given pool exists.
943 :param pool_name: name of the pool to check
946 :raises: :class:`TypeError`, :class:`Error`
947 :returns: bool - whether the pool exists, false otherwise.
949 self.require_state("connected")
951 pool_name = cstr(pool_name, 'pool_name')
953 char *_pool_name = pool_name
956 ret = rados_pool_lookup(self.cluster, _pool_name)
959 elif ret == -errno.ENOENT:
962 raise make_ex(ret, "error looking up pool '%s'" % pool_name)
964 @requires(('pool_name', str_type))
965 def pool_lookup(self, pool_name):
967 Returns a pool's ID based on its name.
969 :param pool_name: name of the pool to look up
972 :raises: :class:`TypeError`, :class:`Error`
973 :returns: int - pool ID, or None if it doesn't exist
975 self.require_state("connected")
976 pool_name = cstr(pool_name, 'pool_name')
978 char *_pool_name = pool_name
981 ret = rados_pool_lookup(self.cluster, _pool_name)
984 elif ret == -errno.ENOENT:
987 raise make_ex(ret, "error looking up pool '%s'" % pool_name)
989 @requires(('pool_id', int))
990 def pool_reverse_lookup(self, pool_id):
992 Returns a pool's name based on its ID.
994 :param pool_id: ID of the pool to look up
997 :raises: :class:`TypeError`, :class:`Error`
998 :returns: string - pool name, or None if it doesn't exist
1000 self.require_state("connected")
1002 int64_t _pool_id = pool_id
1008 name = <char *>realloc_chk(name, size)
1010 ret = rados_pool_reverse_lookup(self.cluster, _pool_id, name, size)
1013 elif ret != -errno.ERANGE and size <= 4096:
1015 elif ret == -errno.ENOENT:
1018 raise make_ex(ret, "error reverse looking up pool '%s'" % pool_id)
1020 return decode_cstr(name)
1025 @requires(('pool_name', str_type), ('crush_rule', opt(int)))
1026 def create_pool(self, pool_name, crush_rule=None):
1029 - with default settings: if crush_rule=None
1030 - with a specific CRUSH rule: crush_rule given
1032 :param pool_name: name of the pool to create
1033 :type pool_name: str
1034 :param crush_rule: rule to use for placement in the new pool
1035 :type crush_rule: int
1037 :raises: :class:`TypeError`, :class:`Error`
1039 self.require_state("connected")
1041 pool_name = cstr(pool_name, 'pool_name')
1043 char *_pool_name = pool_name
1046 if crush_rule is None:
1048 ret = rados_pool_create(self.cluster, _pool_name)
1050 _crush_rule = crush_rule
1052 ret = rados_pool_create_with_crush_rule(self.cluster, _pool_name, _crush_rule)
1054 raise make_ex(ret, "error creating pool '%s'" % pool_name)
1056 @requires(('pool_id', int))
1057 def get_pool_base_tier(self, pool_id):
1061 :returns: base pool, or pool_id if tiering is not configured for the pool
1063 self.require_state("connected")
1065 int64_t base_tier = 0
1066 int64_t _pool_id = pool_id
1069 ret = rados_pool_get_base_tier(self.cluster, _pool_id, &base_tier)
1071 raise make_ex(ret, "get_pool_base_tier(%d)" % pool_id)
1072 return int(base_tier)
1074 @requires(('pool_name', str_type))
1075 def delete_pool(self, pool_name):
1077 Delete a pool and all data inside it.
1079 The pool is removed from the cluster immediately,
1080 but the actual data is deleted in the background.
1082 :param pool_name: name of the pool to delete
1083 :type pool_name: str
1085 :raises: :class:`TypeError`, :class:`Error`
1087 self.require_state("connected")
1089 pool_name = cstr(pool_name, 'pool_name')
1091 char *_pool_name = pool_name
1094 ret = rados_pool_delete(self.cluster, _pool_name)
1096 raise make_ex(ret, "error deleting pool '%s'" % pool_name)
1098 @requires(('pool_id', int))
1099 def get_inconsistent_pgs(self, pool_id):
1101 List inconsistent placement groups in the given pool
1103 :param pool_id: ID of the pool in which PGs are listed
1105 :returns: list - inconsistent placement groups
1107 self.require_state("connected")
1109 int64_t pool = pool_id
1115 pgs = <char *>realloc_chk(pgs, size);
1117 ret = rados_inconsistent_pg_list(self.cluster, pool,
1124 raise make_ex(ret, "error calling inconsistent_pg_list")
1125 return [pg for pg in decode_cstr(pgs[:ret]).split('\0') if pg]
1129 def list_pools(self):
1131 Gets a list of pool names.
1133 :returns: list - of pool names.
1135 self.require_state("connected")
1138 char *c_names = NULL
1142 c_names = <char *>realloc_chk(c_names, size)
1144 ret = rados_pool_list(self.cluster, c_names, size)
1149 return [name for name in decode_cstr(c_names[:ret]).split('\0')
1156 Get the fsid of the cluster as a hexadecimal string.
1158 :raises: :class:`Error`
1159 :returns: str - cluster fsid
1161 self.require_state("connected")
1163 char *ret_buf = NULL
1168 ret_buf = <char *>realloc_chk(ret_buf, buf_len)
1170 ret = rados_cluster_fsid(self.cluster, ret_buf, buf_len)
1171 if ret == -errno.ERANGE:
1172 buf_len = buf_len * 2
1174 raise make_ex(ret, "error getting cluster fsid")
1177 return decode_cstr(ret_buf)
1181 @requires(('ioctx_name', str_type))
1182 def open_ioctx(self, ioctx_name):
1184 Create an io context
1186 The io context allows you to perform operations within a particular
1189 :param ioctx_name: name of the pool
1190 :type ioctx_name: str
1192 :raises: :class:`TypeError`, :class:`Error`
1193 :returns: Ioctx - Rados Ioctx object
1195 self.require_state("connected")
1196 ioctx_name = cstr(ioctx_name, 'ioctx_name')
1199 char *_ioctx_name = ioctx_name
1201 ret = rados_ioctx_create(self.cluster, _ioctx_name, &ioctx)
1203 raise make_ex(ret, "error opening pool '%s'" % ioctx_name)
1204 io = Ioctx(ioctx_name)
1208 @requires(('pool_id', int))
1209 def open_ioctx2(self, pool_id):
1211 Create an io context
1213 The io context allows you to perform operations within a particular
1216 :param pool_id: ID of the pool
1219 :raises: :class:`TypeError`, :class:`Error`
1220 :returns: Ioctx - Rados Ioctx object
1222 self.require_state("connected")
1225 int64_t _pool_id = pool_id
1227 ret = rados_ioctx_create2(self.cluster, _pool_id, &ioctx)
1229 raise make_ex(ret, "error opening pool id '%s'" % pool_id)
1230 io = Ioctx(str(pool_id))
1234 def mon_command(self, cmd, inbuf, timeout=0, target=None):
1236 mon_command[_target](cmd, inbuf, outbuf, outbuflen, outs, outslen)
1237 returns (int ret, string outbuf, string outs)
1239 # NOTE(sileht): timeout is ignored because C API doesn't provide
1240 # timeout argument, but we keep it for backward compat with old python binding
1242 self.require_state("connected")
1243 cmd = cstr_list(cmd, 'c')
1245 if isinstance(target, int):
1246 # NOTE(sileht): looks weird but test_monmap_dump pass int
1247 target = str(target)
1249 target = cstr(target, 'target', opt=True)
1250 inbuf = cstr(inbuf, 'inbuf')
1253 char *_target = opt_str(target)
1254 char **_cmd = to_bytes_array(cmd)
1255 size_t _cmdlen = len(cmd)
1257 char *_inbuf = inbuf
1258 size_t _inbuf_len = len(inbuf)
1268 ret = rados_mon_command_target(self.cluster, _target,
1269 <const char **>_cmd, _cmdlen,
1270 <const char*>_inbuf, _inbuf_len,
1271 &_outbuf, &_outbuf_len,
1275 ret = rados_mon_command(self.cluster,
1276 <const char **>_cmd, _cmdlen,
1277 <const char*>_inbuf, _inbuf_len,
1278 &_outbuf, &_outbuf_len,
1281 my_outs = decode_cstr(_outs[:_outs_len])
1282 my_outbuf = _outbuf[:_outbuf_len]
1284 rados_buffer_free(_outs)
1286 rados_buffer_free(_outbuf)
1287 return (ret, my_outbuf, my_outs)
1291 def osd_command(self, osdid, cmd, inbuf, timeout=0):
1293 osd_command(osdid, cmd, inbuf, outbuf, outbuflen, outs, outslen)
1294 returns (int ret, string outbuf, string outs)
1296 # NOTE(sileht): timeout is ignored because C API doesn't provide
1297 # timeout argument, but we keep it for backward compat with old python binding
1298 self.require_state("connected")
1300 cmd = cstr_list(cmd, 'cmd')
1301 inbuf = cstr(inbuf, 'inbuf')
1305 char **_cmd = to_bytes_array(cmd)
1306 size_t _cmdlen = len(cmd)
1308 char *_inbuf = inbuf
1309 size_t _inbuf_len = len(inbuf)
1318 ret = rados_osd_command(self.cluster, _osdid,
1319 <const char **>_cmd, _cmdlen,
1320 <const char*>_inbuf, _inbuf_len,
1321 &_outbuf, &_outbuf_len,
1324 my_outs = decode_cstr(_outs[:_outs_len])
1325 my_outbuf = _outbuf[:_outbuf_len]
1327 rados_buffer_free(_outs)
1329 rados_buffer_free(_outbuf)
1330 return (ret, my_outbuf, my_outs)
1334 def mgr_command(self, cmd, inbuf, timeout=0):
1336 returns (int ret, string outbuf, string outs)
1338 # NOTE(sileht): timeout is ignored because C API doesn't provide
1339 # timeout argument, but we keep it for backward compat with old python binding
1340 self.require_state("connected")
1342 cmd = cstr_list(cmd, 'cmd')
1343 inbuf = cstr(inbuf, 'inbuf')
1346 char **_cmd = to_bytes_array(cmd)
1347 size_t _cmdlen = len(cmd)
1349 char *_inbuf = inbuf
1350 size_t _inbuf_len = len(inbuf)
1359 ret = rados_mgr_command(self.cluster,
1360 <const char **>_cmd, _cmdlen,
1361 <const char*>_inbuf, _inbuf_len,
1362 &_outbuf, &_outbuf_len,
1365 my_outs = decode_cstr(_outs[:_outs_len])
1366 my_outbuf = _outbuf[:_outbuf_len]
1368 rados_buffer_free(_outs)
1370 rados_buffer_free(_outbuf)
1371 return (ret, my_outbuf, my_outs)
1375 def pg_command(self, pgid, cmd, inbuf, timeout=0):
1377 pg_command(pgid, cmd, inbuf, outbuf, outbuflen, outs, outslen)
1378 returns (int ret, string outbuf, string outs)
1380 # NOTE(sileht): timeout is ignored because C API doesn't provide
1381 # timeout argument, but we keep it for backward compat with old python binding
1382 self.require_state("connected")
1384 pgid = cstr(pgid, 'pgid')
1385 cmd = cstr_list(cmd, 'cmd')
1386 inbuf = cstr(inbuf, 'inbuf')
1390 char **_cmd = to_bytes_array(cmd)
1391 size_t _cmdlen = len(cmd)
1393 char *_inbuf = inbuf
1394 size_t _inbuf_len = len(inbuf)
1403 ret = rados_pg_command(self.cluster, _pgid,
1404 <const char **>_cmd, _cmdlen,
1405 <const char *>_inbuf, _inbuf_len,
1406 &_outbuf, &_outbuf_len,
1409 my_outs = decode_cstr(_outs[:_outs_len])
1410 my_outbuf = _outbuf[:_outbuf_len]
1412 rados_buffer_free(_outs)
1414 rados_buffer_free(_outbuf)
1415 return (ret, my_outbuf, my_outs)
1419 def wait_for_latest_osdmap(self):
1420 self.require_state("connected")
1422 ret = rados_wait_for_latest_osdmap(self.cluster)
1425 def blacklist_add(self, client_address, expire_seconds=0):
1427 Blacklist a client from the OSDs
1429 :param client_address: client address
1430 :type client_address: str
1431 :param expire_seconds: number of seconds to blacklist
1432 :type expire_seconds: int
1434 :raises: :class:`Error`
1436 self.require_state("connected")
1437 client_address = cstr(client_address, 'client_address')
1439 uint32_t _expire_seconds = expire_seconds
1440 char *_client_address = client_address
1443 ret = rados_blacklist_add(self.cluster, _client_address, _expire_seconds)
1445 raise make_ex(ret, "error blacklisting client '%s'" % client_address)
1447 def monitor_log(self, level, callback, arg):
1448 if level not in MONITOR_LEVELS:
1449 raise LogicError("invalid monitor level " + level)
1450 if callback is not None and not callable(callback):
1451 raise LogicError("callback must be a callable function or None")
1453 level = cstr(level, 'level')
1454 cdef char *_level = level
1456 if callback is None:
1458 r = rados_monitor_log(self.cluster, <const char*>_level, NULL, NULL)
1459 self.monitor_callback = None
1460 self.monitor_callback2 = None
1463 cb = (callback, arg)
1464 cdef PyObject* _arg = <PyObject*>cb
1466 r = rados_monitor_log(self.cluster, <const char*>_level,
1467 <rados_log_callback_t>&__monitor_callback, _arg)
1470 raise make_ex(r, 'error calling rados_monitor_log')
1471 # NOTE(sileht): Prevents the callback method from being garbage collected
1472 self.monitor_callback = cb
1473 self.monitor_callback2 = None
1475 def monitor_log2(self, level, callback, arg):
1476 if level not in MONITOR_LEVELS:
1477 raise LogicError("invalid monitor level " + level)
1478 if callback is not None and not callable(callback):
1479 raise LogicError("callback must be a callable function or None")
1481 level = cstr(level, 'level')
1482 cdef char *_level = level
1484 if callback is None:
1486 r = rados_monitor_log2(self.cluster, <const char*>_level, NULL, NULL)
1487 self.monitor_callback = None
1488 self.monitor_callback2 = None
1491 cb = (callback, arg)
1492 cdef PyObject* _arg = <PyObject*>cb
1494 r = rados_monitor_log2(self.cluster, <const char*>_level,
1495 <rados_log_callback2_t>&__monitor_callback2, _arg)
1498 raise make_ex(r, 'error calling rados_monitor_log')
1499 # NOTE(sileht): Prevents the callback method from being garbage collected
1500 self.monitor_callback = None
1501 self.monitor_callback2 = cb
1503 @requires(('service', str_type), ('daemon', str_type), ('metadata', dict))
1504 def service_daemon_register(self, service, daemon, metadata):
1506 :param str service: service name (e.g. "rgw")
1507 :param str daemon: daemon name (e.g. "gwfoo")
1508 :param dict metadata: static metadata about the register daemon
1509 (e.g., the version of Ceph, the kernel version.)
1511 service = cstr(service, 'service')
1512 daemon = cstr(daemon, 'daemon')
1513 metadata_dict = '\0'.join(chain.from_iterable(metadata.items()))
1514 metadata_dict += '\0'
1516 char *_service = service
1517 char *_daemon = daemon
1518 char *_metadata = metadata_dict
1521 ret = rados_service_register(self.cluster, _service, _daemon, _metadata)
1523 raise make_ex(ret, "error calling service_register()")
1525 @requires(('metadata', dict))
1526 def service_daemon_update(self, status):
1527 status_dict = '\0'.join(chain.from_iterable(status.items()))
1530 char *_status = status_dict
1533 ret = rados_service_update_status(self.cluster, _status)
1535 raise make_ex(ret, "error calling service_daemon_update()")
1538 cdef class OmapIterator(object):
1541 cdef public Ioctx ioctx
1542 cdef rados_omap_iter_t ctx
1544 def __cinit__(self, Ioctx ioctx):
1552 Get the next key-value pair in the object
1553 :returns: next rados.OmapItem
1561 ret = rados_omap_get_next(self.ctx, &key_, &val_, &len_)
1564 raise make_ex(ret, "error iterating over the omap")
1566 raise StopIteration()
1567 key = decode_cstr(key_)
1573 def __dealloc__(self):
1575 rados_omap_get_end(self.ctx)
1578 cdef class ObjectIterator(object):
1579 """rados.Ioctx Object iterator"""
1581 cdef rados_list_ctx_t ctx
1583 cdef public object ioctx
1585 def __cinit__(self, Ioctx ioctx):
1589 ret = rados_nobjects_list_open(ioctx.io, &self.ctx)
1591 raise make_ex(ret, "error iterating over the objects in ioctx '%s'"
1599 Get the next object name and locator in the pool
1601 :raises: StopIteration
1602 :returns: next rados.Ioctx Object
1605 const char *key_ = NULL
1606 const char *locator_ = NULL
1607 const char *nspace_ = NULL
1610 ret = rados_nobjects_list_next(self.ctx, &key_, &locator_, &nspace_)
1613 raise StopIteration()
1615 key = decode_cstr(key_)
1616 locator = decode_cstr(locator_) if locator_ != NULL else None
1617 nspace = decode_cstr(nspace_) if nspace_ != NULL else None
1618 return Object(self.ioctx, key, locator, nspace)
1620 def __dealloc__(self):
1622 rados_nobjects_list_close(self.ctx)
1625 cdef class XattrIterator(object):
1626 """Extended attribute iterator"""
1628 cdef rados_xattrs_iter_t it
1631 cdef public Ioctx ioctx
1632 cdef public object oid
1634 def __cinit__(self, Ioctx ioctx, oid):
1636 self.oid = cstr(oid, 'oid')
1637 self._oid = self.oid
1640 ret = rados_getxattrs(ioctx.io, self._oid, &self.it)
1642 raise make_ex(ret, "Failed to get rados xattrs for object %r" % oid)
1649 Get the next xattr on the object
1651 :raises: StopIteration
1652 :returns: pair - of name and value of the next Xattr
1655 const char *name_ = NULL
1656 const char *val_ = NULL
1660 ret = rados_getxattrs_next(self.it, &name_, &val_, &len_)
1662 raise make_ex(ret, "error iterating over the extended attributes \
1663 in '%s'" % self.oid)
1665 raise StopIteration()
1666 name = decode_cstr(name_)
1670 def __dealloc__(self):
1672 rados_getxattrs_end(self.it)
1675 cdef class SnapIterator(object):
1676 """Snapshot iterator"""
1678 cdef public Ioctx ioctx
1680 cdef rados_snap_t *snaps
1684 def __cinit__(self, Ioctx ioctx):
1686 # We don't know how big a buffer we need until we've called the
1687 # function. So use the exponential doubling strategy.
1688 cdef int num_snaps = 10
1690 self.snaps = <rados_snap_t*>realloc_chk(self.snaps,
1692 sizeof(rados_snap_t))
1695 ret = rados_ioctx_snap_list(ioctx.io, self.snaps, num_snaps)
1699 elif ret != -errno.ERANGE:
1700 raise make_ex(ret, "error calling rados_snap_list for \
1701 ioctx '%s'" % self.ioctx.name)
1702 num_snaps = num_snaps * 2
1710 Get the next Snapshot
1712 :raises: :class:`Error`, StopIteration
1713 :returns: Snap - next snapshot
1715 if self.cur_snap >= self.max_snap:
1719 rados_snap_t snap_id = self.snaps[self.cur_snap]
1725 name = <char *>realloc_chk(name, name_len)
1727 ret = rados_ioctx_snap_get_name(self.ioctx.io, snap_id, name, name_len)
1730 elif ret != -errno.ERANGE:
1731 raise make_ex(ret, "rados_snap_get_name error")
1733 name_len = name_len * 2
1735 snap = Snap(self.ioctx, decode_cstr(name[:name_len]).rstrip('\0'), snap_id)
1736 self.cur_snap = self.cur_snap + 1
1742 cdef class Snap(object):
1743 """Snapshot object"""
1744 cdef public Ioctx ioctx
1745 cdef public object name
1747 # NOTE(sileht): old API was storing the ctypes object
1748 # instead of the value ....
1749 cdef public rados_snap_t snap_id
1751 def __cinit__(self, Ioctx ioctx, object name, rados_snap_t snap_id):
1754 self.snap_id = snap_id
1757 return "rados.Snap(ioctx=%s,name=%s,snap_id=%d)" \
1758 % (str(self.ioctx), self.name, self.snap_id)
1760 def get_timestamp(self):
1762 Find when a snapshot in the current pool occurred
1764 :raises: :class:`Error`
1765 :returns: datetime - the data and time the snapshot was created
1767 cdef time_t snap_time
1770 ret = rados_ioctx_snap_get_stamp(self.ioctx.io, self.snap_id, &snap_time)
1772 raise make_ex(ret, "rados_ioctx_snap_get_stamp error")
1773 return datetime.fromtimestamp(snap_time)
1776 cdef class Completion(object):
1777 """completion object"""
1785 rados_callback_t complete_cb
1786 rados_callback_t safe_cb
1787 rados_completion_t rados_comp
1790 def __cinit__(self, Ioctx ioctx, object oncomplete, object onsafe):
1791 self.oncomplete = oncomplete
1792 self.onsafe = onsafe
1797 Is an asynchronous operation safe?
1799 This does not imply that the safe callback has finished.
1801 :returns: True if the operation is safe
1804 ret = rados_aio_is_safe(self.rados_comp)
1807 def is_complete(self):
1809 Has an asynchronous operation completed?
1811 This does not imply that the safe callback has finished.
1813 :returns: True if the operation is completed
1816 ret = rados_aio_is_complete(self.rados_comp)
1819 def wait_for_safe(self):
1821 Wait for an asynchronous operation to be marked safe
1823 This does not imply that the safe callback has finished.
1826 rados_aio_wait_for_safe(self.rados_comp)
1828 def wait_for_complete(self):
1830 Wait for an asynchronous operation to complete
1832 This does not imply that the complete callback has finished.
1835 rados_aio_wait_for_complete(self.rados_comp)
1837 def wait_for_safe_and_cb(self):
1839 Wait for an asynchronous operation to be marked safe and for
1840 the safe callback to have returned
1843 rados_aio_wait_for_safe_and_cb(self.rados_comp)
1845 def wait_for_complete_and_cb(self):
1847 Wait for an asynchronous operation to complete and for the
1848 complete callback to have returned
1850 :returns: whether the operation is completed
1853 ret = rados_aio_wait_for_complete_and_cb(self.rados_comp)
1856 def get_return_value(self):
1858 Get the return value of an asychronous operation
1860 The return value is set when the operation is complete or safe,
1861 whichever comes first.
1863 :returns: int - return value of the operation
1866 ret = rados_aio_get_return_value(self.rados_comp)
1869 def __dealloc__(self):
1871 Release a completion
1873 Call this when you no longer need the completion. It may not be
1874 freed immediately if the operation is not acked and committed.
1876 ref.Py_XDECREF(self.buf)
1878 if self.rados_comp != NULL:
1880 rados_aio_release(self.rados_comp)
1881 self.rados_comp = NULL
1883 def _complete(self):
1884 self.oncomplete(self)
1885 with self.ioctx.lock:
1887 self.ioctx.complete_completions.remove(self)
1891 with self.ioctx.lock:
1893 self.ioctx.safe_completions.remove(self)
1896 with self.ioctx.lock:
1898 self.ioctx.complete_completions.remove(self)
1900 self.ioctx.safe_completions.remove(self)
1903 class OpCtx(object):
1904 def __enter__(self):
1905 return self.create()
1907 def __exit__(self, type, msg, traceback):
1911 cdef class WriteOp(object):
1912 cdef rados_write_op_t write_op
1916 self.write_op = rados_create_write_op()
1921 rados_release_write_op(self.write_op)
1923 @requires(('exclusive', opt(int)))
1924 def new(self, exclusive=None):
1930 int _exclusive = exclusive
1933 rados_write_op_create(self.write_op, _exclusive, NULL)
1941 rados_write_op_remove(self.write_op)
1943 @requires(('flags', int))
1944 def set_flags(self, flags=LIBRADOS_OPERATION_NOFLAG):
1946 Set flags for the last operation added to this write_op.
1947 :para flags: flags to apply to the last operation
1955 rados_write_op_set_flags(self.write_op, _flags)
1957 @requires(('to_write', bytes))
1958 def append(self, to_write):
1960 Append data to an object synchronously
1961 :param to_write: data to write
1962 :type to_write: bytes
1966 char *_to_write = to_write
1967 size_t length = len(to_write)
1970 rados_write_op_append(self.write_op, _to_write, length)
1972 @requires(('to_write', bytes))
1973 def write_full(self, to_write):
1975 Write whole object, atomically replacing it.
1976 :param to_write: data to write
1977 :type to_write: bytes
1981 char *_to_write = to_write
1982 size_t length = len(to_write)
1985 rados_write_op_write_full(self.write_op, _to_write, length)
1987 @requires(('to_write', bytes), ('offset', int))
1988 def write(self, to_write, offset=0):
1991 :param to_write: data to write
1992 :type to_write: bytes
1993 :param offset: byte offset in the object to begin writing at
1998 char *_to_write = to_write
1999 size_t length = len(to_write)
2000 uint64_t _offset = offset
2003 rados_write_op_write(self.write_op, _to_write, length, _offset)
2005 @requires(('version', int))
2006 def assert_version(self, version):
2008 Check if object's version is the expected one.
2009 :param version: expected version of the object
2013 uint64_t _version = version
2016 rados_write_op_assert_version(self.write_op, _version)
2018 @requires(('offset', int), ('length', int))
2019 def zero(self, offset, length):
2021 Zero part of an object.
2022 :param offset: byte offset in the object to begin writing at
2024 :param offset: number of zero to write
2029 size_t _length = length
2030 uint64_t _offset = offset
2033 rados_write_op_zero(self.write_op, _length, _offset)
2035 @requires(('offset', int))
2036 def truncate(self, offset):
2039 :param offset: byte offset in the object to begin truncating at
2044 uint64_t _offset = offset
2047 rados_write_op_truncate(self.write_op, _offset)
2050 class WriteOpCtx(WriteOp, OpCtx):
2051 """write operation context manager"""
2054 cdef class ReadOp(object):
2055 cdef rados_read_op_t read_op
2059 self.read_op = rados_create_read_op()
2064 rados_release_read_op(self.read_op)
2066 @requires(('flags', int))
2067 def set_flags(self, flags=LIBRADOS_OPERATION_NOFLAG):
2069 Set flags for the last operation added to this read_op.
2070 :para flags: flags to apply to the last operation
2078 rados_read_op_set_flags(self.read_op, _flags)
2081 class ReadOpCtx(ReadOp, OpCtx):
2082 """read operation context manager"""
2085 cdef int __aio_safe_cb(rados_completion_t completion, void *args) with gil:
2087 Callback to onsafe() for asynchronous operations
2089 cdef object cb = <object>args
2094 cdef int __aio_complete_cb(rados_completion_t completion, void *args) with gil:
2096 Callback to oncomplete() for asynchronous operations
2098 cdef object cb = <object>args
2103 cdef class Ioctx(object):
2104 """rados.Ioctx object"""
2105 # NOTE(sileht): attributes declared in .pyd
2107 def __init__(self, name):
2111 self.locator_key = ""
2113 self.lock = threading.Lock()
2114 self.safe_completions = []
2115 self.complete_completions = []
2117 def __enter__(self):
2120 def __exit__(self, type_, value, traceback):
2124 def __dealloc__(self):
2127 def __track_completion(self, completion_obj):
2128 if completion_obj.oncomplete:
2130 self.complete_completions.append(completion_obj)
2131 if completion_obj.onsafe:
2133 self.safe_completions.append(completion_obj)
2135 def __get_completion(self, oncomplete, onsafe):
2137 Constructs a completion to use with asynchronous operations
2139 :param oncomplete: what to do when the write is safe and complete in memory
2141 :type oncomplete: completion
2142 :param onsafe: what to do when the write is safe and complete on storage
2144 :type onsafe: completion
2146 :raises: :class:`Error`
2147 :returns: completion object
2150 completion_obj = Completion(self, oncomplete, onsafe)
2153 rados_callback_t complete_cb = NULL
2154 rados_callback_t safe_cb = NULL
2155 rados_completion_t completion
2156 PyObject* p_completion_obj= <PyObject*>completion_obj
2159 complete_cb = <rados_callback_t>&__aio_complete_cb
2161 safe_cb = <rados_callback_t>&__aio_safe_cb
2164 ret = rados_aio_create_completion(p_completion_obj, complete_cb, safe_cb,
2167 raise make_ex(ret, "error getting a completion")
2169 completion_obj.rados_comp = completion
2170 return completion_obj
2172 @requires(('object_name', str_type), ('oncomplete', opt(Callable)))
2173 def aio_stat(self, object_name, oncomplete):
2175 Asynchronously get object stats (size/mtime)
2177 oncomplete will be called with the returned size and mtime
2178 as well as the completion:
2180 oncomplete(completion, size, mtime)
2182 :param object_name: the name of the object to get stats from
2183 :type object_name: str
2184 :param oncomplete: what to do when the stat is complete
2185 :type oncomplete: completion
2187 :raises: :class:`Error`
2188 :returns: completion object
2191 object_name = cstr(object_name, 'object_name')
2194 Completion completion
2195 char *_object_name = object_name
2199 def oncomplete_(completion_v):
2200 cdef Completion _completion_v = completion_v
2201 return_value = _completion_v.get_return_value()
2202 if return_value >= 0:
2203 return oncomplete(_completion_v, psize, time.localtime(pmtime))
2205 return oncomplete(_completion_v, None, None)
2207 completion = self.__get_completion(oncomplete_, None)
2208 self.__track_completion(completion)
2210 ret = rados_aio_stat(self.io, _object_name, completion.rados_comp,
2214 completion._cleanup()
2215 raise make_ex(ret, "error stating %s" % object_name)
2218 @requires(('object_name', str_type), ('to_write', bytes), ('offset', int),
2219 ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)))
2220 def aio_write(self, object_name, to_write, offset=0,
2221 oncomplete=None, onsafe=None):
2223 Write data to an object asynchronously
2225 Queues the write and returns.
2227 :param object_name: name of the object
2228 :type object_name: str
2229 :param to_write: data to write
2230 :type to_write: bytes
2231 :param offset: byte offset in the object to begin writing at
2233 :param oncomplete: what to do when the write is safe and complete in memory
2235 :type oncomplete: completion
2236 :param onsafe: what to do when the write is safe and complete on storage
2238 :type onsafe: completion
2240 :raises: :class:`Error`
2241 :returns: completion object
2244 object_name = cstr(object_name, 'object_name')
2247 Completion completion
2248 char* _object_name = object_name
2249 char* _to_write = to_write
2250 size_t size = len(to_write)
2251 uint64_t _offset = offset
2253 completion = self.__get_completion(oncomplete, onsafe)
2254 self.__track_completion(completion)
2256 ret = rados_aio_write(self.io, _object_name, completion.rados_comp,
2257 _to_write, size, _offset)
2259 completion._cleanup()
2260 raise make_ex(ret, "error writing object %s" % object_name)
2263 @requires(('object_name', str_type), ('to_write', bytes), ('oncomplete', opt(Callable)),
2264 ('onsafe', opt(Callable)))
2265 def aio_write_full(self, object_name, to_write,
2266 oncomplete=None, onsafe=None):
2268 Asynchronously write an entire object
2270 The object is filled with the provided data. If the object exists,
2271 it is atomically truncated and then written.
2272 Queues the write and returns.
2274 :param object_name: name of the object
2275 :type object_name: str
2276 :param to_write: data to write
2278 :param oncomplete: what to do when the write is safe and complete in memory
2280 :type oncomplete: completion
2281 :param onsafe: what to do when the write is safe and complete on storage
2283 :type onsafe: completion
2285 :raises: :class:`Error`
2286 :returns: completion object
2289 object_name = cstr(object_name, 'object_name')
2292 Completion completion
2293 char* _object_name = object_name
2294 char* _to_write = to_write
2295 size_t size = len(to_write)
2297 completion = self.__get_completion(oncomplete, onsafe)
2298 self.__track_completion(completion)
2300 ret = rados_aio_write_full(self.io, _object_name,
2301 completion.rados_comp,
2304 completion._cleanup()
2305 raise make_ex(ret, "error writing object %s" % object_name)
2308 @requires(('object_name', str_type), ('to_append', bytes), ('oncomplete', opt(Callable)),
2309 ('onsafe', opt(Callable)))
2310 def aio_append(self, object_name, to_append, oncomplete=None, onsafe=None):
2312 Asynchronously append data to an object
2314 Queues the write and returns.
2316 :param object_name: name of the object
2317 :type object_name: str
2318 :param to_append: data to append
2319 :type to_append: str
2320 :param offset: byte offset in the object to begin writing at
2322 :param oncomplete: what to do when the write is safe and complete in memory
2324 :type oncomplete: completion
2325 :param onsafe: what to do when the write is safe and complete on storage
2327 :type onsafe: completion
2329 :raises: :class:`Error`
2330 :returns: completion object
2332 object_name = cstr(object_name, 'object_name')
2335 Completion completion
2336 char* _object_name = object_name
2337 char* _to_append = to_append
2338 size_t size = len(to_append)
2340 completion = self.__get_completion(oncomplete, onsafe)
2341 self.__track_completion(completion)
2343 ret = rados_aio_append(self.io, _object_name,
2344 completion.rados_comp,
2347 completion._cleanup()
2348 raise make_ex(ret, "error appending object %s" % object_name)
2351 def aio_flush(self):
2353 Block until all pending writes in an io context are safe
2355 :raises: :class:`Error`
2358 ret = rados_aio_flush(self.io)
2360 raise make_ex(ret, "error flushing")
2362 @requires(('object_name', str_type), ('length', int), ('offset', int),
2363 ('oncomplete', opt(Callable)))
2364 def aio_read(self, object_name, length, offset, oncomplete):
2366 Asynchronously read data from an object
2368 oncomplete will be called with the returned read value as
2369 well as the completion:
2371 oncomplete(completion, data_read)
2373 :param object_name: name of the object to read from
2374 :type object_name: str
2375 :param length: the number of bytes to read
2377 :param offset: byte offset in the object to begin reading from
2379 :param oncomplete: what to do when the read is complete
2380 :type oncomplete: completion
2382 :raises: :class:`Error`
2383 :returns: completion object
2386 object_name = cstr(object_name, 'object_name')
2389 Completion completion
2390 char* _object_name = object_name
2391 uint64_t _offset = offset
2394 size_t _length = length
2396 def oncomplete_(completion_v):
2397 cdef Completion _completion_v = completion_v
2398 return_value = _completion_v.get_return_value()
2399 if return_value > 0 and return_value != length:
2400 _PyBytes_Resize(&_completion_v.buf, return_value)
2401 return oncomplete(_completion_v, <object>_completion_v.buf if return_value >= 0 else None)
2403 completion = self.__get_completion(oncomplete_, None)
2404 completion.buf = PyBytes_FromStringAndSize(NULL, length)
2405 ret_buf = PyBytes_AsString(completion.buf)
2406 self.__track_completion(completion)
2408 ret = rados_aio_read(self.io, _object_name, completion.rados_comp,
2409 ret_buf, _length, _offset)
2411 completion._cleanup()
2412 raise make_ex(ret, "error reading %s" % object_name)
2415 @requires(('object_name', str_type), ('cls', str_type), ('method', str_type),
2416 ('data', bytes), ('length', int),
2417 ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)))
2418 def aio_execute(self, object_name, cls, method, data,
2419 length=8192, oncomplete=None, onsafe=None):
2421 Asynchronously execute an OSD class method on an object.
2423 oncomplete and onsafe will be called with the data returned from
2424 the plugin as well as the completion:
2426 oncomplete(completion, data)
2427 onsafe(completion, data)
2429 :param object_name: name of the object
2430 :type object_name: str
2431 :param cls: name of the object class
2433 :param method: name of the method
2435 :param data: input data
2437 :param length: size of output buffer in bytes (default=8192)
2439 :param oncomplete: what to do when the execution is complete
2440 :type oncomplete: completion
2441 :param onsafe: what to do when the execution is safe and complete
2442 :type onsafe: completion
2444 :raises: :class:`Error`
2445 :returns: completion object
2448 object_name = cstr(object_name, 'object_name')
2449 cls = cstr(cls, 'cls')
2450 method = cstr(method, 'method')
2452 Completion completion
2453 char *_object_name = object_name
2455 char *_method = method
2457 size_t _data_len = len(data)
2460 size_t _length = length
2462 def oncomplete_(completion_v):
2463 cdef Completion _completion_v = completion_v
2464 return_value = _completion_v.get_return_value()
2465 if return_value > 0 and return_value != length:
2466 _PyBytes_Resize(&_completion_v.buf, return_value)
2467 return oncomplete(_completion_v, <object>_completion_v.buf if return_value >= 0 else None)
2469 def onsafe_(completion_v):
2470 cdef Completion _completion_v = completion_v
2471 return_value = _completion_v.get_return_value()
2472 return onsafe(_completion_v, <object>_completion_v.buf if return_value >= 0 else None)
2474 completion = self.__get_completion(oncomplete_ if oncomplete else None, onsafe_ if onsafe else None)
2475 completion.buf = PyBytes_FromStringAndSize(NULL, length)
2476 ret_buf = PyBytes_AsString(completion.buf)
2477 self.__track_completion(completion)
2479 ret = rados_aio_exec(self.io, _object_name, completion.rados_comp,
2480 _cls, _method, _data, _data_len, ret_buf, _length)
2482 completion._cleanup()
2483 raise make_ex(ret, "error executing %s::%s on %s" % (cls, method, object_name))
2486 @requires(('object_name', str_type), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)))
2487 def aio_remove(self, object_name, oncomplete=None, onsafe=None):
2489 Asynchronously remove an object
2491 :param object_name: name of the object to remove
2492 :type object_name: str
2493 :param oncomplete: what to do when the remove is safe and complete in memory
2495 :type oncomplete: completion
2496 :param onsafe: what to do when the remove is safe and complete on storage
2498 :type onsafe: completion
2500 :raises: :class:`Error`
2501 :returns: completion object
2503 object_name = cstr(object_name, 'object_name')
2506 Completion completion
2507 char* _object_name = object_name
2509 completion = self.__get_completion(oncomplete, onsafe)
2510 self.__track_completion(completion)
2512 ret = rados_aio_remove(self.io, _object_name,
2513 completion.rados_comp)
2515 completion._cleanup()
2516 raise make_ex(ret, "error removing %s" % object_name)
2519 def require_ioctx_open(self):
2521 Checks if the rados.Ioctx object state is 'open'
2523 :raises: IoctxStateError
2525 if self.state != "open":
2526 raise IoctxStateError("The pool is %s" % self.state)
2528 @requires(('loc_key', str_type))
2529 def set_locator_key(self, loc_key):
2531 Set the key for mapping objects to pgs within an io context.
2533 The key is used instead of the object name to determine which
2534 placement groups an object is put in. This affects all subsequent
2535 operations of the io context - until a different locator key is
2536 set, all objects in this io context will be placed in the same pg.
2538 :param loc_key: the key to use as the object locator, or NULL to discard
2539 any previously set key
2542 :raises: :class:`TypeError`
2544 self.require_ioctx_open()
2545 cloc_key = cstr(loc_key, 'loc_key')
2546 cdef char *_loc_key = cloc_key
2548 rados_ioctx_locator_set_key(self.io, _loc_key)
2549 self.locator_key = loc_key
2551 def get_locator_key(self):
2553 Get the locator_key of context
2555 :returns: locator_key
2557 return self.locator_key
2559 @requires(('snap_id', long))
2560 def set_read(self, snap_id):
2562 Set the snapshot for reading objects.
2564 To stop to read from snapshot, use set_read(LIBRADOS_SNAP_HEAD)
2566 :param snap_id: the snapshot Id
2569 :raises: :class:`TypeError`
2571 self.require_ioctx_open()
2572 cdef rados_snap_t _snap_id = snap_id
2574 rados_ioctx_snap_set_read(self.io, _snap_id)
2576 @requires(('nspace', str_type))
2577 def set_namespace(self, nspace):
2579 Set the namespace for objects within an io context.
2581 The namespace in addition to the object name fully identifies
2582 an object. This affects all subsequent operations of the io context
2583 - until a different namespace is set, all objects in this io context
2584 will be placed in the same namespace.
2586 :param nspace: the namespace to use, or None/"" for the default namespace
2589 :raises: :class:`TypeError`
2591 self.require_ioctx_open()
2594 cnspace = cstr(nspace, 'nspace')
2595 cdef char *_nspace = cnspace
2597 rados_ioctx_set_namespace(self.io, _nspace)
2598 self.nspace = nspace
2600 def get_namespace(self):
2602 Get the namespace of context
2610 Close a rados.Ioctx object.
2612 This just tells librados that you no longer need to use the io context.
2613 It may not be freed immediately if there are pending asynchronous
2614 requests on it, but you should not use an io context again after
2615 calling this function on it.
2617 if self.state == "open":
2618 self.require_ioctx_open()
2620 rados_ioctx_destroy(self.io)
2621 self.state = "closed"
2624 @requires(('key', str_type), ('data', bytes))
2625 def write(self, key, data, offset=0):
2627 Write data to an object synchronously
2629 :param key: name of the object
2631 :param data: data to write
2633 :param offset: byte offset in the object to begin writing at
2636 :raises: :class:`TypeError`
2637 :raises: :class:`LogicError`
2638 :returns: int - 0 on success
2640 self.require_ioctx_open()
2642 key = cstr(key, 'key')
2646 size_t length = len(data)
2647 uint64_t _offset = offset
2650 ret = rados_write(self.io, _key, _data, length, _offset)
2654 raise make_ex(ret, "Ioctx.write(%s): failed to write %s"
2657 raise LogicError("Ioctx.write(%s): rados_write \
2658 returned %d, but should return zero on success." % (self.name, ret))
2660 @requires(('key', str_type), ('data', bytes))
2661 def write_full(self, key, data):
2663 Write an entire object synchronously.
2665 The object is filled with the provided data. If the object exists,
2666 it is atomically truncated and then written.
2668 :param key: name of the object
2670 :param data: data to write
2673 :raises: :class:`TypeError`
2674 :raises: :class:`Error`
2675 :returns: int - 0 on success
2677 self.require_ioctx_open()
2678 key = cstr(key, 'key')
2682 size_t length = len(data)
2685 ret = rados_write_full(self.io, _key, _data, length)
2689 raise make_ex(ret, "Ioctx.write_full(%s): failed to write %s"
2692 raise LogicError("Ioctx.write_full(%s): rados_write_full \
2693 returned %d, but should return zero on success." % (self.name, ret))
2695 @requires(('key', str_type), ('data', bytes))
2696 def append(self, key, data):
2698 Append data to an object synchronously
2700 :param key: name of the object
2702 :param data: data to write
2705 :raises: :class:`TypeError`
2706 :raises: :class:`LogicError`
2707 :returns: int - 0 on success
2709 self.require_ioctx_open()
2710 key = cstr(key, 'key')
2714 size_t length = len(data)
2717 ret = rados_append(self.io, _key, _data, length)
2721 raise make_ex(ret, "Ioctx.append(%s): failed to append %s"
2724 raise LogicError("Ioctx.append(%s): rados_append \
2725 returned %d, but should return zero on success." % (self.name, ret))
2727 @requires(('key', str_type))
2728 def read(self, key, length=8192, offset=0):
2730 Read data from an object synchronously
2732 :param key: name of the object
2734 :param length: the number of bytes to read (default=8192)
2736 :param offset: byte offset in the object to begin reading at
2739 :raises: :class:`TypeError`
2740 :raises: :class:`Error`
2741 :returns: str - data read from object
2743 self.require_ioctx_open()
2744 key = cstr(key, 'key')
2748 uint64_t _offset = offset
2749 size_t _length = length
2750 PyObject* ret_s = NULL
2752 ret_s = PyBytes_FromStringAndSize(NULL, length)
2754 ret_buf = PyBytes_AsString(ret_s)
2756 ret = rados_read(self.io, _key, ret_buf, _length, _offset)
2758 raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key))
2761 _PyBytes_Resize(&ret_s, ret)
2763 return <object>ret_s
2765 # We DECREF unconditionally: the cast to object above will have
2766 # INCREFed if necessary. This also takes care of exceptions,
2767 # including if _PyString_Resize fails (that will free the string
2768 # itself and set ret_s to NULL, hence XDECREF).
2769 ref.Py_XDECREF(ret_s)
2771 @requires(('key', str_type), ('cls', str_type), ('method', str_type), ('data', bytes))
2772 def execute(self, key, cls, method, data, length=8192):
2774 Execute an OSD class method on an object.
2776 :param key: name of the object
2778 :param cls: name of the object class
2780 :param method: name of the method
2782 :param data: input data
2784 :param length: size of output buffer in bytes (default=8192)
2787 :raises: :class:`TypeError`
2788 :raises: :class:`Error`
2789 :returns: (ret, method output)
2791 self.require_ioctx_open()
2793 key = cstr(key, 'key')
2794 cls = cstr(cls, 'cls')
2795 method = cstr(method, 'method')
2799 char *_method = method
2801 size_t _data_len = len(data)
2804 size_t _length = length
2805 PyObject* ret_s = NULL
2807 ret_s = PyBytes_FromStringAndSize(NULL, length)
2809 ret_buf = PyBytes_AsString(ret_s)
2811 ret = rados_exec(self.io, _key, _cls, _method, _data,
2812 _data_len, ret_buf, _length)
2814 raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key))
2817 _PyBytes_Resize(&ret_s, ret)
2819 return ret, <object>ret_s
2821 # We DECREF unconditionally: the cast to object above will have
2822 # INCREFed if necessary. This also takes care of exceptions,
2823 # including if _PyString_Resize fails (that will free the string
2824 # itself and set ret_s to NULL, hence XDECREF).
2825 ref.Py_XDECREF(ret_s)
2827 def get_stats(self):
2829 Get pool usage statistics
2831 :returns: dict - contains the following keys:
2833 - ``num_bytes`` (int) - size of pool in bytes
2835 - ``num_kb`` (int) - size of pool in kbytes
2837 - ``num_objects`` (int) - number of objects in the pool
2839 - ``num_object_clones`` (int) - number of object clones
2841 - ``num_object_copies`` (int) - number of object copies
2843 - ``num_objects_missing_on_primary`` (int) - number of objets
2846 - ``num_objects_unfound`` (int) - number of unfound objects
2848 - ``num_objects_degraded`` (int) - number of degraded objects
2850 - ``num_rd`` (int) - bytes read
2852 - ``num_rd_kb`` (int) - kbytes read
2854 - ``num_wr`` (int) - bytes written
2856 - ``num_wr_kb`` (int) - kbytes written
2858 self.require_ioctx_open()
2859 cdef rados_pool_stat_t stats
2861 ret = rados_ioctx_pool_stat(self.io, &stats)
2863 raise make_ex(ret, "Ioctx.get_stats(%s): get_stats failed" % self.name)
2864 return {'num_bytes': stats.num_bytes,
2865 'num_kb': stats.num_kb,
2866 'num_objects': stats.num_objects,
2867 'num_object_clones': stats.num_object_clones,
2868 'num_object_copies': stats.num_object_copies,
2869 "num_objects_missing_on_primary": stats.num_objects_missing_on_primary,
2870 "num_objects_unfound": stats.num_objects_unfound,
2871 "num_objects_degraded": stats.num_objects_degraded,
2872 "num_rd": stats.num_rd,
2873 "num_rd_kb": stats.num_rd_kb,
2874 "num_wr": stats.num_wr,
2875 "num_wr_kb": stats.num_wr_kb}
2877 @requires(('key', str_type))
2878 def remove_object(self, key):
2882 This does not delete any snapshots of the object.
2884 :param key: the name of the object to delete
2887 :raises: :class:`TypeError`
2888 :raises: :class:`Error`
2889 :returns: bool - True on success
2891 self.require_ioctx_open()
2892 key = cstr(key, 'key')
2897 ret = rados_remove(self.io, _key)
2899 raise make_ex(ret, "Failed to remove '%s'" % key)
2902 @requires(('key', str_type))
2903 def trunc(self, key, size):
2907 If this enlarges the object, the new area is logically filled with
2908 zeroes. If this shrinks the object, the excess data is removed.
2910 :param key: the name of the object to resize
2912 :param size: the new size of the object in bytes
2915 :raises: :class:`TypeError`
2916 :raises: :class:`Error`
2917 :returns: int - 0 on success, otherwise raises error
2920 self.require_ioctx_open()
2921 key = cstr(key, 'key')
2924 uint64_t _size = size
2927 ret = rados_trunc(self.io, _key, _size)
2929 raise make_ex(ret, "Ioctx.trunc(%s): failed to truncate %s" % (self.name, key))
2932 @requires(('key', str_type))
2933 def stat(self, key):
2935 Get object stats (size/mtime)
2937 :param key: the name of the object to get stats from
2940 :raises: :class:`TypeError`
2941 :raises: :class:`Error`
2942 :returns: (size,timestamp)
2944 self.require_ioctx_open()
2946 key = cstr(key, 'key')
2953 ret = rados_stat(self.io, _key, &psize, &pmtime)
2955 raise make_ex(ret, "Failed to stat %r" % key)
2956 return psize, time.localtime(pmtime)
2958 @requires(('key', str_type), ('xattr_name', str_type))
2959 def get_xattr(self, key, xattr_name):
2961 Get the value of an extended attribute on an object.
2963 :param key: the name of the object to get xattr from
2965 :param xattr_name: which extended attribute to read
2966 :type xattr_name: str
2968 :raises: :class:`TypeError`
2969 :raises: :class:`Error`
2970 :returns: str - value of the xattr
2972 self.require_ioctx_open()
2974 key = cstr(key, 'key')
2975 xattr_name = cstr(xattr_name, 'xattr_name')
2978 char *_xattr_name = xattr_name
2979 size_t ret_length = 4096
2980 char *ret_buf = NULL
2983 while ret_length < 4096 * 1024 * 1024:
2984 ret_buf = <char *>realloc_chk(ret_buf, ret_length)
2986 ret = rados_getxattr(self.io, _key, _xattr_name, ret_buf, ret_length)
2987 if ret == -errno.ERANGE:
2990 raise make_ex(ret, "Failed to get xattr %r" % xattr_name)
2993 return ret_buf[:ret]
2997 @requires(('oid', str_type))
2998 def get_xattrs(self, oid):
3000 Start iterating over xattrs on an object.
3002 :param oid: the name of the object to get xattrs from
3005 :raises: :class:`TypeError`
3006 :raises: :class:`Error`
3007 :returns: XattrIterator
3009 self.require_ioctx_open()
3010 return XattrIterator(self, oid)
3012 @requires(('key', str_type), ('xattr_name', str_type), ('xattr_value', bytes))
3013 def set_xattr(self, key, xattr_name, xattr_value):
3015 Set an extended attribute on an object.
3017 :param key: the name of the object to set xattr to
3019 :param xattr_name: which extended attribute to set
3020 :type xattr_name: str
3021 :param xattr_value: the value of the extended attribute
3022 :type xattr_value: bytes
3024 :raises: :class:`TypeError`
3025 :raises: :class:`Error`
3026 :returns: bool - True on success, otherwise raise an error
3028 self.require_ioctx_open()
3030 key = cstr(key, 'key')
3031 xattr_name = cstr(xattr_name, 'xattr_name')
3034 char *_xattr_name = xattr_name
3035 char *_xattr_value = xattr_value
3036 size_t _xattr_value_len = len(xattr_value)
3039 ret = rados_setxattr(self.io, _key, _xattr_name,
3040 _xattr_value, _xattr_value_len)
3042 raise make_ex(ret, "Failed to set xattr %r" % xattr_name)
3045 @requires(('key', str_type), ('xattr_name', str_type))
3046 def rm_xattr(self, key, xattr_name):
3048 Removes an extended attribute on from an object.
3050 :param key: the name of the object to remove xattr from
3052 :param xattr_name: which extended attribute to remove
3053 :type xattr_name: str
3055 :raises: :class:`TypeError`
3056 :raises: :class:`Error`
3057 :returns: bool - True on success, otherwise raise an error
3059 self.require_ioctx_open()
3061 key = cstr(key, 'key')
3062 xattr_name = cstr(xattr_name, 'xattr_name')
3065 char *_xattr_name = xattr_name
3068 ret = rados_rmxattr(self.io, _key, _xattr_name)
3070 raise make_ex(ret, "Failed to delete key %r xattr %r" %
3074 @requires(('obj', str_type), ('msg', str_type), ('timeout_ms', int))
3075 def notify(self, obj, msg='', timeout_ms=5000):
3077 Send a rados notification to an object.
3079 :param obj: the name of the object to notify
3081 :param msg: optional message to send in the notification
3083 :param timeout_ms: notify timeout (in ms)
3084 :type timeout_ms: int
3086 :raises: :class:`TypeError`
3087 :raises: :class:`Error`
3088 :returns: bool - True on success, otherwise raise an error
3090 self.require_ioctx_open()
3093 obj = cstr(obj, 'obj')
3094 msg = cstr(msg, 'msg')
3098 int _msglen = msglen
3099 uint64_t _timeout_ms = timeout_ms
3102 ret = rados_notify2(self.io, _obj, _msg, _msglen, _timeout_ms,
3105 raise make_ex(ret, "Failed to notify %r" % (obj))
3108 def list_objects(self):
3110 Get ObjectIterator on rados.Ioctx object.
3112 :returns: ObjectIterator
3114 self.require_ioctx_open()
3115 return ObjectIterator(self)
3117 def list_snaps(self):
3119 Get SnapIterator on rados.Ioctx object.
3121 :returns: SnapIterator
3123 self.require_ioctx_open()
3124 return SnapIterator(self)
3126 @requires(('snap_name', str_type))
3127 def create_snap(self, snap_name):
3129 Create a pool-wide snapshot
3131 :param snap_name: the name of the snapshot
3132 :type snap_name: str
3134 :raises: :class:`TypeError`
3135 :raises: :class:`Error`
3137 self.require_ioctx_open()
3138 snap_name = cstr(snap_name, 'snap_name')
3139 cdef char *_snap_name = snap_name
3142 ret = rados_ioctx_snap_create(self.io, _snap_name)
3144 raise make_ex(ret, "Failed to create snap %s" % snap_name)
3146 @requires(('snap_name', str_type))
3147 def remove_snap(self, snap_name):
3149 Removes a pool-wide snapshot
3151 :param snap_name: the name of the snapshot
3152 :type snap_name: str
3154 :raises: :class:`TypeError`
3155 :raises: :class:`Error`
3157 self.require_ioctx_open()
3158 snap_name = cstr(snap_name, 'snap_name')
3159 cdef char *_snap_name = snap_name
3162 ret = rados_ioctx_snap_remove(self.io, _snap_name)
3164 raise make_ex(ret, "Failed to remove snap %s" % snap_name)
3166 @requires(('snap_name', str_type))
3167 def lookup_snap(self, snap_name):
3169 Get the id of a pool snapshot
3171 :param snap_name: the name of the snapshot to lookop
3172 :type snap_name: str
3174 :raises: :class:`TypeError`
3175 :raises: :class:`Error`
3176 :returns: Snap - on success
3178 self.require_ioctx_open()
3179 csnap_name = cstr(snap_name, 'snap_name')
3181 char *_snap_name = csnap_name
3182 rados_snap_t snap_id
3185 ret = rados_ioctx_snap_lookup(self.io, _snap_name, &snap_id)
3187 raise make_ex(ret, "Failed to lookup snap %s" % snap_name)
3188 return Snap(self, snap_name, int(snap_id))
3190 @requires(('oid', str_type), ('snap_name', str_type))
3191 def snap_rollback(self, oid, snap_name):
3193 Rollback an object to a snapshot
3195 :param oid: the name of the object
3197 :param snap_name: the name of the snapshot
3198 :type snap_name: str
3200 :raises: :class:`TypeError`
3201 :raises: :class:`Error`
3203 self.require_ioctx_open()
3204 oid = cstr(oid, 'oid')
3205 snap_name = cstr(snap_name, 'snap_name')
3207 char *_snap_name = snap_name
3211 ret = rados_ioctx_snap_rollback(self.io, _oid, _snap_name)
3213 raise make_ex(ret, "Failed to rollback %s" % oid)
3215 def create_self_managed_snap(self):
3217 Creates a self-managed snapshot
3219 :returns: snap id on success
3221 :raises: :class:`Error`
3223 self.require_ioctx_open()
3225 rados_snap_t _snap_id
3227 ret = rados_ioctx_selfmanaged_snap_create(self.io, &_snap_id)
3229 raise make_ex(ret, "Failed to create self-managed snapshot")
3230 return int(_snap_id)
3232 @requires(('snap_id', int))
3233 def remove_self_managed_snap(self, snap_id):
3235 Removes a self-managed snapshot
3237 :param snap_id: the name of the snapshot
3240 :raises: :class:`TypeError`
3241 :raises: :class:`Error`
3243 self.require_ioctx_open()
3245 rados_snap_t _snap_id = snap_id
3247 ret = rados_ioctx_selfmanaged_snap_remove(self.io, _snap_id)
3249 raise make_ex(ret, "Failed to remove self-managed snapshot")
3251 def set_self_managed_snap_write(self, snaps):
3253 Updates the write context to the specified self-managed
3256 :param snaps: all associated self-managed snapshot ids
3259 :raises: :class:`TypeError`
3260 :raises: :class:`Error`
3262 self.require_ioctx_open()
3266 sorted_snaps = sorted([int(x) for x in snaps], reverse=True)
3267 snap_seq = sorted_snaps[0]
3270 rados_snap_t _snap_seq = snap_seq
3271 rados_snap_t *_snaps = NULL
3272 int _num_snaps = len(sorted_snaps)
3274 _snaps = <rados_snap_t *>malloc(_num_snaps * sizeof(rados_snap_t))
3275 for i in range(len(sorted_snaps)):
3276 _snaps[i] = sorted_snaps[i]
3278 ret = rados_ioctx_selfmanaged_snap_set_write_ctx(self.io,
3283 raise make_ex(ret, "Failed to update snapshot write context")
3287 @requires(('oid', str_type), ('snap_id', int))
3288 def rollback_self_managed_snap(self, oid, snap_id):
3290 Rolls an specific object back to a self-managed snapshot revision
3292 :param oid: the name of the object
3294 :param snap_id: the name of the snapshot
3297 :raises: :class:`TypeError`
3298 :raises: :class:`Error`
3300 self.require_ioctx_open()
3301 oid = cstr(oid, 'oid')
3304 rados_snap_t _snap_id = snap_id
3306 ret = rados_ioctx_selfmanaged_snap_rollback(self.io, _oid, _snap_id)
3308 raise make_ex(ret, "Failed to rollback %s" % oid)
3310 def get_last_version(self):
3312 Return the version of the last object read or written to.
3314 This exposes the internal version number of the last object read or
3315 written via this io context
3317 :returns: version of the last object used
3319 self.require_ioctx_open()
3321 ret = rados_get_last_version(self.io)
3324 def create_write_op(self):
3326 create write operation object.
3327 need call release_write_op after use
3329 return WriteOp().create()
3331 def create_read_op(self):
3333 create read operation object.
3334 need call release_read_op after use
3336 return ReadOp().create()
3338 def release_write_op(self, write_op):
3340 release memory alloc by create_write_op
3344 def release_read_op(self, read_op):
3346 release memory alloc by create_read_op
3347 :para read_op: read_op object
3352 @requires(('write_op', WriteOp), ('keys', tuple), ('values', tuple))
3353 def set_omap(self, write_op, keys, values):
3355 set keys values to write_op
3356 :para write_op: write_operation object
3357 :type write_op: WriteOp
3358 :para keys: a tuple of keys
3360 :para values: a tuple of values
3364 if len(keys) != len(values):
3365 raise Error("Rados(): keys and values must have the same number of items")
3367 keys = cstr_list(keys, 'keys')
3368 values = cstr_list(values, 'values')
3369 lens = [len(v) for v in values]
3371 WriteOp _write_op = write_op
3372 size_t key_num = len(keys)
3373 char **_keys = to_bytes_array(keys)
3374 char **_values = to_bytes_array(values)
3375 size_t *_lens = to_csize_t_array(lens)
3379 rados_write_op_omap_set(_write_op.write_op,
3380 <const char**>_keys,
3381 <const char**>_values,
3382 <const size_t*>_lens, key_num)
3388 @requires(('write_op', WriteOp), ('oid', str_type), ('mtime', opt(int)), ('flags', opt(int)))
3389 def operate_write_op(self, write_op, oid, mtime=0, flags=LIBRADOS_OPERATION_NOFLAG):
3391 execute the real write operation
3392 :para write_op: write operation object
3393 :type write_op: WriteOp
3394 :para oid: object name
3396 :para mtime: the time to set the mtime to, 0 for the current time
3398 :para flags: flags to apply to the entire operation
3402 oid = cstr(oid, 'oid')
3404 WriteOp _write_op = write_op
3406 time_t _mtime = mtime
3410 ret = rados_write_op_operate(_write_op.write_op, self.io, _oid, &_mtime, _flags)
3412 raise make_ex(ret, "Failed to operate write op for oid %s" % oid)
3414 @requires(('write_op', WriteOp), ('oid', str_type), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)), ('mtime', opt(int)), ('flags', opt(int)))
3415 def operate_aio_write_op(self, write_op, oid, oncomplete=None, onsafe=None, mtime=0, flags=LIBRADOS_OPERATION_NOFLAG):
3417 execute the real write operation asynchronously
3418 :para write_op: write operation object
3419 :type write_op: WriteOp
3420 :para oid: object name
3422 :param oncomplete: what to do when the remove is safe and complete in memory
3424 :type oncomplete: completion
3425 :param onsafe: what to do when the remove is safe and complete on storage
3427 :type onsafe: completion
3428 :para mtime: the time to set the mtime to, 0 for the current time
3430 :para flags: flags to apply to the entire operation
3433 :raises: :class:`Error`
3434 :returns: completion object
3437 oid = cstr(oid, 'oid')
3439 WriteOp _write_op = write_op
3441 Completion completion
3442 time_t _mtime = mtime
3445 completion = self.__get_completion(oncomplete, onsafe)
3446 self.__track_completion(completion)
3449 ret = rados_aio_write_op_operate(_write_op.write_op, self.io, completion.rados_comp, _oid,
3452 completion._cleanup()
3453 raise make_ex(ret, "Failed to operate aio write op for oid %s" % oid)
3456 @requires(('read_op', ReadOp), ('oid', str_type), ('flag', opt(int)))
3457 def operate_read_op(self, read_op, oid, flag=LIBRADOS_OPERATION_NOFLAG):
3459 execute the real read operation
3460 :para read_op: read operation object
3461 :type read_op: ReadOp
3462 :para oid: object name
3464 :para flag: flags to apply to the entire operation
3467 oid = cstr(oid, 'oid')
3469 ReadOp _read_op = read_op
3474 ret = rados_read_op_operate(_read_op.read_op, self.io, _oid, _flag)
3476 raise make_ex(ret, "Failed to operate read op for oid %s" % oid)
3478 @requires(('read_op', ReadOp), ('oid', str_type), ('oncomplete', opt(Callable)), ('onsafe', opt(Callable)), ('flag', opt(int)))
3479 def operate_aio_read_op(self, read_op, oid, oncomplete=None, onsafe=None, flag=LIBRADOS_OPERATION_NOFLAG):
3481 execute the real read operation
3482 :para read_op: read operation object
3483 :type read_op: ReadOp
3484 :para oid: object name
3486 :param oncomplete: what to do when the remove is safe and complete in memory
3488 :type oncomplete: completion
3489 :param onsafe: what to do when the remove is safe and complete on storage
3491 :type onsafe: completion
3492 :para flag: flags to apply to the entire operation
3495 oid = cstr(oid, 'oid')
3497 ReadOp _read_op = read_op
3499 Completion completion
3502 completion = self.__get_completion(oncomplete, onsafe)
3503 self.__track_completion(completion)
3506 ret = rados_aio_read_op_operate(_read_op.read_op, self.io, completion.rados_comp, _oid, _flag)
3508 completion._cleanup()
3509 raise make_ex(ret, "Failed to operate aio read op for oid %s" % oid)
3512 @requires(('read_op', ReadOp), ('start_after', str_type), ('filter_prefix', str_type), ('max_return', int))
3513 def get_omap_vals(self, read_op, start_after, filter_prefix, max_return):
3516 :para read_op: read operation object
3517 :type read_op: ReadOp
3518 :para start_after: list keys starting after start_after
3519 :type start_after: str
3520 :para filter_prefix: list only keys beginning with filter_prefix
3521 :type filter_prefix: str
3522 :para max_return: list no more than max_return key/value pairs
3523 :type max_return: int
3524 :returns: an iterator over the requested omap values, return value from this action
3527 start_after = cstr(start_after, 'start_after') if start_after else None
3528 filter_prefix = cstr(filter_prefix, 'filter_prefix') if filter_prefix else None
3530 char *_start_after = opt_str(start_after)
3531 char *_filter_prefix = opt_str(filter_prefix)
3532 ReadOp _read_op = read_op
3533 rados_omap_iter_t iter_addr = NULL
3534 int _max_return = max_return
3537 rados_read_op_omap_get_vals2(_read_op.read_op, _start_after, _filter_prefix,
3538 _max_return, &iter_addr, NULL, NULL)
3539 it = OmapIterator(self)
3541 return it, 0 # 0 is meaningless; there for backward-compat
3543 @requires(('read_op', ReadOp), ('start_after', str_type), ('max_return', int))
3544 def get_omap_keys(self, read_op, start_after, max_return):
3547 :para read_op: read operation object
3548 :type read_op: ReadOp
3549 :para start_after: list keys starting after start_after
3550 :type start_after: str
3551 :para max_return: list no more than max_return key/value pairs
3552 :type max_return: int
3553 :returns: an iterator over the requested omap values, return value from this action
3555 start_after = cstr(start_after, 'start_after') if start_after else None
3557 char *_start_after = opt_str(start_after)
3558 ReadOp _read_op = read_op
3559 rados_omap_iter_t iter_addr = NULL
3560 int _max_return = max_return
3563 rados_read_op_omap_get_keys2(_read_op.read_op, _start_after,
3564 _max_return, &iter_addr, NULL, NULL)
3565 it = OmapIterator(self)
3567 return it, 0 # 0 is meaningless; there for backward-compat
3569 @requires(('read_op', ReadOp), ('keys', tuple))
3570 def get_omap_vals_by_keys(self, read_op, keys):
3572 get the omap values by keys
3573 :para read_op: read operation object
3574 :type read_op: ReadOp
3575 :para keys: input key tuple
3577 :returns: an iterator over the requested omap values, return value from this action
3579 keys = cstr_list(keys, 'keys')
3581 ReadOp _read_op = read_op
3582 rados_omap_iter_t iter_addr
3583 char **_keys = to_bytes_array(keys)
3584 size_t key_num = len(keys)
3588 rados_read_op_omap_get_vals_by_keys(_read_op.read_op,
3589 <const char**>_keys,
3590 key_num, &iter_addr, NULL)
3591 it = OmapIterator(self)
3593 return it, 0 # 0 is meaningless; there for backward-compat
3597 @requires(('write_op', WriteOp), ('keys', tuple))
3598 def remove_omap_keys(self, write_op, keys):
3600 remove omap keys specifiled
3601 :para write_op: write operation object
3602 :type write_op: WriteOp
3603 :para keys: input key tuple
3607 keys = cstr_list(keys, 'keys')
3609 WriteOp _write_op = write_op
3610 size_t key_num = len(keys)
3611 char **_keys = to_bytes_array(keys)
3615 rados_write_op_omap_rm_keys(_write_op.write_op, <const char**>_keys, key_num)
3619 @requires(('write_op', WriteOp))
3620 def clear_omap(self, write_op):
3622 Remove all key/value pairs from an object
3623 :para write_op: write operation object
3624 :type write_op: WriteOp
3628 WriteOp _write_op = write_op
3631 rados_write_op_omap_clear(_write_op.write_op)
3633 @requires(('key', str_type), ('name', str_type), ('cookie', str_type), ('desc', str_type),
3634 ('duration', opt(int)), ('flags', int))
3635 def lock_exclusive(self, key, name, cookie, desc="", duration=None, flags=0):
3638 Take an exclusive lock on an object
3640 :param key: name of the object
3642 :param name: name of the lock
3644 :param cookie: cookie of the lock
3646 :param desc: description of the lock
3648 :param duration: duration of the lock in seconds
3653 :raises: :class:`TypeError`
3654 :raises: :class:`Error`
3656 self.require_ioctx_open()
3658 key = cstr(key, 'key')
3659 name = cstr(name, 'name')
3660 cookie = cstr(cookie, 'cookie')
3661 desc = cstr(desc, 'desc')
3666 char* _cookie = cookie
3668 uint8_t _flags = flags
3671 if duration is None:
3673 ret = rados_lock_exclusive(self.io, _key, _name, _cookie, _desc,
3676 _duration.tv_sec = duration
3678 ret = rados_lock_exclusive(self.io, _key, _name, _cookie, _desc,
3682 raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key))
3684 @requires(('key', str_type), ('name', str_type), ('cookie', str_type), ('tag', str_type),
3685 ('desc', str_type), ('duration', opt(int)), ('flags', int))
3686 def lock_shared(self, key, name, cookie, tag, desc="", duration=None, flags=0):
3689 Take a shared lock on an object
3691 :param key: name of the object
3693 :param name: name of the lock
3695 :param cookie: cookie of the lock
3697 :param tag: tag of the lock
3699 :param desc: description of the lock
3701 :param duration: duration of the lock in seconds
3706 :raises: :class:`TypeError`
3707 :raises: :class:`Error`
3709 self.require_ioctx_open()
3711 key = cstr(key, 'key')
3712 tag = cstr(tag, 'tag')
3713 name = cstr(name, 'name')
3714 cookie = cstr(cookie, 'cookie')
3715 desc = cstr(desc, 'desc')
3721 char* _cookie = cookie
3723 uint8_t _flags = flags
3726 if duration is None:
3728 ret = rados_lock_shared(self.io, _key, _name, _cookie, _tag, _desc,
3731 _duration.tv_sec = duration
3733 ret = rados_lock_shared(self.io, _key, _name, _cookie, _tag, _desc,
3736 raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key))
3738 @requires(('key', str_type), ('name', str_type), ('cookie', str_type))
3739 def unlock(self, key, name, cookie):
3742 Release a shared or exclusive lock on an object
3744 :param key: name of the object
3746 :param name: name of the lock
3748 :param cookie: cookie of the lock
3751 :raises: :class:`TypeError`
3752 :raises: :class:`Error`
3754 self.require_ioctx_open()
3756 key = cstr(key, 'key')
3757 name = cstr(name, 'name')
3758 cookie = cstr(cookie, 'cookie')
3763 char* _cookie = cookie
3766 ret = rados_unlock(self.io, _key, _name, _cookie)
3768 raise make_ex(ret, "Ioctx.rados_lock_exclusive(%s): failed to set lock %s on %s" % (self.name, name, key))
3770 def set_osdmap_full_try(self):
3772 Set global osdmap_full_try label to true
3775 rados_set_osdmap_full_try(self.io)
3777 def unset_osdmap_full_try(self):
3782 rados_unset_osdmap_full_try(self.io)
3784 def application_enable(self, app_name, force=False):
3786 Enable an application on an OSD pool
3788 :param app_name: application name
3790 :param force: False if only a single app should exist per pool
3791 :type expire_seconds: boool
3793 :raises: :class:`Error`
3795 app_name = cstr(app_name, 'app_name')
3797 char *_app_name = app_name
3798 int _force = (1 if force else 0)
3801 ret = rados_application_enable(self.io, _app_name, _force)
3803 raise make_ex(ret, "error enabling application")
3805 def application_list(self):
3807 Returns a list of enabled applications
3809 :returns: list of app name string
3817 apps = <char *>realloc_chk(apps, length)
3819 ret = rados_application_list(self.io, apps, &length)
3821 return [decode_cstr(app) for app in
3822 apps[:length].split(b'\0') if app]
3823 elif ret == -errno.ENOENT:
3825 elif ret == -errno.ERANGE:
3828 raise make_ex(ret, "error listing applications")
3832 def application_metadata_set(self, app_name, key, value):
3834 Sets application metadata on an OSD pool
3836 :param app_name: application name
3838 :param key: metadata key
3840 :param value: metadata value
3843 :raises: :class:`Error`
3845 app_name = cstr(app_name, 'app_name')
3846 key = cstr(key, 'key')
3847 value = cstr(value, 'value')
3849 char *_app_name = app_name
3851 char *_value = value
3854 ret = rados_application_metadata_set(self.io, _app_name, _key,
3857 raise make_ex(ret, "error setting application metadata")
3859 def application_metadata_remove(self, app_name, key):
3861 Remove application metadata from an OSD pool
3863 :param app_name: application name
3865 :param key: metadata key
3868 :raises: :class:`Error`
3870 app_name = cstr(app_name, 'app_name')
3871 key = cstr(key, 'key')
3873 char *_app_name = app_name
3877 ret = rados_application_metadata_remove(self.io, _app_name, _key)
3879 raise make_ex(ret, "error removing application metadata")
3881 def application_metadata_list(self, app_name):
3883 Returns a list of enabled applications
3885 :param app_name: application name
3887 :returns: list of key/value tuples
3889 app_name = cstr(app_name, 'app_name')
3891 char *_app_name = app_name
3892 size_t key_length = 128
3893 size_t val_length = 128
3899 c_keys = <char *>realloc_chk(c_keys, key_length)
3900 c_vals = <char *>realloc_chk(c_vals, val_length)
3902 ret = rados_application_metadata_list(self.io, _app_name,
3903 c_keys, &key_length,
3904 c_vals, &val_length)
3906 keys = [decode_cstr(key) for key in
3907 c_keys[:key_length].split(b'\0')]
3908 vals = [decode_cstr(val) for val in
3909 c_vals[:val_length].split(b'\0')]
3910 return zip(keys, vals)[:-1]
3911 elif ret == -errno.ERANGE:
3914 raise make_ex(ret, "error listing application metadata")
3919 def alignment(self):
3921 Returns pool alignment
3924 Number of alignment bytes required by the current pool, or None if
3925 alignment is not required.
3932 ret = rados_ioctx_pool_requires_alignment2(self.io, &requires)
3934 raise make_ex(ret, "error checking alignment")
3939 ret = rados_ioctx_pool_required_alignment2(self.io, &_alignment)
3941 raise make_ex(ret, "error querying alignment")
3942 alignment = _alignment
3946 def set_object_locator(func):
3947 def retfunc(self, *args, **kwargs):
3948 if self.locator_key is not None:
3949 old_locator = self.ioctx.get_locator_key()
3950 self.ioctx.set_locator_key(self.locator_key)
3951 retval = func(self, *args, **kwargs)
3952 self.ioctx.set_locator_key(old_locator)
3955 return func(self, *args, **kwargs)
3959 def set_object_namespace(func):
3960 def retfunc(self, *args, **kwargs):
3961 if self.nspace is None:
3962 raise LogicError("Namespace not set properly in context")
3963 old_nspace = self.ioctx.get_namespace()
3964 self.ioctx.set_namespace(self.nspace)
3965 retval = func(self, *args, **kwargs)
3966 self.ioctx.set_namespace(old_nspace)
3971 class Object(object):
3972 """Rados object wrapper, makes the object look like a file"""
3973 def __init__(self, ioctx, key, locator_key=None, nspace=None):
3977 self.state = "exists"
3978 self.locator_key = locator_key
3979 self.nspace = "" if nspace is None else nspace
3982 return "rados.Object(ioctx=%s,key=%s,nspace=%s,locator=%s)" % \
3983 (str(self.ioctx), self.key, "--default--"
3984 if self.nspace is "" else self.nspace, self.locator_key)
3986 def require_object_exists(self):
3987 if self.state != "exists":
3988 raise ObjectStateError("The object is %s" % self.state)
3991 @set_object_namespace
3992 def read(self, length=1024 * 1024):
3993 self.require_object_exists()
3994 ret = self.ioctx.read(self.key, length, self.offset)
3995 self.offset += len(ret)
3999 @set_object_namespace
4000 def write(self, string_to_write):
4001 self.require_object_exists()
4002 ret = self.ioctx.write(self.key, string_to_write, self.offset)
4004 self.offset += len(string_to_write)
4008 @set_object_namespace
4010 self.require_object_exists()
4011 self.ioctx.remove_object(self.key)
4012 self.state = "removed"
4015 @set_object_namespace
4017 self.require_object_exists()
4018 return self.ioctx.stat(self.key)
4020 def seek(self, position):
4021 self.require_object_exists()
4022 self.offset = position
4025 @set_object_namespace
4026 def get_xattr(self, xattr_name):
4027 self.require_object_exists()
4028 return self.ioctx.get_xattr(self.key, xattr_name)
4031 @set_object_namespace
4032 def get_xattrs(self):
4033 self.require_object_exists()
4034 return self.ioctx.get_xattrs(self.key)
4037 @set_object_namespace
4038 def set_xattr(self, xattr_name, xattr_value):
4039 self.require_object_exists()
4040 return self.ioctx.set_xattr(self.key, xattr_name, xattr_value)
4043 @set_object_namespace
4044 def rm_xattr(self, xattr_name):
4045 self.require_object_exists()
4046 return self.ioctx.rm_xattr(self.key, xattr_name)
4057 class MonitorLog(object):
4058 # NOTE(sileht): Keep this class for backward compat
4059 # method moved to Rados.monitor_log()
4061 For watching cluster log messages. Instantiate an object and keep
4062 it around while callback is periodically called. Construct with
4063 'level' to monitor 'level' messages (one of MONITOR_LEVELS).
4064 arg will be passed to the callback.
4066 callback will be called with:
4067 arg (given to __init__)
4068 line (the full line, including timestamp, who, level, msg)
4069 who (which entity issued the log message)
4070 timestamp_sec (sec of a struct timespec)
4071 timestamp_nsec (sec of a struct timespec)
4072 seq (sequence number)
4073 level (string representing the level of the log message)
4074 msg (the message itself)
4075 callback's return value is ignored
4077 def __init__(self, cluster, level, callback, arg):
4079 self.callback = callback
4081 self.cluster = cluster
4082 self.cluster.monitor_log(level, callback, arg)