ceph/src/pybind/rgw/rgw.pyx

   1 """
   2 This module is a thin wrapper around rgw_file.
   3 """
   4
   5
   6 from cpython cimport PyObject, ref, exc, array
   7 from libc.stdint cimport *
   8 from libc.stdlib cimport malloc, realloc, free
   9 from cstat cimport stat
  10
  11 IF BUILD_DOC:
  12     include "mock_rgw.pxi"
  13 ELSE:
  14     from c_rgw cimport *
  15     cimport rados
  16
  17 from collections import namedtuple
  18 from datetime import datetime
  19 import errno
  20
  21
  22 cdef extern from "Python.h":
  23     # These are in cpython/string.pxd, but use "object" types instead of
  24     # PyObject*, which invokes assumptions in cpython that we need to
  25     # legitimately break to implement zero-copy string buffers in Image.read().
  26     # This is valid use of the Python API and documented as a special case.
  27     PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL
  28     char* PyBytes_AsString(PyObject *string) except NULL
  29     int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1
  30     void PyEval_InitThreads()
  31
  32
  33 class Error(Exception):
  34     pass
  35
  36
  37 class OSError(Error):
  38     """ `OSError` class, derived from `Error` """
  39     def __init__(self, errno, strerror):
  40         self.errno = errno
  41         self.strerror = strerror
  42
  43     def __str__(self):
  44         return '[Errno {0}] {1}'.format(self.errno, self.strerror)
  45
  46
  47 class PermissionError(OSError):
  48     pass
  49
  50
  51 class ObjectNotFound(OSError):
  52     pass
  53
  54
  55 class NoData(Error):
  56     pass
  57
  58
  59 class ObjectExists(Error):
  60     pass
  61
  62
  63 class IOError(OSError):
  64     pass
  65
  66
  67 class NoSpace(Error):
  68     pass
  69
  70
  71 class InvalidValue(Error):
  72     pass
  73
  74
  75 class OperationNotSupported(Error):
  76     pass
  77
  78
  79 class IncompleteWriteError(Error):
  80     pass
  81
  82
  83 class LibCephFSStateError(Error):
  84     pass
  85
  86 class WouldBlock(Error):
  87     pass
  88
  89 class OutOfRange(Error):
  90     pass
  91
  92 IF UNAME_SYSNAME == "FreeBSD":
  93     cdef errno_to_exception =  {
  94         errno.EPERM      : PermissionError,
  95         errno.ENOENT     : ObjectNotFound,
  96         errno.EIO        : IOError,
  97         errno.ENOSPC     : NoSpace,
  98         errno.EEXIST     : ObjectExists,
  99         errno.ENOATTR    : NoData,
 100         errno.EINVAL     : InvalidValue,
 101         errno.EOPNOTSUPP : OperationNotSupported,
 102         errno.ERANGE     : OutOfRange,
 103         errno.EWOULDBLOCK: WouldBlock,
 104     }
 105 ELSE:
 106     cdef errno_to_exception =  {
 107         errno.EPERM      : PermissionError,
 108         errno.ENOENT     : ObjectNotFound,
 109         errno.EIO        : IOError,
 110         errno.ENOSPC     : NoSpace,
 111         errno.EEXIST     : ObjectExists,
 112         errno.ENODATA    : NoData,
 113         errno.EINVAL     : InvalidValue,
 114         errno.EOPNOTSUPP : OperationNotSupported,
 115         errno.ERANGE     : OutOfRange,
 116         errno.EWOULDBLOCK: WouldBlock,
 117     }
 118
 119
 120 cdef class FileHandle(object):
 121     cdef rgw_file_handle *handler
 122
 123
 124 StatResult = namedtuple('StatResult',
 125                         ["st_dev", "st_ino", "st_mode", "st_nlink", "st_uid",
 126                          "st_gid", "st_rdev", "st_size", "st_blksize",
 127                          "st_blocks", "st_atime", "st_mtime", "st_ctime"])
 128
 129
 130 def cstr(val, name, encoding="utf-8", opt=False):
 131     """
 132     Create a byte string from a Python string
 133
 134     :param basestring val: Python string
 135     :param str name: Name of the string parameter, for exceptions
 136     :param str encoding: Encoding to use
 137     :param bool opt: If True, None is allowed
 138     :rtype: bytes
 139     :raises: :class:`InvalidArgument`
 140     """
 141     if opt and val is None:
 142         return None
 143     if isinstance(val, bytes):
 144         return val
 145     elif isinstance(val, str):
 146         return val.encode(encoding)
 147     else:
 148         raise TypeError('%s must be a string' % name)
 149
 150
 151 cdef make_ex(ret, msg):
 152     """
 153     Translate a librados return code into an exception.
 154
 155     :param ret: the return code
 156     :type ret: int
 157     :param msg: the error message to use
 158     :type msg: str
 159     :returns: a subclass of :class:`Error`
 160     """
 161     ret = abs(ret)
 162     if ret in errno_to_exception:
 163         return errno_to_exception[ret](ret, msg)
 164     else:
 165         return Error(msg + (": error code %d" % ret))
 166
 167
 168 cdef bint readdir_cb(const char *name, void *arg, uint64_t offset, stat *st, uint32_t st_mask, uint32_t flags) \
 169 except? -9000 with gil:
 170     if exc.PyErr_Occurred():
 171         return False
 172     (<object>arg)(name, offset, flags)
 173     return True
 174
 175
 176 class LibCephFSStateError(Error):
 177     pass
 178
 179
 180 cdef class LibRGWFS(object):
 181     """librgwfs python wrapper"""
 182
 183     cdef public object state
 184     cdef public object uid
 185     cdef public object key
 186     cdef public object secret
 187     cdef librgw_t cluster
 188     cdef rgw_fs *fs
 189
 190     def require_state(self, *args):
 191         if self.state in args:
 192             return
 193         raise LibCephFSStateError("You cannot perform that operation on a "
 194                                   "RGWFS object in state %s." % (self.state))
 195
 196     def __cinit__(self, uid, key, secret):
 197         PyEval_InitThreads()
 198         self.state = "umounted"
 199         ret = librgw_create(&self.cluster, 0, NULL)
 200         if ret != 0:
 201             raise make_ex(ret, "error calling librgw_create")
 202         self.uid = cstr(uid, "uid")
 203         self.key = cstr(key, "key")
 204         self.secret = cstr(secret, "secret")
 205
 206     def shutdown(self):
 207         """
 208         Unmount and destroy the ceph mount handle.
 209         """
 210         if self.state in ["mounted"]:
 211             with nogil:
 212                 ret = rgw_umount(self.fs, 0);
 213             if ret != 0:
 214                 raise make_ex(ret, "error calling rgw_unmount")
 215             self.state = "shutdown"
 216
 217     def __enter__(self):
 218         self.mount()
 219         return self
 220
 221     def __exit__(self, type_, value, traceback):
 222         self.shutdown()
 223
 224     def __dealloc__(self):
 225         self.shutdown()
 226
 227     def version(self):
 228         """
 229         Get the version number of the ``librgwfile`` C library.
 230
 231         :returns: a tuple of ``(major, minor, extra)`` components of the
 232                   libcephfs version
 233         """
 234         cdef:
 235             int major = 0
 236             int minor = 0
 237             int extra = 0
 238         with nogil:
 239             rgwfile_version(&major, &minor, &extra)
 240         return (major, minor, extra)
 241
 242     def mount(self):
 243         self.require_state("umounted")
 244         cdef:
 245             char *_uid = self.uid
 246             char *_key = self.key
 247             char *_secret = self.secret
 248         with nogil:
 249             ret = rgw_mount(self.cluster, <const char*>_uid, <const char*>_key,
 250                             <const char*>_secret, &self.fs, 0)
 251         if ret != 0:
 252             raise make_ex(ret, "error calling rgw_mount")
 253         self.state = "mounted"
 254         dir_handler = FileHandle()
 255         dir_handler.handler = <rgw_file_handle*>self.fs.root_fh
 256         return dir_handler
 257
 258     def unmount(self):
 259         self.require_state("mounted")
 260         with nogil:
 261             ret = rgw_umount(self.fs, 0)
 262         if ret != 0:
 263             raise make_ex(ret, "error calling rgw_umount")
 264         self.state = "umounted"
 265
 266     def statfs(self):
 267         self.require_state("mounted")
 268         cdef:
 269             rgw_statvfs statbuf
 270
 271         with nogil:
 272             ret = rgw_statfs(self.fs, <rgw_file_handle*>self.fs.root_fh, &statbuf, 0)
 273         if ret < 0:
 274             raise make_ex(ret, "statfs failed")
 275         cdef uint64_t[:] fsid = statbuf.f_fsid
 276         return {'f_bsize': statbuf.f_bsize,
 277                 'f_frsize': statbuf.f_frsize,
 278                 'f_blocks': statbuf.f_blocks,
 279                 'f_bfree': statbuf.f_bfree,
 280                 'f_bavail': statbuf.f_bavail,
 281                 'f_files': statbuf.f_files,
 282                 'f_ffree': statbuf.f_ffree,
 283                 'f_favail': statbuf.f_favail,
 284                 'f_fsid': fsid,
 285                 'f_flag': statbuf.f_flag,
 286                 'f_namemax': statbuf.f_namemax}
 287
 288
 289     def create(self, FileHandle dir_handler, filename, flags = 0):
 290         self.require_state("mounted")
 291
 292         if not isinstance(flags, int):
 293             raise TypeError("flags must be an integer")
 294
 295         filename = cstr(filename, 'filename')
 296
 297         cdef:
 298             rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
 299             rgw_file_handle *_file_handler
 300             int _flags = flags
 301             char* _filename = filename
 302             stat statbuf
 303
 304         with nogil:
 305             ret = rgw_create(self.fs, _dir_handler, _filename, &statbuf, 0,
 306                              &_file_handler, 0, _flags)
 307         if ret < 0:
 308             raise make_ex(ret, "error in create '%s'" % filename)
 309         with nogil:
 310             ret = rgw_open(self.fs, _file_handler, 0, _flags)
 311         if ret < 0:
 312             raise make_ex(ret, "error in open '%s'" % filename)
 313
 314         file_handler = FileHandle()
 315         file_handler.handler = _file_handler
 316         return file_handler
 317
 318     def mkdir(self, FileHandle dir_handler, dirname, flags = 0):
 319         self.require_state("mounted")
 320         dirname = cstr(dirname, 'dirname')
 321         new_dir_handler = FileHandle()
 322         cdef:
 323             rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
 324             rgw_file_handle *_new_dir_handler
 325             char* _dirname = dirname
 326             int _flags = flags
 327             stat statbuf
 328         with nogil:
 329             ret = rgw_mkdir(self.fs, _dir_handler, _dirname, &statbuf,
 330                             0, &_new_dir_handler, _flags)
 331         if ret < 0:
 332             raise make_ex(ret, "error in mkdir '%s'" % dirname)
 333         new_dir_handler.handler = _new_dir_handler
 334         return new_dir_handler
 335
 336     def rename(self, FileHandle src_handler, src_name, FileHandle dst_handler, dst_name, flags = 0):
 337         self.require_state("mounted")
 338
 339         src_name = cstr(src_name, 'src_name')
 340         dst_name = cstr(dst_name, 'dst_name')
 341
 342         cdef:
 343             rgw_file_handle *_src_dir_handler = <rgw_file_handle*>src_handler.handler
 344             rgw_file_handle *_dst_dir_handler = <rgw_file_handle*>dst_handler.handler
 345             char* _src_name = src_name
 346             char* _dst_name = dst_name
 347             int _flags = flags
 348
 349         with nogil:
 350             ret = rgw_rename(self.fs, _src_dir_handler, _src_name,
 351                              _dst_dir_handler, _dst_name, _flags)
 352         if ret < 0:
 353             raise make_ex(ret, "error in rename '%s' to '%s'" % (src_name,
 354                                                                  dst_name))
 355         return ret
 356
 357     def unlink(self, FileHandle handler, name, flags = 0):
 358         self.require_state("mounted")
 359         name = cstr(name, 'name')
 360         cdef:
 361             rgw_file_handle *_handler = <rgw_file_handle*>handler.handler
 362             int _flags = flags
 363             char* _name = name
 364         with nogil:
 365             ret = rgw_unlink(self.fs, _handler, _name, _flags)
 366         if ret < 0:
 367             raise make_ex(ret, "error in unlink")
 368         return ret
 369
 370     def readdir(self, FileHandle dir_handler, iterate_cb, offset, flags = 0):
 371         self.require_state("mounted")
 372
 373         cdef:
 374             rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
 375             uint64_t _offset = offset
 376             bint _eof
 377             uint32_t _flags = flags
 378         with nogil:
 379             ret = rgw_readdir(self.fs, _dir_handler, &_offset, &readdir_cb,
 380                               <void *>iterate_cb, &_eof, _flags)
 381         if ret < 0:
 382             raise make_ex(ret, "error in readdir")
 383
 384         return (_offset, _eof)
 385
 386     def fstat(self, FileHandle file_handler):
 387         self.require_state("mounted")
 388
 389         cdef:
 390             rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
 391             stat statbuf
 392
 393         with nogil:
 394             ret = rgw_getattr(self.fs, _file_handler, &statbuf, 0)
 395         if ret < 0:
 396             raise make_ex(ret, "error in getattr")
 397         return StatResult(st_dev=statbuf.st_dev, st_ino=statbuf.st_ino,
 398                           st_mode=statbuf.st_mode, st_nlink=statbuf.st_nlink,
 399                           st_uid=statbuf.st_uid, st_gid=statbuf.st_gid,
 400                           st_rdev=statbuf.st_rdev, st_size=statbuf.st_size,
 401                           st_blksize=statbuf.st_blksize,
 402                           st_blocks=statbuf.st_blocks,
 403                           st_atime=datetime.fromtimestamp(statbuf.st_atime),
 404                           st_mtime=datetime.fromtimestamp(statbuf.st_mtime),
 405                           st_ctime=datetime.fromtimestamp(statbuf.st_ctime))
 406
 407     def opendir(self, FileHandle dir_handler, dirname, flags = 0):
 408         self.require_state("mounted")
 409
 410         if not isinstance(flags, int):
 411             raise TypeError("flags must be an integer")
 412
 413         dirname = cstr(dirname, 'dirname')
 414
 415         cdef:
 416             rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
 417             rgw_file_handle *_file_handler
 418             int _flags = flags
 419             char* _dirname = dirname
 420             stat st
 421             uint32_t st_mask = 0
 422
 423         with nogil:
 424             ret = rgw_lookup(self.fs, _dir_handler, _dirname,
 425                              &_file_handler, &st, st_mask, _flags)
 426         if ret < 0:
 427             raise make_ex(ret, "error in open '%s'" % dirname)
 428
 429         file_handler = FileHandle()
 430         file_handler.handler = _file_handler
 431         return file_handler
 432
 433     def open(self, FileHandle dir_handler, filename, flags = 0):
 434         self.require_state("mounted")
 435
 436         if not isinstance(flags, int):
 437             raise TypeError("flags must be an integer")
 438
 439         filename = cstr(filename, 'filename')
 440
 441         cdef:
 442             rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
 443             rgw_file_handle *_file_handler
 444             int _flags = flags
 445             char* _filename = filename
 446             stat st
 447             uint32_t st_mask = 0
 448
 449         with nogil:
 450             ret = rgw_lookup(self.fs, _dir_handler, _filename,
 451                              &_file_handler, &st, st_mask, _flags)
 452         if ret < 0:
 453             raise make_ex(ret, "error in open '%s'" % filename)
 454         with nogil:
 455             ret = rgw_open(self.fs, _file_handler, 0, _flags)
 456         if ret < 0:
 457             raise make_ex(ret, "error in open '%s'" % filename)
 458
 459         file_handler = FileHandle()
 460         file_handler.handler = _file_handler
 461         return file_handler
 462
 463     def close(self, FileHandle file_handler, flags = 0):
 464         self.require_state("mounted")
 465         cdef:
 466             rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
 467             int _flags = flags
 468         with nogil:
 469             ret = rgw_close(self.fs, _file_handler, _flags)
 470         if ret < 0:
 471             raise make_ex(ret, "error in close")
 472
 473     def read(self, FileHandle file_handler, offset, l, flags = 0):
 474         self.require_state("mounted")
 475         if not isinstance(offset, int):
 476             raise TypeError('offset must be an int')
 477         if not isinstance(l, int):
 478             raise TypeError('l must be an int')
 479         cdef:
 480             rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
 481             int64_t _offset = offset
 482             size_t _length = l
 483             size_t _got
 484             int _flags = flags
 485
 486             char *ret_buf
 487             PyObject* ret_s = NULL
 488
 489         ret_s = PyBytes_FromStringAndSize(NULL, _length)
 490         try:
 491             ret_buf = PyBytes_AsString(ret_s)
 492             with nogil:
 493                 ret = rgw_read(self.fs, _file_handler, _offset, _length,
 494                                &_got, ret_buf, _flags)
 495             if ret < 0:
 496                 raise make_ex(ret, "error in read")
 497
 498             if _got < _length:
 499                 _PyBytes_Resize(&ret_s, _got)
 500
 501             return <object>ret_s
 502         finally:
 503             # We DECREF unconditionally: the cast to object above will have
 504             # INCREFed if necessary. This also takes care of exceptions,
 505             # including if _PyString_Resize fails (that will free the string
 506             # itself and set ret_s to NULL, hence XDECREF).
 507             ref.Py_XDECREF(ret_s)
 508
 509
 510     def write(self, FileHandle file_handler, offset, buf, flags = 0):
 511         self.require_state("mounted")
 512         if not isinstance(buf, bytes):
 513             raise TypeError('buf must be a bytes')
 514         if not isinstance(offset, int):
 515             raise TypeError('offset must be an int')
 516
 517         cdef:
 518             rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
 519             char *_data = buf
 520             int64_t _offset = offset
 521
 522             size_t length = len(buf)
 523             int _flags = flags
 524             size_t _written
 525
 526         with nogil:
 527             ret = rgw_write(self.fs, _file_handler, _offset, length, &_written,
 528                             _data, _flags)
 529         if ret < 0:
 530             raise make_ex(ret, "error in write")
 531         return ret
 532
 533
 534     def fsync(self, FileHandle handler, flags = 0):
 535         self.require_state("mounted")
 536
 537         cdef:
 538             rgw_file_handle *_file_handler = <rgw_file_handle*>handler.handler
 539             int _flags = flags
 540         with nogil:
 541             ret = rgw_fsync(self.fs, _file_handler, _flags)
 542
 543         if ret < 0:
 544             raise make_ex(ret, "fsync failed")