ceph/src/libradosstriper/RadosStriperImpl.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2014 Sebastien Ponce <sebastien.ponce@cern.ch>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #ifndef CEPH_LIBRADOSSTRIPER_RADOSSTRIPERIMPL_H
  16 #define CEPH_LIBRADOSSTRIPER_RADOSSTRIPERIMPL_H
  17
  18 #include <string>
  19
  20 #include "include/atomic.h"
  21
  22 #include "include/rados/librados.h"
  23 #include "include/rados/librados.hpp"
  24 #include "include/radosstriper/libradosstriper.h"
  25 #include "include/radosstriper/libradosstriper.hpp"
  26
  27 #include "librados/IoCtxImpl.h"
  28 #include "librados/AioCompletionImpl.h"
  29 #include "common/RefCountedObj.h"
  30
  31 struct libradosstriper::RadosStriperImpl {
  32
  33   /**
  34    * struct handling the data needed to pass to the call back
  35    * function in asynchronous operations
  36    */
  37   struct CompletionData : RefCountedObject {
  38     /// constructor
  39     CompletionData(libradosstriper::RadosStriperImpl * striper,
  40                    const std::string& soid,
  41                    const std::string& lockCookie,
  42                    librados::AioCompletionImpl *userCompletion = 0,
  43                    int n = 1);
  44     /// destructor
  45     ~CompletionData() override;
  46     /// complete method
  47     void complete(int r);
  48     /// striper to be used to handle the write completion
  49     libradosstriper::RadosStriperImpl *m_striper;
  50     /// striped object concerned by the write operation
  51     std::string m_soid;
  52     /// shared lock to be released at completion
  53     std::string m_lockCookie;
  54     /// completion handler
  55     librados::IoCtxImpl::C_aio_Complete *m_ack;
  56   };
  57
  58   /**
  59    * struct handling the data needed to pass to the call back
  60    * function in asynchronous read operations
  61    */
  62   struct ReadCompletionData : CompletionData {
  63     /// bufferlist containing final result
  64     bufferlist* m_bl;
  65     /// extents that will be read
  66     std::vector<ObjectExtent>* m_extents;
  67     /// intermediate results
  68     std::vector<bufferlist>* m_resultbl;
  69     /// return code of read completion, to be remembered until unlocking happened
  70     int m_readRc;
  71     /// completion object for the unlocking of the striped object at the end of the read
  72     librados::AioCompletion *m_unlockCompletion;
  73     /// constructor
  74     ReadCompletionData(libradosstriper::RadosStriperImpl * striper,
  75                        const std::string& soid,
  76                        const std::string& lockCookie,
  77                        librados::AioCompletionImpl *userCompletion,
  78                        bufferlist* bl,
  79                        std::vector<ObjectExtent>* extents,
  80                        std::vector<bufferlist>* resultbl,
  81                        int n);
  82     /// destructor
  83     ~ReadCompletionData() override;
  84     /// complete method for when reading is over
  85     void complete_read(int r);
  86     /// complete method for when object is unlocked
  87     void complete_unlock(int r);
  88   };
  89
  90   /**
  91    * struct handling the data needed to pass to the call back
  92    * function in asynchronous write operations
  93    */
  94   struct WriteCompletionData : CompletionData {
  95     /// safe completion handler
  96     librados::IoCtxImpl::C_aio_Complete *m_safe;
  97     /// return code of write completion, to be remembered until unlocking happened
  98     int m_writeRc;
  99     /// completion object for the unlocking of the striped object at the end of the write
 100     librados::AioCompletion *m_unlockCompletion;
 101     /// constructor
 102     WriteCompletionData(libradosstriper::RadosStriperImpl * striper,
 103                         const std::string& soid,
 104                         const std::string& lockCookie,
 105                         librados::AioCompletionImpl *userCompletion,
 106                         int n);
 107     /// destructor
 108     ~WriteCompletionData() override;
 109     /// complete method for when writing is over
 110     void complete_write(int r);
 111     /// complete method for when object is unlocked
 112     void complete_unlock(int r);
 113     /// safe method
 114     void safe(int r);
 115   };
 116
 117   /**
 118    * struct handling the data needed to pass to the call back
 119    * function in asynchronous read operations of a Rados File
 120    */
 121   struct RadosReadCompletionData : RefCountedObject {
 122     /// constructor
 123     RadosReadCompletionData(MultiAioCompletionImpl *multiAioCompl,
 124                             uint64_t expectedBytes,
 125                             bufferlist *bl,
 126                             CephContext *context,
 127                             int n = 1) :
 128       RefCountedObject(context, n),
 129       m_multiAioCompl(multiAioCompl), m_expectedBytes(expectedBytes), m_bl(bl) {};
 130     /// the multi asynch io completion object to be used
 131     MultiAioCompletionImpl *m_multiAioCompl;
 132     /// the expected number of bytes
 133     uint64_t m_expectedBytes;
 134     /// the bufferlist object where data have been written
 135     bufferlist *m_bl;
 136   };
 137
 138   /**
 139    * struct handling (most of) the data needed to pass to the call back
 140    * function in asynchronous stat operations.
 141    * Inherited by the actual type for adding time information in different
 142    * versions (time_t or struct timespec)
 143    */
 144   struct BasicStatCompletionData : CompletionData {
 145     /// constructor
 146     BasicStatCompletionData(libradosstriper::RadosStriperImpl* striper,
 147                             const std::string& soid,
 148                             librados::AioCompletionImpl *userCompletion,
 149                             libradosstriper::MultiAioCompletionImpl *multiCompletion,
 150                             uint64_t *psize,
 151                             int n = 1) :
 152       CompletionData(striper, soid, "", userCompletion, n),
 153       m_multiCompletion(multiCompletion), m_psize(psize),
 154       m_statRC(0), m_getxattrRC(0) {};
 155     // MultiAioCompletionImpl used to handle the double aysnc
 156     // call in the back (stat + getxattr)
 157     libradosstriper::MultiAioCompletionImpl *m_multiCompletion;
 158     // where to store the size of first objct
 159     // this will be ignored but we need a place to store it when
 160     // async stat is called
 161     uint64_t m_objectSize;
 162     // where to store the file size
 163     uint64_t *m_psize;
 164     /// the bufferlist object used for the getxattr call
 165     bufferlist m_bl;
 166     /// return code of the stat
 167     int m_statRC;
 168     /// return code of the getxattr
 169     int m_getxattrRC;
 170   };
 171
 172   /**
 173    * struct handling the data needed to pass to the call back
 174    * function in asynchronous stat operations.
 175    * Simple templated extension of BasicStatCompletionData.
 176    * The template parameter is the type of the time information
 177    * (used with time_t for stat and struct timespec for stat2)
 178    */
 179   template<class TimeType>
 180   struct StatCompletionData : BasicStatCompletionData {
 181     /// constructor
 182     StatCompletionData(libradosstriper::RadosStriperImpl* striper,
 183                        const std::string& soid,
 184                        librados::AioCompletionImpl *userCompletion,
 185                        libradosstriper::MultiAioCompletionImpl *multiCompletion,
 186                        uint64_t *psize,
 187                        TimeType *pmtime,
 188                        int n = 1) :
 189       BasicStatCompletionData(striper, soid, userCompletion, multiCompletion, psize, n),
 190       m_pmtime(pmtime) {};
 191     // where to store the file time
 192     TimeType *m_pmtime;
 193   };
 194
 195   /**
 196    * struct handling the data needed to pass to the call back
 197    * function in asynchronous remove operations of a Rados File
 198    */
 199   struct RadosRemoveCompletionData : RefCountedObject {
 200     /// constructor
 201     RadosRemoveCompletionData(MultiAioCompletionImpl *multiAioCompl,
 202                               CephContext *context) :
 203       RefCountedObject(context, 2),
 204       m_multiAioCompl(multiAioCompl) {};
 205     /// the multi asynch io completion object to be used
 206     MultiAioCompletionImpl *m_multiAioCompl;
 207   };
 208
 209   struct RemoveCompletionData : CompletionData {
 210     /// removal flags
 211     int flags;
 212     /**
 213      * constructor
 214      * note that the constructed object will take ownership of the lock
 215      */
 216     RemoveCompletionData(libradosstriper::RadosStriperImpl * striper,
 217                          const std::string& soid,
 218                          const std::string& lockCookie,
 219                          librados::AioCompletionImpl *userCompletion,
 220                          int flags = 0);
 221   };
 222
 223   /**
 224    * struct handling the data needed to pass to the call back
 225    * function in asynchronous truncate operations
 226    */
 227   struct TruncateCompletionData : RefCountedObject {
 228     /// constructor
 229     TruncateCompletionData(libradosstriper::RadosStriperImpl* striper,
 230                            const std::string& soid,
 231                            uint64_t size);
 232     /// destructor
 233     ~TruncateCompletionData() override;
 234     /// striper to be used
 235     libradosstriper::RadosStriperImpl *m_striper;
 236     /// striped object concerned by the truncate operation
 237     std::string m_soid;
 238     /// the final size of the truncated object
 239     uint64_t m_size;
 240   };
 241
 242   /**
 243    * exception wrapper around an error code
 244    */
 245   struct ErrorCode {
 246     ErrorCode(int error) : m_code(error) {};
 247     int m_code;
 248   };
 249
 250   /*
 251    * Constructor
 252    * @param cluster_name name of the cluster, can be NULL
 253    * @param client_name has 2 meanings depending on cluster_name
 254    *          - if cluster_name is null : this is the client id
 255    *          - else : this is the full client name in format type.id
 256    */
 257   RadosStriperImpl(librados::IoCtx& ioctx, librados::IoCtxImpl *ioctx_impl);
 258   /// Destructor
 259   ~RadosStriperImpl() {};
 260
 261   // configuration
 262   int setObjectLayoutStripeUnit(unsigned int stripe_unit);
 263   int setObjectLayoutStripeCount(unsigned int stripe_count);
 264   int setObjectLayoutObjectSize(unsigned int object_size);
 265
 266   // xattrs
 267   int getxattr(const object_t& soid, const char *name, bufferlist& bl);
 268   int setxattr(const object_t& soid, const char *name, bufferlist& bl);
 269   int getxattrs(const object_t& soid, map<string, bufferlist>& attrset);
 270   int rmxattr(const object_t& soid, const char *name);
 271
 272   // io
 273   int write(const std::string& soid, const bufferlist& bl, size_t len, uint64_t off);
 274   int append(const std::string& soid, const bufferlist& bl, size_t len);
 275   int write_full(const std::string& soid, const bufferlist& bl);
 276   int read(const std::string& soid, bufferlist* pbl, size_t len, uint64_t off);
 277
 278   // asynchronous io
 279   int aio_write(const std::string& soid, librados::AioCompletionImpl *c,
 280                 const bufferlist& bl, size_t len, uint64_t off);
 281   int aio_append(const std::string& soid, librados::AioCompletionImpl *c,
 282                  const bufferlist& bl, size_t len);
 283   int aio_write_full(const std::string& soid, librados::AioCompletionImpl *c,
 284                      const bufferlist& bl);
 285   int aio_read(const std::string& soid, librados::AioCompletionImpl *c,
 286                bufferlist* pbl, size_t len, uint64_t off);
 287   int aio_read(const std::string& soid, librados::AioCompletionImpl *c,
 288                char* buf, size_t len, uint64_t off);
 289   int aio_flush();
 290
 291   // stat, deletion and truncation
 292   int stat(const std::string& soid, uint64_t *psize, time_t *pmtime);
 293   int stat2(const std::string& soid, uint64_t *psize, struct timespec *pts);
 294   template<class TimeType>
 295   struct StatFunction {
 296     typedef int (librados::IoCtxImpl::*Type) (const object_t& oid,
 297                                               librados::AioCompletionImpl *c,
 298                                               uint64_t *psize, TimeType *pmtime);
 299   };
 300   template<class TimeType>
 301   int aio_generic_stat(const std::string& soid, librados::AioCompletionImpl *c,
 302                        uint64_t *psize, TimeType *pmtime,
 303                        typename StatFunction<TimeType>::Type statFunction);
 304   int aio_stat(const std::string& soid, librados::AioCompletionImpl *c,
 305                uint64_t *psize, time_t *pmtime);
 306   int aio_stat2(const std::string& soid, librados::AioCompletionImpl *c,
 307                 uint64_t *psize, struct timespec *pts);
 308   int remove(const std::string& soid, int flags=0);
 309   int trunc(const std::string& soid, uint64_t size);
 310
 311   // asynchronous remove. Note that the removal is not 100% parallelized :
 312   // the removal of the first rados object of the striped object will be
 313   // done via a syncrhonous call after the completion of all other removals.
 314   // These are done asynchrounously and in parallel
 315   int aio_remove(const std::string& soid, librados::AioCompletionImpl *c, int flags=0);
 316
 317   // reference counting
 318   void get() {
 319     lock.Lock();
 320     m_refCnt ++ ;
 321     lock.Unlock();
 322   }
 323   void put() {
 324     bool deleteme = false;
 325     lock.Lock();
 326     m_refCnt --;
 327     if (m_refCnt == 0)
 328       deleteme = true;
 329     cond.Signal();
 330     lock.Unlock();
 331     if (deleteme)
 332       delete this;
 333   }
 334
 335   // objectid manipulation
 336   std::string getObjectId(const object_t& soid, long long unsigned objectno);
 337
 338   // opening and closing of striped objects
 339   void unlockObject(const std::string& soid,
 340                     const std::string& lockCookie);
 341   void aio_unlockObject(const std::string& soid,
 342                         const std::string& lockCookie,
 343                         librados::AioCompletion *c);
 344
 345   // internal versions of IO method
 346   int write_in_open_object(const std::string& soid,
 347                            const ceph_file_layout& layout,
 348                            const std::string& lockCookie,
 349                            const bufferlist& bl,
 350                            size_t len,
 351                            uint64_t off);
 352   int aio_write_in_open_object(const std::string& soid,
 353                                librados::AioCompletionImpl *c,
 354                                const ceph_file_layout& layout,
 355                                const std::string& lockCookie,
 356                                const bufferlist& bl,
 357                                size_t len,
 358                                uint64_t off);
 359   int internal_aio_write(const std::string& soid,
 360                          libradosstriper::MultiAioCompletionImpl *c,
 361                          const bufferlist& bl,
 362                          size_t len,
 363                          uint64_t off,
 364                          const ceph_file_layout& layout);
 365
 366   int extract_uint32_attr(std::map<std::string, bufferlist> &attrs,
 367                           const std::string& key,
 368                           ceph_le32 *value);
 369
 370   int extract_sizet_attr(std::map<std::string, bufferlist> &attrs,
 371                          const std::string& key,
 372                          size_t *value);
 373
 374   int internal_get_layout_and_size(const std::string& oid,
 375                                    ceph_file_layout *layout,
 376                                    uint64_t *size);
 377
 378   int internal_aio_remove(const std::string& soid,
 379                           libradosstriper::MultiAioCompletionImpl *multi_completion,
 380                           int flags=0);
 381
 382   /**
 383    * opens an existing striped object and takes a shared lock on it
 384    * @return 0 if everything is ok and the lock was taken. -errcode otherwise
 385    * In particulae, if the striped object does not exists, -ENOENT is returned
 386    * In case the return code in not 0, no lock is taken
 387    */
 388   int openStripedObjectForRead(const std::string& soid,
 389                                ceph_file_layout *layout,
 390                                uint64_t *size,
 391                                std::string *lockCookie);
 392
 393   /**
 394    * opens an existing striped object, takes a shared lock on it
 395    * and sets its size to the size it will have after the write.
 396    * In case the striped object does not exists, it will create it by
 397    * calling createOrOpenStripedObject.
 398    * @param layout this is filled with the layout of the file
 399    * @param size new size of the file (together with isFileSizeAbsolute)
 400    * In case of success, this is filled with the size of the file before the opening
 401    * @param isFileSizeAbsolute if false, this means that the given size should
 402    * be added to the current file size (append mode)
 403    * @return 0 if everything is ok and the lock was taken. -errcode otherwise
 404    * In case the return code in not 0, no lock is taken
 405    */
 406   int openStripedObjectForWrite(const std::string& soid,
 407                                 ceph_file_layout *layout,
 408                                 uint64_t *size,
 409                                 std::string *lockCookie,
 410                                 bool isFileSizeAbsolute);
 411   /**
 412    * creates an empty striped object with the given size and opens it calling
 413    * openStripedObjectForWrite, which implies taking a shared lock on it
 414    * Also deals with the cases where the object was created in the mean time
 415    * @param isFileSizeAbsolute if false, this means that the given size should
 416    * be added to the current file size (append mode). This of course only makes
 417    * sense in case the striped object already exists
 418    * @return 0 if everything is ok and the lock was taken. -errcode otherwise
 419    * In case the return code in not 0, no lock is taken
 420    */
 421   int createAndOpenStripedObject(const std::string& soid,
 422                                  ceph_file_layout *layout,
 423                                  uint64_t size,
 424                                  std::string *lockCookie,
 425                                  bool isFileSizeAbsolute);
 426
 427   /**
 428    * truncates an object synchronously. Should only be called with size < original_size
 429    */
 430   int truncate(const std::string& soid,
 431                uint64_t original_size,
 432                uint64_t size,
 433                ceph_file_layout &layout);
 434
 435   /**
 436    * truncates an object asynchronously. Should only be called with size < original_size
 437    * note that the method is not 100% asynchronous, only the removal of rados objects
 438    * is, the (potential) truncation of the rados object residing just at the truncation
 439    * point is synchronous for lack of asynchronous truncation in the rados layer
 440    */
 441   int aio_truncate(const std::string& soid,
 442                    libradosstriper::MultiAioCompletionImpl *c,
 443                    uint64_t original_size,
 444                    uint64_t size,
 445                    ceph_file_layout &layout);
 446
 447   /**
 448    * grows an object (adding 0s). Should only be called with size > original_size
 449    */
 450   int grow(const std::string& soid,
 451            uint64_t original_size,
 452            uint64_t size,
 453            ceph_file_layout &layout);
 454
 455   /**
 456    * creates a unique identifier
 457    */
 458   static std::string getUUID();
 459
 460   CephContext *cct() {
 461     return (CephContext*)m_radosCluster.cct();
 462   }
 463
 464   // reference counting
 465   Cond  cond;
 466   int m_refCnt;
 467   Mutex lock;
 468
 469
 470   // Context
 471   librados::Rados m_radosCluster;
 472   librados::IoCtx m_ioCtx;
 473   librados::IoCtxImpl *m_ioCtxImpl;
 474
 475   // Default layout
 476   ceph_file_layout m_layout;
 477 };
 478
 479 #endif