ceph/src/kv/KeyValueDB.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 #ifndef KEY_VALUE_DB_H
   4 #define KEY_VALUE_DB_H
   5
   6 #include "include/buffer.h"
   7 #include <ostream>
   8 #include <set>
   9 #include <map>
  10 #include <string>
  11 #include "include/memory.h"
  12 #include <boost/scoped_ptr.hpp>
  13 #include "include/encoding.h"
  14 #include "common/Formatter.h"
  15 #include "common/perf_counters.h"
  16
  17 using std::string;
  18 /**
  19  * Defines virtual interface to be implemented by key value store
  20  *
  21  * Kyoto Cabinet or LevelDB should implement this
  22  */
  23 class KeyValueDB {
  24 public:
  25   class TransactionImpl {
  26   public:
  27     /// Set Keys
  28     void set(
  29       const std::string &prefix,                 ///< [in] Prefix for keys
  30       const std::map<std::string, bufferlist> &to_set ///< [in] keys/values to set
  31     ) {
  32       std::map<std::string, bufferlist>::const_iterator it;
  33       for (it = to_set.begin(); it != to_set.end(); ++it)
  34         set(prefix, it->first, it->second);
  35     }
  36
  37     /// Set Keys (via encoded bufferlist)
  38     void set(
  39       const std::string &prefix,      ///< [in] prefix
  40       bufferlist& to_set_bl     ///< [in] encoded key/values to set
  41       ) {
  42       bufferlist::iterator p = to_set_bl.begin();
  43       uint32_t num;
  44       ::decode(num, p);
  45       while (num--) {
  46         string key;
  47         bufferlist value;
  48         ::decode(key, p);
  49         ::decode(value, p);
  50         set(prefix, key, value);
  51       }
  52     }
  53
  54     /// Set Key
  55     virtual void set(
  56       const std::string &prefix,   ///< [in] Prefix for the key
  57       const std::string &k,           ///< [in] Key to set
  58       const bufferlist &bl    ///< [in] Value to set
  59       ) = 0;
  60     virtual void set(
  61       const std::string &prefix,
  62       const char *k,
  63       size_t keylen,
  64       const bufferlist& bl) {
  65       set(prefix, string(k, keylen), bl);
  66     }
  67
  68     /// Removes Keys (via encoded bufferlist)
  69     void rmkeys(
  70       const std::string &prefix,   ///< [in] Prefix to search for
  71       bufferlist &keys_bl ///< [in] Keys to remove
  72     ) {
  73       bufferlist::iterator p = keys_bl.begin();
  74       uint32_t num;
  75       ::decode(num, p);
  76       while (num--) {
  77         string key;
  78         ::decode(key, p);
  79         rmkey(prefix, key);
  80       }
  81     }
  82
  83     /// Removes Keys
  84     void rmkeys(
  85       const std::string &prefix,   ///< [in] Prefix to search for
  86       const std::set<std::string> &keys ///< [in] Keys to remove
  87     ) {
  88       std::set<std::string>::const_iterator it;
  89       for (it = keys.begin(); it != keys.end(); ++it)
  90         rmkey(prefix, *it);
  91     }
  92
  93     /// Remove Key
  94     virtual void rmkey(
  95       const std::string &prefix,   ///< [in] Prefix to search for
  96       const std::string &k            ///< [in] Key to remove
  97       ) = 0;
  98     virtual void rmkey(
  99       const std::string &prefix,   ///< [in] Prefix to search for
 100       const char *k,          ///< [in] Key to remove
 101       size_t keylen
 102       ) {
 103       rmkey(prefix, string(k, keylen));
 104     }
 105
 106     /// Remove Single Key which exists and was not overwritten.
 107     /// This API is only related to performance optimization, and should only be
 108     /// re-implemented by log-insert-merge tree based keyvalue stores(such as RocksDB).
 109     /// If a key is overwritten (by calling set multiple times), then the result
 110     /// of calling rm_single_key on this key is undefined.
 111     virtual void rm_single_key(
 112       const std::string &prefix,   ///< [in] Prefix to search for
 113       const std::string &k            ///< [in] Key to remove
 114       ) { return rmkey(prefix, k);}
 115
 116     /// Removes keys beginning with prefix
 117     virtual void rmkeys_by_prefix(
 118       const std::string &prefix ///< [in] Prefix by which to remove keys
 119       ) = 0;
 120
 121     virtual void rm_range_keys(
 122       const string &prefix,    ///< [in] Prefix by which to remove keys
 123       const string &start,     ///< [in] The start bound of remove keys
 124       const string &end        ///< [in] The start bound of remove keys
 125       ) = 0;
 126
 127     /// Merge value into key
 128     virtual void merge(
 129       const std::string &prefix,   ///< [in] Prefix ==> MUST match some established merge operator
 130       const std::string &key,      ///< [in] Key to be merged
 131       const bufferlist  &value     ///< [in] value to be merged into key
 132     ) { assert(0 == "Not implemented"); }
 133
 134     virtual ~TransactionImpl() {}
 135   };
 136   typedef ceph::shared_ptr< TransactionImpl > Transaction;
 137
 138   /// create a new instance
 139   static KeyValueDB *create(CephContext *cct, const std::string& type,
 140                             const std::string& dir,
 141                             void *p = NULL);
 142
 143   /// test whether we can successfully initialize; may have side effects (e.g., create)
 144   static int test_init(const std::string& type, const std::string& dir);
 145   virtual int init(string option_str="") = 0;
 146   virtual int open(std::ostream &out) = 0;
 147   virtual int create_and_open(std::ostream &out) = 0;
 148   virtual void close() { }
 149
 150   virtual Transaction get_transaction() = 0;
 151   virtual int submit_transaction(Transaction) = 0;
 152   virtual int submit_transaction_sync(Transaction t) {
 153     return submit_transaction(t);
 154   }
 155
 156   /// Retrieve Keys
 157   virtual int get(
 158     const std::string &prefix,        ///< [in] Prefix for key
 159     const std::set<std::string> &key,      ///< [in] Key to retrieve
 160     std::map<std::string, bufferlist> *out ///< [out] Key value retrieved
 161     ) = 0;
 162   virtual int get(const std::string &prefix, ///< [in] prefix
 163                   const std::string &key,    ///< [in] key
 164                   bufferlist *value) {  ///< [out] value
 165     std::set<std::string> ks;
 166     ks.insert(key);
 167     std::map<std::string,bufferlist> om;
 168     int r = get(prefix, ks, &om);
 169     if (om.find(key) != om.end()) {
 170       *value = om[key];
 171     } else {
 172       *value = bufferlist();
 173       r = -ENOENT;
 174     }
 175     return r;
 176   }
 177   virtual int get(const string &prefix,
 178                   const char *key, size_t keylen,
 179                   bufferlist *value) {
 180     return get(prefix, string(key, keylen), value);
 181   }
 182
 183   class GenericIteratorImpl {
 184   public:
 185     virtual int seek_to_first() = 0;
 186     virtual int upper_bound(const std::string &after) = 0;
 187     virtual int lower_bound(const std::string &to) = 0;
 188     virtual bool valid() = 0;
 189     virtual int next(bool validate=true) = 0;
 190     virtual std::string key() = 0;
 191     virtual bufferlist value() = 0;
 192     virtual int status() = 0;
 193     virtual ~GenericIteratorImpl() {}
 194   };
 195
 196   class WholeSpaceIteratorImpl {
 197   public:
 198     virtual int seek_to_first() = 0;
 199     virtual int seek_to_first(const std::string &prefix) = 0;
 200     virtual int seek_to_last() = 0;
 201     virtual int seek_to_last(const std::string &prefix) = 0;
 202     virtual int upper_bound(const std::string &prefix, const std::string &after) = 0;
 203     virtual int lower_bound(const std::string &prefix, const std::string &to) = 0;
 204     virtual bool valid() = 0;
 205     virtual int next() = 0;
 206     virtual int prev() = 0;
 207     virtual std::string key() = 0;
 208     virtual std::pair<std::string,std::string> raw_key() = 0;
 209     virtual bool raw_key_is_prefixed(const std::string &prefix) = 0;
 210     virtual bufferlist value() = 0;
 211     virtual bufferptr value_as_ptr() {
 212       bufferlist bl = value();
 213       if (bl.length()) {
 214         return *bl.buffers().begin();
 215       } else {
 216         return bufferptr();
 217       }
 218     }
 219     virtual int status() = 0;
 220     virtual size_t key_size() {
 221       return 0;
 222     }
 223     virtual size_t value_size() {
 224       return 0;
 225     }
 226     virtual ~WholeSpaceIteratorImpl() { }
 227   };
 228   typedef ceph::shared_ptr< WholeSpaceIteratorImpl > WholeSpaceIterator;
 229
 230   class IteratorImpl : public GenericIteratorImpl {
 231     const std::string prefix;
 232     WholeSpaceIterator generic_iter;
 233   public:
 234     IteratorImpl(const std::string &prefix, WholeSpaceIterator iter) :
 235       prefix(prefix), generic_iter(iter) { }
 236     ~IteratorImpl() override { }
 237
 238     int seek_to_first() override {
 239       return generic_iter->seek_to_first(prefix);
 240     }
 241     int seek_to_last() {
 242       return generic_iter->seek_to_last(prefix);
 243     }
 244     int upper_bound(const std::string &after) override {
 245       return generic_iter->upper_bound(prefix, after);
 246     }
 247     int lower_bound(const std::string &to) override {
 248       return generic_iter->lower_bound(prefix, to);
 249     }
 250     bool valid() override {
 251       if (!generic_iter->valid())
 252         return false;
 253       return generic_iter->raw_key_is_prefixed(prefix);
 254     }
 255     // Note that next() and prev() shouldn't validate iters,
 256     // it's responsibility of caller to ensure they're valid.
 257     int next(bool validate=true) override {
 258       if (validate) {
 259         if (valid())
 260           return generic_iter->next();
 261         return status();
 262       } else {
 263         return generic_iter->next();
 264       }
 265     }
 266
 267     int prev(bool validate=true) {
 268       if (validate) {
 269         if (valid())
 270           return generic_iter->prev();
 271         return status();
 272       } else {
 273         return generic_iter->prev();
 274       }
 275     }
 276     std::string key() override {
 277       return generic_iter->key();
 278     }
 279     std::pair<std::string, std::string> raw_key() {
 280       return generic_iter->raw_key();
 281     }
 282     bufferlist value() override {
 283       return generic_iter->value();
 284     }
 285     bufferptr value_as_ptr() {
 286       return generic_iter->value_as_ptr();
 287     }
 288     int status() override {
 289       return generic_iter->status();
 290     }
 291   };
 292
 293   typedef ceph::shared_ptr< IteratorImpl > Iterator;
 294
 295   WholeSpaceIterator get_iterator() {
 296     return _get_iterator();
 297   }
 298
 299   Iterator get_iterator(const std::string &prefix) {
 300     return std::make_shared<IteratorImpl>(prefix, get_iterator());
 301   }
 302
 303   virtual uint64_t get_estimated_size(std::map<std::string,uint64_t> &extra) = 0;
 304   virtual int get_statfs(struct store_statfs_t *buf) {
 305     return -EOPNOTSUPP;
 306   }
 307
 308   virtual int set_cache_size(uint64_t) {
 309     return -EOPNOTSUPP;
 310   }
 311
 312   virtual ~KeyValueDB() {}
 313
 314   /// compact the underlying store
 315   virtual void compact() {}
 316
 317   /// compact db for all keys with a given prefix
 318   virtual void compact_prefix(const std::string& prefix) {}
 319   /// compact db for all keys with a given prefix, async
 320   virtual void compact_prefix_async(const std::string& prefix) {}
 321   virtual void compact_range(const std::string& prefix,
 322                              const std::string& start, const std::string& end) {}
 323   virtual void compact_range_async(const std::string& prefix,
 324                                    const std::string& start, const std::string& end) {}
 325
 326   // See RocksDB merge operator definition, we support the basic
 327   // associative merge only right now.
 328   class MergeOperator {
 329     public:
 330     /// Merge into a key that doesn't exist
 331     virtual void merge_nonexistent(
 332       const char *rdata, size_t rlen,
 333       std::string *new_value) = 0;
 334     /// Merge into a key that does exist
 335     virtual void merge(
 336       const char *ldata, size_t llen,
 337       const char *rdata, size_t rlen,
 338       std::string *new_value) = 0;
 339     /// We use each operator name and each prefix to construct the overall RocksDB operator name for consistency check at open time.
 340     virtual string name() const = 0;
 341
 342     virtual ~MergeOperator() {}
 343   };
 344
 345   /// Setup one or more operators, this needs to be done BEFORE the DB is opened.
 346   virtual int set_merge_operator(const std::string& prefix,
 347                                  std::shared_ptr<MergeOperator> mop) {
 348     return -EOPNOTSUPP;
 349   }
 350
 351   virtual void get_statistics(Formatter *f) {
 352     return;
 353   }
 354
 355   /**
 356    * Return your perf counters if you have any.  Subclasses are not
 357    * required to implement this, and callers must respect a null return
 358    * value.
 359    */
 360   virtual PerfCounters *get_perf_counters() {
 361     return nullptr;
 362   }
 363 protected:
 364   /// List of matching prefixes and merge operators
 365   std::vector<std::pair<std::string,
 366                         std::shared_ptr<MergeOperator> > > merge_ops;
 367
 368   virtual WholeSpaceIterator _get_iterator() = 0;
 369 };
 370
 371 #endif