ceph/src/rgw/rgw_bucket_layout.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2020 Red Hat, Inc.
   8  *
   9  * This is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License version 2.1, as published by the Free Software
  12  * Foundation.  See file COPYING.
  13  *
  14  */
  15
  16 /* N.B., this header defines fundamental serialized types.  Do not
  17  * introduce changes or include files which can only be compiled in
  18  * radosgw or OSD contexts (e.g., rgw_sal.h, rgw_common.h)
  19  */
  20
  21 #pragma once
  22
  23 #include <optional>
  24 #include <string>
  25 #include "include/encoding.h"
  26 #include "common/ceph_json.h"
  27
  28 namespace rgw {
  29
  30 enum class BucketIndexType : uint8_t {
  31   Normal, // normal hash-based sharded index layout
  32   Indexless, // no bucket index, so listing is unsupported
  33 };
  34
  35 std::string_view to_string(const BucketIndexType& t);
  36 bool parse(std::string_view str, BucketIndexType& t);
  37 void encode_json_impl(const char *name, const BucketIndexType& t, ceph::Formatter *f);
  38 void decode_json_obj(BucketIndexType& t, JSONObj *obj);
  39
  40 inline std::ostream& operator<<(std::ostream& out, const BucketIndexType& t)
  41 {
  42   return out << to_string(t);
  43 }
  44
  45 enum class BucketHashType : uint8_t {
  46   Mod, // rjenkins hash of object name, modulo num_shards
  47 };
  48
  49 std::string_view to_string(const BucketHashType& t);
  50 bool parse(std::string_view str, BucketHashType& t);
  51 void encode_json_impl(const char *name, const BucketHashType& t, ceph::Formatter *f);
  52 void decode_json_obj(BucketHashType& t, JSONObj *obj);
  53
  54 struct bucket_index_normal_layout {
  55   uint32_t num_shards = 1;
  56
  57   BucketHashType hash_type = BucketHashType::Mod;
  58
  59   friend std::ostream& operator<<(std::ostream& out, const bucket_index_normal_layout& l) {
  60     out << "num_shards=" << l.num_shards << ", hash_type=" << to_string(l.hash_type);
  61     return out;
  62   }
  63 };
  64
  65 inline bool operator==(const bucket_index_normal_layout& l,
  66                        const bucket_index_normal_layout& r) {
  67   return l.num_shards == r.num_shards
  68       && l.hash_type == r.hash_type;
  69 }
  70 inline bool operator!=(const bucket_index_normal_layout& l,
  71                        const bucket_index_normal_layout& r) {
  72   return !(l == r);
  73 }
  74
  75 void encode(const bucket_index_normal_layout& l, bufferlist& bl, uint64_t f=0);
  76 void decode(bucket_index_normal_layout& l, bufferlist::const_iterator& bl);
  77 void encode_json_impl(const char *name, const bucket_index_normal_layout& l, ceph::Formatter *f);
  78 void decode_json_obj(bucket_index_normal_layout& l, JSONObj *obj);
  79
  80 struct bucket_index_layout {
  81   BucketIndexType type = BucketIndexType::Normal;
  82
  83   // TODO: variant of layout types?
  84   bucket_index_normal_layout normal;
  85
  86   friend std::ostream& operator<<(std::ostream& out, const bucket_index_layout& l) {
  87     out << "type=" << to_string(l.type) << ", normal=" << l.normal;
  88     return out;
  89   }
  90 };
  91
  92 inline bool operator==(const bucket_index_layout& l,
  93                        const bucket_index_layout& r) {
  94   return l.type == r.type && l.normal == r.normal;
  95 }
  96 inline bool operator!=(const bucket_index_layout& l,
  97                        const bucket_index_layout& r) {
  98   return !(l == r);
  99 }
 100
 101 void encode(const bucket_index_layout& l, bufferlist& bl, uint64_t f=0);
 102 void decode(bucket_index_layout& l, bufferlist::const_iterator& bl);
 103 void encode_json_impl(const char *name, const bucket_index_layout& l, ceph::Formatter *f);
 104 void decode_json_obj(bucket_index_layout& l, JSONObj *obj);
 105
 106 struct bucket_index_layout_generation {
 107   uint64_t gen = 0;
 108   bucket_index_layout layout;
 109
 110   friend std::ostream& operator<<(std::ostream& out, const bucket_index_layout_generation& g) {
 111     out << "gen=" << g.gen;
 112     return out;
 113   }
 114 };
 115
 116 inline bool operator==(const bucket_index_layout_generation& l,
 117                        const bucket_index_layout_generation& r) {
 118   return l.gen == r.gen && l.layout == r.layout;
 119 }
 120 inline bool operator!=(const bucket_index_layout_generation& l,
 121                        const bucket_index_layout_generation& r) {
 122   return !(l == r);
 123 }
 124
 125 void encode(const bucket_index_layout_generation& l, bufferlist& bl, uint64_t f=0);
 126 void decode(bucket_index_layout_generation& l, bufferlist::const_iterator& bl);
 127 void encode_json_impl(const char *name, const bucket_index_layout_generation& l, ceph::Formatter *f);
 128 void decode_json_obj(bucket_index_layout_generation& l, JSONObj *obj);
 129
 130
 131 enum class BucketLogType : uint8_t {
 132   // colocated with bucket index, so the log layout matches the index layout
 133   InIndex,
 134 };
 135
 136 std::string_view to_string(const BucketLogType& t);
 137 bool parse(std::string_view str, BucketLogType& t);
 138 void encode_json_impl(const char *name, const BucketLogType& t, ceph::Formatter *f);
 139 void decode_json_obj(BucketLogType& t, JSONObj *obj);
 140
 141 inline std::ostream& operator<<(std::ostream& out, const BucketLogType &log_type)
 142 {
 143   switch (log_type) {
 144     case BucketLogType::InIndex:
 145       return out << "InIndex";
 146     default:
 147       return out << "Unknown";
 148   }
 149 }
 150
 151 struct bucket_index_log_layout {
 152   uint64_t gen = 0;
 153   bucket_index_normal_layout layout;
 154   operator bucket_index_layout_generation() const {
 155     bucket_index_layout_generation bilg;
 156     bilg.gen = gen;
 157     bilg.layout.type = BucketIndexType::Normal;
 158     bilg.layout.normal = layout;
 159     return bilg;
 160   }
 161 };
 162
 163 void encode(const bucket_index_log_layout& l, bufferlist& bl, uint64_t f=0);
 164 void decode(bucket_index_log_layout& l, bufferlist::const_iterator& bl);
 165 void encode_json_impl(const char *name, const bucket_index_log_layout& l, ceph::Formatter *f);
 166 void decode_json_obj(bucket_index_log_layout& l, JSONObj *obj);
 167
 168 struct bucket_log_layout {
 169   BucketLogType type = BucketLogType::InIndex;
 170
 171   bucket_index_log_layout in_index;
 172
 173   friend std::ostream& operator<<(std::ostream& out, const bucket_log_layout& l) {
 174     out << "type=" << to_string(l.type);
 175     return out;
 176   }
 177 };
 178
 179 void encode(const bucket_log_layout& l, bufferlist& bl, uint64_t f=0);
 180 void decode(bucket_log_layout& l, bufferlist::const_iterator& bl);
 181 void encode_json_impl(const char *name, const bucket_log_layout& l, ceph::Formatter *f);
 182 void decode_json_obj(bucket_log_layout& l, JSONObj *obj);
 183
 184 struct bucket_log_layout_generation {
 185   uint64_t gen = 0;
 186   bucket_log_layout layout;
 187
 188   friend std::ostream& operator<<(std::ostream& out, const bucket_log_layout_generation& g) {
 189     out << "gen=" << g.gen << ", layout=[ " << g.layout << " ]";
 190     return out;
 191   }
 192 };
 193
 194 void encode(const bucket_log_layout_generation& l, bufferlist& bl, uint64_t f=0);
 195 void decode(bucket_log_layout_generation& l, bufferlist::const_iterator& bl);
 196 void encode_json_impl(const char *name, const bucket_log_layout_generation& l, ceph::Formatter *f);
 197 void decode_json_obj(bucket_log_layout_generation& l, JSONObj *obj);
 198
 199 // return a log layout that shares its layout with the index
 200 inline bucket_log_layout_generation log_layout_from_index(
 201     uint64_t gen, const bucket_index_layout_generation& index)
 202 {
 203   return {gen, {BucketLogType::InIndex, {index.gen, index.layout.normal}}};
 204 }
 205
 206 inline auto matches_gen(uint64_t gen)
 207 {
 208   return [gen] (const bucket_log_layout_generation& l) { return l.gen == gen; };
 209 }
 210
 211 inline bucket_index_layout_generation log_to_index_layout(const bucket_log_layout_generation& log_layout)
 212 {
 213   ceph_assert(log_layout.layout.type == BucketLogType::InIndex);
 214   bucket_index_layout_generation index;
 215   index.gen = log_layout.layout.in_index.gen;
 216   index.layout.normal = log_layout.layout.in_index.layout;
 217   return index;
 218 }
 219
 220 enum class BucketReshardState : uint8_t {
 221   None,
 222   InProgress,
 223 };
 224 std::string_view to_string(const BucketReshardState& s);
 225 bool parse(std::string_view str, BucketReshardState& s);
 226 void encode_json_impl(const char *name, const BucketReshardState& s, ceph::Formatter *f);
 227 void decode_json_obj(BucketReshardState& s, JSONObj *obj);
 228
 229 // describes the layout of bucket index objects
 230 struct BucketLayout {
 231   BucketReshardState resharding = BucketReshardState::None;
 232
 233   // current bucket index layout
 234   bucket_index_layout_generation current_index;
 235
 236   // target index layout of a resharding operation
 237   std::optional<bucket_index_layout_generation> target_index;
 238
 239   // history of untrimmed bucket log layout generations, with the current
 240   // generation at the back()
 241   std::vector<bucket_log_layout_generation> logs;
 242
 243   friend std::ostream& operator<<(std::ostream& out, const BucketLayout& l) {
 244     std::stringstream ss;
 245     if (l.target_index) {
 246       ss << *l.target_index;
 247     } else {
 248       ss << "none";
 249     }
 250     out << "resharding=" << to_string(l.resharding) <<
 251       ", current_index=[" << l.current_index << "], target_index=[" <<
 252       ss.str() << "], logs.size()=" << l.logs.size();
 253
 254     return out;
 255   }
 256 };
 257
 258 void encode(const BucketLayout& l, bufferlist& bl, uint64_t f=0);
 259 void decode(BucketLayout& l, bufferlist::const_iterator& bl);
 260 void encode_json_impl(const char *name, const BucketLayout& l, ceph::Formatter *f);
 261 void decode_json_obj(BucketLayout& l, JSONObj *obj);
 262
 263
 264 inline uint32_t num_shards(const bucket_index_normal_layout& index) {
 265   // old buckets used num_shards=0 to mean 1
 266   return index.num_shards > 0 ? index.num_shards : 1;
 267 }
 268 inline uint32_t num_shards(const bucket_index_layout& index) {
 269   ceph_assert(index.type == BucketIndexType::Normal);
 270   return num_shards(index.normal);
 271 }
 272 inline uint32_t num_shards(const bucket_index_layout_generation& index) {
 273   return num_shards(index.layout);
 274 }
 275 inline uint32_t current_num_shards(const BucketLayout& layout) {
 276   return num_shards(layout.current_index);
 277 }
 278 inline bool is_layout_indexless(const bucket_index_layout_generation& layout) {
 279   return layout.layout.type == BucketIndexType::Indexless;
 280 }
 281
 282 } // namespace rgw