ceph/src/cls/rgw/cls_rgw_types.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #ifndef CEPH_CLS_RGW_TYPES_H
   5 #define CEPH_CLS_RGW_TYPES_H
   6
   7 #include <boost/container/flat_map.hpp>
   8 #include "common/ceph_time.h"
   9 #include "common/Formatter.h"
  10
  11 #include "rgw/rgw_basic_types.h"
  12
  13 #define CEPH_RGW_REMOVE 'r'
  14 #define CEPH_RGW_UPDATE 'u'
  15 #define CEPH_RGW_TAG_TIMEOUT 120
  16 #define CEPH_RGW_DIR_SUGGEST_LOG_OP  0x80
  17 #define CEPH_RGW_DIR_SUGGEST_OP_MASK 0x7f
  18
  19 class JSONObj;
  20
  21 namespace ceph {
  22   class Formatter;
  23 }
  24 using ceph::operator <<;
  25
  26 struct rgw_zone_set_entry {
  27   string zone;
  28   std::optional<std::string> location_key;
  29
  30   bool operator<(const rgw_zone_set_entry& e) const {
  31     if (zone < e.zone) {
  32       return true;
  33     }
  34     if (zone > e.zone) {
  35       return false;
  36     }
  37     return (location_key < e.location_key);
  38   }
  39
  40   rgw_zone_set_entry() {}
  41   rgw_zone_set_entry(const string& _zone,
  42                      std::optional<std::string> _location_key) : zone(_zone),
  43                                                                 location_key(_location_key) {}
  44   rgw_zone_set_entry(const string& s) {
  45     from_str(s);
  46   }
  47
  48   void from_str(const string& s);
  49   string to_str() const;
  50
  51   void encode(bufferlist &bl) const;
  52   void decode(bufferlist::const_iterator &bl);
  53
  54   void dump(Formatter *f) const;
  55   void decode_json(JSONObj *obj);
  56 };
  57 WRITE_CLASS_ENCODER(rgw_zone_set_entry)
  58
  59 struct rgw_zone_set {
  60   std::set<rgw_zone_set_entry> entries;
  61
  62   void encode(bufferlist &bl) const {
  63     /* no ENCODE_START, ENCODE_END for backward compatibility */
  64     ceph::encode(entries, bl);
  65   }
  66   void decode(bufferlist::const_iterator &bl) {
  67     /* no DECODE_START, DECODE_END for backward compatibility */
  68     ceph::decode(entries, bl);
  69   }
  70
  71   void insert(const string& zone, std::optional<string> location_key);
  72   bool exists(const string& zone, std::optional<string> location_key) const;
  73 };
  74 WRITE_CLASS_ENCODER(rgw_zone_set)
  75
  76 /* backward compatibility, rgw_zone_set needs to encode/decode the same as std::set */
  77 void encode_json(const char *name, const rgw_zone_set& zs, ceph::Formatter *f);
  78 void decode_json_obj(rgw_zone_set& zs, JSONObj *obj);
  79
  80
  81 enum RGWPendingState {
  82   CLS_RGW_STATE_PENDING_MODIFY = 0,
  83   CLS_RGW_STATE_COMPLETE       = 1,
  84   CLS_RGW_STATE_UNKNOWN        = 2,
  85 };
  86
  87 enum RGWModifyOp {
  88   CLS_RGW_OP_ADD     = 0,
  89   CLS_RGW_OP_DEL     = 1,
  90   CLS_RGW_OP_CANCEL  = 2,
  91   CLS_RGW_OP_UNKNOWN = 3,
  92   CLS_RGW_OP_LINK_OLH        = 4,
  93   CLS_RGW_OP_LINK_OLH_DM     = 5, /* creation of delete marker */
  94   CLS_RGW_OP_UNLINK_INSTANCE = 6,
  95   CLS_RGW_OP_SYNCSTOP        = 7,
  96   CLS_RGW_OP_RESYNC          = 8,
  97 };
  98
  99 enum RGWBILogFlags {
 100   RGW_BILOG_FLAG_VERSIONED_OP = 0x1,
 101 };
 102
 103 enum RGWCheckMTimeType {
 104   CLS_RGW_CHECK_TIME_MTIME_EQ = 0,
 105   CLS_RGW_CHECK_TIME_MTIME_LT = 1,
 106   CLS_RGW_CHECK_TIME_MTIME_LE = 2,
 107   CLS_RGW_CHECK_TIME_MTIME_GT = 3,
 108   CLS_RGW_CHECK_TIME_MTIME_GE = 4,
 109 };
 110
 111 #define ROUND_BLOCK_SIZE 4096
 112
 113 static inline uint64_t cls_rgw_get_rounded_size(uint64_t size) {
 114   return (size + ROUND_BLOCK_SIZE - 1) & ~(ROUND_BLOCK_SIZE - 1);
 115 }
 116
 117 /*
 118  * This takes a string that either wholly contains a delimiter or is a
 119  * path that ends with a delimiter and appends a new character to the
 120  * end such that when a we request bucket-index entries *after* this,
 121  * we'll get the next object after the "subdirectory". This works
 122  * because we append a '\xFF' charater, and no valid UTF-8 character
 123  * can contain that byte, so no valid entries can be skipped.
 124  */
 125 static inline std::string cls_rgw_after_delim(const std::string& path) {
 126   // assert: ! path.empty()
 127   return path + '\xFF';
 128 }
 129
 130 struct rgw_bucket_pending_info {
 131   RGWPendingState state;
 132   ceph::real_time timestamp;
 133   uint8_t op;
 134
 135   rgw_bucket_pending_info() : state(CLS_RGW_STATE_PENDING_MODIFY), op(0) {}
 136
 137   void encode(bufferlist &bl) const {
 138     ENCODE_START(2, 2, bl);
 139     uint8_t s = (uint8_t)state;
 140     encode(s, bl);
 141     encode(timestamp, bl);
 142     encode(op, bl);
 143     ENCODE_FINISH(bl);
 144   }
 145   void decode(bufferlist::const_iterator &bl) {
 146     DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
 147     uint8_t s;
 148     decode(s, bl);
 149     state = (RGWPendingState)s;
 150     decode(timestamp, bl);
 151     decode(op, bl);
 152     DECODE_FINISH(bl);
 153   }
 154   void dump(Formatter *f) const;
 155   void decode_json(JSONObj *obj);
 156   static void generate_test_instances(list<rgw_bucket_pending_info*>& o);
 157 };
 158 WRITE_CLASS_ENCODER(rgw_bucket_pending_info)
 159
 160
 161 // categories of objects stored in a bucket index (b-i) and used to
 162 // differentiate their associated statistics (bucket stats, and in
 163 // some cases user stats)
 164 enum class RGWObjCategory : uint8_t {
 165   None      = 0,  // b-i entries for delete markers; also used in
 166                   // testing and for default values in default
 167                   // constructors
 168
 169   Main      = 1,  // b-i entries for standard objs
 170
 171   Shadow    = 2,  // presumfably intended for multipart shadow
 172                   // uploads; not currently used in the codebase
 173
 174   MultiMeta = 3,  // b-i entries for multipart upload metadata objs
 175 };
 176
 177
 178 struct rgw_bucket_dir_entry_meta {
 179   RGWObjCategory category;
 180   uint64_t size;
 181   ceph::real_time mtime;
 182   string etag;
 183   string owner;
 184   string owner_display_name;
 185   string content_type;
 186   uint64_t accounted_size;
 187   string user_data;
 188   string storage_class;
 189   bool appendable;
 190
 191   rgw_bucket_dir_entry_meta() :
 192     category(RGWObjCategory::None), size(0), accounted_size(0), appendable(false) { }
 193
 194   void encode(bufferlist &bl) const {
 195     ENCODE_START(7, 3, bl);
 196     encode(category, bl);
 197     encode(size, bl);
 198     encode(mtime, bl);
 199     encode(etag, bl);
 200     encode(owner, bl);
 201     encode(owner_display_name, bl);
 202     encode(content_type, bl);
 203     encode(accounted_size, bl);
 204     encode(user_data, bl);
 205     encode(storage_class, bl);
 206     encode(appendable, bl);
 207     ENCODE_FINISH(bl);
 208   }
 209
 210   void decode(bufferlist::const_iterator &bl) {
 211     DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
 212     decode(category, bl);
 213     decode(size, bl);
 214     decode(mtime, bl);
 215     decode(etag, bl);
 216     decode(owner, bl);
 217     decode(owner_display_name, bl);
 218     if (struct_v >= 2)
 219       decode(content_type, bl);
 220     if (struct_v >= 4)
 221       decode(accounted_size, bl);
 222     else
 223       accounted_size = size;
 224     if (struct_v >= 5)
 225       decode(user_data, bl);
 226     if (struct_v >= 6)
 227       decode(storage_class, bl);
 228     if (struct_v >= 7)
 229       decode(appendable, bl);
 230     DECODE_FINISH(bl);
 231   }
 232   void dump(Formatter *f) const;
 233   void decode_json(JSONObj *obj);
 234   static void generate_test_instances(list<rgw_bucket_dir_entry_meta*>& o);
 235 };
 236 WRITE_CLASS_ENCODER(rgw_bucket_dir_entry_meta)
 237
 238 template<class T>
 239 void encode_packed_val(T val, bufferlist& bl)
 240 {
 241   using ceph::encode;
 242   if ((uint64_t)val < 0x80) {
 243     encode((uint8_t)val, bl);
 244   } else {
 245     unsigned char c = 0x80;
 246
 247     if ((uint64_t)val < 0x100) {
 248       c |= 1;
 249       encode(c, bl);
 250       encode((uint8_t)val, bl);
 251     } else if ((uint64_t)val <= 0x10000) {
 252       c |= 2;
 253       encode(c, bl);
 254       encode((uint16_t)val, bl);
 255     } else if ((uint64_t)val <= 0x1000000) {
 256       c |= 4;
 257       encode(c, bl);
 258       encode((uint32_t)val, bl);
 259     } else {
 260       c |= 8;
 261       encode(c, bl);
 262       encode((uint64_t)val, bl);
 263     }
 264   }
 265 }
 266
 267 template<class T>
 268 void decode_packed_val(T& val, bufferlist::const_iterator& bl)
 269 {
 270   using ceph::decode;
 271   unsigned char c;
 272   decode(c, bl);
 273   if (c < 0x80) {
 274     val = c;
 275     return;
 276   }
 277
 278   c &= ~0x80;
 279
 280   switch (c) {
 281     case 1:
 282       {
 283         uint8_t v;
 284         decode(v, bl);
 285         val = v;
 286       }
 287       break;
 288     case 2:
 289       {
 290         uint16_t v;
 291         decode(v, bl);
 292         val = v;
 293       }
 294       break;
 295     case 4:
 296       {
 297         uint32_t v;
 298         decode(v, bl);
 299         val = v;
 300       }
 301       break;
 302     case 8:
 303       {
 304         uint64_t v;
 305         decode(v, bl);
 306         val = v;
 307       }
 308       break;
 309     default:
 310       throw buffer::error();
 311   }
 312 }
 313
 314 struct rgw_bucket_entry_ver {
 315   int64_t pool;
 316   uint64_t epoch;
 317
 318   rgw_bucket_entry_ver() : pool(-1), epoch(0) {}
 319
 320   void encode(bufferlist &bl) const {
 321     ENCODE_START(1, 1, bl);
 322     encode_packed_val(pool, bl);
 323     encode_packed_val(epoch, bl);
 324     ENCODE_FINISH(bl);
 325   }
 326   void decode(bufferlist::const_iterator &bl) {
 327     DECODE_START(1, bl);
 328     decode_packed_val(pool, bl);
 329     decode_packed_val(epoch, bl);
 330     DECODE_FINISH(bl);
 331   }
 332   void dump(Formatter *f) const;
 333   void decode_json(JSONObj *obj);
 334   static void generate_test_instances(list<rgw_bucket_entry_ver*>& o);
 335 };
 336 WRITE_CLASS_ENCODER(rgw_bucket_entry_ver)
 337
 338 struct cls_rgw_obj_key {
 339   string name;
 340   string instance;
 341
 342   cls_rgw_obj_key() {}
 343   cls_rgw_obj_key(const string &_name) : name(_name) {}
 344   cls_rgw_obj_key(const string& n, const string& i) : name(n), instance(i) {}
 345
 346   void set(const string& _name) {
 347     name = _name;
 348   }
 349
 350   bool operator==(const cls_rgw_obj_key& k) const {
 351     return (name.compare(k.name) == 0) &&
 352            (instance.compare(k.instance) == 0);
 353   }
 354   bool operator<(const cls_rgw_obj_key& k) const {
 355     int r = name.compare(k.name);
 356     if (r == 0) {
 357       r = instance.compare(k.instance);
 358     }
 359     return (r < 0);
 360   }
 361   bool operator<=(const cls_rgw_obj_key& k) const {
 362     return !(k < *this);
 363   }
 364   bool empty() const {
 365     return name.empty();
 366   }
 367   void encode(bufferlist &bl) const {
 368     ENCODE_START(1, 1, bl);
 369     encode(name, bl);
 370     encode(instance, bl);
 371     ENCODE_FINISH(bl);
 372   }
 373   void decode(bufferlist::const_iterator &bl) {
 374     DECODE_START(1, bl);
 375     decode(name, bl);
 376     decode(instance, bl);
 377     DECODE_FINISH(bl);
 378   }
 379   void dump(Formatter *f) const {
 380     f->dump_string("name", name);
 381     f->dump_string("instance", instance);
 382   }
 383   void decode_json(JSONObj *obj);
 384   static void generate_test_instances(list<cls_rgw_obj_key*>& ls) {
 385     ls.push_back(new cls_rgw_obj_key);
 386     ls.push_back(new cls_rgw_obj_key);
 387     ls.back()->name = "name";
 388     ls.back()->instance = "instance";
 389   }
 390 };
 391 WRITE_CLASS_ENCODER(cls_rgw_obj_key)
 392
 393
 394 struct rgw_bucket_dir_entry {
 395   /* a versioned object instance */
 396   static constexpr uint16_t FLAG_VER =                0x1;
 397   /* the last object instance of a versioned object */
 398   static constexpr uint16_t FLAG_CURRENT =            0x2;
 399   /* delete marker */
 400   static constexpr uint16_t FLAG_DELETE_MARKER =      0x4;
 401   /* object is versioned, a placeholder for the plain entry */
 402   static constexpr uint16_t FLAG_VER_MARKER =         0x8;
 403   /* object is a proxy; it is not listed in the bucket index but is a
 404    * prefix ending with a delimiter, perhaps common to multiple
 405    * entries; it is only useful when a delimiter is used and
 406    * represents a "subdirectory" (again, ending in a delimiter) that
 407    * may contain one or more actual entries/objects */
 408   static constexpr uint16_t FLAG_COMMON_PREFIX =   0x8000;
 409
 410   cls_rgw_obj_key key;
 411   rgw_bucket_entry_ver ver;
 412   std::string locator;
 413   bool exists;
 414   rgw_bucket_dir_entry_meta meta;
 415   multimap<string, rgw_bucket_pending_info> pending_map;
 416   uint64_t index_ver;
 417   string tag;
 418   uint16_t flags;
 419   uint64_t versioned_epoch;
 420
 421   rgw_bucket_dir_entry() :
 422     exists(false), index_ver(0), flags(0), versioned_epoch(0) {}
 423
 424   void encode(bufferlist &bl) const {
 425     ENCODE_START(8, 3, bl);
 426     encode(key.name, bl);
 427     encode(ver.epoch, bl);
 428     encode(exists, bl);
 429     encode(meta, bl);
 430     encode(pending_map, bl);
 431     encode(locator, bl);
 432     encode(ver, bl);
 433     encode_packed_val(index_ver, bl);
 434     encode(tag, bl);
 435     encode(key.instance, bl);
 436     encode(flags, bl);
 437     encode(versioned_epoch, bl);
 438     ENCODE_FINISH(bl);
 439   }
 440   void decode(bufferlist::const_iterator &bl) {
 441     DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl);
 442     decode(key.name, bl);
 443     decode(ver.epoch, bl);
 444     decode(exists, bl);
 445     decode(meta, bl);
 446     decode(pending_map, bl);
 447     if (struct_v >= 2) {
 448       decode(locator, bl);
 449     }
 450     if (struct_v >= 4) {
 451       decode(ver, bl);
 452     } else {
 453       ver.pool = -1;
 454     }
 455     if (struct_v >= 5) {
 456       decode_packed_val(index_ver, bl);
 457       decode(tag, bl);
 458     }
 459     if (struct_v >= 6) {
 460       decode(key.instance, bl);
 461     }
 462     if (struct_v >= 7) {
 463       decode(flags, bl);
 464     }
 465     if (struct_v >= 8) {
 466       decode(versioned_epoch, bl);
 467     }
 468     DECODE_FINISH(bl);
 469   }
 470
 471   bool is_current() const {
 472     int test_flags =
 473       rgw_bucket_dir_entry::FLAG_VER | rgw_bucket_dir_entry::FLAG_CURRENT;
 474     return (flags & rgw_bucket_dir_entry::FLAG_VER) == 0 ||
 475            (flags & test_flags) == test_flags;
 476   }
 477   bool is_delete_marker() const {
 478     return (flags & rgw_bucket_dir_entry::FLAG_DELETE_MARKER) != 0;
 479   }
 480   bool is_visible() const {
 481     return is_current() && !is_delete_marker();
 482   }
 483   bool is_valid() const {
 484     return (flags & rgw_bucket_dir_entry::FLAG_VER_MARKER) == 0;
 485   }
 486   bool is_common_prefix() const {
 487     return flags & rgw_bucket_dir_entry::FLAG_COMMON_PREFIX;
 488   }
 489
 490   void dump(Formatter *f) const;
 491   void decode_json(JSONObj *obj);
 492   static void generate_test_instances(list<rgw_bucket_dir_entry*>& o);
 493 };
 494 WRITE_CLASS_ENCODER(rgw_bucket_dir_entry)
 495
 496 enum class BIIndexType : uint8_t {
 497   Invalid    = 0,
 498   Plain      = 1,
 499   Instance   = 2,
 500   OLH        = 3,
 501 };
 502
 503 struct rgw_bucket_category_stats;
 504
 505 struct rgw_cls_bi_entry {
 506   BIIndexType type;
 507   string idx;
 508   bufferlist data;
 509
 510   rgw_cls_bi_entry() : type(BIIndexType::Invalid) {}
 511
 512   void encode(bufferlist& bl) const {
 513     ENCODE_START(1, 1, bl);
 514     encode(type, bl);
 515     encode(idx, bl);
 516     encode(data, bl);
 517     ENCODE_FINISH(bl);
 518   }
 519
 520   void decode(bufferlist::const_iterator& bl) {
 521     DECODE_START(1, bl);
 522     uint8_t c;
 523     decode(c, bl);
 524     type = (BIIndexType)c;
 525     decode(idx, bl);
 526     decode(data, bl);
 527     DECODE_FINISH(bl);
 528   }
 529
 530   void dump(Formatter *f) const;
 531   void decode_json(JSONObj *obj, cls_rgw_obj_key *effective_key = NULL);
 532
 533   bool get_info(cls_rgw_obj_key *key, RGWObjCategory *category,
 534                 rgw_bucket_category_stats *accounted_stats);
 535 };
 536 WRITE_CLASS_ENCODER(rgw_cls_bi_entry)
 537
 538 enum OLHLogOp {
 539   CLS_RGW_OLH_OP_UNKNOWN         = 0,
 540   CLS_RGW_OLH_OP_LINK_OLH        = 1,
 541   CLS_RGW_OLH_OP_UNLINK_OLH      = 2, /* object does not exist */
 542   CLS_RGW_OLH_OP_REMOVE_INSTANCE = 3,
 543 };
 544
 545 struct rgw_bucket_olh_log_entry {
 546   uint64_t epoch;
 547   OLHLogOp op;
 548   string op_tag;
 549   cls_rgw_obj_key key;
 550   bool delete_marker;
 551
 552   rgw_bucket_olh_log_entry() : epoch(0), op(CLS_RGW_OLH_OP_UNKNOWN), delete_marker(false) {}
 553
 554
 555   void encode(bufferlist &bl) const {
 556     ENCODE_START(1, 1, bl);
 557     encode(epoch, bl);
 558     encode((__u8)op, bl);
 559     encode(op_tag, bl);
 560     encode(key, bl);
 561     encode(delete_marker, bl);
 562     ENCODE_FINISH(bl);
 563   }
 564   void decode(bufferlist::const_iterator &bl) {
 565     DECODE_START(1, bl);
 566     decode(epoch, bl);
 567     uint8_t c;
 568     decode(c, bl);
 569     op = (OLHLogOp)c;
 570     decode(op_tag, bl);
 571     decode(key, bl);
 572     decode(delete_marker, bl);
 573     DECODE_FINISH(bl);
 574   }
 575   static void generate_test_instances(list<rgw_bucket_olh_log_entry*>& o);
 576   void dump(Formatter *f) const;
 577   void decode_json(JSONObj *obj);
 578 };
 579 WRITE_CLASS_ENCODER(rgw_bucket_olh_log_entry)
 580
 581 struct rgw_bucket_olh_entry {
 582   cls_rgw_obj_key key;
 583   bool delete_marker;
 584   uint64_t epoch;
 585   map<uint64_t, vector<struct rgw_bucket_olh_log_entry> > pending_log;
 586   string tag;
 587   bool exists;
 588   bool pending_removal;
 589
 590   rgw_bucket_olh_entry() : delete_marker(false), epoch(0), exists(false), pending_removal(false) {}
 591
 592   void encode(bufferlist &bl) const {
 593     ENCODE_START(1, 1, bl);
 594     encode(key, bl);
 595     encode(delete_marker, bl);
 596     encode(epoch, bl);
 597     encode(pending_log, bl);
 598     encode(tag, bl);
 599     encode(exists, bl);
 600     encode(pending_removal, bl);
 601     ENCODE_FINISH(bl);
 602   }
 603   void decode(bufferlist::const_iterator &bl) {
 604     DECODE_START(1, bl);
 605     decode(key, bl);
 606     decode(delete_marker, bl);
 607     decode(epoch, bl);
 608     decode(pending_log, bl);
 609     decode(tag, bl);
 610     decode(exists, bl);
 611     decode(pending_removal, bl);
 612     DECODE_FINISH(bl);
 613   }
 614   void dump(Formatter *f) const;
 615   void decode_json(JSONObj *obj);
 616 };
 617 WRITE_CLASS_ENCODER(rgw_bucket_olh_entry)
 618
 619 struct rgw_bi_log_entry {
 620   string id;
 621   string object;
 622   string instance;
 623   ceph::real_time timestamp;
 624   rgw_bucket_entry_ver ver;
 625   RGWModifyOp op;
 626   RGWPendingState state;
 627   uint64_t index_ver;
 628   string tag;
 629   uint16_t bilog_flags;
 630   string owner; /* only being set if it's a delete marker */
 631   string owner_display_name; /* only being set if it's a delete marker */
 632   rgw_zone_set zones_trace;
 633
 634   rgw_bi_log_entry() : op(CLS_RGW_OP_UNKNOWN), state(CLS_RGW_STATE_PENDING_MODIFY), index_ver(0), bilog_flags(0) {}
 635
 636   void encode(bufferlist &bl) const {
 637     ENCODE_START(4, 1, bl);
 638     encode(id, bl);
 639     encode(object, bl);
 640     encode(timestamp, bl);
 641     encode(ver, bl);
 642     encode(tag, bl);
 643     uint8_t c = (uint8_t)op;
 644     encode(c, bl);
 645     c = (uint8_t)state;
 646     encode(c, bl);
 647     encode_packed_val(index_ver, bl);
 648     encode(instance, bl);
 649     encode(bilog_flags, bl);
 650     encode(owner, bl);
 651     encode(owner_display_name, bl);
 652     encode(zones_trace, bl);
 653     ENCODE_FINISH(bl);
 654   }
 655   void decode(bufferlist::const_iterator &bl) {
 656     DECODE_START(4, bl);
 657     decode(id, bl);
 658     decode(object, bl);
 659     decode(timestamp, bl);
 660     decode(ver, bl);
 661     decode(tag, bl);
 662     uint8_t c;
 663     decode(c, bl);
 664     op = (RGWModifyOp)c;
 665     decode(c, bl);
 666     state = (RGWPendingState)c;
 667     decode_packed_val(index_ver, bl);
 668     if (struct_v >= 2) {
 669       decode(instance, bl);
 670       decode(bilog_flags, bl);
 671     }
 672     if (struct_v >= 3) {
 673       decode(owner, bl);
 674       decode(owner_display_name, bl);
 675     }
 676     if (struct_v >= 4) {
 677       decode(zones_trace, bl);
 678     }
 679     DECODE_FINISH(bl);
 680   }
 681   void dump(Formatter *f) const;
 682   void decode_json(JSONObj *obj);
 683   static void generate_test_instances(list<rgw_bi_log_entry*>& o);
 684
 685   bool is_versioned() {
 686     return ((bilog_flags & RGW_BILOG_FLAG_VERSIONED_OP) != 0);
 687   }
 688 };
 689 WRITE_CLASS_ENCODER(rgw_bi_log_entry)
 690
 691 struct rgw_bucket_category_stats {
 692   uint64_t total_size;
 693   uint64_t total_size_rounded;
 694   uint64_t num_entries;
 695   uint64_t actual_size{0}; //< account for compression, encryption
 696
 697   rgw_bucket_category_stats() : total_size(0), total_size_rounded(0), num_entries(0) {}
 698
 699   void encode(bufferlist &bl) const {
 700     ENCODE_START(3, 2, bl);
 701     encode(total_size, bl);
 702     encode(total_size_rounded, bl);
 703     encode(num_entries, bl);
 704     encode(actual_size, bl);
 705     ENCODE_FINISH(bl);
 706   }
 707   void decode(bufferlist::const_iterator &bl) {
 708     DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
 709     decode(total_size, bl);
 710     decode(total_size_rounded, bl);
 711     decode(num_entries, bl);
 712     if (struct_v >= 3) {
 713       decode(actual_size, bl);
 714     } else {
 715       actual_size = total_size;
 716     }
 717     DECODE_FINISH(bl);
 718   }
 719   void dump(Formatter *f) const;
 720   static void generate_test_instances(list<rgw_bucket_category_stats*>& o);
 721 };
 722 WRITE_CLASS_ENCODER(rgw_bucket_category_stats)
 723
 724 enum class cls_rgw_reshard_status : uint8_t {
 725   NOT_RESHARDING  = 0,
 726   IN_PROGRESS     = 1,
 727   DONE            = 2
 728 };
 729
 730 static inline std::string to_string(const cls_rgw_reshard_status status)
 731 {
 732   switch (status) {
 733   case cls_rgw_reshard_status::NOT_RESHARDING:
 734     return "not-resharding";
 735     break;
 736   case cls_rgw_reshard_status::IN_PROGRESS:
 737     return "in-progress";
 738     break;
 739   case cls_rgw_reshard_status::DONE:
 740     return "done";
 741     break;
 742   };
 743   return "Unknown reshard status";
 744 }
 745
 746 struct cls_rgw_bucket_instance_entry {
 747   using RESHARD_STATUS = cls_rgw_reshard_status;
 748
 749   cls_rgw_reshard_status reshard_status{RESHARD_STATUS::NOT_RESHARDING};
 750   string new_bucket_instance_id;
 751   int32_t num_shards{-1};
 752
 753   void encode(bufferlist& bl) const {
 754     ENCODE_START(1, 1, bl);
 755     encode((uint8_t)reshard_status, bl);
 756     encode(new_bucket_instance_id, bl);
 757     encode(num_shards, bl);
 758     ENCODE_FINISH(bl);
 759   }
 760
 761   void decode(bufferlist::const_iterator& bl) {
 762     DECODE_START(1, bl);
 763     uint8_t s;
 764     decode(s, bl);
 765     reshard_status = (cls_rgw_reshard_status)s;
 766     decode(new_bucket_instance_id, bl);
 767     decode(num_shards, bl);
 768     DECODE_FINISH(bl);
 769   }
 770
 771   void dump(Formatter *f) const;
 772   static void generate_test_instances(list<cls_rgw_bucket_instance_entry*>& o);
 773
 774   void clear() {
 775     reshard_status = RESHARD_STATUS::NOT_RESHARDING;
 776     new_bucket_instance_id.clear();
 777   }
 778
 779   void set_status(const string& new_instance_id,
 780                   int32_t new_num_shards,
 781                   cls_rgw_reshard_status s) {
 782     reshard_status = s;
 783     new_bucket_instance_id = new_instance_id;
 784     num_shards = new_num_shards;
 785   }
 786
 787   bool resharding() const {
 788     return reshard_status != RESHARD_STATUS::NOT_RESHARDING;
 789   }
 790   bool resharding_in_progress() const {
 791     return reshard_status == RESHARD_STATUS::IN_PROGRESS;
 792   }
 793 };
 794 WRITE_CLASS_ENCODER(cls_rgw_bucket_instance_entry)
 795
 796 struct rgw_bucket_dir_header {
 797   map<RGWObjCategory, rgw_bucket_category_stats> stats;
 798   uint64_t tag_timeout;
 799   uint64_t ver;
 800   uint64_t master_ver;
 801   string max_marker;
 802   cls_rgw_bucket_instance_entry new_instance;
 803   bool syncstopped;
 804
 805   rgw_bucket_dir_header() : tag_timeout(0), ver(0), master_ver(0), syncstopped(false) {}
 806
 807   void encode(bufferlist &bl) const {
 808     ENCODE_START(7, 2, bl);
 809     encode(stats, bl);
 810     encode(tag_timeout, bl);
 811     encode(ver, bl);
 812     encode(master_ver, bl);
 813     encode(max_marker, bl);
 814     encode(new_instance, bl);
 815     encode(syncstopped,bl);
 816     ENCODE_FINISH(bl);
 817   }
 818   void decode(bufferlist::const_iterator &bl) {
 819     DECODE_START_LEGACY_COMPAT_LEN(6, 2, 2, bl);
 820     decode(stats, bl);
 821     if (struct_v > 2) {
 822       decode(tag_timeout, bl);
 823     } else {
 824       tag_timeout = 0;
 825     }
 826     if (struct_v >= 4) {
 827       decode(ver, bl);
 828       decode(master_ver, bl);
 829     } else {
 830       ver = 0;
 831     }
 832     if (struct_v >= 5) {
 833       decode(max_marker, bl);
 834     }
 835     if (struct_v >= 6) {
 836       decode(new_instance, bl);
 837     } else {
 838       new_instance = cls_rgw_bucket_instance_entry();
 839     }
 840     if (struct_v >= 7) {
 841       decode(syncstopped,bl);
 842     }
 843     DECODE_FINISH(bl);
 844   }
 845   void dump(Formatter *f) const;
 846   static void generate_test_instances(list<rgw_bucket_dir_header*>& o);
 847
 848   bool resharding() const {
 849     return new_instance.resharding();
 850   }
 851   bool resharding_in_progress() const {
 852     return new_instance.resharding_in_progress();
 853   }
 854 };
 855 WRITE_CLASS_ENCODER(rgw_bucket_dir_header)
 856
 857 struct rgw_bucket_dir {
 858   rgw_bucket_dir_header header;
 859   boost::container::flat_map<string, rgw_bucket_dir_entry> m;
 860
 861   void encode(bufferlist &bl) const {
 862     ENCODE_START(2, 2, bl);
 863     encode(header, bl);
 864     encode(m, bl);
 865     ENCODE_FINISH(bl);
 866   }
 867   void decode(bufferlist::const_iterator &bl) {
 868     DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
 869     decode(header, bl);
 870     decode(m, bl);
 871     DECODE_FINISH(bl);
 872   }
 873   void dump(Formatter *f) const;
 874   static void generate_test_instances(list<rgw_bucket_dir*>& o);
 875 };
 876 WRITE_CLASS_ENCODER(rgw_bucket_dir)
 877
 878 struct rgw_usage_data {
 879   uint64_t bytes_sent;
 880   uint64_t bytes_received;
 881   uint64_t ops;
 882   uint64_t successful_ops;
 883
 884   rgw_usage_data() : bytes_sent(0), bytes_received(0), ops(0), successful_ops(0) {}
 885   rgw_usage_data(uint64_t sent, uint64_t received) : bytes_sent(sent), bytes_received(received), ops(0), successful_ops(0) {}
 886
 887   void encode(bufferlist& bl) const {
 888     ENCODE_START(1, 1, bl);
 889     encode(bytes_sent, bl);
 890     encode(bytes_received, bl);
 891     encode(ops, bl);
 892     encode(successful_ops, bl);
 893     ENCODE_FINISH(bl);
 894   }
 895
 896   void decode(bufferlist::const_iterator& bl) {
 897     DECODE_START(1, bl);
 898     decode(bytes_sent, bl);
 899     decode(bytes_received, bl);
 900     decode(ops, bl);
 901     decode(successful_ops, bl);
 902     DECODE_FINISH(bl);
 903   }
 904
 905   void aggregate(const rgw_usage_data& usage) {
 906     bytes_sent += usage.bytes_sent;
 907     bytes_received += usage.bytes_received;
 908     ops += usage.ops;
 909     successful_ops += usage.successful_ops;
 910   }
 911 };
 912 WRITE_CLASS_ENCODER(rgw_usage_data)
 913
 914
 915 struct rgw_usage_log_entry {
 916   rgw_user owner;
 917   rgw_user payer; /* if empty, same as owner */
 918   string bucket;
 919   uint64_t epoch;
 920   rgw_usage_data total_usage; /* this one is kept for backwards compatibility */
 921   map<string, rgw_usage_data> usage_map;
 922
 923   rgw_usage_log_entry() : epoch(0) {}
 924   rgw_usage_log_entry(string& o, string& b) : owner(o), bucket(b), epoch(0) {}
 925   rgw_usage_log_entry(string& o, string& p, string& b) : owner(o), payer(p), bucket(b), epoch(0) {}
 926
 927   void encode(bufferlist& bl) const {
 928     ENCODE_START(3, 1, bl);
 929     encode(owner.to_str(), bl);
 930     encode(bucket, bl);
 931     encode(epoch, bl);
 932     encode(total_usage.bytes_sent, bl);
 933     encode(total_usage.bytes_received, bl);
 934     encode(total_usage.ops, bl);
 935     encode(total_usage.successful_ops, bl);
 936     encode(usage_map, bl);
 937     encode(payer.to_str(), bl);
 938     ENCODE_FINISH(bl);
 939   }
 940
 941
 942    void decode(bufferlist::const_iterator& bl) {
 943     DECODE_START(3, bl);
 944     string s;
 945     decode(s, bl);
 946     owner.from_str(s);
 947     decode(bucket, bl);
 948     decode(epoch, bl);
 949     decode(total_usage.bytes_sent, bl);
 950     decode(total_usage.bytes_received, bl);
 951     decode(total_usage.ops, bl);
 952     decode(total_usage.successful_ops, bl);
 953     if (struct_v < 2) {
 954       usage_map[""] = total_usage;
 955     } else {
 956       decode(usage_map, bl);
 957     }
 958     if (struct_v >= 3) {
 959       string p;
 960       decode(p, bl);
 961       payer.from_str(p);
 962     }
 963     DECODE_FINISH(bl);
 964   }
 965
 966   void aggregate(const rgw_usage_log_entry& e, map<string, bool> *categories = NULL) {
 967     if (owner.empty()) {
 968       owner = e.owner;
 969       bucket = e.bucket;
 970       epoch = e.epoch;
 971       payer = e.payer;
 972     }
 973
 974     map<string, rgw_usage_data>::const_iterator iter;
 975     for (iter = e.usage_map.begin(); iter != e.usage_map.end(); ++iter) {
 976       if (!categories || !categories->size() || categories->count(iter->first)) {
 977         add(iter->first, iter->second);
 978       }
 979     }
 980   }
 981
 982   void sum(rgw_usage_data& usage, map<string, bool>& categories) const {
 983     usage = rgw_usage_data();
 984     for (map<string, rgw_usage_data>::const_iterator iter = usage_map.begin(); iter != usage_map.end(); ++iter) {
 985       if (!categories.size() || categories.count(iter->first)) {
 986         usage.aggregate(iter->second);
 987       }
 988     }
 989   }
 990
 991   void add(const string& category, const rgw_usage_data& data) {
 992     usage_map[category].aggregate(data);
 993     total_usage.aggregate(data);
 994   }
 995
 996   void dump(Formatter* f) const;
 997   static void generate_test_instances(list<rgw_usage_log_entry*>& o);
 998
 999 };
1000 WRITE_CLASS_ENCODER(rgw_usage_log_entry)
1001
1002 struct rgw_usage_log_info {
1003   vector<rgw_usage_log_entry> entries;
1004
1005   void encode(bufferlist& bl) const {
1006     ENCODE_START(1, 1, bl);
1007     encode(entries, bl);
1008     ENCODE_FINISH(bl);
1009   }
1010
1011   void decode(bufferlist::const_iterator& bl) {
1012     DECODE_START(1, bl);
1013     decode(entries, bl);
1014     DECODE_FINISH(bl);
1015   }
1016
1017   rgw_usage_log_info() {}
1018 };
1019 WRITE_CLASS_ENCODER(rgw_usage_log_info)
1020
1021 struct rgw_user_bucket {
1022   string user;
1023   string bucket;
1024
1025   rgw_user_bucket() {}
1026   rgw_user_bucket(const string& u, const string& b) : user(u), bucket(b) {}
1027
1028   void encode(bufferlist& bl) const {
1029     ENCODE_START(1, 1, bl);
1030     encode(user, bl);
1031     encode(bucket, bl);
1032     ENCODE_FINISH(bl);
1033   }
1034
1035   void decode(bufferlist::const_iterator& bl) {
1036     DECODE_START(1, bl);
1037     decode(user, bl);
1038     decode(bucket, bl);
1039     DECODE_FINISH(bl);
1040   }
1041
1042   bool operator<(const rgw_user_bucket& ub2) const {
1043     int comp = user.compare(ub2.user);
1044     if (comp < 0)
1045       return true;
1046     else if (!comp)
1047       return bucket.compare(ub2.bucket) < 0;
1048
1049     return false;
1050   }
1051 };
1052 WRITE_CLASS_ENCODER(rgw_user_bucket)
1053
1054 enum cls_rgw_gc_op {
1055   CLS_RGW_GC_DEL_OBJ,
1056   CLS_RGW_GC_DEL_BUCKET,
1057 };
1058
1059 struct cls_rgw_obj {
1060   string pool;
1061   cls_rgw_obj_key key;
1062   string loc;
1063
1064   cls_rgw_obj() {}
1065   cls_rgw_obj(string& _p, cls_rgw_obj_key& _k) : pool(_p), key(_k) {}
1066
1067   void encode(bufferlist& bl) const {
1068     ENCODE_START(2, 1, bl);
1069     encode(pool, bl);
1070     encode(key.name, bl);
1071     encode(loc, bl);
1072     encode(key, bl);
1073     ENCODE_FINISH(bl);
1074   }
1075
1076   void decode(bufferlist::const_iterator& bl) {
1077     DECODE_START(2, bl);
1078     decode(pool, bl);
1079     decode(key.name, bl);
1080     decode(loc, bl);
1081     if (struct_v >= 2) {
1082       decode(key, bl);
1083     }
1084     DECODE_FINISH(bl);
1085   }
1086
1087   void dump(Formatter *f) const {
1088     f->dump_string("pool", pool);
1089     f->dump_string("oid", key.name);
1090     f->dump_string("key", loc);
1091     f->dump_string("instance", key.instance);
1092   }
1093   static void generate_test_instances(list<cls_rgw_obj*>& ls) {
1094     ls.push_back(new cls_rgw_obj);
1095     ls.push_back(new cls_rgw_obj);
1096     ls.back()->pool = "mypool";
1097     ls.back()->key.name = "myoid";
1098     ls.back()->loc = "mykey";
1099   }
1100 };
1101 WRITE_CLASS_ENCODER(cls_rgw_obj)
1102
1103 struct cls_rgw_obj_chain {
1104   list<cls_rgw_obj> objs;
1105
1106   cls_rgw_obj_chain() {}
1107
1108   void push_obj(const string& pool, const cls_rgw_obj_key& key, const string& loc) {
1109     cls_rgw_obj obj;
1110     obj.pool = pool;
1111     obj.key = key;
1112     obj.loc = loc;
1113     objs.push_back(obj);
1114   }
1115
1116   void encode(bufferlist& bl) const {
1117     ENCODE_START(1, 1, bl);
1118     encode(objs, bl);
1119     ENCODE_FINISH(bl);
1120   }
1121
1122   void decode(bufferlist::const_iterator& bl) {
1123     DECODE_START(1, bl);
1124     decode(objs, bl);
1125     DECODE_FINISH(bl);
1126   }
1127
1128   void dump(Formatter *f) const {
1129     f->open_array_section("objs");
1130     for (list<cls_rgw_obj>::const_iterator p = objs.begin(); p != objs.end(); ++p) {
1131       f->open_object_section("obj");
1132       p->dump(f);
1133       f->close_section();
1134     }
1135     f->close_section();
1136   }
1137   static void generate_test_instances(list<cls_rgw_obj_chain*>& ls) {
1138     ls.push_back(new cls_rgw_obj_chain);
1139   }
1140
1141   bool empty() {
1142     return objs.empty();
1143   }
1144 };
1145 WRITE_CLASS_ENCODER(cls_rgw_obj_chain)
1146
1147 struct cls_rgw_gc_obj_info
1148 {
1149   string tag;
1150   cls_rgw_obj_chain chain;
1151   ceph::real_time time;
1152
1153   cls_rgw_gc_obj_info() {}
1154
1155   void encode(bufferlist& bl) const {
1156     ENCODE_START(1, 1, bl);
1157     encode(tag, bl);
1158     encode(chain, bl);
1159     encode(time, bl);
1160     ENCODE_FINISH(bl);
1161   }
1162
1163   void decode(bufferlist::const_iterator& bl) {
1164     DECODE_START(1, bl);
1165     decode(tag, bl);
1166     decode(chain, bl);
1167     decode(time, bl);
1168     DECODE_FINISH(bl);
1169   }
1170
1171   void dump(Formatter *f) const {
1172     f->dump_string("tag", tag);
1173     f->open_object_section("chain");
1174     chain.dump(f);
1175     f->close_section();
1176     f->dump_stream("time") << time;
1177   }
1178   static void generate_test_instances(list<cls_rgw_gc_obj_info*>& ls) {
1179     ls.push_back(new cls_rgw_gc_obj_info);
1180     ls.push_back(new cls_rgw_gc_obj_info);
1181     ls.back()->tag = "footag";
1182     ceph_timespec ts{init_le32(21), init_le32(32)};
1183     ls.back()->time = ceph::real_clock::from_ceph_timespec(ts);
1184   }
1185 };
1186 WRITE_CLASS_ENCODER(cls_rgw_gc_obj_info)
1187
1188 struct cls_rgw_lc_obj_head
1189 {
1190   time_t start_date = 0;
1191   string marker;
1192
1193   cls_rgw_lc_obj_head() {}
1194
1195   void encode(bufferlist& bl) const {
1196     ENCODE_START(1, 1, bl);
1197     uint64_t t = start_date;
1198     encode(t, bl);
1199     encode(marker, bl);
1200     ENCODE_FINISH(bl);
1201   }
1202
1203   void decode(bufferlist::const_iterator& bl) {
1204     DECODE_START(1, bl);
1205     uint64_t t;
1206     decode(t, bl);
1207     start_date = static_cast<time_t>(t);
1208     decode(marker, bl);
1209     DECODE_FINISH(bl);
1210   }
1211
1212   void dump(Formatter *f) const;
1213   static void generate_test_instances(list<cls_rgw_lc_obj_head*>& ls);
1214 };
1215 WRITE_CLASS_ENCODER(cls_rgw_lc_obj_head)
1216
1217 struct cls_rgw_reshard_entry
1218 {
1219   ceph::real_time time;
1220   string tenant;
1221   string bucket_name;
1222   string bucket_id;
1223   string new_instance_id;
1224   uint32_t old_num_shards{0};
1225   uint32_t new_num_shards{0};
1226
1227   cls_rgw_reshard_entry() {}
1228
1229   void encode(bufferlist& bl) const {
1230     ENCODE_START(1, 1, bl);
1231     encode(time, bl);
1232     encode(tenant, bl);
1233     encode(bucket_name, bl);
1234     encode(bucket_id, bl);
1235     encode(new_instance_id, bl);
1236     encode(old_num_shards, bl);
1237     encode(new_num_shards, bl);
1238     ENCODE_FINISH(bl);
1239   }
1240
1241   void decode(bufferlist::const_iterator& bl) {
1242     DECODE_START(1, bl);
1243     decode(time, bl);
1244     decode(tenant, bl);
1245     decode(bucket_name, bl);
1246     decode(bucket_id, bl);
1247     decode(new_instance_id, bl);
1248     decode(old_num_shards, bl);
1249     decode(new_num_shards, bl);
1250     DECODE_FINISH(bl);
1251   }
1252
1253   void dump(Formatter *f) const;
1254   static void generate_test_instances(list<cls_rgw_reshard_entry*>& o);
1255
1256   static void generate_key(const string& tenant, const string& bucket_name, string *key);
1257   void get_key(string *key) const;
1258 };
1259 WRITE_CLASS_ENCODER(cls_rgw_reshard_entry)
1260
1261 #endif