ceph/src/cls/rgw/cls_rgw_types.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #pragma once
   5
   6 #include <boost/container/flat_map.hpp>
   7 #include "common/ceph_time.h"
   8 #include "common/Formatter.h"
   9
  10 #undef FMT_HEADER_ONLY
  11 #define FMT_HEADER_ONLY 1
  12 #include <fmt/format.h>
  13
  14 #include "rgw/rgw_basic_types.h"
  15
  16 #define CEPH_RGW_REMOVE 'r'
  17 #define CEPH_RGW_UPDATE 'u'
  18 #define CEPH_RGW_TAG_TIMEOUT 120
  19 #define CEPH_RGW_DIR_SUGGEST_LOG_OP  0x80
  20 #define CEPH_RGW_DIR_SUGGEST_OP_MASK 0x7f
  21
  22 class JSONObj;
  23
  24 using ceph::operator <<;
  25
  26 struct rgw_zone_set_entry {
  27   std::string zone;
  28   std::optional<std::string> location_key;
  29
  30   bool operator<(const rgw_zone_set_entry& e) const {
  31     if (zone < e.zone) {
  32       return true;
  33     }
  34     if (zone > e.zone) {
  35       return false;
  36     }
  37     return (location_key < e.location_key);
  38   }
  39
  40   rgw_zone_set_entry() {}
  41   rgw_zone_set_entry(const std::string& _zone,
  42                      std::optional<std::string> _location_key) : zone(_zone),
  43                                                                 location_key(_location_key) {}
  44   rgw_zone_set_entry(const std::string& s) {
  45     from_str(s);
  46   }
  47
  48   void from_str(const std::string& s);
  49   std::string to_str() const;
  50
  51   void encode(ceph::buffer::list &bl) const;
  52   void decode(ceph::buffer::list::const_iterator &bl);
  53
  54   void dump(ceph::Formatter *f) const;
  55   void decode_json(JSONObj *obj);
  56 };
  57 WRITE_CLASS_ENCODER(rgw_zone_set_entry)
  58
  59 struct rgw_zone_set {
  60   std::set<rgw_zone_set_entry> entries;
  61
  62   void encode(ceph::buffer::list &bl) const {
  63     /* no ENCODE_START, ENCODE_END for backward compatibility */
  64     ceph::encode(entries, bl);
  65   }
  66   void decode(ceph::buffer::list::const_iterator &bl) {
  67     /* no DECODE_START, DECODE_END for backward compatibility */
  68     ceph::decode(entries, bl);
  69   }
  70
  71   void insert(const std::string& zone, std::optional<std::string> location_key);
  72   bool exists(const std::string& zone, std::optional<std::string> location_key) const;
  73 };
  74 WRITE_CLASS_ENCODER(rgw_zone_set)
  75
  76 /* backward compatibility, rgw_zone_set needs to encode/decode the same as std::set */
  77 void encode_json(const char *name, const rgw_zone_set& zs, ceph::Formatter *f);
  78 void decode_json_obj(rgw_zone_set& zs, JSONObj *obj);
  79
  80
  81 enum RGWPendingState {
  82   CLS_RGW_STATE_PENDING_MODIFY = 0,
  83   CLS_RGW_STATE_COMPLETE       = 1,
  84   CLS_RGW_STATE_UNKNOWN        = 2,
  85 };
  86
  87 enum RGWModifyOp {
  88   CLS_RGW_OP_ADD     = 0,
  89   CLS_RGW_OP_DEL     = 1,
  90   CLS_RGW_OP_CANCEL  = 2,
  91   CLS_RGW_OP_UNKNOWN = 3,
  92   CLS_RGW_OP_LINK_OLH        = 4,
  93   CLS_RGW_OP_LINK_OLH_DM     = 5, /* creation of delete marker */
  94   CLS_RGW_OP_UNLINK_INSTANCE = 6,
  95   CLS_RGW_OP_SYNCSTOP        = 7,
  96   CLS_RGW_OP_RESYNC          = 8,
  97 };
  98
  99 enum RGWBILogFlags {
 100   RGW_BILOG_FLAG_VERSIONED_OP = 0x1,
 101 };
 102
 103 enum RGWCheckMTimeType {
 104   CLS_RGW_CHECK_TIME_MTIME_EQ = 0,
 105   CLS_RGW_CHECK_TIME_MTIME_LT = 1,
 106   CLS_RGW_CHECK_TIME_MTIME_LE = 2,
 107   CLS_RGW_CHECK_TIME_MTIME_GT = 3,
 108   CLS_RGW_CHECK_TIME_MTIME_GE = 4,
 109 };
 110
 111 #define ROUND_BLOCK_SIZE 4096
 112
 113 inline uint64_t cls_rgw_get_rounded_size(uint64_t size) {
 114   return (size + ROUND_BLOCK_SIZE - 1) & ~(ROUND_BLOCK_SIZE - 1);
 115 }
 116
 117 /*
 118  * This takes a std::string that either wholly contains a delimiter or is a
 119  * path that ends with a delimiter and appends a new character to the
 120  * end such that when a we request bucket-index entries *after* this,
 121  * we'll get the next object after the "subdirectory". This works
 122  * because we append a '\xFF' charater, and no valid UTF-8 character
 123  * can contain that byte, so no valid entries can be skipped.
 124  */
 125 inline std::string cls_rgw_after_delim(const std::string& path) {
 126   // assert: ! path.empty()
 127   return path + '\xFF';
 128 }
 129
 130 struct rgw_bucket_pending_info {
 131   RGWPendingState state;
 132   ceph::real_time timestamp;
 133   uint8_t op;
 134
 135   rgw_bucket_pending_info() : state(CLS_RGW_STATE_PENDING_MODIFY), op(0) {}
 136
 137   void encode(ceph::buffer::list &bl) const {
 138     ENCODE_START(2, 2, bl);
 139     uint8_t s = (uint8_t)state;
 140     encode(s, bl);
 141     encode(timestamp, bl);
 142     encode(op, bl);
 143     ENCODE_FINISH(bl);
 144   }
 145   void decode(ceph::buffer::list::const_iterator &bl) {
 146     DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
 147     uint8_t s;
 148     decode(s, bl);
 149     state = (RGWPendingState)s;
 150     decode(timestamp, bl);
 151     decode(op, bl);
 152     DECODE_FINISH(bl);
 153   }
 154   void dump(ceph::Formatter *f) const;
 155   void decode_json(JSONObj *obj);
 156   static void generate_test_instances(std::list<rgw_bucket_pending_info*>& o);
 157 };
 158 WRITE_CLASS_ENCODER(rgw_bucket_pending_info)
 159
 160
 161 // categories of objects stored in a bucket index (b-i) and used to
 162 // differentiate their associated statistics (bucket stats, and in
 163 // some cases user stats)
 164 enum class RGWObjCategory : uint8_t {
 165   None      = 0,  // b-i entries for delete markers; also used in
 166                   // testing and for default values in default
 167                   // constructors
 168
 169   Main      = 1,  // b-i entries for standard objs
 170
 171   Shadow    = 2,  // presumfably intended for multipart shadow
 172                   // uploads; not currently used in the codebase
 173
 174   MultiMeta = 3,  // b-i entries for multipart upload metadata objs
 175 };
 176
 177
 178 struct rgw_bucket_dir_entry_meta {
 179   RGWObjCategory category;
 180   uint64_t size;
 181   ceph::real_time mtime;
 182   std::string etag;
 183   std::string owner;
 184   std::string owner_display_name;
 185   std::string content_type;
 186   uint64_t accounted_size;
 187   std::string user_data;
 188   std::string storage_class;
 189   bool appendable;
 190
 191   rgw_bucket_dir_entry_meta() :
 192     category(RGWObjCategory::None), size(0), accounted_size(0), appendable(false) { }
 193
 194   void encode(ceph::buffer::list &bl) const {
 195     ENCODE_START(7, 3, bl);
 196     encode(category, bl);
 197     encode(size, bl);
 198     encode(mtime, bl);
 199     encode(etag, bl);
 200     encode(owner, bl);
 201     encode(owner_display_name, bl);
 202     encode(content_type, bl);
 203     encode(accounted_size, bl);
 204     encode(user_data, bl);
 205     encode(storage_class, bl);
 206     encode(appendable, bl);
 207     ENCODE_FINISH(bl);
 208   }
 209
 210   void decode(ceph::buffer::list::const_iterator &bl) {
 211     DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
 212     decode(category, bl);
 213     decode(size, bl);
 214     decode(mtime, bl);
 215     decode(etag, bl);
 216     decode(owner, bl);
 217     decode(owner_display_name, bl);
 218     if (struct_v >= 2)
 219       decode(content_type, bl);
 220     if (struct_v >= 4)
 221       decode(accounted_size, bl);
 222     else
 223       accounted_size = size;
 224     if (struct_v >= 5)
 225       decode(user_data, bl);
 226     if (struct_v >= 6)
 227       decode(storage_class, bl);
 228     if (struct_v >= 7)
 229       decode(appendable, bl);
 230     DECODE_FINISH(bl);
 231   }
 232   void dump(ceph::Formatter *f) const;
 233   void decode_json(JSONObj *obj);
 234   static void generate_test_instances(std::list<rgw_bucket_dir_entry_meta*>& o);
 235 };
 236 WRITE_CLASS_ENCODER(rgw_bucket_dir_entry_meta)
 237
 238 template<class T>
 239 void encode_packed_val(T val, ceph::buffer::list& bl)
 240 {
 241   using ceph::encode;
 242   if ((uint64_t)val < 0x80) {
 243     encode((uint8_t)val, bl);
 244   } else {
 245     unsigned char c = 0x80;
 246
 247     if ((uint64_t)val < 0x100) {
 248       c |= 1;
 249       encode(c, bl);
 250       encode((uint8_t)val, bl);
 251     } else if ((uint64_t)val <= 0x10000) {
 252       c |= 2;
 253       encode(c, bl);
 254       encode((uint16_t)val, bl);
 255     } else if ((uint64_t)val <= 0x1000000) {
 256       c |= 4;
 257       encode(c, bl);
 258       encode((uint32_t)val, bl);
 259     } else {
 260       c |= 8;
 261       encode(c, bl);
 262       encode((uint64_t)val, bl);
 263     }
 264   }
 265 }
 266
 267 template<class T>
 268 void decode_packed_val(T& val, ceph::buffer::list::const_iterator& bl)
 269 {
 270   using ceph::decode;
 271   unsigned char c;
 272   decode(c, bl);
 273   if (c < 0x80) {
 274     val = c;
 275     return;
 276   }
 277
 278   c &= ~0x80;
 279
 280   switch (c) {
 281     case 1:
 282       {
 283         uint8_t v;
 284         decode(v, bl);
 285         val = v;
 286       }
 287       break;
 288     case 2:
 289       {
 290         uint16_t v;
 291         decode(v, bl);
 292         val = v;
 293       }
 294       break;
 295     case 4:
 296       {
 297         uint32_t v;
 298         decode(v, bl);
 299         val = v;
 300       }
 301       break;
 302     case 8:
 303       {
 304         uint64_t v;
 305         decode(v, bl);
 306         val = v;
 307       }
 308       break;
 309     default:
 310       throw ceph::buffer::malformed_input();
 311   }
 312 }
 313
 314 struct rgw_bucket_entry_ver {
 315   int64_t pool;
 316   uint64_t epoch;
 317
 318   rgw_bucket_entry_ver() : pool(-1), epoch(0) {}
 319
 320   void encode(ceph::buffer::list &bl) const {
 321     ENCODE_START(1, 1, bl);
 322     encode_packed_val(pool, bl);
 323     encode_packed_val(epoch, bl);
 324     ENCODE_FINISH(bl);
 325   }
 326   void decode(ceph::buffer::list::const_iterator &bl) {
 327     DECODE_START(1, bl);
 328     decode_packed_val(pool, bl);
 329     decode_packed_val(epoch, bl);
 330     DECODE_FINISH(bl);
 331   }
 332   void dump(ceph::Formatter *f) const;
 333   void decode_json(JSONObj *obj);
 334   static void generate_test_instances(std::list<rgw_bucket_entry_ver*>& o);
 335 };
 336 WRITE_CLASS_ENCODER(rgw_bucket_entry_ver)
 337
 338 struct cls_rgw_obj_key {
 339   std::string name;
 340   std::string instance;
 341
 342   cls_rgw_obj_key() {}
 343   cls_rgw_obj_key(const std::string &_name) : name(_name) {}
 344   cls_rgw_obj_key(const std::string& n, const std::string& i) : name(n), instance(i) {}
 345
 346   std::string to_string() const {
 347     return fmt::format("{}({})", name, instance);
 348   }
 349
 350   bool empty() const {
 351     return name.empty();
 352   }
 353
 354   void set(const std::string& _name) {
 355     name = _name;
 356   }
 357
 358   bool operator==(const cls_rgw_obj_key& k) const {
 359     return (name.compare(k.name) == 0) &&
 360            (instance.compare(k.instance) == 0);
 361   }
 362
 363   bool operator<(const cls_rgw_obj_key& k) const {
 364     int r = name.compare(k.name);
 365     if (r == 0) {
 366       r = instance.compare(k.instance);
 367     }
 368     return (r < 0);
 369   }
 370
 371   bool operator<=(const cls_rgw_obj_key& k) const {
 372     return !(k < *this);
 373   }
 374
 375   std::ostream& operator<<(std::ostream& out) const {
 376     out << to_string();
 377     return out;
 378   }
 379
 380   void encode(ceph::buffer::list &bl) const {
 381     ENCODE_START(1, 1, bl);
 382     encode(name, bl);
 383     encode(instance, bl);
 384     ENCODE_FINISH(bl);
 385   }
 386   void decode(ceph::buffer::list::const_iterator &bl) {
 387     DECODE_START(1, bl);
 388     decode(name, bl);
 389     decode(instance, bl);
 390     DECODE_FINISH(bl);
 391   }
 392   void dump(ceph::Formatter *f) const {
 393     f->dump_string("name", name);
 394     f->dump_string("instance", instance);
 395   }
 396   void decode_json(JSONObj *obj);
 397   static void generate_test_instances(std::list<cls_rgw_obj_key*>& ls) {
 398     ls.push_back(new cls_rgw_obj_key);
 399     ls.push_back(new cls_rgw_obj_key);
 400     ls.back()->name = "name";
 401     ls.back()->instance = "instance";
 402   }
 403 };
 404 WRITE_CLASS_ENCODER(cls_rgw_obj_key)
 405
 406
 407 struct rgw_bucket_dir_entry {
 408   /* a versioned object instance */
 409   static constexpr uint16_t FLAG_VER =                0x1;
 410   /* the last object instance of a versioned object */
 411   static constexpr uint16_t FLAG_CURRENT =            0x2;
 412   /* delete marker */
 413   static constexpr uint16_t FLAG_DELETE_MARKER =      0x4;
 414   /* object is versioned, a placeholder for the plain entry */
 415   static constexpr uint16_t FLAG_VER_MARKER =         0x8;
 416   /* object is a proxy; it is not listed in the bucket index but is a
 417    * prefix ending with a delimiter, perhaps common to multiple
 418    * entries; it is only useful when a delimiter is used and
 419    * represents a "subdirectory" (again, ending in a delimiter) that
 420    * may contain one or more actual entries/objects */
 421   static constexpr uint16_t FLAG_COMMON_PREFIX =   0x8000;
 422
 423   cls_rgw_obj_key key;
 424   rgw_bucket_entry_ver ver;
 425   std::string locator;
 426   bool exists;
 427   rgw_bucket_dir_entry_meta meta;
 428   std::multimap<std::string, rgw_bucket_pending_info> pending_map;
 429   uint64_t index_ver;
 430   std::string tag;
 431   uint16_t flags;
 432   uint64_t versioned_epoch;
 433
 434   rgw_bucket_dir_entry() :
 435     exists(false), index_ver(0), flags(0), versioned_epoch(0) {}
 436
 437   void encode(ceph::buffer::list &bl) const {
 438     ENCODE_START(8, 3, bl);
 439     encode(key.name, bl);
 440     encode(ver.epoch, bl);
 441     encode(exists, bl);
 442     encode(meta, bl);
 443     encode(pending_map, bl);
 444     encode(locator, bl);
 445     encode(ver, bl);
 446     encode_packed_val(index_ver, bl);
 447     encode(tag, bl);
 448     encode(key.instance, bl);
 449     encode(flags, bl);
 450     encode(versioned_epoch, bl);
 451     ENCODE_FINISH(bl);
 452   }
 453   void decode(ceph::buffer::list::const_iterator &bl) {
 454     DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl);
 455     decode(key.name, bl);
 456     decode(ver.epoch, bl);
 457     decode(exists, bl);
 458     decode(meta, bl);
 459     decode(pending_map, bl);
 460     if (struct_v >= 2) {
 461       decode(locator, bl);
 462     }
 463     if (struct_v >= 4) {
 464       decode(ver, bl);
 465     } else {
 466       ver.pool = -1;
 467     }
 468     if (struct_v >= 5) {
 469       decode_packed_val(index_ver, bl);
 470       decode(tag, bl);
 471     }
 472     if (struct_v >= 6) {
 473       decode(key.instance, bl);
 474     }
 475     if (struct_v >= 7) {
 476       decode(flags, bl);
 477     }
 478     if (struct_v >= 8) {
 479       decode(versioned_epoch, bl);
 480     }
 481     DECODE_FINISH(bl);
 482   }
 483
 484   bool is_current() const {
 485     int test_flags =
 486       rgw_bucket_dir_entry::FLAG_VER | rgw_bucket_dir_entry::FLAG_CURRENT;
 487     return (flags & rgw_bucket_dir_entry::FLAG_VER) == 0 ||
 488            (flags & test_flags) == test_flags;
 489   }
 490   bool is_delete_marker() const {
 491     return (flags & rgw_bucket_dir_entry::FLAG_DELETE_MARKER) != 0;
 492   }
 493   bool is_visible() const {
 494     return is_current() && !is_delete_marker();
 495   }
 496   bool is_valid() const {
 497     return (flags & rgw_bucket_dir_entry::FLAG_VER_MARKER) == 0;
 498   }
 499   bool is_common_prefix() const {
 500     return flags & rgw_bucket_dir_entry::FLAG_COMMON_PREFIX;
 501   }
 502
 503   void dump(ceph::Formatter *f) const;
 504   void decode_json(JSONObj *obj);
 505   static void generate_test_instances(std::list<rgw_bucket_dir_entry*>& o);
 506 };
 507 WRITE_CLASS_ENCODER(rgw_bucket_dir_entry)
 508
 509 enum class BIIndexType : uint8_t {
 510   Invalid    = 0,
 511   Plain      = 1,
 512   Instance   = 2,
 513   OLH        = 3,
 514 };
 515
 516 struct rgw_bucket_category_stats;
 517
 518 struct rgw_cls_bi_entry {
 519   BIIndexType type;
 520   std::string idx;
 521   ceph::buffer::list data;
 522
 523   rgw_cls_bi_entry() : type(BIIndexType::Invalid) {}
 524
 525   void encode(ceph::buffer::list& bl) const {
 526     ENCODE_START(1, 1, bl);
 527     encode(type, bl);
 528     encode(idx, bl);
 529     encode(data, bl);
 530     ENCODE_FINISH(bl);
 531   }
 532
 533   void decode(ceph::buffer::list::const_iterator& bl) {
 534     DECODE_START(1, bl);
 535     uint8_t c;
 536     decode(c, bl);
 537     type = (BIIndexType)c;
 538     decode(idx, bl);
 539     decode(data, bl);
 540     DECODE_FINISH(bl);
 541   }
 542
 543   void dump(ceph::Formatter *f) const;
 544   void decode_json(JSONObj *obj, cls_rgw_obj_key *effective_key = NULL);
 545
 546   bool get_info(cls_rgw_obj_key *key, RGWObjCategory *category,
 547                 rgw_bucket_category_stats *accounted_stats);
 548 };
 549 WRITE_CLASS_ENCODER(rgw_cls_bi_entry)
 550
 551 enum OLHLogOp {
 552   CLS_RGW_OLH_OP_UNKNOWN         = 0,
 553   CLS_RGW_OLH_OP_LINK_OLH        = 1,
 554   CLS_RGW_OLH_OP_UNLINK_OLH      = 2, /* object does not exist */
 555   CLS_RGW_OLH_OP_REMOVE_INSTANCE = 3,
 556 };
 557
 558 struct rgw_bucket_olh_log_entry {
 559   uint64_t epoch;
 560   OLHLogOp op;
 561   std::string op_tag;
 562   cls_rgw_obj_key key;
 563   bool delete_marker;
 564
 565   rgw_bucket_olh_log_entry() : epoch(0), op(CLS_RGW_OLH_OP_UNKNOWN), delete_marker(false) {}
 566
 567
 568   void encode(ceph::buffer::list &bl) const {
 569     ENCODE_START(1, 1, bl);
 570     encode(epoch, bl);
 571     encode((__u8)op, bl);
 572     encode(op_tag, bl);
 573     encode(key, bl);
 574     encode(delete_marker, bl);
 575     ENCODE_FINISH(bl);
 576   }
 577   void decode(ceph::buffer::list::const_iterator &bl) {
 578     DECODE_START(1, bl);
 579     decode(epoch, bl);
 580     uint8_t c;
 581     decode(c, bl);
 582     op = (OLHLogOp)c;
 583     decode(op_tag, bl);
 584     decode(key, bl);
 585     decode(delete_marker, bl);
 586     DECODE_FINISH(bl);
 587   }
 588   static void generate_test_instances(std::list<rgw_bucket_olh_log_entry*>& o);
 589   void dump(ceph::Formatter *f) const;
 590   void decode_json(JSONObj *obj);
 591 };
 592 WRITE_CLASS_ENCODER(rgw_bucket_olh_log_entry)
 593
 594 struct rgw_bucket_olh_entry {
 595   cls_rgw_obj_key key;
 596   bool delete_marker;
 597   uint64_t epoch;
 598   std::map<uint64_t, std::vector<struct rgw_bucket_olh_log_entry> > pending_log;
 599   std::string tag;
 600   bool exists;
 601   bool pending_removal;
 602
 603   rgw_bucket_olh_entry() : delete_marker(false), epoch(0), exists(false), pending_removal(false) {}
 604
 605   void encode(ceph::buffer::list &bl) const {
 606     ENCODE_START(1, 1, bl);
 607     encode(key, bl);
 608     encode(delete_marker, bl);
 609     encode(epoch, bl);
 610     encode(pending_log, bl);
 611     encode(tag, bl);
 612     encode(exists, bl);
 613     encode(pending_removal, bl);
 614     ENCODE_FINISH(bl);
 615   }
 616   void decode(ceph::buffer::list::const_iterator &bl) {
 617     DECODE_START(1, bl);
 618     decode(key, bl);
 619     decode(delete_marker, bl);
 620     decode(epoch, bl);
 621     decode(pending_log, bl);
 622     decode(tag, bl);
 623     decode(exists, bl);
 624     decode(pending_removal, bl);
 625     DECODE_FINISH(bl);
 626   }
 627   void dump(ceph::Formatter *f) const;
 628   void decode_json(JSONObj *obj);
 629 };
 630 WRITE_CLASS_ENCODER(rgw_bucket_olh_entry)
 631
 632 struct rgw_bi_log_entry {
 633   std::string id;
 634   std::string object;
 635   std::string instance;
 636   ceph::real_time timestamp;
 637   rgw_bucket_entry_ver ver;
 638   RGWModifyOp op;
 639   RGWPendingState state;
 640   uint64_t index_ver;
 641   std::string tag;
 642   uint16_t bilog_flags;
 643   std::string owner; /* only being set if it's a delete marker */
 644   std::string owner_display_name; /* only being set if it's a delete marker */
 645   rgw_zone_set zones_trace;
 646
 647   rgw_bi_log_entry() : op(CLS_RGW_OP_UNKNOWN), state(CLS_RGW_STATE_PENDING_MODIFY), index_ver(0), bilog_flags(0) {}
 648
 649   void encode(ceph::buffer::list &bl) const {
 650     ENCODE_START(4, 1, bl);
 651     encode(id, bl);
 652     encode(object, bl);
 653     encode(timestamp, bl);
 654     encode(ver, bl);
 655     encode(tag, bl);
 656     uint8_t c = (uint8_t)op;
 657     encode(c, bl);
 658     c = (uint8_t)state;
 659     encode(c, bl);
 660     encode_packed_val(index_ver, bl);
 661     encode(instance, bl);
 662     encode(bilog_flags, bl);
 663     encode(owner, bl);
 664     encode(owner_display_name, bl);
 665     encode(zones_trace, bl);
 666     ENCODE_FINISH(bl);
 667   }
 668   void decode(ceph::buffer::list::const_iterator &bl) {
 669     DECODE_START(4, bl);
 670     decode(id, bl);
 671     decode(object, bl);
 672     decode(timestamp, bl);
 673     decode(ver, bl);
 674     decode(tag, bl);
 675     uint8_t c;
 676     decode(c, bl);
 677     op = (RGWModifyOp)c;
 678     decode(c, bl);
 679     state = (RGWPendingState)c;
 680     decode_packed_val(index_ver, bl);
 681     if (struct_v >= 2) {
 682       decode(instance, bl);
 683       decode(bilog_flags, bl);
 684     }
 685     if (struct_v >= 3) {
 686       decode(owner, bl);
 687       decode(owner_display_name, bl);
 688     }
 689     if (struct_v >= 4) {
 690       decode(zones_trace, bl);
 691     }
 692     DECODE_FINISH(bl);
 693   }
 694   void dump(ceph::Formatter *f) const;
 695   void decode_json(JSONObj *obj);
 696   static void generate_test_instances(std::list<rgw_bi_log_entry*>& o);
 697
 698   bool is_versioned() {
 699     return ((bilog_flags & RGW_BILOG_FLAG_VERSIONED_OP) != 0);
 700   }
 701 };
 702 WRITE_CLASS_ENCODER(rgw_bi_log_entry)
 703
 704 struct rgw_bucket_category_stats {
 705   uint64_t total_size;
 706   uint64_t total_size_rounded;
 707   uint64_t num_entries;
 708   uint64_t actual_size{0}; //< account for compression, encryption
 709
 710   rgw_bucket_category_stats() : total_size(0), total_size_rounded(0), num_entries(0) {}
 711
 712   void encode(ceph::buffer::list &bl) const {
 713     ENCODE_START(3, 2, bl);
 714     encode(total_size, bl);
 715     encode(total_size_rounded, bl);
 716     encode(num_entries, bl);
 717     encode(actual_size, bl);
 718     ENCODE_FINISH(bl);
 719   }
 720   void decode(ceph::buffer::list::const_iterator &bl) {
 721     DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
 722     decode(total_size, bl);
 723     decode(total_size_rounded, bl);
 724     decode(num_entries, bl);
 725     if (struct_v >= 3) {
 726       decode(actual_size, bl);
 727     } else {
 728       actual_size = total_size;
 729     }
 730     DECODE_FINISH(bl);
 731   }
 732   void dump(ceph::Formatter *f) const;
 733   static void generate_test_instances(std::list<rgw_bucket_category_stats*>& o);
 734 };
 735 WRITE_CLASS_ENCODER(rgw_bucket_category_stats)
 736
 737 enum class cls_rgw_reshard_status : uint8_t {
 738   NOT_RESHARDING  = 0,
 739   IN_PROGRESS     = 1,
 740   DONE            = 2
 741 };
 742
 743 inline std::string to_string(const cls_rgw_reshard_status status)
 744 {
 745   switch (status) {
 746   case cls_rgw_reshard_status::NOT_RESHARDING:
 747     return "not-resharding";
 748   case cls_rgw_reshard_status::IN_PROGRESS:
 749     return "in-progress";
 750   case cls_rgw_reshard_status::DONE:
 751     return "done";
 752   };
 753   return "Unknown reshard status";
 754 }
 755
 756 struct cls_rgw_bucket_instance_entry {
 757   using RESHARD_STATUS = cls_rgw_reshard_status;
 758
 759   cls_rgw_reshard_status reshard_status{RESHARD_STATUS::NOT_RESHARDING};
 760   std::string new_bucket_instance_id;
 761   int32_t num_shards{-1};
 762
 763   void encode(ceph::buffer::list& bl) const {
 764     ENCODE_START(1, 1, bl);
 765     encode((uint8_t)reshard_status, bl);
 766     encode(new_bucket_instance_id, bl);
 767     encode(num_shards, bl);
 768     ENCODE_FINISH(bl);
 769   }
 770
 771   void decode(ceph::buffer::list::const_iterator& bl) {
 772     DECODE_START(1, bl);
 773     uint8_t s;
 774     decode(s, bl);
 775     reshard_status = (cls_rgw_reshard_status)s;
 776     decode(new_bucket_instance_id, bl);
 777     decode(num_shards, bl);
 778     DECODE_FINISH(bl);
 779   }
 780
 781   void dump(ceph::Formatter *f) const;
 782   static void generate_test_instances(std::list<cls_rgw_bucket_instance_entry*>& o);
 783
 784   void clear() {
 785     reshard_status = RESHARD_STATUS::NOT_RESHARDING;
 786     new_bucket_instance_id.clear();
 787   }
 788
 789   void set_status(const std::string& new_instance_id,
 790                   int32_t new_num_shards,
 791                   cls_rgw_reshard_status s) {
 792     reshard_status = s;
 793     new_bucket_instance_id = new_instance_id;
 794     num_shards = new_num_shards;
 795   }
 796
 797   bool resharding() const {
 798     return reshard_status != RESHARD_STATUS::NOT_RESHARDING;
 799   }
 800   bool resharding_in_progress() const {
 801     return reshard_status == RESHARD_STATUS::IN_PROGRESS;
 802   }
 803 };
 804 WRITE_CLASS_ENCODER(cls_rgw_bucket_instance_entry)
 805
 806 struct rgw_bucket_dir_header {
 807   std::map<RGWObjCategory, rgw_bucket_category_stats> stats;
 808   uint64_t tag_timeout;
 809   uint64_t ver;
 810   uint64_t master_ver;
 811   std::string max_marker;
 812   cls_rgw_bucket_instance_entry new_instance;
 813   bool syncstopped;
 814
 815   rgw_bucket_dir_header() : tag_timeout(0), ver(0), master_ver(0), syncstopped(false) {}
 816
 817   void encode(ceph::buffer::list &bl) const {
 818     ENCODE_START(7, 2, bl);
 819     encode(stats, bl);
 820     encode(tag_timeout, bl);
 821     encode(ver, bl);
 822     encode(master_ver, bl);
 823     encode(max_marker, bl);
 824     encode(new_instance, bl);
 825     encode(syncstopped,bl);
 826     ENCODE_FINISH(bl);
 827   }
 828   void decode(ceph::buffer::list::const_iterator &bl) {
 829     DECODE_START_LEGACY_COMPAT_LEN(6, 2, 2, bl);
 830     decode(stats, bl);
 831     if (struct_v > 2) {
 832       decode(tag_timeout, bl);
 833     } else {
 834       tag_timeout = 0;
 835     }
 836     if (struct_v >= 4) {
 837       decode(ver, bl);
 838       decode(master_ver, bl);
 839     } else {
 840       ver = 0;
 841     }
 842     if (struct_v >= 5) {
 843       decode(max_marker, bl);
 844     }
 845     if (struct_v >= 6) {
 846       decode(new_instance, bl);
 847     } else {
 848       new_instance = cls_rgw_bucket_instance_entry();
 849     }
 850     if (struct_v >= 7) {
 851       decode(syncstopped,bl);
 852     }
 853     DECODE_FINISH(bl);
 854   }
 855   void dump(ceph::Formatter *f) const;
 856   static void generate_test_instances(std::list<rgw_bucket_dir_header*>& o);
 857
 858   bool resharding() const {
 859     return new_instance.resharding();
 860   }
 861   bool resharding_in_progress() const {
 862     return new_instance.resharding_in_progress();
 863   }
 864 };
 865 WRITE_CLASS_ENCODER(rgw_bucket_dir_header)
 866
 867 struct rgw_bucket_dir {
 868   rgw_bucket_dir_header header;
 869   boost::container::flat_map<std::string, rgw_bucket_dir_entry> m;
 870
 871   void encode(ceph::buffer::list &bl) const {
 872     ENCODE_START(2, 2, bl);
 873     encode(header, bl);
 874     encode(m, bl);
 875     ENCODE_FINISH(bl);
 876   }
 877   void decode(ceph::buffer::list::const_iterator &bl) {
 878     DECODE_START_LEGACY_COMPAT_LEN(2, 2, 2, bl);
 879     decode(header, bl);
 880     decode(m, bl);
 881     DECODE_FINISH(bl);
 882   }
 883   void dump(ceph::Formatter *f) const;
 884   static void generate_test_instances(std::list<rgw_bucket_dir*>& o);
 885 };
 886 WRITE_CLASS_ENCODER(rgw_bucket_dir)
 887
 888 struct rgw_usage_data {
 889   uint64_t bytes_sent;
 890   uint64_t bytes_received;
 891   uint64_t ops;
 892   uint64_t successful_ops;
 893
 894   rgw_usage_data() : bytes_sent(0), bytes_received(0), ops(0), successful_ops(0) {}
 895   rgw_usage_data(uint64_t sent, uint64_t received) : bytes_sent(sent), bytes_received(received), ops(0), successful_ops(0) {}
 896
 897   void encode(ceph::buffer::list& bl) const {
 898     ENCODE_START(1, 1, bl);
 899     encode(bytes_sent, bl);
 900     encode(bytes_received, bl);
 901     encode(ops, bl);
 902     encode(successful_ops, bl);
 903     ENCODE_FINISH(bl);
 904   }
 905
 906   void decode(ceph::buffer::list::const_iterator& bl) {
 907     DECODE_START(1, bl);
 908     decode(bytes_sent, bl);
 909     decode(bytes_received, bl);
 910     decode(ops, bl);
 911     decode(successful_ops, bl);
 912     DECODE_FINISH(bl);
 913   }
 914
 915   void aggregate(const rgw_usage_data& usage) {
 916     bytes_sent += usage.bytes_sent;
 917     bytes_received += usage.bytes_received;
 918     ops += usage.ops;
 919     successful_ops += usage.successful_ops;
 920   }
 921 };
 922 WRITE_CLASS_ENCODER(rgw_usage_data)
 923
 924
 925 struct rgw_usage_log_entry {
 926   rgw_user owner;
 927   rgw_user payer; /* if empty, same as owner */
 928   std::string bucket;
 929   uint64_t epoch;
 930   rgw_usage_data total_usage; /* this one is kept for backwards compatibility */
 931   std::map<std::string, rgw_usage_data> usage_map;
 932
 933   rgw_usage_log_entry() : epoch(0) {}
 934   rgw_usage_log_entry(std::string& o, std::string& b) : owner(o), bucket(b), epoch(0) {}
 935   rgw_usage_log_entry(std::string& o, std::string& p, std::string& b) : owner(o), payer(p), bucket(b), epoch(0) {}
 936
 937   void encode(ceph::buffer::list& bl) const {
 938     ENCODE_START(3, 1, bl);
 939     encode(owner.to_str(), bl);
 940     encode(bucket, bl);
 941     encode(epoch, bl);
 942     encode(total_usage.bytes_sent, bl);
 943     encode(total_usage.bytes_received, bl);
 944     encode(total_usage.ops, bl);
 945     encode(total_usage.successful_ops, bl);
 946     encode(usage_map, bl);
 947     encode(payer.to_str(), bl);
 948     ENCODE_FINISH(bl);
 949   }
 950
 951
 952    void decode(ceph::buffer::list::const_iterator& bl) {
 953     DECODE_START(3, bl);
 954     std::string s;
 955     decode(s, bl);
 956     owner.from_str(s);
 957     decode(bucket, bl);
 958     decode(epoch, bl);
 959     decode(total_usage.bytes_sent, bl);
 960     decode(total_usage.bytes_received, bl);
 961     decode(total_usage.ops, bl);
 962     decode(total_usage.successful_ops, bl);
 963     if (struct_v < 2) {
 964       usage_map[""] = total_usage;
 965     } else {
 966       decode(usage_map, bl);
 967     }
 968     if (struct_v >= 3) {
 969       std::string p;
 970       decode(p, bl);
 971       payer.from_str(p);
 972     }
 973     DECODE_FINISH(bl);
 974   }
 975
 976   void aggregate(const rgw_usage_log_entry& e,
 977                  std::map<std::string, bool> *categories = NULL) {
 978     if (owner.empty()) {
 979       owner = e.owner;
 980       bucket = e.bucket;
 981       epoch = e.epoch;
 982       payer = e.payer;
 983     }
 984
 985     for (auto iter = e.usage_map.begin(); iter != e.usage_map.end(); ++iter) {
 986       if (!categories || !categories->size() || categories->count(iter->first)) {
 987         add(iter->first, iter->second);
 988       }
 989     }
 990   }
 991
 992   void sum(rgw_usage_data& usage,
 993            std::map<std::string, bool>& categories) const {
 994     usage = rgw_usage_data();
 995     for (auto iter = usage_map.begin(); iter != usage_map.end(); ++iter) {
 996       if (!categories.size() || categories.count(iter->first)) {
 997         usage.aggregate(iter->second);
 998       }
 999     }
1000   }
1001
1002   void add(const std::string& category, const rgw_usage_data& data) {
1003     usage_map[category].aggregate(data);
1004     total_usage.aggregate(data);
1005   }
1006
1007   void dump(ceph::Formatter* f) const;
1008   static void generate_test_instances(std::list<rgw_usage_log_entry*>& o);
1009
1010 };
1011 WRITE_CLASS_ENCODER(rgw_usage_log_entry)
1012
1013 struct rgw_usage_log_info {
1014   std::vector<rgw_usage_log_entry> entries;
1015
1016   void encode(ceph::buffer::list& bl) const {
1017     ENCODE_START(1, 1, bl);
1018     encode(entries, bl);
1019     ENCODE_FINISH(bl);
1020   }
1021
1022   void decode(ceph::buffer::list::const_iterator& bl) {
1023     DECODE_START(1, bl);
1024     decode(entries, bl);
1025     DECODE_FINISH(bl);
1026   }
1027
1028   rgw_usage_log_info() {}
1029 };
1030 WRITE_CLASS_ENCODER(rgw_usage_log_info)
1031
1032 struct rgw_user_bucket {
1033   std::string user;
1034   std::string bucket;
1035
1036   rgw_user_bucket() {}
1037   rgw_user_bucket(const std::string& u, const std::string& b) : user(u), bucket(b) {}
1038
1039   void encode(ceph::buffer::list& bl) const {
1040     ENCODE_START(1, 1, bl);
1041     encode(user, bl);
1042     encode(bucket, bl);
1043     ENCODE_FINISH(bl);
1044   }
1045
1046   void decode(ceph::buffer::list::const_iterator& bl) {
1047     DECODE_START(1, bl);
1048     decode(user, bl);
1049     decode(bucket, bl);
1050     DECODE_FINISH(bl);
1051   }
1052
1053   bool operator<(const rgw_user_bucket& ub2) const {
1054     int comp = user.compare(ub2.user);
1055     if (comp < 0)
1056       return true;
1057     else if (!comp)
1058       return bucket.compare(ub2.bucket) < 0;
1059
1060     return false;
1061   }
1062 };
1063 WRITE_CLASS_ENCODER(rgw_user_bucket)
1064
1065 enum cls_rgw_gc_op {
1066   CLS_RGW_GC_DEL_OBJ,
1067   CLS_RGW_GC_DEL_BUCKET,
1068 };
1069
1070 struct cls_rgw_obj {
1071   std::string pool;
1072   cls_rgw_obj_key key;
1073   std::string loc;
1074
1075   cls_rgw_obj() {}
1076   cls_rgw_obj(std::string& _p, cls_rgw_obj_key& _k) : pool(_p), key(_k) {}
1077
1078   void encode(ceph::buffer::list& bl) const {
1079     ENCODE_START(2, 1, bl);
1080     encode(pool, bl);
1081     encode(key.name, bl);
1082     encode(loc, bl);
1083     encode(key, bl);
1084     ENCODE_FINISH(bl);
1085   }
1086
1087   void decode(ceph::buffer::list::const_iterator& bl) {
1088     DECODE_START(2, bl);
1089     decode(pool, bl);
1090     decode(key.name, bl);
1091     decode(loc, bl);
1092     if (struct_v >= 2) {
1093       decode(key, bl);
1094     }
1095     DECODE_FINISH(bl);
1096   }
1097
1098   void dump(ceph::Formatter *f) const {
1099     f->dump_string("pool", pool);
1100     f->dump_string("oid", key.name);
1101     f->dump_string("key", loc);
1102     f->dump_string("instance", key.instance);
1103   }
1104   static void generate_test_instances(std::list<cls_rgw_obj*>& ls) {
1105     ls.push_back(new cls_rgw_obj);
1106     ls.push_back(new cls_rgw_obj);
1107     ls.back()->pool = "mypool";
1108     ls.back()->key.name = "myoid";
1109     ls.back()->loc = "mykey";
1110   }
1111 };
1112 WRITE_CLASS_ENCODER(cls_rgw_obj)
1113
1114 struct cls_rgw_obj_chain {
1115   std::list<cls_rgw_obj> objs;
1116
1117   cls_rgw_obj_chain() {}
1118
1119   void push_obj(const std::string& pool, const cls_rgw_obj_key& key, const std::string& loc) {
1120     cls_rgw_obj obj;
1121     obj.pool = pool;
1122     obj.key = key;
1123     obj.loc = loc;
1124     objs.push_back(obj);
1125   }
1126
1127   void encode(ceph::buffer::list& bl) const {
1128     ENCODE_START(1, 1, bl);
1129     encode(objs, bl);
1130     ENCODE_FINISH(bl);
1131   }
1132
1133   void decode(ceph::buffer::list::const_iterator& bl) {
1134     DECODE_START(1, bl);
1135     decode(objs, bl);
1136     DECODE_FINISH(bl);
1137   }
1138
1139   void dump(ceph::Formatter *f) const {
1140     f->open_array_section("objs");
1141     for (std::list<cls_rgw_obj>::const_iterator p = objs.begin(); p != objs.end(); ++p) {
1142       f->open_object_section("obj");
1143       p->dump(f);
1144       f->close_section();
1145     }
1146     f->close_section();
1147   }
1148   static void generate_test_instances(std::list<cls_rgw_obj_chain*>& ls) {
1149     ls.push_back(new cls_rgw_obj_chain);
1150   }
1151
1152   bool empty() {
1153     return objs.empty();
1154   }
1155 };
1156 WRITE_CLASS_ENCODER(cls_rgw_obj_chain)
1157
1158 struct cls_rgw_gc_obj_info
1159 {
1160   std::string tag;
1161   cls_rgw_obj_chain chain;
1162   ceph::real_time time;
1163
1164   cls_rgw_gc_obj_info() {}
1165
1166   void encode(ceph::buffer::list& bl) const {
1167     ENCODE_START(1, 1, bl);
1168     encode(tag, bl);
1169     encode(chain, bl);
1170     encode(time, bl);
1171     ENCODE_FINISH(bl);
1172   }
1173
1174   void decode(ceph::buffer::list::const_iterator& bl) {
1175     DECODE_START(1, bl);
1176     decode(tag, bl);
1177     decode(chain, bl);
1178     decode(time, bl);
1179     DECODE_FINISH(bl);
1180   }
1181
1182   void dump(ceph::Formatter *f) const {
1183     f->dump_string("tag", tag);
1184     f->open_object_section("chain");
1185     chain.dump(f);
1186     f->close_section();
1187     f->dump_stream("time") << time;
1188   }
1189   static void generate_test_instances(std::list<cls_rgw_gc_obj_info*>& ls) {
1190     ls.push_back(new cls_rgw_gc_obj_info);
1191     ls.push_back(new cls_rgw_gc_obj_info);
1192     ls.back()->tag = "footag";
1193     ceph_timespec ts{init_le32(21), init_le32(32)};
1194     ls.back()->time = ceph::real_clock::from_ceph_timespec(ts);
1195   }
1196 };
1197 WRITE_CLASS_ENCODER(cls_rgw_gc_obj_info)
1198
1199 struct cls_rgw_lc_obj_head
1200 {
1201   time_t start_date = 0;
1202   std::string marker;
1203
1204   cls_rgw_lc_obj_head() {}
1205
1206   void encode(ceph::buffer::list& bl) const {
1207     ENCODE_START(1, 1, bl);
1208     uint64_t t = start_date;
1209     encode(t, bl);
1210     encode(marker, bl);
1211     ENCODE_FINISH(bl);
1212   }
1213
1214   void decode(ceph::buffer::list::const_iterator& bl) {
1215     DECODE_START(1, bl);
1216     uint64_t t;
1217     decode(t, bl);
1218     start_date = static_cast<time_t>(t);
1219     decode(marker, bl);
1220     DECODE_FINISH(bl);
1221   }
1222
1223   void dump(ceph::Formatter *f) const;
1224   static void generate_test_instances(std::list<cls_rgw_lc_obj_head*>& ls);
1225 };
1226 WRITE_CLASS_ENCODER(cls_rgw_lc_obj_head)
1227
1228 struct cls_rgw_lc_entry {
1229   std::string bucket;
1230   uint64_t start_time; // if in_progress
1231   uint32_t status;
1232
1233   cls_rgw_lc_entry()
1234     : start_time(0), status(0) {}
1235
1236   cls_rgw_lc_entry(const cls_rgw_lc_entry& rhs) = default;
1237
1238   cls_rgw_lc_entry(const std::string& b, uint64_t t, uint32_t s)
1239     : bucket(b), start_time(t), status(s) {};
1240
1241   void encode(bufferlist& bl) const {
1242     ENCODE_START(1, 1, bl);
1243     encode(bucket, bl);
1244     encode(start_time, bl);
1245     encode(status, bl);
1246     ENCODE_FINISH(bl);
1247   }
1248
1249   void decode(bufferlist::const_iterator& bl) {
1250     DECODE_START(1, bl);
1251     decode(bucket, bl);
1252     decode(start_time, bl);
1253     decode(status, bl);
1254     DECODE_FINISH(bl);
1255   }
1256 };
1257 WRITE_CLASS_ENCODER(cls_rgw_lc_entry);
1258
1259 struct cls_rgw_reshard_entry
1260 {
1261   ceph::real_time time;
1262   std::string tenant;
1263   std::string bucket_name;
1264   std::string bucket_id;
1265   std::string new_instance_id;
1266   uint32_t old_num_shards{0};
1267   uint32_t new_num_shards{0};
1268
1269   cls_rgw_reshard_entry() {}
1270
1271   void encode(ceph::buffer::list& bl) const {
1272     ENCODE_START(1, 1, bl);
1273     encode(time, bl);
1274     encode(tenant, bl);
1275     encode(bucket_name, bl);
1276     encode(bucket_id, bl);
1277     encode(new_instance_id, bl);
1278     encode(old_num_shards, bl);
1279     encode(new_num_shards, bl);
1280     ENCODE_FINISH(bl);
1281   }
1282
1283   void decode(ceph::buffer::list::const_iterator& bl) {
1284     DECODE_START(1, bl);
1285     decode(time, bl);
1286     decode(tenant, bl);
1287     decode(bucket_name, bl);
1288     decode(bucket_id, bl);
1289     decode(new_instance_id, bl);
1290     decode(old_num_shards, bl);
1291     decode(new_num_shards, bl);
1292     DECODE_FINISH(bl);
1293   }
1294
1295   void dump(ceph::Formatter *f) const;
1296   static void generate_test_instances(std::list<cls_rgw_reshard_entry*>& o);
1297
1298   static void generate_key(const std::string& tenant, const std::string& bucket_name, std::string *key);
1299   void get_key(std::string *key) const;
1300 };
1301 WRITE_CLASS_ENCODER(cls_rgw_reshard_entry)