ceph/src/test/cls_rgw/test_cls_rgw_stats.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <vector>
   5 #include <boost/circular_buffer.hpp>
   6 #include <boost/intrusive/set.hpp>
   7 #include <gtest/gtest.h>
   8 #include "cls/rgw/cls_rgw_client.h"
   9 #include "common/debug.h"
  10 #include "common/dout.h"
  11 #include "common/errno.h"
  12 #include "common/random_string.h"
  13 #include "global/global_context.h"
  14 #include "test/librados/test_cxx.h"
  15
  16 #define dout_subsys ceph_subsys_rgw
  17 #define dout_context g_ceph_context
  18
  19
  20 // simulation parameters:
  21
  22 // total number of index operations to prepare
  23 constexpr size_t max_operations = 2048;
  24 // total number of object names. each operation chooses one at random
  25 constexpr size_t max_entries = 32;
  26 // maximum number of pending operations. once the limit is reached, the oldest
  27 // pending operation is finished before another can start
  28 constexpr size_t max_pending = 16;
  29 // object size is randomly distributed between 0 and 4M
  30 constexpr size_t max_object_size = 4*1024*1024;
  31 // multipart upload threshold
  32 constexpr size_t max_part_size = 1024*1024;
  33
  34
  35 // create/destroy a pool that's shared by all tests in the process
  36 struct RadosEnv : public ::testing::Environment {
  37   static std::optional<std::string> pool_name;
  38  public:
  39   static librados::Rados rados;
  40   static librados::IoCtx ioctx;
  41
  42   void SetUp() override {
  43     // create pool
  44     std::string name = get_temp_pool_name();
  45     ASSERT_EQ("", create_one_pool_pp(name, rados));
  46     pool_name = name;
  47     ASSERT_EQ(rados.ioctx_create(name.c_str(), ioctx), 0);
  48   }
  49   void TearDown() override {
  50     ioctx.close();
  51     if (pool_name) {
  52       ASSERT_EQ(destroy_one_pool_pp(*pool_name, rados), 0);
  53     }
  54   }
  55 };
  56 std::optional<std::string> RadosEnv::pool_name;
  57 librados::Rados RadosEnv::rados;
  58 librados::IoCtx RadosEnv::ioctx;
  59
  60 auto *const rados_env = ::testing::AddGlobalTestEnvironment(new RadosEnv);
  61
  62
  63 std::ostream& operator<<(std::ostream& out, const rgw_bucket_category_stats& c) {
  64   return out << "{count=" << c.num_entries << " size=" << c.total_size << '}';
  65 }
  66
  67
  68 // librados helpers
  69 rgw_bucket_entry_ver last_version(librados::IoCtx& ioctx)
  70 {
  71   rgw_bucket_entry_ver ver;
  72   ver.pool = ioctx.get_id();
  73   ver.epoch = ioctx.get_last_version();
  74   return ver;
  75 }
  76
  77 int index_init(librados::IoCtx& ioctx, const std::string& oid)
  78 {
  79   librados::ObjectWriteOperation op;
  80   cls_rgw_bucket_init_index(op);
  81   return ioctx.operate(oid, &op);
  82 }
  83
  84 int index_prepare(librados::IoCtx& ioctx, const std::string& oid,
  85                   const cls_rgw_obj_key& key, const std::string& tag,
  86                   RGWModifyOp type)
  87 {
  88   librados::ObjectWriteOperation op;
  89   const std::string loc; // empty
  90   constexpr bool log_op = false;
  91   constexpr int flags = 0;
  92   rgw_zone_set zones;
  93   cls_rgw_bucket_prepare_op(op, type, tag, key, loc, log_op, flags, zones);
  94   return ioctx.operate(oid, &op);
  95 }
  96
  97 int index_complete(librados::IoCtx& ioctx, const std::string& oid,
  98                    const cls_rgw_obj_key& key, const std::string& tag,
  99                    RGWModifyOp type, const rgw_bucket_entry_ver& ver,
 100                    const rgw_bucket_dir_entry_meta& meta,
 101                    std::list<cls_rgw_obj_key>* remove_objs)
 102 {
 103   librados::ObjectWriteOperation op;
 104   constexpr bool log_op = false;
 105   constexpr int flags = 0;
 106   constexpr rgw_zone_set* zones = nullptr;
 107   cls_rgw_bucket_complete_op(op, type, tag, ver, key, meta,
 108                              remove_objs, log_op, flags, zones);
 109   return ioctx.operate(oid, &op);
 110 }
 111
 112 void read_stats(librados::IoCtx& ioctx, const std::string& oid,
 113                 rgw_bucket_dir_stats& stats)
 114 {
 115   auto oids = std::map<int, std::string>{{0, oid}};
 116   std::map<int, rgw_cls_list_ret> results;
 117   ASSERT_EQ(0, CLSRGWIssueGetDirHeader(ioctx, oids, results, 8)());
 118   ASSERT_EQ(1, results.size());
 119   stats = std::move(results.begin()->second.dir.header.stats);
 120 }
 121
 122 static void account_entry(rgw_bucket_dir_stats& stats,
 123                           const rgw_bucket_dir_entry_meta& meta)
 124 {
 125   rgw_bucket_category_stats& c = stats[meta.category];
 126   c.num_entries++;
 127   c.total_size += meta.accounted_size;
 128   c.total_size_rounded += cls_rgw_get_rounded_size(meta.accounted_size);
 129   c.actual_size += meta.size;
 130 }
 131
 132 static void unaccount_entry(rgw_bucket_dir_stats& stats,
 133                             const rgw_bucket_dir_entry_meta& meta)
 134 {
 135   rgw_bucket_category_stats& c = stats[meta.category];
 136   c.num_entries--;
 137   c.total_size -= meta.accounted_size;
 138   c.total_size_rounded -= cls_rgw_get_rounded_size(meta.accounted_size);
 139   c.actual_size -= meta.size;
 140 }
 141
 142
 143 // a map of cached dir entries representing the expected state of cls_rgw
 144 struct object : rgw_bucket_dir_entry, boost::intrusive::set_base_hook<> {
 145   explicit object(const cls_rgw_obj_key& key) {
 146     this->key = key;
 147   }
 148 };
 149
 150 struct object_key {
 151   using type = cls_rgw_obj_key;
 152   const type& operator()(const object& o) const { return o.key; }
 153 };
 154
 155 using object_map_base = boost::intrusive::set<object,
 156       boost::intrusive::key_of_value<object_key>>;
 157
 158 struct object_map : object_map_base {
 159   ~object_map() {
 160     clear_and_dispose(std::default_delete<object>{});
 161   }
 162 };
 163
 164
 165 // models a bucket index operation, starting with cls_rgw_bucket_prepare_op().
 166 // stores all of the state necessary to complete the operation, either with
 167 // cls_rgw_bucket_complete_op() or cls_rgw_suggest_changes(). uploads larger
 168 // than max_part_size are modeled as multipart uploads
 169 struct operation {
 170   RGWModifyOp type;
 171   cls_rgw_obj_key key;
 172   std::string tag;
 173   rgw_bucket_entry_ver ver;
 174   std::string upload_id; // empty unless multipart
 175   rgw_bucket_dir_entry_meta meta;
 176 };
 177
 178
 179 class simulator {
 180  public:
 181   simulator(librados::IoCtx& ioctx, std::string oid)
 182       : ioctx(ioctx), oid(std::move(oid)), pending(max_pending)
 183   {
 184     // generate a set of object keys. each operation chooses one at random
 185     keys.reserve(max_entries);
 186     for (size_t i = 0; i < max_entries; i++) {
 187       keys.emplace_back(gen_rand_alphanumeric_upper(g_ceph_context, 12));
 188     }
 189   }
 190
 191   void run();
 192
 193  private:
 194   void start();
 195   int try_start(const cls_rgw_obj_key& key,
 196                 const std::string& tag);
 197
 198   void finish(const operation& op);
 199   void complete(const operation& op, RGWModifyOp type);
 200   void suggest(const operation& op, char suggestion);
 201
 202   int init_multipart(const operation& op);
 203   void complete_multipart(const operation& op);
 204
 205   object_map::iterator find_or_create(const cls_rgw_obj_key& key);
 206
 207   librados::IoCtx& ioctx;
 208   std::string oid;
 209
 210   std::vector<cls_rgw_obj_key> keys;
 211   object_map objects;
 212   boost::circular_buffer<operation> pending;
 213   rgw_bucket_dir_stats stats;
 214 };
 215
 216
 217 void simulator::run()
 218 {
 219   // init the bucket index object
 220   ASSERT_EQ(0, index_init(ioctx, oid));
 221   // run the simulation for N steps
 222   for (size_t i = 0; i < max_operations; i++) {
 223     if (pending.full()) {
 224       // if we're at max_pending, finish the oldest operation
 225       auto& op = pending.front();
 226       finish(op);
 227       pending.pop_front();
 228
 229       // verify bucket stats
 230       rgw_bucket_dir_stats stored_stats;
 231       read_stats(ioctx, oid, stored_stats);
 232
 233       const rgw_bucket_dir_stats& expected_stats = stats;
 234       ASSERT_EQ(expected_stats, stored_stats);
 235     }
 236
 237     // initiate the next operation
 238     start();
 239
 240     // verify bucket stats
 241     rgw_bucket_dir_stats stored_stats;
 242     read_stats(ioctx, oid, stored_stats);
 243
 244     const rgw_bucket_dir_stats& expected_stats = stats;
 245     ASSERT_EQ(expected_stats, stored_stats);
 246   }
 247 }
 248
 249 object_map::iterator simulator::find_or_create(const cls_rgw_obj_key& key)
 250 {
 251   object_map::insert_commit_data commit;
 252   auto result = objects.insert_check(key, std::less<cls_rgw_obj_key>{}, commit);
 253   if (result.second) { // inserting new entry
 254     auto p = new object(key);
 255     result.first = objects.insert_commit(*p, commit);
 256   }
 257   return result.first;
 258 }
 259
 260 int simulator::try_start(const cls_rgw_obj_key& key, const std::string& tag)
 261 {
 262   // choose randomly betwen create and delete
 263   const auto type = static_cast<RGWModifyOp>(
 264       ceph::util::generate_random_number<size_t, size_t>(CLS_RGW_OP_ADD,
 265                                                          CLS_RGW_OP_DEL));
 266   auto op = operation{type, key, tag};
 267
 268   op.meta.category = RGWObjCategory::Main;
 269   op.meta.size = op.meta.accounted_size =
 270       ceph::util::generate_random_number(1, max_object_size);
 271
 272   if (type == CLS_RGW_OP_ADD && op.meta.size > max_part_size) {
 273     // simulate multipart for uploads over the max_part_size threshold
 274     op.upload_id = gen_rand_alphanumeric_upper(g_ceph_context, 12);
 275
 276     int r = init_multipart(op);
 277     if (r != 0) {
 278       derr << "> failed to prepare multipart upload key=" << key
 279           << " upload=" << op.upload_id << " tag=" << tag
 280           << " type=" << type << ": " << cpp_strerror(r) << dendl;
 281       return r;
 282     }
 283
 284     dout(1) << "> prepared multipart upload key=" << key
 285         << " upload=" << op.upload_id << " tag=" << tag
 286         << " type=" << type << " size=" << op.meta.size << dendl;
 287   } else {
 288     // prepare operation
 289     int r = index_prepare(ioctx, oid, op.key, op.tag, op.type);
 290     if (r != 0) {
 291       derr << "> failed to prepare operation key=" << key
 292           << " tag=" << tag << " type=" << type
 293           << ": " << cpp_strerror(r) << dendl;
 294       return r;
 295     }
 296
 297     dout(1) << "> prepared operation key=" << key
 298         << " tag=" << tag << " type=" << type
 299         << " size=" << op.meta.size << dendl;
 300   }
 301   op.ver = last_version(ioctx);
 302
 303   ceph_assert(!pending.full());
 304   pending.push_back(std::move(op));
 305   return 0;
 306 }
 307
 308 void simulator::start()
 309 {
 310   // choose a random object key
 311   const size_t index = ceph::util::generate_random_number(0, keys.size() - 1);
 312   const auto& key = keys[index];
 313   // generate a random tag
 314   const auto tag = gen_rand_alphanumeric_upper(g_ceph_context, 12);
 315
 316   // retry until success. failures don't count towards max_operations
 317   while (try_start(key, tag) != 0)
 318     ;
 319 }
 320
 321 void simulator::finish(const operation& op)
 322 {
 323   if (op.type == CLS_RGW_OP_ADD && !op.upload_id.empty()) {
 324     // multipart uploads either complete or abort based on part uploads
 325     complete_multipart(op);
 326     return;
 327   }
 328
 329   // complete most operations, but finish some with cancel or dir suggest
 330   constexpr int cancel_percent = 10;
 331   constexpr int suggest_update_percent = 10;
 332   constexpr int suggest_remove_percent = 10;
 333
 334   int result = ceph::util::generate_random_number(0, 99);
 335   if (result < cancel_percent) {
 336     complete(op, CLS_RGW_OP_CANCEL);
 337     return;
 338   }
 339   result -= cancel_percent;
 340   if (result < suggest_update_percent) {
 341     suggest(op, CEPH_RGW_UPDATE);
 342     return;
 343   }
 344   result -= suggest_update_percent;
 345   if (result < suggest_remove_percent) {
 346     suggest(op, CEPH_RGW_REMOVE);
 347     return;
 348   }
 349   complete(op, op.type);
 350 }
 351
 352 void simulator::complete(const operation& op, RGWModifyOp type)
 353 {
 354   int r = index_complete(ioctx, oid, op.key, op.tag, type,
 355                          op.ver, op.meta, nullptr);
 356   if (r != 0) {
 357     derr << "< failed to complete operation key=" << op.key
 358         << " tag=" << op.tag << " type=" << op.type
 359         << " size=" << op.meta.size
 360         << ": " << cpp_strerror(r) << dendl;
 361     return;
 362   }
 363
 364   if (type == CLS_RGW_OP_CANCEL) {
 365     dout(1) << "< canceled operation key=" << op.key
 366         << " tag=" << op.tag << " type=" << op.type
 367         << " size=" << op.meta.size << dendl;
 368   } else if (type == CLS_RGW_OP_ADD) {
 369     auto obj = find_or_create(op.key);
 370     if (obj->exists) {
 371       unaccount_entry(stats, obj->meta);
 372     }
 373     obj->exists = true;
 374     obj->meta = op.meta;
 375     account_entry(stats, obj->meta);
 376     dout(1) << "< completed write operation key=" << op.key
 377         << " tag=" << op.tag << " type=" << type
 378         << " size=" << op.meta.size << dendl;
 379   } else {
 380     ceph_assert(type == CLS_RGW_OP_DEL);
 381     auto obj = objects.find(op.key, std::less<cls_rgw_obj_key>{});
 382     if (obj != objects.end()) {
 383       if (obj->exists) {
 384         unaccount_entry(stats, obj->meta);
 385       }
 386       objects.erase_and_dispose(obj, std::default_delete<object>{});
 387     }
 388     dout(1) << "< completed delete operation key=" << op.key
 389         << " tag=" << op.tag << " type=" << type << dendl;
 390   }
 391 }
 392
 393 void simulator::suggest(const operation& op, char suggestion)
 394 {
 395   // read and decode the current dir entry
 396   rgw_cls_bi_entry bi_entry;
 397   int r = cls_rgw_bi_get(ioctx, oid, BIIndexType::Plain, op.key, &bi_entry);
 398   if (r != 0) {
 399     derr << "< no bi entry to suggest for operation key=" << op.key
 400         << " tag=" << op.tag << " type=" << op.type
 401         << " size=" << op.meta.size
 402         << ": " << cpp_strerror(r) << dendl;
 403     return;
 404   }
 405   ASSERT_EQ(bi_entry.type, BIIndexType::Plain);
 406
 407   rgw_bucket_dir_entry entry;
 408   auto p = bi_entry.data.cbegin();
 409   ASSERT_NO_THROW(decode(entry, p));
 410
 411   ASSERT_EQ(entry.key, op.key);
 412
 413   // clear pending info and write it back; this cancels those pending
 414   // operations (we'll see EINVAL when we try to complete them), but dir
 415   // suggest is ignored unless the pending_map is empty
 416   entry.pending_map.clear();
 417
 418   bi_entry.data.clear();
 419   encode(entry, bi_entry.data);
 420
 421   r = cls_rgw_bi_put(ioctx, oid, bi_entry);
 422   ASSERT_EQ(0, r);
 423
 424   // now suggest changes for this entry
 425   entry.ver = last_version(ioctx);
 426   entry.exists = (suggestion == CEPH_RGW_UPDATE);
 427   entry.meta = op.meta;
 428
 429   bufferlist update;
 430   cls_rgw_encode_suggestion(suggestion, entry, update);
 431
 432   librados::ObjectWriteOperation write_op;
 433   cls_rgw_suggest_changes(write_op, update);
 434   r = ioctx.operate(oid, &write_op);
 435   if (r != 0) {
 436     derr << "< failed to suggest operation key=" << op.key
 437         << " tag=" << op.tag << " type=" << op.type
 438         << " size=" << op.meta.size
 439         << ": " << cpp_strerror(r) << dendl;
 440     return;
 441   }
 442
 443   // update our cache accordingly
 444   if (suggestion == CEPH_RGW_UPDATE) {
 445     auto obj = find_or_create(op.key);
 446     if (obj->exists) {
 447       unaccount_entry(stats, obj->meta);
 448     }
 449     obj->exists = true;
 450     obj->meta = op.meta;
 451     account_entry(stats, obj->meta);
 452     dout(1) << "< suggested update operation key=" << op.key
 453         << " tag=" << op.tag << " type=" << op.type
 454         << " size=" << op.meta.size << dendl;
 455   } else {
 456     ceph_assert(suggestion == CEPH_RGW_REMOVE);
 457     auto obj = objects.find(op.key, std::less<cls_rgw_obj_key>{});
 458     if (obj != objects.end()) {
 459       if (obj->exists) {
 460         unaccount_entry(stats, obj->meta);
 461       }
 462       objects.erase_and_dispose(obj, std::default_delete<object>{});
 463     }
 464     dout(1) << "< suggested remove operation key=" << op.key
 465         << " tag=" << op.tag << " type=" << op.type << dendl;
 466   }
 467 }
 468
 469 int simulator::init_multipart(const operation& op)
 470 {
 471   // create (not just prepare) the meta object
 472   const auto meta_key = cls_rgw_obj_key{
 473     fmt::format("_multipart_{}.2~{}.meta", op.key.name, op.upload_id)};
 474   const std::string empty_tag; // empty tag enables complete without prepare
 475   const rgw_bucket_entry_ver empty_ver;
 476   rgw_bucket_dir_entry_meta meta_meta;
 477   meta_meta.category = RGWObjCategory::MultiMeta;
 478   int r = index_complete(ioctx, oid, meta_key, empty_tag, CLS_RGW_OP_ADD,
 479                          empty_ver, meta_meta, nullptr);
 480   if (r != 0) {
 481     derr << "  < failed to create multipart meta key=" << meta_key
 482         << ": " << cpp_strerror(r) << dendl;
 483     return r;
 484   } else {
 485     // account for meta object
 486     auto obj = find_or_create(meta_key);
 487     if (obj->exists) {
 488       unaccount_entry(stats, obj->meta);
 489     }
 490     obj->exists = true;
 491     obj->meta = meta_meta;
 492     account_entry(stats, obj->meta);
 493   }
 494
 495   // prepare part uploads
 496   std::list<cls_rgw_obj_key> remove_objs;
 497   size_t part_id = 0;
 498
 499   size_t remaining = op.meta.size;
 500   while (remaining > max_part_size) {
 501     remaining -= max_part_size;
 502     const auto part_size = std::min(remaining, max_part_size);
 503     const auto part_key = cls_rgw_obj_key{
 504       fmt::format("_multipart_{}.2~{}.{}", op.key.name, op.upload_id, part_id)};
 505     part_id++;
 506
 507     r = index_prepare(ioctx, oid, part_key, op.tag, op.type);
 508     if (r != 0) {
 509       // if part prepare fails, remove the meta object and remove_objs
 510       [[maybe_unused]] int ignored =
 511           index_complete(ioctx, oid, meta_key, empty_tag, CLS_RGW_OP_DEL,
 512                          empty_ver, meta_meta, &remove_objs);
 513       derr << "  > failed to prepare part key=" << part_key
 514           << " size=" << part_size << dendl;
 515       return r; // return the error from prepare
 516     }
 517     dout(1) << "  > prepared part key=" << part_key
 518         << " size=" << part_size << dendl;
 519     remove_objs.push_back(part_key);
 520   }
 521   return 0;
 522 }
 523
 524 void simulator::complete_multipart(const operation& op)
 525 {
 526   const std::string empty_tag; // empty tag enables complete without prepare
 527   const rgw_bucket_entry_ver empty_ver;
 528
 529   // try to finish part uploads
 530   size_t part_id = 0;
 531   std::list<cls_rgw_obj_key> remove_objs;
 532
 533   RGWModifyOp type = op.type; // OP_ADD, or OP_CANCEL for abort
 534
 535   size_t remaining = op.meta.size;
 536   while (remaining > max_part_size) {
 537     remaining -= max_part_size;
 538     const auto part_size = std::min(remaining, max_part_size);
 539     const auto part_key = cls_rgw_obj_key{
 540       fmt::format("_multipart_{}.2~{}.{}", op.key.name, op.upload_id, part_id)};
 541     part_id++;
 542
 543     // cancel 10% of part uploads (and abort the multipart upload)
 544     constexpr int cancel_percent = 10;
 545     const int result = ceph::util::generate_random_number(0, 99);
 546     if (result < cancel_percent) {
 547       type = CLS_RGW_OP_CANCEL; // abort multipart
 548       dout(1) << "  < canceled part key=" << part_key
 549           << " size=" << part_size << dendl;
 550     } else {
 551       rgw_bucket_dir_entry_meta meta;
 552       meta.category = op.meta.category;
 553       meta.size = meta.accounted_size = part_size;
 554
 555       int r = index_complete(ioctx, oid, part_key, op.tag, op.type,
 556                              empty_ver, meta, nullptr);
 557       if (r != 0) {
 558         derr << "  < failed to complete part key=" << part_key
 559             << " size=" << meta.size << ": " << cpp_strerror(r) << dendl;
 560         type = CLS_RGW_OP_CANCEL; // abort multipart
 561       } else {
 562         dout(1) << "  < completed part key=" << part_key
 563             << " size=" << meta.size << dendl;
 564         // account for successful part upload
 565         auto obj = find_or_create(part_key);
 566         if (obj->exists) {
 567           unaccount_entry(stats, obj->meta);
 568         }
 569         obj->exists = true;
 570         obj->meta = meta;
 571         account_entry(stats, obj->meta);
 572       }
 573     }
 574     remove_objs.push_back(part_key);
 575   }
 576
 577   // delete the multipart meta object
 578   const auto meta_key = cls_rgw_obj_key{
 579     fmt::format("_multipart_{}.2~{}.meta", op.key.name, op.upload_id)};
 580   rgw_bucket_dir_entry_meta meta_meta;
 581   meta_meta.category = RGWObjCategory::MultiMeta;
 582
 583   int r = index_complete(ioctx, oid, meta_key, empty_tag, CLS_RGW_OP_DEL,
 584                          empty_ver, meta_meta, nullptr);
 585   if (r != 0) {
 586     derr << "  < failed to remove multipart meta key=" << meta_key
 587         << ": " << cpp_strerror(r) << dendl;
 588   } else {
 589     // unaccount for meta object
 590     auto obj = objects.find(meta_key, std::less<cls_rgw_obj_key>{});
 591     if (obj != objects.end()) {
 592       if (obj->exists) {
 593         unaccount_entry(stats, obj->meta);
 594       }
 595       objects.erase_and_dispose(obj, std::default_delete<object>{});
 596     }
 597   }
 598
 599   // create or cancel the head object
 600   r = index_complete(ioctx, oid, op.key, empty_tag, type,
 601                      empty_ver, op.meta, &remove_objs);
 602   if (r != 0) {
 603     derr << "< failed to complete multipart upload key=" << op.key
 604         << " upload=" << op.upload_id << " tag=" << op.tag
 605         << " type=" << type << " size=" << op.meta.size
 606         << ": " << cpp_strerror(r) << dendl;
 607     return;
 608   }
 609
 610   if (type == CLS_RGW_OP_ADD) {
 611     dout(1) << "< completed multipart upload key=" << op.key
 612         << " upload=" << op.upload_id << " tag=" << op.tag
 613         << " type=" << op.type << " size=" << op.meta.size << dendl;
 614
 615     // account for head stats
 616     auto obj = find_or_create(op.key);
 617     if (obj->exists) {
 618       unaccount_entry(stats, obj->meta);
 619     }
 620     obj->exists = true;
 621     obj->meta = op.meta;
 622     account_entry(stats, obj->meta);
 623   } else {
 624     dout(1) << "< canceled multipart upload key=" << op.key
 625         << " upload=" << op.upload_id << " tag=" << op.tag
 626         << " type=" << op.type << " size=" << op.meta.size << dendl;
 627   }
 628
 629   // unaccount for remove_objs
 630   for (const auto& part_key : remove_objs) {
 631     auto obj = objects.find(part_key, std::less<cls_rgw_obj_key>{});
 632     if (obj != objects.end()) {
 633       if (obj->exists) {
 634         unaccount_entry(stats, obj->meta);
 635       }
 636       objects.erase_and_dispose(obj, std::default_delete<object>{});
 637     }
 638   }
 639 }
 640
 641 TEST(cls_rgw_stats, simulate)
 642 {
 643   const char* bucket_oid = __func__;
 644   auto sim = simulator{RadosEnv::ioctx, bucket_oid};
 645   sim.run();
 646 }