ceph/src/test/cls_rgw/test_cls_rgw.cc

   1 // -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/types.h"
   5 #include "cls/rgw/cls_rgw_client.h"
   6 #include "cls/rgw/cls_rgw_ops.h"
   7
   8 #include "gtest/gtest.h"
   9 #include "test/librados/test_cxx.h"
  10 #include "global/global_context.h"
  11
  12 #include <errno.h>
  13 #include <string>
  14 #include <vector>
  15 #include <map>
  16 #include <set>
  17
  18 using namespace librados;
  19
  20 librados::Rados rados;
  21 librados::IoCtx ioctx;
  22 string pool_name;
  23
  24
  25 /* must be the first test! */
  26 TEST(cls_rgw, init)
  27 {
  28   pool_name = get_temp_pool_name();
  29   /* create pool */
  30   ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
  31   ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
  32 }
  33
  34
  35 string str_int(string s, int i)
  36 {
  37   char buf[32];
  38   snprintf(buf, sizeof(buf), "-%d", i);
  39   s.append(buf);
  40
  41   return s;
  42 }
  43
  44
  45 class OpMgr {
  46   vector<ObjectOperation *> ops;
  47
  48 public:
  49   OpMgr() {}
  50   ~OpMgr() {
  51     vector<ObjectOperation *>::iterator iter;
  52     for (iter = ops.begin(); iter != ops.end(); ++iter) {
  53       ObjectOperation *op = *iter;
  54       delete op;
  55     }
  56   }
  57
  58   ObjectReadOperation *read_op() {
  59     ObjectReadOperation *op = new ObjectReadOperation;
  60     ops.push_back(op);
  61     return op;
  62   }
  63
  64   ObjectWriteOperation *write_op() {
  65     ObjectWriteOperation *op = new ObjectWriteOperation;
  66     ops.push_back(op);
  67     return op;
  68   }
  69 };
  70
  71 void test_stats(librados::IoCtx& ioctx, string& oid, RGWObjCategory category, uint64_t num_entries, uint64_t total_size)
  72 {
  73   map<int, struct rgw_cls_list_ret> results;
  74   map<int, string> oids;
  75   oids[0] = oid;
  76   ASSERT_EQ(0, CLSRGWIssueGetDirHeader(ioctx, oids, results, 8)());
  77
  78   uint64_t entries = 0;
  79   uint64_t size = 0;
  80   map<int, struct rgw_cls_list_ret>::iterator iter = results.begin();
  81   for (; iter != results.end(); ++iter) {
  82     entries += (iter->second).dir.header.stats[category].num_entries;
  83     size += (iter->second).dir.header.stats[category].total_size;
  84   }
  85   ASSERT_EQ(total_size, size);
  86   ASSERT_EQ(num_entries, entries);
  87 }
  88
  89 void index_prepare(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag,
  90                    string& obj, string& loc, uint16_t bi_flags = 0, bool log_op = true)
  91 {
  92   ObjectWriteOperation *op = mgr.write_op();
  93   cls_rgw_obj_key key(obj, string());
  94   rgw_zone_set zones_trace;
  95   cls_rgw_bucket_prepare_op(*op, index_op, tag, key, loc, log_op, bi_flags, zones_trace);
  96   ASSERT_EQ(0, ioctx.operate(oid, op));
  97 }
  98
  99 void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag,
 100                     int epoch, string& obj, rgw_bucket_dir_entry_meta& meta, uint16_t bi_flags = 0, bool log_op = true)
 101 {
 102   ObjectWriteOperation *op = mgr.write_op();
 103   cls_rgw_obj_key key(obj, string());
 104   rgw_bucket_entry_ver ver;
 105   ver.pool = ioctx.get_id();
 106   ver.epoch = epoch;
 107   meta.accounted_size = meta.size;
 108   cls_rgw_bucket_complete_op(*op, index_op, tag, ver, key, meta, nullptr, log_op, bi_flags, nullptr);
 109   ASSERT_EQ(0, ioctx.operate(oid, op));
 110 }
 111
 112 TEST(cls_rgw, index_basic)
 113 {
 114   string bucket_oid = str_int("bucket", 0);
 115
 116   OpMgr mgr;
 117
 118   ObjectWriteOperation *op = mgr.write_op();
 119   cls_rgw_bucket_init_index(*op);
 120   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 121
 122   uint64_t epoch = 1;
 123
 124   uint64_t obj_size = 1024;
 125
 126 #define NUM_OBJS 10
 127   for (int i = 0; i < NUM_OBJS; i++) {
 128     string obj = str_int("obj", i);
 129     string tag = str_int("tag", i);
 130     string loc = str_int("loc", i);
 131
 132     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
 133
 134     test_stats(ioctx, bucket_oid, RGWObjCategory::None, i, obj_size * i);
 135
 136     op = mgr.write_op();
 137     rgw_bucket_dir_entry_meta meta;
 138     meta.category = RGWObjCategory::None;
 139     meta.size = obj_size;
 140     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta);
 141   }
 142
 143   test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS,
 144              obj_size * NUM_OBJS);
 145 }
 146
 147 TEST(cls_rgw, index_multiple_obj_writers)
 148 {
 149   string bucket_oid = str_int("bucket", 1);
 150
 151   OpMgr mgr;
 152
 153   ObjectWriteOperation *op = mgr.write_op();
 154   cls_rgw_bucket_init_index(*op);
 155   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 156
 157   uint64_t obj_size = 1024;
 158
 159   string obj = str_int("obj", 0);
 160   string loc = str_int("loc", 0);
 161   /* multi prepare on a single object */
 162   for (int i = 0; i < NUM_OBJS; i++) {
 163     string tag = str_int("tag", i);
 164
 165     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
 166
 167     test_stats(ioctx, bucket_oid, RGWObjCategory::None, 0, 0);
 168   }
 169
 170   for (int i = NUM_OBJS; i > 0; i--) {
 171     string tag = str_int("tag", i - 1);
 172
 173     rgw_bucket_dir_entry_meta meta;
 174     meta.category = RGWObjCategory::None;
 175     meta.size = obj_size * i;
 176
 177     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, i, obj, meta);
 178
 179     /* verify that object size doesn't change, as we went back with epoch */
 180     test_stats(ioctx, bucket_oid, RGWObjCategory::None, 1,
 181                obj_size * NUM_OBJS);
 182   }
 183 }
 184
 185 TEST(cls_rgw, index_remove_object)
 186 {
 187   string bucket_oid = str_int("bucket", 2);
 188
 189   OpMgr mgr;
 190
 191   ObjectWriteOperation *op = mgr.write_op();
 192   cls_rgw_bucket_init_index(*op);
 193   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 194
 195   uint64_t obj_size = 1024;
 196   uint64_t total_size = 0;
 197
 198   int epoch = 0;
 199
 200   /* prepare multiple objects */
 201   for (int i = 0; i < NUM_OBJS; i++) {
 202     string obj = str_int("obj", i);
 203     string tag = str_int("tag", i);
 204     string loc = str_int("loc", i);
 205
 206     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
 207
 208     test_stats(ioctx, bucket_oid, RGWObjCategory::None, i, total_size);
 209
 210     rgw_bucket_dir_entry_meta meta;
 211     meta.category = RGWObjCategory::None;
 212     meta.size = i * obj_size;
 213     total_size += i * obj_size;
 214
 215     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
 216
 217     test_stats(ioctx, bucket_oid, RGWObjCategory::None, i + 1, total_size);
 218   }
 219
 220   int i = NUM_OBJS / 2;
 221   string tag_remove = "tag-rm";
 222   string tag_modify = "tag-mod";
 223   string obj = str_int("obj", i);
 224   string loc = str_int("loc", i);
 225
 226   /* prepare both removal and modification on the same object */
 227   index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc);
 228   index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, obj, loc);
 229
 230   test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS, total_size);
 231
 232   rgw_bucket_dir_entry_meta meta;
 233
 234   /* complete object removal */
 235   index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta);
 236
 237   /* verify stats correct */
 238   total_size -= i * obj_size;
 239   test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS - 1, total_size);
 240
 241   meta.size = 512;
 242   meta.category = RGWObjCategory::None;
 243
 244   /* complete object modification */
 245   index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta);
 246
 247   /* verify stats correct */
 248   total_size += meta.size;
 249   test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS, total_size);
 250
 251
 252   /* prepare both removal and modification on the same object, this time we'll
 253    * first complete modification then remove*/
 254   index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc);
 255   index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_modify, obj, loc);
 256
 257   /* complete modification */
 258   total_size -= meta.size;
 259   meta.size = i * obj_size * 2;
 260   meta.category = RGWObjCategory::None;
 261
 262   /* complete object modification */
 263   index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta);
 264
 265   /* verify stats correct */
 266   total_size += meta.size;
 267   test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS, total_size);
 268
 269   /* complete object removal */
 270   index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta);
 271
 272   /* verify stats correct */
 273   total_size -= meta.size;
 274   test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS - 1,
 275              total_size);
 276 }
 277
 278 TEST(cls_rgw, index_suggest)
 279 {
 280   string bucket_oid = str_int("bucket", 3);
 281
 282   OpMgr mgr;
 283
 284   ObjectWriteOperation *op = mgr.write_op();
 285   cls_rgw_bucket_init_index(*op);
 286   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 287
 288   uint64_t total_size = 0;
 289
 290   int epoch = 0;
 291
 292   int num_objs = 100;
 293
 294   uint64_t obj_size = 1024;
 295
 296   /* create multiple objects */
 297   for (int i = 0; i < num_objs; i++) {
 298     string obj = str_int("obj", i);
 299     string tag = str_int("tag", i);
 300     string loc = str_int("loc", i);
 301
 302     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
 303
 304     test_stats(ioctx, bucket_oid, RGWObjCategory::None, i, total_size);
 305
 306     rgw_bucket_dir_entry_meta meta;
 307     meta.category = RGWObjCategory::None;
 308     meta.size = obj_size;
 309     total_size += meta.size;
 310
 311     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
 312
 313     test_stats(ioctx, bucket_oid, RGWObjCategory::None, i + 1, total_size);
 314   }
 315
 316   /* prepare (without completion) some of the objects */
 317   for (int i = 0; i < num_objs; i += 2) {
 318     string obj = str_int("obj", i);
 319     string tag = str_int("tag-prepare", i);
 320     string loc = str_int("loc", i);
 321
 322     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
 323
 324     test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs, total_size);
 325   }
 326
 327   int actual_num_objs = num_objs;
 328   /* remove half of the objects */
 329   for (int i = num_objs / 2; i < num_objs; i++) {
 330     string obj = str_int("obj", i);
 331     string tag = str_int("tag-rm", i);
 332     string loc = str_int("loc", i);
 333
 334     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
 335
 336     test_stats(ioctx, bucket_oid, RGWObjCategory::None, actual_num_objs, total_size);
 337
 338     rgw_bucket_dir_entry_meta meta;
 339     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_DEL, tag, ++epoch, obj, meta);
 340
 341     total_size -= obj_size;
 342     actual_num_objs--;
 343     test_stats(ioctx, bucket_oid, RGWObjCategory::None, actual_num_objs, total_size);
 344   }
 345
 346   bufferlist updates;
 347
 348   for (int i = 0; i < num_objs; i += 2) {
 349     string obj = str_int("obj", i);
 350     string tag = str_int("tag-rm", i);
 351     string loc = str_int("loc", i);
 352
 353     rgw_bucket_dir_entry dirent;
 354     dirent.key.name = obj;
 355     dirent.locator = loc;
 356     dirent.exists = (i < num_objs / 2); // we removed half the objects
 357     dirent.meta.size = 1024;
 358     dirent.meta.accounted_size = 1024;
 359
 360     char suggest_op = (i < num_objs / 2 ? CEPH_RGW_UPDATE : CEPH_RGW_REMOVE);
 361     cls_rgw_encode_suggestion(suggest_op, dirent, updates);
 362   }
 363
 364   map<int, string> bucket_objs;
 365   bucket_objs[0] = bucket_oid;
 366   int r = CLSRGWIssueSetTagTimeout(ioctx, bucket_objs, 8 /* max aio */, 1)();
 367   ASSERT_EQ(0, r);
 368
 369   sleep(1);
 370
 371   /* suggest changes! */
 372   op = mgr.write_op();
 373   cls_rgw_suggest_changes(*op, updates);
 374   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 375
 376   /* suggest changes twice! */
 377   op = mgr.write_op();
 378   cls_rgw_suggest_changes(*op, updates);
 379   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 380
 381   test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs / 2, total_size);
 382 }
 383
 384 /*
 385  * This case is used to test whether get_obj_vals will
 386  * return all validate utf8 objnames and filter out those
 387  * in BI_PREFIX_CHAR private namespace.
 388  */
 389 TEST(cls_rgw, index_list)
 390 {
 391   string bucket_oid = str_int("bucket", 4);
 392
 393   OpMgr mgr;
 394
 395   ObjectWriteOperation *op = mgr.write_op();
 396   cls_rgw_bucket_init_index(*op);
 397   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 398
 399   uint64_t epoch = 1;
 400   uint64_t obj_size = 1024;
 401   const int num_objs = 4;
 402   const string keys[num_objs] = {
 403     /* single byte utf8 character */
 404     { static_cast<char>(0x41) },
 405     /* double byte utf8 character */
 406     { static_cast<char>(0xCF), static_cast<char>(0x8F) },
 407     /* treble byte utf8 character */
 408     { static_cast<char>(0xDF), static_cast<char>(0x8F), static_cast<char>(0x8F) },
 409     /* quadruble byte utf8 character */
 410     { static_cast<char>(0xF7), static_cast<char>(0x8F), static_cast<char>(0x8F), static_cast<char>(0x8F) },
 411   };
 412
 413   for (int i = 0; i < num_objs; i++) {
 414     string obj = keys[i];
 415     string tag = str_int("tag", i);
 416     string loc = str_int("loc", i);
 417
 418     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc, 0 /* bi_falgs */, false /* log_op */);
 419
 420     op = mgr.write_op();
 421     rgw_bucket_dir_entry_meta meta;
 422     meta.category = RGWObjCategory::None;
 423     meta.size = obj_size;
 424     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta, 0 /* bi_falgs */, false /* log_op */);
 425   }
 426
 427   map<string, bufferlist> entries;
 428   /* insert 998 omap key starts with BI_PREFIX_CHAR,
 429    * so bucket list first time will get one key before 0x80 and one key after */
 430   for (int i = 0; i < 998; ++i) {
 431     char buf[10];
 432     snprintf(buf, sizeof(buf), "%c%s%d", 0x80, "1000_", i);
 433     entries.emplace(string{buf}, bufferlist{});
 434   }
 435   ioctx.omap_set(bucket_oid, entries);
 436
 437   test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs, obj_size * num_objs);
 438
 439   map<int, string> oids = { {0, bucket_oid} };
 440   map<int, struct rgw_cls_list_ret> list_results;
 441   cls_rgw_obj_key start_key("", "");
 442   int r = CLSRGWIssueBucketList(ioctx, start_key, "", 1000, true, oids, list_results, 1)();
 443
 444   ASSERT_EQ(r, 0);
 445   ASSERT_EQ(1u, list_results.size());
 446
 447   auto it = list_results.begin();
 448   auto m = (it->second).dir.m;
 449
 450   ASSERT_EQ(4u, m.size());
 451   int i = 0;
 452   for(auto it2 = m.begin(); it2 != m.end(); it2++, i++)
 453     ASSERT_EQ(it2->first.compare(keys[i]), 0);
 454 }
 455
 456
 457 TEST(cls_rgw, bi_list)
 458 {
 459   string bucket_oid = str_int("bucket", 5);
 460
 461  CephContext *cct = reinterpret_cast<CephContext *>(ioctx.cct());
 462
 463   OpMgr mgr;
 464
 465   ObjectWriteOperation *op = mgr.write_op();
 466   cls_rgw_bucket_init_index(*op);
 467   ASSERT_EQ(0, ioctx.operate(bucket_oid, op));
 468
 469   string name;
 470   string marker;
 471   uint64_t max = 10;
 472   list<rgw_cls_bi_entry> entries;
 473   bool is_truncated;
 474
 475   int ret = cls_rgw_bi_list(ioctx, bucket_oid, name, marker, max, &entries,
 476                             &is_truncated);
 477   ASSERT_EQ(ret, 0);
 478   ASSERT_EQ(entries.size(), 0u);
 479   ASSERT_EQ(is_truncated, false);
 480
 481   uint64_t epoch = 1;
 482   uint64_t obj_size = 1024;
 483   uint64_t num_objs = 35;
 484
 485   for (uint64_t i = 0; i < num_objs; i++) {
 486     string obj = str_int("obj", i);
 487     string tag = str_int("tag", i);
 488     string loc = str_int("loc", i);
 489     index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc, RGW_BILOG_FLAG_VERSIONED_OP);
 490     op = mgr.write_op();
 491     rgw_bucket_dir_entry_meta meta;
 492     meta.category = RGWObjCategory::None;
 493     meta.size = obj_size;
 494     index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta, RGW_BILOG_FLAG_VERSIONED_OP);
 495   }
 496
 497   ret = cls_rgw_bi_list(ioctx, bucket_oid, name, marker, num_objs + 10, &entries,
 498                             &is_truncated);
 499   ASSERT_EQ(ret, 0);
 500   if (cct->_conf->osd_max_omap_entries_per_request < num_objs) {
 501     ASSERT_EQ(entries.size(), cct->_conf->osd_max_omap_entries_per_request);
 502   } else {
 503     ASSERT_EQ(entries.size(), num_objs);
 504   }
 505
 506   uint64_t num_entries = 0;
 507
 508   is_truncated = true;
 509   while(is_truncated) {
 510     ret = cls_rgw_bi_list(ioctx, bucket_oid, name, marker, max, &entries,
 511                           &is_truncated);
 512     ASSERT_EQ(ret, 0);
 513     if (is_truncated) {
 514       ASSERT_EQ(entries.size(), std::min(max, cct->_conf->osd_max_omap_entries_per_request));
 515     } else {
 516       ASSERT_EQ(entries.size(), num_objs - num_entries);
 517     }
 518     num_entries += entries.size();
 519     marker = entries.back().idx;
 520   }
 521
 522   ret = cls_rgw_bi_list(ioctx, bucket_oid, name, marker, max, &entries,
 523                         &is_truncated);
 524   ASSERT_EQ(ret, 0);
 525   ASSERT_EQ(entries.size(), 0u);
 526   ASSERT_EQ(is_truncated, false);
 527
 528   if (cct->_conf->osd_max_omap_entries_per_request < 15) {
 529     num_entries = 0;
 530     max = 15;
 531     is_truncated = true;
 532     marker.clear();
 533     while(is_truncated) {
 534       ret = cls_rgw_bi_list(ioctx, bucket_oid, name, marker, max, &entries,
 535                             &is_truncated);
 536       ASSERT_EQ(ret, 0);
 537       if (is_truncated) {
 538         ASSERT_EQ(entries.size(), cct->_conf->osd_max_omap_entries_per_request);
 539       } else {
 540         ASSERT_EQ(entries.size(), num_objs - num_entries);
 541       }
 542       num_entries += entries.size();
 543       marker = entries.back().idx;
 544     }
 545   }
 546
 547   ret = cls_rgw_bi_list(ioctx, bucket_oid, name, marker, max, &entries,
 548                         &is_truncated);
 549   ASSERT_EQ(ret, 0);
 550   ASSERT_EQ(entries.size(), 0u);
 551   ASSERT_EQ(is_truncated, false);
 552 }
 553
 554 /* test garbage collection */
 555 static void create_obj(cls_rgw_obj& obj, int i, int j)
 556 {
 557   char buf[32];
 558   snprintf(buf, sizeof(buf), "-%d.%d", i, j);
 559   obj.pool = "pool";
 560   obj.pool.append(buf);
 561   obj.key.name = "oid";
 562   obj.key.name.append(buf);
 563   obj.loc = "loc";
 564   obj.loc.append(buf);
 565 }
 566
 567 static bool cmp_objs(cls_rgw_obj& obj1, cls_rgw_obj& obj2)
 568 {
 569   return (obj1.pool == obj2.pool) &&
 570          (obj1.key == obj2.key) &&
 571          (obj1.loc == obj2.loc);
 572 }
 573
 574
 575 TEST(cls_rgw, gc_set)
 576 {
 577   /* add chains */
 578   string oid = "obj";
 579   for (int i = 0; i < 10; i++) {
 580     char buf[32];
 581     snprintf(buf, sizeof(buf), "chain-%d", i);
 582     string tag = buf;
 583     librados::ObjectWriteOperation op;
 584     cls_rgw_gc_obj_info info;
 585
 586     cls_rgw_obj obj1, obj2;
 587     create_obj(obj1, i, 1);
 588     create_obj(obj2, i, 2);
 589     info.chain.objs.push_back(obj1);
 590     info.chain.objs.push_back(obj2);
 591
 592     op.create(false); // create object
 593
 594     info.tag = tag;
 595     cls_rgw_gc_set_entry(op, 0, info);
 596
 597     ASSERT_EQ(0, ioctx.operate(oid, &op));
 598   }
 599
 600   bool truncated;
 601   list<cls_rgw_gc_obj_info> entries;
 602   string marker;
 603   string next_marker;
 604
 605   /* list chains, verify truncated */
 606   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated, next_marker));
 607   ASSERT_EQ(8, (int)entries.size());
 608   ASSERT_EQ(1, truncated);
 609
 610   entries.clear();
 611   next_marker.clear();
 612
 613   /* list all chains, verify not truncated */
 614   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, true, entries, &truncated, next_marker));
 615   ASSERT_EQ(10, (int)entries.size());
 616   ASSERT_EQ(0, truncated);
 617
 618   /* verify all chains are valid */
 619   list<cls_rgw_gc_obj_info>::iterator iter = entries.begin();
 620   for (int i = 0; i < 10; i++, ++iter) {
 621     cls_rgw_gc_obj_info& entry = *iter;
 622
 623     /* create expected chain name */
 624     char buf[32];
 625     snprintf(buf, sizeof(buf), "chain-%d", i);
 626     string tag = buf;
 627
 628     /* verify chain name as expected */
 629     ASSERT_EQ(entry.tag, tag);
 630
 631     /* verify expected num of objects in chain */
 632     ASSERT_EQ(2, (int)entry.chain.objs.size());
 633
 634     list<cls_rgw_obj>::iterator oiter = entry.chain.objs.begin();
 635     cls_rgw_obj obj1, obj2;
 636
 637     /* create expected objects */
 638     create_obj(obj1, i, 1);
 639     create_obj(obj2, i, 2);
 640
 641     /* assign returned object names */
 642     cls_rgw_obj& ret_obj1 = *oiter++;
 643     cls_rgw_obj& ret_obj2 = *oiter;
 644
 645     /* verify objects are as expected */
 646     ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1));
 647     ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2));
 648   }
 649 }
 650
 651 TEST(cls_rgw, gc_list)
 652 {
 653   /* add chains */
 654   string oid = "obj";
 655   for (int i = 0; i < 10; i++) {
 656     char buf[32];
 657     snprintf(buf, sizeof(buf), "chain-%d", i);
 658     string tag = buf;
 659     librados::ObjectWriteOperation op;
 660     cls_rgw_gc_obj_info info;
 661
 662     cls_rgw_obj obj1, obj2;
 663     create_obj(obj1, i, 1);
 664     create_obj(obj2, i, 2);
 665     info.chain.objs.push_back(obj1);
 666     info.chain.objs.push_back(obj2);
 667
 668     op.create(false); // create object
 669
 670     info.tag = tag;
 671     cls_rgw_gc_set_entry(op, 0, info);
 672
 673     ASSERT_EQ(0, ioctx.operate(oid, &op));
 674   }
 675
 676   bool truncated;
 677   list<cls_rgw_gc_obj_info> entries;
 678   list<cls_rgw_gc_obj_info> entries2;
 679   string marker;
 680   string next_marker;
 681
 682   /* list chains, verify truncated */
 683   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated, next_marker));
 684   ASSERT_EQ(8, (int)entries.size());
 685   ASSERT_EQ(1, truncated);
 686
 687   marker = next_marker;
 688   next_marker.clear();
 689
 690   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries2, &truncated, next_marker));
 691   ASSERT_EQ(2, (int)entries2.size());
 692   ASSERT_EQ(0, truncated);
 693
 694   entries.splice(entries.end(), entries2);
 695
 696   /* verify all chains are valid */
 697   list<cls_rgw_gc_obj_info>::iterator iter = entries.begin();
 698   for (int i = 0; i < 10; i++, ++iter) {
 699     cls_rgw_gc_obj_info& entry = *iter;
 700
 701     /* create expected chain name */
 702     char buf[32];
 703     snprintf(buf, sizeof(buf), "chain-%d", i);
 704     string tag = buf;
 705
 706     /* verify chain name as expected */
 707     ASSERT_EQ(entry.tag, tag);
 708
 709     /* verify expected num of objects in chain */
 710     ASSERT_EQ(2, (int)entry.chain.objs.size());
 711
 712     list<cls_rgw_obj>::iterator oiter = entry.chain.objs.begin();
 713     cls_rgw_obj obj1, obj2;
 714
 715     /* create expected objects */
 716     create_obj(obj1, i, 1);
 717     create_obj(obj2, i, 2);
 718
 719     /* assign returned object names */
 720     cls_rgw_obj& ret_obj1 = *oiter++;
 721     cls_rgw_obj& ret_obj2 = *oiter;
 722
 723     /* verify objects are as expected */
 724     ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1));
 725     ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2));
 726   }
 727 }
 728
 729 TEST(cls_rgw, gc_defer)
 730 {
 731   librados::IoCtx ioctx;
 732   librados::Rados rados;
 733
 734   string gc_pool_name = get_temp_pool_name();
 735   /* create pool */
 736   ASSERT_EQ("", create_one_pool_pp(gc_pool_name, rados));
 737   ASSERT_EQ(0, rados.ioctx_create(gc_pool_name.c_str(), ioctx));
 738
 739   string oid = "obj";
 740   string tag = "mychain";
 741
 742   librados::ObjectWriteOperation op;
 743   cls_rgw_gc_obj_info info;
 744
 745   op.create(false);
 746
 747   info.tag = tag;
 748
 749   /* create chain */
 750   cls_rgw_gc_set_entry(op, 0, info);
 751
 752   ASSERT_EQ(0, ioctx.operate(oid, &op));
 753
 754   bool truncated;
 755   list<cls_rgw_gc_obj_info> entries;
 756   string marker;
 757   string next_marker;
 758
 759   /* list chains, verify num entries as expected */
 760   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
 761   ASSERT_EQ(1, (int)entries.size());
 762   ASSERT_EQ(0, truncated);
 763
 764   librados::ObjectWriteOperation op2;
 765
 766   /* defer chain */
 767   cls_rgw_gc_defer_entry(op2, 5, tag);
 768   ASSERT_EQ(0, ioctx.operate(oid, &op2));
 769
 770   entries.clear();
 771   next_marker.clear();
 772
 773   /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */
 774   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
 775   ASSERT_EQ(0, (int)entries.size());
 776   ASSERT_EQ(0, truncated);
 777
 778   /* wait enough */
 779   sleep(5);
 780   next_marker.clear();
 781
 782   /* verify list shows deferred entry */
 783   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
 784   ASSERT_EQ(1, (int)entries.size());
 785   ASSERT_EQ(0, truncated);
 786
 787   librados::ObjectWriteOperation op3;
 788   vector<string> tags;
 789   tags.push_back(tag);
 790
 791   /* remove chain */
 792   cls_rgw_gc_remove(op3, tags);
 793   ASSERT_EQ(0, ioctx.operate(oid, &op3));
 794
 795   entries.clear();
 796   next_marker.clear();
 797
 798   /* verify entry was removed */
 799   ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
 800   ASSERT_EQ(0, (int)entries.size());
 801   ASSERT_EQ(0, truncated);
 802
 803   /* remove pool */
 804   ioctx.close();
 805   ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name, rados));
 806 }
 807
 808 auto populate_usage_log_info(std::string user, std::string payer, int total_usage_entries)
 809 {
 810   rgw_usage_log_info info;
 811
 812   for (int i=0; i < total_usage_entries; i++){
 813     auto bucket = str_int("bucket", i);
 814     info.entries.emplace_back(rgw_usage_log_entry(user, payer, bucket));
 815   }
 816
 817   return info;
 818 }
 819
 820 auto gen_usage_log_info(std::string payer, std::string bucket, int total_usage_entries)
 821 {
 822   rgw_usage_log_info info;
 823   for (int i=0; i < total_usage_entries; i++){
 824     auto user = str_int("user", i);
 825     info.entries.emplace_back(rgw_usage_log_entry(user, payer, bucket));
 826   }
 827
 828   return info;
 829 }
 830
 831 TEST(cls_rgw, usage_basic)
 832 {
 833   string oid="usage.1";
 834   string user="user1";
 835   uint64_t start_epoch{0}, end_epoch{(uint64_t) -1};
 836   int total_usage_entries = 512;
 837   uint64_t max_entries = 2000;
 838   string payer;
 839
 840   auto info = populate_usage_log_info(user, payer, total_usage_entries);
 841   ObjectWriteOperation op;
 842   cls_rgw_usage_log_add(op, info);
 843   ASSERT_EQ(0, ioctx.operate(oid, &op));
 844
 845   string read_iter;
 846   map <rgw_user_bucket, rgw_usage_log_entry> usage, usage2;
 847   bool truncated;
 848
 849
 850   int ret = cls_rgw_usage_log_read(ioctx, oid, user, "", start_epoch, end_epoch,
 851                                    max_entries, read_iter, usage, &truncated);
 852   // read the entries, and see that we have all the added entries
 853   ASSERT_EQ(0, ret);
 854   ASSERT_FALSE(truncated);
 855   ASSERT_EQ(static_cast<uint64_t>(total_usage_entries), usage.size());
 856
 857   // delete and read to assert that we've deleted all the values
 858   ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx, oid, user, "", start_epoch, end_epoch));
 859
 860
 861   ret = cls_rgw_usage_log_read(ioctx, oid, user, "", start_epoch, end_epoch,
 862                                max_entries, read_iter, usage2, &truncated);
 863   ASSERT_EQ(0, ret);
 864   ASSERT_EQ(0u, usage2.size());
 865
 866   // add and read to assert that bucket option is valid for usage reading
 867   string bucket1 = "bucket-usage-1";
 868   string bucket2 = "bucket-usage-2";
 869   info = gen_usage_log_info(payer, bucket1, 100);
 870   cls_rgw_usage_log_add(op, info);
 871   ASSERT_EQ(0, ioctx.operate(oid, &op));
 872
 873   info = gen_usage_log_info(payer, bucket2, 100);
 874   cls_rgw_usage_log_add(op, info);
 875   ASSERT_EQ(0, ioctx.operate(oid, &op));
 876   ret = cls_rgw_usage_log_read(ioctx, oid, "", bucket1, start_epoch, end_epoch,
 877                               max_entries, read_iter, usage2, &truncated);
 878   ASSERT_EQ(0, ret);
 879   ASSERT_EQ(100u, usage2.size());
 880
 881   // delete and read to assert that bucket option is valid for usage trim
 882   ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx, oid, "", bucket1, start_epoch, end_epoch));
 883
 884   ret = cls_rgw_usage_log_read(ioctx, oid, "", bucket1, start_epoch, end_epoch,
 885                                max_entries, read_iter, usage2, &truncated);
 886   ASSERT_EQ(0, ret);
 887   ASSERT_EQ(0u, usage2.size());
 888   ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx, oid, "", bucket2, start_epoch, end_epoch));
 889 }
 890
 891 TEST(cls_rgw, usage_clear_no_obj)
 892 {
 893   string user="user1";
 894   string oid="usage.10";
 895   librados::ObjectWriteOperation op;
 896   cls_rgw_usage_log_clear(op);
 897   int ret = ioctx.operate(oid, &op);
 898   ASSERT_EQ(0, ret);
 899
 900 }
 901
 902 TEST(cls_rgw, usage_clear)
 903 {
 904   string user="user1";
 905   string payer;
 906   string oid="usage.10";
 907   librados::ObjectWriteOperation op;
 908   int max_entries=2000;
 909
 910   auto info = populate_usage_log_info(user, payer, max_entries);
 911
 912   cls_rgw_usage_log_add(op, info);
 913   ASSERT_EQ(0, ioctx.operate(oid, &op));
 914
 915   ObjectWriteOperation op2;
 916   cls_rgw_usage_log_clear(op2);
 917   int ret = ioctx.operate(oid, &op2);
 918   ASSERT_EQ(0, ret);
 919
 920   map <rgw_user_bucket, rgw_usage_log_entry> usage;
 921   bool truncated;
 922   uint64_t start_epoch{0}, end_epoch{(uint64_t) -1};
 923   string read_iter;
 924   ret = cls_rgw_usage_log_read(ioctx, oid, user, "", start_epoch, end_epoch,
 925                                max_entries, read_iter, usage, &truncated);
 926   ASSERT_EQ(0, ret);
 927   ASSERT_EQ(0u, usage.size());
 928
 929 }
 930
 931
 932 /* must be last test! */
 933
 934 TEST(cls_rgw, finalize)
 935 {
 936   /* remove pool */
 937   ioctx.close();
 938   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
 939 }