1 // -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/types.h"
5 #include "cls/rgw/cls_rgw_client.h"
6 #include "cls/rgw/cls_rgw_ops.h"
8 #include "gtest/gtest.h"
9 #include "test/librados/test_cxx.h"
10 #include "global/global_context.h"
11 #include "common/ceph_context.h"
19 using namespace librados
;
21 // creates a temporary pool and initializes an IoCtx shared by all tests
22 class cls_rgw
: public ::testing::Test
{
23 static librados::Rados rados
;
24 static std::string pool_name
;
26 static librados::IoCtx ioctx
;
28 static void SetUpTestCase() {
29 pool_name
= get_temp_pool_name();
31 ASSERT_EQ("", create_one_pool_pp(pool_name
, rados
));
32 ASSERT_EQ(0, rados
.ioctx_create(pool_name
.c_str(), ioctx
));
34 static void TearDownTestCase() {
37 ASSERT_EQ(0, destroy_one_pool_pp(pool_name
, rados
));
40 librados::Rados
cls_rgw::rados
;
41 std::string
cls_rgw::pool_name
;
42 librados::IoCtx
cls_rgw::ioctx
;
45 string
str_int(string s
, int i
)
48 snprintf(buf
, sizeof(buf
), "-%d", i
);
54 void test_stats(librados::IoCtx
& ioctx
, string
& oid
, RGWObjCategory category
, uint64_t num_entries
, uint64_t total_size
)
56 map
<int, struct rgw_cls_list_ret
> results
;
57 map
<int, string
> oids
;
59 ASSERT_EQ(0, CLSRGWIssueGetDirHeader(ioctx
, oids
, results
, 8)());
63 map
<int, struct rgw_cls_list_ret
>::iterator iter
= results
.begin();
64 for (; iter
!= results
.end(); ++iter
) {
65 entries
+= (iter
->second
).dir
.header
.stats
[category
].num_entries
;
66 size
+= (iter
->second
).dir
.header
.stats
[category
].total_size
;
68 ASSERT_EQ(total_size
, size
);
69 ASSERT_EQ(num_entries
, entries
);
72 void index_prepare(librados::IoCtx
& ioctx
, string
& oid
, RGWModifyOp index_op
,
73 string
& tag
, const cls_rgw_obj_key
& key
, string
& loc
,
74 uint16_t bi_flags
= 0, bool log_op
= true)
76 ObjectWriteOperation op
;
77 rgw_zone_set zones_trace
;
78 cls_rgw_bucket_prepare_op(op
, index_op
, tag
, key
, loc
, log_op
, bi_flags
, zones_trace
);
79 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
82 void index_complete(librados::IoCtx
& ioctx
, string
& oid
, RGWModifyOp index_op
,
83 string
& tag
, int epoch
, const cls_rgw_obj_key
& key
,
84 rgw_bucket_dir_entry_meta
& meta
, uint16_t bi_flags
= 0,
87 ObjectWriteOperation op
;
88 rgw_bucket_entry_ver ver
;
89 ver
.pool
= ioctx
.get_id();
91 meta
.accounted_size
= meta
.size
;
92 cls_rgw_bucket_complete_op(op
, index_op
, tag
, ver
, key
, meta
, nullptr, log_op
, bi_flags
, nullptr);
93 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
94 if (!key
.instance
.empty()) {
97 rgw_zone_set zone_set
;
98 ASSERT_EQ(0, cls_rgw_bucket_link_olh(ioctx
, oid
, key
, olh_tag
,
99 false, tag
, &meta
, epoch
,
100 ceph::real_time
{}, true, true, zone_set
));
104 TEST_F(cls_rgw
, index_basic
)
106 string bucket_oid
= str_int("bucket", 0);
108 ObjectWriteOperation op
;
109 cls_rgw_bucket_init_index(op
);
110 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
114 uint64_t obj_size
= 1024;
117 for (int i
= 0; i
< NUM_OBJS
; i
++) {
118 cls_rgw_obj_key obj
= str_int("obj", i
);
119 string tag
= str_int("tag", i
);
120 string loc
= str_int("loc", i
);
122 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
124 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, i
, obj_size
* i
);
126 rgw_bucket_dir_entry_meta meta
;
127 meta
.category
= RGWObjCategory::None
;
128 meta
.size
= obj_size
;
129 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, epoch
, obj
, meta
);
132 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, NUM_OBJS
,
133 obj_size
* NUM_OBJS
);
136 TEST_F(cls_rgw
, index_multiple_obj_writers
)
138 string bucket_oid
= str_int("bucket", 1);
140 ObjectWriteOperation op
;
141 cls_rgw_bucket_init_index(op
);
142 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
144 uint64_t obj_size
= 1024;
146 cls_rgw_obj_key obj
= str_int("obj", 0);
147 string loc
= str_int("loc", 0);
148 /* multi prepare on a single object */
149 for (int i
= 0; i
< NUM_OBJS
; i
++) {
150 string tag
= str_int("tag", i
);
152 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
154 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, 0, 0);
157 for (int i
= NUM_OBJS
; i
> 0; i
--) {
158 string tag
= str_int("tag", i
- 1);
160 rgw_bucket_dir_entry_meta meta
;
161 meta
.category
= RGWObjCategory::None
;
162 meta
.size
= obj_size
* i
;
164 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, i
, obj
, meta
);
166 /* verify that object size doesn't change, as we went back with epoch */
167 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, 1,
168 obj_size
* NUM_OBJS
);
172 TEST_F(cls_rgw
, index_remove_object
)
174 string bucket_oid
= str_int("bucket", 2);
176 ObjectWriteOperation op
;
177 cls_rgw_bucket_init_index(op
);
178 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
180 uint64_t obj_size
= 1024;
181 uint64_t total_size
= 0;
185 /* prepare multiple objects */
186 for (int i
= 0; i
< NUM_OBJS
; i
++) {
187 cls_rgw_obj_key obj
= str_int("obj", i
);
188 string tag
= str_int("tag", i
);
189 string loc
= str_int("loc", i
);
191 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
193 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, i
, total_size
);
195 rgw_bucket_dir_entry_meta meta
;
196 meta
.category
= RGWObjCategory::None
;
197 meta
.size
= i
* obj_size
;
198 total_size
+= i
* obj_size
;
200 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, ++epoch
, obj
, meta
);
202 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, i
+ 1, total_size
);
205 int i
= NUM_OBJS
/ 2;
206 string tag_remove
= "tag-rm";
207 string tag_modify
= "tag-mod";
208 cls_rgw_obj_key obj
= str_int("obj", i
);
209 string loc
= str_int("loc", i
);
211 /* prepare both removal and modification on the same object */
212 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_DEL
, tag_remove
, obj
, loc
);
213 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag_modify
, obj
, loc
);
215 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, NUM_OBJS
, total_size
);
217 rgw_bucket_dir_entry_meta meta
;
219 /* complete object removal */
220 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_DEL
, tag_remove
, ++epoch
, obj
, meta
);
222 /* verify stats correct */
223 total_size
-= i
* obj_size
;
224 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, NUM_OBJS
- 1, total_size
);
227 meta
.category
= RGWObjCategory::None
;
229 /* complete object modification */
230 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag_modify
, ++epoch
, obj
, meta
);
232 /* verify stats correct */
233 total_size
+= meta
.size
;
234 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, NUM_OBJS
, total_size
);
237 /* prepare both removal and modification on the same object, this time we'll
238 * first complete modification then remove*/
239 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_DEL
, tag_remove
, obj
, loc
);
240 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_DEL
, tag_modify
, obj
, loc
);
242 /* complete modification */
243 total_size
-= meta
.size
;
244 meta
.size
= i
* obj_size
* 2;
245 meta
.category
= RGWObjCategory::None
;
247 /* complete object modification */
248 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag_modify
, ++epoch
, obj
, meta
);
250 /* verify stats correct */
251 total_size
+= meta
.size
;
252 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, NUM_OBJS
, total_size
);
254 /* complete object removal */
255 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_DEL
, tag_remove
, ++epoch
, obj
, meta
);
257 /* verify stats correct */
258 total_size
-= meta
.size
;
259 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, NUM_OBJS
- 1,
263 TEST_F(cls_rgw
, index_suggest
)
265 string bucket_oid
= str_int("bucket", 3);
267 ObjectWriteOperation op
;
268 cls_rgw_bucket_init_index(op
);
269 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
271 uint64_t total_size
= 0;
277 uint64_t obj_size
= 1024;
279 /* create multiple objects */
280 for (int i
= 0; i
< num_objs
; i
++) {
281 cls_rgw_obj_key obj
= str_int("obj", i
);
282 string tag
= str_int("tag", i
);
283 string loc
= str_int("loc", i
);
285 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
287 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, i
, total_size
);
289 rgw_bucket_dir_entry_meta meta
;
290 meta
.category
= RGWObjCategory::None
;
291 meta
.size
= obj_size
;
292 total_size
+= meta
.size
;
294 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, ++epoch
, obj
, meta
);
296 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, i
+ 1, total_size
);
299 /* prepare (without completion) some of the objects */
300 for (int i
= 0; i
< num_objs
; i
+= 2) {
301 cls_rgw_obj_key obj
= str_int("obj", i
);
302 string tag
= str_int("tag-prepare", i
);
303 string loc
= str_int("loc", i
);
305 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
307 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, num_objs
, total_size
);
310 int actual_num_objs
= num_objs
;
311 /* remove half of the objects */
312 for (int i
= num_objs
/ 2; i
< num_objs
; i
++) {
313 cls_rgw_obj_key obj
= str_int("obj", i
);
314 string tag
= str_int("tag-rm", i
);
315 string loc
= str_int("loc", i
);
317 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
319 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, actual_num_objs
, total_size
);
321 rgw_bucket_dir_entry_meta meta
;
322 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_DEL
, tag
, ++epoch
, obj
, meta
);
324 total_size
-= obj_size
;
326 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, actual_num_objs
, total_size
);
331 for (int i
= 0; i
< num_objs
; i
+= 2) {
332 cls_rgw_obj_key obj
= str_int("obj", i
);
333 string tag
= str_int("tag-rm", i
);
334 string loc
= str_int("loc", i
);
336 rgw_bucket_dir_entry dirent
;
337 dirent
.key
.name
= obj
.name
;
338 dirent
.locator
= loc
;
339 dirent
.exists
= (i
< num_objs
/ 2); // we removed half the objects
340 dirent
.meta
.size
= 1024;
341 dirent
.meta
.accounted_size
= 1024;
343 char suggest_op
= (i
< num_objs
/ 2 ? CEPH_RGW_UPDATE
: CEPH_RGW_REMOVE
);
344 cls_rgw_encode_suggestion(suggest_op
, dirent
, updates
);
347 map
<int, string
> bucket_objs
;
348 bucket_objs
[0] = bucket_oid
;
349 int r
= CLSRGWIssueSetTagTimeout(ioctx
, bucket_objs
, 8 /* max aio */, 1)();
354 /* suggest changes! */
356 ObjectWriteOperation op
;
357 cls_rgw_suggest_changes(op
, updates
);
358 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
360 /* suggest changes twice! */
362 ObjectWriteOperation op
;
363 cls_rgw_suggest_changes(op
, updates
);
364 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
366 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
, num_objs
/ 2, total_size
);
371 * This case is used to test whether get_obj_vals will
372 * return all validate utf8 objnames and filter out those
373 * in BI_PREFIX_CHAR private namespace.
375 TEST_F(cls_rgw
, index_list
)
377 string bucket_oid
= str_int("bucket", 4);
379 ObjectWriteOperation op
;
380 cls_rgw_bucket_init_index(op
);
381 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
384 uint64_t obj_size
= 1024;
385 const int num_objs
= 4;
386 const string keys
[num_objs
] = {
387 /* single byte utf8 character */
388 { static_cast<char>(0x41) },
389 /* double byte utf8 character */
390 { static_cast<char>(0xCF), static_cast<char>(0x8F) },
391 /* treble byte utf8 character */
392 { static_cast<char>(0xDF), static_cast<char>(0x8F), static_cast<char>(0x8F) },
393 /* quadruble byte utf8 character */
394 { static_cast<char>(0xF7), static_cast<char>(0x8F), static_cast<char>(0x8F), static_cast<char>(0x8F) },
397 for (int i
= 0; i
< num_objs
; i
++) {
398 string obj
= keys
[i
];
399 string tag
= str_int("tag", i
);
400 string loc
= str_int("loc", i
);
402 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
,
403 0 /* bi_flags */, false /* log_op */);
405 rgw_bucket_dir_entry_meta meta
;
406 meta
.category
= RGWObjCategory::None
;
407 meta
.size
= obj_size
;
408 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, epoch
, obj
, meta
,
409 0 /* bi_flags */, false /* log_op */);
412 map
<string
, bufferlist
> entries
;
413 /* insert 998 omap key starts with BI_PREFIX_CHAR,
414 * so bucket list first time will get one key before 0x80 and one key after */
415 for (int i
= 0; i
< 998; ++i
) {
417 snprintf(buf
, sizeof(buf
), "%c%s%d", 0x80, "1000_", i
);
418 entries
.emplace(string
{buf
}, bufferlist
{});
420 ioctx
.omap_set(bucket_oid
, entries
);
422 test_stats(ioctx
, bucket_oid
, RGWObjCategory::None
,
423 num_objs
, obj_size
* num_objs
);
425 map
<int, string
> oids
= { {0, bucket_oid
} };
426 map
<int, struct rgw_cls_list_ret
> list_results
;
427 cls_rgw_obj_key
start_key("", "");
429 string empty_delimiter
;
430 int r
= CLSRGWIssueBucketList(ioctx
, start_key
,
431 empty_prefix
, empty_delimiter
,
432 1000, true, oids
, list_results
, 1)();
434 ASSERT_EQ(1u, list_results
.size());
436 auto it
= list_results
.begin();
437 auto m
= (it
->second
).dir
.m
;
439 ASSERT_EQ(4u, m
.size());
441 for(auto it2
= m
.cbegin(); it2
!= m
.cend(); it2
++, i
++) {
442 ASSERT_EQ(it2
->first
.compare(keys
[i
]), 0);
448 * This case is used to test when bucket index list that includes a
449 * delimiter can handle the first chunk ending in a delimiter.
451 TEST_F(cls_rgw
, index_list_delimited
)
453 string bucket_oid
= str_int("bucket", 7);
455 ObjectWriteOperation op
;
456 cls_rgw_bucket_init_index(op
);
457 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
460 uint64_t obj_size
= 1024;
461 const int file_num_objs
= 5;
462 const int dir_num_objs
= 1005;
464 std::vector
<std::string
> file_prefixes
=
465 { "a", "c", "e", "g", "i", "k", "m", "o", "q", "s", "u" };
466 std::vector
<std::string
> dir_prefixes
=
467 { "b/", "d/", "f/", "h/", "j/", "l/", "n/", "p/", "r/", "t/" };
469 rgw_bucket_dir_entry_meta meta
;
470 meta
.category
= RGWObjCategory::None
;
471 meta
.size
= obj_size
;
473 // create top-level files
474 for (const auto& p
: file_prefixes
) {
475 for (int i
= 0; i
< file_num_objs
; i
++) {
476 string tag
= str_int("tag", i
);
477 string loc
= str_int("loc", i
);
478 const string obj
= str_int(p
, i
);
480 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
,
481 0 /* bi_flags */, false /* log_op */);
483 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, epoch
, obj
, meta
,
484 0 /* bi_flags */, false /* log_op */);
488 // create large directories
489 for (const auto& p
: dir_prefixes
) {
490 for (int i
= 0; i
< dir_num_objs
; i
++) {
491 string tag
= str_int("tag", i
);
492 string loc
= str_int("loc", i
);
493 const string obj
= p
+ str_int("f", i
);
495 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
,
496 0 /* bi_flags */, false /* log_op */);
498 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, epoch
, obj
, meta
,
499 0 /* bi_flags */, false /* log_op */);
503 map
<int, string
> oids
= { {0, bucket_oid
} };
504 map
<int, struct rgw_cls_list_ret
> list_results
;
505 cls_rgw_obj_key
start_key("", "");
506 const string empty_prefix
;
507 const string delimiter
= "/";
508 int r
= CLSRGWIssueBucketList(ioctx
, start_key
,
509 empty_prefix
, delimiter
,
510 1000, true, oids
, list_results
, 1)();
512 ASSERT_EQ(1u, list_results
.size()) <<
513 "Because we only have one bucket index shard, we should "
514 "only get one list_result.";
516 auto it
= list_results
.begin();
517 auto id_entry_map
= it
->second
.dir
.m
;
518 bool truncated
= it
->second
.is_truncated
;
520 // the cls code will make 4 tries to get 1000 entries; however
521 // because each of the subdirectories is so large, each attempt will
522 // only retrieve the first part of the subdirectory
524 ASSERT_EQ(48u, id_entry_map
.size()) <<
525 "We should get 40 top-level entries and the tops of 8 \"subdirectories\".";
526 ASSERT_EQ(true, truncated
) << "We did not get all entries.";
528 ASSERT_EQ("a-0", id_entry_map
.cbegin()->first
);
529 ASSERT_EQ("p/", id_entry_map
.crbegin()->first
);
531 // now let's get the rest of the entries
533 list_results
.clear();
535 cls_rgw_obj_key
start_key2("p/", "");
536 r
= CLSRGWIssueBucketList(ioctx
, start_key2
,
537 empty_prefix
, delimiter
,
538 1000, true, oids
, list_results
, 1)();
541 it
= list_results
.begin();
542 id_entry_map
= it
->second
.dir
.m
;
543 truncated
= it
->second
.is_truncated
;
545 ASSERT_EQ(17u, id_entry_map
.size()) <<
546 "We should get 15 top-level entries and the tops of 2 \"subdirectories\".";
547 ASSERT_EQ(false, truncated
) << "We now have all entries.";
549 ASSERT_EQ("q-0", id_entry_map
.cbegin()->first
);
550 ASSERT_EQ("u-4", id_entry_map
.crbegin()->first
);
554 TEST_F(cls_rgw
, bi_list
)
556 string bucket_oid
= str_int("bucket", 5);
558 CephContext
*cct
= reinterpret_cast<CephContext
*>(ioctx
.cct());
560 ObjectWriteOperation op
;
561 cls_rgw_bucket_init_index(op
);
562 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
567 list
<rgw_cls_bi_entry
> entries
;
570 int ret
= cls_rgw_bi_list(ioctx
, bucket_oid
, name
, marker
, max
, &entries
,
573 ASSERT_EQ(entries
.size(), 0u) <<
574 "The listing of an empty bucket as 0 entries.";
575 ASSERT_EQ(is_truncated
, false) <<
576 "The listing of an empty bucket is not truncated.";
579 uint64_t obj_size
= 1024;
580 uint64_t num_objs
= 35;
582 for (uint64_t i
= 0; i
< num_objs
; i
++) {
583 string obj
= str_int(i
% 4 ? "obj" : "об'єкт", i
);
584 string tag
= str_int("tag", i
);
585 string loc
= str_int("loc", i
);
586 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
,
587 RGW_BILOG_FLAG_VERSIONED_OP
);
589 rgw_bucket_dir_entry_meta meta
;
590 meta
.category
= RGWObjCategory::None
;
591 meta
.size
= obj_size
;
592 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, epoch
, obj
, meta
,
593 RGW_BILOG_FLAG_VERSIONED_OP
);
596 ret
= cls_rgw_bi_list(ioctx
, bucket_oid
, name
, marker
, num_objs
+ 10, &entries
,
600 ASSERT_LT(entries
.size(), num_objs
);
602 ASSERT_EQ(entries
.size(), num_objs
);
605 uint64_t num_entries
= 0;
608 while(is_truncated
) {
609 ret
= cls_rgw_bi_list(ioctx
, bucket_oid
, name
, marker
, max
, &entries
,
613 ASSERT_LT(entries
.size(), num_objs
- num_entries
);
615 ASSERT_EQ(entries
.size(), num_objs
- num_entries
);
617 num_entries
+= entries
.size();
618 marker
= entries
.back().idx
;
621 ret
= cls_rgw_bi_list(ioctx
, bucket_oid
, name
, marker
, max
, &entries
,
624 ASSERT_EQ(entries
.size(), 0u);
625 ASSERT_EQ(is_truncated
, false);
627 if (cct
->_conf
->osd_max_omap_entries_per_request
< 15) {
632 while(is_truncated
) {
633 ret
= cls_rgw_bi_list(ioctx
, bucket_oid
, name
, marker
, max
, &entries
,
637 ASSERT_LT(entries
.size(), num_objs
- num_entries
);
639 ASSERT_EQ(entries
.size(), num_objs
- num_entries
);
641 num_entries
+= entries
.size();
642 marker
= entries
.back().idx
;
646 ret
= cls_rgw_bi_list(ioctx
, bucket_oid
, name
, marker
, max
, &entries
,
649 ASSERT_EQ(entries
.size(), 0u);
650 ASSERT_EQ(is_truncated
, false);
653 /* test garbage collection */
654 static void create_obj(cls_rgw_obj
& obj
, int i
, int j
)
657 snprintf(buf
, sizeof(buf
), "-%d.%d", i
, j
);
659 obj
.pool
.append(buf
);
660 obj
.key
.name
= "oid";
661 obj
.key
.name
.append(buf
);
666 static bool cmp_objs(cls_rgw_obj
& obj1
, cls_rgw_obj
& obj2
)
668 return (obj1
.pool
== obj2
.pool
) &&
669 (obj1
.key
== obj2
.key
) &&
670 (obj1
.loc
== obj2
.loc
);
674 TEST_F(cls_rgw
, gc_set
)
678 for (int i
= 0; i
< 10; i
++) {
680 snprintf(buf
, sizeof(buf
), "chain-%d", i
);
682 librados::ObjectWriteOperation op
;
683 cls_rgw_gc_obj_info info
;
685 cls_rgw_obj obj1
, obj2
;
686 create_obj(obj1
, i
, 1);
687 create_obj(obj2
, i
, 2);
688 info
.chain
.objs
.push_back(obj1
);
689 info
.chain
.objs
.push_back(obj2
);
691 op
.create(false); // create object
694 cls_rgw_gc_set_entry(op
, 0, info
);
696 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
700 list
<cls_rgw_gc_obj_info
> entries
;
704 /* list chains, verify truncated */
705 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 8, true, entries
, &truncated
, next_marker
));
706 ASSERT_EQ(8, (int)entries
.size());
707 ASSERT_EQ(1, truncated
);
712 /* list all chains, verify not truncated */
713 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 10, true, entries
, &truncated
, next_marker
));
714 ASSERT_EQ(10, (int)entries
.size());
715 ASSERT_EQ(0, truncated
);
717 /* verify all chains are valid */
718 list
<cls_rgw_gc_obj_info
>::iterator iter
= entries
.begin();
719 for (int i
= 0; i
< 10; i
++, ++iter
) {
720 cls_rgw_gc_obj_info
& entry
= *iter
;
722 /* create expected chain name */
724 snprintf(buf
, sizeof(buf
), "chain-%d", i
);
727 /* verify chain name as expected */
728 ASSERT_EQ(entry
.tag
, tag
);
730 /* verify expected num of objects in chain */
731 ASSERT_EQ(2, (int)entry
.chain
.objs
.size());
733 list
<cls_rgw_obj
>::iterator oiter
= entry
.chain
.objs
.begin();
734 cls_rgw_obj obj1
, obj2
;
736 /* create expected objects */
737 create_obj(obj1
, i
, 1);
738 create_obj(obj2
, i
, 2);
740 /* assign returned object names */
741 cls_rgw_obj
& ret_obj1
= *oiter
++;
742 cls_rgw_obj
& ret_obj2
= *oiter
;
744 /* verify objects are as expected */
745 ASSERT_EQ(1, (int)cmp_objs(obj1
, ret_obj1
));
746 ASSERT_EQ(1, (int)cmp_objs(obj2
, ret_obj2
));
750 TEST_F(cls_rgw
, gc_list
)
754 for (int i
= 0; i
< 10; i
++) {
756 snprintf(buf
, sizeof(buf
), "chain-%d", i
);
758 librados::ObjectWriteOperation op
;
759 cls_rgw_gc_obj_info info
;
761 cls_rgw_obj obj1
, obj2
;
762 create_obj(obj1
, i
, 1);
763 create_obj(obj2
, i
, 2);
764 info
.chain
.objs
.push_back(obj1
);
765 info
.chain
.objs
.push_back(obj2
);
767 op
.create(false); // create object
770 cls_rgw_gc_set_entry(op
, 0, info
);
772 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
776 list
<cls_rgw_gc_obj_info
> entries
;
777 list
<cls_rgw_gc_obj_info
> entries2
;
781 /* list chains, verify truncated */
782 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 8, true, entries
, &truncated
, next_marker
));
783 ASSERT_EQ(8, (int)entries
.size());
784 ASSERT_EQ(1, truncated
);
786 marker
= next_marker
;
789 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 8, true, entries2
, &truncated
, next_marker
));
790 ASSERT_EQ(2, (int)entries2
.size());
791 ASSERT_EQ(0, truncated
);
793 entries
.splice(entries
.end(), entries2
);
795 /* verify all chains are valid */
796 list
<cls_rgw_gc_obj_info
>::iterator iter
= entries
.begin();
797 for (int i
= 0; i
< 10; i
++, ++iter
) {
798 cls_rgw_gc_obj_info
& entry
= *iter
;
800 /* create expected chain name */
802 snprintf(buf
, sizeof(buf
), "chain-%d", i
);
805 /* verify chain name as expected */
806 ASSERT_EQ(entry
.tag
, tag
);
808 /* verify expected num of objects in chain */
809 ASSERT_EQ(2, (int)entry
.chain
.objs
.size());
811 list
<cls_rgw_obj
>::iterator oiter
= entry
.chain
.objs
.begin();
812 cls_rgw_obj obj1
, obj2
;
814 /* create expected objects */
815 create_obj(obj1
, i
, 1);
816 create_obj(obj2
, i
, 2);
818 /* assign returned object names */
819 cls_rgw_obj
& ret_obj1
= *oiter
++;
820 cls_rgw_obj
& ret_obj2
= *oiter
;
822 /* verify objects are as expected */
823 ASSERT_EQ(1, (int)cmp_objs(obj1
, ret_obj1
));
824 ASSERT_EQ(1, (int)cmp_objs(obj2
, ret_obj2
));
828 TEST_F(cls_rgw
, gc_defer
)
830 librados::IoCtx ioctx
;
831 librados::Rados rados
;
833 string gc_pool_name
= get_temp_pool_name();
835 ASSERT_EQ("", create_one_pool_pp(gc_pool_name
, rados
));
836 ASSERT_EQ(0, rados
.ioctx_create(gc_pool_name
.c_str(), ioctx
));
839 string tag
= "mychain";
841 librados::ObjectWriteOperation op
;
842 cls_rgw_gc_obj_info info
;
849 cls_rgw_gc_set_entry(op
, 0, info
);
851 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
854 list
<cls_rgw_gc_obj_info
> entries
;
858 /* list chains, verify num entries as expected */
859 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 1, true, entries
, &truncated
, next_marker
));
860 ASSERT_EQ(1, (int)entries
.size());
861 ASSERT_EQ(0, truncated
);
863 librados::ObjectWriteOperation op2
;
866 cls_rgw_gc_defer_entry(op2
, 5, tag
);
867 ASSERT_EQ(0, ioctx
.operate(oid
, &op2
));
872 /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */
873 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 1, true, entries
, &truncated
, next_marker
));
874 ASSERT_EQ(0, (int)entries
.size());
875 ASSERT_EQ(0, truncated
);
881 /* verify list shows deferred entry */
882 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 1, true, entries
, &truncated
, next_marker
));
883 ASSERT_EQ(1, (int)entries
.size());
884 ASSERT_EQ(0, truncated
);
886 librados::ObjectWriteOperation op3
;
891 cls_rgw_gc_remove(op3
, tags
);
892 ASSERT_EQ(0, ioctx
.operate(oid
, &op3
));
897 /* verify entry was removed */
898 ASSERT_EQ(0, cls_rgw_gc_list(ioctx
, oid
, marker
, 1, true, entries
, &truncated
, next_marker
));
899 ASSERT_EQ(0, (int)entries
.size());
900 ASSERT_EQ(0, truncated
);
904 ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name
, rados
));
907 auto populate_usage_log_info(std::string user
, std::string payer
, int total_usage_entries
)
909 rgw_usage_log_info info
;
911 for (int i
=0; i
< total_usage_entries
; i
++){
912 auto bucket
= str_int("bucket", i
);
913 info
.entries
.emplace_back(rgw_usage_log_entry(user
, payer
, bucket
));
919 auto gen_usage_log_info(std::string payer
, std::string bucket
, int total_usage_entries
)
921 rgw_usage_log_info info
;
922 for (int i
=0; i
< total_usage_entries
; i
++){
923 auto user
= str_int("user", i
);
924 info
.entries
.emplace_back(rgw_usage_log_entry(user
, payer
, bucket
));
930 TEST_F(cls_rgw
, usage_basic
)
932 string oid
="usage.1";
934 uint64_t start_epoch
{0}, end_epoch
{(uint64_t) -1};
935 int total_usage_entries
= 512;
936 uint64_t max_entries
= 2000;
939 auto info
= populate_usage_log_info(user
, payer
, total_usage_entries
);
940 ObjectWriteOperation op
;
941 cls_rgw_usage_log_add(op
, info
);
942 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
945 map
<rgw_user_bucket
, rgw_usage_log_entry
> usage
, usage2
;
949 int ret
= cls_rgw_usage_log_read(ioctx
, oid
, user
, "", start_epoch
, end_epoch
,
950 max_entries
, read_iter
, usage
, &truncated
);
951 // read the entries, and see that we have all the added entries
953 ASSERT_FALSE(truncated
);
954 ASSERT_EQ(static_cast<uint64_t>(total_usage_entries
), usage
.size());
956 // delete and read to assert that we've deleted all the values
957 ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx
, oid
, user
, "", start_epoch
, end_epoch
));
960 ret
= cls_rgw_usage_log_read(ioctx
, oid
, user
, "", start_epoch
, end_epoch
,
961 max_entries
, read_iter
, usage2
, &truncated
);
963 ASSERT_EQ(0u, usage2
.size());
965 // add and read to assert that bucket option is valid for usage reading
966 string bucket1
= "bucket-usage-1";
967 string bucket2
= "bucket-usage-2";
968 info
= gen_usage_log_info(payer
, bucket1
, 100);
969 cls_rgw_usage_log_add(op
, info
);
970 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
972 info
= gen_usage_log_info(payer
, bucket2
, 100);
973 cls_rgw_usage_log_add(op
, info
);
974 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
975 ret
= cls_rgw_usage_log_read(ioctx
, oid
, "", bucket1
, start_epoch
, end_epoch
,
976 max_entries
, read_iter
, usage2
, &truncated
);
978 ASSERT_EQ(100u, usage2
.size());
980 // delete and read to assert that bucket option is valid for usage trim
981 ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx
, oid
, "", bucket1
, start_epoch
, end_epoch
));
983 ret
= cls_rgw_usage_log_read(ioctx
, oid
, "", bucket1
, start_epoch
, end_epoch
,
984 max_entries
, read_iter
, usage2
, &truncated
);
986 ASSERT_EQ(0u, usage2
.size());
987 ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx
, oid
, "", bucket2
, start_epoch
, end_epoch
));
990 TEST_F(cls_rgw
, usage_clear_no_obj
)
993 string oid
="usage.10";
994 librados::ObjectWriteOperation op
;
995 cls_rgw_usage_log_clear(op
);
996 int ret
= ioctx
.operate(oid
, &op
);
1001 TEST_F(cls_rgw
, usage_clear
)
1003 string user
="user1";
1005 string oid
="usage.10";
1006 librados::ObjectWriteOperation op
;
1007 int max_entries
=2000;
1009 auto info
= populate_usage_log_info(user
, payer
, max_entries
);
1011 cls_rgw_usage_log_add(op
, info
);
1012 ASSERT_EQ(0, ioctx
.operate(oid
, &op
));
1014 ObjectWriteOperation op2
;
1015 cls_rgw_usage_log_clear(op2
);
1016 int ret
= ioctx
.operate(oid
, &op2
);
1019 map
<rgw_user_bucket
, rgw_usage_log_entry
> usage
;
1021 uint64_t start_epoch
{0}, end_epoch
{(uint64_t) -1};
1023 ret
= cls_rgw_usage_log_read(ioctx
, oid
, user
, "", start_epoch
, end_epoch
,
1024 max_entries
, read_iter
, usage
, &truncated
);
1026 ASSERT_EQ(0u, usage
.size());
1029 static int bilog_list(librados::IoCtx
& ioctx
, const std::string
& oid
,
1030 cls_rgw_bi_log_list_ret
*result
)
1033 librados::ObjectReadOperation op
;
1034 cls_rgw_bilog_list(op
, "", 128, result
, &retcode
);
1035 int ret
= ioctx
.operate(oid
, &op
, nullptr);
1042 static int bilog_trim(librados::IoCtx
& ioctx
, const std::string
& oid
,
1043 const std::string
& start_marker
,
1044 const std::string
& end_marker
)
1046 librados::ObjectWriteOperation op
;
1047 cls_rgw_bilog_trim(op
, start_marker
, end_marker
);
1048 return ioctx
.operate(oid
, &op
);
1051 TEST_F(cls_rgw
, bi_log_trim
)
1053 string bucket_oid
= str_int("bucket", 6);
1055 ObjectWriteOperation op
;
1056 cls_rgw_bucket_init_index(op
);
1057 ASSERT_EQ(0, ioctx
.operate(bucket_oid
, &op
));
1059 // create 10 versioned entries. this generates instance and olh bi entries,
1060 // allowing us to check that bilog trim doesn't remove any of those
1061 for (int i
= 0; i
< 10; i
++) {
1062 cls_rgw_obj_key obj
{str_int("obj", i
), "inst"};
1063 string tag
= str_int("tag", i
);
1064 string loc
= str_int("loc", i
);
1066 index_prepare(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, obj
, loc
);
1067 rgw_bucket_dir_entry_meta meta
;
1068 index_complete(ioctx
, bucket_oid
, CLS_RGW_OP_ADD
, tag
, 1, obj
, meta
);
1072 list
<rgw_cls_bi_entry
> entries
;
1073 bool truncated
{false};
1074 ASSERT_EQ(0, cls_rgw_bi_list(ioctx
, bucket_oid
, "", "", 128,
1075 &entries
, &truncated
));
1076 // prepare/complete/instance/olh entry for each
1077 EXPECT_EQ(40u, entries
.size());
1078 EXPECT_FALSE(truncated
);
1081 vector
<rgw_bi_log_entry
> bilog1
;
1083 cls_rgw_bi_log_list_ret bilog
;
1084 ASSERT_EQ(0, bilog_list(ioctx
, bucket_oid
, &bilog
));
1085 // complete/olh entry for each
1086 EXPECT_EQ(20u, bilog
.entries
.size());
1088 bilog1
.assign(std::make_move_iterator(bilog
.entries
.begin()),
1089 std::make_move_iterator(bilog
.entries
.end()));
1091 // trim front of bilog
1093 const std::string from
= "";
1094 const std::string to
= bilog1
[0].id
;
1095 ASSERT_EQ(0, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1096 cls_rgw_bi_log_list_ret bilog
;
1097 ASSERT_EQ(0, bilog_list(ioctx
, bucket_oid
, &bilog
));
1098 EXPECT_EQ(19u, bilog
.entries
.size());
1099 EXPECT_EQ(bilog1
[1].id
, bilog
.entries
.begin()->id
);
1100 ASSERT_EQ(-ENODATA
, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1102 // trim back of bilog
1104 const std::string from
= bilog1
[18].id
;
1105 const std::string to
= "9";
1106 ASSERT_EQ(0, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1107 cls_rgw_bi_log_list_ret bilog
;
1108 ASSERT_EQ(0, bilog_list(ioctx
, bucket_oid
, &bilog
));
1109 EXPECT_EQ(18u, bilog
.entries
.size());
1110 EXPECT_EQ(bilog1
[18].id
, bilog
.entries
.rbegin()->id
);
1111 ASSERT_EQ(-ENODATA
, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1113 // trim middle of bilog
1115 const std::string from
= bilog1
[13].id
;
1116 const std::string to
= bilog1
[14].id
;
1117 ASSERT_EQ(0, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1118 cls_rgw_bi_log_list_ret bilog
;
1119 ASSERT_EQ(0, bilog_list(ioctx
, bucket_oid
, &bilog
));
1120 EXPECT_EQ(17u, bilog
.entries
.size());
1121 ASSERT_EQ(-ENODATA
, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1125 const std::string from
= "";
1126 const std::string to
= "9";
1127 ASSERT_EQ(0, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1128 cls_rgw_bi_log_list_ret bilog
;
1129 ASSERT_EQ(0, bilog_list(ioctx
, bucket_oid
, &bilog
));
1130 EXPECT_EQ(0u, bilog
.entries
.size());
1131 ASSERT_EQ(-ENODATA
, bilog_trim(ioctx
, bucket_oid
, from
, to
));
1133 // bi list should be the same
1135 list
<rgw_cls_bi_entry
> entries
;
1136 bool truncated
{false};
1137 ASSERT_EQ(0, cls_rgw_bi_list(ioctx
, bucket_oid
, "", "", 128,
1138 &entries
, &truncated
));
1139 EXPECT_EQ(40u, entries
.size());
1140 EXPECT_FALSE(truncated
);