1 // -*- mode:C; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/int_types.h"
5 #include "include/types.h"
12 #include "common/ceph_time.h"
13 #include "objclass/objclass.h"
14 #include "cls/rgw/cls_rgw_ops.h"
15 #include "cls/rgw/cls_rgw_const.h"
16 #include "common/Clock.h"
17 #include "common/strtol.h"
18 #include "common/escape.h"
20 #include "global/global_context.h"
21 #include "include/compat.h"
27 #define BI_PREFIX_CHAR 0x80
29 #define BI_BUCKET_OBJS_INDEX 0
30 #define BI_BUCKET_LOG_INDEX 1
31 #define BI_BUCKET_OBJ_INSTANCE_INDEX 2
32 #define BI_BUCKET_OLH_DATA_INDEX 3
34 #define BI_BUCKET_LAST_INDEX 4
36 static string bucket_index_prefixes
[] = { "", /* special handling for the objs list index */
37 "0_", /* bucket log index */
38 "1000_", /* obj instance index */
39 "1001_", /* olh data index */
41 /* this must be the last index */
44 static bool bi_is_objs_index(const string
& s
) {
45 return ((unsigned char)s
[0] != BI_PREFIX_CHAR
);
48 int bi_entry_type(const string
& s
)
50 if (bi_is_objs_index(s
)) {
51 return BI_BUCKET_OBJS_INDEX
;
55 i
< sizeof(bucket_index_prefixes
) / sizeof(bucket_index_prefixes
[0]);
57 const string
& t
= bucket_index_prefixes
[i
];
59 if (s
.compare(1, t
.size(), t
) == 0) {
67 static bool bi_entry_gt(const string
& first
, const string
& second
)
69 int fi
= bi_entry_type(first
);
70 int si
= bi_entry_type(second
);
78 return first
> second
;
81 static void get_time_key(real_time
& ut
, string
*key
)
84 ceph_timespec ts
= ceph::real_clock::to_ceph_timespec(ut
);
85 snprintf(buf
, 32, "%011llu.%09u", (unsigned long long)ts
.tv_sec
, (unsigned int)ts
.tv_nsec
);
89 static void get_index_ver_key(cls_method_context_t hctx
, uint64_t index_ver
, string
*key
)
92 snprintf(buf
, sizeof(buf
), "%011llu.%llu.%d", (unsigned long long)index_ver
,
93 (unsigned long long)cls_current_version(hctx
),
94 cls_current_subop_num(hctx
));
98 static void bi_log_prefix(string
& key
)
100 key
= BI_PREFIX_CHAR
;
101 key
.append(bucket_index_prefixes
[BI_BUCKET_LOG_INDEX
]);
104 static void bi_log_index_key(cls_method_context_t hctx
, string
& key
, string
& id
, uint64_t index_ver
)
107 get_index_ver_key(hctx
, index_ver
, &id
);
111 static int log_index_operation(cls_method_context_t hctx
, cls_rgw_obj_key
& obj_key
, RGWModifyOp op
,
112 string
& tag
, real_time
& timestamp
,
113 rgw_bucket_entry_ver
& ver
, RGWPendingState state
, uint64_t index_ver
,
114 string
& max_marker
, uint16_t bilog_flags
, string
*owner
, string
*owner_display_name
)
118 struct rgw_bi_log_entry entry
;
120 entry
.object
= obj_key
.name
;
121 entry
.instance
= obj_key
.instance
;
122 entry
.timestamp
= timestamp
;
126 entry
.index_ver
= index_ver
;
128 entry
.bilog_flags
= bilog_flags
;
130 entry
.owner
= *owner
;
132 if (owner_display_name
) {
133 entry
.owner_display_name
= *owner_display_name
;
137 bi_log_index_key(hctx
, key
, entry
.id
, index_ver
);
141 if (entry
.id
> max_marker
)
142 max_marker
= entry
.id
;
144 return cls_cxx_map_set_val(hctx
, key
, &bl
);
148 * read list of objects, skips objects in the ugly namespace
150 static int get_obj_vals(cls_method_context_t hctx
, const string
& start
, const string
& filter_prefix
,
151 int num_entries
, map
<string
, bufferlist
> *pkeys
)
153 int ret
= cls_cxx_map_get_vals(hctx
, start
, filter_prefix
, num_entries
, pkeys
);
160 map
<string
, bufferlist
>::reverse_iterator last_element
= pkeys
->rbegin();
161 if ((unsigned char)last_element
->first
[0] < BI_PREFIX_CHAR
) {
162 /* nothing to see here, move along */
166 map
<string
, bufferlist
>::iterator first_element
= pkeys
->begin();
167 if ((unsigned char)first_element
->first
[0] > BI_PREFIX_CHAR
) {
171 /* let's rebuild the list, only keep entries we're interested in */
172 map
<string
, bufferlist
> old_keys
;
173 old_keys
.swap(*pkeys
);
175 for (map
<string
, bufferlist
>::iterator iter
= old_keys
.begin(); iter
!= old_keys
.end(); ++iter
) {
176 if ((unsigned char)iter
->first
[0] != BI_PREFIX_CHAR
) {
177 (*pkeys
)[iter
->first
] = iter
->second
;
181 if (num_entries
== (int)pkeys
->size())
184 map
<string
, bufferlist
> new_keys
;
185 char c
[] = { (char)(BI_PREFIX_CHAR
+ 1), 0 };
186 string new_start
= c
;
188 /* now get some more keys */
189 ret
= cls_cxx_map_get_vals(hctx
, new_start
, filter_prefix
, num_entries
- pkeys
->size(), &new_keys
);
193 for (map
<string
, bufferlist
>::iterator iter
= new_keys
.begin(); iter
!= new_keys
.end(); ++iter
) {
194 (*pkeys
)[iter
->first
] = iter
->second
;
201 * get a monotonically decreasing string representation.
202 * For num = x, num = y, where x > y, str(x) < str(y)
203 * Another property is that string size starts short and grows as num increases
205 static void decreasing_str(uint64_t num
, string
*str
)
208 if (num
< 0x10) { /* 16 */
209 snprintf(buf
, sizeof(buf
), "9%02lld", 15 - (long long)num
);
210 } else if (num
< 0x100) { /* 256 */
211 snprintf(buf
, sizeof(buf
), "8%03lld", 255 - (long long)num
);
212 } else if (num
< 0x1000) /* 4096 */ {
213 snprintf(buf
, sizeof(buf
), "7%04lld", 4095 - (long long)num
);
214 } else if (num
< 0x10000) /* 65536 */ {
215 snprintf(buf
, sizeof(buf
), "6%05lld", 65535 - (long long)num
);
216 } else if (num
< 0x100000000) /* 4G */ {
217 snprintf(buf
, sizeof(buf
), "5%010lld", 0xFFFFFFFF - (long long)num
);
219 snprintf(buf
, sizeof(buf
), "4%020lld", (long long)-num
);
226 * we now hold two different indexes for objects. The first one holds the list of objects in the
227 * order that we want them to be listed. The second one only holds the objects instances (for
228 * versioned objects), and they're not arranged in any particular order.
229 * When listing objects we'll use the first index, when doing operations on the objects themselves
230 * we'll use the second index. Note that regular objects only map to the first index anyway
233 static void get_list_index_key(struct rgw_bucket_dir_entry
& entry
, string
*index_key
)
235 *index_key
= entry
.key
.name
;
238 decreasing_str(entry
.versioned_epoch
, &ver_str
);
239 string
instance_delim("\0i", 2);
240 string
ver_delim("\0v", 2);
242 index_key
->append(ver_delim
);
243 index_key
->append(ver_str
);
244 index_key
->append(instance_delim
);
245 index_key
->append(entry
.key
.instance
);
248 static void encode_obj_versioned_data_key(const cls_rgw_obj_key
& key
, string
*index_key
, bool append_delete_marker_suffix
= false)
250 *index_key
= BI_PREFIX_CHAR
;
251 index_key
->append(bucket_index_prefixes
[BI_BUCKET_OBJ_INSTANCE_INDEX
]);
252 index_key
->append(key
.name
);
253 string
delim("\0i", 2);
254 index_key
->append(delim
);
255 index_key
->append(key
.instance
);
256 if (append_delete_marker_suffix
) {
258 index_key
->append(dm
);
262 static void encode_obj_index_key(const cls_rgw_obj_key
& key
, string
*index_key
)
264 if (key
.instance
.empty()) {
265 *index_key
= key
.name
;
267 encode_obj_versioned_data_key(key
, index_key
);
271 static void encode_olh_data_key(const cls_rgw_obj_key
& key
, string
*index_key
)
273 *index_key
= BI_PREFIX_CHAR
;
274 index_key
->append(bucket_index_prefixes
[BI_BUCKET_OLH_DATA_INDEX
]);
275 index_key
->append(key
.name
);
279 static int read_index_entry(cls_method_context_t hctx
, string
& name
, T
*entry
);
281 static int encode_list_index_key(cls_method_context_t hctx
, const cls_rgw_obj_key
& key
, string
*index_key
)
283 if (key
.instance
.empty()) {
284 *index_key
= key
.name
;
288 string obj_index_key
;
289 encode_obj_index_key(key
, &obj_index_key
);
291 rgw_bucket_dir_entry entry
;
293 int ret
= read_index_entry(hctx
, obj_index_key
, &entry
);
294 if (ret
== -ENOENT
) {
295 /* couldn't find the entry, set key value after the current object */
296 char buf
[2] = { 0x1, 0 };
298 *index_key
= key
.name
+ s
;
302 CLS_LOG(1, "ERROR: encode_list_index_key(): cls_cxx_map_get_val returned %d\n", ret
);
306 get_list_index_key(entry
, index_key
);
311 static void split_key(const string
& key
, list
<string
>& vals
)
314 const char *p
= key
.c_str();
315 while (pos
< key
.size()) {
316 size_t len
= strlen(p
);
324 * list index key structure:
326 * <obj name>\0[v<ver>\0i<instance id>]
328 static void decode_list_index_key(const string
& index_key
, cls_rgw_obj_key
*key
, uint64_t *ver
)
330 size_t len
= strlen(index_key
.c_str());
332 key
->instance
.clear();
335 if (len
== index_key
.size()) {
336 key
->name
= index_key
;
341 split_key(index_key
, vals
);
343 assert(!vals
.empty());
345 list
<string
>::iterator iter
= vals
.begin();
349 assert(iter
!= vals
.end());
351 for (; iter
!= vals
.end(); ++iter
) {
354 key
->instance
= val
.substr(1);
355 } else if (val
[0] == 'v') {
357 const char *s
= val
.c_str() + 1;
358 *ver
= strict_strtoll(s
, 10, &err
);
364 static int read_bucket_header(cls_method_context_t hctx
, struct rgw_bucket_dir_header
*header
)
367 int rc
= cls_cxx_map_read_header(hctx
, &bl
);
370 bufferlist::iterator iter
= bl
.begin();
372 ::decode(*header
, iter
);
373 } catch (buffer::error
& err
) {
374 CLS_LOG(1, "ERROR: read_bucket_header(): failed to decode header\n");
381 int rgw_bucket_list(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
383 bufferlist::iterator iter
= in
->begin();
385 struct rgw_cls_list_op op
;
388 } catch (buffer::error
& err
) {
389 CLS_LOG(1, "ERROR: rgw_bucket_list(): failed to decode request\n");
393 struct rgw_cls_list_ret ret
;
394 struct rgw_bucket_dir
& new_dir
= ret
.dir
;
395 int rc
= read_bucket_header(hctx
, &new_dir
.header
);
397 CLS_LOG(1, "ERROR: rgw_bucket_list(): failed to read header\n");
401 map
<string
, bufferlist
> keys
;
402 std::map
<string
, bufferlist
>::iterator kiter
;
404 encode_list_index_key(hctx
, op
.start_obj
, &start_key
);
406 uint32_t left_to_read
= op
.num_entries
+ 1;
409 rc
= get_obj_vals(hctx
, start_key
, op
.filter_prefix
, left_to_read
, &keys
);
413 std::map
<string
, struct rgw_bucket_dir_entry
>& m
= new_dir
.m
;
417 for (kiter
= keys
.begin(); kiter
!= keys
.end(); ++kiter
) {
418 struct rgw_bucket_dir_entry entry
;
420 if (!bi_is_objs_index(kiter
->first
)) {
425 bufferlist
& entrybl
= kiter
->second
;
426 bufferlist::iterator eiter
= entrybl
.begin();
428 ::decode(entry
, eiter
);
429 } catch (buffer::error
& err
) {
430 CLS_LOG(1, "ERROR: rgw_bucket_list(): failed to decode entry, key=%s\n", kiter
->first
.c_str());
436 decode_list_index_key(kiter
->first
, &key
, &ver
);
438 start_key
= kiter
->first
;
439 CLS_LOG(20, "start_key=%s len=%zu", start_key
.c_str(), start_key
.size());
441 if (!entry
.is_valid()) {
442 CLS_LOG(20, "entry %s[%s] is not valid\n", key
.name
.c_str(), key
.instance
.c_str());
446 if (!op
.list_versions
&& !entry
.is_visible()) {
447 CLS_LOG(20, "entry %s[%s] is not visible\n", key
.name
.c_str(), key
.instance
.c_str());
450 if (m
.size() < op
.num_entries
) {
451 m
[kiter
->first
] = entry
;
455 CLS_LOG(20, "got entry %s[%s] m.size()=%d\n", key
.name
.c_str(), key
.instance
.c_str(), (int)m
.size());
457 } while (left_to_read
> 0 && !done
);
459 ret
.is_truncated
= (left_to_read
== 0) && /* we found more entries than we were requested, meaning response is truncated */
466 static int check_index(cls_method_context_t hctx
, struct rgw_bucket_dir_header
*existing_header
, struct rgw_bucket_dir_header
*calc_header
)
468 int rc
= read_bucket_header(hctx
, existing_header
);
470 CLS_LOG(1, "ERROR: check_index(): failed to read header\n");
474 calc_header
->tag_timeout
= existing_header
->tag_timeout
;
475 calc_header
->ver
= existing_header
->ver
;
477 map
<string
, bufferlist
> keys
;
479 string filter_prefix
;
481 #define CHECK_CHUNK_SIZE 1000
485 rc
= get_obj_vals(hctx
, start_obj
, filter_prefix
, CHECK_CHUNK_SIZE
, &keys
);
489 std::map
<string
, bufferlist
>::iterator kiter
= keys
.begin();
490 for (; kiter
!= keys
.end(); ++kiter
) {
491 if (!bi_is_objs_index(kiter
->first
)) {
496 struct rgw_bucket_dir_entry entry
;
497 bufferlist::iterator eiter
= kiter
->second
.begin();
499 ::decode(entry
, eiter
);
500 } catch (buffer::error
& err
) {
501 CLS_LOG(1, "ERROR: rgw_bucket_list(): failed to decode entry, key=%s\n", kiter
->first
.c_str());
504 struct rgw_bucket_category_stats
& stats
= calc_header
->stats
[entry
.meta
.category
];
506 stats
.total_size
+= entry
.meta
.accounted_size
;
507 stats
.total_size_rounded
+= cls_rgw_get_rounded_size(entry
.meta
.accounted_size
);
508 stats
.actual_size
+= entry
.meta
.size
;
510 start_obj
= kiter
->first
;
512 } while (keys
.size() == CHECK_CHUNK_SIZE
&& !done
);
517 int rgw_bucket_check_index(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
519 struct rgw_cls_check_index_ret ret
;
521 int rc
= check_index(hctx
, &ret
.existing_header
, &ret
.calculated_header
);
530 static int write_bucket_header(cls_method_context_t hctx
, struct rgw_bucket_dir_header
*header
)
534 bufferlist header_bl
;
535 ::encode(*header
, header_bl
);
536 return cls_cxx_map_write_header(hctx
, &header_bl
);
540 int rgw_bucket_rebuild_index(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
542 struct rgw_bucket_dir_header existing_header
;
543 struct rgw_bucket_dir_header calc_header
;
544 int rc
= check_index(hctx
, &existing_header
, &calc_header
);
548 return write_bucket_header(hctx
, &calc_header
);
551 int rgw_bucket_update_stats(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
554 rgw_cls_bucket_update_stats_op op
;
555 auto iter
= in
->begin();
558 } catch (buffer::error
& err
) {
559 CLS_LOG(1, "ERROR: %s(): failed to decode request\n", __func__
);
563 struct rgw_bucket_dir_header header
;
564 int rc
= read_bucket_header(hctx
, &header
);
566 CLS_LOG(1, "ERROR: %s(): failed to read header\n", __func__
);
570 for (auto& s
: op
.stats
) {
571 auto& dest
= header
.stats
[s
.first
];
575 dest
.total_size
+= s
.second
.total_size
;
576 dest
.total_size_rounded
+= s
.second
.total_size_rounded
;
577 dest
.num_entries
+= s
.second
.num_entries
;
581 return write_bucket_header(hctx
, &header
);
584 int rgw_bucket_init_index(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
586 bufferlist::iterator iter
;
588 bufferlist header_bl
;
589 int rc
= cls_cxx_map_read_header(hctx
, &header_bl
);
600 if (header_bl
.length() != 0) {
601 CLS_LOG(1, "ERROR: index already initialized\n");
607 return write_bucket_header(hctx
, &dir
.header
);
610 int rgw_bucket_set_tag_timeout(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
613 rgw_cls_tag_timeout_op op
;
614 bufferlist::iterator iter
= in
->begin();
617 } catch (buffer::error
& err
) {
618 CLS_LOG(1, "ERROR: rgw_bucket_set_tag_timeout(): failed to decode request\n");
622 struct rgw_bucket_dir_header header
;
623 int rc
= read_bucket_header(hctx
, &header
);
625 CLS_LOG(1, "ERROR: rgw_bucket_set_tag_timeout(): failed to read header\n");
629 header
.tag_timeout
= op
.tag_timeout
;
631 return write_bucket_header(hctx
, &header
);
634 static int read_key_entry(cls_method_context_t hctx
, cls_rgw_obj_key
& key
, string
*idx
, struct rgw_bucket_dir_entry
*entry
,
635 bool special_delete_marker_name
= false);
637 int rgw_bucket_prepare_op(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
640 rgw_cls_obj_prepare_op op
;
641 bufferlist::iterator iter
= in
->begin();
644 } catch (buffer::error
& err
) {
645 CLS_LOG(1, "ERROR: rgw_bucket_prepare_op(): failed to decode request\n");
649 if (op
.tag
.empty()) {
650 CLS_LOG(1, "ERROR: tag is empty\n");
654 CLS_LOG(1, "rgw_bucket_prepare_op(): request: op=%d name=%s instance=%s tag=%s\n",
655 op
.op
, op
.key
.name
.c_str(), op
.key
.instance
.c_str(), op
.tag
.c_str());
660 struct rgw_bucket_dir_entry entry
;
661 int rc
= read_key_entry(hctx
, op
.key
, &idx
, &entry
);
662 if (rc
< 0 && rc
!= -ENOENT
)
665 bool noent
= (rc
== -ENOENT
);
669 if (noent
) { // no entry, initialize fields
671 entry
.ver
= rgw_bucket_entry_ver();
672 entry
.exists
= false;
673 entry
.locator
= op
.locator
;
676 // fill in proper state
677 struct rgw_bucket_pending_info info
;
678 info
.timestamp
= real_clock::now();
679 info
.state
= CLS_RGW_STATE_PENDING_MODIFY
;
681 entry
.pending_map
.insert(pair
<string
, rgw_bucket_pending_info
>(op
.tag
, info
));
683 struct rgw_bucket_dir_header header
;
684 rc
= read_bucket_header(hctx
, &header
);
686 CLS_LOG(1, "ERROR: rgw_bucket_prepare_op(): failed to read header\n");
691 rc
= log_index_operation(hctx
, op
.key
, op
.op
, op
.tag
, entry
.meta
.mtime
,
692 entry
.ver
, info
.state
, header
.ver
, header
.max_marker
, op
.bilog_flags
, NULL
, NULL
);
697 // write out new key to disk
699 ::encode(entry
, info_bl
);
700 rc
= cls_cxx_map_set_val(hctx
, idx
, &info_bl
);
704 return write_bucket_header(hctx
, &header
);
707 static void unaccount_entry(struct rgw_bucket_dir_header
& header
, struct rgw_bucket_dir_entry
& entry
)
709 struct rgw_bucket_category_stats
& stats
= header
.stats
[entry
.meta
.category
];
711 stats
.total_size
-= entry
.meta
.accounted_size
;
712 stats
.total_size_rounded
-= cls_rgw_get_rounded_size(entry
.meta
.accounted_size
);
713 stats
.actual_size
-= entry
.meta
.size
;
716 static void log_entry(const char *func
, const char *str
, struct rgw_bucket_dir_entry
*entry
)
718 CLS_LOG(1, "%s(): %s: ver=%ld:%llu name=%s instance=%s locator=%s\n", func
, str
,
719 (long)entry
->ver
.pool
, (unsigned long long)entry
->ver
.epoch
,
720 entry
->key
.name
.c_str(), entry
->key
.instance
.c_str(), entry
->locator
.c_str());
723 static void log_entry(const char *func
, const char *str
, struct rgw_bucket_olh_entry
*entry
)
725 CLS_LOG(1, "%s(): %s: epoch=%llu name=%s instance=%s tag=%s\n", func
, str
,
726 (unsigned long long)entry
->epoch
, entry
->key
.name
.c_str(), entry
->key
.instance
.c_str(),
731 static int read_index_entry(cls_method_context_t hctx
, string
& name
, T
*entry
)
733 bufferlist current_entry
;
734 int rc
= cls_cxx_map_get_val(hctx
, name
, ¤t_entry
);
739 bufferlist::iterator cur_iter
= current_entry
.begin();
741 ::decode(*entry
, cur_iter
);
742 } catch (buffer::error
& err
) {
743 CLS_LOG(1, "ERROR: read_index_entry(): failed to decode entry\n");
747 log_entry(__func__
, "existing entry", entry
);
751 static int read_key_entry(cls_method_context_t hctx
, cls_rgw_obj_key
& key
, string
*idx
, struct rgw_bucket_dir_entry
*entry
,
752 bool special_delete_marker_name
)
754 encode_obj_index_key(key
, idx
);
755 int rc
= read_index_entry(hctx
, *idx
, entry
);
760 if (key
.instance
.empty() &&
761 entry
->flags
& RGW_BUCKET_DIRENT_FLAG_VER_MARKER
) {
762 /* we only do it where key.instance is empty. In this case the delete marker will have a
763 * separate entry in the index to avoid collisions with the actual object, as it's mutable
765 if (special_delete_marker_name
) {
766 encode_obj_versioned_data_key(key
, idx
, true);
767 rc
= read_index_entry(hctx
, *idx
, entry
);
772 encode_obj_versioned_data_key(key
, idx
);
773 rc
= read_index_entry(hctx
, *idx
, entry
);
775 *entry
= rgw_bucket_dir_entry(); /* need to reset entry because we initialized it earlier */
783 int rgw_bucket_complete_op(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
786 rgw_cls_obj_complete_op op
;
787 bufferlist::iterator iter
= in
->begin();
790 } catch (buffer::error
& err
) {
791 CLS_LOG(1, "ERROR: rgw_bucket_complete_op(): failed to decode request\n");
794 CLS_LOG(1, "rgw_bucket_complete_op(): request: op=%d name=%s instance=%s ver=%lu:%llu tag=%s\n",
795 op
.op
, op
.key
.name
.c_str(), op
.key
.instance
.c_str(),
796 (unsigned long)op
.ver
.pool
, (unsigned long long)op
.ver
.epoch
,
799 struct rgw_bucket_dir_header header
;
800 int rc
= read_bucket_header(hctx
, &header
);
802 CLS_LOG(1, "ERROR: rgw_bucket_complete_op(): failed to read header\n");
806 struct rgw_bucket_dir_entry entry
;
810 rc
= read_key_entry(hctx
, op
.key
, &idx
, &entry
);
814 entry
.meta
= op
.meta
;
815 entry
.locator
= op
.locator
;
821 entry
.index_ver
= header
.ver
;
822 entry
.flags
= (entry
.key
.instance
.empty() ? 0 : RGW_BUCKET_DIRENT_FLAG_VER
); /* resetting entry flags, entry might have been previously a delete marker */
825 map
<string
, struct rgw_bucket_pending_info
>::iterator pinter
= entry
.pending_map
.find(op
.tag
);
826 if (pinter
== entry
.pending_map
.end()) {
827 CLS_LOG(1, "ERROR: couldn't find tag for pending operation\n");
830 entry
.pending_map
.erase(pinter
);
834 bufferlist update_bl
;
836 if (op
.tag
.size() && op
.op
== CLS_RGW_OP_CANCEL
) {
837 CLS_LOG(1, "rgw_bucket_complete_op(): cancel requested\n");
839 } else if (op
.ver
.pool
== entry
.ver
.pool
&&
840 op
.ver
.epoch
&& op
.ver
.epoch
<= entry
.ver
.epoch
) {
841 CLS_LOG(1, "rgw_bucket_complete_op(): skipping request, old epoch\n");
848 rc
= log_index_operation(hctx
, op
.key
, op
.op
, op
.tag
, entry
.meta
.mtime
, entry
.ver
,
849 CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
, op
.bilog_flags
, NULL
, NULL
);
855 bufferlist new_key_bl
;
856 ::encode(entry
, new_key_bl
);
857 return cls_cxx_map_set_val(hctx
, idx
, &new_key_bl
);
864 unaccount_entry(header
, entry
);
868 switch ((int)op
.op
) {
870 entry
.meta
= op
.meta
;
872 if (!entry
.pending_map
.size()) {
873 int ret
= cls_cxx_map_remove_key(hctx
, idx
);
877 entry
.exists
= false;
878 bufferlist new_key_bl
;
879 ::encode(entry
, new_key_bl
);
880 int ret
= cls_cxx_map_set_val(hctx
, idx
, &new_key_bl
);
890 struct rgw_bucket_dir_entry_meta
& meta
= op
.meta
;
891 struct rgw_bucket_category_stats
& stats
= header
.stats
[meta
.category
];
897 stats
.total_size
+= meta
.accounted_size
;
898 stats
.total_size_rounded
+= cls_rgw_get_rounded_size(meta
.accounted_size
);
899 stats
.actual_size
+= meta
.size
;
900 bufferlist new_key_bl
;
901 ::encode(entry
, new_key_bl
);
902 int ret
= cls_cxx_map_set_val(hctx
, idx
, &new_key_bl
);
910 rc
= log_index_operation(hctx
, op
.key
, op
.op
, op
.tag
, entry
.meta
.mtime
, entry
.ver
,
911 CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
, op
.bilog_flags
, NULL
, NULL
);
916 list
<cls_rgw_obj_key
>::iterator remove_iter
;
917 CLS_LOG(20, "rgw_bucket_complete_op(): remove_objs.size()=%d\n", (int)op
.remove_objs
.size());
918 for (remove_iter
= op
.remove_objs
.begin(); remove_iter
!= op
.remove_objs
.end(); ++remove_iter
) {
919 cls_rgw_obj_key
& remove_key
= *remove_iter
;
920 CLS_LOG(1, "rgw_bucket_complete_op(): removing entries, read_index_entry name=%s instance=%s\n",
921 remove_key
.name
.c_str(), remove_key
.instance
.c_str());
922 struct rgw_bucket_dir_entry remove_entry
;
924 int ret
= read_key_entry(hctx
, remove_key
, &k
, &remove_entry
);
926 CLS_LOG(1, "rgw_bucket_complete_op(): removing entries, read_index_entry name=%s instance=%s ret=%d\n",
927 remove_key
.name
.c_str(), remove_key
.instance
.c_str(), ret
);
930 CLS_LOG(0, "rgw_bucket_complete_op(): entry.name=%s entry.instance=%s entry.meta.category=%d\n",
931 remove_entry
.key
.name
.c_str(), remove_entry
.key
.instance
.c_str(), remove_entry
.meta
.category
);
932 unaccount_entry(header
, remove_entry
);
935 rc
= log_index_operation(hctx
, remove_key
, CLS_RGW_OP_DEL
, op
.tag
, remove_entry
.meta
.mtime
,
936 remove_entry
.ver
, CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
, op
.bilog_flags
, NULL
, NULL
);
941 ret
= cls_cxx_map_remove_key(hctx
, k
);
943 CLS_LOG(1, "rgw_bucket_complete_op(): cls_cxx_map_remove_key, failed to remove entry, name=%s instance=%s read_index_entry ret=%d\n", remove_key
.name
.c_str(), remove_key
.instance
.c_str(), rc
);
948 return write_bucket_header(hctx
, &header
);
952 static int write_entry(cls_method_context_t hctx
, T
& entry
, const string
& key
)
956 return cls_cxx_map_set_val(hctx
, key
, &bl
);
959 static int read_olh(cls_method_context_t hctx
,cls_rgw_obj_key
& obj_key
, struct rgw_bucket_olh_entry
*olh_data_entry
, string
*index_key
, bool *found
)
961 cls_rgw_obj_key olh_key
;
962 olh_key
.name
= obj_key
.name
;
964 encode_olh_data_key(olh_key
, index_key
);
965 int ret
= read_index_entry(hctx
, *index_key
, olh_data_entry
);
966 if (ret
< 0 && ret
!= -ENOENT
) {
967 CLS_LOG(0, "ERROR: read_index_entry() olh_key=%s ret=%d", olh_key
.name
.c_str(), ret
);
971 *found
= (ret
!= -ENOENT
);
976 static void update_olh_log(struct rgw_bucket_olh_entry
& olh_data_entry
, OLHLogOp op
, const string
& op_tag
,
977 cls_rgw_obj_key
& key
, bool delete_marker
, uint64_t epoch
)
979 vector
<rgw_bucket_olh_log_entry
>& log
= olh_data_entry
.pending_log
[olh_data_entry
.epoch
];
980 rgw_bucket_olh_log_entry log_entry
;
981 log_entry
.epoch
= epoch
;
983 log_entry
.op_tag
= op_tag
;
985 log_entry
.delete_marker
= delete_marker
;
986 log
.push_back(log_entry
);
989 static string
escape_str(const string
& s
)
991 int len
= escape_json_attr_len(s
.c_str(), s
.size());
993 escape_json_attr(s
.c_str(), s
.size(), escaped
);
994 return string(escaped
);
997 static int write_obj_instance_entry(cls_method_context_t hctx
, struct rgw_bucket_dir_entry
& instance_entry
, const string
& instance_idx
)
999 CLS_LOG(20, "write_entry() instance=%s idx=%s flags=%d", escape_str(instance_entry
.key
.instance
).c_str(), instance_idx
.c_str(), instance_entry
.flags
);
1000 /* write the instance entry */
1001 int ret
= write_entry(hctx
, instance_entry
, instance_idx
);
1003 CLS_LOG(0, "ERROR: write_entry() instance_key=%s ret=%d", escape_str(instance_idx
).c_str(), ret
);
1010 * write object instance entry, and if needed also the list entry
1012 static int write_obj_entries(cls_method_context_t hctx
, struct rgw_bucket_dir_entry
& instance_entry
, const string
& instance_idx
)
1014 int ret
= write_obj_instance_entry(hctx
, instance_entry
, instance_idx
);
1018 string instance_list_idx
;
1019 get_list_index_key(instance_entry
, &instance_list_idx
);
1021 if (instance_idx
!= instance_list_idx
) {
1022 CLS_LOG(20, "write_entry() idx=%s flags=%d", escape_str(instance_list_idx
).c_str(), instance_entry
.flags
);
1023 /* write a new list entry for the object instance */
1024 ret
= write_entry(hctx
, instance_entry
, instance_list_idx
);
1026 CLS_LOG(0, "ERROR: write_entry() instance=%s instance_list_idx=%s ret=%d", instance_entry
.key
.instance
.c_str(), instance_list_idx
.c_str(), ret
);
1034 class BIVerObjEntry
{
1035 cls_method_context_t hctx
;
1036 cls_rgw_obj_key key
;
1037 string instance_idx
;
1039 struct rgw_bucket_dir_entry instance_entry
;
1044 BIVerObjEntry(cls_method_context_t
& _hctx
, const cls_rgw_obj_key
& _key
) : hctx(_hctx
), key(_key
), initialized(false) {
1047 int init(bool check_delete_marker
= true) {
1048 int ret
= read_key_entry(hctx
, key
, &instance_idx
, &instance_entry
,
1049 check_delete_marker
&& key
.instance
.empty()); /* this is potentially a delete marker, for null objects we
1050 keep separate instance entry for the delete markers */
1053 CLS_LOG(0, "ERROR: read_key_entry() idx=%s ret=%d", instance_idx
.c_str(), ret
);
1057 CLS_LOG(20, "read instance_entry key.name=%s key.instance=%s flags=%d", instance_entry
.key
.name
.c_str(), instance_entry
.key
.instance
.c_str(), instance_entry
.flags
);
1061 rgw_bucket_dir_entry
& get_dir_entry() {
1062 return instance_entry
;
1065 void init_as_delete_marker(rgw_bucket_dir_entry_meta
& meta
) {
1066 /* a deletion marker, need to initialize it, there's no instance entry for it yet */
1067 instance_entry
.key
= key
;
1068 instance_entry
.flags
= RGW_BUCKET_DIRENT_FLAG_DELETE_MARKER
;
1069 instance_entry
.meta
= meta
;
1070 instance_entry
.tag
= "delete-marker";
1075 void set_epoch(uint64_t epoch
) {
1076 instance_entry
.versioned_epoch
= epoch
;
1079 int unlink_list_entry() {
1081 /* this instance has a previous list entry, remove that entry */
1082 get_list_index_key(instance_entry
, &list_idx
);
1083 CLS_LOG(20, "unlink_list_entry() list_idx=%s", escape_str(list_idx
).c_str());
1084 int ret
= cls_cxx_map_remove_key(hctx
, list_idx
);
1086 CLS_LOG(0, "ERROR: cls_cxx_map_remove_key() list_idx=%s ret=%d", list_idx
.c_str(), ret
);
1093 /* remove the instance entry */
1094 CLS_LOG(20, "unlink() idx=%s", escape_str(instance_idx
).c_str());
1095 int ret
= cls_cxx_map_remove_key(hctx
, instance_idx
);
1097 CLS_LOG(0, "ERROR: cls_cxx_map_remove_key() instance_idx=%s ret=%d", instance_idx
.c_str(), ret
);
1103 int write_entries(uint64_t flags_set
, uint64_t flags_reset
) {
1110 instance_entry
.flags
&= ~flags_reset
;
1111 instance_entry
.flags
|= flags_set
;
1113 /* write the instance and list entries */
1114 bool special_delete_marker_key
= (instance_entry
.is_delete_marker() && instance_entry
.key
.instance
.empty());
1115 encode_obj_versioned_data_key(key
, &instance_idx
, special_delete_marker_key
);
1116 int ret
= write_obj_entries(hctx
, instance_entry
, instance_idx
);
1118 CLS_LOG(0, "ERROR: write_obj_entries() instance_idx=%s ret=%d", instance_idx
.c_str(), ret
);
1125 int write(uint64_t epoch
, bool current
) {
1126 if (instance_entry
.versioned_epoch
> 0) {
1127 CLS_LOG(20, "%s(): instance_entry.versioned_epoch=%d epoch=%d", __func__
, (int)instance_entry
.versioned_epoch
, (int)epoch
);
1128 /* this instance has a previous list entry, remove that entry */
1129 int ret
= unlink_list_entry();
1135 uint64_t flags
= RGW_BUCKET_DIRENT_FLAG_VER
;
1137 flags
|= RGW_BUCKET_DIRENT_FLAG_CURRENT
;
1140 instance_entry
.versioned_epoch
= epoch
;
1141 return write_entries(flags
, 0);
1144 int demote_current() {
1145 return write_entries(0, RGW_BUCKET_DIRENT_FLAG_CURRENT
);
1148 bool is_delete_marker() {
1149 return instance_entry
.is_delete_marker();
1152 int find_next_key(cls_rgw_obj_key
*next_key
, bool *found
) {
1154 /* this instance has a previous list entry, remove that entry */
1155 get_list_index_key(instance_entry
, &list_idx
);
1156 /* this is the current head, need to update! */
1157 map
<string
, bufferlist
> keys
;
1158 string filter
= key
.name
; /* list key starts with key name, filter it to avoid a case where we cross to
1159 different namespace */
1160 int ret
= cls_cxx_map_get_vals(hctx
, list_idx
, filter
, 1, &keys
);
1165 if (keys
.size() < 1) {
1170 rgw_bucket_dir_entry next_entry
;
1172 map
<string
, bufferlist
>::reverse_iterator last
= keys
.rbegin();
1174 bufferlist::iterator iter
= last
->second
.begin();
1175 ::decode(next_entry
, iter
);
1176 } catch (buffer::error
& err
) {
1177 CLS_LOG(0, "ERROR; failed to decode entry: %s", last
->first
.c_str());
1181 *found
= (key
.name
== next_entry
.key
.name
);
1183 *next_key
= next_entry
.key
;
1190 return instance_entry
.meta
.mtime
;
1196 cls_method_context_t hctx
;
1197 cls_rgw_obj_key key
;
1199 string olh_data_idx
;
1200 struct rgw_bucket_olh_entry olh_data_entry
;
1204 BIOLHEntry(cls_method_context_t
& _hctx
, const cls_rgw_obj_key
& _key
) : hctx(_hctx
), key(_key
), initialized(false) { }
1206 int init(bool *exists
) {
1208 int ret
= read_olh(hctx
, key
, &olh_data_entry
, &olh_data_idx
, exists
);
1217 bool apply_epoch(uint64_t candidate_epoch
) {
1218 if (candidate_epoch
< olh_data_entry
.epoch
) {
1222 olh_data_entry
.epoch
= candidate_epoch
;
1226 bool start_modify(uint64_t candidate_epoch
) {
1227 if (candidate_epoch
) {
1228 if (candidate_epoch
< olh_data_entry
.epoch
) {
1229 return false; /* olh cannot be modified, old epoch */
1231 olh_data_entry
.epoch
= candidate_epoch
;
1233 if (olh_data_entry
.epoch
== 0) {
1234 olh_data_entry
.epoch
= 2; /* versioned epoch should start with 2, 1 is reserved to converted plain entries */
1236 olh_data_entry
.epoch
++;
1242 uint64_t get_epoch() {
1243 return olh_data_entry
.epoch
;
1246 rgw_bucket_olh_entry
& get_entry() {
1247 return olh_data_entry
;
1250 void update(cls_rgw_obj_key
& key
, bool delete_marker
) {
1251 olh_data_entry
.delete_marker
= delete_marker
;
1252 olh_data_entry
.key
= key
;
1256 /* write the olh data entry */
1257 int ret
= write_entry(hctx
, olh_data_entry
, olh_data_idx
);
1259 CLS_LOG(0, "ERROR: write_entry() olh_key=%s ret=%d", olh_data_idx
.c_str(), ret
);
1266 void update_log(OLHLogOp op
, const string
& op_tag
, cls_rgw_obj_key
& key
, bool delete_marker
, uint64_t epoch
= 0) {
1268 epoch
= olh_data_entry
.epoch
;
1270 update_olh_log(olh_data_entry
, op
, op_tag
, key
, delete_marker
, epoch
);
1273 bool exists() { return olh_data_entry
.exists
; }
1275 void set_exists(bool exists
) {
1276 olh_data_entry
.exists
= exists
;
1279 bool pending_removal() { return olh_data_entry
.pending_removal
; }
1281 void set_pending_removal(bool pending_removal
) {
1282 olh_data_entry
.pending_removal
= pending_removal
;
1285 const string
& get_tag() { return olh_data_entry
.tag
; }
1286 void set_tag(const string
& tag
) {
1287 olh_data_entry
.tag
= tag
;
1291 static int write_version_marker(cls_method_context_t hctx
, cls_rgw_obj_key
& key
)
1293 struct rgw_bucket_dir_entry entry
;
1295 entry
.flags
= RGW_BUCKET_DIRENT_FLAG_VER_MARKER
;
1296 int ret
= write_entry(hctx
, entry
, key
.name
);
1298 CLS_LOG(0, "ERROR: write_entry returned ret=%d", ret
);
1305 * plain entries are the ones who were created when bucket was not versioned,
1306 * if we override these objects, we need to convert these to versioned entries -- ones that have
1307 * both data entry, and listing key. Their version is going to be empty though
1309 static int convert_plain_entry_to_versioned(cls_method_context_t hctx
, cls_rgw_obj_key
& key
, bool demote_current
, bool instance_only
)
1311 if (!key
.instance
.empty()) {
1315 struct rgw_bucket_dir_entry entry
;
1318 int ret
= read_key_entry(hctx
, key
, &orig_idx
, &entry
);
1319 if (ret
!= -ENOENT
) {
1321 CLS_LOG(0, "ERROR: read_key_entry() returned ret=%d", ret
);
1325 entry
.versioned_epoch
= 1; /* converted entries are always 1 */
1326 entry
.flags
|= RGW_BUCKET_DIRENT_FLAG_VER
;
1328 if (demote_current
) {
1329 entry
.flags
&= ~RGW_BUCKET_DIRENT_FLAG_CURRENT
;
1333 encode_obj_versioned_data_key(key
, &new_idx
);
1335 if (instance_only
) {
1336 ret
= write_obj_instance_entry(hctx
, entry
, new_idx
);
1338 ret
= write_obj_entries(hctx
, entry
, new_idx
);
1341 CLS_LOG(0, "ERROR: write_obj_entries new_idx=%s returned %d", new_idx
.c_str(), ret
);
1346 ret
= write_version_marker(hctx
, key
);
1355 * link an object version to an olh, update the relevant index entries. It will also handle the
1356 * deletion marker case. We have a few entries that we need to take care of. For object 'foo',
1357 * instance BAR, we'd update the following (not actual encoding):
1358 * - olh data: [BI_BUCKET_OLH_DATA_INDEX]foo
1359 * - object instance data: [BI_BUCKET_OBJ_INSTANCE_INDEX]foo,BAR
1360 * - object instance list entry: foo,123,BAR
1362 * The instance list entry needs to be ordered by newer to older, so we generate an appropriate
1363 * number string that follows the name.
1364 * The top instance for each object is marked appropriately.
1365 * We generate instance entry for deletion markers here, as they are not created prior.
1367 static int rgw_bucket_link_olh(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1369 string olh_data_idx
;
1370 string instance_idx
;
1373 rgw_cls_link_olh_op op
;
1374 bufferlist::iterator iter
= in
->begin();
1377 } catch (buffer::error
& err
) {
1378 CLS_LOG(0, "ERROR: rgw_bucket_link_olh_op(): failed to decode request\n");
1382 BIVerObjEntry
obj(hctx
, op
.key
);
1383 BIOLHEntry
olh(hctx
, op
.key
);
1385 /* read instance entry */
1386 int ret
= obj
.init(op
.delete_marker
);
1387 bool existed
= (ret
== 0);
1388 if (ret
== -ENOENT
&& op
.delete_marker
) {
1395 if (existed
&& !real_clock::is_zero(op
.unmod_since
)) {
1396 struct timespec mtime
= ceph::real_clock::to_timespec(obj
.mtime());
1397 struct timespec unmod
= ceph::real_clock::to_timespec(op
.unmod_since
);
1398 if (!op
.high_precision_time
) {
1402 if (mtime
>= unmod
) {
1403 return 0; /* no need to set error, we just return 0 and avoid writing to the bi log */
1410 * Special handling for null instance object / delete-marker. For these objects we're going to
1411 * have separate instances for a data object vs. delete-marker to avoid collisions. We now check
1412 * if we got to overwrite a previous entry, and in that case we'll remove its list entry.
1414 if (op
.key
.instance
.empty()) {
1415 BIVerObjEntry
other_obj(hctx
, op
.key
);
1416 ret
= other_obj
.init(!op
.delete_marker
); /* try reading the other null versioned entry */
1417 existed
= (ret
>= 0 && !other_obj
.is_delete_marker());
1418 if (ret
>= 0 && other_obj
.is_delete_marker() != op
.delete_marker
) {
1419 ret
= other_obj
.unlink_list_entry();
1423 ret
= other_obj
.unlink();
1429 removing
= existed
&& op
.delete_marker
;
1431 removing
= (existed
&& !obj
.is_delete_marker() && op
.delete_marker
);
1434 if (op
.delete_marker
) {
1435 /* a deletion marker, need to initialize entry as such */
1436 obj
.init_as_delete_marker(op
.meta
);
1441 ret
= olh
.init(&olh_found
);
1446 if (!olh
.start_modify(op
.olh_epoch
)) {
1447 ret
= obj
.write(op
.olh_epoch
, false);
1452 olh
.update_log(CLS_RGW_OLH_OP_REMOVE_INSTANCE
, op
.op_tag
, op
.key
, false, op
.olh_epoch
);
1458 const string
& olh_tag
= olh
.get_tag();
1459 if (op
.olh_tag
!= olh_tag
) {
1460 if (!olh
.pending_removal()) {
1461 CLS_LOG(5, "NOTICE: op.olh_tag (%s) != olh.tag (%s)", op
.olh_tag
.c_str(), olh_tag
.c_str());
1464 /* if pending removal, this is a new olh instance */
1465 olh
.set_tag(op
.olh_tag
);
1468 rgw_bucket_olh_entry
& olh_entry
= olh
.get_entry();
1469 /* found olh, previous instance is no longer the latest, need to update */
1470 if (!(olh_entry
.key
== op
.key
)) {
1471 BIVerObjEntry
old_obj(hctx
, olh_entry
.key
);
1473 ret
= old_obj
.demote_current();
1475 CLS_LOG(0, "ERROR: could not demote current on previous key ret=%d", ret
);
1480 olh
.set_pending_removal(false);
1482 bool instance_only
= (op
.key
.instance
.empty() && op
.delete_marker
);
1483 cls_rgw_obj_key
key(op
.key
.name
);
1484 ret
= convert_plain_entry_to_versioned(hctx
, key
, true, instance_only
);
1486 CLS_LOG(0, "ERROR: convert_plain_entry_to_versioned ret=%d", ret
);
1489 olh
.set_tag(op
.olh_tag
);
1492 /* update the olh log */
1493 olh
.update_log(CLS_RGW_OLH_OP_LINK_OLH
, op
.op_tag
, op
.key
, op
.delete_marker
);
1495 olh
.update_log(CLS_RGW_OLH_OP_REMOVE_INSTANCE
, op
.op_tag
, op
.key
, false);
1498 olh
.update(op
.key
, op
.delete_marker
);
1500 olh
.set_exists(true);
1504 CLS_LOG(0, "ERROR: failed to update olh ret=%d", ret
);
1508 /* write the instance and list entries */
1509 ret
= obj
.write(olh
.get_epoch(), true);
1514 struct rgw_bucket_dir_header header
;
1515 ret
= read_bucket_header(hctx
, &header
);
1517 CLS_LOG(1, "ERROR: rgw_bucket_unlink_instance(): failed to read header\n");
1522 rgw_bucket_dir_entry
& entry
= obj
.get_dir_entry();
1524 rgw_bucket_entry_ver ver
;
1525 ver
.epoch
= (op
.olh_epoch
? op
.olh_epoch
: olh
.get_epoch());
1527 string
*powner
= NULL
;
1528 string
*powner_display_name
= NULL
;
1530 if (op
.delete_marker
) {
1531 powner
= &entry
.meta
.owner
;
1532 powner_display_name
= &entry
.meta
.owner_display_name
;
1535 RGWModifyOp operation
= (op
.delete_marker
? CLS_RGW_OP_LINK_OLH_DM
: CLS_RGW_OP_LINK_OLH
);
1536 ret
= log_index_operation(hctx
, op
.key
, operation
, op
.op_tag
,
1537 entry
.meta
.mtime
, ver
,
1538 CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
, op
.bilog_flags
| RGW_BILOG_FLAG_VERSIONED_OP
,
1539 powner
, powner_display_name
);
1544 return write_bucket_header(hctx
, &header
); /* updates header version */
1547 static int rgw_bucket_unlink_instance(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1549 string olh_data_idx
;
1550 string instance_idx
;
1553 rgw_cls_unlink_instance_op op
;
1554 bufferlist::iterator iter
= in
->begin();
1557 } catch (buffer::error
& err
) {
1558 CLS_LOG(0, "ERROR: rgw_bucket_rm_obj_instance_op(): failed to decode request\n");
1562 cls_rgw_obj_key dest_key
= op
.key
;
1563 if (dest_key
.instance
== "null") {
1564 dest_key
.instance
.clear();
1567 BIVerObjEntry
obj(hctx
, dest_key
);
1568 BIOLHEntry
olh(hctx
, dest_key
);
1570 int ret
= obj
.init();
1571 if (ret
== -ENOENT
) {
1572 return 0; /* already removed */
1575 CLS_LOG(0, "ERROR: obj.init() returned ret=%d", ret
);
1580 ret
= olh
.init(&olh_found
);
1582 CLS_LOG(0, "ERROR: olh.init() returned ret=%d", ret
);
1587 bool instance_only
= false;
1588 cls_rgw_obj_key
key(dest_key
.name
);
1589 ret
= convert_plain_entry_to_versioned(hctx
, key
, true, instance_only
);
1591 CLS_LOG(0, "ERROR: convert_plain_entry_to_versioned ret=%d", ret
);
1594 olh
.update(dest_key
, false);
1595 olh
.set_tag(op
.olh_tag
);
1600 if (!olh
.start_modify(op
.olh_epoch
)) {
1601 ret
= obj
.unlink_list_entry();
1606 if (!obj
.is_delete_marker()) {
1607 olh
.update_log(CLS_RGW_OLH_OP_REMOVE_INSTANCE
, op
.op_tag
, op
.key
, false, op
.olh_epoch
);
1613 rgw_bucket_olh_entry
& olh_entry
= olh
.get_entry();
1614 cls_rgw_obj_key
& olh_key
= olh_entry
.key
;
1615 CLS_LOG(20, "%s(): updating olh log: existing olh entry: %s[%s] (delete_marker=%d)", __func__
,
1616 olh_key
.name
.c_str(), olh_key
.instance
.c_str(), olh_entry
.delete_marker
);
1618 if (olh_key
== dest_key
) {
1619 /* this is the current head, need to update! */
1620 cls_rgw_obj_key next_key
;
1622 ret
= obj
.find_next_key(&next_key
, &found
);
1624 CLS_LOG(0, "ERROR: obj.find_next_key() returned ret=%d", ret
);
1629 BIVerObjEntry
next(hctx
, next_key
);
1630 ret
= next
.write(olh
.get_epoch(), true);
1632 CLS_LOG(0, "ERROR: next.write() returned ret=%d", ret
);
1636 CLS_LOG(20, "%s(): updating olh log: link olh -> %s[%s] (is_delete=%d)", __func__
,
1637 next_key
.name
.c_str(), next_key
.instance
.c_str(), (int)next
.is_delete_marker());
1639 olh
.update(next_key
, next
.is_delete_marker());
1640 olh
.update_log(CLS_RGW_OLH_OP_LINK_OLH
, op
.op_tag
, next_key
, next
.is_delete_marker());
1642 /* next_key is empty */
1643 olh
.update(next_key
, false);
1644 olh
.update_log(CLS_RGW_OLH_OP_UNLINK_OLH
, op
.op_tag
, next_key
, false);
1645 olh
.set_exists(false);
1646 olh
.set_pending_removal(true);
1650 if (!obj
.is_delete_marker()) {
1651 olh
.update_log(CLS_RGW_OLH_OP_REMOVE_INSTANCE
, op
.op_tag
, op
.key
, false);
1653 /* this is a delete marker, it's our responsibility to remove its instance entry */
1660 ret
= obj
.unlink_list_entry();
1670 struct rgw_bucket_dir_header header
;
1671 ret
= read_bucket_header(hctx
, &header
);
1673 CLS_LOG(1, "ERROR: rgw_bucket_unlink_instance(): failed to read header\n");
1678 rgw_bucket_entry_ver ver
;
1679 ver
.epoch
= (op
.olh_epoch
? op
.olh_epoch
: olh
.get_epoch());
1681 real_time mtime
= real_clock::now(); /* mtime has no real meaning in instance removal context */
1682 ret
= log_index_operation(hctx
, op
.key
, CLS_RGW_OP_UNLINK_INSTANCE
, op
.op_tag
,
1684 CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
,
1685 op
.bilog_flags
| RGW_BILOG_FLAG_VERSIONED_OP
, NULL
, NULL
);
1690 return write_bucket_header(hctx
, &header
); /* updates header version */
1693 static int rgw_bucket_read_olh_log(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1696 rgw_cls_read_olh_log_op op
;
1697 bufferlist::iterator iter
= in
->begin();
1700 } catch (buffer::error
& err
) {
1701 CLS_LOG(0, "ERROR: rgw_bucket_read_olh_log(): failed to decode request\n");
1705 if (!op
.olh
.instance
.empty()) {
1706 CLS_LOG(1, "bad key passed in (non empty instance)");
1710 struct rgw_bucket_olh_entry olh_data_entry
;
1711 string olh_data_key
;
1712 encode_olh_data_key(op
.olh
, &olh_data_key
);
1713 int ret
= read_index_entry(hctx
, olh_data_key
, &olh_data_entry
);
1714 if (ret
< 0 && ret
!= -ENOENT
) {
1715 CLS_LOG(0, "ERROR: read_index_entry() olh_key=%s ret=%d", olh_data_key
.c_str(), ret
);
1719 if (olh_data_entry
.tag
!= op
.olh_tag
) {
1720 CLS_LOG(1, "NOTICE: %s(): olh_tag_mismatch olh_data_entry.tag=%s op.olh_tag=%s", __func__
, olh_data_entry
.tag
.c_str(), op
.olh_tag
.c_str());
1724 rgw_cls_read_olh_log_ret op_ret
;
1726 #define MAX_OLH_LOG_ENTRIES 1000
1727 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> >& log
= olh_data_entry
.pending_log
;
1729 if (log
.begin()->first
> op
.ver_marker
&& log
.size() <= MAX_OLH_LOG_ENTRIES
) {
1731 op_ret
.is_truncated
= false;
1733 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> >::iterator iter
= log
.upper_bound(op
.ver_marker
);
1735 for (int i
= 0; i
< MAX_OLH_LOG_ENTRIES
&& iter
!= log
.end(); ++i
, ++iter
) {
1736 op_ret
.log
[iter
->first
] = iter
->second
;
1738 op_ret
.is_truncated
= (iter
!= log
.end());
1741 ::encode(op_ret
, *out
);
1746 static int rgw_bucket_trim_olh_log(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1749 rgw_cls_trim_olh_log_op op
;
1750 bufferlist::iterator iter
= in
->begin();
1753 } catch (buffer::error
& err
) {
1754 CLS_LOG(0, "ERROR: rgw_bucket_trim_olh_log(): failed to decode request\n");
1758 if (!op
.olh
.instance
.empty()) {
1759 CLS_LOG(1, "bad key passed in (non empty instance)");
1763 /* read olh entry */
1764 struct rgw_bucket_olh_entry olh_data_entry
;
1765 string olh_data_key
;
1766 encode_olh_data_key(op
.olh
, &olh_data_key
);
1767 int ret
= read_index_entry(hctx
, olh_data_key
, &olh_data_entry
);
1768 if (ret
< 0 && ret
!= -ENOENT
) {
1769 CLS_LOG(0, "ERROR: read_index_entry() olh_key=%s ret=%d", olh_data_key
.c_str(), ret
);
1773 if (olh_data_entry
.tag
!= op
.olh_tag
) {
1774 CLS_LOG(1, "NOTICE: %s(): olh_tag_mismatch olh_data_entry.tag=%s op.olh_tag=%s", __func__
, olh_data_entry
.tag
.c_str(), op
.olh_tag
.c_str());
1778 /* remove all versions up to and including ver from the pending map */
1779 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> >& log
= olh_data_entry
.pending_log
;
1780 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> >::iterator liter
= log
.begin();
1781 while (liter
!= log
.end() && liter
->first
<= op
.ver
) {
1782 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> >::iterator rm_iter
= liter
;
1787 /* write the olh data entry */
1788 ret
= write_entry(hctx
, olh_data_entry
, olh_data_key
);
1790 CLS_LOG(0, "ERROR: write_entry() olh_key=%s ret=%d", olh_data_key
.c_str(), ret
);
1797 static int rgw_bucket_clear_olh(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1800 rgw_cls_bucket_clear_olh_op op
;
1801 bufferlist::iterator iter
= in
->begin();
1804 } catch (buffer::error
& err
) {
1805 CLS_LOG(0, "ERROR: rgw_bucket_clear_olh(): failed to decode request\n");
1809 if (!op
.key
.instance
.empty()) {
1810 CLS_LOG(1, "bad key passed in (non empty instance)");
1814 /* read olh entry */
1815 struct rgw_bucket_olh_entry olh_data_entry
;
1816 string olh_data_key
;
1817 encode_olh_data_key(op
.key
, &olh_data_key
);
1818 int ret
= read_index_entry(hctx
, olh_data_key
, &olh_data_entry
);
1819 if (ret
< 0 && ret
!= -ENOENT
) {
1820 CLS_LOG(0, "ERROR: read_index_entry() olh_key=%s ret=%d", olh_data_key
.c_str(), ret
);
1824 if (olh_data_entry
.tag
!= op
.olh_tag
) {
1825 CLS_LOG(1, "NOTICE: %s(): olh_tag_mismatch olh_data_entry.tag=%s op.olh_tag=%s", __func__
, olh_data_entry
.tag
.c_str(), op
.olh_tag
.c_str());
1829 ret
= cls_cxx_map_remove_key(hctx
, olh_data_key
);
1831 CLS_LOG(1, "NOTICE: %s(): can't remove key %s ret=%d", __func__
, olh_data_key
.c_str(), ret
);
1835 rgw_bucket_dir_entry plain_entry
;
1837 /* read plain entry, make sure it's a versioned place holder */
1838 ret
= read_index_entry(hctx
, op
.key
.name
, &plain_entry
);
1839 if (ret
== -ENOENT
) {
1840 /* we're done, no entry existing */
1844 CLS_LOG(0, "ERROR: read_index_entry key=%s ret=%d", op
.key
.name
.c_str(), ret
);
1848 if ((plain_entry
.flags
& RGW_BUCKET_DIRENT_FLAG_VER_MARKER
) == 0) {
1849 /* it's not a version marker, don't remove it */
1853 ret
= cls_cxx_map_remove_key(hctx
, op
.key
.name
);
1855 CLS_LOG(1, "NOTICE: %s(): can't remove key %s ret=%d", __func__
, op
.key
.name
.c_str(), ret
);
1862 int rgw_dir_suggest_changes(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1864 CLS_LOG(1, "rgw_dir_suggest_changes()");
1866 bufferlist header_bl
;
1867 struct rgw_bucket_dir_header header
;
1868 bool header_changed
= false;
1870 int rc
= read_bucket_header(hctx
, &header
);
1872 CLS_LOG(1, "ERROR: rgw_dir_suggest_changes(): failed to read header\n");
1876 timespan
tag_timeout(header
.tag_timeout
? header
.tag_timeout
: CEPH_RGW_TAG_TIMEOUT
);
1878 bufferlist::iterator in_iter
= in
->begin();
1880 while (!in_iter
.end()) {
1882 rgw_bucket_dir_entry cur_change
;
1883 rgw_bucket_dir_entry cur_disk
;
1885 ::decode(op
, in_iter
);
1886 ::decode(cur_change
, in_iter
);
1887 } catch (buffer::error
& err
) {
1888 CLS_LOG(1, "ERROR: rgw_dir_suggest_changes(): failed to decode request\n");
1892 bufferlist cur_disk_bl
;
1893 string cur_change_key
;
1894 encode_obj_index_key(cur_change
.key
, &cur_change_key
);
1895 int ret
= cls_cxx_map_get_val(hctx
, cur_change_key
, &cur_disk_bl
);
1896 if (ret
< 0 && ret
!= -ENOENT
)
1899 if (cur_disk_bl
.length()) {
1900 bufferlist::iterator cur_disk_iter
= cur_disk_bl
.begin();
1902 ::decode(cur_disk
, cur_disk_iter
);
1903 } catch (buffer::error
& error
) {
1904 CLS_LOG(1, "ERROR: rgw_dir_suggest_changes(): failed to decode cur_disk\n");
1908 real_time cur_time
= real_clock::now();
1909 map
<string
, struct rgw_bucket_pending_info
>::iterator iter
=
1910 cur_disk
.pending_map
.begin();
1911 while(iter
!= cur_disk
.pending_map
.end()) {
1912 map
<string
, struct rgw_bucket_pending_info
>::iterator cur_iter
=iter
++;
1913 if (cur_time
> (cur_iter
->second
.timestamp
+ timespan(tag_timeout
))) {
1914 cur_disk
.pending_map
.erase(cur_iter
);
1919 CLS_LOG(20, "cur_disk.pending_map.empty()=%d op=%d cur_disk.exists=%d cur_change.pending_map.size()=%d cur_change.exists=%d\n",
1920 cur_disk
.pending_map
.empty(), (int)op
, cur_disk
.exists
,
1921 (int)cur_change
.pending_map
.size(), cur_change
.exists
);
1923 if (cur_disk
.pending_map
.empty()) {
1924 if (cur_disk
.exists
) {
1925 struct rgw_bucket_category_stats
& old_stats
= header
.stats
[cur_disk
.meta
.category
];
1926 CLS_LOG(10, "total_entries: %" PRId64
" -> %" PRId64
"\n", old_stats
.num_entries
, old_stats
.num_entries
- 1);
1927 old_stats
.num_entries
--;
1928 old_stats
.total_size
-= cur_disk
.meta
.accounted_size
;
1929 old_stats
.total_size_rounded
-= cls_rgw_get_rounded_size(cur_disk
.meta
.accounted_size
);
1930 old_stats
.actual_size
-= cur_disk
.meta
.size
;
1931 header_changed
= true;
1933 struct rgw_bucket_category_stats
& stats
=
1934 header
.stats
[cur_change
.meta
.category
];
1935 bool log_op
= (op
& CEPH_RGW_DIR_SUGGEST_LOG_OP
) != 0;
1936 op
&= CEPH_RGW_DIR_SUGGEST_OP_MASK
;
1938 case CEPH_RGW_REMOVE
:
1939 CLS_LOG(10, "CEPH_RGW_REMOVE name=%s instance=%s\n", cur_change
.key
.name
.c_str(), cur_change
.key
.instance
.c_str());
1940 ret
= cls_cxx_map_remove_key(hctx
, cur_change_key
);
1943 if (log_op
&& cur_disk
.exists
) {
1944 ret
= log_index_operation(hctx
, cur_disk
.key
, CLS_RGW_OP_DEL
, cur_disk
.tag
, cur_disk
.meta
.mtime
,
1945 cur_disk
.ver
, CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
, 0, NULL
, NULL
);
1947 CLS_LOG(0, "ERROR: %s(): failed to log operation ret=%d", __func__
, ret
);
1952 case CEPH_RGW_UPDATE
:
1953 CLS_LOG(10, "CEPH_RGW_UPDATE name=%s instance=%s total_entries: %" PRId64
" -> %" PRId64
"\n",
1954 cur_change
.key
.name
.c_str(), cur_change
.key
.instance
.c_str(), stats
.num_entries
, stats
.num_entries
+ 1);
1955 stats
.num_entries
++;
1956 stats
.total_size
+= cur_change
.meta
.accounted_size
;
1957 stats
.total_size_rounded
+= cls_rgw_get_rounded_size(cur_change
.meta
.accounted_size
);
1958 stats
.actual_size
+= cur_change
.meta
.size
;
1959 header_changed
= true;
1960 cur_change
.index_ver
= header
.ver
;
1961 bufferlist cur_state_bl
;
1962 ::encode(cur_change
, cur_state_bl
);
1963 ret
= cls_cxx_map_set_val(hctx
, cur_change_key
, &cur_state_bl
);
1967 ret
= log_index_operation(hctx
, cur_change
.key
, CLS_RGW_OP_ADD
, cur_change
.tag
, cur_change
.meta
.mtime
,
1968 cur_change
.ver
, CLS_RGW_STATE_COMPLETE
, header
.ver
, header
.max_marker
, 0, NULL
, NULL
);
1970 CLS_LOG(0, "ERROR: %s(): failed to log operation ret=%d", __func__
, ret
);
1980 if (header_changed
) {
1981 return write_bucket_header(hctx
, &header
);
1986 static int rgw_obj_remove(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
1989 rgw_cls_obj_remove_op op
;
1990 bufferlist::iterator iter
= in
->begin();
1993 } catch (buffer::error
& err
) {
1994 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
1998 if (op
.keep_attr_prefixes
.empty()) {
1999 return cls_cxx_remove(hctx
);
2002 map
<string
, bufferlist
> attrset
;
2003 int ret
= cls_cxx_getxattrs(hctx
, &attrset
);
2004 if (ret
< 0 && ret
!= -ENOENT
) {
2005 CLS_LOG(0, "ERROR: %s(): cls_cxx_getxattrs() returned %d", __func__
, ret
);
2009 map
<string
, bufferlist
> new_attrs
;
2010 for (list
<string
>::iterator iter
= op
.keep_attr_prefixes
.begin();
2011 iter
!= op
.keep_attr_prefixes
.end(); ++iter
) {
2012 string
& check_prefix
= *iter
;
2014 for (map
<string
, bufferlist
>::iterator aiter
= attrset
.lower_bound(check_prefix
);
2015 aiter
!= attrset
.end(); ++aiter
) {
2016 const string
& attr
= aiter
->first
;
2018 if (attr
.substr(0, check_prefix
.size()) > check_prefix
) {
2022 new_attrs
[attr
] = aiter
->second
;
2026 CLS_LOG(20, "%s(): removing object", __func__
);
2027 ret
= cls_cxx_remove(hctx
);
2029 CLS_LOG(0, "ERROR: %s(): cls_cxx_remove returned %d", __func__
, ret
);
2033 if (new_attrs
.empty()) {
2034 /* no data to keep */
2038 ret
= cls_cxx_create(hctx
, false);
2040 CLS_LOG(0, "ERROR: %s(): cls_cxx_create returned %d", __func__
, ret
);
2044 for (map
<string
, bufferlist
>::iterator aiter
= new_attrs
.begin();
2045 aiter
!= new_attrs
.end(); ++aiter
) {
2046 const string
& attr
= aiter
->first
;
2048 ret
= cls_cxx_setxattr(hctx
, attr
.c_str(), &aiter
->second
);
2049 CLS_LOG(20, "%s(): setting attr: %s", __func__
, attr
.c_str());
2051 CLS_LOG(0, "ERROR: %s(): cls_cxx_setxattr (attr=%s) returned %d", __func__
, attr
.c_str(), ret
);
2059 static int rgw_obj_store_pg_ver(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2062 rgw_cls_obj_store_pg_ver_op op
;
2063 bufferlist::iterator iter
= in
->begin();
2066 } catch (buffer::error
& err
) {
2067 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
2072 uint64_t ver
= cls_current_version(hctx
);
2074 int ret
= cls_cxx_setxattr(hctx
, op
.attr
.c_str(), &bl
);
2076 CLS_LOG(0, "ERROR: %s(): cls_cxx_setxattr (attr=%s) returned %d", __func__
, op
.attr
.c_str(), ret
);
2083 static int rgw_obj_check_attrs_prefix(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2086 rgw_cls_obj_check_attrs_prefix op
;
2087 bufferlist::iterator iter
= in
->begin();
2090 } catch (buffer::error
& err
) {
2091 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
2095 if (op
.check_prefix
.empty()) {
2099 map
<string
, bufferlist
> attrset
;
2100 int ret
= cls_cxx_getxattrs(hctx
, &attrset
);
2101 if (ret
< 0 && ret
!= -ENOENT
) {
2102 CLS_LOG(0, "ERROR: %s(): cls_cxx_getxattrs() returned %d", __func__
, ret
);
2108 for (map
<string
, bufferlist
>::iterator aiter
= attrset
.lower_bound(op
.check_prefix
);
2109 aiter
!= attrset
.end(); ++aiter
) {
2110 const string
& attr
= aiter
->first
;
2112 if (attr
.substr(0, op
.check_prefix
.size()) > op
.check_prefix
) {
2119 if (exist
== op
.fail_if_exist
) {
2126 static int rgw_obj_check_mtime(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2129 rgw_cls_obj_check_mtime op
;
2130 bufferlist::iterator iter
= in
->begin();
2133 } catch (buffer::error
& err
) {
2134 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
2139 int ret
= cls_cxx_stat2(hctx
, NULL
, &obj_ut
);
2140 if (ret
< 0 && ret
!= -ENOENT
) {
2141 CLS_LOG(0, "ERROR: %s(): cls_cxx_stat() returned %d", __func__
, ret
);
2144 if (ret
== -ENOENT
) {
2145 CLS_LOG(10, "object does not exist, skipping check");
2148 ceph_timespec obj_ts
= ceph::real_clock::to_ceph_timespec(obj_ut
);
2149 ceph_timespec op_ts
= ceph::real_clock::to_ceph_timespec(op
.mtime
);
2151 if (!op
.high_precision_time
) {
2156 CLS_LOG(10, "%s: obj_ut=%lld.%06lld op.mtime=%lld.%06lld", __func__
,
2157 (long long)obj_ts
.tv_sec
, (long long)obj_ts
.tv_nsec
,
2158 (long long)op_ts
.tv_sec
, (long long)op_ts
.tv_nsec
);
2163 case CLS_RGW_CHECK_TIME_MTIME_EQ
:
2164 check
= (obj_ts
== op_ts
);
2166 case CLS_RGW_CHECK_TIME_MTIME_LT
:
2167 check
= (obj_ts
< op_ts
);
2169 case CLS_RGW_CHECK_TIME_MTIME_LE
:
2170 check
= (obj_ts
<= op_ts
);
2172 case CLS_RGW_CHECK_TIME_MTIME_GT
:
2173 check
= (obj_ts
> op_ts
);
2175 case CLS_RGW_CHECK_TIME_MTIME_GE
:
2176 check
= (obj_ts
>= op_ts
);
2189 static int rgw_bi_get_op(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2192 rgw_cls_bi_get_op op
;
2193 bufferlist::iterator iter
= in
->begin();
2196 } catch (buffer::error
& err
) {
2197 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
2208 encode_obj_index_key(op
.key
, &idx
);
2211 encode_olh_data_key(op
.key
, &idx
);
2214 CLS_LOG(10, "%s(): invalid key type encoding: %d", __func__
, op
.type
);
2218 rgw_cls_bi_get_ret op_ret
;
2220 rgw_cls_bi_entry
& entry
= op_ret
.entry
;
2222 entry
.type
= op
.type
;
2225 int r
= cls_cxx_map_get_val(hctx
, idx
, &entry
.data
);
2227 CLS_LOG(10, "%s(): cls_cxx_map_get_val() returned %d", __func__
, r
);
2231 ::encode(op_ret
, *out
);
2236 static int rgw_bi_put_op(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2239 rgw_cls_bi_put_op op
;
2240 bufferlist::iterator iter
= in
->begin();
2243 } catch (buffer::error
& err
) {
2244 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
2248 rgw_cls_bi_entry
& entry
= op
.entry
;
2250 int r
= cls_cxx_map_set_val(hctx
, entry
.idx
, &entry
.data
);
2252 CLS_LOG(0, "ERROR: %s(): cls_cxx_map_set_val() returned r=%d", __func__
, r
);
2258 static int list_plain_entries(cls_method_context_t hctx
, const string
& name
, const string
& marker
, uint32_t max
,
2259 list
<rgw_cls_bi_entry
> *entries
)
2261 string filter
= name
;
2262 string start_key
= marker
;
2264 string end_key
; // stop listing at bi_log_prefix
2265 bi_log_prefix(end_key
);
2268 map
<string
, bufferlist
> keys
;
2270 if (count
>= (int)max
) {
2274 #define BI_GET_NUM_KEYS 128
2275 int ret
= cls_cxx_map_get_vals(hctx
, start_key
, filter
, BI_GET_NUM_KEYS
, &keys
);
2280 map
<string
, bufferlist
>::iterator iter
;
2281 for (iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
2282 if (iter
->first
>= end_key
) {
2283 /* past the end of plain namespace */
2287 rgw_cls_bi_entry entry
;
2288 entry
.type
= PlainIdx
;
2289 entry
.idx
= iter
->first
;
2290 entry
.data
= iter
->second
;
2292 bufferlist::iterator biter
= entry
.data
.begin();
2294 rgw_bucket_dir_entry e
;
2297 } catch (buffer::error
& err
) {
2298 CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__
);
2302 CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__
, escape_str(entry
.idx
).c_str(), escape_str(e
.key
.name
).c_str());
2304 if (!name
.empty() && e
.key
.name
!= name
) {
2308 entries
->push_back(entry
);
2310 if (count
>= (int)max
) {
2313 start_key
= entry
.idx
;
2315 } while (!keys
.empty());
2320 static int list_instance_entries(cls_method_context_t hctx
, const string
& name
, const string
& marker
, uint32_t max
,
2321 list
<rgw_cls_bi_entry
> *entries
)
2323 cls_rgw_obj_key
key(name
);
2324 string first_instance_idx
;
2325 encode_obj_versioned_data_key(key
, &first_instance_idx
);
2328 if (!name
.empty()) {
2329 start_key
= first_instance_idx
;
2331 start_key
= BI_PREFIX_CHAR
;
2332 start_key
.append(bucket_index_prefixes
[BI_BUCKET_OBJ_INSTANCE_INDEX
]);
2334 string filter
= start_key
;
2335 if (bi_entry_gt(marker
, start_key
)) {
2339 map
<string
, bufferlist
> keys
;
2340 bool started
= true;
2342 if (count
>= (int)max
) {
2346 #define BI_GET_NUM_KEYS 128
2349 ret
= cls_cxx_map_get_val(hctx
, start_key
, &keys
[start_key
]);
2350 if (ret
== -ENOENT
) {
2351 ret
= cls_cxx_map_get_vals(hctx
, start_key
, string(), BI_GET_NUM_KEYS
, &keys
);
2355 ret
= cls_cxx_map_get_vals(hctx
, start_key
, string(), BI_GET_NUM_KEYS
, &keys
);
2357 CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__
, escape_str(start_key
).c_str(), escape_str(first_instance_idx
).c_str(), (int)keys
.size());
2362 map
<string
, bufferlist
>::iterator iter
;
2363 for (iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
2364 rgw_cls_bi_entry entry
;
2365 entry
.type
= InstanceIdx
;
2366 entry
.idx
= iter
->first
;
2367 entry
.data
= iter
->second
;
2369 if (!filter
.empty() && entry
.idx
.compare(0, filter
.size(), filter
) != 0) {
2373 CLS_LOG(20, "%s(): entry.idx=%s", __func__
, escape_str(entry
.idx
).c_str());
2375 bufferlist::iterator biter
= entry
.data
.begin();
2377 rgw_bucket_dir_entry e
;
2380 } catch (buffer::error
& err
) {
2381 CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__
, entry
.data
.length());
2385 if (!name
.empty() && e
.key
.name
!= name
) {
2389 entries
->push_back(entry
);
2391 start_key
= entry
.idx
;
2393 } while (!keys
.empty());
2398 static int list_olh_entries(cls_method_context_t hctx
, const string
& name
, const string
& marker
, uint32_t max
,
2399 list
<rgw_cls_bi_entry
> *entries
)
2401 cls_rgw_obj_key
key(name
);
2402 string first_instance_idx
;
2403 encode_olh_data_key(key
, &first_instance_idx
);
2406 if (!name
.empty()) {
2407 start_key
= first_instance_idx
;
2409 start_key
= BI_PREFIX_CHAR
;
2410 start_key
.append(bucket_index_prefixes
[BI_BUCKET_OLH_DATA_INDEX
]);
2412 string filter
= start_key
;
2413 if (bi_entry_gt(marker
, start_key
)) {
2417 map
<string
, bufferlist
> keys
;
2418 bool started
= true;
2420 if (count
>= (int)max
) {
2424 #define BI_GET_NUM_KEYS 128
2427 ret
= cls_cxx_map_get_val(hctx
, start_key
, &keys
[start_key
]);
2428 if (ret
== -ENOENT
) {
2429 ret
= cls_cxx_map_get_vals(hctx
, start_key
, string(), BI_GET_NUM_KEYS
, &keys
);
2433 ret
= cls_cxx_map_get_vals(hctx
, start_key
, string(), BI_GET_NUM_KEYS
, &keys
);
2435 CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__
, escape_str(start_key
).c_str(), escape_str(first_instance_idx
).c_str(), (int)keys
.size());
2440 map
<string
, bufferlist
>::iterator iter
;
2441 for (iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
2442 rgw_cls_bi_entry entry
;
2443 entry
.type
= OLHIdx
;
2444 entry
.idx
= iter
->first
;
2445 entry
.data
= iter
->second
;
2447 if (!filter
.empty() && entry
.idx
.compare(0, filter
.size(), filter
) != 0) {
2451 CLS_LOG(20, "%s(): entry.idx=%s", __func__
, escape_str(entry
.idx
).c_str());
2453 bufferlist::iterator biter
= entry
.data
.begin();
2455 rgw_bucket_olh_entry e
;
2458 } catch (buffer::error
& err
) {
2459 CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__
, entry
.data
.length());
2463 if (!name
.empty() && e
.key
.name
!= name
) {
2467 entries
->push_back(entry
);
2469 start_key
= entry
.idx
;
2471 } while (!keys
.empty());
2476 static int rgw_bi_list_op(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2479 rgw_cls_bi_list_op op
;
2480 bufferlist::iterator iter
= in
->begin();
2483 } catch (buffer::error
& err
) {
2484 CLS_LOG(0, "ERROR: %s(): failed to decode request", __func__
);
2488 rgw_cls_bi_list_ret op_ret
;
2490 string filter
= op
.name
;
2491 #define MAX_BI_LIST_ENTRIES 1000
2492 int32_t max
= (op
.max
< MAX_BI_LIST_ENTRIES
? op
.max
: MAX_BI_LIST_ENTRIES
) + 1; /* one extra entry for identifying truncation */
2493 string start_key
= op
.marker
;
2494 int ret
= list_plain_entries(hctx
, op
.name
, op
.marker
, max
, &op_ret
.entries
);
2496 CLS_LOG(0, "ERROR: %s(): list_plain_entries retured ret=%d", __func__
, ret
);
2501 CLS_LOG(20, "found %d plain entries", count
);
2503 ret
= list_instance_entries(hctx
, op
.name
, op
.marker
, max
- count
, &op_ret
.entries
);
2505 CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__
, ret
);
2511 ret
= list_olh_entries(hctx
, op
.name
, op
.marker
, max
- count
, &op_ret
.entries
);
2513 CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__
, ret
);
2519 op_ret
.is_truncated
= (count
>= max
);
2520 while (count
>= max
) {
2521 op_ret
.entries
.pop_back();
2525 ::encode(op_ret
, *out
);
2530 int bi_log_record_decode(bufferlist
& bl
, rgw_bi_log_entry
& e
)
2532 bufferlist::iterator iter
= bl
.begin();
2535 } catch (buffer::error
& err
) {
2536 CLS_LOG(0, "ERROR: failed to decode rgw_bi_log_entry");
2542 static int bi_log_iterate_entries(cls_method_context_t hctx
, const string
& marker
, const string
& end_marker
,
2543 string
& key_iter
, uint32_t max_entries
, bool *truncated
,
2544 int (*cb
)(cls_method_context_t
, const string
&, rgw_bi_log_entry
&, void *),
2547 CLS_LOG(10, "bi_log_iterate_range");
2549 map
<string
, bufferlist
> keys
;
2550 string filter_prefix
, end_key
;
2558 if (key_iter
.empty()) {
2559 key
= BI_PREFIX_CHAR
;
2560 key
.append(bucket_index_prefixes
[BI_BUCKET_LOG_INDEX
]);
2565 start_key
= key_iter
;
2568 if (end_marker
.empty()) {
2569 end_key
= BI_PREFIX_CHAR
;
2570 end_key
.append(bucket_index_prefixes
[BI_BUCKET_LOG_INDEX
+ 1]);
2572 end_key
= BI_PREFIX_CHAR
;
2573 end_key
.append(bucket_index_prefixes
[BI_BUCKET_LOG_INDEX
]);
2574 end_key
.append(end_marker
);
2577 CLS_LOG(0, "bi_log_iterate_entries start_key=%s end_key=%s\n", start_key
.c_str(), end_key
.c_str());
2582 #define BI_NUM_KEYS 128
2583 int ret
= cls_cxx_map_get_vals(hctx
, start_key
, filter
, BI_NUM_KEYS
, &keys
);
2587 map
<string
, bufferlist
>::iterator iter
= keys
.begin();
2588 if (iter
== keys
.end())
2591 for (; iter
!= keys
.end(); ++iter
) {
2592 const string
& key
= iter
->first
;
2595 CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key
.c_str(), (int)iter
->second
.length());
2597 if (key
.compare(end_key
) > 0)
2600 ret
= bi_log_record_decode(iter
->second
, e
);
2604 if (max_entries
&& (i
>= max_entries
)) {
2611 ret
= cb(hctx
, key
, e
, param
);
2618 start_key
= iter
->first
;
2623 static int bi_log_list_cb(cls_method_context_t hctx
, const string
& key
, rgw_bi_log_entry
& info
, void *param
)
2625 list
<rgw_bi_log_entry
> *l
= (list
<rgw_bi_log_entry
> *)param
;
2630 static int bi_log_list_entries(cls_method_context_t hctx
, const string
& marker
,
2631 uint32_t max
, list
<rgw_bi_log_entry
>& entries
, bool *truncated
)
2635 int ret
= bi_log_iterate_entries(hctx
, marker
, end_marker
,
2636 key_iter
, max
, truncated
,
2637 bi_log_list_cb
, &entries
);
2641 static int rgw_bi_log_list(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2643 bufferlist::iterator in_iter
= in
->begin();
2645 cls_rgw_bi_log_list_op op
;
2647 ::decode(op
, in_iter
);
2648 } catch (buffer::error
& err
) {
2649 CLS_LOG(1, "ERROR: rgw_bi_log_list(): failed to decode entry\n");
2653 cls_rgw_bi_log_list_ret op_ret
;
2654 int ret
= bi_log_list_entries(hctx
, op
.marker
, op
.max
, op_ret
.entries
, &op_ret
.truncated
);
2658 ::encode(op_ret
, *out
);
2663 static int bi_log_list_trim_cb(cls_method_context_t hctx
, const string
& key
, rgw_bi_log_entry
& info
, void *param
)
2665 list
<rgw_bi_log_entry
> *entries
= (list
<rgw_bi_log_entry
> *)param
;
2667 entries
->push_back(info
);
2671 static int bi_log_remove_entry(cls_method_context_t hctx
, rgw_bi_log_entry
& entry
)
2674 key
= BI_PREFIX_CHAR
;
2675 key
.append(bucket_index_prefixes
[BI_BUCKET_LOG_INDEX
]);
2676 key
.append(entry
.id
);
2677 return cls_cxx_map_remove_key(hctx
, key
);
2680 static int bi_log_list_trim_entries(cls_method_context_t hctx
,
2681 const string
& start_marker
, const string
& end_marker
,
2682 list
<rgw_bi_log_entry
>& entries
, bool *truncated
)
2685 #define MAX_TRIM_ENTRIES 1000 /* max entries to trim in a single operation */
2686 int ret
= bi_log_iterate_entries(hctx
, start_marker
, end_marker
,
2687 key_iter
, MAX_TRIM_ENTRIES
, truncated
,
2688 bi_log_list_trim_cb
, &entries
);
2692 static int rgw_bi_log_trim(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2694 bufferlist::iterator in_iter
= in
->begin();
2696 cls_rgw_bi_log_trim_op op
;
2698 ::decode(op
, in_iter
);
2699 } catch (buffer::error
& err
) {
2700 CLS_LOG(1, "ERROR: rgw_bi_log_list(): failed to decode entry\n");
2704 cls_rgw_bi_log_list_ret op_ret
;
2705 list
<rgw_bi_log_entry
> entries
;
2706 #define MAX_TRIM_ENTRIES 1000 /* don't do more than that in a single operation */
2708 int ret
= bi_log_list_trim_entries(hctx
, op
.start_marker
, op
.end_marker
, entries
, &truncated
);
2712 if (entries
.empty())
2715 list
<rgw_bi_log_entry
>::iterator iter
;
2716 for (iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
2717 rgw_bi_log_entry
& entry
= *iter
;
2719 ret
= bi_log_remove_entry(hctx
, entry
);
2727 static void usage_record_prefix_by_time(uint64_t epoch
, string
& key
)
2730 snprintf(buf
, sizeof(buf
), "%011llu", (long long unsigned)epoch
);
2734 static void usage_record_prefix_by_user(string
& user
, uint64_t epoch
, string
& key
)
2736 char buf
[user
.size() + 32];
2737 snprintf(buf
, sizeof(buf
), "%s_%011llu_", user
.c_str(), (long long unsigned)epoch
);
2741 static void usage_record_name_by_time(uint64_t epoch
, const string
& user
, string
& bucket
, string
& key
)
2743 char buf
[32 + user
.size() + bucket
.size()];
2744 snprintf(buf
, sizeof(buf
), "%011llu_%s_%s", (long long unsigned)epoch
, user
.c_str(), bucket
.c_str());
2748 static void usage_record_name_by_user(const string
& user
, uint64_t epoch
, string
& bucket
, string
& key
)
2750 char buf
[32 + user
.size() + bucket
.size()];
2751 snprintf(buf
, sizeof(buf
), "%s_%011llu_%s", user
.c_str(), (long long unsigned)epoch
, bucket
.c_str());
2755 static int usage_record_decode(bufferlist
& record_bl
, rgw_usage_log_entry
& e
)
2757 bufferlist::iterator kiter
= record_bl
.begin();
2760 } catch (buffer::error
& err
) {
2761 CLS_LOG(1, "ERROR: usage_record_decode(): failed to decode record_bl\n");
2768 int rgw_user_usage_log_add(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2770 CLS_LOG(10, "rgw_user_usage_log_add()");
2772 bufferlist::iterator in_iter
= in
->begin();
2773 rgw_cls_usage_log_add_op op
;
2776 ::decode(op
, in_iter
);
2777 } catch (buffer::error
& err
) {
2778 CLS_LOG(1, "ERROR: rgw_user_usage_log_add(): failed to decode request\n");
2782 rgw_usage_log_info
& info
= op
.info
;
2783 vector
<rgw_usage_log_entry
>::iterator iter
;
2785 for (iter
= info
.entries
.begin(); iter
!= info
.entries
.end(); ++iter
) {
2786 rgw_usage_log_entry
& entry
= *iter
;
2789 rgw_user
*puser
= (entry
.payer
.empty() ? &entry
.owner
: &entry
.payer
);
2791 usage_record_name_by_time(entry
.epoch
, puser
->to_str(), entry
.bucket
, key_by_time
);
2793 CLS_LOG(10, "rgw_user_usage_log_add user=%s bucket=%s\n", puser
->to_str().c_str(), entry
.bucket
.c_str());
2795 bufferlist record_bl
;
2796 int ret
= cls_cxx_map_get_val(hctx
, key_by_time
, &record_bl
);
2797 if (ret
< 0 && ret
!= -ENOENT
) {
2798 CLS_LOG(1, "ERROR: rgw_user_usage_log_add(): cls_cxx_map_read_key returned %d\n", ret
);
2802 rgw_usage_log_entry e
;
2803 ret
= usage_record_decode(record_bl
, e
);
2806 CLS_LOG(10, "rgw_user_usage_log_add aggregating existing bucket\n");
2810 bufferlist new_record_bl
;
2811 ::encode(entry
, new_record_bl
);
2812 ret
= cls_cxx_map_set_val(hctx
, key_by_time
, &new_record_bl
);
2817 usage_record_name_by_user(puser
->to_str(), entry
.epoch
, entry
.bucket
, key_by_user
);
2818 ret
= cls_cxx_map_set_val(hctx
, key_by_user
, &new_record_bl
);
2826 static int usage_iterate_range(cls_method_context_t hctx
, uint64_t start
, uint64_t end
,
2827 string
& user
, string
& key_iter
, uint32_t max_entries
, bool *truncated
,
2828 int (*cb
)(cls_method_context_t
, const string
&, rgw_usage_log_entry
&, void *),
2831 CLS_LOG(10, "usage_iterate_range");
2833 map
<string
, bufferlist
> keys
;
2835 string filter_prefix
;
2836 string start_key
, end_key
;
2837 bool by_user
= !user
.empty();
2845 usage_record_prefix_by_time(end
, end_key
);
2848 user_key
.append("_");
2851 if (key_iter
.empty()) {
2853 usage_record_prefix_by_user(user
, start
, start_key
);
2855 usage_record_prefix_by_time(start
, start_key
);
2858 start_key
= key_iter
;
2862 CLS_LOG(20, "usage_iterate_range start_key=%s", start_key
.c_str());
2863 int ret
= cls_cxx_map_get_vals(hctx
, start_key
, filter_prefix
, NUM_KEYS
, &keys
);
2868 map
<string
, bufferlist
>::iterator iter
= keys
.begin();
2869 if (iter
== keys
.end())
2872 for (; iter
!= keys
.end(); ++iter
) {
2873 const string
& key
= iter
->first
;
2874 rgw_usage_log_entry e
;
2876 if (!by_user
&& key
.compare(end_key
) >= 0) {
2877 CLS_LOG(20, "usage_iterate_range reached key=%s, done", key
.c_str());
2881 if (by_user
&& key
.compare(0, user_key
.size(), user_key
) != 0) {
2882 CLS_LOG(20, "usage_iterate_range reached key=%s, done", key
.c_str());
2886 ret
= usage_record_decode(iter
->second
, e
);
2890 if (e
.epoch
< start
)
2893 /* keys are sorted by epoch, so once we're past end we're done */
2897 ret
= cb(hctx
, key
, e
, param
);
2903 if (max_entries
&& (i
> max_entries
)) {
2904 CLS_LOG(20, "usage_iterate_range reached max_entries (%d), done", max_entries
);
2911 start_key
= iter
->first
;
2916 static int usage_log_read_cb(cls_method_context_t hctx
, const string
& key
, rgw_usage_log_entry
& entry
, void *param
)
2918 map
<rgw_user_bucket
, rgw_usage_log_entry
> *usage
= (map
<rgw_user_bucket
, rgw_usage_log_entry
> *)param
;
2920 if (!entry
.payer
.empty()) {
2921 puser
= &entry
.payer
;
2923 puser
= &entry
.owner
;
2925 rgw_user_bucket
ub(puser
->to_str(), entry
.bucket
);
2926 rgw_usage_log_entry
& le
= (*usage
)[ub
];
2927 le
.aggregate(entry
);
2932 int rgw_user_usage_log_read(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2934 CLS_LOG(10, "rgw_user_usage_log_read()");
2936 bufferlist::iterator in_iter
= in
->begin();
2937 rgw_cls_usage_log_read_op op
;
2940 ::decode(op
, in_iter
);
2941 } catch (buffer::error
& err
) {
2942 CLS_LOG(1, "ERROR: rgw_user_usage_log_read(): failed to decode request\n");
2946 rgw_cls_usage_log_read_ret ret_info
;
2947 map
<rgw_user_bucket
, rgw_usage_log_entry
> *usage
= &ret_info
.usage
;
2948 string iter
= op
.iter
;
2949 #define MAX_ENTRIES 1000
2950 uint32_t max_entries
= (op
.max_entries
? op
.max_entries
: MAX_ENTRIES
);
2951 int ret
= usage_iterate_range(hctx
, op
.start_epoch
, op
.end_epoch
, op
.owner
, iter
, max_entries
, &ret_info
.truncated
, usage_log_read_cb
, (void *)usage
);
2955 if (ret_info
.truncated
)
2956 ret_info
.next_iter
= iter
;
2958 ::encode(ret_info
, *out
);
2962 static int usage_log_trim_cb(cls_method_context_t hctx
, const string
& key
, rgw_usage_log_entry
& entry
, void *param
)
2967 string o
= entry
.owner
.to_str();
2968 usage_record_name_by_time(entry
.epoch
, o
, entry
.bucket
, key_by_time
);
2969 usage_record_name_by_user(o
, entry
.epoch
, entry
.bucket
, key_by_user
);
2971 int ret
= cls_cxx_map_remove_key(hctx
, key_by_time
);
2975 return cls_cxx_map_remove_key(hctx
, key_by_user
);
2978 int rgw_user_usage_log_trim(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
2980 CLS_LOG(10, "rgw_user_usage_log_trim()");
2982 /* only continue if object exists! */
2983 int ret
= cls_cxx_stat(hctx
, NULL
, NULL
);
2987 bufferlist::iterator in_iter
= in
->begin();
2988 rgw_cls_usage_log_trim_op op
;
2991 ::decode(op
, in_iter
);
2992 } catch (buffer::error
& err
) {
2993 CLS_LOG(1, "ERROR: rgw_user_log_usage_log_trim(): failed to decode request\n");
2998 ret
= usage_iterate_range(hctx
, op
.start_epoch
, op
.end_epoch
, op
.user
, iter
, 0, NULL
, usage_log_trim_cb
, NULL
);
3006 * We hold the garbage collection chain data under two different indexes: the first 'name' index
3007 * keeps them under a unique tag that represents the chains, and a second 'time' index keeps
3008 * them by their expiration timestamp
3010 #define GC_OBJ_NAME_INDEX 0
3011 #define GC_OBJ_TIME_INDEX 1
3013 static string gc_index_prefixes
[] = { "0_",
3016 static void prepend_index_prefix(const string
& src
, int index
, string
*dest
)
3018 *dest
= gc_index_prefixes
[index
];
3022 static int gc_omap_get(cls_method_context_t hctx
, int type
, const string
& key
, cls_rgw_gc_obj_info
*info
)
3025 prepend_index_prefix(key
, type
, &index
);
3028 int ret
= cls_cxx_map_get_val(hctx
, index
, &bl
);
3033 bufferlist::iterator iter
= bl
.begin();
3034 ::decode(*info
, iter
);
3035 } catch (buffer::error
& err
) {
3036 CLS_LOG(0, "ERROR: rgw_cls_gc_omap_get(): failed to decode index=%s\n", index
.c_str());
3042 static int gc_omap_set(cls_method_context_t hctx
, int type
, const string
& key
, const cls_rgw_gc_obj_info
*info
)
3045 ::encode(*info
, bl
);
3047 string index
= gc_index_prefixes
[type
];
3050 int ret
= cls_cxx_map_set_val(hctx
, index
, &bl
);
3057 static int gc_omap_remove(cls_method_context_t hctx
, int type
, const string
& key
)
3059 string index
= gc_index_prefixes
[type
];
3062 int ret
= cls_cxx_map_remove_key(hctx
, index
);
3069 static bool key_in_index(const string
& key
, int index_type
)
3071 const string
& prefix
= gc_index_prefixes
[index_type
];
3072 return (key
.compare(0, prefix
.size(), prefix
) == 0);
3076 static int gc_update_entry(cls_method_context_t hctx
, uint32_t expiration_secs
,
3077 cls_rgw_gc_obj_info
& info
)
3079 cls_rgw_gc_obj_info old_info
;
3080 int ret
= gc_omap_get(hctx
, GC_OBJ_NAME_INDEX
, info
.tag
, &old_info
);
3083 get_time_key(old_info
.time
, &key
);
3084 ret
= gc_omap_remove(hctx
, GC_OBJ_TIME_INDEX
, key
);
3085 if (ret
< 0 && ret
!= -ENOENT
) {
3086 CLS_LOG(0, "ERROR: failed to remove key=%s\n", key
.c_str());
3090 info
.time
= ceph::real_clock::now();
3091 info
.time
+= make_timespan(expiration_secs
);
3092 ret
= gc_omap_set(hctx
, GC_OBJ_NAME_INDEX
, info
.tag
, &info
);
3097 get_time_key(info
.time
, &key
);
3098 ret
= gc_omap_set(hctx
, GC_OBJ_TIME_INDEX
, key
, &info
);
3105 CLS_LOG(0, "ERROR: gc_set_entry error info.tag=%s, ret=%d\n", info
.tag
.c_str(), ret
);
3106 gc_omap_remove(hctx
, GC_OBJ_NAME_INDEX
, info
.tag
);
3110 static int gc_defer_entry(cls_method_context_t hctx
, const string
& tag
, uint32_t expiration_secs
)
3112 cls_rgw_gc_obj_info info
;
3113 int ret
= gc_omap_get(hctx
, GC_OBJ_NAME_INDEX
, tag
, &info
);
3118 return gc_update_entry(hctx
, expiration_secs
, info
);
3121 int gc_record_decode(bufferlist
& bl
, cls_rgw_gc_obj_info
& e
)
3123 bufferlist::iterator iter
= bl
.begin();
3126 } catch (buffer::error
& err
) {
3127 CLS_LOG(0, "ERROR: failed to decode cls_rgw_gc_obj_info");
3133 static int rgw_cls_gc_set_entry(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3135 bufferlist::iterator in_iter
= in
->begin();
3137 cls_rgw_gc_set_entry_op op
;
3139 ::decode(op
, in_iter
);
3140 } catch (buffer::error
& err
) {
3141 CLS_LOG(1, "ERROR: rgw_cls_gc_set_entry(): failed to decode entry\n");
3145 return gc_update_entry(hctx
, op
.expiration_secs
, op
.info
);
3148 static int rgw_cls_gc_defer_entry(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3150 bufferlist::iterator in_iter
= in
->begin();
3152 cls_rgw_gc_defer_entry_op op
;
3154 ::decode(op
, in_iter
);
3155 } catch (buffer::error
& err
) {
3156 CLS_LOG(1, "ERROR: rgw_cls_gc_defer_entry(): failed to decode entry\n");
3160 return gc_defer_entry(hctx
, op
.tag
, op
.expiration_secs
);
3163 static int gc_iterate_entries(cls_method_context_t hctx
, const string
& marker
, bool expired_only
,
3164 string
& key_iter
, uint32_t max_entries
, bool *truncated
,
3165 int (*cb
)(cls_method_context_t
, const string
&, cls_rgw_gc_obj_info
&, void *),
3168 CLS_LOG(10, "gc_iterate_range");
3170 map
<string
, bufferlist
> keys
;
3171 string filter_prefix
, end_key
;
3179 if (key_iter
.empty()) {
3180 prepend_index_prefix(marker
, GC_OBJ_TIME_INDEX
, &start_key
);
3182 start_key
= key_iter
;
3186 real_time now
= ceph::real_clock::now();
3188 get_time_key(now
, &now_str
);
3189 prepend_index_prefix(now_str
, GC_OBJ_TIME_INDEX
, &end_key
);
3191 CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key
.c_str());
3197 #define GC_NUM_KEYS 32
3198 int ret
= cls_cxx_map_get_vals(hctx
, start_key
, filter
, GC_NUM_KEYS
, &keys
);
3203 map
<string
, bufferlist
>::iterator iter
= keys
.begin();
3204 if (iter
== keys
.end())
3207 for (; iter
!= keys
.end(); ++iter
) {
3208 const string
& key
= iter
->first
;
3209 cls_rgw_gc_obj_info e
;
3211 CLS_LOG(10, "gc_iterate_entries key=%s\n", key
.c_str());
3213 if (!end_key
.empty() && key
.compare(end_key
) >= 0)
3216 if (!key_in_index(key
, GC_OBJ_TIME_INDEX
))
3219 ret
= gc_record_decode(iter
->second
, e
);
3223 if (max_entries
&& (i
>= max_entries
)) {
3230 ret
= cb(hctx
, key
, e
, param
);
3237 start_key
= iter
->first
;
3242 static int gc_list_cb(cls_method_context_t hctx
, const string
& key
, cls_rgw_gc_obj_info
& info
, void *param
)
3244 list
<cls_rgw_gc_obj_info
> *l
= (list
<cls_rgw_gc_obj_info
> *)param
;
3249 static int gc_list_entries(cls_method_context_t hctx
, const string
& marker
,
3250 uint32_t max
, bool expired_only
,
3251 list
<cls_rgw_gc_obj_info
>& entries
, bool *truncated
)
3254 int ret
= gc_iterate_entries(hctx
, marker
, expired_only
,
3255 key_iter
, max
, truncated
,
3256 gc_list_cb
, &entries
);
3260 static int rgw_cls_gc_list(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3262 bufferlist::iterator in_iter
= in
->begin();
3264 cls_rgw_gc_list_op op
;
3266 ::decode(op
, in_iter
);
3267 } catch (buffer::error
& err
) {
3268 CLS_LOG(1, "ERROR: rgw_cls_gc_list(): failed to decode entry\n");
3272 cls_rgw_gc_list_ret op_ret
;
3273 int ret
= gc_list_entries(hctx
, op
.marker
, op
.max
, op
.expired_only
, op_ret
.entries
, &op_ret
.truncated
);
3277 ::encode(op_ret
, *out
);
3282 static int gc_remove(cls_method_context_t hctx
, list
<string
>& tags
)
3284 list
<string
>::iterator iter
;
3286 for (iter
= tags
.begin(); iter
!= tags
.end(); ++iter
) {
3287 string
& tag
= *iter
;
3288 cls_rgw_gc_obj_info info
;
3289 int ret
= gc_omap_get(hctx
, GC_OBJ_NAME_INDEX
, tag
, &info
);
3290 if (ret
== -ENOENT
) {
3291 CLS_LOG(0, "couldn't find tag in name index tag=%s\n", tag
.c_str());
3299 get_time_key(info
.time
, &time_key
);
3300 ret
= gc_omap_remove(hctx
, GC_OBJ_TIME_INDEX
, time_key
);
3301 if (ret
< 0 && ret
!= -ENOENT
)
3303 if (ret
== -ENOENT
) {
3304 CLS_LOG(0, "couldn't find key in time index key=%s\n", time_key
.c_str());
3307 ret
= gc_omap_remove(hctx
, GC_OBJ_NAME_INDEX
, tag
);
3308 if (ret
< 0 && ret
!= -ENOENT
)
3315 static int rgw_cls_gc_remove(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3317 bufferlist::iterator in_iter
= in
->begin();
3319 cls_rgw_gc_remove_op op
;
3321 ::decode(op
, in_iter
);
3322 } catch (buffer::error
& err
) {
3323 CLS_LOG(1, "ERROR: rgw_cls_gc_remove(): failed to decode entry\n");
3327 return gc_remove(hctx
, op
.tags
);
3330 static int rgw_cls_lc_set_entry(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3332 bufferlist::iterator in_iter
= in
->begin();
3334 cls_rgw_lc_set_entry_op op
;
3336 ::decode(op
, in_iter
);
3337 } catch (buffer::error
& err
) {
3338 CLS_LOG(1, "ERROR: rgw_cls_lc_set_entry(): failed to decode entry\n");
3343 ::encode(op
.entry
, bl
);
3345 int ret
= cls_cxx_map_set_val(hctx
, op
.entry
.first
, &bl
);
3349 static int rgw_cls_lc_rm_entry(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3351 bufferlist::iterator in_iter
= in
->begin();
3353 cls_rgw_lc_rm_entry_op op
;
3355 ::decode(op
, in_iter
);
3356 } catch (buffer::error
& err
) {
3357 CLS_LOG(1, "ERROR: rgw_cls_lc_rm_entry(): failed to decode entry\n");
3362 ::encode(op
.entry
, bl
);
3364 int ret
= cls_cxx_map_remove_key(hctx
, op
.entry
.first
);
3368 static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3370 bufferlist::iterator in_iter
= in
->begin();
3371 cls_rgw_lc_get_next_entry_ret op_ret
;
3372 cls_rgw_lc_get_next_entry_op op
;
3374 ::decode(op
, in_iter
);
3375 } catch (buffer::error
& err
) {
3376 CLS_LOG(1, "ERROR: rgw_cls_lc_get_next_entry: failed to decode op\n");
3380 map
<string
, bufferlist
> vals
;
3381 string filter_prefix
;
3382 int ret
= cls_cxx_map_get_vals(hctx
, op
.marker
, filter_prefix
, 1, &vals
);
3385 map
<string
, bufferlist
>::iterator it
;
3386 pair
<string
, int> entry
;
3387 if (!vals
.empty()) {
3389 in_iter
= it
->second
.begin();
3391 ::decode(entry
, in_iter
);
3392 } catch (buffer::error
& err
) {
3393 CLS_LOG(1, "ERROR: rgw_cls_lc_get_next_entry(): failed to decode entry\n");
3397 op_ret
.entry
= entry
;
3398 ::encode(op_ret
, *out
);
3402 static int rgw_cls_lc_list_entries(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3404 cls_rgw_lc_list_entries_op op
;
3405 bufferlist::iterator in_iter
= in
->begin();
3407 ::decode(op
, in_iter
);
3408 } catch (buffer::error
& err
) {
3409 CLS_LOG(1, "ERROR: rgw_cls_lc_list_entries(): failed to decode op\n");
3413 cls_rgw_lc_list_entries_ret op_ret
;
3414 bufferlist::iterator iter
;
3415 map
<string
, bufferlist
> vals
;
3416 string filter_prefix
;
3417 int ret
= cls_cxx_map_get_vals(hctx
, op
.marker
, filter_prefix
, op
.max_entries
, &vals
);
3420 map
<string
, bufferlist
>::iterator it
;
3421 pair
<string
, int> entry
;
3422 for (it
= vals
.begin(); it
!= vals
.end(); ++it
) {
3423 iter
= it
->second
.begin();
3425 ::decode(entry
, iter
);
3426 } catch (buffer::error
& err
) {
3427 CLS_LOG(1, "ERROR: rgw_cls_lc_list_entries(): failed to decode entry\n");
3430 op_ret
.entries
.insert(entry
);
3432 ::encode(op_ret
, *out
);
3436 static int rgw_cls_lc_put_head(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3438 bufferlist::iterator in_iter
= in
->begin();
3440 cls_rgw_lc_put_head_op op
;
3442 ::decode(op
, in_iter
);
3443 } catch (buffer::error
& err
) {
3444 CLS_LOG(1, "ERROR: rgw_cls_lc_put_head(): failed to decode entry\n");
3449 ::encode(op
.head
, bl
);
3450 int ret
= cls_cxx_map_write_header(hctx
,&bl
);
3454 static int rgw_cls_lc_get_head(cls_method_context_t hctx
, bufferlist
*in
, bufferlist
*out
)
3457 int ret
= cls_cxx_map_read_header(hctx
, &bl
);
3460 cls_rgw_lc_obj_head head
;
3461 if (bl
.length() != 0) {
3462 bufferlist::iterator iter
= bl
.begin();
3464 ::decode(head
, iter
);
3465 } catch (buffer::error
& err
) {
3466 CLS_LOG(0, "ERROR: rgw_cls_lc_get_head(): failed to decode entry %s\n",err
.what());
3470 head
.start_date
= 0;
3471 head
.marker
.clear();
3473 cls_rgw_lc_get_head_ret op_ret
;
3475 ::encode(op_ret
, *out
);
3481 CLS_LOG(1, "Loaded rgw class!");
3483 cls_handle_t h_class
;
3484 cls_method_handle_t h_rgw_bucket_init_index
;
3485 cls_method_handle_t h_rgw_bucket_set_tag_timeout
;
3486 cls_method_handle_t h_rgw_bucket_list
;
3487 cls_method_handle_t h_rgw_bucket_check_index
;
3488 cls_method_handle_t h_rgw_bucket_rebuild_index
;
3489 cls_method_handle_t h_rgw_bucket_update_stats
;
3490 cls_method_handle_t h_rgw_bucket_prepare_op
;
3491 cls_method_handle_t h_rgw_bucket_complete_op
;
3492 cls_method_handle_t h_rgw_bucket_link_olh
;
3493 cls_method_handle_t h_rgw_bucket_unlink_instance_op
;
3494 cls_method_handle_t h_rgw_bucket_read_olh_log
;
3495 cls_method_handle_t h_rgw_bucket_trim_olh_log
;
3496 cls_method_handle_t h_rgw_bucket_clear_olh
;
3497 cls_method_handle_t h_rgw_obj_remove
;
3498 cls_method_handle_t h_rgw_obj_store_pg_ver
;
3499 cls_method_handle_t h_rgw_obj_check_attrs_prefix
;
3500 cls_method_handle_t h_rgw_obj_check_mtime
;
3501 cls_method_handle_t h_rgw_bi_get_op
;
3502 cls_method_handle_t h_rgw_bi_put_op
;
3503 cls_method_handle_t h_rgw_bi_list_op
;
3504 cls_method_handle_t h_rgw_bi_log_list_op
;
3505 cls_method_handle_t h_rgw_dir_suggest_changes
;
3506 cls_method_handle_t h_rgw_user_usage_log_add
;
3507 cls_method_handle_t h_rgw_user_usage_log_read
;
3508 cls_method_handle_t h_rgw_user_usage_log_trim
;
3509 cls_method_handle_t h_rgw_gc_set_entry
;
3510 cls_method_handle_t h_rgw_gc_list
;
3511 cls_method_handle_t h_rgw_gc_remove
;
3512 cls_method_handle_t h_rgw_lc_set_entry
;
3513 cls_method_handle_t h_rgw_lc_rm_entry
;
3514 cls_method_handle_t h_rgw_lc_get_next_entry
;
3515 cls_method_handle_t h_rgw_lc_put_head
;
3516 cls_method_handle_t h_rgw_lc_get_head
;
3517 cls_method_handle_t h_rgw_lc_list_entries
;
3520 cls_register(RGW_CLASS
, &h_class
);
3523 cls_register_cxx_method(h_class
, RGW_BUCKET_INIT_INDEX
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_init_index
, &h_rgw_bucket_init_index
);
3524 cls_register_cxx_method(h_class
, RGW_BUCKET_SET_TAG_TIMEOUT
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_set_tag_timeout
, &h_rgw_bucket_set_tag_timeout
);
3525 cls_register_cxx_method(h_class
, RGW_BUCKET_LIST
, CLS_METHOD_RD
, rgw_bucket_list
, &h_rgw_bucket_list
);
3526 cls_register_cxx_method(h_class
, RGW_BUCKET_CHECK_INDEX
, CLS_METHOD_RD
, rgw_bucket_check_index
, &h_rgw_bucket_check_index
);
3527 cls_register_cxx_method(h_class
, RGW_BUCKET_REBUILD_INDEX
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_rebuild_index
, &h_rgw_bucket_rebuild_index
);
3528 cls_register_cxx_method(h_class
, RGW_BUCKET_UPDATE_STATS
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_update_stats
, &h_rgw_bucket_update_stats
);
3529 cls_register_cxx_method(h_class
, RGW_BUCKET_PREPARE_OP
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_prepare_op
, &h_rgw_bucket_prepare_op
);
3530 cls_register_cxx_method(h_class
, RGW_BUCKET_COMPLETE_OP
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_complete_op
, &h_rgw_bucket_complete_op
);
3531 cls_register_cxx_method(h_class
, RGW_BUCKET_LINK_OLH
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_link_olh
, &h_rgw_bucket_link_olh
);
3532 cls_register_cxx_method(h_class
, RGW_BUCKET_UNLINK_INSTANCE
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_unlink_instance
, &h_rgw_bucket_unlink_instance_op
);
3533 cls_register_cxx_method(h_class
, RGW_BUCKET_READ_OLH_LOG
, CLS_METHOD_RD
, rgw_bucket_read_olh_log
, &h_rgw_bucket_read_olh_log
);
3534 cls_register_cxx_method(h_class
, RGW_BUCKET_TRIM_OLH_LOG
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_trim_olh_log
, &h_rgw_bucket_trim_olh_log
);
3535 cls_register_cxx_method(h_class
, RGW_BUCKET_CLEAR_OLH
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bucket_clear_olh
, &h_rgw_bucket_clear_olh
);
3537 cls_register_cxx_method(h_class
, RGW_OBJ_REMOVE
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_obj_remove
, &h_rgw_obj_remove
);
3538 cls_register_cxx_method(h_class
, RGW_OBJ_STORE_PG_VER
, CLS_METHOD_WR
, rgw_obj_store_pg_ver
, &h_rgw_obj_store_pg_ver
);
3539 cls_register_cxx_method(h_class
, RGW_OBJ_CHECK_ATTRS_PREFIX
, CLS_METHOD_RD
, rgw_obj_check_attrs_prefix
, &h_rgw_obj_check_attrs_prefix
);
3540 cls_register_cxx_method(h_class
, RGW_OBJ_CHECK_MTIME
, CLS_METHOD_RD
, rgw_obj_check_mtime
, &h_rgw_obj_check_mtime
);
3542 cls_register_cxx_method(h_class
, RGW_BI_GET
, CLS_METHOD_RD
, rgw_bi_get_op
, &h_rgw_bi_get_op
);
3543 cls_register_cxx_method(h_class
, RGW_BI_PUT
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bi_put_op
, &h_rgw_bi_put_op
);
3544 cls_register_cxx_method(h_class
, RGW_BI_LIST
, CLS_METHOD_RD
, rgw_bi_list_op
, &h_rgw_bi_list_op
);
3546 cls_register_cxx_method(h_class
, RGW_BI_LOG_LIST
, CLS_METHOD_RD
, rgw_bi_log_list
, &h_rgw_bi_log_list_op
);
3547 cls_register_cxx_method(h_class
, RGW_BI_LOG_TRIM
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_bi_log_trim
, &h_rgw_bi_log_list_op
);
3548 cls_register_cxx_method(h_class
, RGW_DIR_SUGGEST_CHANGES
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_dir_suggest_changes
, &h_rgw_dir_suggest_changes
);
3551 cls_register_cxx_method(h_class
, RGW_USER_USAGE_LOG_ADD
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_user_usage_log_add
, &h_rgw_user_usage_log_add
);
3552 cls_register_cxx_method(h_class
, RGW_USER_USAGE_LOG_READ
, CLS_METHOD_RD
, rgw_user_usage_log_read
, &h_rgw_user_usage_log_read
);
3553 cls_register_cxx_method(h_class
, RGW_USER_USAGE_LOG_TRIM
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_user_usage_log_trim
, &h_rgw_user_usage_log_trim
);
3555 /* garbage collection */
3556 cls_register_cxx_method(h_class
, RGW_GC_SET_ENTRY
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_cls_gc_set_entry
, &h_rgw_gc_set_entry
);
3557 cls_register_cxx_method(h_class
, RGW_GC_DEFER_ENTRY
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_cls_gc_defer_entry
, &h_rgw_gc_set_entry
);
3558 cls_register_cxx_method(h_class
, RGW_GC_LIST
, CLS_METHOD_RD
, rgw_cls_gc_list
, &h_rgw_gc_list
);
3559 cls_register_cxx_method(h_class
, RGW_GC_REMOVE
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_cls_gc_remove
, &h_rgw_gc_remove
);
3561 /* lifecycle bucket list */
3562 cls_register_cxx_method(h_class
, RGW_LC_SET_ENTRY
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_cls_lc_set_entry
, &h_rgw_lc_set_entry
);
3563 cls_register_cxx_method(h_class
, RGW_LC_RM_ENTRY
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_cls_lc_rm_entry
, &h_rgw_lc_rm_entry
);
3564 cls_register_cxx_method(h_class
, RGW_LC_GET_NEXT_ENTRY
, CLS_METHOD_RD
, rgw_cls_lc_get_next_entry
, &h_rgw_lc_get_next_entry
);
3565 cls_register_cxx_method(h_class
, RGW_LC_PUT_HEAD
, CLS_METHOD_RD
| CLS_METHOD_WR
, rgw_cls_lc_put_head
, &h_rgw_lc_put_head
);
3566 cls_register_cxx_method(h_class
, RGW_LC_GET_HEAD
, CLS_METHOD_RD
, rgw_cls_lc_get_head
, &h_rgw_lc_get_head
);
3567 cls_register_cxx_method(h_class
, RGW_LC_LIST_ENTRIES
, CLS_METHOD_RD
, rgw_cls_lc_list_entries
, &h_rgw_lc_list_entries
);