1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
10 #include <boost/format.hpp>
12 #include "common/errno.h"
13 #include "common/ceph_json.h"
14 #include "include/scope_guard.h"
16 #include "rgw_datalog.h"
17 #include "rgw_rados.h"
20 #include "rgw_acl_s3.h"
21 #include "rgw_tag_s3.h"
23 #include "include/types.h"
24 #include "rgw_bucket.h"
26 #include "rgw_string.h"
27 #include "rgw_multi.h"
29 #include "rgw_bucket_sync.h"
31 #include "services/svc_zone.h"
32 #include "services/svc_sys_obj.h"
33 #include "services/svc_bucket.h"
34 #include "services/svc_bucket_sync.h"
35 #include "services/svc_meta.h"
36 #include "services/svc_meta_be_sobj.h"
37 #include "services/svc_user.h"
38 #include "services/svc_cls.h"
39 #include "services/svc_bilog_rados.h"
41 #include "include/rados/librados.hpp"
42 // until everything is moved from rgw_common
43 #include "rgw_common.h"
44 #include "rgw_reshard.h"
46 #include "rgw_bucket_layout.h"
47 #include "common/errno.h"
49 // stolen from src/cls/version/cls_version.cc
50 #define VERSION_ATTR "ceph.objclass.version"
52 #include "cls/user/cls_user_types.h"
54 #include "rgw_sal_rados.h"
56 #define dout_context g_ceph_context
57 #define dout_subsys ceph_subsys_rgw
59 #define BUCKET_TAG_TIMEOUT 30
63 // default number of entries to list with each bucket listing call
64 // (use marker to bridge between calls)
65 static constexpr size_t listing_max_entries
= 1000;
67 void init_bucket(rgw_bucket
*b
, const char *t
, const char *n
, const char *dp
, const char *ip
, const char *m
, const char *id
)
73 b
->explicit_placement
.data_pool
= rgw_pool(dp
);
74 b
->explicit_placement
.index_pool
= rgw_pool(ip
);
78 * The tenant_name is always returned on purpose. May be empty, of course.
80 static void parse_bucket(const string
& bucket
,
83 string
*bucket_instance
= nullptr /* optional */)
86 * expected format: [tenant/]bucket:bucket_instance
88 int pos
= bucket
.find('/');
90 *tenant_name
= bucket
.substr(0, pos
);
94 string bn
= bucket
.substr(pos
+ 1);
97 *bucket_name
= std::move(bn
);
100 *bucket_name
= bn
.substr(0, pos
);
101 if (bucket_instance
) {
102 *bucket_instance
= bn
.substr(pos
+ 1);
106 * deal with the possible tenant:bucket:bucket_instance case
108 if (tenant_name
->empty()) {
109 pos
= bucket_instance
->find(':');
111 *tenant_name
= *bucket_name
;
112 *bucket_name
= bucket_instance
->substr(0, pos
);
113 *bucket_instance
= bucket_instance
->substr(pos
+ 1);
119 * Note that this is not a reversal of parse_bucket(). That one deals
120 * with the syntax we need in metadata and such. This one deals with
121 * the representation in RADOS pools. We chose '/' because it's not
122 * acceptable in bucket names and thus qualified buckets cannot conflict
123 * with the legacy or S3 buckets.
125 std::string
rgw_make_bucket_entry_name(const std::string
& tenant_name
,
126 const std::string
& bucket_name
) {
127 std::string bucket_entry
;
129 if (bucket_name
.empty()) {
130 bucket_entry
.clear();
131 } else if (tenant_name
.empty()) {
132 bucket_entry
= bucket_name
;
134 bucket_entry
= tenant_name
+ "/" + bucket_name
;
141 * Tenants are separated from buckets in URLs by a colon in S3.
142 * This function is not to be used on Swift URLs, not even for COPY arguments.
144 int rgw_parse_url_bucket(const string
&bucket
, const string
& auth_tenant
,
145 string
&tenant_name
, string
&bucket_name
) {
147 int pos
= bucket
.find(':');
150 * N.B.: We allow ":bucket" syntax with explicit empty tenant in order
151 * to refer to the legacy tenant, in case users in new named tenants
152 * want to access old global buckets.
154 tenant_name
= bucket
.substr(0, pos
);
155 bucket_name
= bucket
.substr(pos
+ 1);
156 if (bucket_name
.empty()) {
157 return -ERR_INVALID_BUCKET_NAME
;
160 tenant_name
= auth_tenant
;
161 bucket_name
= bucket
;
166 int rgw_bucket_parse_bucket_instance(const string
& bucket_instance
, string
*bucket_name
, string
*bucket_id
, int *shard_id
)
168 auto pos
= bucket_instance
.rfind(':');
169 if (pos
== string::npos
) {
173 string first
= bucket_instance
.substr(0, pos
);
174 string second
= bucket_instance
.substr(pos
+ 1);
176 pos
= first
.find(':');
178 if (pos
== string::npos
) {
180 *bucket_name
= first
;
185 *bucket_name
= first
.substr(0, pos
);
186 *bucket_id
= first
.substr(pos
+ 1);
189 *shard_id
= strict_strtol(second
.c_str(), 10, &err
);
197 // parse key in format: [tenant/]name:instance[:shard_id]
198 int rgw_bucket_parse_bucket_key(CephContext
*cct
, const string
& key
,
199 rgw_bucket
*bucket
, int *shard_id
)
201 std::string_view name
{key
};
202 std::string_view instance
;
205 auto pos
= name
.find('/');
206 if (pos
!= string::npos
) {
207 auto tenant
= name
.substr(0, pos
);
208 bucket
->tenant
.assign(tenant
.begin(), tenant
.end());
209 name
= name
.substr(pos
+ 1);
211 bucket
->tenant
.clear();
214 // split name:instance
215 pos
= name
.find(':');
216 if (pos
!= string::npos
) {
217 instance
= name
.substr(pos
+ 1);
218 name
= name
.substr(0, pos
);
220 bucket
->name
.assign(name
.begin(), name
.end());
222 // split instance:shard
223 pos
= instance
.find(':');
224 if (pos
== string::npos
) {
225 bucket
->bucket_id
.assign(instance
.begin(), instance
.end());
233 auto shard
= instance
.substr(pos
+ 1);
235 auto id
= strict_strtol(shard
.data(), 10, &err
);
238 ldout(cct
, 0) << "ERROR: failed to parse bucket shard '"
239 << instance
.data() << "': " << err
<< dendl
;
247 instance
= instance
.substr(0, pos
);
248 bucket
->bucket_id
.assign(instance
.begin(), instance
.end());
252 static void dump_mulipart_index_results(list
<rgw_obj_index_key
>& objs_to_unlink
,
255 for (const auto& o
: objs_to_unlink
) {
256 f
->dump_string("object", o
.name
);
260 void check_bad_user_bucket_mapping(rgw::sal::Store
* store
, rgw::sal::User
& user
,
263 const DoutPrefixProvider
*dpp
)
265 rgw::sal::BucketList user_buckets
;
268 CephContext
*cct
= store
->ctx();
270 size_t max_entries
= cct
->_conf
->rgw_list_buckets_max_chunk
;
273 int ret
= user
.list_buckets(dpp
, marker
, string(), max_entries
, false, user_buckets
, y
);
275 ldout(store
->ctx(), 0) << "failed to read user buckets: "
276 << cpp_strerror(-ret
) << dendl
;
280 map
<string
, std::unique_ptr
<rgw::sal::Bucket
>>& buckets
= user_buckets
.get_buckets();
281 for (auto i
= buckets
.begin();
286 auto& bucket
= i
->second
;
288 std::unique_ptr
<rgw::sal::Bucket
> actual_bucket
;
289 int r
= store
->get_bucket(dpp
, &user
, user
.get_tenant(), bucket
->get_name(), &actual_bucket
, null_yield
);
291 ldout(store
->ctx(), 0) << "could not get bucket info for bucket=" << bucket
<< dendl
;
295 if (actual_bucket
->get_name().compare(bucket
->get_name()) != 0 ||
296 actual_bucket
->get_tenant().compare(bucket
->get_tenant()) != 0 ||
297 actual_bucket
->get_marker().compare(bucket
->get_marker()) != 0 ||
298 actual_bucket
->get_bucket_id().compare(bucket
->get_bucket_id()) != 0) {
299 cout
<< "bucket info mismatch: expected " << actual_bucket
<< " got " << bucket
<< std::endl
;
301 cout
<< "fixing" << std::endl
;
302 r
= actual_bucket
->chown(dpp
, user
, null_yield
);
304 cerr
<< "failed to fix bucket: " << cpp_strerror(-r
) << std::endl
;
309 } while (user_buckets
.is_truncated());
312 // returns true if entry is in the empty namespace. note: function
313 // type conforms to type RGWBucketListNameFilter
314 bool rgw_bucket_object_check_filter(const std::string
& oid
)
316 const static std::string empty_ns
;
317 rgw_obj_key key
; // thrown away but needed for parsing
318 return rgw_obj_key::oid_to_key_in_ns(oid
, &key
, empty_ns
);
321 int rgw_remove_object(const DoutPrefixProvider
*dpp
, rgw::sal::Store
* store
, rgw::sal::Bucket
* bucket
, rgw_obj_key
& key
)
323 RGWObjectCtx
rctx(store
);
325 if (key
.instance
.empty()) {
326 key
.instance
= "null";
329 std::unique_ptr
<rgw::sal::Object
> object
= bucket
->get_object(key
);
331 return object
->delete_object(dpp
, &rctx
, null_yield
);
334 static void set_err_msg(std::string
*sink
, std::string msg
)
336 if (sink
&& !msg
.empty())
340 int RGWBucket::init(rgw::sal::Store
* _store
, RGWBucketAdminOpState
& op_state
,
341 optional_yield y
, const DoutPrefixProvider
*dpp
, std::string
*err_msg
)
344 set_err_msg(err_msg
, "no storage!");
350 std::string bucket_name
= op_state
.get_bucket_name();
352 if (bucket_name
.empty() && op_state
.get_user_id().empty())
355 user
= store
->get_user(op_state
.get_user_id());
356 std::string tenant
= user
->get_tenant();
358 // split possible tenant/name
359 auto pos
= bucket_name
.find('/');
360 if (pos
!= string::npos
) {
361 tenant
= bucket_name
.substr(0, pos
);
362 bucket_name
= bucket_name
.substr(pos
+ 1);
365 int r
= store
->get_bucket(dpp
, user
.get(), tenant
, bucket_name
, &bucket
, y
);
367 set_err_msg(err_msg
, "failed to fetch bucket info for bucket=" + bucket_name
);
371 op_state
.set_bucket(bucket
->clone());
373 if (!rgw::sal::User::empty(user
.get())) {
374 r
= user
->load_user(dpp
, y
);
376 set_err_msg(err_msg
, "failed to fetch user info");
381 op_state
.display_name
= user
->get_display_name();
387 bool rgw_find_bucket_by_id(const DoutPrefixProvider
*dpp
, CephContext
*cct
, rgw::sal::Store
* store
,
388 const string
& marker
, const string
& bucket_id
, rgw_bucket
* bucket_out
)
391 bool truncated
= false;
394 int ret
= store
->meta_list_keys_init(dpp
, "bucket.instance", marker
, &handle
);
396 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
397 store
->meta_list_keys_complete(handle
);
402 ret
= store
->meta_list_keys_next(dpp
, handle
, 1000, keys
, &truncated
);
404 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
405 store
->meta_list_keys_complete(handle
);
408 for (list
<string
>::iterator iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
410 ret
= rgw_bucket_parse_bucket_key(cct
, s
, bucket_out
, nullptr);
414 if (bucket_id
== bucket_out
->bucket_id
) {
415 store
->meta_list_keys_complete(handle
);
420 store
->meta_list_keys_complete(handle
);
424 int RGWBucket::chown(RGWBucketAdminOpState
& op_state
, const string
& marker
,
425 optional_yield y
, const DoutPrefixProvider
*dpp
, std::string
*err_msg
)
427 /* User passed in by rgw_admin is the new user; get the current user and set it in
429 std::unique_ptr
<rgw::sal::User
> old_user
= store
->get_user(bucket
->get_info().owner
);
430 bucket
->set_owner(old_user
.get());
432 return rgw_chown_bucket_and_objects(store
, bucket
.get(), user
.get(), marker
, err_msg
, dpp
, y
);
435 int RGWBucket::set_quota(RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
, std::string
*err_msg
)
437 bucket
= op_state
.get_bucket()->clone();
439 bucket
->get_info().quota
= op_state
.quota
;
440 int r
= bucket
->put_info(dpp
, false, real_time());
442 set_err_msg(err_msg
, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r
));
448 int RGWBucket::remove_object(const DoutPrefixProvider
*dpp
, RGWBucketAdminOpState
& op_state
, std::string
*err_msg
)
450 std::string object_name
= op_state
.get_object_name();
452 rgw_obj_key
key(object_name
);
454 bucket
= op_state
.get_bucket()->clone();
456 int ret
= rgw_remove_object(dpp
, store
, bucket
.get(), key
);
458 set_err_msg(err_msg
, "unable to remove object" + cpp_strerror(-ret
));
465 static void dump_bucket_index(const vector
<rgw_bucket_dir_entry
>& objs
, Formatter
*f
)
467 for (auto iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
468 f
->dump_string("object", iter
->key
.name
);
472 static void dump_bucket_usage(map
<RGWObjCategory
, RGWStorageStats
>& stats
, Formatter
*formatter
)
474 map
<RGWObjCategory
, RGWStorageStats
>::iterator iter
;
476 formatter
->open_object_section("usage");
477 for (iter
= stats
.begin(); iter
!= stats
.end(); ++iter
) {
478 RGWStorageStats
& s
= iter
->second
;
479 formatter
->open_object_section(to_string(iter
->first
));
481 formatter
->close_section();
483 formatter
->close_section();
486 static void dump_index_check(map
<RGWObjCategory
, RGWStorageStats
> existing_stats
,
487 map
<RGWObjCategory
, RGWStorageStats
> calculated_stats
,
488 Formatter
*formatter
)
490 formatter
->open_object_section("check_result");
491 formatter
->open_object_section("existing_header");
492 dump_bucket_usage(existing_stats
, formatter
);
493 formatter
->close_section();
494 formatter
->open_object_section("calculated_header");
495 dump_bucket_usage(calculated_stats
, formatter
);
496 formatter
->close_section();
497 formatter
->close_section();
500 int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState
& op_state
,
501 RGWFormatterFlusher
& flusher
,
502 const DoutPrefixProvider
*dpp
, std::string
*err_msg
)
504 bool fix_index
= op_state
.will_fix_index();
507 map
<string
, bool> meta_objs
;
508 map
<rgw_obj_index_key
, string
> all_objs
;
510 bucket
= op_state
.get_bucket()->clone();
512 rgw::sal::Bucket::ListParams params
;
514 params
.list_versions
= true;
515 params
.ns
= RGW_OBJ_NS_MULTIPART
;
518 rgw::sal::Bucket::ListResults results
;
519 int r
= bucket
->list(dpp
, params
, listing_max_entries
, results
, null_yield
);
521 set_err_msg(err_msg
, "failed to list objects in bucket=" + bucket
->get_name() +
522 " err=" + cpp_strerror(-r
));
526 is_truncated
= results
.is_truncated
;
528 vector
<rgw_bucket_dir_entry
>::iterator iter
;
529 for (iter
= results
.objs
.begin(); iter
!= results
.objs
.end(); ++iter
) {
530 rgw_obj_index_key key
= iter
->key
;
531 rgw_obj
obj(bucket
->get_key(), key
);
532 string oid
= obj
.get_oid();
534 int pos
= oid
.find_last_of('.');
536 /* obj has no suffix */
540 string name
= oid
.substr(0, pos
);
541 string suffix
= oid
.substr(pos
+ 1);
543 if (suffix
.compare("meta") == 0) {
544 meta_objs
[name
] = true;
546 all_objs
[key
] = name
;
550 } while (is_truncated
);
552 list
<rgw_obj_index_key
> objs_to_unlink
;
553 Formatter
*f
= flusher
.get_formatter();
555 f
->open_array_section("invalid_multipart_entries");
557 for (auto aiter
= all_objs
.begin(); aiter
!= all_objs
.end(); ++aiter
) {
558 string
& name
= aiter
->second
;
560 if (meta_objs
.find(name
) == meta_objs
.end()) {
561 objs_to_unlink
.push_back(aiter
->first
);
564 if (objs_to_unlink
.size() > listing_max_entries
) {
566 int r
= bucket
->remove_objs_from_index(dpp
, objs_to_unlink
);
568 set_err_msg(err_msg
, "ERROR: remove_obj_from_index() returned error: " +
574 dump_mulipart_index_results(objs_to_unlink
, flusher
.get_formatter());
576 objs_to_unlink
.clear();
581 int r
= bucket
->remove_objs_from_index(dpp
, objs_to_unlink
);
583 set_err_msg(err_msg
, "ERROR: remove_obj_from_index() returned error: " +
590 dump_mulipart_index_results(objs_to_unlink
, f
);
597 int RGWBucket::check_object_index(const DoutPrefixProvider
*dpp
,
598 RGWBucketAdminOpState
& op_state
,
599 RGWFormatterFlusher
& flusher
,
601 std::string
*err_msg
)
604 bool fix_index
= op_state
.will_fix_index();
607 set_err_msg(err_msg
, "check-objects flag requires fix index enabled");
611 bucket
->set_tag_timeout(dpp
, BUCKET_TAG_TIMEOUT
);
613 rgw::sal::Bucket::ListResults results
;
614 results
.is_truncated
= true;
616 Formatter
*formatter
= flusher
.get_formatter();
617 formatter
->open_object_section("objects");
618 while (results
.is_truncated
) {
619 rgw::sal::Bucket::ListParams params
;
620 params
.marker
= results
.next_marker
;
621 params
.force_check_filter
= rgw_bucket_object_check_filter
;
623 int r
= bucket
->list(dpp
, params
, listing_max_entries
, results
, y
);
628 set_err_msg(err_msg
, "ERROR: failed operation r=" + cpp_strerror(-r
));
631 dump_bucket_index(results
.objs
, formatter
);
635 formatter
->close_section();
637 bucket
->set_tag_timeout(dpp
, 0);
643 int RGWBucket::check_index(const DoutPrefixProvider
*dpp
,
644 RGWBucketAdminOpState
& op_state
,
645 map
<RGWObjCategory
, RGWStorageStats
>& existing_stats
,
646 map
<RGWObjCategory
, RGWStorageStats
>& calculated_stats
,
647 std::string
*err_msg
)
649 bool fix_index
= op_state
.will_fix_index();
651 int r
= bucket
->check_index(dpp
, existing_stats
, calculated_stats
);
653 set_err_msg(err_msg
, "failed to check index error=" + cpp_strerror(-r
));
658 r
= bucket
->rebuild_index(dpp
);
660 set_err_msg(err_msg
, "failed to rebuild index err=" + cpp_strerror(-r
));
668 int RGWBucket::sync(RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
, std::string
*err_msg
)
670 if (!store
->is_meta_master()) {
671 set_err_msg(err_msg
, "ERROR: failed to update bucket sync: only allowed on meta master zone");
674 bool sync
= op_state
.will_sync_bucket();
676 bucket
->get_info().flags
&= ~BUCKET_DATASYNC_DISABLED
;
678 bucket
->get_info().flags
|= BUCKET_DATASYNC_DISABLED
;
681 int r
= bucket
->put_info(dpp
, false, real_time());
683 set_err_msg(err_msg
, "ERROR: failed writing bucket instance info:" + cpp_strerror(-r
));
687 int shards_num
= bucket
->get_info().layout
.current_index
.layout
.normal
.num_shards
? bucket
->get_info().layout
.current_index
.layout
.normal
.num_shards
: 1;
688 int shard_id
= bucket
->get_info().layout
.current_index
.layout
.normal
.num_shards
? 0 : -1;
691 r
= static_cast<rgw::sal::RadosStore
*>(store
)->svc()->bilog_rados
->log_stop(dpp
, bucket
->get_info(), -1);
693 set_err_msg(err_msg
, "ERROR: failed writing stop bilog:" + cpp_strerror(-r
));
697 r
= static_cast<rgw::sal::RadosStore
*>(store
)->svc()->bilog_rados
->log_start(dpp
, bucket
->get_info(), -1);
699 set_err_msg(err_msg
, "ERROR: failed writing resync bilog:" + cpp_strerror(-r
));
704 for (int i
= 0; i
< shards_num
; ++i
, ++shard_id
) {
705 r
= static_cast<rgw::sal::RadosStore
*>(store
)->svc()->datalog_rados
->add_entry(dpp
, bucket
->get_info(), shard_id
);
707 set_err_msg(err_msg
, "ERROR: failed writing data log:" + cpp_strerror(-r
));
716 int RGWBucket::policy_bl_to_stream(bufferlist
& bl
, ostream
& o
)
718 RGWAccessControlPolicy_S3
policy(g_ceph_context
);
719 int ret
= decode_bl(bl
, policy
);
721 ldout(store
->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl
;
727 int rgw_object_get_attr(const DoutPrefixProvider
*dpp
,
728 rgw::sal::Store
* store
, rgw::sal::Object
* obj
,
729 const char* attr_name
, bufferlist
& out_bl
, optional_yield y
)
731 RGWObjectCtx
obj_ctx(store
);
732 std::unique_ptr
<rgw::sal::Object::ReadOp
> rop
= obj
->get_read_op(&obj_ctx
);
734 return rop
->get_attr(dpp
, attr_name
, out_bl
, y
);
737 int RGWBucket::get_policy(RGWBucketAdminOpState
& op_state
, RGWAccessControlPolicy
& policy
, optional_yield y
, const DoutPrefixProvider
*dpp
)
740 std::string object_name
= op_state
.get_object_name();
742 bucket
= op_state
.get_bucket()->clone();
744 if (!object_name
.empty()) {
746 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(rgw_obj_key(object_name
));
748 ret
= rgw_object_get_attr(dpp
, store
, obj
.get(), RGW_ATTR_ACL
, bl
, y
);
753 ret
= decode_bl(bl
, policy
);
755 ldout(store
->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl
;
760 map
<string
, bufferlist
>::iterator aiter
= bucket
->get_attrs().find(RGW_ATTR_ACL
);
761 if (aiter
== bucket
->get_attrs().end()) {
765 ret
= decode_bl(aiter
->second
, policy
);
767 ldout(store
->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl
;
774 int RGWBucketAdminOp::get_policy(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
,
775 RGWAccessControlPolicy
& policy
, const DoutPrefixProvider
*dpp
)
779 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
);
783 ret
= bucket
.get_policy(op_state
, policy
, null_yield
, dpp
);
790 /* Wrappers to facilitate RESTful interface */
793 int RGWBucketAdminOp::get_policy(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
,
794 RGWFormatterFlusher
& flusher
, const DoutPrefixProvider
*dpp
)
796 RGWAccessControlPolicy
policy(store
->ctx());
798 int ret
= get_policy(store
, op_state
, policy
, dpp
);
802 Formatter
*formatter
= flusher
.get_formatter();
806 formatter
->open_object_section("policy");
807 policy
.dump(formatter
);
808 formatter
->close_section();
815 int RGWBucketAdminOp::dump_s3_policy(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
,
816 ostream
& os
, const DoutPrefixProvider
*dpp
)
818 RGWAccessControlPolicy_S3
policy(store
->ctx());
820 int ret
= get_policy(store
, op_state
, policy
, dpp
);
829 int RGWBucketAdminOp::unlink(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
)
833 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
);
837 return static_cast<rgw::sal::RadosStore
*>(store
)->ctl()->bucket
->unlink_bucket(op_state
.get_user_id(), op_state
.get_bucket()->get_info().bucket
, null_yield
, dpp
, true);
840 int RGWBucketAdminOp::link(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
, string
*err
)
842 if (!op_state
.is_user_op()) {
843 set_err_msg(err
, "empty user id");
848 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
, err
);
852 string bucket_id
= op_state
.get_bucket_id();
853 std::string display_name
= op_state
.get_user_display_name();
854 std::unique_ptr
<rgw::sal::Bucket
> loc_bucket
;
855 std::unique_ptr
<rgw::sal::Bucket
> old_bucket
;
857 loc_bucket
= op_state
.get_bucket()->clone();
859 if (!bucket_id
.empty() && bucket_id
!= loc_bucket
->get_bucket_id()) {
861 "specified bucket id does not match " + loc_bucket
->get_bucket_id());
865 old_bucket
= loc_bucket
->clone();
867 loc_bucket
->get_key().tenant
= op_state
.get_user_id().tenant
;
869 if (!op_state
.new_bucket_name
.empty()) {
870 auto pos
= op_state
.new_bucket_name
.find('/');
871 if (pos
!= string::npos
) {
872 loc_bucket
->get_key().tenant
= op_state
.new_bucket_name
.substr(0, pos
);
873 loc_bucket
->get_key().name
= op_state
.new_bucket_name
.substr(pos
+ 1);
875 loc_bucket
->get_key().name
= op_state
.new_bucket_name
;
879 RGWObjVersionTracker objv_tracker
;
880 RGWObjVersionTracker old_version
= loc_bucket
->get_info().objv_tracker
;
882 map
<string
, bufferlist
>::iterator aiter
= loc_bucket
->get_attrs().find(RGW_ATTR_ACL
);
883 if (aiter
== loc_bucket
->get_attrs().end()) {
884 // should never happen; only pre-argonaut buckets lacked this.
885 ldpp_dout(dpp
, 0) << "WARNING: can't bucket link because no acl on bucket=" << old_bucket
<< dendl
;
887 "While crossing the Anavros you have displeased the goddess Hera."
888 " You must sacrifice your ancient bucket " + loc_bucket
->get_bucket_id());
891 bufferlist
& aclbl
= aiter
->second
;
892 RGWAccessControlPolicy policy
;
895 auto iter
= aclbl
.cbegin();
896 decode(policy
, iter
);
897 owner
= policy
.get_owner();
898 } catch (buffer::error
& e
) {
899 set_err_msg(err
, "couldn't decode policy");
903 int r
= static_cast<rgw::sal::RadosStore
*>(store
)->ctl()->bucket
->unlink_bucket(owner
.get_id(), old_bucket
->get_info().bucket
, null_yield
, dpp
, false);
905 set_err_msg(err
, "could not unlink policy from user " + owner
.get_id().to_str());
909 // now update the user for the bucket...
910 if (display_name
.empty()) {
911 ldpp_dout(dpp
, 0) << "WARNING: user " << op_state
.get_user_id() << " has no display name set" << dendl
;
914 RGWAccessControlPolicy policy_instance
;
915 policy_instance
.create_default(op_state
.get_user_id(), display_name
);
916 owner
= policy_instance
.get_owner();
919 policy_instance
.encode(aclbl
);
921 bool exclusive
= false;
922 loc_bucket
->get_info().owner
= op_state
.get_user_id();
923 if (*loc_bucket
!= *old_bucket
) {
924 loc_bucket
->get_info().bucket
= loc_bucket
->get_key();
925 loc_bucket
->get_info().objv_tracker
.version_for_read()->ver
= 0;
929 r
= loc_bucket
->put_info(dpp
, exclusive
, ceph::real_time());
931 set_err_msg(err
, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r
));
936 RGWBucketEntryPoint ep
;
937 ep
.bucket
= loc_bucket
->get_info().bucket
;
938 ep
.owner
= op_state
.get_user_id();
939 ep
.creation_time
= loc_bucket
->get_info().creation_time
;
941 rgw::sal::Attrs ep_attrs
;
942 rgw_ep_info ep_data
{ep
, ep_attrs
};
944 r
= static_cast<rgw::sal::RadosStore
*>(store
)->ctl()->bucket
->link_bucket(op_state
.get_user_id(), loc_bucket
->get_info().bucket
, loc_bucket
->get_info().creation_time
, null_yield
, dpp
, true, &ep_data
);
946 set_err_msg(err
, "failed to relink bucket");
950 if (*loc_bucket
!= *old_bucket
) {
951 // like RGWRados::delete_bucket -- excepting no bucket_index work.
952 r
= static_cast<rgw::sal::RadosStore
*>(store
)->ctl()->bucket
->remove_bucket_entrypoint_info(
953 old_bucket
->get_key(), null_yield
, dpp
,
954 RGWBucketCtl::Bucket::RemoveParams()
955 .set_objv_tracker(&ep_data
.ep_objv
));
957 set_err_msg(err
, "failed to unlink old bucket " + old_bucket
->get_tenant() + "/" + old_bucket
->get_name());
960 r
= static_cast<rgw::sal::RadosStore
*>(store
)->ctl()->bucket
->remove_bucket_instance_info(
961 old_bucket
->get_key(), old_bucket
->get_info(),
963 RGWBucketCtl::BucketInstance::RemoveParams()
964 .set_objv_tracker(&ep_data
.ep_objv
));
966 set_err_msg(err
, "failed to unlink old bucket " + old_bucket
->get_tenant() + "/" + old_bucket
->get_name());
974 int RGWBucketAdminOp::chown(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
, const string
& marker
, const DoutPrefixProvider
*dpp
, string
*err
)
978 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
, err
);
982 return bucket
.chown(op_state
, marker
, null_yield
, dpp
, err
);
986 int RGWBucketAdminOp::check_index(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
,
987 RGWFormatterFlusher
& flusher
, optional_yield y
, const DoutPrefixProvider
*dpp
)
990 map
<RGWObjCategory
, RGWStorageStats
> existing_stats
;
991 map
<RGWObjCategory
, RGWStorageStats
> calculated_stats
;
996 ret
= bucket
.init(store
, op_state
, null_yield
, dpp
);
1000 Formatter
*formatter
= flusher
.get_formatter();
1003 ret
= bucket
.check_bad_index_multipart(op_state
, flusher
, dpp
);
1007 ret
= bucket
.check_object_index(dpp
, op_state
, flusher
, y
);
1011 ret
= bucket
.check_index(dpp
, op_state
, existing_stats
, calculated_stats
);
1015 dump_index_check(existing_stats
, calculated_stats
, formatter
);
1021 int RGWBucketAdminOp::remove_bucket(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
,
1022 optional_yield y
, const DoutPrefixProvider
*dpp
,
1023 bool bypass_gc
, bool keep_index_consistent
)
1025 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1026 std::unique_ptr
<rgw::sal::User
> user
= store
->get_user(op_state
.get_user_id());
1028 int ret
= store
->get_bucket(dpp
, user
.get(), user
->get_tenant(), op_state
.get_bucket_name(),
1034 ret
= bucket
->remove_bucket_bypass_gc(op_state
.get_max_aio(), keep_index_consistent
, y
, dpp
);
1036 ret
= bucket
->remove_bucket(dpp
, op_state
.will_delete_children(),
1042 int RGWBucketAdminOp::remove_object(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
)
1046 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
);
1050 return bucket
.remove_object(dpp
, op_state
);
1053 int RGWBucketAdminOp::sync_bucket(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
, string
*err_msg
)
1056 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
, err_msg
);
1061 return bucket
.sync(op_state
, dpp
, err_msg
);
1064 static int bucket_stats(rgw::sal::Store
* store
,
1065 const std::string
& tenant_name
,
1066 const std::string
& bucket_name
,
1067 Formatter
*formatter
,
1068 const DoutPrefixProvider
*dpp
)
1070 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1071 map
<RGWObjCategory
, RGWStorageStats
> stats
;
1074 int ret
= store
->get_bucket(dpp
, nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1079 string bucket_ver
, master_ver
;
1081 ret
= bucket
->read_stats(dpp
, RGW_NO_SHARD
, &bucket_ver
, &master_ver
, stats
, &max_marker
);
1083 cerr
<< "error getting bucket stats bucket=" << bucket
->get_name() << " ret=" << ret
<< std::endl
;
1088 utime_t
ctime_ut(bucket
->get_creation_time());
1090 formatter
->open_object_section("stats");
1091 formatter
->dump_string("bucket", bucket
->get_name());
1092 formatter
->dump_int("num_shards",
1093 bucket
->get_info().layout
.current_index
.layout
.normal
.num_shards
);
1094 formatter
->dump_string("tenant", bucket
->get_tenant());
1095 formatter
->dump_string("zonegroup", bucket
->get_info().zonegroup
);
1096 formatter
->dump_string("placement_rule", bucket
->get_info().placement_rule
.to_str());
1097 ::encode_json("explicit_placement", bucket
->get_key().explicit_placement
, formatter
);
1098 formatter
->dump_string("id", bucket
->get_bucket_id());
1099 formatter
->dump_string("marker", bucket
->get_marker());
1100 formatter
->dump_stream("index_type") << bucket
->get_info().layout
.current_index
.layout
.type
;
1101 ::encode_json("owner", bucket
->get_info().owner
, formatter
);
1102 formatter
->dump_string("ver", bucket_ver
);
1103 formatter
->dump_string("master_ver", master_ver
);
1104 ut
.gmtime(formatter
->dump_stream("mtime"));
1105 ctime_ut
.gmtime(formatter
->dump_stream("creation_time"));
1106 formatter
->dump_string("max_marker", max_marker
);
1107 dump_bucket_usage(stats
, formatter
);
1108 encode_json("bucket_quota", bucket
->get_info().quota
, formatter
);
1111 auto iter
= bucket
->get_attrs().find(RGW_ATTR_TAGS
);
1112 if (iter
!= bucket
->get_attrs().end()) {
1113 RGWObjTagSet_S3 tagset
;
1114 bufferlist::const_iterator piter
{&iter
->second
};
1116 tagset
.decode(piter
);
1117 tagset
.dump(formatter
);
1118 } catch (buffer::error
& err
) {
1119 cerr
<< "ERROR: caught buffer:error, couldn't decode TagSet" << std::endl
;
1123 // TODO: bucket CORS
1125 formatter
->close_section();
1130 int RGWBucketAdminOp::limit_check(rgw::sal::Store
* store
,
1131 RGWBucketAdminOpState
& op_state
,
1132 const std::list
<std::string
>& user_ids
,
1133 RGWFormatterFlusher
& flusher
, optional_yield y
,
1134 const DoutPrefixProvider
*dpp
,
1138 const size_t max_entries
=
1139 store
->ctx()->_conf
->rgw_list_buckets_max_chunk
;
1141 const size_t safe_max_objs_per_shard
=
1142 store
->ctx()->_conf
->rgw_safe_max_objects_per_shard
;
1144 uint16_t shard_warn_pct
=
1145 store
->ctx()->_conf
->rgw_shard_warning_threshold
;
1146 if (shard_warn_pct
> 100)
1147 shard_warn_pct
= 90;
1149 Formatter
*formatter
= flusher
.get_formatter();
1152 formatter
->open_array_section("users");
1154 for (const auto& user_id
: user_ids
) {
1156 formatter
->open_object_section("user");
1157 formatter
->dump_string("user_id", user_id
);
1158 formatter
->open_array_section("buckets");
1161 rgw::sal::BucketList buckets
;
1163 std::unique_ptr
<rgw::sal::User
> user
= store
->get_user(rgw_user(user_id
));
1165 ret
= user
->list_buckets(dpp
, marker
, string(), max_entries
, false, buckets
, y
);
1170 map
<string
, std::unique_ptr
<rgw::sal::Bucket
>>& m_buckets
= buckets
.get_buckets();
1172 for (const auto& iter
: m_buckets
) {
1173 auto& bucket
= iter
.second
;
1174 uint32_t num_shards
= 1;
1175 uint64_t num_objects
= 0;
1177 marker
= bucket
->get_name(); /* Casey's location for marker update,
1178 * as we may now not reach the end of
1181 ret
= bucket
->load_bucket(dpp
, null_yield
);
1185 /* need stats for num_entries */
1186 string bucket_ver
, master_ver
;
1187 std::map
<RGWObjCategory
, RGWStorageStats
> stats
;
1188 ret
= bucket
->read_stats(dpp
, RGW_NO_SHARD
, &bucket_ver
, &master_ver
, stats
, nullptr);
1193 for (const auto& s
: stats
) {
1194 num_objects
+= s
.second
.num_objects
;
1197 num_shards
= bucket
->get_info().layout
.current_index
.layout
.normal
.num_shards
;
1198 uint64_t objs_per_shard
=
1199 (num_shards
) ? num_objects
/num_shards
: num_objects
;
1203 uint64_t fill_pct
= objs_per_shard
* 100 / safe_max_objs_per_shard
;
1204 if (fill_pct
> 100) {
1205 ss
<< "OVER " << fill_pct
<< "%";
1207 } else if (fill_pct
>= shard_warn_pct
) {
1208 ss
<< "WARN " << fill_pct
<< "%";
1215 if (warn
|| !warnings_only
) {
1216 formatter
->open_object_section("bucket");
1217 formatter
->dump_string("bucket", bucket
->get_name());
1218 formatter
->dump_string("tenant", bucket
->get_tenant());
1219 formatter
->dump_int("num_objects", num_objects
);
1220 formatter
->dump_int("num_shards", num_shards
);
1221 formatter
->dump_int("objects_per_shard", objs_per_shard
);
1222 formatter
->dump_string("fill_status", ss
.str());
1223 formatter
->close_section();
1227 formatter
->flush(cout
);
1228 } while (buckets
.is_truncated()); /* foreach: bucket */
1230 formatter
->close_section();
1231 formatter
->close_section();
1232 formatter
->flush(cout
);
1234 } /* foreach: user_id */
1236 formatter
->close_section();
1237 formatter
->flush(cout
);
1240 } /* RGWBucketAdminOp::limit_check */
1242 int RGWBucketAdminOp::info(rgw::sal::Store
* store
,
1243 RGWBucketAdminOpState
& op_state
,
1244 RGWFormatterFlusher
& flusher
,
1246 const DoutPrefixProvider
*dpp
)
1250 const std::string
& bucket_name
= op_state
.get_bucket_name();
1251 if (!bucket_name
.empty()) {
1252 ret
= bucket
.init(store
, op_state
, null_yield
, dpp
);
1254 return -ERR_NO_SUCH_BUCKET
;
1259 Formatter
*formatter
= flusher
.get_formatter();
1262 CephContext
*cct
= store
->ctx();
1264 const size_t max_entries
= cct
->_conf
->rgw_list_buckets_max_chunk
;
1266 const bool show_stats
= op_state
.will_fetch_stats();
1267 const rgw_user
& user_id
= op_state
.get_user_id();
1268 if (op_state
.is_user_op()) {
1269 formatter
->open_array_section("buckets");
1271 rgw::sal::BucketList buckets
;
1272 std::unique_ptr
<rgw::sal::User
> user
= store
->get_user(op_state
.get_user_id());
1274 const std::string empty_end_marker
;
1275 constexpr bool no_need_stats
= false; // set need_stats to false
1278 ret
= user
->list_buckets(dpp
, marker
, empty_end_marker
, max_entries
,
1279 no_need_stats
, buckets
, y
);
1284 const std::string
* marker_cursor
= nullptr;
1285 map
<string
, std::unique_ptr
<rgw::sal::Bucket
>>& m
= buckets
.get_buckets();
1287 for (const auto& i
: m
) {
1288 const std::string
& obj_name
= i
.first
;
1289 if (!bucket_name
.empty() && bucket_name
!= obj_name
) {
1294 bucket_stats(store
, user_id
.tenant
, obj_name
, formatter
, dpp
);
1296 formatter
->dump_string("bucket", obj_name
);
1299 marker_cursor
= &obj_name
;
1301 if (marker_cursor
) {
1302 marker
= *marker_cursor
;
1306 } while (buckets
.is_truncated());
1308 formatter
->close_section();
1309 } else if (!bucket_name
.empty()) {
1310 ret
= bucket_stats(store
, user_id
.tenant
, bucket_name
, formatter
, dpp
);
1315 void *handle
= nullptr;
1316 bool truncated
= true;
1318 formatter
->open_array_section("buckets");
1319 ret
= store
->meta_list_keys_init(dpp
, "bucket", string(), &handle
);
1320 while (ret
== 0 && truncated
) {
1321 std::list
<std::string
> buckets
;
1322 constexpr int max_keys
= 1000;
1323 ret
= store
->meta_list_keys_next(dpp
, handle
, max_keys
, buckets
,
1325 for (auto& bucket_name
: buckets
) {
1327 bucket_stats(store
, user_id
.tenant
, bucket_name
, formatter
, dpp
);
1329 formatter
->dump_string("bucket", bucket_name
);
1333 store
->meta_list_keys_complete(handle
);
1335 formatter
->close_section();
1343 int RGWBucketAdminOp::set_quota(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
, const DoutPrefixProvider
*dpp
)
1347 int ret
= bucket
.init(store
, op_state
, null_yield
, dpp
);
1350 return bucket
.set_quota(op_state
, dpp
);
1353 static int purge_bucket_instance(rgw::sal::Store
* store
, const RGWBucketInfo
& bucket_info
, const DoutPrefixProvider
*dpp
)
1355 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1356 int ret
= store
->get_bucket(nullptr, bucket_info
, &bucket
);
1360 return bucket
->purge_instance(dpp
);
1363 inline auto split_tenant(const std::string
& bucket_name
){
1364 auto p
= bucket_name
.find('/');
1365 if(p
!= std::string::npos
) {
1366 return std::make_pair(bucket_name
.substr(0,p
), bucket_name
.substr(p
+1));
1368 return std::make_pair(std::string(), bucket_name
);
1371 using bucket_instance_ls
= std::vector
<RGWBucketInfo
>;
1372 void get_stale_instances(rgw::sal::Store
* store
, const std::string
& bucket_name
,
1373 const vector
<std::string
>& lst
,
1374 bucket_instance_ls
& stale_instances
,
1375 const DoutPrefixProvider
*dpp
)
1378 bucket_instance_ls other_instances
;
1379 // first iterate over the entries, and pick up the done buckets; these
1380 // are guaranteed to be stale
1381 for (const auto& bucket_instance
: lst
){
1382 RGWBucketInfo binfo
;
1383 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1385 rgw_bucket_parse_bucket_key(store
->ctx(), bucket_instance
, &rbucket
, nullptr);
1386 int r
= store
->get_bucket(dpp
, nullptr, rbucket
, &bucket
, null_yield
);
1388 // this can only happen if someone deletes us right when we're processing
1389 ldpp_dout(dpp
, -1) << "Bucket instance is invalid: " << bucket_instance
1390 << cpp_strerror(-r
) << dendl
;
1393 binfo
= bucket
->get_info();
1394 if (binfo
.reshard_status
== cls_rgw_reshard_status::DONE
)
1395 stale_instances
.emplace_back(std::move(binfo
));
1397 other_instances
.emplace_back(std::move(binfo
));
1401 // Read the cur bucket info, if the bucket doesn't exist we can simply return
1402 // all the instances
1403 auto [tenant
, bname
] = split_tenant(bucket_name
);
1404 RGWBucketInfo cur_bucket_info
;
1405 std::unique_ptr
<rgw::sal::Bucket
> cur_bucket
;
1406 int r
= store
->get_bucket(dpp
, nullptr, tenant
, bname
, &cur_bucket
, null_yield
);
1409 // bucket doesn't exist, everything is stale then
1410 stale_instances
.insert(std::end(stale_instances
),
1411 std::make_move_iterator(other_instances
.begin()),
1412 std::make_move_iterator(other_instances
.end()));
1414 // all bets are off if we can't read the bucket, just return the sureshot stale instances
1415 ldpp_dout(dpp
, -1) << "error: reading bucket info for bucket: "
1416 << bname
<< cpp_strerror(-r
) << dendl
;
1421 // Don't process further in this round if bucket is resharding
1422 cur_bucket_info
= cur_bucket
->get_info();
1423 if (cur_bucket_info
.reshard_status
== cls_rgw_reshard_status::IN_PROGRESS
)
1426 other_instances
.erase(std::remove_if(other_instances
.begin(), other_instances
.end(),
1427 [&cur_bucket_info
](const RGWBucketInfo
& b
){
1428 return (b
.bucket
.bucket_id
== cur_bucket_info
.bucket
.bucket_id
||
1429 b
.bucket
.bucket_id
== cur_bucket_info
.new_bucket_instance_id
);
1431 other_instances
.end());
1433 // check if there are still instances left
1434 if (other_instances
.empty()) {
1438 // Now we have a bucket with instances where the reshard status is none, this
1439 // usually happens when the reshard process couldn't complete, lockdown the
1440 // bucket and walk through these instances to make sure no one else interferes
1443 RGWBucketReshardLock
reshard_lock(static_cast<rgw::sal::RadosStore
*>(store
), cur_bucket
->get_info(), true);
1444 r
= reshard_lock
.lock(dpp
);
1446 // most likely bucket is under reshard, return the sureshot stale instances
1447 ldpp_dout(dpp
, 5) << __func__
1448 << "failed to take reshard lock; reshard underway likey" << dendl
;
1451 auto sg
= make_scope_guard([&reshard_lock
](){ reshard_lock
.unlock();} );
1452 // this should be fast enough that we may not need to renew locks and check
1453 // exit status?, should we read the values of the instances again?
1454 stale_instances
.insert(std::end(stale_instances
),
1455 std::make_move_iterator(other_instances
.begin()),
1456 std::make_move_iterator(other_instances
.end()));
1462 static int process_stale_instances(rgw::sal::Store
* store
, RGWBucketAdminOpState
& op_state
,
1463 RGWFormatterFlusher
& flusher
,
1464 const DoutPrefixProvider
*dpp
,
1465 std::function
<void(const bucket_instance_ls
&,
1467 rgw::sal::Store
*)> process_f
)
1471 Formatter
*formatter
= flusher
.get_formatter();
1472 static constexpr auto default_max_keys
= 1000;
1474 int ret
= store
->meta_list_keys_init(dpp
, "bucket.instance", marker
, &handle
);
1476 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
1482 formatter
->open_array_section("keys");
1483 auto g
= make_scope_guard([&store
, &handle
, &formatter
]() {
1484 store
->meta_list_keys_complete(handle
);
1485 formatter
->close_section(); // keys
1486 formatter
->flush(cout
);
1490 list
<std::string
> keys
;
1492 ret
= store
->meta_list_keys_next(dpp
, handle
, default_max_keys
, keys
, &truncated
);
1493 if (ret
< 0 && ret
!= -ENOENT
) {
1494 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
1496 } if (ret
!= -ENOENT
) {
1497 // partition the list of buckets by buckets as the listing is un sorted,
1498 // since it would minimize the reads to bucket_info
1499 std::unordered_map
<std::string
, std::vector
<std::string
>> bucket_instance_map
;
1500 for (auto &key
: keys
) {
1501 auto pos
= key
.find(':');
1502 if(pos
!= std::string::npos
)
1503 bucket_instance_map
[key
.substr(0,pos
)].emplace_back(std::move(key
));
1505 for (const auto& kv
: bucket_instance_map
) {
1506 bucket_instance_ls stale_lst
;
1507 get_stale_instances(store
, kv
.first
, kv
.second
, stale_lst
, dpp
);
1508 process_f(stale_lst
, formatter
, store
);
1511 } while (truncated
);
1516 int RGWBucketAdminOp::list_stale_instances(rgw::sal::Store
* store
,
1517 RGWBucketAdminOpState
& op_state
,
1518 RGWFormatterFlusher
& flusher
,
1519 const DoutPrefixProvider
*dpp
)
1521 auto process_f
= [](const bucket_instance_ls
& lst
,
1522 Formatter
*formatter
,
1524 for (const auto& binfo
: lst
)
1525 formatter
->dump_string("key", binfo
.bucket
.get_key());
1527 return process_stale_instances(store
, op_state
, flusher
, dpp
, process_f
);
1531 int RGWBucketAdminOp::clear_stale_instances(rgw::sal::Store
* store
,
1532 RGWBucketAdminOpState
& op_state
,
1533 RGWFormatterFlusher
& flusher
,
1534 const DoutPrefixProvider
*dpp
)
1536 auto process_f
= [dpp
](const bucket_instance_ls
& lst
,
1537 Formatter
*formatter
,
1538 rgw::sal::Store
* store
){
1539 for (const auto &binfo
: lst
) {
1540 int ret
= purge_bucket_instance(store
, binfo
, dpp
);
1542 auto md_key
= "bucket.instance:" + binfo
.bucket
.get_key();
1543 ret
= store
->meta_remove(dpp
, md_key
, null_yield
);
1545 formatter
->open_object_section("delete_status");
1546 formatter
->dump_string("bucket_instance", binfo
.bucket
.get_key());
1547 formatter
->dump_int("status", -ret
);
1548 formatter
->close_section();
1552 return process_stale_instances(store
, op_state
, flusher
, dpp
, process_f
);
1555 static int fix_single_bucket_lc(rgw::sal::Store
* store
,
1556 const std::string
& tenant_name
,
1557 const std::string
& bucket_name
,
1558 const DoutPrefixProvider
*dpp
)
1560 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1561 int ret
= store
->get_bucket(dpp
, nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1563 // TODO: Should we handle the case where the bucket could've been removed between
1564 // listing and fetching?
1568 return rgw::lc::fix_lc_shard_entry(dpp
, store
, store
->get_rgwlc()->get_lc(), bucket
.get());
1571 static void format_lc_status(Formatter
* formatter
,
1572 const std::string
& tenant_name
,
1573 const std::string
& bucket_name
,
1576 formatter
->open_object_section("bucket_entry");
1577 std::string entry
= tenant_name
.empty() ? bucket_name
: tenant_name
+ "/" + bucket_name
;
1578 formatter
->dump_string("bucket", entry
);
1579 formatter
->dump_int("status", status
);
1580 formatter
->close_section(); // bucket_entry
1583 static void process_single_lc_entry(rgw::sal::Store
* store
,
1584 Formatter
*formatter
,
1585 const std::string
& tenant_name
,
1586 const std::string
& bucket_name
,
1587 const DoutPrefixProvider
*dpp
)
1589 int ret
= fix_single_bucket_lc(store
, tenant_name
, bucket_name
, dpp
);
1590 format_lc_status(formatter
, tenant_name
, bucket_name
, -ret
);
1593 int RGWBucketAdminOp::fix_lc_shards(rgw::sal::Store
* store
,
1594 RGWBucketAdminOpState
& op_state
,
1595 RGWFormatterFlusher
& flusher
,
1596 const DoutPrefixProvider
*dpp
)
1600 Formatter
*formatter
= flusher
.get_formatter();
1601 static constexpr auto default_max_keys
= 1000;
1604 if (const std::string
& bucket_name
= op_state
.get_bucket_name();
1605 ! bucket_name
.empty()) {
1606 const rgw_user user_id
= op_state
.get_user_id();
1607 process_single_lc_entry(store
, formatter
, user_id
.tenant
, bucket_name
, dpp
);
1608 formatter
->flush(cout
);
1610 int ret
= store
->meta_list_keys_init(dpp
, "bucket", marker
, &handle
);
1612 std::cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
1617 formatter
->open_array_section("lc_fix_status");
1618 auto sg
= make_scope_guard([&store
, &handle
, &formatter
](){
1619 store
->meta_list_keys_complete(handle
);
1620 formatter
->close_section(); // lc_fix_status
1621 formatter
->flush(cout
);
1624 list
<std::string
> keys
;
1625 ret
= store
->meta_list_keys_next(dpp
, handle
, default_max_keys
, keys
, &truncated
);
1626 if (ret
< 0 && ret
!= -ENOENT
) {
1627 std::cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
1629 } if (ret
!= -ENOENT
) {
1630 for (const auto &key
:keys
) {
1631 auto [tenant_name
, bucket_name
] = split_tenant(key
);
1632 process_single_lc_entry(store
, formatter
, tenant_name
, bucket_name
, dpp
);
1635 formatter
->flush(cout
); // regularly flush every 1k entries
1636 } while (truncated
);
1644 static bool has_object_expired(const DoutPrefixProvider
*dpp
,
1645 rgw::sal::Store
* store
,
1646 rgw::sal::Bucket
* bucket
,
1647 const rgw_obj_key
& key
, utime_t
& delete_at
)
1649 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(key
);
1650 bufferlist delete_at_bl
;
1652 int ret
= rgw_object_get_attr(dpp
, store
, obj
.get(), RGW_ATTR_DELETE_AT
, delete_at_bl
, null_yield
);
1654 return false; // no delete at attr, proceed
1657 ret
= decode_bl(delete_at_bl
, delete_at
);
1659 return false; // failed to parse
1662 if (delete_at
<= ceph_clock_now() && !delete_at
.is_zero()) {
1669 static int fix_bucket_obj_expiry(const DoutPrefixProvider
*dpp
,
1670 rgw::sal::Store
* store
,
1671 rgw::sal::Bucket
* bucket
,
1672 RGWFormatterFlusher
& flusher
, bool dry_run
)
1674 if (bucket
->get_key().bucket_id
== bucket
->get_key().marker
) {
1675 ldpp_dout(dpp
, -1) << "Not a resharded bucket skipping" << dendl
;
1676 return 0; // not a resharded bucket, move along
1679 Formatter
*formatter
= flusher
.get_formatter();
1680 formatter
->open_array_section("expired_deletion_status");
1681 auto sg
= make_scope_guard([&formatter
] {
1682 formatter
->close_section();
1683 formatter
->flush(std::cout
);
1686 rgw::sal::Bucket::ListParams params
;
1687 rgw::sal::Bucket::ListResults results
;
1689 params
.list_versions
= bucket
->versioned();
1690 params
.allow_unordered
= true;
1693 int ret
= bucket
->list(dpp
, params
, listing_max_entries
, results
, null_yield
);
1695 ldpp_dout(dpp
, -1) << "ERROR failed to list objects in the bucket" << dendl
;
1698 for (const auto& obj
: results
.objs
) {
1699 rgw_obj_key
key(obj
.key
);
1701 if (has_object_expired(dpp
, store
, bucket
, key
, delete_at
)) {
1702 formatter
->open_object_section("object_status");
1703 formatter
->dump_string("object", key
.name
);
1704 formatter
->dump_stream("delete_at") << delete_at
;
1707 ret
= rgw_remove_object(dpp
, store
, bucket
, key
);
1708 formatter
->dump_int("status", ret
);
1711 formatter
->close_section(); // object_status
1714 formatter
->flush(cout
); // regularly flush every 1k entries
1715 } while (results
.is_truncated
);
1720 int RGWBucketAdminOp::fix_obj_expiry(rgw::sal::Store
* store
,
1721 RGWBucketAdminOpState
& op_state
,
1722 RGWFormatterFlusher
& flusher
,
1723 const DoutPrefixProvider
*dpp
, bool dry_run
)
1725 RGWBucket admin_bucket
;
1726 int ret
= admin_bucket
.init(store
, op_state
, null_yield
, dpp
);
1728 ldpp_dout(dpp
, -1) << "failed to initialize bucket" << dendl
;
1731 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1732 ret
= store
->get_bucket(nullptr, admin_bucket
.get_bucket_info(), &bucket
);
1737 return fix_bucket_obj_expiry(dpp
, store
, bucket
.get(), flusher
, dry_run
);
1740 void RGWBucketCompleteInfo::dump(Formatter
*f
) const {
1741 encode_json("bucket_info", info
, f
);
1742 encode_json("attrs", attrs
, f
);
1745 void RGWBucketCompleteInfo::decode_json(JSONObj
*obj
) {
1746 JSONDecoder::decode_json("bucket_info", info
, obj
);
1747 JSONDecoder::decode_json("attrs", attrs
, obj
);
1750 class RGWBucketMetadataHandler
: public RGWBucketMetadataHandlerBase
{
1753 RGWSI_Bucket
*bucket
{nullptr};
1757 RGWBucketCtl
*bucket
{nullptr};
1760 RGWBucketMetadataHandler() {}
1762 void init(RGWSI_Bucket
*bucket_svc
,
1763 RGWBucketCtl
*bucket_ctl
) override
{
1764 base_init(bucket_svc
->ctx(),
1765 bucket_svc
->get_ep_be_handler().get());
1766 svc
.bucket
= bucket_svc
;
1767 ctl
.bucket
= bucket_ctl
;
1770 string
get_type() override
{ return "bucket"; }
1772 RGWMetadataObject
*get_meta_obj(JSONObj
*jo
, const obj_version
& objv
, const ceph::real_time
& mtime
) override
{
1773 RGWBucketEntryPoint be
;
1776 decode_json_obj(be
, jo
);
1777 } catch (JSONDecoder::err
& e
) {
1781 return new RGWBucketEntryMetadataObject(be
, objv
, mtime
);
1784 int do_get(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWMetadataObject
**obj
, optional_yield y
, const DoutPrefixProvider
*dpp
) override
{
1785 RGWObjVersionTracker ot
;
1786 RGWBucketEntryPoint be
;
1789 map
<string
, bufferlist
> attrs
;
1791 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
1793 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, entry
, &be
, &ot
, &mtime
, &attrs
, y
, dpp
);
1797 RGWBucketEntryMetadataObject
*mdo
= new RGWBucketEntryMetadataObject(be
, ot
.read_version
, mtime
, std::move(attrs
));
1804 int do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
1805 RGWMetadataObject
*obj
,
1806 RGWObjVersionTracker
& objv_tracker
,
1808 const DoutPrefixProvider
*dpp
,
1809 RGWMDLogSyncType type
, bool from_remote_zone
) override
;
1811 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
,
1812 optional_yield y
, const DoutPrefixProvider
*dpp
) override
{
1813 RGWBucketEntryPoint be
;
1815 real_time orig_mtime
;
1817 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
1819 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, entry
, &be
, &objv_tracker
, &orig_mtime
, nullptr, y
, dpp
);
1824 * We're unlinking the bucket but we don't want to update the entrypoint here - we're removing
1825 * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal
1826 * will incorrectly fail.
1828 ret
= ctl
.bucket
->unlink_bucket(be
.owner
, be
.bucket
, y
, dpp
, false);
1830 ldpp_dout(dpp
, -1) << "could not unlink bucket=" << entry
<< " owner=" << be
.owner
<< dendl
;
1833 ret
= svc
.bucket
->remove_bucket_entrypoint_info(ctx
, entry
, &objv_tracker
, y
, dpp
);
1835 ldpp_dout(dpp
, -1) << "could not delete bucket=" << entry
<< dendl
;
1841 int call(std::function
<int(RGWSI_Bucket_EP_Ctx
& ctx
)> f
) {
1842 return call(nullopt
, f
);
1845 int call(std::optional
<RGWSI_MetaBackend_CtxParams
> bectx_params
,
1846 std::function
<int(RGWSI_Bucket_EP_Ctx
& ctx
)> f
) {
1847 return be_handler
->call(bectx_params
, [&](RGWSI_MetaBackend_Handler::Op
*op
) {
1848 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
1854 class RGWMetadataHandlerPut_Bucket
: public RGWMetadataHandlerPut_SObj
1856 RGWBucketMetadataHandler
*bhandler
;
1857 RGWBucketEntryMetadataObject
*obj
;
1859 RGWMetadataHandlerPut_Bucket(RGWBucketMetadataHandler
*_handler
,
1860 RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
1861 RGWMetadataObject
*_obj
, RGWObjVersionTracker
& objv_tracker
,
1863 RGWMDLogSyncType type
, bool from_remote_zone
) : RGWMetadataHandlerPut_SObj(_handler
, op
, entry
, obj
, objv_tracker
, y
, type
, from_remote_zone
),
1864 bhandler(_handler
) {
1865 obj
= static_cast<RGWBucketEntryMetadataObject
*>(_obj
);
1867 ~RGWMetadataHandlerPut_Bucket() {}
1869 void encode_obj(bufferlist
*bl
) override
{
1870 obj
->get_ep().encode(*bl
);
1873 int put_checked(const DoutPrefixProvider
*dpp
) override
;
1874 int put_post(const DoutPrefixProvider
*dpp
) override
;
1877 int RGWBucketMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
1878 RGWMetadataObject
*obj
,
1879 RGWObjVersionTracker
& objv_tracker
,
1881 const DoutPrefixProvider
*dpp
,
1882 RGWMDLogSyncType type
, bool from_remote_zone
)
1884 RGWMetadataHandlerPut_Bucket
put_op(this, op
, entry
, obj
, objv_tracker
, y
, type
, from_remote_zone
);
1885 return do_put_operate(&put_op
, dpp
);
1888 int RGWMetadataHandlerPut_Bucket::put_checked(const DoutPrefixProvider
*dpp
)
1890 RGWBucketEntryMetadataObject
*orig_obj
= static_cast<RGWBucketEntryMetadataObject
*>(old_obj
);
1893 obj
->set_pattrs(&orig_obj
->get_attrs());
1896 auto& be
= obj
->get_ep();
1897 auto mtime
= obj
->get_mtime();
1898 auto pattrs
= obj
->get_pattrs();
1900 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
1902 return bhandler
->svc
.bucket
->store_bucket_entrypoint_info(ctx
, entry
,
1912 int RGWMetadataHandlerPut_Bucket::put_post(const DoutPrefixProvider
*dpp
)
1914 auto& be
= obj
->get_ep();
1920 ret
= bhandler
->ctl
.bucket
->link_bucket(be
.owner
, be
.bucket
, be
.creation_time
, y
, dpp
, false);
1922 ret
= bhandler
->ctl
.bucket
->unlink_bucket(be
.owner
, be
.bucket
, y
, dpp
, false);
1928 static void get_md5_digest(const RGWBucketEntryPoint
*be
, string
& md5_digest
) {
1930 char md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
1931 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
1934 Formatter
*f
= new JSONFormatter(false);
1939 // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes
1940 hash
.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
);
1941 hash
.Update((const unsigned char *)bl
.c_str(), bl
.length());
1944 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, md5
);
1951 #define ARCHIVE_META_ATTR RGW_ATTR_PREFIX "zone.archive.info"
1953 struct archive_meta_info
{
1954 rgw_bucket orig_bucket
;
1956 bool from_attrs(CephContext
*cct
, map
<string
, bufferlist
>& attrs
) {
1957 auto iter
= attrs
.find(ARCHIVE_META_ATTR
);
1958 if (iter
== attrs
.end()) {
1962 auto bliter
= iter
->second
.cbegin();
1965 } catch (buffer::error
& err
) {
1966 ldout(cct
, 0) << "ERROR: failed to decode archive meta info" << dendl
;
1973 void store_in_attrs(map
<string
, bufferlist
>& attrs
) const {
1974 encode(attrs
[ARCHIVE_META_ATTR
]);
1977 void encode(bufferlist
& bl
) const {
1978 ENCODE_START(1, 1, bl
);
1979 encode(orig_bucket
, bl
);
1983 void decode(bufferlist::const_iterator
& bl
) {
1984 DECODE_START(1, bl
);
1985 decode(orig_bucket
, bl
);
1989 WRITE_CLASS_ENCODER(archive_meta_info
)
1991 class RGWArchiveBucketMetadataHandler
: public RGWBucketMetadataHandler
{
1993 RGWArchiveBucketMetadataHandler() {}
1995 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
,
1996 optional_yield y
, const DoutPrefixProvider
*dpp
) override
{
1997 auto cct
= svc
.bucket
->ctx();
1999 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
2001 ldpp_dout(dpp
, 5) << "SKIP: bucket removal is not allowed on archive zone: bucket:" << entry
<< " ... proceeding to rename" << dendl
;
2003 string tenant_name
, bucket_name
;
2004 parse_bucket(entry
, &tenant_name
, &bucket_name
);
2005 rgw_bucket entry_bucket
;
2006 entry_bucket
.tenant
= tenant_name
;
2007 entry_bucket
.name
= bucket_name
;
2011 /* read original entrypoint */
2013 RGWBucketEntryPoint be
;
2014 map
<string
, bufferlist
> attrs
;
2015 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, entry
, &be
, &objv_tracker
, &mtime
, &attrs
, y
, dpp
);
2020 string bi_meta_name
= RGWSI_Bucket::get_bi_meta_key(be
.bucket
);
2022 /* read original bucket instance info */
2024 map
<string
, bufferlist
> attrs_m
;
2025 ceph::real_time orig_mtime
;
2026 RGWBucketInfo old_bi
;
2028 ret
= ctl
.bucket
->read_bucket_instance_info(be
.bucket
, &old_bi
, y
, dpp
, RGWBucketCtl::BucketInstance::GetParams()
2029 .set_mtime(&orig_mtime
)
2030 .set_attrs(&attrs_m
));
2035 archive_meta_info ami
;
2037 if (!ami
.from_attrs(svc
.bucket
->ctx(), attrs_m
)) {
2038 ami
.orig_bucket
= old_bi
.bucket
;
2039 ami
.store_in_attrs(attrs_m
);
2042 /* generate a new bucket instance. We could have avoided this if we could just point a new
2043 * bucket entry point to the old bucket instance, however, due to limitation in the way
2044 * we index buckets under the user, bucket entrypoint and bucket instance of the same
2045 * bucket need to have the same name, so we need to copy the old bucket instance into
2046 * to a new entry with the new name
2049 string new_bucket_name
;
2051 RGWBucketInfo new_bi
= old_bi
;
2052 RGWBucketEntryPoint new_be
= be
;
2056 get_md5_digest(&new_be
, md5_digest
);
2057 new_bucket_name
= ami
.orig_bucket
.name
+ "-deleted-" + md5_digest
;
2059 new_bi
.bucket
.name
= new_bucket_name
;
2060 new_bi
.objv_tracker
.clear();
2062 new_be
.bucket
.name
= new_bucket_name
;
2064 ret
= ctl
.bucket
->store_bucket_instance_info(be
.bucket
, new_bi
, y
, dpp
, RGWBucketCtl::BucketInstance::PutParams()
2065 .set_exclusive(false)
2066 .set_mtime(orig_mtime
)
2067 .set_attrs(&attrs_m
)
2068 .set_orig_info(&old_bi
));
2070 ldpp_dout(dpp
, 0) << "ERROR: failed to put new bucket instance info for bucket=" << new_bi
.bucket
<< " ret=" << ret
<< dendl
;
2074 /* store a new entrypoint */
2076 RGWObjVersionTracker ot
;
2077 ot
.generate_new_write_ver(cct
);
2079 ret
= svc
.bucket
->store_bucket_entrypoint_info(ctx
, RGWSI_Bucket::get_entrypoint_meta_key(new_be
.bucket
),
2080 new_be
, true, mtime
, &attrs
, nullptr, y
, dpp
);
2082 ldpp_dout(dpp
, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be
.bucket
<< " ret=" << ret
<< dendl
;
2086 /* link new bucket */
2088 ret
= ctl
.bucket
->link_bucket(new_be
.owner
, new_be
.bucket
, new_be
.creation_time
, y
, dpp
, false);
2090 ldpp_dout(dpp
, 0) << "ERROR: failed to link new bucket for bucket=" << new_be
.bucket
<< " ret=" << ret
<< dendl
;
2094 /* clean up old stuff */
2096 ret
= ctl
.bucket
->unlink_bucket(be
.owner
, entry_bucket
, y
, dpp
, false);
2098 ldpp_dout(dpp
, -1) << "could not unlink bucket=" << entry
<< " owner=" << be
.owner
<< dendl
;
2101 // if (ret == -ECANCELED) it means that there was a race here, and someone
2102 // wrote to the bucket entrypoint just before we removed it. The question is
2103 // whether it was a newly created bucket entrypoint ... in which case we
2104 // should ignore the error and move forward, or whether it is a higher version
2105 // of the same bucket instance ... in which we should retry
2106 ret
= svc
.bucket
->remove_bucket_entrypoint_info(ctx
,
2107 RGWSI_Bucket::get_entrypoint_meta_key(be
.bucket
),
2112 ldpp_dout(dpp
, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be
.bucket
<< " ret=" << ret
<< dendl
;
2116 ret
= ctl
.bucket
->remove_bucket_instance_info(be
.bucket
, old_bi
, y
, dpp
);
2118 ldpp_dout(dpp
, -1) << "could not delete bucket=" << entry
<< dendl
;
2127 int do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2128 RGWMetadataObject
*obj
,
2129 RGWObjVersionTracker
& objv_tracker
,
2130 optional_yield y
, const DoutPrefixProvider
*dpp
,
2131 RGWMDLogSyncType type
, bool from_remote_zone
) override
{
2132 if (entry
.find("-deleted-") != string::npos
) {
2133 RGWObjVersionTracker ot
;
2134 RGWMetadataObject
*robj
;
2135 int ret
= do_get(op
, entry
, &robj
, y
, dpp
);
2136 if (ret
!= -ENOENT
) {
2140 ot
.read_version
= robj
->get_version();
2143 ret
= do_remove(op
, entry
, ot
, y
, dpp
);
2150 return RGWBucketMetadataHandler::do_put(op
, entry
, obj
,
2151 objv_tracker
, y
, dpp
, type
, from_remote_zone
);
2156 class RGWBucketInstanceMetadataHandler
: public RGWBucketInstanceMetadataHandlerBase
{
2157 int read_bucket_instance_entry(RGWSI_Bucket_BI_Ctx
& ctx
,
2158 const string
& entry
,
2159 RGWBucketCompleteInfo
*bi
,
2160 ceph::real_time
*pmtime
,
2162 const DoutPrefixProvider
*dpp
) {
2163 return svc
.bucket
->read_bucket_instance_info(ctx
,
2173 RGWSI_Zone
*zone
{nullptr};
2174 RGWSI_Bucket
*bucket
{nullptr};
2175 RGWSI_BucketIndex
*bi
{nullptr};
2178 RGWBucketInstanceMetadataHandler() {}
2180 void init(RGWSI_Zone
*zone_svc
,
2181 RGWSI_Bucket
*bucket_svc
,
2182 RGWSI_BucketIndex
*bi_svc
) override
{
2183 base_init(bucket_svc
->ctx(),
2184 bucket_svc
->get_bi_be_handler().get());
2185 svc
.zone
= zone_svc
;
2186 svc
.bucket
= bucket_svc
;
2190 string
get_type() override
{ return "bucket.instance"; }
2192 RGWMetadataObject
*get_meta_obj(JSONObj
*jo
, const obj_version
& objv
, const ceph::real_time
& mtime
) override
{
2193 RGWBucketCompleteInfo bci
;
2196 decode_json_obj(bci
, jo
);
2197 } catch (JSONDecoder::err
& e
) {
2201 return new RGWBucketInstanceMetadataObject(bci
, objv
, mtime
);
2204 int do_get(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWMetadataObject
**obj
, optional_yield y
, const DoutPrefixProvider
*dpp
) override
{
2205 RGWBucketCompleteInfo bci
;
2208 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2210 int ret
= svc
.bucket
->read_bucket_instance_info(ctx
, entry
, &bci
.info
, &mtime
, &bci
.attrs
, y
, dpp
);
2214 RGWBucketInstanceMetadataObject
*mdo
= new RGWBucketInstanceMetadataObject(bci
, bci
.info
.objv_tracker
.read_version
, mtime
);
2221 int do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2222 RGWMetadataObject
*_obj
, RGWObjVersionTracker
& objv_tracker
,
2223 optional_yield y
, const DoutPrefixProvider
*dpp
,
2224 RGWMDLogSyncType sync_type
, bool from_remote_zone
) override
;
2226 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
,
2227 optional_yield y
, const DoutPrefixProvider
*dpp
) override
{
2228 RGWBucketCompleteInfo bci
;
2230 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2232 int ret
= read_bucket_instance_entry(ctx
, entry
, &bci
, nullptr, y
, dpp
);
2233 if (ret
< 0 && ret
!= -ENOENT
)
2236 return svc
.bucket
->remove_bucket_instance_info(ctx
, entry
, bci
.info
, &bci
.info
.objv_tracker
, y
, dpp
);
2239 int call(std::function
<int(RGWSI_Bucket_BI_Ctx
& ctx
)> f
) {
2240 return call(nullopt
, f
);
2243 int call(std::optional
<RGWSI_MetaBackend_CtxParams
> bectx_params
,
2244 std::function
<int(RGWSI_Bucket_BI_Ctx
& ctx
)> f
) {
2245 return be_handler
->call(bectx_params
, [&](RGWSI_MetaBackend_Handler::Op
*op
) {
2246 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2252 class RGWMetadataHandlerPut_BucketInstance
: public RGWMetadataHandlerPut_SObj
2255 RGWBucketInstanceMetadataHandler
*bihandler
;
2256 RGWBucketInstanceMetadataObject
*obj
;
2258 RGWMetadataHandlerPut_BucketInstance(CephContext
*_cct
,
2259 RGWBucketInstanceMetadataHandler
*_handler
,
2260 RGWSI_MetaBackend_Handler::Op
*_op
, string
& entry
,
2261 RGWMetadataObject
*_obj
, RGWObjVersionTracker
& objv_tracker
,
2263 RGWMDLogSyncType type
, bool from_remote_zone
) : RGWMetadataHandlerPut_SObj(_handler
, _op
, entry
, obj
, objv_tracker
, y
, type
, from_remote_zone
),
2264 cct(_cct
), bihandler(_handler
) {
2265 obj
= static_cast<RGWBucketInstanceMetadataObject
*>(_obj
);
2267 auto& bci
= obj
->get_bci();
2268 obj
->set_pattrs(&bci
.attrs
);
2271 void encode_obj(bufferlist
*bl
) override
{
2272 obj
->get_bucket_info().encode(*bl
);
2275 int put_check(const DoutPrefixProvider
*dpp
) override
;
2276 int put_checked(const DoutPrefixProvider
*dpp
) override
;
2277 int put_post(const DoutPrefixProvider
*dpp
) override
;
2280 int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op
*op
,
2282 RGWMetadataObject
*obj
,
2283 RGWObjVersionTracker
& objv_tracker
,
2285 const DoutPrefixProvider
*dpp
,
2286 RGWMDLogSyncType type
, bool from_remote_zone
)
2288 RGWMetadataHandlerPut_BucketInstance
put_op(svc
.bucket
->ctx(), this, op
, entry
, obj
,
2289 objv_tracker
, y
, type
, from_remote_zone
);
2290 return do_put_operate(&put_op
, dpp
);
2293 void init_default_bucket_layout(CephContext
*cct
, rgw::BucketLayout
& layout
,
2294 const RGWZone
& zone
,
2295 std::optional
<uint32_t> shards
,
2296 std::optional
<rgw::BucketIndexType
> type
) {
2297 layout
.current_index
.gen
= 0;
2298 layout
.current_index
.layout
.normal
.hash_type
= rgw::BucketHashType::Mod
;
2300 layout
.current_index
.layout
.type
=
2301 type
.value_or(rgw::BucketIndexType::Normal
);
2304 layout
.current_index
.layout
.normal
.num_shards
= *shards
;
2305 } else if (cct
->_conf
->rgw_override_bucket_index_max_shards
> 0) {
2306 layout
.current_index
.layout
.normal
.num_shards
=
2307 cct
->_conf
->rgw_override_bucket_index_max_shards
;
2309 layout
.current_index
.layout
.normal
.num_shards
=
2310 zone
.bucket_index_max_shards
;
2313 if (layout
.current_index
.layout
.type
== rgw::BucketIndexType::Normal
) {
2314 layout
.logs
.push_back(log_layout_from_index(
2315 layout
.current_index
.gen
,
2316 layout
.current_index
.layout
.normal
));
2320 int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider
*dpp
)
2324 RGWBucketCompleteInfo
& bci
= obj
->get_bci();
2326 RGWBucketInstanceMetadataObject
*orig_obj
= static_cast<RGWBucketInstanceMetadataObject
*>(old_obj
);
2328 RGWBucketCompleteInfo
*old_bci
= (orig_obj
? &orig_obj
->get_bci() : nullptr);
2330 const bool exists
= (!!orig_obj
);
2332 if (from_remote_zone
) {
2333 // don't sync bucket layout changes
2335 auto& bci_index
= bci
.info
.layout
.current_index
.layout
;
2336 auto index_type
= bci_index
.type
;
2337 auto num_shards
= bci_index
.normal
.num_shards
;
2338 init_default_bucket_layout(cct
, bci
.info
.layout
,
2339 bihandler
->svc
.zone
->get_zone(),
2340 num_shards
, index_type
);
2342 bci
.info
.layout
= old_bci
->info
.layout
;
2346 if (!exists
|| old_bci
->info
.bucket
.bucket_id
!= bci
.info
.bucket
.bucket_id
) {
2347 /* a new bucket, we need to select a new bucket placement for it */
2350 string bucket_instance
;
2351 parse_bucket(entry
, &tenant_name
, &bucket_name
, &bucket_instance
);
2353 RGWZonePlacementInfo rule_info
;
2354 bci
.info
.bucket
.name
= bucket_name
;
2355 bci
.info
.bucket
.bucket_id
= bucket_instance
;
2356 bci
.info
.bucket
.tenant
= tenant_name
;
2357 // if the sync module never writes data, don't require the zone to specify all placement targets
2358 if (bihandler
->svc
.zone
->sync_module_supports_writes()) {
2359 ret
= bihandler
->svc
.zone
->select_bucket_location_by_rule(dpp
, bci
.info
.placement_rule
, &rule_info
, y
);
2361 ldpp_dout(dpp
, 0) << "ERROR: select_bucket_placement() returned " << ret
<< dendl
;
2365 bci
.info
.layout
.current_index
.layout
.type
= rule_info
.index_type
;
2367 /* existing bucket, keep its placement */
2368 bci
.info
.bucket
.explicit_placement
= old_bci
->info
.bucket
.explicit_placement
;
2369 bci
.info
.placement_rule
= old_bci
->info
.placement_rule
;
2372 /* record the read version (if any), store the new version */
2373 bci
.info
.objv_tracker
.read_version
= objv_tracker
.read_version
;
2374 bci
.info
.objv_tracker
.write_version
= objv_tracker
.write_version
;
2379 int RGWMetadataHandlerPut_BucketInstance::put_checked(const DoutPrefixProvider
*dpp
)
2381 RGWBucketInstanceMetadataObject
*orig_obj
= static_cast<RGWBucketInstanceMetadataObject
*>(old_obj
);
2383 RGWBucketInfo
*orig_info
= (orig_obj
? &orig_obj
->get_bucket_info() : nullptr);
2385 auto& info
= obj
->get_bucket_info();
2386 auto mtime
= obj
->get_mtime();
2387 auto pattrs
= obj
->get_pattrs();
2389 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2391 return bihandler
->svc
.bucket
->store_bucket_instance_info(ctx
,
2402 int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider
*dpp
)
2404 RGWBucketCompleteInfo
& bci
= obj
->get_bci();
2406 objv_tracker
= bci
.info
.objv_tracker
;
2408 int ret
= bihandler
->svc
.bi
->init_index(dpp
, bci
.info
);
2413 return STATUS_APPLIED
;
2416 class RGWArchiveBucketInstanceMetadataHandler
: public RGWBucketInstanceMetadataHandler
{
2418 RGWArchiveBucketInstanceMetadataHandler() {}
2420 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
, optional_yield y
, const DoutPrefixProvider
*dpp
) override
{
2421 ldpp_dout(dpp
, 0) << "SKIP: bucket instance removal is not allowed on archive zone: bucket.instance:" << entry
<< dendl
;
2426 RGWBucketCtl::RGWBucketCtl(RGWSI_Zone
*zone_svc
,
2427 RGWSI_Bucket
*bucket_svc
,
2428 RGWSI_Bucket_Sync
*bucket_sync_svc
,
2429 RGWSI_BucketIndex
*bi_svc
) : cct(zone_svc
->ctx())
2431 svc
.zone
= zone_svc
;
2432 svc
.bucket
= bucket_svc
;
2433 svc
.bucket_sync
= bucket_sync_svc
;
2437 void RGWBucketCtl::init(RGWUserCtl
*user_ctl
,
2438 RGWBucketMetadataHandler
*_bm_handler
,
2439 RGWBucketInstanceMetadataHandler
*_bmi_handler
,
2440 RGWDataChangesLog
*datalog
,
2441 const DoutPrefixProvider
*dpp
)
2443 ctl
.user
= user_ctl
;
2445 bm_handler
= _bm_handler
;
2446 bmi_handler
= _bmi_handler
;
2448 bucket_be_handler
= bm_handler
->get_be_handler();
2449 bi_be_handler
= bmi_handler
->get_be_handler();
2451 datalog
->set_bucket_filter(
2452 [this](const rgw_bucket
& bucket
, optional_yield y
, const DoutPrefixProvider
*dpp
) {
2453 return bucket_exports_data(bucket
, y
, dpp
);
2457 int RGWBucketCtl::call(std::function
<int(RGWSI_Bucket_X_Ctx
& ctx
)> f
) {
2458 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ep_ctx
) {
2459 return bmi_handler
->call([&](RGWSI_Bucket_BI_Ctx
& bi_ctx
) {
2460 RGWSI_Bucket_X_Ctx ctx
{ep_ctx
, bi_ctx
};
2466 int RGWBucketCtl::read_bucket_entrypoint_info(const rgw_bucket
& bucket
,
2467 RGWBucketEntryPoint
*info
,
2468 optional_yield y
, const DoutPrefixProvider
*dpp
,
2469 const Bucket::GetParams
& params
)
2471 return bm_handler
->call(params
.bectx_params
, [&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2472 return svc
.bucket
->read_bucket_entrypoint_info(ctx
,
2473 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2475 params
.objv_tracker
,
2481 params
.refresh_version
);
2485 int RGWBucketCtl::store_bucket_entrypoint_info(const rgw_bucket
& bucket
,
2486 RGWBucketEntryPoint
& info
,
2488 const DoutPrefixProvider
*dpp
,
2489 const Bucket::PutParams
& params
)
2491 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2492 return svc
.bucket
->store_bucket_entrypoint_info(ctx
,
2493 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2498 params
.objv_tracker
,
2504 int RGWBucketCtl::remove_bucket_entrypoint_info(const rgw_bucket
& bucket
,
2506 const DoutPrefixProvider
*dpp
,
2507 const Bucket::RemoveParams
& params
)
2509 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2510 return svc
.bucket
->remove_bucket_entrypoint_info(ctx
,
2511 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2512 params
.objv_tracker
,
2518 int RGWBucketCtl::read_bucket_instance_info(const rgw_bucket
& bucket
,
2519 RGWBucketInfo
*info
,
2521 const DoutPrefixProvider
*dpp
,
2522 const BucketInstance::GetParams
& params
)
2524 int ret
= bmi_handler
->call(params
.bectx_params
, [&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2525 return svc
.bucket
->read_bucket_instance_info(ctx
,
2526 RGWSI_Bucket::get_bi_meta_key(bucket
),
2533 params
.refresh_version
);
2540 if (params
.objv_tracker
) {
2541 *params
.objv_tracker
= info
->objv_tracker
;
2547 int RGWBucketCtl::read_bucket_info(const rgw_bucket
& bucket
,
2548 RGWBucketInfo
*info
,
2550 const DoutPrefixProvider
*dpp
,
2551 const BucketInstance::GetParams
& params
,
2552 RGWObjVersionTracker
*ep_objv_tracker
)
2554 const rgw_bucket
*b
= &bucket
;
2556 std::optional
<RGWBucketEntryPoint
> ep
;
2558 if (b
->bucket_id
.empty()) {
2561 int r
= read_bucket_entrypoint_info(*b
, &(*ep
), y
, dpp
, RGWBucketCtl::Bucket::GetParams()
2562 .set_bectx_params(params
.bectx_params
)
2563 .set_objv_tracker(ep_objv_tracker
));
2571 int ret
= bmi_handler
->call(params
.bectx_params
, [&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2572 return svc
.bucket
->read_bucket_instance_info(ctx
,
2573 RGWSI_Bucket::get_bi_meta_key(*b
),
2579 params
.refresh_version
);
2586 if (params
.objv_tracker
) {
2587 *params
.objv_tracker
= info
->objv_tracker
;
2593 int RGWBucketCtl::do_store_bucket_instance_info(RGWSI_Bucket_BI_Ctx
& ctx
,
2594 const rgw_bucket
& bucket
,
2595 RGWBucketInfo
& info
,
2597 const DoutPrefixProvider
*dpp
,
2598 const BucketInstance::PutParams
& params
)
2600 if (params
.objv_tracker
) {
2601 info
.objv_tracker
= *params
.objv_tracker
;
2604 return svc
.bucket
->store_bucket_instance_info(ctx
,
2605 RGWSI_Bucket::get_bi_meta_key(bucket
),
2615 int RGWBucketCtl::store_bucket_instance_info(const rgw_bucket
& bucket
,
2616 RGWBucketInfo
& info
,
2618 const DoutPrefixProvider
*dpp
,
2619 const BucketInstance::PutParams
& params
)
2621 return bmi_handler
->call([&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2622 return do_store_bucket_instance_info(ctx
, bucket
, info
, y
, dpp
, params
);
2626 int RGWBucketCtl::remove_bucket_instance_info(const rgw_bucket
& bucket
,
2627 RGWBucketInfo
& info
,
2629 const DoutPrefixProvider
*dpp
,
2630 const BucketInstance::RemoveParams
& params
)
2632 if (params
.objv_tracker
) {
2633 info
.objv_tracker
= *params
.objv_tracker
;
2636 return bmi_handler
->call([&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2637 return svc
.bucket
->remove_bucket_instance_info(ctx
,
2638 RGWSI_Bucket::get_bi_meta_key(bucket
),
2646 int RGWBucketCtl::do_store_linked_bucket_info(RGWSI_Bucket_X_Ctx
& ctx
,
2647 RGWBucketInfo
& info
,
2648 RGWBucketInfo
*orig_info
,
2649 bool exclusive
, real_time mtime
,
2650 obj_version
*pep_objv
,
2651 map
<string
, bufferlist
> *pattrs
,
2652 bool create_entry_point
,
2653 optional_yield y
, const DoutPrefixProvider
*dpp
)
2655 bool create_head
= !info
.has_instance_obj
|| create_entry_point
;
2657 int ret
= svc
.bucket
->store_bucket_instance_info(ctx
.bi
,
2658 RGWSI_Bucket::get_bi_meta_key(info
.bucket
),
2669 return 0; /* done! */
2671 RGWBucketEntryPoint entry_point
;
2672 entry_point
.bucket
= info
.bucket
;
2673 entry_point
.owner
= info
.owner
;
2674 entry_point
.creation_time
= info
.creation_time
;
2675 entry_point
.linked
= true;
2676 RGWObjVersionTracker ot
;
2677 if (pep_objv
&& !pep_objv
->tag
.empty()) {
2678 ot
.write_version
= *pep_objv
;
2680 ot
.generate_new_write_ver(cct
);
2682 *pep_objv
= ot
.write_version
;
2685 ret
= svc
.bucket
->store_bucket_entrypoint_info(ctx
.ep
,
2686 RGWSI_Bucket::get_entrypoint_meta_key(info
.bucket
),
2699 int RGWBucketCtl::convert_old_bucket_info(RGWSI_Bucket_X_Ctx
& ctx
,
2700 const rgw_bucket
& bucket
,
2702 const DoutPrefixProvider
*dpp
)
2704 RGWBucketEntryPoint entry_point
;
2706 RGWObjVersionTracker ot
;
2707 map
<string
, bufferlist
> attrs
;
2709 auto cct
= svc
.bucket
->ctx();
2711 ldpp_dout(dpp
, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket
<< dendl
;
2713 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
.ep
,
2714 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2715 &entry_point
, &ot
, &ep_mtime
, &attrs
, y
, dpp
);
2717 ldpp_dout(dpp
, 0) << "ERROR: get_bucket_entrypoint_info() returned " << ret
<< " bucket=" << bucket
<< dendl
;
2721 if (!entry_point
.has_bucket_info
) {
2722 /* already converted! */
2726 info
= entry_point
.old_bucket_info
;
2728 ot
.generate_new_write_ver(cct
);
2730 ret
= do_store_linked_bucket_info(ctx
, info
, nullptr, false, ep_mtime
, &ot
.write_version
, &attrs
, true, y
, dpp
);
2732 ldpp_dout(dpp
, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret
<< dendl
;
2739 int RGWBucketCtl::set_bucket_instance_attrs(RGWBucketInfo
& bucket_info
,
2740 map
<string
, bufferlist
>& attrs
,
2741 RGWObjVersionTracker
*objv_tracker
,
2743 const DoutPrefixProvider
*dpp
)
2745 return call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
2746 rgw_bucket
& bucket
= bucket_info
.bucket
;
2748 if (!bucket_info
.has_instance_obj
) {
2749 /* an old bucket object, need to convert it */
2750 int ret
= convert_old_bucket_info(ctx
, bucket
, y
, dpp
);
2752 ldpp_dout(dpp
, 0) << "ERROR: failed converting old bucket info: " << ret
<< dendl
;
2757 return do_store_bucket_instance_info(ctx
.bi
,
2762 BucketInstance::PutParams().set_attrs(&attrs
)
2763 .set_objv_tracker(objv_tracker
)
2764 .set_orig_info(&bucket_info
));
2769 int RGWBucketCtl::link_bucket(const rgw_user
& user_id
,
2770 const rgw_bucket
& bucket
,
2771 ceph::real_time creation_time
,
2773 const DoutPrefixProvider
*dpp
,
2774 bool update_entrypoint
,
2777 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2778 return do_link_bucket(ctx
, user_id
, bucket
, creation_time
,
2779 update_entrypoint
, pinfo
, y
, dpp
);
2783 int RGWBucketCtl::do_link_bucket(RGWSI_Bucket_EP_Ctx
& ctx
,
2784 const rgw_user
& user_id
,
2785 const rgw_bucket
& bucket
,
2786 ceph::real_time creation_time
,
2787 bool update_entrypoint
,
2790 const DoutPrefixProvider
*dpp
)
2794 RGWBucketEntryPoint ep
;
2795 RGWObjVersionTracker ot
;
2796 RGWObjVersionTracker
& rot
= (pinfo
) ? pinfo
->ep_objv
: ot
;
2797 map
<string
, bufferlist
> attrs
, *pattrs
= nullptr;
2800 if (update_entrypoint
) {
2801 meta_key
= RGWSI_Bucket::get_entrypoint_meta_key(bucket
);
2804 pattrs
= &pinfo
->attrs
;
2806 ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
,
2811 if (ret
< 0 && ret
!= -ENOENT
) {
2812 ldpp_dout(dpp
, 0) << "ERROR: store->get_bucket_entrypoint_info() returned: "
2813 << cpp_strerror(-ret
) << dendl
;
2819 ret
= ctl
.user
->add_bucket(dpp
, user_id
, bucket
, creation_time
, y
);
2821 ldpp_dout(dpp
, 0) << "ERROR: error adding bucket to user directory:"
2822 << " user=" << user_id
2823 << " bucket=" << bucket
2824 << " err=" << cpp_strerror(-ret
)
2829 if (!update_entrypoint
)
2835 ret
= svc
.bucket
->store_bucket_entrypoint_info(
2836 ctx
, meta_key
, ep
, false, real_time(), pattrs
, &rot
, y
, dpp
);
2843 int r
= do_unlink_bucket(ctx
, user_id
, bucket
, true, y
, dpp
);
2845 ldpp_dout(dpp
, 0) << "ERROR: failed unlinking bucket on error cleanup: "
2846 << cpp_strerror(-r
) << dendl
;
2851 int RGWBucketCtl::unlink_bucket(const rgw_user
& user_id
, const rgw_bucket
& bucket
, optional_yield y
, const DoutPrefixProvider
*dpp
, bool update_entrypoint
)
2853 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2854 return do_unlink_bucket(ctx
, user_id
, bucket
, update_entrypoint
, y
, dpp
);
2858 int RGWBucketCtl::do_unlink_bucket(RGWSI_Bucket_EP_Ctx
& ctx
,
2859 const rgw_user
& user_id
,
2860 const rgw_bucket
& bucket
,
2861 bool update_entrypoint
,
2863 const DoutPrefixProvider
*dpp
)
2865 int ret
= ctl
.user
->remove_bucket(dpp
, user_id
, bucket
, y
);
2867 ldpp_dout(dpp
, 0) << "ERROR: error removing bucket from directory: "
2868 << cpp_strerror(-ret
)<< dendl
;
2871 if (!update_entrypoint
)
2874 RGWBucketEntryPoint ep
;
2875 RGWObjVersionTracker ot
;
2876 map
<string
, bufferlist
> attrs
;
2877 string meta_key
= RGWSI_Bucket::get_entrypoint_meta_key(bucket
);
2878 ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, meta_key
, &ep
, &ot
, nullptr, &attrs
, y
, dpp
);
2887 if (ep
.owner
!= user_id
) {
2888 ldpp_dout(dpp
, 0) << "bucket entry point user mismatch, can't unlink bucket: " << ep
.owner
<< " != " << user_id
<< dendl
;
2893 return svc
.bucket
->store_bucket_entrypoint_info(ctx
, meta_key
, ep
, false, real_time(), &attrs
, &ot
, y
, dpp
);
2896 int RGWBucketCtl::read_bucket_stats(const rgw_bucket
& bucket
,
2897 RGWBucketEnt
*result
,
2899 const DoutPrefixProvider
*dpp
)
2901 return call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
2902 return svc
.bucket
->read_bucket_stats(ctx
, bucket
, result
, y
, dpp
);
2906 int RGWBucketCtl::read_buckets_stats(map
<string
, RGWBucketEnt
>& m
,
2907 optional_yield y
, const DoutPrefixProvider
*dpp
)
2909 return call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
2910 return svc
.bucket
->read_buckets_stats(ctx
, m
, y
, dpp
);
2914 int RGWBucketCtl::sync_user_stats(const DoutPrefixProvider
*dpp
,
2915 const rgw_user
& user_id
,
2916 const RGWBucketInfo
& bucket_info
,
2924 int r
= svc
.bi
->read_stats(dpp
, bucket_info
, pent
, null_yield
);
2926 ldpp_dout(dpp
, 20) << __func__
<< "(): failed to read bucket stats (r=" << r
<< ")" << dendl
;
2930 return ctl
.user
->flush_bucket_stats(dpp
, user_id
, *pent
, y
);
2933 int RGWBucketCtl::get_sync_policy_handler(std::optional
<rgw_zone_id
> zone
,
2934 std::optional
<rgw_bucket
> bucket
,
2935 RGWBucketSyncPolicyHandlerRef
*phandler
,
2937 const DoutPrefixProvider
*dpp
)
2939 int r
= call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
2940 return svc
.bucket_sync
->get_policy_handler(ctx
, zone
, bucket
, phandler
, y
, dpp
);
2943 ldpp_dout(dpp
, 20) << __func__
<< "(): failed to get policy handler for bucket=" << bucket
<< " (r=" << r
<< ")" << dendl
;
2949 int RGWBucketCtl::bucket_exports_data(const rgw_bucket
& bucket
,
2951 const DoutPrefixProvider
*dpp
)
2954 RGWBucketSyncPolicyHandlerRef handler
;
2956 int r
= get_sync_policy_handler(std::nullopt
, bucket
, &handler
, y
, dpp
);
2961 return handler
->bucket_exports_data();
2964 int RGWBucketCtl::bucket_imports_data(const rgw_bucket
& bucket
,
2965 optional_yield y
, const DoutPrefixProvider
*dpp
)
2968 RGWBucketSyncPolicyHandlerRef handler
;
2970 int r
= get_sync_policy_handler(std::nullopt
, bucket
, &handler
, y
, dpp
);
2975 return handler
->bucket_imports_data();
2978 RGWBucketMetadataHandlerBase
*RGWBucketMetaHandlerAllocator::alloc()
2980 return new RGWBucketMetadataHandler();
2983 RGWBucketInstanceMetadataHandlerBase
*RGWBucketInstanceMetaHandlerAllocator::alloc()
2985 return new RGWBucketInstanceMetadataHandler();
2988 RGWBucketMetadataHandlerBase
*RGWArchiveBucketMetaHandlerAllocator::alloc()
2990 return new RGWArchiveBucketMetadataHandler();
2993 RGWBucketInstanceMetadataHandlerBase
*RGWArchiveBucketInstanceMetaHandlerAllocator::alloc()
2995 return new RGWArchiveBucketInstanceMetadataHandler();
2999 void RGWBucketEntryPoint::generate_test_instances(list
<RGWBucketEntryPoint
*>& o
)
3001 RGWBucketEntryPoint
*bp
= new RGWBucketEntryPoint();
3002 init_bucket(&bp
->bucket
, "tenant", "bucket", "pool", ".index.pool", "marker", "10");
3003 bp
->owner
= "owner";
3004 bp
->creation_time
= ceph::real_clock::from_ceph_timespec({ceph_le32(2), ceph_le32(3)});
3007 o
.push_back(new RGWBucketEntryPoint
);
3010 void RGWBucketEntryPoint::dump(Formatter
*f
) const
3012 encode_json("bucket", bucket
, f
);
3013 encode_json("owner", owner
, f
);
3014 utime_t
ut(creation_time
);
3015 encode_json("creation_time", ut
, f
);
3016 encode_json("linked", linked
, f
);
3017 encode_json("has_bucket_info", has_bucket_info
, f
);
3018 if (has_bucket_info
) {
3019 encode_json("old_bucket_info", old_bucket_info
, f
);
3023 void RGWBucketEntryPoint::decode_json(JSONObj
*obj
) {
3024 JSONDecoder::decode_json("bucket", bucket
, obj
);
3025 JSONDecoder::decode_json("owner", owner
, obj
);
3027 JSONDecoder::decode_json("creation_time", ut
, obj
);
3028 creation_time
= ut
.to_real_time();
3029 JSONDecoder::decode_json("linked", linked
, obj
);
3030 JSONDecoder::decode_json("has_bucket_info", has_bucket_info
, obj
);
3031 if (has_bucket_info
) {
3032 JSONDecoder::decode_json("old_bucket_info", old_bucket_info
, obj
);
3036 int rgw_chown_bucket_and_objects(rgw::sal::Store
* store
, rgw::sal::Bucket
* bucket
,
3037 rgw::sal::User
* new_user
,
3038 const std::string
& marker
, std::string
*err_msg
,
3039 const DoutPrefixProvider
*dpp
, optional_yield y
)
3041 /* Chown on the bucket */
3042 int ret
= bucket
->chown(dpp
, *new_user
, y
);
3044 set_err_msg(err_msg
, "Failed to change object ownership: " + cpp_strerror(-ret
));
3047 /* Now chown on all the objects in the bucket */
3048 map
<string
, bool> common_prefixes
;
3050 rgw::sal::Bucket::ListParams params
;
3051 rgw::sal::Bucket::ListResults results
;
3053 params
.list_versions
= true;
3054 params
.allow_unordered
= true;
3055 params
.marker
= marker
;
3058 int max_entries
= 1000;
3060 //Loop through objects and update object acls to point to bucket owner
3063 results
.objs
.clear();
3064 ret
= bucket
->list(dpp
, params
, max_entries
, results
, y
);
3066 ldpp_dout(dpp
, 0) << "ERROR: list objects failed: " << cpp_strerror(-ret
) << dendl
;
3070 params
.marker
= results
.next_marker
;
3071 count
+= results
.objs
.size();
3073 for (const auto& obj
: results
.objs
) {
3074 std::unique_ptr
<rgw::sal::Object
> r_obj
= bucket
->get_object(obj
.key
);
3076 ret
= r_obj
->chown(*new_user
, dpp
, y
);
3078 ldpp_dout(dpp
, 0) << "ERROR: chown failed on " << r_obj
<< " :" << cpp_strerror(-ret
) << dendl
;
3082 cerr
<< count
<< " objects processed in " << bucket
3083 << ". Next marker " << params
.marker
.name
<< std::endl
;
3084 } while(results
.is_truncated
);