1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
10 #include <boost/format.hpp>
12 #include "common/errno.h"
13 #include "common/ceph_json.h"
14 #include "include/scope_guard.h"
16 #include "rgw_datalog.h"
17 #include "rgw_rados.h"
20 #include "rgw_acl_s3.h"
21 #include "rgw_tag_s3.h"
23 #include "include/types.h"
24 #include "rgw_bucket.h"
26 #include "rgw_string.h"
27 #include "rgw_multi.h"
29 #include "rgw_bucket_sync.h"
31 #include "services/svc_zone.h"
32 #include "services/svc_sys_obj.h"
33 #include "services/svc_bucket.h"
34 #include "services/svc_bucket_sync.h"
35 #include "services/svc_meta.h"
36 #include "services/svc_meta_be_sobj.h"
37 #include "services/svc_user.h"
38 #include "services/svc_cls.h"
39 #include "services/svc_bilog_rados.h"
41 #include "include/rados/librados.hpp"
42 // until everything is moved from rgw_common
43 #include "rgw_common.h"
44 #include "rgw_reshard.h"
46 #include "rgw_bucket_layout.h"
48 // stolen from src/cls/version/cls_version.cc
49 #define VERSION_ATTR "ceph.objclass.version"
51 #include "cls/user/cls_user_types.h"
54 #include "rgw_sal_rados.h"
56 #define dout_context g_ceph_context
57 #define dout_subsys ceph_subsys_rgw
59 #define BUCKET_TAG_TIMEOUT 30
61 // default number of entries to list with each bucket listing call
62 // (use marker to bridge between calls)
63 static constexpr size_t listing_max_entries
= 1000;
67 * The tenant_name is always returned on purpose. May be empty, of course.
69 static void parse_bucket(const string
& bucket
,
72 string
*bucket_instance
= nullptr /* optional */)
75 * expected format: [tenant/]bucket:bucket_instance
77 int pos
= bucket
.find('/');
79 *tenant_name
= bucket
.substr(0, pos
);
83 string bn
= bucket
.substr(pos
+ 1);
86 *bucket_name
= std::move(bn
);
89 *bucket_name
= bn
.substr(0, pos
);
90 if (bucket_instance
) {
91 *bucket_instance
= bn
.substr(pos
+ 1);
95 * deal with the possible tenant:bucket:bucket_instance case
97 if (tenant_name
->empty()) {
98 pos
= bucket_instance
->find(':');
100 *tenant_name
= *bucket_name
;
101 *bucket_name
= bucket_instance
->substr(0, pos
);
102 *bucket_instance
= bucket_instance
->substr(pos
+ 1);
108 * Note that this is not a reversal of parse_bucket(). That one deals
109 * with the syntax we need in metadata and such. This one deals with
110 * the representation in RADOS pools. We chose '/' because it's not
111 * acceptable in bucket names and thus qualified buckets cannot conflict
112 * with the legacy or S3 buckets.
114 std::string
rgw_make_bucket_entry_name(const std::string
& tenant_name
,
115 const std::string
& bucket_name
) {
116 std::string bucket_entry
;
118 if (bucket_name
.empty()) {
119 bucket_entry
.clear();
120 } else if (tenant_name
.empty()) {
121 bucket_entry
= bucket_name
;
123 bucket_entry
= tenant_name
+ "/" + bucket_name
;
130 * Tenants are separated from buckets in URLs by a colon in S3.
131 * This function is not to be used on Swift URLs, not even for COPY arguments.
133 void rgw_parse_url_bucket(const string
&bucket
, const string
& auth_tenant
,
134 string
&tenant_name
, string
&bucket_name
) {
136 int pos
= bucket
.find(':');
139 * N.B.: We allow ":bucket" syntax with explicit empty tenant in order
140 * to refer to the legacy tenant, in case users in new named tenants
141 * want to access old global buckets.
143 tenant_name
= bucket
.substr(0, pos
);
144 bucket_name
= bucket
.substr(pos
+ 1);
146 tenant_name
= auth_tenant
;
147 bucket_name
= bucket
;
152 * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
153 * Returns: 0 on success, -ERR# on failure.
155 int rgw_read_user_buckets(rgw::sal::RGWRadosStore
* store
,
156 const rgw_user
& user_id
,
157 rgw::sal::RGWBucketList
& buckets
,
158 const string
& marker
,
159 const string
& end_marker
,
164 rgw::sal::RGWRadosUser
user(store
, user_id
);
165 return user
.list_buckets(marker
, end_marker
, max
, need_stats
, buckets
, y
);
168 int rgw_bucket_parse_bucket_instance(const string
& bucket_instance
, string
*bucket_name
, string
*bucket_id
, int *shard_id
)
170 auto pos
= bucket_instance
.rfind(':');
171 if (pos
== string::npos
) {
175 string first
= bucket_instance
.substr(0, pos
);
176 string second
= bucket_instance
.substr(pos
+ 1);
178 pos
= first
.find(':');
180 if (pos
== string::npos
) {
182 *bucket_name
= first
;
187 *bucket_name
= first
.substr(0, pos
);
188 *bucket_id
= first
.substr(pos
+ 1);
191 *shard_id
= strict_strtol(second
.c_str(), 10, &err
);
199 // parse key in format: [tenant/]name:instance[:shard_id]
200 int rgw_bucket_parse_bucket_key(CephContext
*cct
, const string
& key
,
201 rgw_bucket
*bucket
, int *shard_id
)
203 std::string_view name
{key
};
204 std::string_view instance
;
207 auto pos
= name
.find('/');
208 if (pos
!= string::npos
) {
209 auto tenant
= name
.substr(0, pos
);
210 bucket
->tenant
.assign(tenant
.begin(), tenant
.end());
211 name
= name
.substr(pos
+ 1);
213 bucket
->tenant
.clear();
216 // split name:instance
217 pos
= name
.find(':');
218 if (pos
!= string::npos
) {
219 instance
= name
.substr(pos
+ 1);
220 name
= name
.substr(0, pos
);
222 bucket
->name
.assign(name
.begin(), name
.end());
224 // split instance:shard
225 pos
= instance
.find(':');
226 if (pos
== string::npos
) {
227 bucket
->bucket_id
.assign(instance
.begin(), instance
.end());
235 auto shard
= instance
.substr(pos
+ 1);
237 auto id
= strict_strtol(shard
.data(), 10, &err
);
240 ldout(cct
, 0) << "ERROR: failed to parse bucket shard '"
241 << instance
.data() << "': " << err
<< dendl
;
249 instance
= instance
.substr(0, pos
);
250 bucket
->bucket_id
.assign(instance
.begin(), instance
.end());
254 static void dump_mulipart_index_results(list
<rgw_obj_index_key
>& objs_to_unlink
,
257 for (const auto& o
: objs_to_unlink
) {
258 f
->dump_string("object", o
.name
);
262 void check_bad_user_bucket_mapping(rgw::sal::RGWRadosStore
*store
, const rgw_user
& user_id
,
266 rgw::sal::RGWBucketList user_buckets
;
267 rgw::sal::RGWRadosUser
user(store
, user_id
);
270 CephContext
*cct
= store
->ctx();
272 size_t max_entries
= cct
->_conf
->rgw_list_buckets_max_chunk
;
275 int ret
= user
.list_buckets(marker
, string(), max_entries
, false, user_buckets
, y
);
277 ldout(store
->ctx(), 0) << "failed to read user buckets: "
278 << cpp_strerror(-ret
) << dendl
;
282 map
<string
, std::unique_ptr
<rgw::sal::RGWBucket
>>& buckets
= user_buckets
.get_buckets();
283 for (auto i
= buckets
.begin();
288 auto& bucket
= i
->second
;
290 RGWBucketInfo bucket_info
;
292 int r
= store
->getRados()->get_bucket_info(store
->svc(), user_id
.tenant
, bucket
->get_name(), bucket_info
, &mtime
, null_yield
);
294 ldout(store
->ctx(), 0) << "could not get bucket info for bucket=" << bucket
<< dendl
;
298 rgw_bucket
& actual_bucket
= bucket_info
.bucket
;
300 if (actual_bucket
.name
.compare(bucket
->get_name()) != 0 ||
301 actual_bucket
.tenant
.compare(bucket
->get_tenant()) != 0 ||
302 actual_bucket
.marker
.compare(bucket
->get_marker()) != 0 ||
303 actual_bucket
.bucket_id
.compare(bucket
->get_bucket_id()) != 0) {
304 cout
<< "bucket info mismatch: expected " << actual_bucket
<< " got " << bucket
<< std::endl
;
306 cout
<< "fixing" << std::endl
;
307 r
= store
->ctl()->bucket
->link_bucket(user_id
, actual_bucket
,
308 bucket_info
.creation_time
,
311 cerr
<< "failed to fix bucket: " << cpp_strerror(-r
) << std::endl
;
316 } while (user_buckets
.is_truncated());
319 // note: function type conforms to RGWRados::check_filter_t
320 bool rgw_bucket_object_check_filter(const string
& oid
)
324 return rgw_obj_key::oid_to_key_in_ns(oid
, &key
, ns
);
327 int rgw_remove_object(rgw::sal::RGWRadosStore
*store
, const RGWBucketInfo
& bucket_info
, const rgw_bucket
& bucket
, rgw_obj_key
& key
)
329 RGWObjectCtx
rctx(store
);
331 if (key
.instance
.empty()) {
332 key
.instance
= "null";
335 rgw_obj
obj(bucket
, key
);
337 return store
->getRados()->delete_obj(rctx
, bucket_info
, obj
, bucket_info
.versioning_status());
340 static int aio_wait(librados::AioCompletion
*handle
)
342 librados::AioCompletion
*c
= (librados::AioCompletion
*)handle
;
343 c
->wait_for_complete();
344 int ret
= c
->get_return_value();
349 static int drain_handles(list
<librados::AioCompletion
*>& pending
)
352 while (!pending
.empty()) {
353 librados::AioCompletion
*handle
= pending
.front();
355 int r
= aio_wait(handle
);
363 int rgw_remove_bucket_bypass_gc(rgw::sal::RGWRadosStore
*store
, rgw_bucket
& bucket
,
364 int concurrent_max
, bool keep_index_consistent
,
368 map
<RGWObjCategory
, RGWStorageStats
> stats
;
369 std::vector
<rgw_bucket_dir_entry
> objs
;
370 map
<string
, bool> common_prefixes
;
372 RGWObjectCtx
obj_ctx(store
);
373 CephContext
*cct
= store
->ctx();
375 string bucket_ver
, master_ver
;
377 ret
= store
->getRados()->get_bucket_info(store
->svc(), bucket
.tenant
, bucket
.name
, info
, NULL
, null_yield
);
381 ret
= store
->getRados()->get_bucket_stats(info
, RGW_NO_SHARD
, &bucket_ver
, &master_ver
, stats
, NULL
);
385 string prefix
, delimiter
;
387 ret
= abort_bucket_multiparts(store
, cct
, info
, prefix
, delimiter
);
392 RGWRados::Bucket
target(store
->getRados(), info
);
393 RGWRados::Bucket::List
list_op(&target
);
395 list_op
.params
.list_versions
= true;
396 list_op
.params
.allow_unordered
= true;
398 std::list
<librados::AioCompletion
*> handles
;
400 int max_aio
= concurrent_max
;
401 bool is_truncated
= true;
403 while (is_truncated
) {
405 ret
= list_op
.list_objects(listing_max_entries
, &objs
, &common_prefixes
,
406 &is_truncated
, null_yield
);
410 std::vector
<rgw_bucket_dir_entry
>::iterator it
= objs
.begin();
411 for (; it
!= objs
.end(); ++it
) {
412 RGWObjState
*astate
= NULL
;
413 rgw_obj
obj(bucket
, (*it
).key
);
415 ret
= store
->getRados()->get_obj_state(&obj_ctx
, info
, obj
, &astate
, false, y
);
416 if (ret
== -ENOENT
) {
417 dout(1) << "WARNING: cannot find obj state for obj " << obj
.get_oid() << dendl
;
421 lderr(store
->ctx()) << "ERROR: get obj state returned with error " << ret
<< dendl
;
425 if (astate
->manifest
) {
426 RGWObjManifest
& manifest
= *astate
->manifest
;
427 RGWObjManifest::obj_iterator miter
= manifest
.obj_begin();
428 rgw_obj head_obj
= manifest
.get_obj();
429 rgw_raw_obj raw_head_obj
;
430 store
->getRados()->obj_to_raw(info
.placement_rule
, head_obj
, &raw_head_obj
);
433 for (; miter
!= manifest
.obj_end() && max_aio
--; ++miter
) {
435 ret
= drain_handles(handles
);
437 lderr(store
->ctx()) << "ERROR: could not drain handles as aio completion returned with " << ret
<< dendl
;
440 max_aio
= concurrent_max
;
443 rgw_raw_obj last_obj
= miter
.get_location().get_raw_obj(store
);
444 if (last_obj
== raw_head_obj
) {
445 // have the head obj deleted at the end
449 ret
= store
->getRados()->delete_raw_obj_aio(last_obj
, handles
);
451 lderr(store
->ctx()) << "ERROR: delete obj aio failed with " << ret
<< dendl
;
454 } // for all shadow objs
456 ret
= store
->getRados()->delete_obj_aio(head_obj
, info
, astate
, handles
, keep_index_consistent
, null_yield
);
458 lderr(store
->ctx()) << "ERROR: delete obj aio failed with " << ret
<< dendl
;
464 ret
= drain_handles(handles
);
466 lderr(store
->ctx()) << "ERROR: could not drain handles as aio completion returned with " << ret
<< dendl
;
469 max_aio
= concurrent_max
;
471 obj_ctx
.invalidate(obj
);
472 } // for all RGW objects
475 ret
= drain_handles(handles
);
477 lderr(store
->ctx()) << "ERROR: could not drain handles as aio completion returned with " << ret
<< dendl
;
481 ret
= store
->ctl()->bucket
->sync_user_stats(info
.owner
, info
, y
);
483 dout(1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret
<< dendl
;
486 RGWObjVersionTracker objv_tracker
;
488 // this function can only be run if caller wanted children to be
489 // deleted, so we can ignore the check for children as any that
490 // remain are detritus from a prior bug
491 ret
= store
->getRados()->delete_bucket(info
, objv_tracker
, y
, false);
493 lderr(store
->ctx()) << "ERROR: could not remove bucket " << bucket
.name
<< dendl
;
497 ret
= store
->ctl()->bucket
->unlink_bucket(info
.owner
, bucket
, null_yield
, false);
499 lderr(store
->ctx()) << "ERROR: unable to remove user bucket information" << dendl
;
505 static void set_err_msg(std::string
*sink
, std::string msg
)
507 if (sink
&& !msg
.empty())
511 int RGWBucket::init(rgw::sal::RGWRadosStore
*storage
, RGWBucketAdminOpState
& op_state
,
512 optional_yield y
, std::string
*err_msg
,
513 map
<string
, bufferlist
> *pattrs
)
516 set_err_msg(err_msg
, "no storage!");
522 rgw_user user_id
= op_state
.get_user_id();
523 bucket
.tenant
= user_id
.tenant
;
524 bucket
.name
= op_state
.get_bucket_name();
526 if (bucket
.name
.empty() && user_id
.empty())
529 // split possible tenant/name
530 auto pos
= bucket
.name
.find('/');
531 if (pos
!= string::npos
) {
532 bucket
.tenant
= bucket
.name
.substr(0, pos
);
533 bucket
.name
= bucket
.name
.substr(pos
+ 1);
536 if (!bucket
.name
.empty()) {
537 int r
= store
->ctl()->bucket
->read_bucket_info(
538 bucket
, &bucket_info
, y
,
539 RGWBucketCtl::BucketInstance::GetParams().set_attrs(pattrs
),
542 set_err_msg(err_msg
, "failed to fetch bucket info for bucket=" + bucket
.name
);
546 op_state
.set_bucket(bucket_info
.bucket
);
549 if (!user_id
.empty()) {
550 int r
= store
->ctl()->user
->get_info_by_uid(user_id
, &user_info
, y
);
552 set_err_msg(err_msg
, "failed to fetch user info");
556 op_state
.display_name
= user_info
.display_name
;
563 bool rgw_find_bucket_by_id(CephContext
*cct
, RGWMetadataManager
*mgr
,
564 const string
& marker
, const string
& bucket_id
, rgw_bucket
* bucket_out
)
567 bool truncated
= false;
570 int ret
= mgr
->list_keys_init("bucket.instance", marker
, &handle
);
572 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
573 mgr
->list_keys_complete(handle
);
578 ret
= mgr
->list_keys_next(handle
, 1000, keys
, &truncated
);
580 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
581 mgr
->list_keys_complete(handle
);
584 for (list
<string
>::iterator iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
586 ret
= rgw_bucket_parse_bucket_key(cct
, s
, bucket_out
, nullptr);
590 if (bucket_id
== bucket_out
->bucket_id
) {
591 mgr
->list_keys_complete(handle
);
596 mgr
->list_keys_complete(handle
);
600 int RGWBucket::link(RGWBucketAdminOpState
& op_state
, optional_yield y
,
601 map
<string
, bufferlist
>& attrs
, std::string
*err_msg
)
603 if (!op_state
.is_user_op()) {
604 set_err_msg(err_msg
, "empty user id");
608 string bucket_id
= op_state
.get_bucket_id();
610 std::string display_name
= op_state
.get_user_display_name();
611 rgw_bucket
& bucket
= op_state
.get_bucket();
612 if (!bucket_id
.empty() && bucket_id
!= bucket
.bucket_id
) {
614 "specified bucket id does not match " + bucket
.bucket_id
);
617 rgw_bucket old_bucket
= bucket
;
618 rgw_user user_id
= op_state
.get_user_id();
619 bucket
.tenant
= user_id
.tenant
;
620 if (!op_state
.new_bucket_name
.empty()) {
621 auto pos
= op_state
.new_bucket_name
.find('/');
622 if (pos
!= string::npos
) {
623 bucket
.tenant
= op_state
.new_bucket_name
.substr(0, pos
);
624 bucket
.name
= op_state
.new_bucket_name
.substr(pos
+ 1);
626 bucket
.name
= op_state
.new_bucket_name
;
630 RGWObjVersionTracker objv_tracker
;
631 RGWObjVersionTracker old_version
= bucket_info
.objv_tracker
;
633 map
<string
, bufferlist
>::iterator aiter
= attrs
.find(RGW_ATTR_ACL
);
634 if (aiter
== attrs
.end()) {
635 // should never happen; only pre-argonaut buckets lacked this.
636 ldout(store
->ctx(), 0) << "WARNING: can't bucket link because no acl on bucket=" << old_bucket
.name
<< dendl
;
638 "While crossing the Anavros you have displeased the goddess Hera."
639 " You must sacrifice your ancient bucket " + bucket
.bucket_id
);
642 bufferlist
& aclbl
= aiter
->second
;
643 RGWAccessControlPolicy policy
;
646 auto iter
= aclbl
.cbegin();
647 decode(policy
, iter
);
648 owner
= policy
.get_owner();
649 } catch (buffer::error
& err
) {
650 set_err_msg(err_msg
, "couldn't decode policy");
654 auto bucket_ctl
= store
->ctl()->bucket
;
655 int r
= bucket_ctl
->unlink_bucket(owner
.get_id(), old_bucket
, y
, false);
657 set_err_msg(err_msg
, "could not unlink policy from user " + owner
.get_id().to_str());
661 // now update the user for the bucket...
662 if (display_name
.empty()) {
663 ldout(store
->ctx(), 0) << "WARNING: user " << user_info
.user_id
<< " has no display name set" << dendl
;
666 RGWAccessControlPolicy policy_instance
;
667 policy_instance
.create_default(user_info
.user_id
, display_name
);
668 owner
= policy_instance
.get_owner();
671 policy_instance
.encode(aclbl
);
673 auto instance_params
= RGWBucketCtl::BucketInstance::PutParams().set_attrs(&attrs
);
675 bucket_info
.owner
= user_info
.user_id
;
676 if (bucket
!= old_bucket
) {
677 bucket_info
.bucket
= bucket
;
678 bucket_info
.objv_tracker
.version_for_read()->ver
= 0;
679 instance_params
.set_exclusive(true);
682 r
= bucket_ctl
->store_bucket_instance_info(bucket
, bucket_info
, y
, instance_params
);
684 set_err_msg(err_msg
, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r
));
688 RGWBucketEntryPoint ep
;
689 ep
.bucket
= bucket_info
.bucket
;
690 ep
.owner
= user_info
.user_id
;
691 ep
.creation_time
= bucket_info
.creation_time
;
693 map
<string
, bufferlist
> ep_attrs
;
694 rgw_ep_info ep_data
{ep
, ep_attrs
};
697 r
= store
->ctl()->bucket
->link_bucket(user_info
.user_id
,
702 set_err_msg(err_msg
, "failed to relink bucket");
706 if (bucket
!= old_bucket
) {
707 // like RGWRados::delete_bucket -- excepting no bucket_index work.
708 r
= bucket_ctl
->remove_bucket_entrypoint_info(old_bucket
, y
,
709 RGWBucketCtl::Bucket::RemoveParams()
710 .set_objv_tracker(&ep_data
.ep_objv
));
712 set_err_msg(err_msg
, "failed to unlink old bucket endpoint " + old_bucket
.tenant
+ "/" + old_bucket
.name
);
716 r
= bucket_ctl
->remove_bucket_instance_info(old_bucket
, bucket_info
, y
,
717 RGWBucketCtl::BucketInstance::RemoveParams()
718 .set_objv_tracker(&old_version
));
720 set_err_msg(err_msg
, "failed to unlink old bucket info");
728 int RGWBucket::chown(RGWBucketAdminOpState
& op_state
, const string
& marker
,
729 optional_yield y
, std::string
*err_msg
)
731 int ret
= store
->ctl()->bucket
->chown(store
, bucket_info
, user_info
.user_id
,
732 user_info
.display_name
, marker
, y
);
734 set_err_msg(err_msg
, "Failed to change object ownership: " + cpp_strerror(-ret
));
740 int RGWBucket::unlink(RGWBucketAdminOpState
& op_state
, optional_yield y
, std::string
*err_msg
)
742 rgw_bucket bucket
= op_state
.get_bucket();
744 if (!op_state
.is_user_op()) {
745 set_err_msg(err_msg
, "could not fetch user or user bucket info");
749 int r
= store
->ctl()->bucket
->unlink_bucket(user_info
.user_id
, bucket
, y
);
751 set_err_msg(err_msg
, "error unlinking bucket" + cpp_strerror(-r
));
757 int RGWBucket::set_quota(RGWBucketAdminOpState
& op_state
, std::string
*err_msg
)
759 rgw_bucket bucket
= op_state
.get_bucket();
760 RGWBucketInfo bucket_info
;
761 map
<string
, bufferlist
> attrs
;
762 int r
= store
->getRados()->get_bucket_info(store
->svc(), bucket
.tenant
, bucket
.name
, bucket_info
, NULL
, null_yield
, &attrs
);
764 set_err_msg(err_msg
, "could not get bucket info for bucket=" + bucket
.name
+ ": " + cpp_strerror(-r
));
768 bucket_info
.quota
= op_state
.quota
;
769 r
= store
->getRados()->put_bucket_instance_info(bucket_info
, false, real_time(), &attrs
);
771 set_err_msg(err_msg
, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r
));
777 int RGWBucket::remove_object(RGWBucketAdminOpState
& op_state
, std::string
*err_msg
)
779 rgw_bucket bucket
= op_state
.get_bucket();
780 std::string object_name
= op_state
.get_object_name();
782 rgw_obj_key
key(object_name
);
784 int ret
= rgw_remove_object(store
, bucket_info
, bucket
, key
);
786 set_err_msg(err_msg
, "unable to remove object" + cpp_strerror(-ret
));
793 static void dump_bucket_index(const RGWRados::ent_map_t
& result
, Formatter
*f
)
795 for (auto iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
796 f
->dump_string("object", iter
->first
);
800 static void dump_bucket_usage(map
<RGWObjCategory
, RGWStorageStats
>& stats
, Formatter
*formatter
)
802 map
<RGWObjCategory
, RGWStorageStats
>::iterator iter
;
804 formatter
->open_object_section("usage");
805 for (iter
= stats
.begin(); iter
!= stats
.end(); ++iter
) {
806 RGWStorageStats
& s
= iter
->second
;
807 const char *cat_name
= rgw_obj_category_name(iter
->first
);
808 formatter
->open_object_section(cat_name
);
810 formatter
->close_section();
812 formatter
->close_section();
815 static void dump_index_check(map
<RGWObjCategory
, RGWStorageStats
> existing_stats
,
816 map
<RGWObjCategory
, RGWStorageStats
> calculated_stats
,
817 Formatter
*formatter
)
819 formatter
->open_object_section("check_result");
820 formatter
->open_object_section("existing_header");
821 dump_bucket_usage(existing_stats
, formatter
);
822 formatter
->close_section();
823 formatter
->open_object_section("calculated_header");
824 dump_bucket_usage(calculated_stats
, formatter
);
825 formatter
->close_section();
826 formatter
->close_section();
829 int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState
& op_state
,
830 RGWFormatterFlusher
& flusher
,std::string
*err_msg
)
832 bool fix_index
= op_state
.will_fix_index();
833 rgw_bucket bucket
= op_state
.get_bucket();
835 map
<string
, bool> common_prefixes
;
838 map
<string
, bool> meta_objs
;
839 map
<rgw_obj_index_key
, string
> all_objs
;
841 RGWBucketInfo bucket_info
;
842 auto obj_ctx
= store
->svc()->sysobj
->init_obj_ctx();
843 int r
= store
->getRados()->get_bucket_instance_info(obj_ctx
, bucket
, bucket_info
, nullptr, nullptr, null_yield
);
845 ldout(store
->ctx(), 0) << "ERROR: " << __func__
<< "(): get_bucket_instance_info(bucket=" << bucket
<< ") returned r=" << r
<< dendl
;
849 RGWRados::Bucket
target(store
->getRados(), bucket_info
);
850 RGWRados::Bucket::List
list_op(&target
);
852 list_op
.params
.list_versions
= true;
853 list_op
.params
.ns
= RGW_OBJ_NS_MULTIPART
;
856 vector
<rgw_bucket_dir_entry
> result
;
857 int r
= list_op
.list_objects(listing_max_entries
, &result
,
858 &common_prefixes
, &is_truncated
, null_yield
);
860 set_err_msg(err_msg
, "failed to list objects in bucket=" + bucket
.name
+
861 " err=" + cpp_strerror(-r
));
866 vector
<rgw_bucket_dir_entry
>::iterator iter
;
867 for (iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
868 rgw_obj_index_key key
= iter
->key
;
869 rgw_obj
obj(bucket
, key
);
870 string oid
= obj
.get_oid();
872 int pos
= oid
.find_last_of('.');
874 /* obj has no suffix */
878 string name
= oid
.substr(0, pos
);
879 string suffix
= oid
.substr(pos
+ 1);
881 if (suffix
.compare("meta") == 0) {
882 meta_objs
[name
] = true;
884 all_objs
[key
] = name
;
888 } while (is_truncated
);
890 list
<rgw_obj_index_key
> objs_to_unlink
;
891 Formatter
*f
= flusher
.get_formatter();
893 f
->open_array_section("invalid_multipart_entries");
895 for (auto aiter
= all_objs
.begin(); aiter
!= all_objs
.end(); ++aiter
) {
896 string
& name
= aiter
->second
;
898 if (meta_objs
.find(name
) == meta_objs
.end()) {
899 objs_to_unlink
.push_back(aiter
->first
);
902 if (objs_to_unlink
.size() > listing_max_entries
) {
904 int r
= store
->getRados()->remove_objs_from_index(bucket_info
, objs_to_unlink
);
906 set_err_msg(err_msg
, "ERROR: remove_obj_from_index() returned error: " +
912 dump_mulipart_index_results(objs_to_unlink
, flusher
.get_formatter());
914 objs_to_unlink
.clear();
919 int r
= store
->getRados()->remove_objs_from_index(bucket_info
, objs_to_unlink
);
921 set_err_msg(err_msg
, "ERROR: remove_obj_from_index() returned error: " +
928 dump_mulipart_index_results(objs_to_unlink
, f
);
935 int RGWBucket::check_object_index(RGWBucketAdminOpState
& op_state
,
936 RGWFormatterFlusher
& flusher
,
938 std::string
*err_msg
)
941 bool fix_index
= op_state
.will_fix_index();
944 set_err_msg(err_msg
, "check-objects flag requires fix index enabled");
948 store
->getRados()->cls_obj_set_bucket_tag_timeout(bucket_info
, BUCKET_TAG_TIMEOUT
);
951 string empty_delimiter
;
952 rgw_obj_index_key marker
;
953 bool is_truncated
= true;
954 bool cls_filtered
= true;
956 Formatter
*formatter
= flusher
.get_formatter();
957 formatter
->open_object_section("objects");
958 uint16_t expansion_factor
= 1;
959 while (is_truncated
) {
960 RGWRados::ent_map_t result
;
961 result
.reserve(listing_max_entries
);
963 int r
= store
->getRados()->cls_bucket_list_ordered(
964 bucket_info
, RGW_NO_SHARD
, marker
, prefix
, empty_delimiter
,
965 listing_max_entries
, true, expansion_factor
,
966 result
, &is_truncated
, &cls_filtered
, &marker
,
967 y
, rgw_bucket_object_check_filter
);
970 } else if (r
< 0 && r
!= -ENOENT
) {
971 set_err_msg(err_msg
, "ERROR: failed operation r=" + cpp_strerror(-r
));
974 if (result
.size() < listing_max_entries
/ 8) {
976 } else if (result
.size() > listing_max_entries
* 7 / 8 &&
977 expansion_factor
> 1) {
981 dump_bucket_index(result
, formatter
);
985 formatter
->close_section();
987 store
->getRados()->cls_obj_set_bucket_tag_timeout(bucket_info
, 0);
993 int RGWBucket::check_index(RGWBucketAdminOpState
& op_state
,
994 map
<RGWObjCategory
, RGWStorageStats
>& existing_stats
,
995 map
<RGWObjCategory
, RGWStorageStats
>& calculated_stats
,
996 std::string
*err_msg
)
998 bool fix_index
= op_state
.will_fix_index();
1000 int r
= store
->getRados()->bucket_check_index(bucket_info
, &existing_stats
, &calculated_stats
);
1002 set_err_msg(err_msg
, "failed to check index error=" + cpp_strerror(-r
));
1007 r
= store
->getRados()->bucket_rebuild_index(bucket_info
);
1009 set_err_msg(err_msg
, "failed to rebuild index err=" + cpp_strerror(-r
));
1017 int RGWBucket::sync(RGWBucketAdminOpState
& op_state
, map
<string
, bufferlist
> *attrs
, std::string
*err_msg
)
1019 if (!store
->svc()->zone
->is_meta_master()) {
1020 set_err_msg(err_msg
, "ERROR: failed to update bucket sync: only allowed on meta master zone");
1023 bool sync
= op_state
.will_sync_bucket();
1025 bucket_info
.flags
&= ~BUCKET_DATASYNC_DISABLED
;
1027 bucket_info
.flags
|= BUCKET_DATASYNC_DISABLED
;
1030 int r
= store
->getRados()->put_bucket_instance_info(bucket_info
, false, real_time(), attrs
);
1032 set_err_msg(err_msg
, "ERROR: failed writing bucket instance info:" + cpp_strerror(-r
));
1036 int shards_num
= bucket_info
.layout
.current_index
.layout
.normal
.num_shards
? bucket_info
.layout
.current_index
.layout
.normal
.num_shards
: 1;
1037 int shard_id
= bucket_info
.layout
.current_index
.layout
.normal
.num_shards
? 0 : -1;
1040 r
= store
->svc()->bilog_rados
->log_stop(bucket_info
, -1);
1042 set_err_msg(err_msg
, "ERROR: failed writing stop bilog:" + cpp_strerror(-r
));
1046 r
= store
->svc()->bilog_rados
->log_start(bucket_info
, -1);
1048 set_err_msg(err_msg
, "ERROR: failed writing resync bilog:" + cpp_strerror(-r
));
1053 for (int i
= 0; i
< shards_num
; ++i
, ++shard_id
) {
1054 r
= store
->svc()->datalog_rados
->add_entry(bucket_info
, shard_id
);
1056 set_err_msg(err_msg
, "ERROR: failed writing data log:" + cpp_strerror(-r
));
1065 int RGWBucket::policy_bl_to_stream(bufferlist
& bl
, ostream
& o
)
1067 RGWAccessControlPolicy_S3
policy(g_ceph_context
);
1068 int ret
= decode_bl(bl
, policy
);
1070 ldout(store
->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl
;
1076 int rgw_object_get_attr(rgw::sal::RGWRadosStore
* store
, const RGWBucketInfo
& bucket_info
,
1077 const rgw_obj
& obj
, const char* attr_name
,
1078 bufferlist
& out_bl
, optional_yield y
)
1080 RGWObjectCtx
obj_ctx(store
);
1081 RGWRados::Object
op_target(store
->getRados(), bucket_info
, obj_ctx
, obj
);
1082 RGWRados::Object::Read
rop(&op_target
);
1084 return rop
.get_attr(attr_name
, out_bl
, y
);
1087 int RGWBucket::get_policy(RGWBucketAdminOpState
& op_state
, RGWAccessControlPolicy
& policy
, optional_yield y
)
1089 std::string object_name
= op_state
.get_object_name();
1090 rgw_bucket bucket
= op_state
.get_bucket();
1092 RGWBucketInfo bucket_info
;
1093 map
<string
, bufferlist
> attrs
;
1094 int ret
= store
->getRados()->get_bucket_info(store
->svc(), bucket
.tenant
, bucket
.name
, bucket_info
, NULL
, null_yield
, &attrs
);
1099 if (!object_name
.empty()) {
1101 rgw_obj
obj(bucket
, object_name
);
1103 ret
= rgw_object_get_attr(store
, bucket_info
, obj
, RGW_ATTR_ACL
, bl
, y
);
1108 ret
= decode_bl(bl
, policy
);
1110 ldout(store
->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl
;
1115 map
<string
, bufferlist
>::iterator aiter
= attrs
.find(RGW_ATTR_ACL
);
1116 if (aiter
== attrs
.end()) {
1120 ret
= decode_bl(aiter
->second
, policy
);
1122 ldout(store
->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl
;
1129 int RGWBucketAdminOp::get_policy(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
,
1130 RGWAccessControlPolicy
& policy
)
1134 int ret
= bucket
.init(store
, op_state
, null_yield
);
1138 ret
= bucket
.get_policy(op_state
, policy
, null_yield
);
1145 /* Wrappers to facilitate RESTful interface */
1148 int RGWBucketAdminOp::get_policy(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
,
1149 RGWFormatterFlusher
& flusher
)
1151 RGWAccessControlPolicy
policy(store
->ctx());
1153 int ret
= get_policy(store
, op_state
, policy
);
1157 Formatter
*formatter
= flusher
.get_formatter();
1161 formatter
->open_object_section("policy");
1162 policy
.dump(formatter
);
1163 formatter
->close_section();
1170 int RGWBucketAdminOp::dump_s3_policy(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
,
1173 RGWAccessControlPolicy_S3
policy(store
->ctx());
1175 int ret
= get_policy(store
, op_state
, policy
);
1184 int RGWBucketAdminOp::unlink(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
)
1188 int ret
= bucket
.init(store
, op_state
, null_yield
);
1192 return bucket
.unlink(op_state
, null_yield
);
1195 int RGWBucketAdminOp::link(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
, string
*err
)
1198 map
<string
, bufferlist
> attrs
;
1200 int ret
= bucket
.init(store
, op_state
, null_yield
, err
, &attrs
);
1204 return bucket
.link(op_state
, null_yield
, attrs
, err
);
1208 int RGWBucketAdminOp::chown(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
, const string
& marker
, string
*err
)
1211 map
<string
, bufferlist
> attrs
;
1213 int ret
= bucket
.init(store
, op_state
, null_yield
, err
, &attrs
);
1217 ret
= bucket
.link(op_state
, null_yield
, attrs
, err
);
1221 return bucket
.chown(op_state
, marker
, null_yield
, err
);
1225 int RGWBucketAdminOp::check_index(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
,
1226 RGWFormatterFlusher
& flusher
, optional_yield y
)
1229 map
<RGWObjCategory
, RGWStorageStats
> existing_stats
;
1230 map
<RGWObjCategory
, RGWStorageStats
> calculated_stats
;
1235 ret
= bucket
.init(store
, op_state
, null_yield
);
1239 Formatter
*formatter
= flusher
.get_formatter();
1242 ret
= bucket
.check_bad_index_multipart(op_state
, flusher
);
1246 ret
= bucket
.check_object_index(op_state
, flusher
, y
);
1250 ret
= bucket
.check_index(op_state
, existing_stats
, calculated_stats
);
1254 dump_index_check(existing_stats
, calculated_stats
, formatter
);
1260 int RGWBucketAdminOp::remove_bucket(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
,
1261 optional_yield y
, bool bypass_gc
, bool keep_index_consistent
)
1263 std::unique_ptr
<rgw::sal::RGWBucket
> bucket
;
1264 std::unique_ptr
<rgw::sal::RGWUser
> user
= store
->get_user(op_state
.get_user_id());
1266 int ret
= store
->get_bucket(user
.get(), user
->get_tenant(), op_state
.get_bucket_name(),
1272 ret
= rgw_remove_bucket_bypass_gc(store
, bucket
->get_key(), op_state
.get_max_aio(), keep_index_consistent
, y
);
1274 ret
= bucket
->remove_bucket(op_state
.will_delete_children(), string(), string(),
1280 int RGWBucketAdminOp::remove_object(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
)
1284 int ret
= bucket
.init(store
, op_state
, null_yield
);
1288 return bucket
.remove_object(op_state
);
1291 int RGWBucketAdminOp::sync_bucket(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
, string
*err_msg
)
1294 map
<string
, bufferlist
> attrs
;
1295 int ret
= bucket
.init(store
, op_state
, null_yield
, err_msg
, &attrs
);
1300 return bucket
.sync(op_state
, &attrs
, err_msg
);
1303 static int bucket_stats(rgw::sal::RGWRadosStore
*store
,
1304 const std::string
& tenant_name
,
1305 const std::string
& bucket_name
,
1306 Formatter
*formatter
)
1308 RGWBucketInfo bucket_info
;
1309 map
<RGWObjCategory
, RGWStorageStats
> stats
;
1310 map
<string
, bufferlist
> attrs
;
1313 int r
= store
->getRados()->get_bucket_info(store
->svc(),
1314 tenant_name
, bucket_name
, bucket_info
,
1315 &mtime
, null_yield
, &attrs
);
1320 rgw_bucket
& bucket
= bucket_info
.bucket
;
1322 string bucket_ver
, master_ver
;
1324 int ret
= store
->getRados()->get_bucket_stats(bucket_info
, RGW_NO_SHARD
,
1325 &bucket_ver
, &master_ver
, stats
,
1328 cerr
<< "error getting bucket stats bucket=" << bucket
.name
<< " ret=" << ret
<< std::endl
;
1333 utime_t
ctime_ut(bucket_info
.creation_time
);
1335 formatter
->open_object_section("stats");
1336 formatter
->dump_string("bucket", bucket
.name
);
1337 formatter
->dump_int("num_shards",
1338 bucket_info
.layout
.current_index
.layout
.normal
.num_shards
);
1339 formatter
->dump_string("tenant", bucket
.tenant
);
1340 formatter
->dump_string("zonegroup", bucket_info
.zonegroup
);
1341 formatter
->dump_string("placement_rule", bucket_info
.placement_rule
.to_str());
1342 ::encode_json("explicit_placement", bucket
.explicit_placement
, formatter
);
1343 formatter
->dump_string("id", bucket
.bucket_id
);
1344 formatter
->dump_string("marker", bucket
.marker
);
1345 formatter
->dump_stream("index_type") << bucket_info
.layout
.current_index
.layout
.type
;
1346 ::encode_json("owner", bucket_info
.owner
, formatter
);
1347 formatter
->dump_string("ver", bucket_ver
);
1348 formatter
->dump_string("master_ver", master_ver
);
1349 ut
.gmtime(formatter
->dump_stream("mtime"));
1350 ctime_ut
.gmtime(formatter
->dump_stream("creation_time"));
1351 formatter
->dump_string("max_marker", max_marker
);
1352 dump_bucket_usage(stats
, formatter
);
1353 encode_json("bucket_quota", bucket_info
.quota
, formatter
);
1356 auto iter
= attrs
.find(RGW_ATTR_TAGS
);
1357 if (iter
!= attrs
.end()) {
1358 RGWObjTagSet_S3 tagset
;
1359 bufferlist::const_iterator piter
{&iter
->second
};
1361 tagset
.decode(piter
);
1362 tagset
.dump(formatter
);
1363 } catch (buffer::error
& err
) {
1364 cerr
<< "ERROR: caught buffer:error, couldn't decode TagSet" << std::endl
;
1368 // TODO: bucket CORS
1370 formatter
->close_section();
1375 int RGWBucketAdminOp::limit_check(rgw::sal::RGWRadosStore
*store
,
1376 RGWBucketAdminOpState
& op_state
,
1377 const std::list
<std::string
>& user_ids
,
1378 RGWFormatterFlusher
& flusher
, optional_yield y
,
1382 const size_t max_entries
=
1383 store
->ctx()->_conf
->rgw_list_buckets_max_chunk
;
1385 const size_t safe_max_objs_per_shard
=
1386 store
->ctx()->_conf
->rgw_safe_max_objects_per_shard
;
1388 uint16_t shard_warn_pct
=
1389 store
->ctx()->_conf
->rgw_shard_warning_threshold
;
1390 if (shard_warn_pct
> 100)
1391 shard_warn_pct
= 90;
1393 Formatter
*formatter
= flusher
.get_formatter();
1396 formatter
->open_array_section("users");
1398 for (const auto& user_id
: user_ids
) {
1400 formatter
->open_object_section("user");
1401 formatter
->dump_string("user_id", user_id
);
1402 formatter
->open_array_section("buckets");
1405 rgw::sal::RGWBucketList buckets
;
1407 rgw::sal::RGWRadosUser
user(store
, rgw_user(user_id
));
1409 ret
= user
.list_buckets(marker
, string(), max_entries
, false, buckets
, y
);
1414 map
<string
, std::unique_ptr
<rgw::sal::RGWBucket
>>& m_buckets
= buckets
.get_buckets();
1416 for (const auto& iter
: m_buckets
) {
1417 auto& bucket
= iter
.second
;
1418 uint32_t num_shards
= 1;
1419 uint64_t num_objects
= 0;
1421 /* need info for num_shards */
1424 marker
= bucket
->get_name(); /* Casey's location for marker update,
1425 * as we may now not reach the end of
1428 ret
= store
->getRados()->get_bucket_info(store
->svc(), bucket
->get_tenant(),
1429 bucket
->get_name(), info
, nullptr,
1434 /* need stats for num_entries */
1435 string bucket_ver
, master_ver
;
1436 std::map
<RGWObjCategory
, RGWStorageStats
> stats
;
1437 ret
= store
->getRados()->get_bucket_stats(info
, RGW_NO_SHARD
, &bucket_ver
,
1438 &master_ver
, stats
, nullptr);
1443 for (const auto& s
: stats
) {
1444 num_objects
+= s
.second
.num_objects
;
1447 num_shards
= info
.layout
.current_index
.layout
.normal
.num_shards
;
1448 uint64_t objs_per_shard
=
1449 (num_shards
) ? num_objects
/num_shards
: num_objects
;
1453 uint64_t fill_pct
= objs_per_shard
* 100 / safe_max_objs_per_shard
;
1454 if (fill_pct
> 100) {
1455 ss
<< "OVER " << fill_pct
<< "%";
1457 } else if (fill_pct
>= shard_warn_pct
) {
1458 ss
<< "WARN " << fill_pct
<< "%";
1465 if (warn
|| !warnings_only
) {
1466 formatter
->open_object_section("bucket");
1467 formatter
->dump_string("bucket", bucket
->get_name());
1468 formatter
->dump_string("tenant", bucket
->get_tenant());
1469 formatter
->dump_int("num_objects", num_objects
);
1470 formatter
->dump_int("num_shards", num_shards
);
1471 formatter
->dump_int("objects_per_shard", objs_per_shard
);
1472 formatter
->dump_string("fill_status", ss
.str());
1473 formatter
->close_section();
1477 formatter
->flush(cout
);
1478 } while (buckets
.is_truncated()); /* foreach: bucket */
1480 formatter
->close_section();
1481 formatter
->close_section();
1482 formatter
->flush(cout
);
1484 } /* foreach: user_id */
1486 formatter
->close_section();
1487 formatter
->flush(cout
);
1490 } /* RGWBucketAdminOp::limit_check */
1492 int RGWBucketAdminOp::info(rgw::sal::RGWRadosStore
*store
,
1493 RGWBucketAdminOpState
& op_state
,
1494 RGWFormatterFlusher
& flusher
,
1499 const std::string
& bucket_name
= op_state
.get_bucket_name();
1500 if (!bucket_name
.empty()) {
1501 ret
= bucket
.init(store
, op_state
, null_yield
);
1503 return -ERR_NO_SUCH_BUCKET
;
1508 Formatter
*formatter
= flusher
.get_formatter();
1511 CephContext
*cct
= store
->ctx();
1513 const size_t max_entries
= cct
->_conf
->rgw_list_buckets_max_chunk
;
1515 const bool show_stats
= op_state
.will_fetch_stats();
1516 const rgw_user
& user_id
= op_state
.get_user_id();
1517 if (op_state
.is_user_op()) {
1518 formatter
->open_array_section("buckets");
1520 rgw::sal::RGWBucketList buckets
;
1521 rgw::sal::RGWRadosUser
user(store
, op_state
.get_user_id());
1523 const std::string empty_end_marker
;
1524 constexpr bool no_need_stats
= false; // set need_stats to false
1527 ret
= user
.list_buckets(marker
, empty_end_marker
, max_entries
,
1528 no_need_stats
, buckets
, y
);
1533 const std::string
* marker_cursor
= nullptr;
1534 map
<string
, std::unique_ptr
<rgw::sal::RGWBucket
>>& m
= buckets
.get_buckets();
1536 for (const auto& i
: m
) {
1537 const std::string
& obj_name
= i
.first
;
1538 if (!bucket_name
.empty() && bucket_name
!= obj_name
) {
1543 bucket_stats(store
, user_id
.tenant
, obj_name
, formatter
);
1545 formatter
->dump_string("bucket", obj_name
);
1548 marker_cursor
= &obj_name
;
1550 if (marker_cursor
) {
1551 marker
= *marker_cursor
;
1555 } while (buckets
.is_truncated());
1557 formatter
->close_section();
1558 } else if (!bucket_name
.empty()) {
1559 ret
= bucket_stats(store
, user_id
.tenant
, bucket_name
, formatter
);
1564 void *handle
= nullptr;
1565 bool truncated
= true;
1567 formatter
->open_array_section("buckets");
1568 ret
= store
->ctl()->meta
.mgr
->list_keys_init("bucket", &handle
);
1569 while (ret
== 0 && truncated
) {
1570 std::list
<std::string
> buckets
;
1571 constexpr int max_keys
= 1000;
1572 ret
= store
->ctl()->meta
.mgr
->list_keys_next(handle
, max_keys
, buckets
,
1574 for (auto& bucket_name
: buckets
) {
1576 bucket_stats(store
, user_id
.tenant
, bucket_name
, formatter
);
1578 formatter
->dump_string("bucket", bucket_name
);
1582 store
->ctl()->meta
.mgr
->list_keys_complete(handle
);
1584 formatter
->close_section();
1592 int RGWBucketAdminOp::set_quota(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
)
1596 int ret
= bucket
.init(store
, op_state
, null_yield
);
1599 return bucket
.set_quota(op_state
);
1602 static int purge_bucket_instance(rgw::sal::RGWRadosStore
*store
, const RGWBucketInfo
& bucket_info
)
1604 int max_shards
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? bucket_info
.layout
.current_index
.layout
.normal
.num_shards
: 1);
1605 for (int i
= 0; i
< max_shards
; i
++) {
1606 RGWRados::BucketShard
bs(store
->getRados());
1607 int shard_id
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? i
: -1);
1608 int ret
= bs
.init(bucket_info
.bucket
, shard_id
, bucket_info
.layout
.current_index
, nullptr);
1610 cerr
<< "ERROR: bs.init(bucket=" << bucket_info
.bucket
<< ", shard=" << shard_id
1611 << "): " << cpp_strerror(-ret
) << std::endl
;
1614 ret
= store
->getRados()->bi_remove(bs
);
1616 cerr
<< "ERROR: failed to remove bucket index object: "
1617 << cpp_strerror(-ret
) << std::endl
;
1624 inline auto split_tenant(const std::string
& bucket_name
){
1625 auto p
= bucket_name
.find('/');
1626 if(p
!= std::string::npos
) {
1627 return std::make_pair(bucket_name
.substr(0,p
), bucket_name
.substr(p
+1));
1629 return std::make_pair(std::string(), bucket_name
);
1632 using bucket_instance_ls
= std::vector
<RGWBucketInfo
>;
1633 void get_stale_instances(rgw::sal::RGWRadosStore
*store
, const std::string
& bucket_name
,
1634 const vector
<std::string
>& lst
,
1635 bucket_instance_ls
& stale_instances
)
1638 auto obj_ctx
= store
->svc()->sysobj
->init_obj_ctx();
1640 bucket_instance_ls other_instances
;
1641 // first iterate over the entries, and pick up the done buckets; these
1642 // are guaranteed to be stale
1643 for (const auto& bucket_instance
: lst
){
1644 RGWBucketInfo binfo
;
1645 int r
= store
->getRados()->get_bucket_instance_info(obj_ctx
, bucket_instance
,
1646 binfo
, nullptr,nullptr, null_yield
);
1648 // this can only happen if someone deletes us right when we're processing
1649 lderr(store
->ctx()) << "Bucket instance is invalid: " << bucket_instance
1650 << cpp_strerror(-r
) << dendl
;
1653 if (binfo
.reshard_status
== cls_rgw_reshard_status::DONE
)
1654 stale_instances
.emplace_back(std::move(binfo
));
1656 other_instances
.emplace_back(std::move(binfo
));
1660 // Read the cur bucket info, if the bucket doesn't exist we can simply return
1661 // all the instances
1662 auto [tenant
, bucket
] = split_tenant(bucket_name
);
1663 RGWBucketInfo cur_bucket_info
;
1664 int r
= store
->getRados()->get_bucket_info(store
->svc(), tenant
, bucket
, cur_bucket_info
, nullptr, null_yield
);
1667 // bucket doesn't exist, everything is stale then
1668 stale_instances
.insert(std::end(stale_instances
),
1669 std::make_move_iterator(other_instances
.begin()),
1670 std::make_move_iterator(other_instances
.end()));
1672 // all bets are off if we can't read the bucket, just return the sureshot stale instances
1673 lderr(store
->ctx()) << "error: reading bucket info for bucket: "
1674 << bucket
<< cpp_strerror(-r
) << dendl
;
1679 // Don't process further in this round if bucket is resharding
1680 if (cur_bucket_info
.reshard_status
== cls_rgw_reshard_status::IN_PROGRESS
)
1683 other_instances
.erase(std::remove_if(other_instances
.begin(), other_instances
.end(),
1684 [&cur_bucket_info
](const RGWBucketInfo
& b
){
1685 return (b
.bucket
.bucket_id
== cur_bucket_info
.bucket
.bucket_id
||
1686 b
.bucket
.bucket_id
== cur_bucket_info
.new_bucket_instance_id
);
1688 other_instances
.end());
1690 // check if there are still instances left
1691 if (other_instances
.empty()) {
1695 // Now we have a bucket with instances where the reshard status is none, this
1696 // usually happens when the reshard process couldn't complete, lockdown the
1697 // bucket and walk through these instances to make sure no one else interferes
1700 RGWBucketReshardLock
reshard_lock(store
, cur_bucket_info
, true);
1701 r
= reshard_lock
.lock();
1703 // most likely bucket is under reshard, return the sureshot stale instances
1704 ldout(store
->ctx(), 5) << __func__
1705 << "failed to take reshard lock; reshard underway likey" << dendl
;
1708 auto sg
= make_scope_guard([&reshard_lock
](){ reshard_lock
.unlock();} );
1709 // this should be fast enough that we may not need to renew locks and check
1710 // exit status?, should we read the values of the instances again?
1711 stale_instances
.insert(std::end(stale_instances
),
1712 std::make_move_iterator(other_instances
.begin()),
1713 std::make_move_iterator(other_instances
.end()));
1719 static int process_stale_instances(rgw::sal::RGWRadosStore
*store
, RGWBucketAdminOpState
& op_state
,
1720 RGWFormatterFlusher
& flusher
,
1721 std::function
<void(const bucket_instance_ls
&,
1723 rgw::sal::RGWRadosStore
*)> process_f
)
1727 Formatter
*formatter
= flusher
.get_formatter();
1728 static constexpr auto default_max_keys
= 1000;
1730 int ret
= store
->ctl()->meta
.mgr
->list_keys_init("bucket.instance", marker
, &handle
);
1732 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
1738 formatter
->open_array_section("keys");
1739 auto g
= make_scope_guard([&store
, &handle
, &formatter
]() {
1740 store
->ctl()->meta
.mgr
->list_keys_complete(handle
);
1741 formatter
->close_section(); // keys
1742 formatter
->flush(cout
);
1746 list
<std::string
> keys
;
1748 ret
= store
->ctl()->meta
.mgr
->list_keys_next(handle
, default_max_keys
, keys
, &truncated
);
1749 if (ret
< 0 && ret
!= -ENOENT
) {
1750 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
1752 } if (ret
!= -ENOENT
) {
1753 // partition the list of buckets by buckets as the listing is un sorted,
1754 // since it would minimize the reads to bucket_info
1755 std::unordered_map
<std::string
, std::vector
<std::string
>> bucket_instance_map
;
1756 for (auto &key
: keys
) {
1757 auto pos
= key
.find(':');
1758 if(pos
!= std::string::npos
)
1759 bucket_instance_map
[key
.substr(0,pos
)].emplace_back(std::move(key
));
1761 for (const auto& kv
: bucket_instance_map
) {
1762 bucket_instance_ls stale_lst
;
1763 get_stale_instances(store
, kv
.first
, kv
.second
, stale_lst
);
1764 process_f(stale_lst
, formatter
, store
);
1767 } while (truncated
);
1772 int RGWBucketAdminOp::list_stale_instances(rgw::sal::RGWRadosStore
*store
,
1773 RGWBucketAdminOpState
& op_state
,
1774 RGWFormatterFlusher
& flusher
)
1776 auto process_f
= [](const bucket_instance_ls
& lst
,
1777 Formatter
*formatter
,
1778 rgw::sal::RGWRadosStore
*){
1779 for (const auto& binfo
: lst
)
1780 formatter
->dump_string("key", binfo
.bucket
.get_key());
1782 return process_stale_instances(store
, op_state
, flusher
, process_f
);
1786 int RGWBucketAdminOp::clear_stale_instances(rgw::sal::RGWRadosStore
*store
,
1787 RGWBucketAdminOpState
& op_state
,
1788 RGWFormatterFlusher
& flusher
)
1790 auto process_f
= [](const bucket_instance_ls
& lst
,
1791 Formatter
*formatter
,
1792 rgw::sal::RGWRadosStore
*store
){
1793 for (const auto &binfo
: lst
) {
1794 int ret
= purge_bucket_instance(store
, binfo
);
1796 auto md_key
= "bucket.instance:" + binfo
.bucket
.get_key();
1797 ret
= store
->ctl()->meta
.mgr
->remove(md_key
, null_yield
);
1799 formatter
->open_object_section("delete_status");
1800 formatter
->dump_string("bucket_instance", binfo
.bucket
.get_key());
1801 formatter
->dump_int("status", -ret
);
1802 formatter
->close_section();
1806 return process_stale_instances(store
, op_state
, flusher
, process_f
);
1809 static int fix_single_bucket_lc(rgw::sal::RGWRadosStore
*store
,
1810 const std::string
& tenant_name
,
1811 const std::string
& bucket_name
)
1813 RGWBucketInfo bucket_info
;
1814 map
<std::string
, bufferlist
> bucket_attrs
;
1815 int ret
= store
->getRados()->get_bucket_info(store
->svc(), tenant_name
, bucket_name
,
1816 bucket_info
, nullptr, null_yield
, &bucket_attrs
);
1818 // TODO: Should we handle the case where the bucket could've been removed between
1819 // listing and fetching?
1823 return rgw::lc::fix_lc_shard_entry(store
, store
->get_rgwlc()->get_lc(), bucket_info
,
1827 static void format_lc_status(Formatter
* formatter
,
1828 const std::string
& tenant_name
,
1829 const std::string
& bucket_name
,
1832 formatter
->open_object_section("bucket_entry");
1833 std::string entry
= tenant_name
.empty() ? bucket_name
: tenant_name
+ "/" + bucket_name
;
1834 formatter
->dump_string("bucket", entry
);
1835 formatter
->dump_int("status", status
);
1836 formatter
->close_section(); // bucket_entry
1839 static void process_single_lc_entry(rgw::sal::RGWRadosStore
*store
,
1840 Formatter
*formatter
,
1841 const std::string
& tenant_name
,
1842 const std::string
& bucket_name
)
1844 int ret
= fix_single_bucket_lc(store
, tenant_name
, bucket_name
);
1845 format_lc_status(formatter
, tenant_name
, bucket_name
, -ret
);
1848 int RGWBucketAdminOp::fix_lc_shards(rgw::sal::RGWRadosStore
*store
,
1849 RGWBucketAdminOpState
& op_state
,
1850 RGWFormatterFlusher
& flusher
)
1854 Formatter
*formatter
= flusher
.get_formatter();
1855 static constexpr auto default_max_keys
= 1000;
1858 if (const std::string
& bucket_name
= op_state
.get_bucket_name();
1859 ! bucket_name
.empty()) {
1860 const rgw_user user_id
= op_state
.get_user_id();
1861 process_single_lc_entry(store
, formatter
, user_id
.tenant
, bucket_name
);
1862 formatter
->flush(cout
);
1864 int ret
= store
->ctl()->meta
.mgr
->list_keys_init("bucket", marker
, &handle
);
1866 std::cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
1871 formatter
->open_array_section("lc_fix_status");
1872 auto sg
= make_scope_guard([&store
, &handle
, &formatter
](){
1873 store
->ctl()->meta
.mgr
->list_keys_complete(handle
);
1874 formatter
->close_section(); // lc_fix_status
1875 formatter
->flush(cout
);
1878 list
<std::string
> keys
;
1879 ret
= store
->ctl()->meta
.mgr
->list_keys_next(handle
, default_max_keys
, keys
, &truncated
);
1880 if (ret
< 0 && ret
!= -ENOENT
) {
1881 std::cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
1883 } if (ret
!= -ENOENT
) {
1884 for (const auto &key
:keys
) {
1885 auto [tenant_name
, bucket_name
] = split_tenant(key
);
1886 process_single_lc_entry(store
, formatter
, tenant_name
, bucket_name
);
1889 formatter
->flush(cout
); // regularly flush every 1k entries
1890 } while (truncated
);
1898 static bool has_object_expired(rgw::sal::RGWRadosStore
*store
,
1899 const RGWBucketInfo
& bucket_info
,
1900 const rgw_obj_key
& key
, utime_t
& delete_at
)
1902 rgw_obj
obj(bucket_info
.bucket
, key
);
1903 bufferlist delete_at_bl
;
1905 int ret
= rgw_object_get_attr(store
, bucket_info
, obj
, RGW_ATTR_DELETE_AT
, delete_at_bl
, null_yield
);
1907 return false; // no delete at attr, proceed
1910 ret
= decode_bl(delete_at_bl
, delete_at
);
1912 return false; // failed to parse
1915 if (delete_at
<= ceph_clock_now() && !delete_at
.is_zero()) {
1922 static int fix_bucket_obj_expiry(rgw::sal::RGWRadosStore
*store
,
1923 const RGWBucketInfo
& bucket_info
,
1924 RGWFormatterFlusher
& flusher
, bool dry_run
)
1926 if (bucket_info
.bucket
.bucket_id
== bucket_info
.bucket
.marker
) {
1927 lderr(store
->ctx()) << "Not a resharded bucket skipping" << dendl
;
1928 return 0; // not a resharded bucket, move along
1931 Formatter
*formatter
= flusher
.get_formatter();
1932 formatter
->open_array_section("expired_deletion_status");
1933 auto sg
= make_scope_guard([&formatter
] {
1934 formatter
->close_section();
1935 formatter
->flush(std::cout
);
1938 RGWRados::Bucket
target(store
->getRados(), bucket_info
);
1939 RGWRados::Bucket::List
list_op(&target
);
1941 list_op
.params
.list_versions
= bucket_info
.versioned();
1942 list_op
.params
.allow_unordered
= true;
1944 bool is_truncated
{false};
1946 std::vector
<rgw_bucket_dir_entry
> objs
;
1948 int ret
= list_op
.list_objects(listing_max_entries
, &objs
, nullptr,
1949 &is_truncated
, null_yield
);
1951 lderr(store
->ctx()) << "ERROR failed to list objects in the bucket" << dendl
;
1954 for (const auto& obj
: objs
) {
1955 rgw_obj_key
key(obj
.key
);
1957 if (has_object_expired(store
, bucket_info
, key
, delete_at
)) {
1958 formatter
->open_object_section("object_status");
1959 formatter
->dump_string("object", key
.name
);
1960 formatter
->dump_stream("delete_at") << delete_at
;
1963 ret
= rgw_remove_object(store
, bucket_info
, bucket_info
.bucket
, key
);
1964 formatter
->dump_int("status", ret
);
1967 formatter
->close_section(); // object_status
1970 formatter
->flush(cout
); // regularly flush every 1k entries
1971 } while (is_truncated
);
1976 int RGWBucketAdminOp::fix_obj_expiry(rgw::sal::RGWRadosStore
*store
,
1977 RGWBucketAdminOpState
& op_state
,
1978 RGWFormatterFlusher
& flusher
, bool dry_run
)
1980 RGWBucket admin_bucket
;
1981 int ret
= admin_bucket
.init(store
, op_state
, null_yield
);
1983 lderr(store
->ctx()) << "failed to initialize bucket" << dendl
;
1987 return fix_bucket_obj_expiry(store
, admin_bucket
.get_bucket_info(), flusher
, dry_run
);
1990 void RGWBucketCompleteInfo::dump(Formatter
*f
) const {
1991 encode_json("bucket_info", info
, f
);
1992 encode_json("attrs", attrs
, f
);
1995 void RGWBucketCompleteInfo::decode_json(JSONObj
*obj
) {
1996 JSONDecoder::decode_json("bucket_info", info
, obj
);
1997 JSONDecoder::decode_json("attrs", attrs
, obj
);
2000 class RGWBucketMetadataHandler
: public RGWBucketMetadataHandlerBase
{
2003 RGWSI_Bucket
*bucket
{nullptr};
2007 RGWBucketCtl
*bucket
{nullptr};
2010 RGWBucketMetadataHandler() {}
2012 void init(RGWSI_Bucket
*bucket_svc
,
2013 RGWBucketCtl
*bucket_ctl
) override
{
2014 base_init(bucket_svc
->ctx(),
2015 bucket_svc
->get_ep_be_handler().get());
2016 svc
.bucket
= bucket_svc
;
2017 ctl
.bucket
= bucket_ctl
;
2020 string
get_type() override
{ return "bucket"; }
2022 RGWMetadataObject
*get_meta_obj(JSONObj
*jo
, const obj_version
& objv
, const ceph::real_time
& mtime
) override
{
2023 RGWBucketEntryPoint be
;
2026 decode_json_obj(be
, jo
);
2027 } catch (JSONDecoder::err
& e
) {
2031 return new RGWBucketEntryMetadataObject(be
, objv
, mtime
);
2034 int do_get(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWMetadataObject
**obj
, optional_yield y
) override
{
2035 RGWObjVersionTracker ot
;
2036 RGWBucketEntryPoint be
;
2039 map
<string
, bufferlist
> attrs
;
2041 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
2043 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, entry
, &be
, &ot
, &mtime
, &attrs
, y
);
2047 RGWBucketEntryMetadataObject
*mdo
= new RGWBucketEntryMetadataObject(be
, ot
.read_version
, mtime
, std::move(attrs
));
2054 int do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2055 RGWMetadataObject
*obj
,
2056 RGWObjVersionTracker
& objv_tracker
,
2058 RGWMDLogSyncType type
, bool from_remote_zone
) override
;
2060 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
,
2061 optional_yield y
) override
{
2062 RGWBucketEntryPoint be
;
2064 real_time orig_mtime
;
2066 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
2068 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, entry
, &be
, &objv_tracker
, &orig_mtime
, nullptr, y
);
2073 * We're unlinking the bucket but we don't want to update the entrypoint here - we're removing
2074 * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal
2075 * will incorrectly fail.
2077 ret
= ctl
.bucket
->unlink_bucket(be
.owner
, be
.bucket
, y
, false);
2079 lderr(svc
.bucket
->ctx()) << "could not unlink bucket=" << entry
<< " owner=" << be
.owner
<< dendl
;
2082 ret
= svc
.bucket
->remove_bucket_entrypoint_info(ctx
, entry
, &objv_tracker
, y
);
2084 lderr(svc
.bucket
->ctx()) << "could not delete bucket=" << entry
<< dendl
;
2090 int call(std::function
<int(RGWSI_Bucket_EP_Ctx
& ctx
)> f
) {
2091 return call(nullopt
, f
);
2094 int call(std::optional
<RGWSI_MetaBackend_CtxParams
> bectx_params
,
2095 std::function
<int(RGWSI_Bucket_EP_Ctx
& ctx
)> f
) {
2096 return be_handler
->call(bectx_params
, [&](RGWSI_MetaBackend_Handler::Op
*op
) {
2097 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
2103 class RGWMetadataHandlerPut_Bucket
: public RGWMetadataHandlerPut_SObj
2105 RGWBucketMetadataHandler
*bhandler
;
2106 RGWBucketEntryMetadataObject
*obj
;
2108 RGWMetadataHandlerPut_Bucket(RGWBucketMetadataHandler
*_handler
,
2109 RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2110 RGWMetadataObject
*_obj
, RGWObjVersionTracker
& objv_tracker
,
2112 RGWMDLogSyncType type
, bool from_remote_zone
) : RGWMetadataHandlerPut_SObj(_handler
, op
, entry
, obj
, objv_tracker
, y
, type
, from_remote_zone
),
2113 bhandler(_handler
) {
2114 obj
= static_cast<RGWBucketEntryMetadataObject
*>(_obj
);
2116 ~RGWMetadataHandlerPut_Bucket() {}
2118 void encode_obj(bufferlist
*bl
) override
{
2119 obj
->get_ep().encode(*bl
);
2122 int put_checked() override
;
2123 int put_post() override
;
2126 int RGWBucketMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2127 RGWMetadataObject
*obj
,
2128 RGWObjVersionTracker
& objv_tracker
,
2130 RGWMDLogSyncType type
, bool from_remote_zone
)
2132 RGWMetadataHandlerPut_Bucket
put_op(this, op
, entry
, obj
, objv_tracker
, y
, type
, from_remote_zone
);
2133 return do_put_operate(&put_op
);
2136 int RGWMetadataHandlerPut_Bucket::put_checked()
2138 RGWBucketEntryMetadataObject
*orig_obj
= static_cast<RGWBucketEntryMetadataObject
*>(old_obj
);
2141 obj
->set_pattrs(&orig_obj
->get_attrs());
2144 auto& be
= obj
->get_ep();
2145 auto mtime
= obj
->get_mtime();
2146 auto pattrs
= obj
->get_pattrs();
2148 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
2150 return bhandler
->svc
.bucket
->store_bucket_entrypoint_info(ctx
, entry
,
2159 int RGWMetadataHandlerPut_Bucket::put_post()
2161 auto& be
= obj
->get_ep();
2167 ret
= bhandler
->ctl
.bucket
->link_bucket(be
.owner
, be
.bucket
, be
.creation_time
, y
, false);
2169 ret
= bhandler
->ctl
.bucket
->unlink_bucket(be
.owner
, be
.bucket
, y
, false);
2175 static void get_md5_digest(const RGWBucketEntryPoint
*be
, string
& md5_digest
) {
2177 char md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
2178 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
2181 Formatter
*f
= new JSONFormatter(false);
2186 hash
.Update((const unsigned char *)bl
.c_str(), bl
.length());
2189 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, md5
);
2196 #define ARCHIVE_META_ATTR RGW_ATTR_PREFIX "zone.archive.info"
2198 struct archive_meta_info
{
2199 rgw_bucket orig_bucket
;
2201 bool from_attrs(CephContext
*cct
, map
<string
, bufferlist
>& attrs
) {
2202 auto iter
= attrs
.find(ARCHIVE_META_ATTR
);
2203 if (iter
== attrs
.end()) {
2207 auto bliter
= iter
->second
.cbegin();
2210 } catch (buffer::error
& err
) {
2211 ldout(cct
, 0) << "ERROR: failed to decode archive meta info" << dendl
;
2218 void store_in_attrs(map
<string
, bufferlist
>& attrs
) const {
2219 encode(attrs
[ARCHIVE_META_ATTR
]);
2222 void encode(bufferlist
& bl
) const {
2223 ENCODE_START(1, 1, bl
);
2224 encode(orig_bucket
, bl
);
2228 void decode(bufferlist::const_iterator
& bl
) {
2229 DECODE_START(1, bl
);
2230 decode(orig_bucket
, bl
);
2234 WRITE_CLASS_ENCODER(archive_meta_info
)
2236 class RGWArchiveBucketMetadataHandler
: public RGWBucketMetadataHandler
{
2238 RGWArchiveBucketMetadataHandler() {}
2240 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
,
2241 optional_yield y
) override
{
2242 auto cct
= svc
.bucket
->ctx();
2244 RGWSI_Bucket_EP_Ctx
ctx(op
->ctx());
2246 ldout(cct
, 5) << "SKIP: bucket removal is not allowed on archive zone: bucket:" << entry
<< " ... proceeding to rename" << dendl
;
2248 string tenant_name
, bucket_name
;
2249 parse_bucket(entry
, &tenant_name
, &bucket_name
);
2250 rgw_bucket entry_bucket
;
2251 entry_bucket
.tenant
= tenant_name
;
2252 entry_bucket
.name
= bucket_name
;
2256 /* read original entrypoint */
2258 RGWBucketEntryPoint be
;
2259 map
<string
, bufferlist
> attrs
;
2260 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, entry
, &be
, &objv_tracker
, &mtime
, &attrs
, y
);
2265 string bi_meta_name
= RGWSI_Bucket::get_bi_meta_key(be
.bucket
);
2267 /* read original bucket instance info */
2269 map
<string
, bufferlist
> attrs_m
;
2270 ceph::real_time orig_mtime
;
2271 RGWBucketInfo old_bi
;
2273 ret
= ctl
.bucket
->read_bucket_instance_info(be
.bucket
, &old_bi
, y
, RGWBucketCtl::BucketInstance::GetParams()
2274 .set_mtime(&orig_mtime
)
2275 .set_attrs(&attrs_m
));
2280 archive_meta_info ami
;
2282 if (!ami
.from_attrs(svc
.bucket
->ctx(), attrs_m
)) {
2283 ami
.orig_bucket
= old_bi
.bucket
;
2284 ami
.store_in_attrs(attrs_m
);
2287 /* generate a new bucket instance. We could have avoided this if we could just point a new
2288 * bucket entry point to the old bucket instance, however, due to limitation in the way
2289 * we index buckets under the user, bucket entrypoint and bucket instance of the same
2290 * bucket need to have the same name, so we need to copy the old bucket instance into
2291 * to a new entry with the new name
2294 string new_bucket_name
;
2296 RGWBucketInfo new_bi
= old_bi
;
2297 RGWBucketEntryPoint new_be
= be
;
2301 get_md5_digest(&new_be
, md5_digest
);
2302 new_bucket_name
= ami
.orig_bucket
.name
+ "-deleted-" + md5_digest
;
2304 new_bi
.bucket
.name
= new_bucket_name
;
2305 new_bi
.objv_tracker
.clear();
2307 new_be
.bucket
.name
= new_bucket_name
;
2309 ret
= ctl
.bucket
->store_bucket_instance_info(be
.bucket
, new_bi
, y
, RGWBucketCtl::BucketInstance::PutParams()
2310 .set_exclusive(false)
2311 .set_mtime(orig_mtime
)
2312 .set_attrs(&attrs_m
)
2313 .set_orig_info(&old_bi
));
2315 ldout(cct
, 0) << "ERROR: failed to put new bucket instance info for bucket=" << new_bi
.bucket
<< " ret=" << ret
<< dendl
;
2319 /* store a new entrypoint */
2321 RGWObjVersionTracker ot
;
2322 ot
.generate_new_write_ver(cct
);
2324 ret
= svc
.bucket
->store_bucket_entrypoint_info(ctx
, RGWSI_Bucket::get_entrypoint_meta_key(new_be
.bucket
),
2325 new_be
, true, mtime
, &attrs
, nullptr, y
);
2327 ldout(cct
, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be
.bucket
<< " ret=" << ret
<< dendl
;
2331 /* link new bucket */
2333 ret
= ctl
.bucket
->link_bucket(new_be
.owner
, new_be
.bucket
, new_be
.creation_time
, y
, false);
2335 ldout(cct
, 0) << "ERROR: failed to link new bucket for bucket=" << new_be
.bucket
<< " ret=" << ret
<< dendl
;
2339 /* clean up old stuff */
2341 ret
= ctl
.bucket
->unlink_bucket(be
.owner
, entry_bucket
, y
, false);
2343 lderr(cct
) << "could not unlink bucket=" << entry
<< " owner=" << be
.owner
<< dendl
;
2346 // if (ret == -ECANCELED) it means that there was a race here, and someone
2347 // wrote to the bucket entrypoint just before we removed it. The question is
2348 // whether it was a newly created bucket entrypoint ... in which case we
2349 // should ignore the error and move forward, or whether it is a higher version
2350 // of the same bucket instance ... in which we should retry
2351 ret
= svc
.bucket
->remove_bucket_entrypoint_info(ctx
,
2352 RGWSI_Bucket::get_entrypoint_meta_key(be
.bucket
),
2356 ldout(cct
, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be
.bucket
<< " ret=" << ret
<< dendl
;
2360 ret
= ctl
.bucket
->remove_bucket_instance_info(be
.bucket
, old_bi
, y
);
2362 lderr(cct
) << "could not delete bucket=" << entry
<< dendl
;
2371 int do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2372 RGWMetadataObject
*obj
,
2373 RGWObjVersionTracker
& objv_tracker
,
2375 RGWMDLogSyncType type
, bool from_remote_zone
) override
{
2376 if (entry
.find("-deleted-") != string::npos
) {
2377 RGWObjVersionTracker ot
;
2378 RGWMetadataObject
*robj
;
2379 int ret
= do_get(op
, entry
, &robj
, y
);
2380 if (ret
!= -ENOENT
) {
2384 ot
.read_version
= robj
->get_version();
2387 ret
= do_remove(op
, entry
, ot
, y
);
2394 return RGWBucketMetadataHandler::do_put(op
, entry
, obj
,
2395 objv_tracker
, y
, type
, from_remote_zone
);
2400 class RGWBucketInstanceMetadataHandler
: public RGWBucketInstanceMetadataHandlerBase
{
2401 int read_bucket_instance_entry(RGWSI_Bucket_BI_Ctx
& ctx
,
2402 const string
& entry
,
2403 RGWBucketCompleteInfo
*bi
,
2404 ceph::real_time
*pmtime
,
2406 return svc
.bucket
->read_bucket_instance_info(ctx
,
2415 RGWSI_Zone
*zone
{nullptr};
2416 RGWSI_Bucket
*bucket
{nullptr};
2417 RGWSI_BucketIndex
*bi
{nullptr};
2420 RGWBucketInstanceMetadataHandler() {}
2422 void init(RGWSI_Zone
*zone_svc
,
2423 RGWSI_Bucket
*bucket_svc
,
2424 RGWSI_BucketIndex
*bi_svc
) override
{
2425 base_init(bucket_svc
->ctx(),
2426 bucket_svc
->get_bi_be_handler().get());
2427 svc
.zone
= zone_svc
;
2428 svc
.bucket
= bucket_svc
;
2432 string
get_type() override
{ return "bucket.instance"; }
2434 RGWMetadataObject
*get_meta_obj(JSONObj
*jo
, const obj_version
& objv
, const ceph::real_time
& mtime
) override
{
2435 RGWBucketCompleteInfo bci
;
2438 decode_json_obj(bci
, jo
);
2439 } catch (JSONDecoder::err
& e
) {
2443 return new RGWBucketInstanceMetadataObject(bci
, objv
, mtime
);
2446 int do_get(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWMetadataObject
**obj
, optional_yield y
) override
{
2447 RGWBucketCompleteInfo bci
;
2450 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2452 int ret
= svc
.bucket
->read_bucket_instance_info(ctx
, entry
, &bci
.info
, &mtime
, &bci
.attrs
, y
);
2456 RGWBucketInstanceMetadataObject
*mdo
= new RGWBucketInstanceMetadataObject(bci
, bci
.info
.objv_tracker
.read_version
, mtime
);
2463 int do_put(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
,
2464 RGWMetadataObject
*_obj
, RGWObjVersionTracker
& objv_tracker
,
2466 RGWMDLogSyncType sync_type
, bool from_remote_zone
) override
;
2468 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
,
2469 optional_yield y
) override
{
2470 RGWBucketCompleteInfo bci
;
2472 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2474 int ret
= read_bucket_instance_entry(ctx
, entry
, &bci
, nullptr, y
);
2475 if (ret
< 0 && ret
!= -ENOENT
)
2478 return svc
.bucket
->remove_bucket_instance_info(ctx
, entry
, bci
.info
, &bci
.info
.objv_tracker
, y
);
2481 int call(std::function
<int(RGWSI_Bucket_BI_Ctx
& ctx
)> f
) {
2482 return call(nullopt
, f
);
2485 int call(std::optional
<RGWSI_MetaBackend_CtxParams
> bectx_params
,
2486 std::function
<int(RGWSI_Bucket_BI_Ctx
& ctx
)> f
) {
2487 return be_handler
->call(bectx_params
, [&](RGWSI_MetaBackend_Handler::Op
*op
) {
2488 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2494 class RGWMetadataHandlerPut_BucketInstance
: public RGWMetadataHandlerPut_SObj
2497 RGWBucketInstanceMetadataHandler
*bihandler
;
2498 RGWBucketInstanceMetadataObject
*obj
;
2500 RGWMetadataHandlerPut_BucketInstance(CephContext
*_cct
,
2501 RGWBucketInstanceMetadataHandler
*_handler
,
2502 RGWSI_MetaBackend_Handler::Op
*_op
, string
& entry
,
2503 RGWMetadataObject
*_obj
, RGWObjVersionTracker
& objv_tracker
,
2505 RGWMDLogSyncType type
, bool from_remote_zone
) : RGWMetadataHandlerPut_SObj(_handler
, _op
, entry
, obj
, objv_tracker
, y
, type
, from_remote_zone
),
2506 cct(_cct
), bihandler(_handler
) {
2507 obj
= static_cast<RGWBucketInstanceMetadataObject
*>(_obj
);
2509 auto& bci
= obj
->get_bci();
2510 obj
->set_pattrs(&bci
.attrs
);
2513 void encode_obj(bufferlist
*bl
) override
{
2514 obj
->get_bucket_info().encode(*bl
);
2517 int put_check() override
;
2518 int put_checked() override
;
2519 int put_post() override
;
2522 int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op
*op
,
2524 RGWMetadataObject
*obj
,
2525 RGWObjVersionTracker
& objv_tracker
,
2527 RGWMDLogSyncType type
, bool from_remote_zone
)
2529 RGWMetadataHandlerPut_BucketInstance
put_op(svc
.bucket
->ctx(), this, op
, entry
, obj
,
2530 objv_tracker
, y
, type
, from_remote_zone
);
2531 return do_put_operate(&put_op
);
2534 void init_default_bucket_layout(CephContext
*cct
, rgw::BucketLayout
& layout
,
2535 const RGWZone
& zone
,
2536 std::optional
<uint32_t> shards
,
2537 std::optional
<rgw::BucketIndexType
> type
) {
2538 layout
.current_index
.gen
= 0;
2539 layout
.current_index
.layout
.normal
.hash_type
= rgw::BucketHashType::Mod
;
2541 layout
.current_index
.layout
.type
=
2542 type
.value_or(rgw::BucketIndexType::Normal
);
2545 layout
.current_index
.layout
.normal
.num_shards
= *shards
;
2546 } else if (cct
->_conf
->rgw_override_bucket_index_max_shards
> 0) {
2547 layout
.current_index
.layout
.normal
.num_shards
=
2548 cct
->_conf
->rgw_override_bucket_index_max_shards
;
2550 layout
.current_index
.layout
.normal
.num_shards
=
2551 zone
.bucket_index_max_shards
;
2554 if (layout
.current_index
.layout
.type
== rgw::BucketIndexType::Normal
) {
2555 layout
.logs
.push_back(log_layout_from_index(
2556 layout
.current_index
.gen
,
2557 layout
.current_index
.layout
.normal
));
2561 int RGWMetadataHandlerPut_BucketInstance::put_check()
2565 RGWBucketCompleteInfo
& bci
= obj
->get_bci();
2567 RGWBucketInstanceMetadataObject
*orig_obj
= static_cast<RGWBucketInstanceMetadataObject
*>(old_obj
);
2569 RGWBucketCompleteInfo
*old_bci
= (orig_obj
? &orig_obj
->get_bci() : nullptr);
2571 const bool exists
= (!!orig_obj
);
2573 if (from_remote_zone
) {
2574 // don't sync bucket layout changes
2576 auto& bci_index
= bci
.info
.layout
.current_index
.layout
;
2577 auto index_type
= bci_index
.type
;
2578 auto num_shards
= bci_index
.normal
.num_shards
;
2579 init_default_bucket_layout(cct
, bci
.info
.layout
,
2580 bihandler
->svc
.zone
->get_zone(),
2581 num_shards
, index_type
);
2583 bci
.info
.layout
= old_bci
->info
.layout
;
2587 if (!exists
|| old_bci
->info
.bucket
.bucket_id
!= bci
.info
.bucket
.bucket_id
) {
2588 /* a new bucket, we need to select a new bucket placement for it */
2591 string bucket_instance
;
2592 parse_bucket(entry
, &tenant_name
, &bucket_name
, &bucket_instance
);
2594 RGWZonePlacementInfo rule_info
;
2595 bci
.info
.bucket
.name
= bucket_name
;
2596 bci
.info
.bucket
.bucket_id
= bucket_instance
;
2597 bci
.info
.bucket
.tenant
= tenant_name
;
2598 // if the sync module never writes data, don't require the zone to specify all placement targets
2599 if (bihandler
->svc
.zone
->sync_module_supports_writes()) {
2600 ret
= bihandler
->svc
.zone
->select_bucket_location_by_rule(bci
.info
.placement_rule
, &rule_info
, y
);
2602 ldout(cct
, 0) << "ERROR: select_bucket_placement() returned " << ret
<< dendl
;
2606 bci
.info
.layout
.current_index
.layout
.type
= rule_info
.index_type
;
2608 /* existing bucket, keep its placement */
2609 bci
.info
.bucket
.explicit_placement
= old_bci
->info
.bucket
.explicit_placement
;
2610 bci
.info
.placement_rule
= old_bci
->info
.placement_rule
;
2613 /* record the read version (if any), store the new version */
2614 bci
.info
.objv_tracker
.read_version
= objv_tracker
.read_version
;
2615 bci
.info
.objv_tracker
.write_version
= objv_tracker
.write_version
;
2620 int RGWMetadataHandlerPut_BucketInstance::put_checked()
2622 RGWBucketInstanceMetadataObject
*orig_obj
= static_cast<RGWBucketInstanceMetadataObject
*>(old_obj
);
2624 RGWBucketInfo
*orig_info
= (orig_obj
? &orig_obj
->get_bucket_info() : nullptr);
2626 auto& info
= obj
->get_bucket_info();
2627 auto mtime
= obj
->get_mtime();
2628 auto pattrs
= obj
->get_pattrs();
2630 RGWSI_Bucket_BI_Ctx
ctx(op
->ctx());
2632 return bihandler
->svc
.bucket
->store_bucket_instance_info(ctx
,
2642 int RGWMetadataHandlerPut_BucketInstance::put_post()
2644 RGWBucketCompleteInfo
& bci
= obj
->get_bci();
2646 objv_tracker
= bci
.info
.objv_tracker
;
2648 int ret
= bihandler
->svc
.bi
->init_index(bci
.info
);
2653 return STATUS_APPLIED
;
2656 class RGWArchiveBucketInstanceMetadataHandler
: public RGWBucketInstanceMetadataHandler
{
2658 RGWArchiveBucketInstanceMetadataHandler() {}
2660 int do_remove(RGWSI_MetaBackend_Handler::Op
*op
, string
& entry
, RGWObjVersionTracker
& objv_tracker
, optional_yield y
) override
{
2661 ldout(cct
, 0) << "SKIP: bucket instance removal is not allowed on archive zone: bucket.instance:" << entry
<< dendl
;
2666 RGWBucketCtl::RGWBucketCtl(RGWSI_Zone
*zone_svc
,
2667 RGWSI_Bucket
*bucket_svc
,
2668 RGWSI_Bucket_Sync
*bucket_sync_svc
,
2669 RGWSI_BucketIndex
*bi_svc
) : cct(zone_svc
->ctx())
2671 svc
.zone
= zone_svc
;
2672 svc
.bucket
= bucket_svc
;
2673 svc
.bucket_sync
= bucket_sync_svc
;
2677 void RGWBucketCtl::init(RGWUserCtl
*user_ctl
,
2678 RGWBucketMetadataHandler
*_bm_handler
,
2679 RGWBucketInstanceMetadataHandler
*_bmi_handler
,
2680 RGWDataChangesLog
*datalog
)
2682 ctl
.user
= user_ctl
;
2684 bm_handler
= _bm_handler
;
2685 bmi_handler
= _bmi_handler
;
2687 bucket_be_handler
= bm_handler
->get_be_handler();
2688 bi_be_handler
= bmi_handler
->get_be_handler();
2690 datalog
->set_bucket_filter(
2691 [this](const rgw_bucket
& bucket
, optional_yield y
) {
2692 return bucket_exports_data(bucket
, y
);
2696 int RGWBucketCtl::call(std::function
<int(RGWSI_Bucket_X_Ctx
& ctx
)> f
) {
2697 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ep_ctx
) {
2698 return bmi_handler
->call([&](RGWSI_Bucket_BI_Ctx
& bi_ctx
) {
2699 RGWSI_Bucket_X_Ctx ctx
{ep_ctx
, bi_ctx
};
2705 int RGWBucketCtl::read_bucket_entrypoint_info(const rgw_bucket
& bucket
,
2706 RGWBucketEntryPoint
*info
,
2708 const Bucket::GetParams
& params
)
2710 return bm_handler
->call(params
.bectx_params
, [&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2711 return svc
.bucket
->read_bucket_entrypoint_info(ctx
,
2712 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2714 params
.objv_tracker
,
2719 params
.refresh_version
);
2723 int RGWBucketCtl::store_bucket_entrypoint_info(const rgw_bucket
& bucket
,
2724 RGWBucketEntryPoint
& info
,
2726 const Bucket::PutParams
& params
)
2728 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2729 return svc
.bucket
->store_bucket_entrypoint_info(ctx
,
2730 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2735 params
.objv_tracker
,
2740 int RGWBucketCtl::remove_bucket_entrypoint_info(const rgw_bucket
& bucket
,
2742 const Bucket::RemoveParams
& params
)
2744 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2745 return svc
.bucket
->remove_bucket_entrypoint_info(ctx
,
2746 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2747 params
.objv_tracker
,
2752 int RGWBucketCtl::read_bucket_instance_info(const rgw_bucket
& bucket
,
2753 RGWBucketInfo
*info
,
2755 const BucketInstance::GetParams
& params
)
2757 int ret
= bmi_handler
->call(params
.bectx_params
, [&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2758 return svc
.bucket
->read_bucket_instance_info(ctx
,
2759 RGWSI_Bucket::get_bi_meta_key(bucket
),
2765 params
.refresh_version
);
2772 if (params
.objv_tracker
) {
2773 *params
.objv_tracker
= info
->objv_tracker
;
2779 int RGWBucketCtl::read_bucket_info(const rgw_bucket
& bucket
,
2780 RGWBucketInfo
*info
,
2782 const BucketInstance::GetParams
& params
,
2783 RGWObjVersionTracker
*ep_objv_tracker
)
2785 const rgw_bucket
*b
= &bucket
;
2787 std::optional
<RGWBucketEntryPoint
> ep
;
2789 if (b
->bucket_id
.empty()) {
2792 int r
= read_bucket_entrypoint_info(*b
, &(*ep
), y
, RGWBucketCtl::Bucket::GetParams()
2793 .set_bectx_params(params
.bectx_params
)
2794 .set_objv_tracker(ep_objv_tracker
));
2802 int ret
= bmi_handler
->call(params
.bectx_params
, [&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2803 return svc
.bucket
->read_bucket_instance_info(ctx
,
2804 RGWSI_Bucket::get_bi_meta_key(*b
),
2810 params
.refresh_version
);
2817 if (params
.objv_tracker
) {
2818 *params
.objv_tracker
= info
->objv_tracker
;
2824 int RGWBucketCtl::do_store_bucket_instance_info(RGWSI_Bucket_BI_Ctx
& ctx
,
2825 const rgw_bucket
& bucket
,
2826 RGWBucketInfo
& info
,
2828 const BucketInstance::PutParams
& params
)
2830 if (params
.objv_tracker
) {
2831 info
.objv_tracker
= *params
.objv_tracker
;
2834 return svc
.bucket
->store_bucket_instance_info(ctx
,
2835 RGWSI_Bucket::get_bi_meta_key(bucket
),
2844 int RGWBucketCtl::store_bucket_instance_info(const rgw_bucket
& bucket
,
2845 RGWBucketInfo
& info
,
2847 const BucketInstance::PutParams
& params
)
2849 return bmi_handler
->call([&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2850 return do_store_bucket_instance_info(ctx
, bucket
, info
, y
, params
);
2854 int RGWBucketCtl::remove_bucket_instance_info(const rgw_bucket
& bucket
,
2855 RGWBucketInfo
& info
,
2857 const BucketInstance::RemoveParams
& params
)
2859 if (params
.objv_tracker
) {
2860 info
.objv_tracker
= *params
.objv_tracker
;
2863 return bmi_handler
->call([&](RGWSI_Bucket_BI_Ctx
& ctx
) {
2864 return svc
.bucket
->remove_bucket_instance_info(ctx
,
2865 RGWSI_Bucket::get_bi_meta_key(bucket
),
2872 int RGWBucketCtl::do_store_linked_bucket_info(RGWSI_Bucket_X_Ctx
& ctx
,
2873 RGWBucketInfo
& info
,
2874 RGWBucketInfo
*orig_info
,
2875 bool exclusive
, real_time mtime
,
2876 obj_version
*pep_objv
,
2877 map
<string
, bufferlist
> *pattrs
,
2878 bool create_entry_point
,
2881 bool create_head
= !info
.has_instance_obj
|| create_entry_point
;
2883 int ret
= svc
.bucket
->store_bucket_instance_info(ctx
.bi
,
2884 RGWSI_Bucket::get_bi_meta_key(info
.bucket
),
2895 return 0; /* done! */
2897 RGWBucketEntryPoint entry_point
;
2898 entry_point
.bucket
= info
.bucket
;
2899 entry_point
.owner
= info
.owner
;
2900 entry_point
.creation_time
= info
.creation_time
;
2901 entry_point
.linked
= true;
2902 RGWObjVersionTracker ot
;
2903 if (pep_objv
&& !pep_objv
->tag
.empty()) {
2904 ot
.write_version
= *pep_objv
;
2906 ot
.generate_new_write_ver(cct
);
2908 *pep_objv
= ot
.write_version
;
2911 ret
= svc
.bucket
->store_bucket_entrypoint_info(ctx
.ep
,
2912 RGWSI_Bucket::get_entrypoint_meta_key(info
.bucket
),
2924 int RGWBucketCtl::convert_old_bucket_info(RGWSI_Bucket_X_Ctx
& ctx
,
2925 const rgw_bucket
& bucket
,
2928 RGWBucketEntryPoint entry_point
;
2930 RGWObjVersionTracker ot
;
2931 map
<string
, bufferlist
> attrs
;
2933 auto cct
= svc
.bucket
->ctx();
2935 ldout(cct
, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket
<< dendl
;
2937 int ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
.ep
,
2938 RGWSI_Bucket::get_entrypoint_meta_key(bucket
),
2939 &entry_point
, &ot
, &ep_mtime
, &attrs
, y
);
2941 ldout(cct
, 0) << "ERROR: get_bucket_entrypoint_info() returned " << ret
<< " bucket=" << bucket
<< dendl
;
2945 if (!entry_point
.has_bucket_info
) {
2946 /* already converted! */
2950 info
= entry_point
.old_bucket_info
;
2952 ot
.generate_new_write_ver(cct
);
2954 ret
= do_store_linked_bucket_info(ctx
, info
, nullptr, false, ep_mtime
, &ot
.write_version
, &attrs
, true, y
);
2956 ldout(cct
, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret
<< dendl
;
2963 int RGWBucketCtl::set_bucket_instance_attrs(RGWBucketInfo
& bucket_info
,
2964 map
<string
, bufferlist
>& attrs
,
2965 RGWObjVersionTracker
*objv_tracker
,
2968 return call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
2969 rgw_bucket
& bucket
= bucket_info
.bucket
;
2971 if (!bucket_info
.has_instance_obj
) {
2972 /* an old bucket object, need to convert it */
2973 int ret
= convert_old_bucket_info(ctx
, bucket
, y
);
2975 ldout(cct
, 0) << "ERROR: failed converting old bucket info: " << ret
<< dendl
;
2980 return do_store_bucket_instance_info(ctx
.bi
,
2984 BucketInstance::PutParams().set_attrs(&attrs
)
2985 .set_objv_tracker(objv_tracker
)
2986 .set_orig_info(&bucket_info
));
2991 int RGWBucketCtl::link_bucket(const rgw_user
& user_id
,
2992 const rgw_bucket
& bucket
,
2993 ceph::real_time creation_time
,
2995 bool update_entrypoint
,
2998 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
2999 return do_link_bucket(ctx
, user_id
, bucket
, creation_time
,
3000 update_entrypoint
, pinfo
, y
);
3004 int RGWBucketCtl::do_link_bucket(RGWSI_Bucket_EP_Ctx
& ctx
,
3005 const rgw_user
& user_id
,
3006 const rgw_bucket
& bucket
,
3007 ceph::real_time creation_time
,
3008 bool update_entrypoint
,
3014 RGWBucketEntryPoint ep
;
3015 RGWObjVersionTracker ot
;
3016 RGWObjVersionTracker
& rot
= (pinfo
) ? pinfo
->ep_objv
: ot
;
3017 map
<string
, bufferlist
> attrs
, *pattrs
= nullptr;
3020 if (update_entrypoint
) {
3021 meta_key
= RGWSI_Bucket::get_entrypoint_meta_key(bucket
);
3024 pattrs
= &pinfo
->attrs
;
3026 ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
,
3031 if (ret
< 0 && ret
!= -ENOENT
) {
3032 ldout(cct
, 0) << "ERROR: store->get_bucket_entrypoint_info() returned: "
3033 << cpp_strerror(-ret
) << dendl
;
3039 ret
= ctl
.user
->add_bucket(user_id
, bucket
, creation_time
, y
);
3041 ldout(cct
, 0) << "ERROR: error adding bucket to user directory:"
3042 << " user=" << user_id
3043 << " bucket=" << bucket
3044 << " err=" << cpp_strerror(-ret
)
3049 if (!update_entrypoint
)
3055 ret
= svc
.bucket
->store_bucket_entrypoint_info(
3056 ctx
, meta_key
, ep
, false, real_time(), pattrs
, &rot
, y
);
3063 int r
= do_unlink_bucket(ctx
, user_id
, bucket
, true, y
);
3065 ldout(cct
, 0) << "ERROR: failed unlinking bucket on error cleanup: "
3066 << cpp_strerror(-r
) << dendl
;
3071 int RGWBucketCtl::unlink_bucket(const rgw_user
& user_id
, const rgw_bucket
& bucket
, optional_yield y
, bool update_entrypoint
)
3073 return bm_handler
->call([&](RGWSI_Bucket_EP_Ctx
& ctx
) {
3074 return do_unlink_bucket(ctx
, user_id
, bucket
, update_entrypoint
, y
);
3078 int RGWBucketCtl::do_unlink_bucket(RGWSI_Bucket_EP_Ctx
& ctx
,
3079 const rgw_user
& user_id
,
3080 const rgw_bucket
& bucket
,
3081 bool update_entrypoint
,
3084 int ret
= ctl
.user
->remove_bucket(user_id
, bucket
, y
);
3086 ldout(cct
, 0) << "ERROR: error removing bucket from directory: "
3087 << cpp_strerror(-ret
)<< dendl
;
3090 if (!update_entrypoint
)
3093 RGWBucketEntryPoint ep
;
3094 RGWObjVersionTracker ot
;
3095 map
<string
, bufferlist
> attrs
;
3096 string meta_key
= RGWSI_Bucket::get_entrypoint_meta_key(bucket
);
3097 ret
= svc
.bucket
->read_bucket_entrypoint_info(ctx
, meta_key
, &ep
, &ot
, nullptr, &attrs
, y
);
3106 if (ep
.owner
!= user_id
) {
3107 ldout(cct
, 0) << "bucket entry point user mismatch, can't unlink bucket: " << ep
.owner
<< " != " << user_id
<< dendl
;
3112 return svc
.bucket
->store_bucket_entrypoint_info(ctx
, meta_key
, ep
, false, real_time(), &attrs
, &ot
, y
);
3115 int RGWBucketCtl::set_acl(ACLOwner
& owner
, rgw_bucket
& bucket
,
3116 RGWBucketInfo
& bucket_info
, bufferlist
& bl
,
3119 // set owner and acl
3120 bucket_info
.owner
= owner
.get_id();
3121 std::map
<std::string
, bufferlist
> attrs
{{RGW_ATTR_ACL
, bl
}};
3123 int r
= store_bucket_instance_info(bucket
, bucket_info
, y
,
3124 BucketInstance::PutParams().set_attrs(&attrs
));
3126 cerr
<< "ERROR: failed to set bucket owner: " << cpp_strerror(-r
) << std::endl
;
3133 // TODO: remove RGWRados dependency for bucket listing
3134 int RGWBucketCtl::chown(rgw::sal::RGWRadosStore
*store
, RGWBucketInfo
& bucket_info
,
3135 const rgw_user
& user_id
, const std::string
& display_name
,
3136 const std::string
& marker
, optional_yield y
)
3138 RGWObjectCtx
obj_ctx(store
);
3139 std::vector
<rgw_bucket_dir_entry
> objs
;
3140 map
<string
, bool> common_prefixes
;
3142 RGWRados::Bucket
target(store
->getRados(), bucket_info
);
3143 RGWRados::Bucket::List
list_op(&target
);
3145 list_op
.params
.list_versions
= true;
3146 list_op
.params
.allow_unordered
= true;
3147 list_op
.params
.marker
= marker
;
3149 bool is_truncated
= false;
3151 int max_entries
= 1000;
3153 //Loop through objects and update object acls to point to bucket owner
3157 int ret
= list_op
.list_objects(max_entries
, &objs
, &common_prefixes
, &is_truncated
, y
);
3159 ldout(store
->ctx(), 0) << "ERROR: list objects failed: " << cpp_strerror(-ret
) << dendl
;
3163 list_op
.params
.marker
= list_op
.get_next_marker();
3164 count
+= objs
.size();
3166 for (const auto& obj
: objs
) {
3168 rgw_obj
r_obj(bucket_info
.bucket
, obj
.key
);
3169 RGWRados::Object
op_target(store
->getRados(), bucket_info
, obj_ctx
, r_obj
);
3170 RGWRados::Object::Read
read_op(&op_target
);
3172 map
<string
, bufferlist
> attrs
;
3173 read_op
.params
.attrs
= &attrs
;
3174 ret
= read_op
.prepare(y
);
3176 ldout(store
->ctx(), 0) << "ERROR: failed to read object " << obj
.key
.name
<< cpp_strerror(-ret
) << dendl
;
3179 const auto& aiter
= attrs
.find(RGW_ATTR_ACL
);
3180 if (aiter
== attrs
.end()) {
3181 ldout(store
->ctx(), 0) << "ERROR: no acls found for object " << obj
.key
.name
<< " .Continuing with next object." << dendl
;
3184 bufferlist
& bl
= aiter
->second
;
3185 RGWAccessControlPolicy
policy(store
->ctx());
3189 owner
= policy
.get_owner();
3190 } catch (buffer::error
& err
) {
3191 ldout(store
->ctx(), 0) << "ERROR: decode policy failed" << err
.what()
3196 //Get the ACL from the policy
3197 RGWAccessControlList
& acl
= policy
.get_acl();
3199 //Remove grant that is set to old owner
3200 acl
.remove_canon_user_grant(owner
.get_id());
3202 //Create a grant and add grant
3204 grant
.set_canon(user_id
, display_name
, RGW_PERM_FULL_CONTROL
);
3205 acl
.add_grant(&grant
);
3207 //Update the ACL owner to the new user
3208 owner
.set_id(user_id
);
3209 owner
.set_name(display_name
);
3210 policy
.set_owner(owner
);
3215 obj_ctx
.set_atomic(r_obj
);
3216 ret
= store
->getRados()->set_attr(&obj_ctx
, bucket_info
, r_obj
, RGW_ATTR_ACL
, bl
);
3218 ldout(store
->ctx(), 0) << "ERROR: modify attr failed " << cpp_strerror(-ret
) << dendl
;
3223 cerr
<< count
<< " objects processed in " << bucket_info
.bucket
.name
3224 << ". Next marker " << list_op
.params
.marker
.name
<< std::endl
;
3225 } while(is_truncated
);
3229 int RGWBucketCtl::read_bucket_stats(const rgw_bucket
& bucket
,
3230 RGWBucketEnt
*result
,
3233 return call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
3234 return svc
.bucket
->read_bucket_stats(ctx
, bucket
, result
, y
);
3238 int RGWBucketCtl::read_buckets_stats(map
<string
, RGWBucketEnt
>& m
,
3241 return call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
3242 return svc
.bucket
->read_buckets_stats(ctx
, m
, y
);
3246 int RGWBucketCtl::sync_user_stats(const rgw_user
& user_id
,
3247 const RGWBucketInfo
& bucket_info
,
3255 int r
= svc
.bi
->read_stats(bucket_info
, pent
, null_yield
);
3257 ldout(cct
, 20) << __func__
<< "(): failed to read bucket stats (r=" << r
<< ")" << dendl
;
3261 return ctl
.user
->flush_bucket_stats(user_id
, *pent
, y
);
3264 int RGWBucketCtl::get_sync_policy_handler(std::optional
<rgw_zone_id
> zone
,
3265 std::optional
<rgw_bucket
> bucket
,
3266 RGWBucketSyncPolicyHandlerRef
*phandler
,
3269 int r
= call([&](RGWSI_Bucket_X_Ctx
& ctx
) {
3270 return svc
.bucket_sync
->get_policy_handler(ctx
, zone
, bucket
, phandler
, y
);
3273 ldout(cct
, 20) << __func__
<< "(): failed to get policy handler for bucket=" << bucket
<< " (r=" << r
<< ")" << dendl
;
3279 int RGWBucketCtl::bucket_exports_data(const rgw_bucket
& bucket
,
3283 RGWBucketSyncPolicyHandlerRef handler
;
3285 int r
= get_sync_policy_handler(std::nullopt
, bucket
, &handler
, y
);
3290 return handler
->bucket_exports_data();
3293 int RGWBucketCtl::bucket_imports_data(const rgw_bucket
& bucket
,
3297 RGWBucketSyncPolicyHandlerRef handler
;
3299 int r
= get_sync_policy_handler(std::nullopt
, bucket
, &handler
, y
);
3304 return handler
->bucket_imports_data();
3307 RGWBucketMetadataHandlerBase
*RGWBucketMetaHandlerAllocator::alloc()
3309 return new RGWBucketMetadataHandler();
3312 RGWBucketInstanceMetadataHandlerBase
*RGWBucketInstanceMetaHandlerAllocator::alloc()
3314 return new RGWBucketInstanceMetadataHandler();
3317 RGWBucketMetadataHandlerBase
*RGWArchiveBucketMetaHandlerAllocator::alloc()
3319 return new RGWArchiveBucketMetadataHandler();
3322 RGWBucketInstanceMetadataHandlerBase
*RGWArchiveBucketInstanceMetaHandlerAllocator::alloc()
3324 return new RGWArchiveBucketInstanceMetadataHandler();