1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
5 * Ceph - scalable distributed file system
7 * Copyright (C) 2020 Red Hat, Inc.
9 * This is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License version 2.1, as published by the Free Software
12 * Foundation. See file COPYING.
18 #include <system_error>
22 #include <boost/algorithm/string.hpp>
23 #include <boost/process.hpp>
25 #include "common/Clock.h"
26 #include "common/errno.h"
29 #include "rgw_sal_rados.h"
30 #include "rgw_bucket.h"
31 #include "rgw_multi.h"
33 #include "rgw_acl_s3.h"
35 #include "rgw_aio_throttle.h"
36 #include "rgw_tracer.h"
39 #include "rgw_rest_conn.h"
40 #include "rgw_service.h"
42 #include "rgw_lc_tier.h"
43 #include "rgw_rest_admin.h"
44 #include "rgw_rest_bucket.h"
45 #include "rgw_rest_metadata.h"
46 #include "rgw_rest_log.h"
47 #include "rgw_rest_config.h"
48 #include "rgw_rest_ratelimit.h"
49 #include "rgw_rest_realm.h"
50 #include "rgw_rest_user.h"
51 #include "services/svc_sys_obj.h"
52 #include "services/svc_meta.h"
53 #include "services/svc_meta_be_sobj.h"
54 #include "services/svc_cls.h"
55 #include "services/svc_zone.h"
56 #include "services/svc_tier_rados.h"
57 #include "services/svc_quota.h"
58 #include "services/svc_config_key.h"
59 #include "services/svc_zone_utils.h"
60 #include "services/svc_role_rados.h"
61 #include "services/svc_user.h"
62 #include "cls/rgw/cls_rgw_client.h"
64 #include "rgw_pubsub.h"
66 #define dout_subsys ceph_subsys_rgw
70 static string mp_ns
= RGW_OBJ_NS_MULTIPART
;
74 // default number of entries to list with each bucket listing call
75 // (use marker to bridge between calls)
76 static constexpr size_t listing_max_entries
= 1000;
77 static std::string pubsub_oid_prefix
= "pubsub.";
79 static int decode_policy(CephContext
* cct
,
81 RGWAccessControlPolicy
* policy
)
83 auto iter
= bl
.cbegin();
86 } catch (buffer::error
& err
) {
87 ldout(cct
, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl
;
90 if (cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
91 ldout(cct
, 15) << __func__
<< " Read AccessControlPolicy";
92 RGWAccessControlPolicy_S3
* s3policy
= static_cast<RGWAccessControlPolicy_S3
*>(policy
);
93 s3policy
->to_xml(*_dout
);
99 static int rgw_op_get_bucket_policy_from_attr(const DoutPrefixProvider
* dpp
,
103 RGWAccessControlPolicy
* policy
,
106 auto aiter
= bucket_attrs
.find(RGW_ATTR_ACL
);
108 if (aiter
!= bucket_attrs
.end()) {
109 int ret
= decode_policy(store
->ctx(), aiter
->second
, policy
);
113 ldout(store
->ctx(), 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl
;
114 /* object exists, but policy is broken */
115 int r
= user
->load_user(dpp
, y
);
119 policy
->create_default(user
->get_id(), user
->get_display_name());
124 static int drain_aio(std::list
<librados::AioCompletion
*>& handles
)
127 while (!handles
.empty()) {
128 librados::AioCompletion
* handle
= handles
.front();
130 handle
->wait_for_complete();
131 int r
= handle
->get_return_value();
140 int RadosCompletions::drain()
142 return drain_aio(handles
);
145 int RadosUser::list_buckets(const DoutPrefixProvider
* dpp
, const std::string
& marker
,
146 const std::string
& end_marker
, uint64_t max
, bool need_stats
,
147 BucketList
&buckets
, optional_yield y
)
149 RGWUserBuckets ulist
;
150 bool is_truncated
= false;
154 ret
= store
->ctl()->user
->list_buckets(dpp
, info
.user_id
, marker
, end_marker
, max
,
155 need_stats
, &ulist
, &is_truncated
, y
);
159 buckets
.set_truncated(is_truncated
);
160 for (const auto& ent
: ulist
.get_buckets()) {
161 buckets
.add(std::unique_ptr
<Bucket
>(new RadosBucket(this->store
, ent
.second
, this)));
167 int RadosUser::create_bucket(const DoutPrefixProvider
* dpp
,
169 const std::string
& zonegroup_id
,
170 rgw_placement_rule
& placement_rule
,
171 std::string
& swift_ver_location
,
172 const RGWQuotaInfo
* pquota_info
,
173 const RGWAccessControlPolicy
& policy
,
176 obj_version
& ep_objv
,
178 bool obj_lock_enabled
,
181 std::unique_ptr
<Bucket
>* bucket_out
,
186 RGWBucketInfo master_info
;
187 rgw_bucket
* pmaster_bucket
;
188 uint32_t* pmaster_num_shards
;
189 real_time creation_time
;
190 std::unique_ptr
<Bucket
> bucket
;
191 obj_version objv
,* pobjv
= NULL
;
193 /* If it exists, look it up; otherwise create it */
194 ret
= store
->get_bucket(dpp
, this, b
, &bucket
, y
);
195 if (ret
< 0 && ret
!= -ENOENT
)
198 if (ret
!= -ENOENT
) {
199 RGWAccessControlPolicy
old_policy(store
->ctx());
201 if (swift_ver_location
.empty()) {
202 swift_ver_location
= bucket
->get_info().swift_ver_location
;
204 placement_rule
.inherit_from(bucket
->get_info().placement_rule
);
206 // don't allow changes to the acl policy
207 int r
= rgw_op_get_bucket_policy_from_attr(dpp
, store
, this, bucket
->get_attrs(),
209 if (r
>= 0 && old_policy
!= policy
) {
210 bucket_out
->swap(bucket
);
214 bucket
= std::unique_ptr
<Bucket
>(new RadosBucket(store
, b
, this));
216 bucket
->set_attrs(attrs
);
219 if (!store
->svc()->zone
->is_meta_master()) {
221 ret
= store
->forward_request_to_master(dpp
, this, NULL
, in_data
, &jp
, req_info
, y
);
226 JSONDecoder::decode_json("entry_point_object_ver", ep_objv
, &jp
);
227 JSONDecoder::decode_json("object_ver", objv
, &jp
);
228 JSONDecoder::decode_json("bucket_info", master_info
, &jp
);
229 ldpp_dout(dpp
, 20) << "parsed: objv.tag=" << objv
.tag
<< " objv.ver=" << objv
.ver
<< dendl
;
230 std::time_t ctime
= ceph::real_clock::to_time_t(master_info
.creation_time
);
231 ldpp_dout(dpp
, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime
), "%F %T") << dendl
;
232 pmaster_bucket
= &master_info
.bucket
;
233 creation_time
= master_info
.creation_time
;
234 pmaster_num_shards
= &master_info
.layout
.current_index
.layout
.normal
.num_shards
;
236 if (master_info
.obj_lock_enabled()) {
237 info
.flags
= BUCKET_VERSIONED
| BUCKET_OBJ_LOCK_ENABLED
;
240 pmaster_bucket
= NULL
;
241 pmaster_num_shards
= NULL
;
242 if (obj_lock_enabled
)
243 info
.flags
= BUCKET_VERSIONED
| BUCKET_OBJ_LOCK_ENABLED
;
246 std::string zid
= zonegroup_id
;
248 zid
= store
->svc()->zone
->get_zonegroup().get_id();
252 rgw_placement_rule selected_placement_rule
;
253 ret
= store
->svc()->zone
->select_bucket_placement(dpp
, this->get_info(),
255 &selected_placement_rule
, nullptr, y
);
256 if (selected_placement_rule
!= info
.placement_rule
) {
258 bucket_out
->swap(bucket
);
263 ret
= store
->getRados()->create_bucket(this->get_info(), bucket
->get_key(),
264 zid
, placement_rule
, swift_ver_location
, pquota_info
,
265 attrs
, info
, pobjv
, &ep_objv
, creation_time
,
266 pmaster_bucket
, pmaster_num_shards
, y
, dpp
,
268 if (ret
== -EEXIST
) {
270 /* bucket already existed, might have raced with another bucket creation,
271 * or might be partial bucket creation that never completed. Read existing
272 * bucket info, verify that the reported bucket owner is the current user.
273 * If all is ok then update the user's list of buckets. Otherwise inform
274 * client about a name conflict.
276 if (info
.owner
.compare(this->get_id()) != 0) {
280 } else if (ret
!= 0) {
285 bucket
->set_version(ep_objv
);
286 bucket
->get_info() = info
;
288 RadosBucket
* rbucket
= static_cast<RadosBucket
*>(bucket
.get());
289 ret
= rbucket
->link(dpp
, this, y
, false);
290 if (ret
&& !*existed
&& ret
!= -EEXIST
) {
291 /* if it exists (or previously existed), don't remove it! */
292 ret
= rbucket
->unlink(dpp
, this, y
);
294 ldpp_dout(dpp
, 0) << "WARNING: failed to unlink bucket: ret=" << ret
297 } else if (ret
== -EEXIST
|| (ret
== 0 && *existed
)) {
298 ret
= -ERR_BUCKET_EXISTS
;
301 bucket_out
->swap(bucket
);
306 int RadosUser::read_attrs(const DoutPrefixProvider
* dpp
, optional_yield y
)
308 return store
->ctl()->user
->get_attrs_by_uid(dpp
, get_id(), &attrs
, y
, &objv_tracker
);
311 int RadosUser::merge_and_store_attrs(const DoutPrefixProvider
* dpp
, Attrs
& new_attrs
, optional_yield y
)
313 for(auto& it
: new_attrs
) {
314 attrs
[it
.first
] = it
.second
;
316 return store_user(dpp
, y
, false);
319 int RadosUser::read_stats(const DoutPrefixProvider
*dpp
,
320 optional_yield y
, RGWStorageStats
* stats
,
321 ceph::real_time
* last_stats_sync
,
322 ceph::real_time
* last_stats_update
)
324 return store
->ctl()->user
->read_stats(dpp
, get_id(), stats
, y
, last_stats_sync
, last_stats_update
);
327 int RadosUser::read_stats_async(const DoutPrefixProvider
*dpp
, RGWGetUserStats_CB
* cb
)
329 return store
->svc()->user
->read_stats_async(dpp
, get_id(), cb
);
332 int RadosUser::complete_flush_stats(const DoutPrefixProvider
*dpp
, optional_yield y
)
334 return store
->svc()->user
->complete_flush_stats(dpp
, get_id(), y
);
337 int RadosUser::read_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
,
338 uint32_t max_entries
, bool* is_truncated
,
339 RGWUsageIter
& usage_iter
,
340 map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
)
342 std::string bucket_name
;
343 return store
->getRados()->read_usage(dpp
, get_id(), bucket_name
, start_epoch
,
344 end_epoch
, max_entries
, is_truncated
,
348 int RadosUser::trim_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
)
350 std::string bucket_name
;
352 return store
->getRados()->trim_usage(dpp
, get_id(), bucket_name
, start_epoch
, end_epoch
);
355 int RadosUser::load_user(const DoutPrefixProvider
* dpp
, optional_yield y
)
357 return store
->ctl()->user
->get_info_by_uid(dpp
, info
.user_id
, &info
, y
, RGWUserCtl::GetParams().set_objv_tracker(&objv_tracker
).set_attrs(&attrs
));
360 int RadosUser::store_user(const DoutPrefixProvider
* dpp
, optional_yield y
, bool exclusive
, RGWUserInfo
* old_info
)
362 return store
->ctl()->user
->store_info(dpp
, info
, y
,
363 RGWUserCtl::PutParams().set_objv_tracker(&objv_tracker
)
364 .set_exclusive(exclusive
)
366 .set_old_info(old_info
));
369 int RadosUser::remove_user(const DoutPrefixProvider
* dpp
, optional_yield y
)
371 return store
->ctl()->user
->remove_info(dpp
, info
, y
,
372 RGWUserCtl::RemoveParams().set_objv_tracker(&objv_tracker
));
375 int RadosUser::verify_mfa(const std::string
& mfa_str
, bool* verified
,
376 const DoutPrefixProvider
* dpp
, optional_yield y
)
378 vector
<string
> params
;
379 get_str_vec(mfa_str
, " ", params
);
381 if (params
.size() != 2) {
382 ldpp_dout(dpp
, 5) << "NOTICE: invalid mfa string provided: " << mfa_str
<< dendl
;
386 string
& serial
= params
[0];
387 string
& pin
= params
[1];
389 auto i
= info
.mfa_ids
.find(serial
);
390 if (i
== info
.mfa_ids
.end()) {
391 ldpp_dout(dpp
, 5) << "NOTICE: user does not have mfa device with serial=" << serial
<< dendl
;
395 int ret
= store
->svc()->cls
->mfa
.check_mfa(dpp
, info
.user_id
, serial
, pin
, y
);
397 ldpp_dout(dpp
, 20) << "NOTICE: failed to check MFA, serial=" << serial
<< dendl
;
406 RadosBucket::~RadosBucket() {}
408 int RadosBucket::remove_bucket(const DoutPrefixProvider
* dpp
,
409 bool delete_children
,
410 bool forward_to_master
,
417 ret
= load_bucket(dpp
, y
);
423 params
.list_versions
= true;
424 params
.allow_unordered
= true;
429 results
.objs
.clear();
431 ret
= list(dpp
, params
, 1000, results
, y
);
436 if (!results
.objs
.empty() && !delete_children
) {
437 ldpp_dout(dpp
, -1) << "ERROR: could not remove non-empty bucket " << info
.bucket
.name
<<
442 for (const auto& obj
: results
.objs
) {
443 rgw_obj_key
key(obj
.key
);
445 ret
= rgw_remove_object(dpp
, store
, this, key
);
446 if (ret
< 0 && ret
!= -ENOENT
) {
450 } while(results
.is_truncated
);
452 ret
= abort_multiparts(dpp
, store
->ctx());
457 // remove lifecycle config, if any (XXX note could be made generic)
458 (void) store
->getRados()->get_lc()->remove_bucket_config(
461 ret
= store
->ctl()->bucket
->sync_user_stats(dpp
, info
.owner
, info
, y
, nullptr);
463 ldout(store
->ctx(), 1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret
<< dendl
;
466 RGWObjVersionTracker ot
;
468 // if we deleted children above we will force delete, as any that
469 // remain is detrius from a prior bug
470 ret
= store
->getRados()->delete_bucket(info
, ot
, y
, dpp
, !delete_children
);
472 ldpp_dout(dpp
, -1) << "ERROR: could not remove bucket " <<
473 info
.bucket
.name
<< dendl
;
477 // if bucket has notification definitions associated with it
478 // they should be removed (note that any pending notifications on the bucket are still going to be sent)
479 const RGWPubSub
ps(store
, info
.owner
.tenant
);
480 const RGWPubSub::Bucket
ps_bucket(ps
, this);
481 const auto ps_ret
= ps_bucket
.remove_notifications(dpp
, y
);
482 if (ps_ret
< 0 && ps_ret
!= -ENOENT
) {
483 ldpp_dout(dpp
, -1) << "ERROR: unable to remove notifications from bucket. ret=" << ps_ret
<< dendl
;
486 ret
= store
->ctl()->bucket
->unlink_bucket(info
.owner
, info
.bucket
, y
, dpp
, false);
488 ldpp_dout(dpp
, -1) << "ERROR: unable to remove user bucket information" << dendl
;
491 if (forward_to_master
) {
493 ret
= store
->forward_request_to_master(dpp
, owner
, &ot
.read_version
, in_data
, nullptr, *req_info
, y
);
495 if (ret
== -ENOENT
) {
496 /* adjust error, we want to return with NoSuchBucket and not
498 ret
= -ERR_NO_SUCH_BUCKET
;
507 int RadosBucket::remove_bucket_bypass_gc(int concurrent_max
, bool
508 keep_index_consistent
,
509 optional_yield y
, const
510 DoutPrefixProvider
*dpp
)
513 map
<RGWObjCategory
, RGWStorageStats
> stats
;
514 map
<string
, bool> common_prefixes
;
515 RGWObjectCtx
obj_ctx(store
);
516 CephContext
*cct
= store
->ctx();
518 string bucket_ver
, master_ver
;
520 ret
= load_bucket(dpp
, y
);
524 const auto& index
= info
.get_current_index();
525 ret
= read_stats(dpp
, index
, RGW_NO_SHARD
, &bucket_ver
, &master_ver
, stats
, NULL
);
529 ret
= abort_multiparts(dpp
, cct
);
534 rgw::sal::Bucket::ListParams params
;
535 rgw::sal::Bucket::ListResults results
;
537 params
.list_versions
= true;
538 params
.allow_unordered
= true;
540 std::list
<librados::AioCompletion
*> handles
;
542 int max_aio
= concurrent_max
;
543 results
.is_truncated
= true;
545 while (results
.is_truncated
) {
546 ret
= list(dpp
, params
, listing_max_entries
, results
, y
);
550 std::vector
<rgw_bucket_dir_entry
>::iterator it
= results
.objs
.begin();
551 for (; it
!= results
.objs
.end(); ++it
) {
552 RGWObjState
*astate
= NULL
;
553 RGWObjManifest
*amanifest
= nullptr;
554 rgw_obj obj
{get_key(), it
->key
};
556 ret
= store
->getRados()->get_obj_state(dpp
, &obj_ctx
, get_info(),
557 obj
, &astate
, &amanifest
,
559 if (ret
== -ENOENT
) {
560 ldpp_dout(dpp
, 1) << "WARNING: cannot find obj state for obj " << obj
<< dendl
;
564 ldpp_dout(dpp
, -1) << "ERROR: get obj state returned with error " << ret
<< dendl
;
569 RGWObjManifest
& manifest
= *amanifest
;
570 RGWObjManifest::obj_iterator miter
= manifest
.obj_begin(dpp
);
571 const rgw_obj head_obj
= manifest
.get_obj();
572 rgw_raw_obj raw_head_obj
;
573 store
->get_raw_obj(manifest
.get_head_placement_rule(), head_obj
, &raw_head_obj
);
575 for (; miter
!= manifest
.obj_end(dpp
) && max_aio
--; ++miter
) {
577 ret
= drain_aio(handles
);
579 ldpp_dout(dpp
, -1) << "ERROR: could not drain handles as aio completion returned with " << ret
<< dendl
;
582 max_aio
= concurrent_max
;
585 rgw_raw_obj last_obj
= miter
.get_location().get_raw_obj(store
->getRados());
586 if (last_obj
== raw_head_obj
) {
587 // have the head obj deleted at the end
591 ret
= store
->getRados()->delete_raw_obj_aio(dpp
, last_obj
, handles
);
593 ldpp_dout(dpp
, -1) << "ERROR: delete obj aio failed with " << ret
<< dendl
;
596 } // for all shadow objs
598 ret
= store
->getRados()->delete_obj_aio(dpp
, head_obj
, get_info(), astate
,
599 handles
, keep_index_consistent
, y
);
601 ldpp_dout(dpp
, -1) << "ERROR: delete obj aio failed with " << ret
<< dendl
;
607 ret
= drain_aio(handles
);
609 ldpp_dout(dpp
, -1) << "ERROR: could not drain handles as aio completion returned with " << ret
<< dendl
;
612 max_aio
= concurrent_max
;
614 obj_ctx
.invalidate(obj
);
615 } // for all RGW objects in results
616 } // while is_truncated
618 ret
= drain_aio(handles
);
620 ldpp_dout(dpp
, -1) << "ERROR: could not drain handles as aio completion returned with " << ret
<< dendl
;
624 sync_user_stats(dpp
, y
);
626 ldpp_dout(dpp
, 1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret
<< dendl
;
629 RGWObjVersionTracker objv_tracker
;
631 // this function can only be run if caller wanted children to be
632 // deleted, so we can ignore the check for children as any that
633 // remain are detritus from a prior bug
634 ret
= remove_bucket(dpp
, true, false, nullptr, y
);
636 ldpp_dout(dpp
, -1) << "ERROR: could not remove bucket " << this << dendl
;
643 int RadosBucket::load_bucket(const DoutPrefixProvider
* dpp
, optional_yield y
, bool get_stats
)
647 RGWSI_MetaBackend_CtxParams bectx_params
= RGWSI_MetaBackend_CtxParams_SObj();
648 RGWObjVersionTracker ep_ot
;
649 if (info
.bucket
.bucket_id
.empty()) {
650 ret
= store
->ctl()->bucket
->read_bucket_info(info
.bucket
, &info
, y
, dpp
,
651 RGWBucketCtl::BucketInstance::GetParams()
654 .set_bectx_params(bectx_params
),
657 ret
= store
->ctl()->bucket
->read_bucket_instance_info(info
.bucket
, &info
, y
, dpp
,
658 RGWBucketCtl::BucketInstance::GetParams()
661 .set_bectx_params(bectx_params
));
667 bucket_version
= ep_ot
.read_version
;
670 ret
= store
->ctl()->bucket
->read_bucket_stats(info
.bucket
, &ent
, y
, dpp
);
676 int RadosBucket::read_stats(const DoutPrefixProvider
*dpp
,
677 const bucket_index_layout_generation
& idx_layout
,
678 int shard_id
, std::string
* bucket_ver
, std::string
* master_ver
,
679 std::map
<RGWObjCategory
, RGWStorageStats
>& stats
,
680 std::string
* max_marker
, bool* syncstopped
)
682 return store
->getRados()->get_bucket_stats(dpp
, info
, idx_layout
, shard_id
, bucket_ver
, master_ver
, stats
, max_marker
, syncstopped
);
685 int RadosBucket::read_stats_async(const DoutPrefixProvider
*dpp
,
686 const bucket_index_layout_generation
& idx_layout
,
687 int shard_id
, RGWGetBucketStats_CB
* ctx
)
689 return store
->getRados()->get_bucket_stats_async(dpp
, get_info(), idx_layout
, shard_id
, ctx
);
692 int RadosBucket::sync_user_stats(const DoutPrefixProvider
*dpp
, optional_yield y
)
694 return store
->ctl()->bucket
->sync_user_stats(dpp
, owner
->get_id(), info
, y
, &ent
);
697 int RadosBucket::update_container_stats(const DoutPrefixProvider
* dpp
)
700 map
<std::string
, RGWBucketEnt
> m
;
702 m
[info
.bucket
.name
] = ent
;
703 ret
= store
->getRados()->update_containers_stats(m
, dpp
);
709 map
<std::string
, RGWBucketEnt
>::iterator iter
= m
.find(info
.bucket
.name
);
713 ent
.count
= iter
->second
.count
;
714 ent
.size
= iter
->second
.size
;
715 ent
.size_rounded
= iter
->second
.size_rounded
;
716 ent
.creation_time
= iter
->second
.creation_time
;
717 ent
.placement_rule
= std::move(iter
->second
.placement_rule
);
719 info
.creation_time
= ent
.creation_time
;
720 info
.placement_rule
= ent
.placement_rule
;
725 int RadosBucket::check_bucket_shards(const DoutPrefixProvider
* dpp
)
727 return store
->getRados()->check_bucket_shards(info
, info
.bucket
, get_count(), dpp
);
730 int RadosBucket::link(const DoutPrefixProvider
* dpp
, User
* new_user
, optional_yield y
, bool update_entrypoint
, RGWObjVersionTracker
* objv
)
732 RGWBucketEntryPoint ep
;
733 ep
.bucket
= info
.bucket
;
734 ep
.owner
= new_user
->get_id();
735 ep
.creation_time
= get_creation_time();
738 rgw_ep_info ep_data
{ep
, ep_attrs
};
740 int r
= store
->ctl()->bucket
->link_bucket(new_user
->get_id(), info
.bucket
,
741 get_creation_time(), y
, dpp
, update_entrypoint
,
747 *objv
= ep_data
.ep_objv
;
752 int RadosBucket::unlink(const DoutPrefixProvider
* dpp
, User
* new_user
, optional_yield y
, bool update_entrypoint
)
754 return store
->ctl()->bucket
->unlink_bucket(new_user
->get_id(), info
.bucket
, y
, dpp
, update_entrypoint
);
757 int RadosBucket::chown(const DoutPrefixProvider
* dpp
, User
& new_user
, optional_yield y
)
759 std::string obj_marker
;
763 ldpp_dout(dpp
, 0) << __func__
<< " Cannot chown without an owner " << dendl
;
767 r
= this->unlink(dpp
, owner
, y
);
772 return this->link(dpp
, &new_user
, y
);
775 int RadosBucket::put_info(const DoutPrefixProvider
* dpp
, bool exclusive
, ceph::real_time _mtime
)
778 return store
->getRados()->put_bucket_instance_info(info
, exclusive
, mtime
, &attrs
, dpp
, null_yield
);
781 /* Make sure to call get_bucket_info() if you need it first */
782 bool RadosBucket::is_owner(User
* user
)
784 return (info
.owner
.compare(user
->get_id()) == 0);
787 int RadosBucket::check_empty(const DoutPrefixProvider
* dpp
, optional_yield y
)
789 return store
->getRados()->check_bucket_empty(dpp
, info
, y
);
792 int RadosBucket::check_quota(const DoutPrefixProvider
*dpp
, RGWQuota
& quota
, uint64_t obj_size
,
793 optional_yield y
, bool check_size_only
)
795 return store
->getRados()->check_quota(dpp
, info
.owner
, get_key(),
796 quota
, obj_size
, y
, check_size_only
);
799 int RadosBucket::merge_and_store_attrs(const DoutPrefixProvider
* dpp
, Attrs
& new_attrs
, optional_yield y
)
801 for(auto& it
: new_attrs
) {
802 attrs
[it
.first
] = it
.second
;
804 return store
->ctl()->bucket
->set_bucket_instance_attrs(get_info(),
805 new_attrs
, &get_info().objv_tracker
, y
, dpp
);
808 int RadosBucket::try_refresh_info(const DoutPrefixProvider
* dpp
, ceph::real_time
* pmtime
)
810 return store
->getRados()->try_refresh_bucket_info(info
, pmtime
, dpp
, &attrs
);
813 int RadosBucket::read_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
,
814 uint32_t max_entries
, bool* is_truncated
,
815 RGWUsageIter
& usage_iter
,
816 map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
)
818 return store
->getRados()->read_usage(dpp
, owner
->get_id(), get_name(), start_epoch
,
819 end_epoch
, max_entries
, is_truncated
,
823 int RadosBucket::trim_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
)
825 return store
->getRados()->trim_usage(dpp
, owner
->get_id(), get_name(), start_epoch
, end_epoch
);
828 int RadosBucket::remove_objs_from_index(const DoutPrefixProvider
*dpp
, std::list
<rgw_obj_index_key
>& objs_to_unlink
)
830 return store
->getRados()->remove_objs_from_index(dpp
, info
, objs_to_unlink
);
833 int RadosBucket::check_index(const DoutPrefixProvider
*dpp
, std::map
<RGWObjCategory
, RGWStorageStats
>& existing_stats
, std::map
<RGWObjCategory
, RGWStorageStats
>& calculated_stats
)
835 return store
->getRados()->bucket_check_index(dpp
, info
, &existing_stats
, &calculated_stats
);
838 int RadosBucket::rebuild_index(const DoutPrefixProvider
*dpp
)
840 return store
->getRados()->bucket_rebuild_index(dpp
, info
);
843 int RadosBucket::set_tag_timeout(const DoutPrefixProvider
*dpp
, uint64_t timeout
)
845 return store
->getRados()->cls_obj_set_bucket_tag_timeout(dpp
, info
, timeout
);
848 int RadosBucket::purge_instance(const DoutPrefixProvider
* dpp
)
850 int max_shards
= (info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? info
.layout
.current_index
.layout
.normal
.num_shards
: 1);
851 for (int i
= 0; i
< max_shards
; i
++) {
852 RGWRados::BucketShard
bs(store
->getRados());
853 int shard_id
= (info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? i
: -1);
854 int ret
= bs
.init(dpp
, info
, info
.layout
.current_index
, shard_id
);
856 cerr
<< "ERROR: bs.init(bucket=" << info
.bucket
<< ", shard=" << shard_id
857 << "): " << cpp_strerror(-ret
) << std::endl
;
860 ret
= store
->getRados()->bi_remove(dpp
, bs
);
862 cerr
<< "ERROR: failed to remove bucket index object: "
863 << cpp_strerror(-ret
) << std::endl
;
870 int RadosBucket::set_acl(const DoutPrefixProvider
* dpp
, RGWAccessControlPolicy
&acl
, optional_yield y
)
876 map
<string
, bufferlist
>& attrs
= get_attrs();
878 attrs
[RGW_ATTR_ACL
] = aclbl
;
879 info
.owner
= acl
.get_owner().get_id();
881 int r
= store
->ctl()->bucket
->store_bucket_instance_info(info
.bucket
,
883 RGWBucketCtl::BucketInstance::PutParams().set_attrs(&attrs
));
885 cerr
<< "ERROR: failed to set bucket owner: " << cpp_strerror(-r
) << std::endl
;
892 std::unique_ptr
<Object
> RadosBucket::get_object(const rgw_obj_key
& k
)
894 return std::make_unique
<RadosObject
>(this->store
, k
, this);
897 int RadosBucket::list(const DoutPrefixProvider
* dpp
, ListParams
& params
, int max
, ListResults
& results
, optional_yield y
)
899 RGWRados::Bucket
target(store
->getRados(), get_info());
900 if (params
.shard_id
>= 0) {
901 target
.set_shard_id(params
.shard_id
);
903 RGWRados::Bucket::List
list_op(&target
);
905 list_op
.params
.prefix
= params
.prefix
;
906 list_op
.params
.delim
= params
.delim
;
907 list_op
.params
.marker
= params
.marker
;
908 list_op
.params
.ns
= params
.ns
;
909 list_op
.params
.end_marker
= params
.end_marker
;
910 list_op
.params
.ns
= params
.ns
;
911 list_op
.params
.enforce_ns
= params
.enforce_ns
;
912 list_op
.params
.access_list_filter
= params
.access_list_filter
;
913 list_op
.params
.force_check_filter
= params
.force_check_filter
;
914 list_op
.params
.list_versions
= params
.list_versions
;
915 list_op
.params
.allow_unordered
= params
.allow_unordered
;
917 int ret
= list_op
.list_objects(dpp
, max
, &results
.objs
, &results
.common_prefixes
, &results
.is_truncated
, y
);
919 results
.next_marker
= list_op
.get_next_marker();
920 params
.marker
= results
.next_marker
;
926 std::unique_ptr
<MultipartUpload
> RadosBucket::get_multipart_upload(
927 const std::string
& oid
,
928 std::optional
<std::string
> upload_id
,
929 ACLOwner owner
, ceph::real_time mtime
)
931 return std::make_unique
<RadosMultipartUpload
>(this->store
, this, oid
, upload_id
,
932 std::move(owner
), mtime
);
935 int RadosBucket::list_multiparts(const DoutPrefixProvider
*dpp
,
936 const string
& prefix
,
939 const int& max_uploads
,
940 vector
<std::unique_ptr
<MultipartUpload
>>& uploads
,
941 map
<string
, bool> *common_prefixes
,
944 rgw::sal::Bucket::ListParams params
;
945 rgw::sal::Bucket::ListResults results
;
946 MultipartMetaFilter mp_filter
;
948 params
.prefix
= prefix
;
949 params
.delim
= delim
;
950 params
.marker
= marker
;
951 params
.ns
= RGW_OBJ_NS_MULTIPART
;
952 params
.access_list_filter
= &mp_filter
;
954 int ret
= list(dpp
, params
, max_uploads
, results
, null_yield
);
959 if (!results
.objs
.empty()) {
960 for (const rgw_bucket_dir_entry
& dentry
: results
.objs
) {
961 rgw_obj_key
key(dentry
.key
);
962 ACLOwner
owner(rgw_user(dentry
.meta
.owner
));
963 owner
.set_name(dentry
.meta
.owner_display_name
);
964 uploads
.push_back(this->get_multipart_upload(key
.name
,
965 std::nullopt
, std::move(owner
), dentry
.meta
.mtime
));
968 if (common_prefixes
) {
969 *common_prefixes
= std::move(results
.common_prefixes
);
971 *is_truncated
= results
.is_truncated
;
972 marker
= params
.marker
.name
;
977 int RadosBucket::abort_multiparts(const DoutPrefixProvider
* dpp
,
980 constexpr int max
= 1000;
981 int ret
, num_deleted
= 0;
982 vector
<std::unique_ptr
<MultipartUpload
>> uploads
;
986 const std::string empty_delim
;
987 const std::string empty_prefix
;
990 ret
= list_multiparts(dpp
, empty_prefix
, marker
, empty_delim
,
991 max
, uploads
, nullptr, &is_truncated
);
993 ldpp_dout(dpp
, 0) << __func__
<<
994 " ERROR : calling list_bucket_multiparts; ret=" << ret
<<
995 "; bucket=\"" << this << "\"" << dendl
;
998 ldpp_dout(dpp
, 20) << __func__
<<
999 " INFO: aborting and cleaning up multipart upload(s); bucket=\"" <<
1000 this << "\"; uploads.size()=" << uploads
.size() <<
1001 "; is_truncated=" << is_truncated
<< dendl
;
1003 if (!uploads
.empty()) {
1004 for (const auto& upload
: uploads
) {
1005 ret
= upload
->abort(dpp
, cct
);
1007 // we're doing a best-effort; if something cannot be found,
1008 // log it and keep moving forward
1009 if (ret
!= -ENOENT
&& ret
!= -ERR_NO_SUCH_UPLOAD
) {
1010 ldpp_dout(dpp
, 0) << __func__
<<
1011 " ERROR : failed to abort and clean-up multipart upload \"" <<
1012 upload
->get_meta() << "\"" << dendl
;
1015 ldpp_dout(dpp
, 10) << __func__
<<
1016 " NOTE : unable to find part(s) of "
1017 "aborted multipart upload of \"" << upload
->get_meta() <<
1018 "\" for cleaning up" << dendl
;
1024 ldpp_dout(dpp
, 0) << __func__
<<
1025 " WARNING : aborted " << num_deleted
<<
1026 " incomplete multipart uploads" << dendl
;
1029 } while (is_truncated
);
1034 std::string
RadosBucket::topics_oid() const {
1035 return pubsub_oid_prefix
+ get_tenant() + ".bucket." + get_name() + "/" + get_marker();
1038 int RadosBucket::read_topics(rgw_pubsub_bucket_topics
& notifications
,
1039 RGWObjVersionTracker
* objv_tracker
, optional_yield y
, const DoutPrefixProvider
*dpp
)
1042 const int ret
= rgw_get_system_obj(store
->svc()->sysobj
,
1043 store
->svc()->zone
->get_zone_params().log_pool
,
1047 nullptr, y
, dpp
, nullptr);
1052 auto iter
= bl
.cbegin();
1054 decode(notifications
, iter
);
1055 } catch (buffer::error
& err
) {
1056 ldpp_dout(dpp
, 20) << " failed to decode bucket notifications from oid: " << topics_oid() << ". for bucket: "
1057 << get_name() << ". error: " << err
.what() << dendl
;
1064 int RadosBucket::write_topics(const rgw_pubsub_bucket_topics
& notifications
,
1065 RGWObjVersionTracker
* objv_tracker
, optional_yield y
, const DoutPrefixProvider
*dpp
) {
1067 encode(notifications
, bl
);
1069 return rgw_put_system_obj(dpp
, store
->svc()->sysobj
,
1070 store
->svc()->zone
->get_zone_params().log_pool
,
1072 bl
, false, objv_tracker
, real_time(), y
);
1075 int RadosBucket::remove_topics(RGWObjVersionTracker
* objv_tracker
,
1076 optional_yield y
, const DoutPrefixProvider
*dpp
) {
1077 return rgw_delete_system_obj(dpp
, store
->svc()->sysobj
,
1078 store
->svc()->zone
->get_zone_params().log_pool
,
1083 std::unique_ptr
<User
> RadosStore::get_user(const rgw_user
&u
)
1085 return std::make_unique
<RadosUser
>(this, u
);
1088 std::string
RadosStore::get_cluster_id(const DoutPrefixProvider
* dpp
, optional_yield y
)
1090 return getRados()->get_cluster_fsid(dpp
, y
);
1093 int RadosStore::get_user_by_access_key(const DoutPrefixProvider
* dpp
, const std::string
& key
, optional_yield y
, std::unique_ptr
<User
>* user
)
1097 RGWObjVersionTracker objv_tracker
;
1099 int r
= ctl()->user
->get_info_by_access_key(dpp
, key
, &uinfo
, y
, RGWUserCtl::GetParams().set_objv_tracker(&objv_tracker
));
1103 u
= new RadosUser(this, uinfo
);
1107 u
->get_version_tracker() = objv_tracker
;
1113 int RadosStore::get_user_by_email(const DoutPrefixProvider
* dpp
, const std::string
& email
, optional_yield y
, std::unique_ptr
<User
>* user
)
1117 RGWObjVersionTracker objv_tracker
;
1119 int r
= ctl()->user
->get_info_by_email(dpp
, email
, &uinfo
, y
, RGWUserCtl::GetParams().set_objv_tracker(&objv_tracker
));
1123 u
= new RadosUser(this, uinfo
);
1127 u
->get_version_tracker() = objv_tracker
;
1133 int RadosStore::get_user_by_swift(const DoutPrefixProvider
* dpp
, const std::string
& user_str
, optional_yield y
, std::unique_ptr
<User
>* user
)
1137 RGWObjVersionTracker objv_tracker
;
1139 int r
= ctl()->user
->get_info_by_swift(dpp
, user_str
, &uinfo
, y
, RGWUserCtl::GetParams().set_objv_tracker(&objv_tracker
));
1143 u
= new RadosUser(this, uinfo
);
1147 u
->get_version_tracker() = objv_tracker
;
1153 std::unique_ptr
<Object
> RadosStore::get_object(const rgw_obj_key
& k
)
1155 return std::make_unique
<RadosObject
>(this, k
);
1158 int RadosStore::get_bucket(const DoutPrefixProvider
* dpp
, User
* u
, const rgw_bucket
& b
, std::unique_ptr
<Bucket
>* bucket
, optional_yield y
)
1163 bp
= new RadosBucket(this, b
, u
);
1164 ret
= bp
->load_bucket(dpp
, y
);
1174 int RadosStore::get_bucket(User
* u
, const RGWBucketInfo
& i
, std::unique_ptr
<Bucket
>* bucket
)
1178 bp
= new RadosBucket(this, i
, u
);
1179 /* Don't need to fetch the bucket info, use the provided one */
1185 int RadosStore::get_bucket(const DoutPrefixProvider
* dpp
, User
* u
, const std::string
& tenant
, const std::string
& name
, std::unique_ptr
<Bucket
>* bucket
, optional_yield y
)
1192 return get_bucket(dpp
, u
, b
, bucket
, y
);
1195 bool RadosStore::is_meta_master()
1197 return svc()->zone
->is_meta_master();
1200 int RadosStore::forward_request_to_master(const DoutPrefixProvider
*dpp
, User
* user
, obj_version
* objv
,
1201 bufferlist
& in_data
,
1202 JSONParser
* jp
, req_info
& info
,
1205 if (is_meta_master()) {
1206 /* We're master, don't forward */
1210 if (!svc()->zone
->get_master_conn()) {
1211 ldpp_dout(dpp
, 0) << "rest connection is invalid" << dendl
;
1214 ldpp_dout(dpp
, 0) << "sending request to master zonegroup" << dendl
;
1215 bufferlist response
;
1216 std::string uid_str
= user
->get_id().to_str();
1217 #define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response
1218 int ret
= svc()->zone
->get_master_conn()->forward(dpp
, rgw_user(uid_str
), info
,
1219 objv
, MAX_REST_RESPONSE
,
1220 &in_data
, &response
, y
);
1224 ldpp_dout(dpp
, 20) << "response: " << response
.c_str() << dendl
;
1225 if (jp
&& !jp
->parse(response
.c_str(), response
.length())) {
1226 ldpp_dout(dpp
, 0) << "failed parsing response from master zonegroup" << dendl
;
1233 int RadosStore::forward_iam_request_to_master(const DoutPrefixProvider
*dpp
, const RGWAccessKey
& key
, obj_version
* objv
,
1234 bufferlist
& in_data
,
1235 RGWXMLDecoder::XMLParser
* parser
, req_info
& info
,
1238 if (is_meta_master()) {
1239 /* We're master, don't forward */
1243 if (!svc()->zone
->get_master_conn()) {
1244 ldpp_dout(dpp
, 0) << "rest connection is invalid" << dendl
;
1247 ldpp_dout(dpp
, 0) << "sending request to master zonegroup" << dendl
;
1248 bufferlist response
;
1249 #define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response
1250 int ret
= svc()->zone
->get_master_conn()->forward_iam_request(dpp
, key
, info
,
1251 objv
, MAX_REST_RESPONSE
,
1252 &in_data
, &response
, y
);
1256 ldpp_dout(dpp
, 20) << "response: " << response
.c_str() << dendl
;
1258 std::string r
= response
.c_str();
1259 std::string str_to_search
= """;
1260 std::string str_to_replace
= "\"";
1261 boost::replace_all(r
, str_to_search
, str_to_replace
);
1262 ldpp_dout(dpp
, 20) << "r: " << r
.c_str() << dendl
;
1264 if (parser
&& !parser
->parse(r
.c_str(), r
.length(), 1)) {
1265 ldpp_dout(dpp
, 0) << "ERROR: failed to parse response from master zonegroup" << dendl
;
1272 std::string
RadosStore::zone_unique_id(uint64_t unique_num
)
1274 return svc()->zone_utils
->unique_id(unique_num
);
1277 std::string
RadosStore::zone_unique_trans_id(const uint64_t unique_num
)
1279 return svc()->zone_utils
->unique_trans_id(unique_num
);
1282 int RadosStore::get_zonegroup(const std::string
& id
,
1283 std::unique_ptr
<ZoneGroup
>* zonegroup
)
1287 int r
= svc()->zone
->get_zonegroup(id
, rzg
);
1291 zg
= new RadosZoneGroup(this, rzg
);
1295 zonegroup
->reset(zg
);
1299 int RadosStore::list_all_zones(const DoutPrefixProvider
* dpp
, std::list
<std::string
>& zone_ids
)
1301 return svc()->zone
->list_zones(dpp
, zone_ids
);
1304 int RadosStore::cluster_stat(RGWClusterStat
& stats
)
1306 rados_cluster_stat_t rados_stats
;
1309 ret
= rados
->get_rados_handle()->cluster_stat(rados_stats
);
1313 stats
.kb
= rados_stats
.kb
;
1314 stats
.kb_used
= rados_stats
.kb_used
;
1315 stats
.kb_avail
= rados_stats
.kb_avail
;
1316 stats
.num_objects
= rados_stats
.num_objects
;
1321 std::unique_ptr
<Lifecycle
> RadosStore::get_lifecycle(void)
1323 return std::make_unique
<RadosLifecycle
>(this);
1326 std::unique_ptr
<Completions
> RadosStore::get_completions(void)
1328 return std::make_unique
<RadosCompletions
>();
1331 std::unique_ptr
<Notification
> RadosStore::get_notification(
1332 rgw::sal::Object
* obj
, rgw::sal::Object
* src_obj
, req_state
* s
, rgw::notify::EventType event_type
, optional_yield y
, const std::string
* object_name
)
1334 return std::make_unique
<RadosNotification
>(s
, this, obj
, src_obj
, s
, event_type
, y
, object_name
);
1337 std::unique_ptr
<Notification
> RadosStore::get_notification(const DoutPrefixProvider
* dpp
, rgw::sal::Object
* obj
, rgw::sal::Object
* src_obj
, rgw::notify::EventType event_type
, rgw::sal::Bucket
* _bucket
, std::string
& _user_id
, std::string
& _user_tenant
, std::string
& _req_id
, optional_yield y
)
1339 return std::make_unique
<RadosNotification
>(dpp
, this, obj
, src_obj
, event_type
, _bucket
, _user_id
, _user_tenant
, _req_id
, y
);
1342 std::string
RadosStore::topics_oid(const std::string
& tenant
) const {
1343 return pubsub_oid_prefix
+ tenant
;
1346 int RadosStore::read_topics(const std::string
& tenant
, rgw_pubsub_topics
& topics
, RGWObjVersionTracker
* objv_tracker
,
1347 optional_yield y
, const DoutPrefixProvider
*dpp
) {
1349 const int ret
= rgw_get_system_obj(svc()->sysobj
,
1350 svc()->zone
->get_zone_params().log_pool
,
1354 nullptr, y
, dpp
, nullptr);
1359 auto iter
= bl
.cbegin();
1361 decode(topics
, iter
);
1362 } catch (buffer::error
& err
) {
1363 ldpp_dout(dpp
, 20) << " failed to decode topics from oid: " << topics_oid(tenant
) <<
1364 ". error: " << err
.what() << dendl
;
1371 int RadosStore::write_topics(const std::string
& tenant
, const rgw_pubsub_topics
& topics
, RGWObjVersionTracker
* objv_tracker
,
1372 optional_yield y
, const DoutPrefixProvider
*dpp
) {
1376 return rgw_put_system_obj(dpp
, svc()->sysobj
,
1377 svc()->zone
->get_zone_params().log_pool
,
1379 bl
, false, objv_tracker
, real_time(), y
);
1382 int RadosStore::remove_topics(const std::string
& tenant
, RGWObjVersionTracker
* objv_tracker
,
1383 optional_yield y
, const DoutPrefixProvider
*dpp
) {
1384 return rgw_delete_system_obj(dpp
, svc()->sysobj
,
1385 svc()->zone
->get_zone_params().log_pool
,
1390 int RadosStore::delete_raw_obj(const DoutPrefixProvider
*dpp
, const rgw_raw_obj
& obj
)
1392 return rados
->delete_raw_obj(dpp
, obj
);
1395 int RadosStore::delete_raw_obj_aio(const DoutPrefixProvider
*dpp
, const rgw_raw_obj
& obj
, Completions
* aio
)
1397 RadosCompletions
* raio
= static_cast<RadosCompletions
*>(aio
);
1399 return rados
->delete_raw_obj_aio(dpp
, obj
, raio
->handles
);
1402 void RadosStore::get_raw_obj(const rgw_placement_rule
& placement_rule
, const rgw_obj
& obj
, rgw_raw_obj
* raw_obj
)
1404 rados
->obj_to_raw(placement_rule
, obj
, raw_obj
);
1407 int RadosStore::get_raw_chunk_size(const DoutPrefixProvider
* dpp
, const rgw_raw_obj
& obj
, uint64_t* chunk_size
)
1409 return rados
->get_max_chunk_size(obj
.pool
, chunk_size
, dpp
);
1412 int RadosStore::initialize(CephContext
*cct
, const DoutPrefixProvider
*dpp
)
1414 std::unique_ptr
<ZoneGroup
> zg
=
1415 std::make_unique
<RadosZoneGroup
>(this, svc()->zone
->get_zonegroup());
1416 zone
= make_unique
<RadosZone
>(this, std::move(zg
));
1420 int RadosStore::log_usage(const DoutPrefixProvider
*dpp
, map
<rgw_user_bucket
, RGWUsageBatch
>& usage_info
)
1422 return rados
->log_usage(dpp
, usage_info
);
1425 int RadosStore::log_op(const DoutPrefixProvider
*dpp
, std::string
& oid
, bufferlist
& bl
)
1427 rgw_raw_obj
obj(svc()->zone
->get_zone_params().log_pool
, oid
);
1429 int ret
= rados
->append_async(dpp
, obj
, bl
.length(), bl
);
1430 if (ret
== -ENOENT
) {
1431 ret
= rados
->create_pool(dpp
, svc()->zone
->get_zone_params().log_pool
);
1435 ret
= rados
->append_async(dpp
, obj
, bl
.length(), bl
);
1441 int RadosStore::register_to_service_map(const DoutPrefixProvider
*dpp
, const std::string
& daemon_type
,
1442 const map
<std::string
, std::string
>& meta
)
1444 return rados
->register_to_service_map(dpp
, daemon_type
, meta
);
1447 void RadosStore::get_quota(RGWQuota
& quota
)
1449 quota
.bucket_quota
= svc()->quota
->get_bucket_quota();
1450 quota
.user_quota
= svc()->quota
->get_user_quota();
1453 void RadosStore::get_ratelimit(RGWRateLimitInfo
& bucket_ratelimit
, RGWRateLimitInfo
& user_ratelimit
, RGWRateLimitInfo
& anon_ratelimit
)
1455 bucket_ratelimit
= svc()->zone
->get_current_period().get_config().bucket_ratelimit
;
1456 user_ratelimit
= svc()->zone
->get_current_period().get_config().user_ratelimit
;
1457 anon_ratelimit
= svc()->zone
->get_current_period().get_config().anon_ratelimit
;
1460 int RadosStore::set_buckets_enabled(const DoutPrefixProvider
* dpp
, vector
<rgw_bucket
>& buckets
, bool enabled
)
1462 return rados
->set_buckets_enabled(buckets
, enabled
, dpp
);
1465 int RadosStore::get_sync_policy_handler(const DoutPrefixProvider
* dpp
,
1466 std::optional
<rgw_zone_id
> zone
,
1467 std::optional
<rgw_bucket
> bucket
,
1468 RGWBucketSyncPolicyHandlerRef
* phandler
,
1471 return ctl()->bucket
->get_sync_policy_handler(zone
, bucket
, phandler
, y
, dpp
);
1474 RGWDataSyncStatusManager
* RadosStore::get_data_sync_manager(const rgw_zone_id
& source_zone
)
1476 return rados
->get_data_sync_manager(source_zone
);
1479 int RadosStore::read_all_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
,
1480 uint32_t max_entries
, bool* is_truncated
,
1481 RGWUsageIter
& usage_iter
,
1482 map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
)
1485 std::string bucket_name
;
1487 return rados
->read_usage(dpp
, uid
, bucket_name
, start_epoch
, end_epoch
, max_entries
,
1488 is_truncated
, usage_iter
, usage
);
1491 int RadosStore::trim_all_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
)
1494 std::string bucket_name
;
1496 return rados
->trim_usage(dpp
, uid
, bucket_name
, start_epoch
, end_epoch
);
1499 int RadosStore::get_config_key_val(std::string name
, bufferlist
* bl
)
1501 return svc()->config_key
->get(name
, true, bl
);
1504 int RadosStore::meta_list_keys_init(const DoutPrefixProvider
*dpp
, const std::string
& section
, const std::string
& marker
, void** phandle
)
1506 return ctl()->meta
.mgr
->list_keys_init(dpp
, section
, marker
, phandle
);
1509 int RadosStore::meta_list_keys_next(const DoutPrefixProvider
*dpp
, void* handle
, int max
, list
<std::string
>& keys
, bool* truncated
)
1511 return ctl()->meta
.mgr
->list_keys_next(dpp
, handle
, max
, keys
, truncated
);
1514 void RadosStore::meta_list_keys_complete(void* handle
)
1516 ctl()->meta
.mgr
->list_keys_complete(handle
);
1519 std::string
RadosStore::meta_get_marker(void* handle
)
1521 return ctl()->meta
.mgr
->get_marker(handle
);
1524 int RadosStore::meta_remove(const DoutPrefixProvider
* dpp
, std::string
& metadata_key
, optional_yield y
)
1526 return ctl()->meta
.mgr
->remove(metadata_key
, y
, dpp
);
1529 void RadosStore::finalize(void)
1535 void RadosStore::register_admin_apis(RGWRESTMgr
* mgr
)
1537 mgr
->register_resource("user", new RGWRESTMgr_User
);
1538 mgr
->register_resource("bucket", new RGWRESTMgr_Bucket
);
1539 /*Registering resource for /admin/metadata */
1540 mgr
->register_resource("metadata", new RGWRESTMgr_Metadata
);
1541 mgr
->register_resource("log", new RGWRESTMgr_Log
);
1542 /* XXX These may become global when cbodley is done with his zone work */
1543 mgr
->register_resource("config", new RGWRESTMgr_Config
);
1544 mgr
->register_resource("realm", new RGWRESTMgr_Realm
);
1545 mgr
->register_resource("ratelimit", new RGWRESTMgr_Ratelimit
);
1548 std::unique_ptr
<LuaManager
> RadosStore::get_lua_manager()
1550 return std::make_unique
<RadosLuaManager
>(this);
1553 std::unique_ptr
<RGWRole
> RadosStore::get_role(std::string name
,
1556 std::string trust_policy
,
1557 std::string max_session_duration_str
,
1558 std::multimap
<std::string
,std::string
> tags
)
1560 return std::make_unique
<RadosRole
>(this, name
, tenant
, path
, trust_policy
, max_session_duration_str
, tags
);
1563 std::unique_ptr
<RGWRole
> RadosStore::get_role(std::string id
)
1565 return std::make_unique
<RadosRole
>(this, id
);
1568 std::unique_ptr
<RGWRole
> RadosStore::get_role(const RGWRoleInfo
& info
)
1570 return std::make_unique
<RadosRole
>(this, info
);
1573 int RadosStore::get_roles(const DoutPrefixProvider
*dpp
,
1575 const std::string
& path_prefix
,
1576 const std::string
& tenant
,
1577 vector
<std::unique_ptr
<RGWRole
>>& roles
)
1579 auto pool
= svc()->zone
->get_zone_params().roles_pool
;
1582 // List all roles if path prefix is empty
1583 if (! path_prefix
.empty()) {
1584 prefix
= tenant
+ RGWRole::role_path_oid_prefix
+ path_prefix
;
1586 prefix
= tenant
+ RGWRole::role_path_oid_prefix
;
1589 //Get the filtered objects
1590 list
<std::string
> result
;
1592 RGWListRawObjsCtx ctx
;
1594 list
<std::string
> oids
;
1595 int r
= rados
->list_raw_objects(dpp
, pool
, prefix
, 1000, ctx
, oids
, &is_truncated
);
1597 ldpp_dout(dpp
, 0) << "ERROR: listing filtered objects failed: "
1598 << prefix
<< ": " << cpp_strerror(-r
) << dendl
;
1601 for (const auto& iter
: oids
) {
1602 result
.push_back(iter
.substr(RGWRole::role_path_oid_prefix
.size()));
1604 } while (is_truncated
);
1606 for (const auto& it
: result
) {
1607 //Find the role oid prefix from the end
1608 size_t pos
= it
.rfind(RGWRole::role_oid_prefix
);
1609 if (pos
== std::string::npos
) {
1612 // Split the result into path and info_oid + id
1613 std::string path
= it
.substr(0, pos
);
1615 /*Make sure that prefix is part of path (False results could've been returned)
1616 because of the role info oid + id appended to the path)*/
1617 if(path_prefix
.empty() || path
.find(path_prefix
) != std::string::npos
) {
1618 //Get id from info oid prefix + id
1619 std::string id
= it
.substr(pos
+ RGWRole::role_oid_prefix
.length());
1621 std::unique_ptr
<rgw::sal::RGWRole
> role
= get_role(id
);
1622 int ret
= role
->read_info(dpp
, y
);
1626 roles
.push_back(std::move(role
));
1633 std::unique_ptr
<RGWOIDCProvider
> RadosStore::get_oidc_provider()
1635 return std::make_unique
<RadosOIDCProvider
>(this);
1638 int RadosStore::get_oidc_providers(const DoutPrefixProvider
*dpp
,
1639 const std::string
& tenant
,
1640 vector
<std::unique_ptr
<RGWOIDCProvider
>>& providers
)
1642 std::string prefix
= tenant
+ RGWOIDCProvider::oidc_url_oid_prefix
;
1643 auto pool
= svc()->zone
->get_zone_params().oidc_pool
;
1645 //Get the filtered objects
1646 list
<std::string
> result
;
1648 RGWListRawObjsCtx ctx
;
1650 list
<std::string
> oids
;
1651 int r
= rados
->list_raw_objects(dpp
, pool
, prefix
, 1000, ctx
, oids
, &is_truncated
);
1653 ldpp_dout(dpp
, 0) << "ERROR: listing filtered objects failed: OIDC pool: "
1654 << pool
.name
<< ": " << prefix
<< ": " << cpp_strerror(-r
) << dendl
;
1657 for (const auto& iter
: oids
) {
1658 std::unique_ptr
<rgw::sal::RGWOIDCProvider
> provider
= get_oidc_provider();
1661 r
= rgw_get_system_obj(svc()->sysobj
, pool
, iter
, bl
, nullptr, nullptr, null_yield
, dpp
);
1668 auto iter
= bl
.cbegin();
1669 decode(*provider
, iter
);
1670 } catch (buffer::error
& err
) {
1671 ldpp_dout(dpp
, 0) << "ERROR: failed to decode oidc provider info from pool: "
1672 << pool
.name
<< ": " << iter
<< dendl
;
1676 providers
.push_back(std::move(provider
));
1678 } while (is_truncated
);
1683 std::unique_ptr
<Writer
> RadosStore::get_append_writer(const DoutPrefixProvider
*dpp
,
1685 rgw::sal::Object
* obj
,
1686 const rgw_user
& owner
,
1687 const rgw_placement_rule
*ptail_placement_rule
,
1688 const std::string
& unique_tag
,
1690 uint64_t *cur_accounted_size
)
1692 RGWBucketInfo
& bucket_info
= obj
->get_bucket()->get_info();
1693 RGWObjectCtx
& obj_ctx
= static_cast<RadosObject
*>(obj
)->get_ctx();
1694 auto aio
= rgw::make_throttle(ctx()->_conf
->rgw_put_obj_min_window_size
, y
);
1695 return std::make_unique
<RadosAppendWriter
>(dpp
, y
,
1696 bucket_info
, obj_ctx
, obj
->get_obj(),
1697 this, std::move(aio
), owner
,
1698 ptail_placement_rule
,
1699 unique_tag
, position
,
1700 cur_accounted_size
);
1703 std::unique_ptr
<Writer
> RadosStore::get_atomic_writer(const DoutPrefixProvider
*dpp
,
1705 rgw::sal::Object
* obj
,
1706 const rgw_user
& owner
,
1707 const rgw_placement_rule
*ptail_placement_rule
,
1709 const std::string
& unique_tag
)
1711 RGWBucketInfo
& bucket_info
= obj
->get_bucket()->get_info();
1712 RGWObjectCtx
& obj_ctx
= static_cast<RadosObject
*>(obj
)->get_ctx();
1713 auto aio
= rgw::make_throttle(ctx()->_conf
->rgw_put_obj_min_window_size
, y
);
1714 return std::make_unique
<RadosAtomicWriter
>(dpp
, y
,
1715 bucket_info
, obj_ctx
, obj
->get_obj(),
1716 this, std::move(aio
), owner
,
1717 ptail_placement_rule
,
1718 olh_epoch
, unique_tag
);
1721 const std::string
& RadosStore::get_compression_type(const rgw_placement_rule
& rule
)
1723 return svc()->zone
->get_zone_params().get_compression_type(rule
);
1726 bool RadosStore::valid_placement(const rgw_placement_rule
& rule
)
1728 return svc()->zone
->get_zone_params().valid_placement(rule
);
1731 int RadosStore::get_obj_head_ioctx(const DoutPrefixProvider
*dpp
, const RGWBucketInfo
& bucket_info
, const rgw_obj
& obj
, librados::IoCtx
* ioctx
)
1733 return rados
->get_obj_head_ioctx(dpp
, bucket_info
, obj
, ioctx
);
1736 RadosObject::~RadosObject()
1738 if (rados_ctx_owned
)
1742 int RadosObject::get_obj_state(const DoutPrefixProvider
* dpp
, RGWObjState
**pstate
, optional_yield y
, bool follow_olh
)
1744 int ret
= store
->getRados()->get_obj_state(dpp
, rados_ctx
, bucket
->get_info(), get_obj(), pstate
, &manifest
, follow_olh
, y
);
1749 /* Don't overwrite obj, atomic, or prefetch */
1750 rgw_obj obj
= get_obj();
1751 bool is_atomic
= state
.is_atomic
;
1752 bool prefetch_data
= state
.prefetch_data
;
1757 state
.is_atomic
= is_atomic
;
1758 state
.prefetch_data
= prefetch_data
;
1762 int RadosObject::read_attrs(const DoutPrefixProvider
* dpp
, RGWRados::Object::Read
&read_op
, optional_yield y
, rgw_obj
* target_obj
)
1764 read_op
.params
.attrs
= &state
.attrset
;
1765 read_op
.params
.target_obj
= target_obj
;
1766 read_op
.params
.obj_size
= &state
.size
;
1767 read_op
.params
.lastmod
= &state
.mtime
;
1769 return read_op
.prepare(y
, dpp
);
1772 int RadosObject::set_obj_attrs(const DoutPrefixProvider
* dpp
, Attrs
* setattrs
, Attrs
* delattrs
, optional_yield y
)
1775 return store
->getRados()->set_attrs(dpp
, rados_ctx
,
1778 setattrs
? *setattrs
: empty
,
1779 delattrs
? delattrs
: nullptr,
1783 int RadosObject::get_obj_attrs(optional_yield y
, const DoutPrefixProvider
* dpp
, rgw_obj
* target_obj
)
1785 RGWRados::Object
op_target(store
->getRados(), bucket
->get_info(), *rados_ctx
, get_obj());
1786 RGWRados::Object::Read
read_op(&op_target
);
1788 return read_attrs(dpp
, read_op
, y
, target_obj
);
1791 int RadosObject::modify_obj_attrs(const char* attr_name
, bufferlist
& attr_val
, optional_yield y
, const DoutPrefixProvider
* dpp
)
1793 rgw_obj target
= get_obj();
1794 rgw_obj save
= get_obj();
1795 int r
= get_obj_attrs(y
, dpp
, &target
);
1800 /* Temporarily set target */
1803 state
.attrset
[attr_name
] = attr_val
;
1804 r
= set_obj_attrs(dpp
, &state
.attrset
, nullptr, y
);
1805 /* Restore target */
1811 int RadosObject::delete_obj_attrs(const DoutPrefixProvider
* dpp
, const char* attr_name
, optional_yield y
)
1817 rmattr
[attr_name
] = bl
;
1818 return set_obj_attrs(dpp
, nullptr, &rmattr
, y
);
1821 bool RadosObject::is_expired() {
1822 auto iter
= state
.attrset
.find(RGW_ATTR_DELETE_AT
);
1823 if (iter
== state
.attrset
.end()) {
1828 auto bufit
= iter
->second
.cbegin();
1829 decode(delete_at
, bufit
);
1830 } catch (buffer::error
& err
) {
1831 ldout(store
->ctx(), 0) << "ERROR: " << __func__
<< ": failed to decode " RGW_ATTR_DELETE_AT
" attr" << dendl
;
1835 return delete_at
<= ceph_clock_now() && !delete_at
.is_zero();
1838 void RadosObject::gen_rand_obj_instance_name()
1840 store
->getRados()->gen_rand_obj_instance_name(&state
.obj
.key
);
1843 void RadosObject::raw_obj_to_obj(const rgw_raw_obj
& raw_obj
)
1845 rgw_obj tobj
= get_obj();
1846 RGWSI_Tier_RADOS::raw_obj_to_obj(get_bucket()->get_key(), raw_obj
, &tobj
);
1850 void RadosObject::get_raw_obj(rgw_raw_obj
* raw_obj
)
1852 store
->getRados()->obj_to_raw((bucket
->get_info()).placement_rule
, get_obj(), raw_obj
);
1855 int RadosObject::omap_get_vals(const DoutPrefixProvider
*dpp
, const std::string
& marker
, uint64_t count
,
1856 std::map
<std::string
, bufferlist
> *m
,
1857 bool* pmore
, optional_yield y
)
1859 rgw_raw_obj raw_obj
;
1860 get_raw_obj(&raw_obj
);
1861 auto sysobj
= store
->svc()->sysobj
->get_obj(raw_obj
);
1863 return sysobj
.omap().get_vals(dpp
, marker
, count
, m
, pmore
, y
);
1866 int RadosObject::omap_get_all(const DoutPrefixProvider
*dpp
, std::map
<std::string
, bufferlist
> *m
,
1869 rgw_raw_obj raw_obj
;
1870 get_raw_obj(&raw_obj
);
1871 auto sysobj
= store
->svc()->sysobj
->get_obj(raw_obj
);
1873 return sysobj
.omap().get_all(dpp
, m
, y
);
1876 int RadosObject::omap_get_vals_by_keys(const DoutPrefixProvider
*dpp
, const std::string
& oid
,
1877 const std::set
<std::string
>& keys
,
1881 rgw_raw_obj head_obj
;
1882 librados::IoCtx cur_ioctx
;
1883 rgw_obj obj
= get_obj();
1885 store
->getRados()->obj_to_raw(bucket
->get_placement_rule(), obj
, &head_obj
);
1886 ret
= store
->get_obj_head_ioctx(dpp
, bucket
->get_info(), obj
, &cur_ioctx
);
1891 return cur_ioctx
.omap_get_vals_by_keys(oid
, keys
, vals
);
1894 int RadosObject::omap_set_val_by_key(const DoutPrefixProvider
*dpp
, const std::string
& key
, bufferlist
& val
,
1895 bool must_exist
, optional_yield y
)
1897 rgw_raw_obj raw_meta_obj
;
1898 rgw_obj obj
= get_obj();
1900 store
->getRados()->obj_to_raw(bucket
->get_placement_rule(), obj
, &raw_meta_obj
);
1902 auto sysobj
= store
->svc()->sysobj
->get_obj(raw_meta_obj
);
1904 return sysobj
.omap().set_must_exist(must_exist
).set(dpp
, key
, val
, y
);
1907 int RadosObject::chown(User
& new_user
, const DoutPrefixProvider
* dpp
, optional_yield y
)
1909 int r
= get_obj_attrs(y
, dpp
);
1911 ldpp_dout(dpp
, 0) << "ERROR: failed to read object attrs " << get_name() << cpp_strerror(-r
) << dendl
;
1915 const auto& aiter
= get_attrs().find(RGW_ATTR_ACL
);
1916 if (aiter
== get_attrs().end()) {
1917 ldpp_dout(dpp
, 0) << "ERROR: no acls found for object " << get_name() << dendl
;
1921 bufferlist
& bl
= aiter
->second
;
1922 RGWAccessControlPolicy
policy(store
->ctx());
1924 auto bliter
= bl
.cbegin();
1926 policy
.decode(bliter
);
1927 owner
= policy
.get_owner();
1928 } catch (buffer::error
& err
) {
1929 ldpp_dout(dpp
, 0) << "ERROR: decode policy failed" << err
.what()
1934 //Get the ACL from the policy
1935 RGWAccessControlList
& acl
= policy
.get_acl();
1937 //Remove grant that is set to old owner
1938 acl
.remove_canon_user_grant(owner
.get_id());
1940 //Create a grant and add grant
1942 grant
.set_canon(new_user
.get_id(), new_user
.get_display_name(), RGW_PERM_FULL_CONTROL
);
1943 acl
.add_grant(&grant
);
1945 //Update the ACL owner to the new user
1946 owner
.set_id(new_user
.get_id());
1947 owner
.set_name(new_user
.get_display_name());
1948 policy
.set_owner(owner
);
1954 map
<string
, bufferlist
> attrs
;
1955 attrs
[RGW_ATTR_ACL
] = bl
;
1956 r
= set_obj_attrs(dpp
, &attrs
, nullptr, y
);
1958 ldpp_dout(dpp
, 0) << "ERROR: modify attr failed " << cpp_strerror(-r
) << dendl
;
1965 std::unique_ptr
<MPSerializer
> RadosObject::get_serializer(const DoutPrefixProvider
*dpp
, const std::string
& lock_name
)
1967 return std::make_unique
<MPRadosSerializer
>(dpp
, store
, this, lock_name
);
1970 int RadosObject::transition(Bucket
* bucket
,
1971 const rgw_placement_rule
& placement_rule
,
1972 const real_time
& mtime
,
1974 const DoutPrefixProvider
* dpp
,
1977 return store
->getRados()->transition_obj(*rados_ctx
, bucket
->get_info(), get_obj(), placement_rule
, mtime
, olh_epoch
, dpp
, y
);
1980 int RadosObject::transition_to_cloud(Bucket
* bucket
,
1981 rgw::sal::PlacementTier
* tier
,
1982 rgw_bucket_dir_entry
& o
,
1983 std::set
<std::string
>& cloud_targets
,
1986 const DoutPrefixProvider
* dpp
,
1990 rgw::sal::RadosPlacementTier
* rtier
= static_cast<rgw::sal::RadosPlacementTier
*>(tier
);
1991 string id
= "cloudid";
1992 string endpoint
= rtier
->get_rt().t
.s3
.endpoint
;
1993 RGWAccessKey key
= rtier
->get_rt().t
.s3
.key
;
1994 string region
= rtier
->get_rt().t
.s3
.region
;
1995 HostStyle host_style
= rtier
->get_rt().t
.s3
.host_style
;
1996 string bucket_name
= rtier
->get_rt().t
.s3
.target_path
;
1997 const rgw::sal::ZoneGroup
& zonegroup
= store
->get_zone()->get_zonegroup();
1999 if (bucket_name
.empty()) {
2000 bucket_name
= "rgwx-" + zonegroup
.get_name() + "-" + tier
->get_storage_class() +
2002 boost::algorithm::to_lower(bucket_name
);
2005 /* Create RGW REST connection */
2006 S3RESTConn
conn(cct
, id
, { endpoint
}, key
, zonegroup
.get_id(), region
, host_style
);
2008 RGWLCCloudTierCtx
tier_ctx(cct
, dpp
, o
, store
, bucket
->get_info(),
2009 this, conn
, bucket_name
,
2010 rtier
->get_rt().t
.s3
.target_storage_class
);
2011 tier_ctx
.acl_mappings
= rtier
->get_rt().t
.s3
.acl_mappings
;
2012 tier_ctx
.multipart_min_part_size
= rtier
->get_rt().t
.s3
.multipart_min_part_size
;
2013 tier_ctx
.multipart_sync_threshold
= rtier
->get_rt().t
.s3
.multipart_sync_threshold
;
2014 tier_ctx
.storage_class
= tier
->get_storage_class();
2016 ldpp_dout(dpp
, 0) << "Transitioning object(" << o
.key
<< ") to the cloud endpoint(" << endpoint
<< ")" << dendl
;
2018 /* Transition object to cloud end point */
2019 int ret
= rgw_cloud_tier_transfer_object(tier_ctx
, cloud_targets
);
2022 ldpp_dout(dpp
, 0) << "ERROR: failed to transfer object(" << o
.key
<< ") to the cloud endpoint(" << endpoint
<< ") ret=" << ret
<< dendl
;
2026 if (update_object
) {
2027 real_time read_mtime
;
2029 std::unique_ptr
<rgw::sal::Object::ReadOp
> read_op(get_read_op());
2030 read_op
->params
.lastmod
= &read_mtime
;
2032 ret
= read_op
->prepare(y
, dpp
);
2034 ldpp_dout(dpp
, 0) << "ERROR: Updating tier object(" << o
.key
<< ") failed ret=" << ret
<< dendl
;
2038 if (read_mtime
!= tier_ctx
.o
.meta
.mtime
) {
2040 ldpp_dout(dpp
, 0) << "ERROR: Updating tier object(" << o
.key
<< ") failed ret=" << -ECANCELED
<< dendl
;
2044 rgw_placement_rule target_placement
;
2045 target_placement
.inherit_from(tier_ctx
.bucket_info
.placement_rule
);
2046 target_placement
.storage_class
= tier
->get_storage_class();
2048 ret
= write_cloud_tier(dpp
, y
, tier_ctx
.o
.versioned_epoch
,
2049 tier
, tier_ctx
.is_multipart_upload
,
2050 target_placement
, tier_ctx
.obj
);
2057 int RadosObject::write_cloud_tier(const DoutPrefixProvider
* dpp
,
2060 PlacementTier
* tier
,
2061 bool is_multipart_upload
,
2062 rgw_placement_rule
& target_placement
,
2065 rgw::sal::RadosPlacementTier
* rtier
= static_cast<rgw::sal::RadosPlacementTier
*>(tier
);
2066 map
<string
, bufferlist
> attrs
= get_attrs();
2067 RGWRados::Object
op_target(store
->getRados(), bucket
->get_info(), *rados_ctx
, get_obj());
2068 RGWRados::Object::Write
obj_op(&op_target
);
2070 obj_op
.meta
.modify_tail
= true;
2071 obj_op
.meta
.flags
= PUT_OBJ_CREATE
;
2072 obj_op
.meta
.category
= RGWObjCategory::CloudTiered
;
2073 obj_op
.meta
.delete_at
= real_time();
2075 obj_op
.meta
.data
= &blo
;
2076 obj_op
.meta
.if_match
= NULL
;
2077 obj_op
.meta
.if_nomatch
= NULL
;
2078 obj_op
.meta
.user_data
= NULL
;
2079 obj_op
.meta
.zones_trace
= NULL
;
2080 obj_op
.meta
.delete_at
= real_time();
2081 obj_op
.meta
.olh_epoch
= olh_epoch
;
2083 RGWObjManifest
*pmanifest
;
2084 RGWObjManifest manifest
;
2086 pmanifest
= &manifest
;
2087 RGWObjTier tier_config
;
2088 tier_config
.name
= tier
->get_storage_class();
2089 tier_config
.tier_placement
= rtier
->get_rt();
2090 tier_config
.is_multipart_upload
= is_multipart_upload
;
2092 pmanifest
->set_tier_type("cloud-s3");
2093 pmanifest
->set_tier_config(tier_config
);
2095 /* check if its necessary */
2096 pmanifest
->set_head(target_placement
, head_obj
->get_obj(), 0);
2097 pmanifest
->set_tail_placement(target_placement
, head_obj
->get_obj().bucket
);
2098 pmanifest
->set_obj_size(0);
2099 obj_op
.meta
.manifest
= pmanifest
;
2101 /* update storage class */
2103 bl
.append(tier
->get_storage_class());
2104 attrs
[RGW_ATTR_STORAGE_CLASS
] = bl
;
2106 attrs
.erase(RGW_ATTR_ID_TAG
);
2107 attrs
.erase(RGW_ATTR_TAIL_TAG
);
2109 return obj_op
.write_meta(dpp
, 0, 0, attrs
, y
);
2112 int RadosObject::get_max_chunk_size(const DoutPrefixProvider
* dpp
, rgw_placement_rule placement_rule
, uint64_t* max_chunk_size
, uint64_t* alignment
)
2114 return store
->getRados()->get_max_chunk_size(placement_rule
, get_obj(), max_chunk_size
, dpp
, alignment
);
2117 void RadosObject::get_max_aligned_size(uint64_t size
, uint64_t alignment
,
2120 store
->getRados()->get_max_aligned_size(size
, alignment
, max_size
);
2123 bool RadosObject::placement_rules_match(rgw_placement_rule
& r1
, rgw_placement_rule
& r2
)
2133 if (!store
->getRados()->get_obj_data_pool(r1
, obj
, &p1
)) {
2136 if (!store
->getRados()->get_obj_data_pool(r2
, obj
, &p2
)) {
2143 int RadosObject::dump_obj_layout(const DoutPrefixProvider
*dpp
, optional_yield y
, Formatter
* f
)
2146 RGWObjManifest
*amanifest
{nullptr};
2147 rgw_raw_obj head_obj
;
2149 RGWRados::Object
op_target(store
->getRados(), bucket
->get_info(), *rados_ctx
, get_obj());
2150 RGWRados::Object::Read
parent_op(&op_target
);
2153 parent_op
.params
.obj_size
= &obj_size
;
2154 parent_op
.params
.attrs
= &get_attrs();
2156 ret
= parent_op
.prepare(y
, dpp
);
2161 head_obj
= parent_op
.state
.head_obj
;
2163 ret
= op_target
.get_manifest(dpp
, &amanifest
, y
);
2168 ::encode_json("head", head_obj
, f
);
2169 ::encode_json("manifest", *amanifest
, f
);
2170 f
->open_array_section("data_location");
2171 for (auto miter
= amanifest
->obj_begin(dpp
); miter
!= amanifest
->obj_end(dpp
); ++miter
) {
2172 f
->open_object_section("obj");
2173 rgw_raw_obj raw_loc
= miter
.get_location().get_raw_obj(store
->getRados());
2174 uint64_t ofs
= miter
.get_ofs();
2175 uint64_t left
= amanifest
->get_obj_size() - ofs
;
2176 ::encode_json("ofs", miter
.get_ofs(), f
);
2177 ::encode_json("loc", raw_loc
, f
);
2178 ::encode_json("loc_ofs", miter
.location_ofs(), f
);
2179 uint64_t loc_size
= miter
.get_stripe_size();
2180 if (loc_size
> left
) {
2183 ::encode_json("loc_size", loc_size
, f
);
2191 std::unique_ptr
<Object::ReadOp
> RadosObject::get_read_op()
2193 return std::make_unique
<RadosObject::RadosReadOp
>(this, rados_ctx
);
2196 RadosObject::RadosReadOp::RadosReadOp(RadosObject
*_source
, RGWObjectCtx
*_rctx
) :
2199 op_target(_source
->store
->getRados(),
2200 _source
->get_bucket()->get_info(),
2201 *static_cast<RGWObjectCtx
*>(rctx
),
2202 _source
->get_obj()),
2203 parent_op(&op_target
)
2206 int RadosObject::RadosReadOp::prepare(optional_yield y
, const DoutPrefixProvider
* dpp
)
2210 parent_op
.conds
.mod_ptr
= params
.mod_ptr
;
2211 parent_op
.conds
.unmod_ptr
= params
.unmod_ptr
;
2212 parent_op
.conds
.high_precision_time
= params
.high_precision_time
;
2213 parent_op
.conds
.mod_zone_id
= params
.mod_zone_id
;
2214 parent_op
.conds
.mod_pg_ver
= params
.mod_pg_ver
;
2215 parent_op
.conds
.if_match
= params
.if_match
;
2216 parent_op
.conds
.if_nomatch
= params
.if_nomatch
;
2217 parent_op
.params
.lastmod
= params
.lastmod
;
2218 parent_op
.params
.target_obj
= params
.target_obj
;
2219 parent_op
.params
.obj_size
= &obj_size
;
2220 parent_op
.params
.attrs
= &source
->get_attrs();
2222 int ret
= parent_op
.prepare(y
, dpp
);
2226 source
->set_key(parent_op
.state
.obj
.key
);
2227 source
->set_obj_size(obj_size
);
2232 int RadosObject::RadosReadOp::read(int64_t ofs
, int64_t end
, bufferlist
& bl
, optional_yield y
, const DoutPrefixProvider
* dpp
)
2234 return parent_op
.read(ofs
, end
, bl
, y
, dpp
);
2237 int RadosObject::RadosReadOp::get_attr(const DoutPrefixProvider
* dpp
, const char* name
, bufferlist
& dest
, optional_yield y
)
2239 return parent_op
.get_attr(dpp
, name
, dest
, y
);
2242 std::unique_ptr
<Object::DeleteOp
> RadosObject::get_delete_op()
2244 return std::make_unique
<RadosObject::RadosDeleteOp
>(this);
2247 RadosObject::RadosDeleteOp::RadosDeleteOp(RadosObject
*_source
) :
2249 op_target(_source
->store
->getRados(),
2250 _source
->get_bucket()->get_info(),
2252 _source
->get_obj()),
2253 parent_op(&op_target
)
2256 int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider
* dpp
, optional_yield y
)
2258 parent_op
.params
.bucket_owner
= params
.bucket_owner
.get_id();
2259 parent_op
.params
.versioning_status
= params
.versioning_status
;
2260 parent_op
.params
.obj_owner
= params
.obj_owner
;
2261 parent_op
.params
.olh_epoch
= params
.olh_epoch
;
2262 parent_op
.params
.marker_version_id
= params
.marker_version_id
;
2263 parent_op
.params
.bilog_flags
= params
.bilog_flags
;
2264 parent_op
.params
.remove_objs
= params
.remove_objs
;
2265 parent_op
.params
.expiration_time
= params
.expiration_time
;
2266 parent_op
.params
.unmod_since
= params
.unmod_since
;
2267 parent_op
.params
.mtime
= params
.mtime
;
2268 parent_op
.params
.high_precision_time
= params
.high_precision_time
;
2269 parent_op
.params
.zones_trace
= params
.zones_trace
;
2270 parent_op
.params
.abortmp
= params
.abortmp
;
2271 parent_op
.params
.parts_accounted_size
= params
.parts_accounted_size
;
2273 int ret
= parent_op
.delete_obj(y
, dpp
);
2277 result
.delete_marker
= parent_op
.result
.delete_marker
;
2278 result
.version_id
= parent_op
.result
.version_id
;
2283 int RadosObject::delete_object(const DoutPrefixProvider
* dpp
,
2285 bool prevent_versioning
)
2287 RGWRados::Object
del_target(store
->getRados(), bucket
->get_info(), *rados_ctx
, get_obj());
2288 RGWRados::Object::Delete
del_op(&del_target
);
2290 del_op
.params
.bucket_owner
= bucket
->get_info().owner
;
2291 del_op
.params
.versioning_status
= prevent_versioning
? 0 : bucket
->get_info().versioning_status();
2293 return del_op
.delete_obj(y
, dpp
);
2296 int RadosObject::delete_obj_aio(const DoutPrefixProvider
* dpp
, RGWObjState
* astate
,
2297 Completions
* aio
, bool keep_index_consistent
,
2300 RadosCompletions
* raio
= static_cast<RadosCompletions
*>(aio
);
2302 return store
->getRados()->delete_obj_aio(dpp
, get_obj(), bucket
->get_info(), astate
,
2303 raio
->handles
, keep_index_consistent
, y
);
2306 int RadosObject::copy_object(User
* user
,
2308 const rgw_zone_id
& source_zone
,
2309 rgw::sal::Object
* dest_object
,
2310 rgw::sal::Bucket
* dest_bucket
,
2311 rgw::sal::Bucket
* src_bucket
,
2312 const rgw_placement_rule
& dest_placement
,
2313 ceph::real_time
* src_mtime
,
2314 ceph::real_time
* mtime
,
2315 const ceph::real_time
* mod_ptr
,
2316 const ceph::real_time
* unmod_ptr
,
2317 bool high_precision_time
,
2318 const char* if_match
,
2319 const char* if_nomatch
,
2323 RGWObjCategory category
,
2325 boost::optional
<ceph::real_time
> delete_at
,
2326 std::string
* version_id
,
2329 void (*progress_cb
)(off_t
, void *),
2330 void* progress_data
,
2331 const DoutPrefixProvider
* dpp
,
2334 return store
->getRados()->copy_obj(*rados_ctx
,
2338 dest_object
->get_obj(),
2340 dest_bucket
->get_info(),
2341 src_bucket
->get_info(),
2347 high_precision_time
,
2350 static_cast<RGWRados::AttrsMod
>(attrs_mod
),
2355 (delete_at
? *delete_at
: real_time()),
2365 int RadosObject::RadosReadOp::iterate(const DoutPrefixProvider
* dpp
, int64_t ofs
, int64_t end
, RGWGetDataCB
* cb
, optional_yield y
)
2367 return parent_op
.iterate(dpp
, ofs
, end
, cb
, y
);
2370 int RadosObject::swift_versioning_restore(bool& restored
,
2371 const DoutPrefixProvider
* dpp
)
2373 rgw_obj obj
= get_obj();
2374 return store
->getRados()->swift_versioning_restore(*rados_ctx
,
2375 bucket
->get_owner()->get_id(),
2382 int RadosObject::swift_versioning_copy(const DoutPrefixProvider
* dpp
, optional_yield y
)
2384 return store
->getRados()->swift_versioning_copy(*rados_ctx
,
2385 bucket
->get_info().owner
,
2392 int RadosMultipartUpload::cleanup_part_history(const DoutPrefixProvider
* dpp
,
2394 RadosMultipartPart
*part
,
2395 list
<rgw_obj_index_key
>& remove_objs
)
2397 cls_rgw_obj_chain chain
;
2398 for (auto& ppfx
: part
->get_past_prefixes()) {
2400 past_obj
.init_ns(bucket
->get_key(), ppfx
+ "." + std::to_string(part
->info
.num
), mp_ns
);
2401 rgw_obj_index_key past_key
;
2402 past_obj
.key
.get_index_key(&past_key
);
2403 // Remove past upload part objects from index, too.
2404 remove_objs
.push_back(past_key
);
2406 RGWObjManifest manifest
= part
->get_manifest();
2407 manifest
.set_prefix(ppfx
);
2408 RGWObjManifest::obj_iterator miter
= manifest
.obj_begin(dpp
);
2409 for (; miter
!= manifest
.obj_end(dpp
); ++miter
) {
2410 rgw_raw_obj raw_part_obj
= miter
.get_location().get_raw_obj(store
->getRados());
2411 cls_rgw_obj_key
part_key(raw_part_obj
.oid
);
2412 chain
.push_obj(raw_part_obj
.pool
.to_str(), part_key
, raw_part_obj
.loc
);
2415 if (store
->getRados()->get_gc() == nullptr) {
2416 // Delete objects inline if gc hasn't been initialised (in case when bypass gc is specified)
2417 store
->getRados()->delete_objs_inline(dpp
, chain
, mp_obj
.get_upload_id());
2419 // use upload id as tag and do it synchronously
2420 auto [ret
, leftover_chain
] = store
->getRados()->send_chain_to_gc(chain
, mp_obj
.get_upload_id());
2421 if (ret
< 0 && leftover_chain
) {
2422 ldpp_dout(dpp
, 5) << __func__
<< ": gc->send_chain() returned " << ret
<< dendl
;
2423 if (ret
== -ENOENT
) {
2424 return -ERR_NO_SUCH_UPLOAD
;
2426 // Delete objects inline if send chain to gc fails
2427 store
->getRados()->delete_objs_inline(dpp
, *leftover_chain
, mp_obj
.get_upload_id());
2434 int RadosMultipartUpload::abort(const DoutPrefixProvider
*dpp
, CephContext
*cct
)
2436 std::unique_ptr
<rgw::sal::Object
> meta_obj
= get_meta_obj();
2437 meta_obj
->set_in_extra_data(true);
2438 meta_obj
->set_hash_source(mp_obj
.get_key());
2439 cls_rgw_obj_chain chain
;
2440 list
<rgw_obj_index_key
> remove_objs
;
2444 uint64_t parts_accounted_size
= 0;
2447 ret
= list_parts(dpp
, cct
, 1000, marker
, &marker
, &truncated
);
2449 ldpp_dout(dpp
, 20) << __func__
<< ": RadosMultipartUpload::list_parts returned " <<
2451 return (ret
== -ENOENT
) ? -ERR_NO_SUCH_UPLOAD
: ret
;
2454 for (auto part_it
= parts
.begin();
2455 part_it
!= parts
.end();
2457 RadosMultipartPart
* obj_part
= dynamic_cast<RadosMultipartPart
*>(part_it
->second
.get());
2458 if (obj_part
->info
.manifest
.empty()) {
2459 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(
2460 rgw_obj_key(obj_part
->oid
, std::string(), RGW_OBJ_NS_MULTIPART
));
2461 obj
->set_hash_source(mp_obj
.get_key());
2462 ret
= obj
->delete_object(dpp
, null_yield
);
2463 if (ret
< 0 && ret
!= -ENOENT
)
2466 auto target
= meta_obj
->get_obj();
2467 store
->getRados()->update_gc_chain(dpp
, target
, obj_part
->info
.manifest
, &chain
);
2468 RGWObjManifest::obj_iterator oiter
= obj_part
->info
.manifest
.obj_begin(dpp
);
2469 if (oiter
!= obj_part
->info
.manifest
.obj_end(dpp
)) {
2470 std::unique_ptr
<rgw::sal::Object
> head
= bucket
->get_object(rgw_obj_key());
2471 rgw_raw_obj raw_head
= oiter
.get_location().get_raw_obj(store
->getRados());
2472 dynamic_cast<rgw::sal::RadosObject
*>(head
.get())->raw_obj_to_obj(raw_head
);
2474 rgw_obj_index_key key
;
2475 head
->get_key().get_index_key(&key
);
2476 remove_objs
.push_back(key
);
2478 cleanup_part_history(dpp
, null_yield
, obj_part
, remove_objs
);
2481 parts_accounted_size
+= obj_part
->info
.accounted_size
;
2483 } while (truncated
);
2485 if (store
->getRados()->get_gc() == nullptr) {
2486 //Delete objects inline if gc hasn't been initialised (in case when bypass gc is specified)
2487 store
->getRados()->delete_objs_inline(dpp
, chain
, mp_obj
.get_upload_id());
2489 /* use upload id as tag and do it synchronously */
2490 auto [ret
, leftover_chain
] = store
->getRados()->send_chain_to_gc(chain
, mp_obj
.get_upload_id());
2491 if (ret
< 0 && leftover_chain
) {
2492 ldpp_dout(dpp
, 5) << __func__
<< ": gc->send_chain() returned " << ret
<< dendl
;
2493 if (ret
== -ENOENT
) {
2494 return -ERR_NO_SUCH_UPLOAD
;
2496 //Delete objects inline if send chain to gc fails
2497 store
->getRados()->delete_objs_inline(dpp
, *leftover_chain
, mp_obj
.get_upload_id());
2501 std::unique_ptr
<rgw::sal::Object::DeleteOp
> del_op
= meta_obj
->get_delete_op();
2502 del_op
->params
.bucket_owner
= bucket
->get_acl_owner();
2503 del_op
->params
.versioning_status
= 0;
2504 if (!remove_objs
.empty()) {
2505 del_op
->params
.remove_objs
= &remove_objs
;
2508 del_op
->params
.abortmp
= true;
2509 del_op
->params
.parts_accounted_size
= parts_accounted_size
;
2511 // and also remove the metadata obj
2512 ret
= del_op
->delete_obj(dpp
, null_yield
);
2514 ldpp_dout(dpp
, 20) << __func__
<< ": del_op.delete_obj returned " <<
2517 return (ret
== -ENOENT
) ? -ERR_NO_SUCH_UPLOAD
: ret
;
2520 std::unique_ptr
<rgw::sal::Object
> RadosMultipartUpload::get_meta_obj()
2522 return bucket
->get_object(rgw_obj_key(get_meta(), string(), mp_ns
));
2525 int RadosMultipartUpload::init(const DoutPrefixProvider
*dpp
, optional_yield y
, ACLOwner
& owner
, rgw_placement_rule
& dest_placement
, rgw::sal::Attrs
& attrs
)
2528 std::string oid
= mp_obj
.get_key();
2529 RGWObjectCtx
obj_ctx(store
);
2533 string tmp_obj_name
;
2534 std::unique_ptr
<rgw::sal::Object
> obj
;
2535 gen_rand_alphanumeric(store
->ctx(), buf
, sizeof(buf
) - 1);
2536 std::string upload_id
= MULTIPART_UPLOAD_ID_PREFIX
; /* v2 upload id */
2537 upload_id
.append(buf
);
2539 mp_obj
.init(oid
, upload_id
);
2540 tmp_obj_name
= mp_obj
.get_meta();
2542 obj
= bucket
->get_object(rgw_obj_key(tmp_obj_name
, string(), mp_ns
));
2543 // the meta object will be indexed with 0 size, we c
2544 obj
->set_in_extra_data(true);
2545 obj
->set_hash_source(oid
);
2547 RGWRados::Object
op_target(store
->getRados(),
2548 obj
->get_bucket()->get_info(),
2549 obj_ctx
, obj
->get_obj());
2550 RGWRados::Object::Write
obj_op(&op_target
);
2552 op_target
.set_versioning_disabled(true); /* no versioning for multipart meta */
2553 obj_op
.meta
.owner
= owner
.get_id();
2554 obj_op
.meta
.category
= RGWObjCategory::MultiMeta
;
2555 obj_op
.meta
.flags
= PUT_OBJ_CREATE_EXCL
;
2556 obj_op
.meta
.mtime
= &mtime
;
2558 multipart_upload_info upload_info
;
2559 upload_info
.dest_placement
= dest_placement
;
2562 encode(upload_info
, bl
);
2563 obj_op
.meta
.data
= &bl
;
2565 ret
= obj_op
.write_meta(dpp
, bl
.length(), 0, attrs
, y
);
2566 } while (ret
== -EEXIST
);
2571 int RadosMultipartUpload::list_parts(const DoutPrefixProvider
*dpp
, CephContext
*cct
,
2572 int num_parts
, int marker
,
2573 int *next_marker
, bool *truncated
,
2574 bool assume_unsorted
)
2576 map
<string
, bufferlist
> parts_map
;
2577 map
<string
, bufferlist
>::iterator iter
;
2579 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(
2580 rgw_obj_key(get_meta(), std::string(), RGW_OBJ_NS_MULTIPART
));
2581 obj
->set_in_extra_data(true);
2583 bool sorted_omap
= is_v2_upload_id(get_upload_id()) && !assume_unsorted
;
2593 snprintf(buf
, sizeof(buf
), "%08d", marker
);
2596 ret
= obj
->omap_get_vals(dpp
, p
, num_parts
+ 1, &parts_map
,
2597 nullptr, null_yield
);
2599 ret
= obj
->omap_get_all(dpp
, &parts_map
, null_yield
);
2608 uint32_t expected_next
= marker
+ 1;
2610 for (i
= 0, iter
= parts_map
.begin();
2611 (i
< num_parts
|| !sorted_omap
) && iter
!= parts_map
.end();
2613 bufferlist
& bl
= iter
->second
;
2614 auto bli
= bl
.cbegin();
2615 std::unique_ptr
<RadosMultipartPart
> part
= std::make_unique
<RadosMultipartPart
>();
2617 decode(part
->info
, bli
);
2618 } catch (buffer::error
& err
) {
2619 ldpp_dout(dpp
, 0) << "ERROR: could not part info, caught buffer::error" <<
2624 if (part
->info
.num
!= expected_next
) {
2625 /* ouch, we expected a specific part num here, but we got a
2626 * different one. Either a part is missing, or it could be a
2627 * case of mixed rgw versions working on the same upload,
2628 * where one gateway doesn't support correctly sorted omap
2629 * keys for multipart upload just assume data is unsorted.
2631 return list_parts(dpp
, cct
, num_parts
, marker
, next_marker
, truncated
, true);
2636 (int)part
->info
.num
> marker
) {
2637 last_num
= part
->info
.num
;
2638 parts
[part
->info
.num
] = std::move(part
);
2644 *truncated
= (iter
!= parts_map
.end());
2647 /* rebuild a map with only num_parts entries */
2648 std::map
<uint32_t, std::unique_ptr
<MultipartPart
>> new_parts
;
2649 std::map
<uint32_t, std::unique_ptr
<MultipartPart
>>::iterator piter
;
2650 for (i
= 0, piter
= parts
.begin();
2651 i
< num_parts
&& piter
!= parts
.end();
2653 last_num
= piter
->first
;
2654 new_parts
[piter
->first
] = std::move(piter
->second
);
2658 *truncated
= (piter
!= parts
.end());
2661 parts
.swap(new_parts
);
2665 *next_marker
= last_num
;
2671 int RadosMultipartUpload::complete(const DoutPrefixProvider
*dpp
,
2672 optional_yield y
, CephContext
* cct
,
2673 map
<int, string
>& part_etags
,
2674 list
<rgw_obj_index_key
>& remove_objs
,
2675 uint64_t& accounted_size
, bool& compressed
,
2676 RGWCompressionInfo
& cs_info
, off_t
& ofs
,
2677 std::string
& tag
, ACLOwner
& owner
,
2679 rgw::sal::Object
* target_obj
)
2681 char final_etag
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
2682 char final_etag_str
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 16];
2686 // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes
2687 hash
.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
);
2691 int total_parts
= 0;
2692 int handled_parts
= 0;
2693 int max_parts
= 1000;
2695 uint64_t min_part_size
= cct
->_conf
->rgw_multipart_min_part_size
;
2696 auto etags_iter
= part_etags
.begin();
2697 rgw::sal::Attrs attrs
= target_obj
->get_attrs();
2700 ret
= list_parts(dpp
, cct
, max_parts
, marker
, &marker
, &truncated
);
2701 if (ret
== -ENOENT
) {
2702 ret
= -ERR_NO_SUCH_UPLOAD
;
2707 total_parts
+= parts
.size();
2708 if (!truncated
&& total_parts
!= (int)part_etags
.size()) {
2709 ldpp_dout(dpp
, 0) << "NOTICE: total parts mismatch: have: " << total_parts
2710 << " expected: " << part_etags
.size() << dendl
;
2711 ret
= -ERR_INVALID_PART
;
2715 for (auto obj_iter
= parts
.begin(); etags_iter
!= part_etags
.end() && obj_iter
!= parts
.end(); ++etags_iter
, ++obj_iter
, ++handled_parts
) {
2716 RadosMultipartPart
* part
= dynamic_cast<rgw::sal::RadosMultipartPart
*>(obj_iter
->second
.get());
2717 uint64_t part_size
= part
->get_size();
2718 if (handled_parts
< (int)part_etags
.size() - 1 &&
2719 part_size
< min_part_size
) {
2720 ret
= -ERR_TOO_SMALL
;
2724 char petag
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
2725 if (etags_iter
->first
!= (int)obj_iter
->first
) {
2726 ldpp_dout(dpp
, 0) << "NOTICE: parts num mismatch: next requested: "
2727 << etags_iter
->first
<< " next uploaded: "
2728 << obj_iter
->first
<< dendl
;
2729 ret
= -ERR_INVALID_PART
;
2732 string part_etag
= rgw_string_unquote(etags_iter
->second
);
2733 if (part_etag
.compare(part
->get_etag()) != 0) {
2734 ldpp_dout(dpp
, 0) << "NOTICE: etag mismatch: part: " << etags_iter
->first
2735 << " etag: " << etags_iter
->second
<< dendl
;
2736 ret
= -ERR_INVALID_PART
;
2740 hex_to_buf(part
->get_etag().c_str(), petag
,
2741 CEPH_CRYPTO_MD5_DIGESTSIZE
);
2742 hash
.Update((const unsigned char *)petag
, sizeof(petag
));
2744 RGWUploadPartInfo
& obj_part
= part
->info
;
2746 /* update manifest for part */
2747 string oid
= mp_obj
.get_part(part
->info
.num
);
2749 src_obj
.init_ns(bucket
->get_key(), oid
, mp_ns
);
2751 if (obj_part
.manifest
.empty()) {
2752 ldpp_dout(dpp
, 0) << "ERROR: empty manifest for object part: obj="
2753 << src_obj
<< dendl
;
2754 ret
= -ERR_INVALID_PART
;
2757 manifest
.append(dpp
, obj_part
.manifest
, store
->svc()->zone
->get_zonegroup(), store
->svc()->zone
->get_zone_params());
2758 auto manifest_prefix
= part
->info
.manifest
.get_prefix();
2759 if (not manifest_prefix
.empty()) {
2760 // It has an explicit prefix. Override the default one.
2761 src_obj
.init_ns(bucket
->get_key(), manifest_prefix
+ "." + std::to_string(part
->info
.num
), mp_ns
);
2765 bool part_compressed
= (obj_part
.cs_info
.compression_type
!= "none");
2766 if ((handled_parts
> 0) &&
2767 ((part_compressed
!= compressed
) ||
2768 (cs_info
.compression_type
!= obj_part
.cs_info
.compression_type
))) {
2769 ldpp_dout(dpp
, 0) << "ERROR: compression type was changed during multipart upload ("
2770 << cs_info
.compression_type
<< ">>" << obj_part
.cs_info
.compression_type
<< ")" << dendl
;
2771 ret
= -ERR_INVALID_PART
;
2775 if (part_compressed
) {
2776 int64_t new_ofs
; // offset in compression data for new part
2777 if (cs_info
.blocks
.size() > 0)
2778 new_ofs
= cs_info
.blocks
.back().new_ofs
+ cs_info
.blocks
.back().len
;
2781 for (const auto& block
: obj_part
.cs_info
.blocks
) {
2782 compression_block cb
;
2783 cb
.old_ofs
= block
.old_ofs
+ cs_info
.orig_size
;
2784 cb
.new_ofs
= new_ofs
;
2786 cs_info
.blocks
.push_back(cb
);
2787 new_ofs
= cb
.new_ofs
+ cb
.len
;
2790 cs_info
.compression_type
= obj_part
.cs_info
.compression_type
;
2791 cs_info
.orig_size
+= obj_part
.cs_info
.orig_size
;
2795 rgw_obj_index_key remove_key
;
2796 src_obj
.key
.get_index_key(&remove_key
);
2798 remove_objs
.push_back(remove_key
);
2800 cleanup_part_history(dpp
, y
, part
, remove_objs
);
2802 ofs
+= obj_part
.size
;
2803 accounted_size
+= obj_part
.accounted_size
;
2805 } while (truncated
);
2806 hash
.Final((unsigned char *)final_etag
);
2808 buf_to_hex((unsigned char *)final_etag
, sizeof(final_etag
), final_etag_str
);
2809 snprintf(&final_etag_str
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2],
2810 sizeof(final_etag_str
) - CEPH_CRYPTO_MD5_DIGESTSIZE
* 2,
2811 "-%lld", (long long)part_etags
.size());
2812 etag
= final_etag_str
;
2813 ldpp_dout(dpp
, 10) << "calculated etag: " << etag
<< dendl
;
2815 etag_bl
.append(etag
);
2817 attrs
[RGW_ATTR_ETAG
] = etag_bl
;
2820 // write compression attribute to full object
2822 encode(cs_info
, tmp
);
2823 attrs
[RGW_ATTR_COMPRESSION
] = tmp
;
2826 target_obj
->set_atomic();
2828 RGWRados::Object
op_target(store
->getRados(),
2829 target_obj
->get_bucket()->get_info(),
2830 dynamic_cast<RadosObject
*>(target_obj
)->get_ctx(),
2831 target_obj
->get_obj());
2832 RGWRados::Object::Write
obj_op(&op_target
);
2834 obj_op
.meta
.manifest
= &manifest
;
2835 obj_op
.meta
.remove_objs
= &remove_objs
;
2837 obj_op
.meta
.ptag
= &tag
; /* use req_id as operation tag */
2838 obj_op
.meta
.owner
= owner
.get_id();
2839 obj_op
.meta
.flags
= PUT_OBJ_CREATE
;
2840 obj_op
.meta
.modify_tail
= true;
2841 obj_op
.meta
.completeMultipart
= true;
2842 obj_op
.meta
.olh_epoch
= olh_epoch
;
2844 ret
= obj_op
.write_meta(dpp
, ofs
, accounted_size
, attrs
, y
);
2851 int RadosMultipartUpload::get_info(const DoutPrefixProvider
*dpp
, optional_yield y
, rgw_placement_rule
** rule
, rgw::sal::Attrs
* attrs
)
2853 if (!rule
&& !attrs
) {
2858 if (!placement
.empty()) {
2861 /* Don't need attrs, done */
2869 /* We need either attributes or placement, so we need a read */
2870 std::unique_ptr
<rgw::sal::Object
> meta_obj
;
2871 meta_obj
= get_meta_obj();
2872 meta_obj
->set_in_extra_data(true);
2874 multipart_upload_info upload_info
;
2877 /* Read the obj head which contains the multipart_upload_info */
2878 std::unique_ptr
<rgw::sal::Object::ReadOp
> read_op
= meta_obj
->get_read_op();
2879 meta_obj
->set_prefetch_data();
2881 int ret
= read_op
->prepare(y
, dpp
);
2883 if (ret
== -ENOENT
) {
2884 return -ERR_NO_SUCH_UPLOAD
;
2889 extract_span_context(meta_obj
->get_attrs(), trace_ctx
);
2892 /* Attrs are filled in by prepare */
2893 *attrs
= meta_obj
->get_attrs();
2894 if (!rule
|| *rule
!= nullptr) {
2895 /* placement was cached; don't actually read */
2900 /* Now read the placement from the head */
2901 ret
= read_op
->read(0, store
->ctx()->_conf
->rgw_max_chunk_size
, headbl
, y
, dpp
);
2903 if (ret
== -ENOENT
) {
2904 return -ERR_NO_SUCH_UPLOAD
;
2909 if (headbl
.length() <= 0) {
2910 return -ERR_NO_SUCH_UPLOAD
;
2913 /* Decode multipart_upload_info */
2914 auto hiter
= headbl
.cbegin();
2916 decode(upload_info
, hiter
);
2917 } catch (buffer::error
& err
) {
2918 ldpp_dout(dpp
, 0) << "ERROR: failed to decode multipart upload info" << dendl
;
2921 placement
= upload_info
.dest_placement
;
2927 std::unique_ptr
<Writer
> RadosMultipartUpload::get_writer(
2928 const DoutPrefixProvider
*dpp
,
2930 rgw::sal::Object
* obj
,
2931 const rgw_user
& owner
,
2932 const rgw_placement_rule
*ptail_placement_rule
,
2934 const std::string
& part_num_str
)
2936 RGWBucketInfo
& bucket_info
= obj
->get_bucket()->get_info();
2937 RGWObjectCtx
& obj_ctx
= static_cast<RadosObject
*>(obj
)->get_ctx();
2938 auto aio
= rgw::make_throttle(store
->ctx()->_conf
->rgw_put_obj_min_window_size
, y
);
2939 return std::make_unique
<RadosMultipartWriter
>(dpp
, y
, get_upload_id(),
2940 bucket_info
, obj_ctx
,
2941 obj
->get_obj(), store
, std::move(aio
), owner
,
2942 ptail_placement_rule
, part_num
, part_num_str
);
2945 MPRadosSerializer::MPRadosSerializer(const DoutPrefixProvider
*dpp
, RadosStore
* store
, RadosObject
* obj
, const std::string
& lock_name
) :
2949 rgw_raw_obj raw_obj
;
2951 obj
->get_raw_obj(&raw_obj
);
2953 store
->getRados()->get_obj_data_pool(obj
->get_bucket()->get_placement_rule(),
2954 obj
->get_obj(), &meta_pool
);
2955 store
->getRados()->open_pool_ctx(dpp
, meta_pool
, ioctx
, true, true);
2958 int MPRadosSerializer::try_lock(const DoutPrefixProvider
*dpp
, utime_t dur
, optional_yield y
)
2961 lock
.set_duration(dur
);
2962 lock
.lock_exclusive(&op
);
2963 int ret
= rgw_rados_operate(dpp
, ioctx
, oid
, &op
, y
);
2970 LCRadosSerializer::LCRadosSerializer(RadosStore
* store
, const std::string
& _oid
, const std::string
& lock_name
, const std::string
& cookie
) :
2971 StoreLCSerializer(_oid
),
2974 ioctx
= &store
->getRados()->lc_pool_ctx
;
2975 lock
.set_cookie(cookie
);
2978 int LCRadosSerializer::try_lock(const DoutPrefixProvider
*dpp
, utime_t dur
, optional_yield y
)
2980 lock
.set_duration(dur
);
2981 return lock
.lock_exclusive(ioctx
, oid
);
2984 int RadosLifecycle::get_entry(const std::string
& oid
, const std::string
& marker
,
2985 std::unique_ptr
<LCEntry
>* entry
)
2987 cls_rgw_lc_entry cls_entry
;
2988 int ret
= cls_rgw_lc_get_entry(*store
->getRados()->get_lc_pool_ctx(), oid
, marker
, cls_entry
);
2993 e
= new StoreLCEntry(cls_entry
.bucket
, cls_entry
.start_time
, cls_entry
.status
);
3001 int RadosLifecycle::get_next_entry(const std::string
& oid
, const std::string
& marker
,
3002 std::unique_ptr
<LCEntry
>* entry
)
3004 cls_rgw_lc_entry cls_entry
;
3005 int ret
= cls_rgw_lc_get_next_entry(*store
->getRados()->get_lc_pool_ctx(), oid
, marker
,
3012 e
= new StoreLCEntry(cls_entry
.bucket
, cls_entry
.start_time
, cls_entry
.status
);
3020 int RadosLifecycle::set_entry(const std::string
& oid
, LCEntry
& entry
)
3022 cls_rgw_lc_entry cls_entry
;
3024 cls_entry
.bucket
= entry
.get_bucket();
3025 cls_entry
.start_time
= entry
.get_start_time();
3026 cls_entry
.status
= entry
.get_status();
3028 return cls_rgw_lc_set_entry(*store
->getRados()->get_lc_pool_ctx(), oid
, cls_entry
);
3031 int RadosLifecycle::list_entries(const std::string
& oid
, const std::string
& marker
,
3032 uint32_t max_entries
, std::vector
<std::unique_ptr
<LCEntry
>>& entries
)
3036 vector
<cls_rgw_lc_entry
> cls_entries
;
3037 int ret
= cls_rgw_lc_list(*store
->getRados()->get_lc_pool_ctx(), oid
, marker
, max_entries
, cls_entries
);
3042 for (auto& entry
: cls_entries
) {
3043 entries
.push_back(std::make_unique
<StoreLCEntry
>(entry
.bucket
, oid
,
3044 entry
.start_time
, entry
.status
));
3050 int RadosLifecycle::rm_entry(const std::string
& oid
, LCEntry
& entry
)
3052 cls_rgw_lc_entry cls_entry
;
3054 cls_entry
.bucket
= entry
.get_bucket();
3055 cls_entry
.start_time
= entry
.get_start_time();
3056 cls_entry
.status
= entry
.get_status();
3058 return cls_rgw_lc_rm_entry(*store
->getRados()->get_lc_pool_ctx(), oid
, cls_entry
);
3061 int RadosLifecycle::get_head(const std::string
& oid
, std::unique_ptr
<LCHead
>* head
)
3063 cls_rgw_lc_obj_head cls_head
;
3064 int ret
= cls_rgw_lc_get_head(*store
->getRados()->get_lc_pool_ctx(), oid
, cls_head
);
3069 h
= new StoreLCHead(cls_head
.start_date
, cls_head
.shard_rollover_date
, cls_head
.marker
);
3077 int RadosLifecycle::put_head(const std::string
& oid
, LCHead
& head
)
3079 cls_rgw_lc_obj_head cls_head
;
3081 cls_head
.marker
= head
.get_marker();
3082 cls_head
.start_date
= head
.get_start_date();
3083 cls_head
.shard_rollover_date
= head
.get_shard_rollover_date();
3085 return cls_rgw_lc_put_head(*store
->getRados()->get_lc_pool_ctx(), oid
, cls_head
);
3088 std::unique_ptr
<LCSerializer
> RadosLifecycle::get_serializer(const std::string
& lock_name
,
3089 const std::string
& oid
,
3090 const std::string
& cookie
)
3092 return std::make_unique
<LCRadosSerializer
>(store
, oid
, lock_name
, cookie
);
3095 int RadosNotification::publish_reserve(const DoutPrefixProvider
*dpp
, RGWObjTags
* obj_tags
)
3097 return rgw::notify::publish_reserve(dpp
, event_type
, res
, obj_tags
);
3100 int RadosNotification::publish_commit(const DoutPrefixProvider
* dpp
, uint64_t size
,
3101 const ceph::real_time
& mtime
, const std::string
& etag
, const std::string
& version
)
3103 return rgw::notify::publish_commit(obj
, size
, mtime
, etag
, version
, event_type
, res
, dpp
);
3106 int RadosAtomicWriter::prepare(optional_yield y
)
3108 return processor
.prepare(y
);
3111 int RadosAtomicWriter::process(bufferlist
&& data
, uint64_t offset
)
3113 return processor
.process(std::move(data
), offset
);
3116 int RadosAtomicWriter::complete(size_t accounted_size
, const std::string
& etag
,
3117 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
3118 std::map
<std::string
, bufferlist
>& attrs
,
3119 ceph::real_time delete_at
,
3120 const char *if_match
, const char *if_nomatch
,
3121 const std::string
*user_data
,
3122 rgw_zone_set
*zones_trace
, bool *canceled
,
3125 return processor
.complete(accounted_size
, etag
, mtime
, set_mtime
, attrs
, delete_at
,
3126 if_match
, if_nomatch
, user_data
, zones_trace
, canceled
, y
);
3129 int RadosAppendWriter::prepare(optional_yield y
)
3131 return processor
.prepare(y
);
3134 int RadosAppendWriter::process(bufferlist
&& data
, uint64_t offset
)
3136 return processor
.process(std::move(data
), offset
);
3139 int RadosAppendWriter::complete(size_t accounted_size
, const std::string
& etag
,
3140 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
3141 std::map
<std::string
, bufferlist
>& attrs
,
3142 ceph::real_time delete_at
,
3143 const char *if_match
, const char *if_nomatch
,
3144 const std::string
*user_data
,
3145 rgw_zone_set
*zones_trace
, bool *canceled
,
3148 return processor
.complete(accounted_size
, etag
, mtime
, set_mtime
, attrs
, delete_at
,
3149 if_match
, if_nomatch
, user_data
, zones_trace
, canceled
, y
);
3152 int RadosMultipartWriter::prepare(optional_yield y
)
3154 return processor
.prepare(y
);
3157 int RadosMultipartWriter::process(bufferlist
&& data
, uint64_t offset
)
3159 return processor
.process(std::move(data
), offset
);
3162 int RadosMultipartWriter::complete(size_t accounted_size
, const std::string
& etag
,
3163 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
3164 std::map
<std::string
, bufferlist
>& attrs
,
3165 ceph::real_time delete_at
,
3166 const char *if_match
, const char *if_nomatch
,
3167 const std::string
*user_data
,
3168 rgw_zone_set
*zones_trace
, bool *canceled
,
3171 return processor
.complete(accounted_size
, etag
, mtime
, set_mtime
, attrs
, delete_at
,
3172 if_match
, if_nomatch
, user_data
, zones_trace
, canceled
, y
);
3175 const std::string
& RadosZoneGroup::get_endpoint() const
3177 if (!group
.endpoints
.empty()) {
3178 return group
.endpoints
.front();
3180 // use zonegroup's master zone endpoints
3181 auto z
= group
.zones
.find(group
.master_zone
);
3182 if (z
!= group
.zones
.end() && !z
->second
.endpoints
.empty()) {
3183 return z
->second
.endpoints
.front();
3189 bool RadosZoneGroup::placement_target_exists(std::string
& target
) const
3191 return !!group
.placement_targets
.count(target
);
3194 int RadosZoneGroup::get_placement_target_names(std::set
<std::string
>& names
) const
3196 for (const auto& target
: group
.placement_targets
) {
3197 names
.emplace(target
.second
.name
);
3203 int RadosZoneGroup::get_placement_tier(const rgw_placement_rule
& rule
,
3204 std::unique_ptr
<PlacementTier
>* tier
)
3206 std::map
<std::string
, RGWZoneGroupPlacementTarget
>::const_iterator titer
;
3207 titer
= group
.placement_targets
.find(rule
.name
);
3208 if (titer
== group
.placement_targets
.end()) {
3212 const auto& target_rule
= titer
->second
;
3213 std::map
<std::string
, RGWZoneGroupPlacementTier
>::const_iterator ttier
;
3214 ttier
= target_rule
.tier_targets
.find(rule
.storage_class
);
3215 if (ttier
== target_rule
.tier_targets
.end()) {
3221 t
= new RadosPlacementTier(store
, ttier
->second
);
3229 int RadosZoneGroup::get_zone_by_id(const std::string
& id
, std::unique_ptr
<Zone
>* zone
)
3231 RGWZone
* rz
= store
->svc()->zone
->find_zone(id
);
3235 Zone
* z
= new RadosZone(store
, clone(), *rz
);
3240 int RadosZoneGroup::get_zone_by_name(const std::string
& name
, std::unique_ptr
<Zone
>* zone
)
3243 int ret
= store
->svc()->zone
->find_zone_id_by_name(name
, &id
);
3247 RGWZone
* rz
= store
->svc()->zone
->find_zone(id
.id
);
3251 Zone
* z
= new RadosZone(store
, clone(), *rz
);
3256 int RadosZoneGroup::list_zones(std::list
<std::string
>& zone_ids
)
3258 for (const auto& entry
: group
.zones
)
3260 zone_ids
.push_back(entry
.second
.id
);
3265 std::unique_ptr
<Zone
> RadosZone::clone()
3268 return std::make_unique
<RadosZone
>(store
, group
->clone());
3270 return std::make_unique
<RadosZone
>(store
, group
->clone(), rgw_zone
);
3273 const std::string
& RadosZone::get_id()
3276 return store
->svc()->zone
->zone_id().id
;
3281 const std::string
& RadosZone::get_name() const
3284 return store
->svc()->zone
->zone_name();
3286 return rgw_zone
.name
;
3289 bool RadosZone::is_writeable()
3292 return store
->svc()->zone
->zone_is_writeable();
3294 return !rgw_zone
.read_only
;
3297 bool RadosZone::get_redirect_endpoint(std::string
* endpoint
)
3300 return store
->svc()->zone
->get_redirect_zone_endpoint(endpoint
);
3302 endpoint
= &rgw_zone
.redirect_zone
;
3306 bool RadosZone::has_zonegroup_api(const std::string
& api
) const
3308 return store
->svc()->zone
->has_zonegroup_api(api
);
3311 const std::string
& RadosZone::get_current_period_id()
3313 return store
->svc()->zone
->get_current_period_id();
3316 const RGWAccessKey
& RadosZone::get_system_key()
3318 return store
->svc()->zone
->get_zone_params().system_key
;
3321 const std::string
& RadosZone::get_realm_name()
3323 return store
->svc()->zone
->get_realm().get_name();
3326 const std::string
& RadosZone::get_realm_id()
3328 return store
->svc()->zone
->get_realm().get_id();
3331 const std::string_view
RadosZone::get_tier_type()
3334 return store
->svc()->zone
->get_zone().tier_type
;
3339 RGWBucketSyncPolicyHandlerRef
RadosZone::get_sync_policy_handler()
3341 return store
->svc()->zone
->get_sync_policy_handler(get_id());
3344 RadosLuaManager::RadosLuaManager(RadosStore
* _s
) :
3346 pool((store
->svc() && store
->svc()->zone
) ? store
->svc()->zone
->get_zone_params().log_pool
: rgw_pool())
3349 int RadosLuaManager::get_script(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
, std::string
& script
)
3352 ldpp_dout(dpp
, 10) << "WARNING: missing pool when reading lua script " << dendl
;
3357 int r
= rgw_get_system_obj(store
->svc()->sysobj
, pool
, key
, bl
, nullptr, nullptr, y
, dpp
);
3362 auto iter
= bl
.cbegin();
3364 ceph::decode(script
, iter
);
3365 } catch (buffer::error
& err
) {
3372 int RadosLuaManager::put_script(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
, const std::string
& script
)
3375 ldpp_dout(dpp
, 10) << "WARNING: missing pool when writing lua script " << dendl
;
3379 ceph::encode(script
, bl
);
3381 int r
= rgw_put_system_obj(dpp
, store
->svc()->sysobj
, pool
, key
, bl
, false, nullptr, real_time(), y
);
3389 int RadosLuaManager::del_script(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
)
3392 ldpp_dout(dpp
, 10) << "WARNING: missing pool when deleting lua script " << dendl
;
3395 int r
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, key
, nullptr, y
);
3396 if (r
< 0 && r
!= -ENOENT
) {
3403 const std::string PACKAGE_LIST_OBJECT_NAME
= "lua_package_allowlist";
3405 int RadosLuaManager::add_package(const DoutPrefixProvider
*dpp
, optional_yield y
, const std::string
& package_name
)
3407 // add package to list
3408 const bufferlist empty_bl
;
3409 std::map
<std::string
, bufferlist
> new_package
{{package_name
, empty_bl
}};
3410 librados::ObjectWriteOperation op
;
3411 op
.omap_set(new_package
);
3412 auto ret
= rgw_rados_operate(dpp
, *(store
->getRados()->get_lc_pool_ctx()),
3413 PACKAGE_LIST_OBJECT_NAME
, &op
, y
);
3421 int RadosLuaManager::remove_package(const DoutPrefixProvider
*dpp
, optional_yield y
, const std::string
& package_name
)
3423 librados::ObjectWriteOperation op
;
3424 size_t pos
= package_name
.find(" ");
3425 if (pos
!= package_name
.npos
) {
3426 // remove specfic version of the the package
3427 op
.omap_rm_keys(std::set
<std::string
>({package_name
}));
3428 auto ret
= rgw_rados_operate(dpp
, *(store
->getRados()->get_lc_pool_ctx()),
3429 PACKAGE_LIST_OBJECT_NAME
, &op
, y
);
3435 // otherwise, remove any existing versions of the package
3436 rgw::lua::packages_t packages
;
3437 auto ret
= list_packages(dpp
, y
, packages
);
3438 if (ret
< 0 && ret
!= -ENOENT
) {
3441 for(const auto& package
: packages
) {
3442 const std::string package_no_version
= package
.substr(0, package
.find(" "));
3443 if (package_no_version
.compare(package_name
) == 0) {
3444 op
.omap_rm_keys(std::set
<std::string
>({package
}));
3445 ret
= rgw_rados_operate(dpp
, *(store
->getRados()->get_lc_pool_ctx()),
3446 PACKAGE_LIST_OBJECT_NAME
, &op
, y
);
3455 int RadosLuaManager::list_packages(const DoutPrefixProvider
*dpp
, optional_yield y
, rgw::lua::packages_t
& packages
)
3457 constexpr auto max_chunk
= 1024U;
3458 std::string start_after
;
3462 librados::ObjectReadOperation op
;
3463 rgw::lua::packages_t packages_chunk
;
3464 op
.omap_get_keys2(start_after
, max_chunk
, &packages_chunk
, &more
, &rval
);
3465 const auto ret
= rgw_rados_operate(dpp
, *(store
->getRados()->get_lc_pool_ctx()),
3466 PACKAGE_LIST_OBJECT_NAME
, &op
, nullptr, y
);
3472 packages
.merge(packages_chunk
);
3478 int RadosOIDCProvider::store_url(const DoutPrefixProvider
*dpp
, const std::string
& url
, bool exclusive
, optional_yield y
)
3480 auto sysobj
= store
->svc()->sysobj
;
3481 std::string oid
= tenant
+ get_url_oid_prefix() + url
;
3486 return rgw_put_system_obj(dpp
, sysobj
, store
->svc()->zone
->get_zone_params().oidc_pool
, oid
, bl
, exclusive
, nullptr, real_time(), y
);
3489 int RadosOIDCProvider::read_url(const DoutPrefixProvider
*dpp
, const std::string
& url
, const std::string
& tenant
)
3491 auto sysobj
= store
->svc()->sysobj
;
3492 auto& pool
= store
->svc()->zone
->get_zone_params().oidc_pool
;
3493 std::string oid
= tenant
+ get_url_oid_prefix() + url
;
3496 int ret
= rgw_get_system_obj(sysobj
, pool
, oid
, bl
, nullptr, nullptr, null_yield
, dpp
);
3503 auto iter
= bl
.cbegin();
3504 decode(*this, iter
);
3505 } catch (buffer::error
& err
) {
3506 ldpp_dout(dpp
, 0) << "ERROR: failed to decode oidc provider info from pool: " << pool
.name
<<
3507 ": " << url
<< dendl
;
3514 int RadosOIDCProvider::delete_obj(const DoutPrefixProvider
*dpp
, optional_yield y
)
3516 auto& pool
= store
->svc()->zone
->get_zone_params().oidc_pool
;
3518 std::string url
, tenant
;
3519 auto ret
= get_tenant_url_from_arn(tenant
, url
);
3521 ldpp_dout(dpp
, 0) << "ERROR: failed to parse arn" << dendl
;
3525 if (this->tenant
!= tenant
) {
3526 ldpp_dout(dpp
, 0) << "ERROR: tenant in arn doesn't match that of user " << this->tenant
<< ", "
3527 << tenant
<< ": " << dendl
;
3532 std::string oid
= tenant
+ get_url_oid_prefix() + url
;
3533 ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3535 ldpp_dout(dpp
, 0) << "ERROR: deleting oidc url from pool: " << pool
.name
<< ": "
3536 << provider_url
<< ": " << cpp_strerror(-ret
) << dendl
;
3542 int RadosRole::store_info(const DoutPrefixProvider
*dpp
, bool exclusive
, optional_yield y
)
3550 encode(this->info
, bl
);
3552 if (!this->info
.tags
.empty()) {
3554 encode(this->info
.tags
, bl_tags
);
3555 map
<string
, bufferlist
> attrs
;
3556 attrs
.emplace("tagging", bl_tags
);
3558 RGWSI_MBSObj_PutParams
params(bl
, &attrs
, info
.mtime
, exclusive
);
3559 std::unique_ptr
<RGWSI_MetaBackend::Context
> ctx(store
->svc()->role
->svc
.meta_be
->alloc_ctx());
3560 ctx
->init(store
->svc()->role
->get_be_handler());
3561 return store
->svc()->role
->svc
.meta_be
->put(ctx
.get(), oid
, params
, &info
.objv_tracker
, y
, dpp
);
3563 RGWSI_MBSObj_PutParams
params(bl
, nullptr, info
.mtime
, exclusive
);
3564 std::unique_ptr
<RGWSI_MetaBackend::Context
> ctx(store
->svc()->role
->svc
.meta_be
->alloc_ctx());
3565 ctx
->init(store
->svc()->role
->get_be_handler());
3566 return store
->svc()->role
->svc
.meta_be
->put(ctx
.get(), oid
, params
, &info
.objv_tracker
, y
, dpp
);
3570 int RadosRole::store_name(const DoutPrefixProvider
*dpp
, bool exclusive
, optional_yield y
)
3572 auto sysobj
= store
->svc()->sysobj
;
3573 RGWNameToId nameToId
;
3574 nameToId
.obj_id
= info
.id
;
3576 std::string oid
= info
.tenant
+ get_names_oid_prefix() + info
.name
;
3580 encode(nameToId
, bl
);
3582 return rgw_put_system_obj(dpp
, sysobj
, store
->svc()->zone
->get_zone_params().roles_pool
, oid
, bl
, exclusive
, &info
.objv_tracker
, real_time(), y
);
3585 int RadosRole::store_path(const DoutPrefixProvider
*dpp
, bool exclusive
, optional_yield y
)
3587 auto sysobj
= store
->svc()->sysobj
;
3588 std::string oid
= info
.tenant
+ get_path_oid_prefix() + info
.path
+ get_info_oid_prefix() + info
.id
;
3592 return rgw_put_system_obj(dpp
, sysobj
, store
->svc()->zone
->get_zone_params().roles_pool
, oid
, bl
, exclusive
, &info
.objv_tracker
, real_time(), y
);
3595 int RadosRole::read_id(const DoutPrefixProvider
*dpp
, const std::string
& role_name
, const std::string
& tenant
, std::string
& role_id
, optional_yield y
)
3597 auto sysobj
= store
->svc()->sysobj
;
3598 std::string oid
= info
.tenant
+ get_names_oid_prefix() + role_name
;
3601 int ret
= rgw_get_system_obj(sysobj
, store
->svc()->zone
->get_zone_params().roles_pool
, oid
, bl
, nullptr, nullptr, y
, dpp
);
3606 RGWNameToId nameToId
;
3608 auto iter
= bl
.cbegin();
3610 decode(nameToId
, iter
);
3611 } catch (buffer::error
& err
) {
3612 ldpp_dout(dpp
, 0) << "ERROR: failed to decode role from Role pool: " << role_name
<< dendl
;
3615 role_id
= nameToId
.obj_id
;
3619 int RadosRole::read_name(const DoutPrefixProvider
*dpp
, optional_yield y
)
3621 auto sysobj
= store
->svc()->sysobj
;
3622 std::string oid
= info
.tenant
+ get_names_oid_prefix() + info
.name
;
3625 int ret
= rgw_get_system_obj(sysobj
, store
->svc()->zone
->get_zone_params().roles_pool
, oid
, bl
, nullptr, nullptr, y
, dpp
);
3627 ldpp_dout(dpp
, 0) << "ERROR: failed reading role name from Role pool: " << info
.name
<<
3628 ": " << cpp_strerror(-ret
) << dendl
;
3632 RGWNameToId nameToId
;
3635 auto iter
= bl
.cbegin();
3636 decode(nameToId
, iter
);
3637 } catch (buffer::error
& err
) {
3638 ldpp_dout(dpp
, 0) << "ERROR: failed to decode role name from Role pool: " << info
.name
<< dendl
;
3641 info
.id
= nameToId
.obj_id
;
3645 int RadosRole::read_info(const DoutPrefixProvider
*dpp
, optional_yield y
)
3650 ldpp_dout(dpp
, 20) << "INFO: oid in read_info is: " << oid
<< dendl
;
3654 RGWSI_MBSObj_GetParams
params(&bl
, &info
.attrs
, &info
.mtime
);
3655 std::unique_ptr
<RGWSI_MetaBackend::Context
> ctx(store
->svc()->role
->svc
.meta_be
->alloc_ctx());
3656 ctx
->init(store
->svc()->role
->get_be_handler());
3657 int ret
= store
->svc()->role
->svc
.meta_be
->get(ctx
.get(), oid
, params
, &info
.objv_tracker
, y
, dpp
, true);
3659 ldpp_dout(dpp
, 0) << "ERROR: failed reading role info from Role pool: " << info
.id
<< ": " << cpp_strerror(-ret
) << dendl
;
3665 auto iter
= bl
.cbegin();
3666 decode(this->info
, iter
);
3667 } catch (buffer::error
& err
) {
3668 ldpp_dout(dpp
, 0) << "ERROR: failed to decode role info from Role pool: " << info
.id
<< dendl
;
3672 auto it
= info
.attrs
.find("tagging");
3673 if (it
!= info
.attrs
.end()) {
3674 bufferlist bl_tags
= it
->second
;
3677 auto iter
= bl_tags
.cbegin();
3678 decode(info
.tags
, iter
);
3679 } catch (buffer::error
& err
) {
3680 ldpp_dout(dpp
, 0) << "ERROR: failed to decode attrs" << info
.id
<< dendl
;
3688 int RadosRole::create(const DoutPrefixProvider
*dpp
, bool exclusive
, const std::string
& role_id
, optional_yield y
)
3692 if (! validate_input(dpp
)) {
3696 if (!role_id
.empty()) {
3700 /* check to see the name is not used */
3701 ret
= read_id(dpp
, info
.name
, info
.tenant
, info
.id
, y
);
3702 if (exclusive
&& ret
== 0) {
3703 ldpp_dout(dpp
, 0) << "ERROR: name " << info
.name
<< " already in use for role id "
3704 << info
.id
<< dendl
;
3706 } else if ( ret
< 0 && ret
!= -ENOENT
) {
3707 ldpp_dout(dpp
, 0) << "failed reading role id " << info
.id
<< ": "
3708 << cpp_strerror(-ret
) << dendl
;
3712 if (info
.id
.empty()) {
3713 /* create unique id */
3716 new_uuid
.generate_random();
3717 new_uuid
.print(uuid_str
);
3722 info
.arn
= role_arn_prefix
+ info
.tenant
+ ":role" + info
.path
+ info
.name
;
3725 real_clock::time_point t
= real_clock::now();
3728 real_clock::to_timeval(t
, tv
);
3732 gmtime_r(&tv
.tv_sec
, &result
);
3733 strftime(buf
,30,"%Y-%m-%dT%H:%M:%S", &result
);
3734 sprintf(buf
+ strlen(buf
),".%dZ",(int)tv
.tv_usec
/1000);
3735 info
.creation_date
.assign(buf
, strlen(buf
));
3737 auto& pool
= store
->svc()->zone
->get_zone_params().roles_pool
;
3738 ret
= store_info(dpp
, exclusive
, y
);
3740 ldpp_dout(dpp
, 0) << "ERROR: storing role info in Role pool: "
3741 << info
.id
<< ": " << cpp_strerror(-ret
) << dendl
;
3745 ret
= store_name(dpp
, exclusive
, y
);
3747 ldpp_dout(dpp
, 0) << "ERROR: storing role name in Role pool: "
3748 << info
.name
<< ": " << cpp_strerror(-ret
) << dendl
;
3750 //Delete the role info that was stored in the previous call
3751 std::string oid
= get_info_oid_prefix() + info
.id
;
3752 int info_ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3754 ldpp_dout(dpp
, 0) << "ERROR: cleanup of role id from Role pool: "
3755 << info
.id
<< ": " << cpp_strerror(-info_ret
) << dendl
;
3760 ret
= store_path(dpp
, exclusive
, y
);
3762 ldpp_dout(dpp
, 0) << "ERROR: storing role path in Role pool: "
3763 << info
.path
<< ": " << cpp_strerror(-ret
) << dendl
;
3764 //Delete the role info that was stored in the previous call
3765 std::string oid
= get_info_oid_prefix() + info
.id
;
3766 int info_ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3768 ldpp_dout(dpp
, 0) << "ERROR: cleanup of role id from Role pool: "
3769 << info
.id
<< ": " << cpp_strerror(-info_ret
) << dendl
;
3771 //Delete role name that was stored in previous call
3772 oid
= info
.tenant
+ get_names_oid_prefix() + info
.name
;
3773 int name_ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3775 ldpp_dout(dpp
, 0) << "ERROR: cleanup of role name from Role pool: "
3776 << info
.name
<< ": " << cpp_strerror(-name_ret
) << dendl
;
3783 int RadosRole::delete_obj(const DoutPrefixProvider
*dpp
, optional_yield y
)
3785 auto& pool
= store
->svc()->zone
->get_zone_params().roles_pool
;
3787 int ret
= read_name(dpp
, y
);
3792 ret
= read_info(dpp
, y
);
3797 if (! info
.perm_policy_map
.empty()) {
3798 return -ERR_DELETE_CONFLICT
;
3802 std::string oid
= get_info_oid_prefix() + info
.id
;
3803 ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3805 ldpp_dout(dpp
, 0) << "ERROR: deleting role id from Role pool: "
3806 << info
.id
<< ": " << cpp_strerror(-ret
) << dendl
;
3810 oid
= info
.tenant
+ get_names_oid_prefix() + info
.name
;
3811 ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3813 ldpp_dout(dpp
, 0) << "ERROR: deleting role name from Role pool: "
3814 << info
.name
<< ": " << cpp_strerror(-ret
) << dendl
;
3818 oid
= info
.tenant
+ get_path_oid_prefix() + info
.path
+ get_info_oid_prefix() + info
.id
;
3819 ret
= rgw_delete_system_obj(dpp
, store
->svc()->sysobj
, pool
, oid
, nullptr, y
);
3821 ldpp_dout(dpp
, 0) << "ERROR: deleting role path from Role pool: "
3822 << info
.path
<< ": " << cpp_strerror(-ret
) << dendl
;
3827 } // namespace rgw::sal
3831 void* newRadosStore(void)
3833 rgw::sal::RadosStore
* store
= new rgw::sal::RadosStore();
3835 RGWRados
* rados
= new RGWRados();
3838 delete store
; store
= nullptr;
3840 store
->setRados(rados
);
3841 rados
->set_store(store
);