1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
6 #include <system_error>
11 #include <boost/algorithm/string/predicate.hpp>
12 #include <boost/bind.hpp>
13 #include <boost/optional.hpp>
14 #include <boost/utility/in_place_factory.hpp>
15 #include <boost/utility/string_view.hpp>
17 #include "include/scope_guard.h"
18 #include "common/Clock.h"
19 #include "common/armor.h"
20 #include "common/errno.h"
21 #include "common/mime.h"
22 #include "common/utf8.h"
23 #include "common/ceph_json.h"
24 #include "common/static_ptr.h"
26 #include "rgw_rados.h"
31 #include "rgw_acl_s3.h"
32 #include "rgw_acl_swift.h"
33 #include "rgw_aio_throttle.h"
35 #include "rgw_bucket.h"
37 #include "rgw_multi.h"
38 #include "rgw_multi_del.h"
40 #include "rgw_cors_s3.h"
41 #include "rgw_rest_conn.h"
42 #include "rgw_rest_s3.h"
44 #include "rgw_client_io.h"
45 #include "rgw_compression.h"
47 #include "rgw_tag_s3.h"
48 #include "rgw_putobj_processor.h"
49 #include "rgw_crypt.h"
50 #include "rgw_perf_counters.h"
51 #include "rgw_notify.h"
52 #include "rgw_notify_event_type.h"
54 #include "services/svc_zone.h"
55 #include "services/svc_quota.h"
56 #include "services/svc_sys_obj.h"
58 #include "cls/lock/cls_lock_client.h"
59 #include "cls/rgw/cls_rgw_client.h"
62 #include "include/ceph_assert.h"
64 #include "compressor/Compressor.h"
67 #define TRACEPOINT_DEFINE
68 #define TRACEPOINT_PROBE_DYNAMIC_LINKAGE
69 #include "tracing/rgw_op.h"
70 #undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE
71 #undef TRACEPOINT_DEFINE
73 #define tracepoint(...)
76 #define dout_context g_ceph_context
77 #define dout_subsys ceph_subsys_rgw
79 using namespace librados
;
80 using ceph::crypto::MD5
;
81 using boost::optional
;
85 using rgw::IAM::Effect
;
86 using rgw::IAM::Policy
;
88 using rgw::IAM::Policy
;
90 static string mp_ns
= RGW_OBJ_NS_MULTIPART
;
91 static string shadow_ns
= RGW_OBJ_NS_SHADOW
;
93 static void forward_req_info(CephContext
*cct
, req_info
& info
, const std::string
& bucket_name
);
94 static int forward_request_to_master(struct req_state
*s
, obj_version
*objv
, RGWRados
*store
,
95 bufferlist
& in_data
, JSONParser
*jp
, req_info
*forward_info
= nullptr);
97 static MultipartMetaFilter mp_filter
;
99 // this probably should belong in the rgw_iam_policy_keywords, I'll get it to it
101 static constexpr auto S3_EXISTING_OBJTAG
= "s3:ExistingObjectTag";
103 int RGWGetObj::parse_range(void)
106 string
rs(range_str
);
110 ignore_invalid_range
= s
->cct
->_conf
->rgw_ignore_get_invalid_range
;
111 partial_content
= false;
113 size_t pos
= rs
.find("bytes=");
114 if (pos
== string::npos
) {
116 while (isspace(rs
[pos
]))
119 while (isalpha(rs
[end
]))
121 if (strncasecmp(rs
.c_str(), "bytes", end
- pos
) != 0)
123 while (isspace(rs
[end
]))
127 rs
= rs
.substr(end
+ 1);
129 rs
= rs
.substr(pos
+ 6); /* size of("bytes=") */
132 if (pos
== string::npos
)
135 partial_content
= true;
137 ofs_str
= rs
.substr(0, pos
);
138 end_str
= rs
.substr(pos
+ 1);
139 if (end_str
.length()) {
140 end
= atoll(end_str
.c_str());
145 if (ofs_str
.length()) {
146 ofs
= atoll(ofs_str
.c_str());
147 } else { // RFC2616 suffix-byte-range-spec
152 if (end
>= 0 && end
< ofs
)
159 if (ignore_invalid_range
) {
160 partial_content
= false;
163 range_parsed
= false; // allow retry
170 static int decode_policy(CephContext
*cct
,
172 RGWAccessControlPolicy
*policy
)
174 auto iter
= bl
.cbegin();
176 policy
->decode(iter
);
177 } catch (buffer::error
& err
) {
178 ldout(cct
, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl
;
181 if (cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
182 ldout(cct
, 15) << __func__
<< " Read AccessControlPolicy";
183 RGWAccessControlPolicy_S3
*s3policy
= static_cast<RGWAccessControlPolicy_S3
*>(policy
);
184 s3policy
->to_xml(*_dout
);
191 static int get_user_policy_from_attr(CephContext
* const cct
,
192 RGWRados
* const store
,
193 map
<string
, bufferlist
>& attrs
,
194 RGWAccessControlPolicy
& policy
/* out */)
196 auto aiter
= attrs
.find(RGW_ATTR_ACL
);
197 if (aiter
!= attrs
.end()) {
198 int ret
= decode_policy(cct
, aiter
->second
, &policy
);
209 static int get_bucket_instance_policy_from_attr(CephContext
*cct
,
211 RGWBucketInfo
& bucket_info
,
212 map
<string
, bufferlist
>& bucket_attrs
,
213 RGWAccessControlPolicy
*policy
)
215 map
<string
, bufferlist
>::iterator aiter
= bucket_attrs
.find(RGW_ATTR_ACL
);
217 if (aiter
!= bucket_attrs
.end()) {
218 int ret
= decode_policy(cct
, aiter
->second
, policy
);
222 ldout(cct
, 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl
;
224 /* object exists, but policy is broken */
225 int r
= rgw_get_user_info_by_uid(store
, bucket_info
.owner
, uinfo
);
229 policy
->create_default(bucket_info
.owner
, uinfo
.display_name
);
234 static int get_obj_policy_from_attr(CephContext
*cct
,
236 RGWObjectCtx
& obj_ctx
,
237 RGWBucketInfo
& bucket_info
,
238 map
<string
, bufferlist
>& bucket_attrs
,
239 RGWAccessControlPolicy
*policy
,
240 string
*storage_class
,
246 RGWRados::Object
op_target(store
, bucket_info
, obj_ctx
, obj
);
247 RGWRados::Object::Read
rop(&op_target
);
249 ret
= rop
.get_attr(RGW_ATTR_ACL
, bl
);
251 ret
= decode_policy(cct
, bl
, policy
);
254 } else if (ret
== -ENODATA
) {
255 /* object exists, but policy is broken */
256 ldout(cct
, 0) << "WARNING: couldn't find acl header for object, generating default" << dendl
;
258 ret
= rgw_get_user_info_by_uid(store
, bucket_info
.owner
, uinfo
);
262 policy
->create_default(bucket_info
.owner
, uinfo
.display_name
);
267 int r
= rop
.get_attr(RGW_ATTR_STORAGE_CLASS
, scbl
);
269 *storage_class
= scbl
.to_str();
271 storage_class
->clear();
280 * Get the AccessControlPolicy for an object off of disk.
281 * policy: must point to a valid RGWACL, and will be filled upon return.
282 * bucket: name of the bucket containing the object.
283 * object: name of the object to get the ACL for.
284 * Returns: 0 on success, -ERR# otherwise.
286 int rgw_op_get_bucket_policy_from_attr(CephContext
*cct
,
288 RGWBucketInfo
& bucket_info
,
289 map
<string
, bufferlist
>& bucket_attrs
,
290 RGWAccessControlPolicy
*policy
)
292 return get_bucket_instance_policy_from_attr(cct
, store
, bucket_info
, bucket_attrs
, policy
);
295 static boost::optional
<Policy
> get_iam_policy_from_attr(CephContext
* cct
,
297 map
<string
, bufferlist
>& attrs
,
298 const string
& tenant
) {
299 auto i
= attrs
.find(RGW_ATTR_IAM_POLICY
);
300 if (i
!= attrs
.end()) {
301 return Policy(cct
, tenant
, i
->second
);
307 vector
<Policy
> get_iam_user_policy_from_attr(CephContext
* cct
,
309 map
<string
, bufferlist
>& attrs
,
310 const string
& tenant
) {
311 vector
<Policy
> policies
;
312 if (auto it
= attrs
.find(RGW_ATTR_USER_POLICY
); it
!= attrs
.end()) {
313 bufferlist out_bl
= attrs
[RGW_ATTR_USER_POLICY
];
314 map
<string
, string
> policy_map
;
315 decode(policy_map
, out_bl
);
316 for (auto& it
: policy_map
) {
317 bufferlist bl
= bufferlist::static_from_string(it
.second
);
318 Policy
p(cct
, tenant
, bl
);
319 policies
.push_back(std::move(p
));
325 static int get_obj_attrs(RGWRados
*store
, struct req_state
*s
, const rgw_obj
& obj
, map
<string
, bufferlist
>& attrs
, rgw_obj
*target_obj
= nullptr)
327 RGWRados::Object
op_target(store
, s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
328 RGWRados::Object::Read
read_op(&op_target
);
330 read_op
.params
.attrs
= &attrs
;
331 read_op
.params
.target_obj
= target_obj
;
333 return read_op
.prepare();
336 static int get_obj_head(RGWRados
*store
, struct req_state
*s
,
338 map
<string
, bufferlist
> *attrs
,
341 store
->set_prefetch_data(s
->obj_ctx
, obj
);
343 RGWRados::Object
op_target(store
, s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
344 RGWRados::Object::Read
read_op(&op_target
);
346 read_op
.params
.attrs
= attrs
;
348 int ret
= read_op
.prepare();
357 ret
= read_op
.read(0, s
->cct
->_conf
->rgw_max_chunk_size
, *pbl
);
362 struct multipart_upload_info
364 rgw_placement_rule dest_placement
;
366 void encode(bufferlist
& bl
) const {
367 ENCODE_START(1, 1, bl
);
368 encode(dest_placement
, bl
);
372 void decode(bufferlist::const_iterator
& bl
) {
374 decode(dest_placement
, bl
);
378 WRITE_CLASS_ENCODER(multipart_upload_info
)
380 static int get_multipart_info(RGWRados
*store
, struct req_state
*s
,
382 RGWAccessControlPolicy
*policy
,
383 map
<string
, bufferlist
> *attrs
,
384 multipart_upload_info
*upload_info
)
389 bufferlist
*pheadbl
= (upload_info
? &headbl
: nullptr);
391 int op_ret
= get_obj_head(store
, s
, obj
, attrs
, pheadbl
);
393 if (op_ret
== -ENOENT
) {
394 return -ERR_NO_SUCH_UPLOAD
;
399 if (upload_info
&& headbl
.length() > 0) {
400 auto hiter
= headbl
.cbegin();
402 decode(*upload_info
, hiter
);
403 } catch (buffer::error
& err
) {
404 ldpp_dout(s
, 0) << "ERROR: failed to decode multipart upload info" << dendl
;
409 if (policy
&& attrs
) {
410 for (auto& iter
: *attrs
) {
411 string name
= iter
.first
;
412 if (name
.compare(RGW_ATTR_ACL
) == 0) {
413 bufferlist
& bl
= iter
.second
;
414 auto bli
= bl
.cbegin();
416 decode(*policy
, bli
);
417 } catch (buffer::error
& err
) {
418 ldpp_dout(s
, 0) << "ERROR: could not decode policy" << dendl
;
429 static int get_multipart_info(RGWRados
*store
, struct req_state
*s
,
430 const string
& meta_oid
,
431 RGWAccessControlPolicy
*policy
,
432 map
<string
, bufferlist
> *attrs
,
433 multipart_upload_info
*upload_info
)
435 map
<string
, bufferlist
>::iterator iter
;
439 meta_obj
.init_ns(s
->bucket
, meta_oid
, mp_ns
);
440 meta_obj
.set_in_extra_data(true);
442 return get_multipart_info(store
, s
, meta_obj
, policy
, attrs
, upload_info
);
445 static int modify_obj_attr(RGWRados
*store
, struct req_state
*s
, const rgw_obj
& obj
, const char* attr_name
, bufferlist
& attr_val
)
447 map
<string
, bufferlist
> attrs
;
448 RGWRados::Object
op_target(store
, s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
449 RGWRados::Object::Read
read_op(&op_target
);
451 read_op
.params
.attrs
= &attrs
;
453 int r
= read_op
.prepare();
457 store
->set_atomic(s
->obj_ctx
, read_op
.state
.obj
);
458 attrs
[attr_name
] = attr_val
;
459 return store
->set_attrs(s
->obj_ctx
, s
->bucket_info
, read_op
.state
.obj
, attrs
, NULL
);
462 static int read_bucket_policy(RGWRados
*store
,
464 RGWBucketInfo
& bucket_info
,
465 map
<string
, bufferlist
>& bucket_attrs
,
466 RGWAccessControlPolicy
*policy
,
469 if (!s
->system_request
&& bucket_info
.flags
& BUCKET_SUSPENDED
) {
470 ldpp_dout(s
, 0) << "NOTICE: bucket " << bucket_info
.bucket
.name
471 << " is suspended" << dendl
;
472 return -ERR_USER_SUSPENDED
;
475 if (bucket
.name
.empty()) {
479 int ret
= rgw_op_get_bucket_policy_from_attr(s
->cct
, store
, bucket_info
, bucket_attrs
, policy
);
480 if (ret
== -ENOENT
) {
481 ret
= -ERR_NO_SUCH_BUCKET
;
487 static int read_obj_policy(RGWRados
*store
,
489 RGWBucketInfo
& bucket_info
,
490 map
<string
, bufferlist
>& bucket_attrs
,
491 RGWAccessControlPolicy
* acl
,
492 string
*storage_class
,
493 boost::optional
<Policy
>& policy
,
498 upload_id
= s
->info
.args
.get("uploadId");
501 if (!s
->system_request
&& bucket_info
.flags
& BUCKET_SUSPENDED
) {
502 ldpp_dout(s
, 0) << "NOTICE: bucket " << bucket_info
.bucket
.name
503 << " is suspended" << dendl
;
504 return -ERR_USER_SUSPENDED
;
507 if (!upload_id
.empty()) {
508 /* multipart upload */
509 RGWMPObj
mp(object
.name
, upload_id
);
510 string oid
= mp
.get_meta();
511 obj
.init_ns(bucket
, oid
, mp_ns
);
512 obj
.set_in_extra_data(true);
514 obj
= rgw_obj(bucket
, object
);
516 policy
= get_iam_policy_from_attr(s
->cct
, store
, bucket_attrs
, bucket
.tenant
);
518 RGWObjectCtx
*obj_ctx
= static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
519 int ret
= get_obj_policy_from_attr(s
->cct
, store
, *obj_ctx
,
520 bucket_info
, bucket_attrs
, acl
, storage_class
, obj
);
521 if (ret
== -ENOENT
) {
522 /* object does not exist checking the bucket's ACL to make sure
523 that we send a proper error code */
524 RGWAccessControlPolicy
bucket_policy(s
->cct
);
525 ret
= rgw_op_get_bucket_policy_from_attr(s
->cct
, store
, bucket_info
, bucket_attrs
, &bucket_policy
);
529 const rgw_user
& bucket_owner
= bucket_policy
.get_owner().get_id();
530 if (bucket_owner
.compare(s
->user
->user_id
) != 0 &&
531 ! s
->auth
.identity
->is_admin_of(bucket_owner
)) {
533 auto r
= policy
->eval(s
->env
, *s
->auth
.identity
, rgw::IAM::s3ListBucket
, ARN(bucket
));
534 if (r
== Effect::Allow
)
536 if (r
== Effect::Deny
)
539 if (! bucket_policy
.verify_permission(s
, *s
->auth
.identity
, s
->perm_mask
, RGW_PERM_READ
))
552 * Get the AccessControlPolicy for an user, bucket or object off of disk.
553 * s: The req_state to draw information from.
554 * only_bucket: If true, reads the user and bucket ACLs rather than the object ACL.
555 * Returns: 0 on success, -ERR# otherwise.
557 int rgw_build_bucket_policies(RGWRados
* store
, struct req_state
* s
)
561 RGWUserInfo bucket_owner_info
;
562 auto obj_ctx
= store
->svc
.sysobj
->init_obj_ctx();
564 string bi
= s
->info
.args
.get(RGW_SYS_PARAM_PREFIX
"bucket-instance");
566 ret
= rgw_bucket_parse_bucket_instance(bi
, &s
->bucket_instance_id
, &s
->bucket_instance_shard_id
);
572 if(s
->dialect
.compare("s3") == 0) {
573 s
->bucket_acl
= std::make_unique
<RGWAccessControlPolicy_S3
>(s
->cct
);
574 } else if(s
->dialect
.compare("swift") == 0) {
575 /* We aren't allocating the account policy for those operations using
576 * the Swift's infrastructure that don't really need req_state::user.
577 * Typical example here is the implementation of /info. */
578 if (!s
->user
->user_id
.empty()) {
579 s
->user_acl
= std::make_unique
<RGWAccessControlPolicy_SWIFTAcct
>(s
->cct
);
581 s
->bucket_acl
= std::make_unique
<RGWAccessControlPolicy_SWIFT
>(s
->cct
);
583 s
->bucket_acl
= std::make_unique
<RGWAccessControlPolicy
>(s
->cct
);
586 /* check if copy source is within the current domain */
587 if (!s
->src_bucket_name
.empty()) {
588 RGWBucketInfo source_info
;
590 if (s
->bucket_instance_id
.empty()) {
591 ret
= store
->get_bucket_info(obj_ctx
, s
->src_tenant_name
, s
->src_bucket_name
, source_info
, NULL
);
593 ret
= store
->get_bucket_instance_info(obj_ctx
, s
->bucket_instance_id
, source_info
, NULL
, NULL
);
596 string
& zonegroup
= source_info
.zonegroup
;
597 s
->local_source
= store
->svc
.zone
->get_zonegroup().equals(zonegroup
);
603 std::string display_name
;
606 s
->user
->display_name
,
609 if (!s
->bucket_name
.empty()) {
610 s
->bucket_exists
= true;
611 if (s
->bucket_instance_id
.empty()) {
612 ret
= store
->get_bucket_info(obj_ctx
, s
->bucket_tenant
, s
->bucket_name
,
613 s
->bucket_info
, &s
->bucket_mtime
,
616 ret
= store
->get_bucket_instance_info(obj_ctx
, s
->bucket_instance_id
,
617 s
->bucket_info
, &s
->bucket_mtime
,
621 if (ret
!= -ENOENT
) {
623 rgw_make_bucket_entry_name(s
->bucket_tenant
, s
->bucket_name
, bucket_log
);
624 ldpp_dout(s
, 0) << "NOTICE: couldn't get bucket from bucket_name (name="
625 << bucket_log
<< ")" << dendl
;
628 s
->bucket_exists
= false;
630 s
->bucket
= s
->bucket_info
.bucket
;
632 if (s
->bucket_exists
) {
633 ret
= read_bucket_policy(store
, s
, s
->bucket_info
, s
->bucket_attrs
,
634 s
->bucket_acl
.get(), s
->bucket
);
636 s
->bucket_info
.owner
,
637 s
->bucket_acl
->get_owner().get_display_name(),
640 s
->bucket_acl
->create_default(s
->user
->user_id
, s
->user
->display_name
);
641 ret
= -ERR_NO_SUCH_BUCKET
;
644 s
->bucket_owner
= s
->bucket_acl
->get_owner();
646 RGWZoneGroup zonegroup
;
647 int r
= store
->svc
.zone
->get_zonegroup(s
->bucket_info
.zonegroup
, zonegroup
);
649 if (!zonegroup
.endpoints
.empty()) {
650 s
->zonegroup_endpoint
= zonegroup
.endpoints
.front();
652 // use zonegroup's master zone endpoints
653 auto z
= zonegroup
.zones
.find(zonegroup
.master_zone
);
654 if (z
!= zonegroup
.zones
.end() && !z
->second
.endpoints
.empty()) {
655 s
->zonegroup_endpoint
= z
->second
.endpoints
.front();
658 s
->zonegroup_name
= zonegroup
.get_name();
660 if (r
< 0 && ret
== 0) {
664 if (s
->bucket_exists
&& !store
->svc
.zone
->get_zonegroup().equals(s
->bucket_info
.zonegroup
)) {
665 ldpp_dout(s
, 0) << "NOTICE: request for data in a different zonegroup ("
666 << s
->bucket_info
.zonegroup
<< " != "
667 << store
->svc
.zone
->get_zonegroup().get_id() << ")" << dendl
;
668 /* we now need to make sure that the operation actually requires copy source, that is
669 * it's a copy operation
671 if (store
->svc
.zone
->get_zonegroup().is_master_zonegroup() && s
->system_request
) {
672 /*If this is the master, don't redirect*/
673 } else if (s
->op_type
== RGW_OP_GET_BUCKET_LOCATION
) {
674 /* If op is get bucket location, don't redirect */
675 } else if (!s
->local_source
||
676 (s
->op
!= OP_PUT
&& s
->op
!= OP_COPY
) ||
678 return -ERR_PERMANENT_REDIRECT
;
682 /* init dest placement -- only if bucket exists, otherwise request is either not relevant, or
683 * it's a create_bucket request, in which case the op will deal with the placement later */
684 if (s
->bucket_exists
) {
685 s
->dest_placement
.storage_class
= s
->info
.storage_class
;
686 s
->dest_placement
.inherit_from(s
->bucket_info
.placement_rule
);
688 if (!store
->svc
.zone
->get_zone_params().valid_placement(s
->dest_placement
)) {
689 ldpp_dout(s
, 0) << "NOTICE: invalid dest placement: " << s
->dest_placement
.to_str() << dendl
;
695 /* handle user ACL only for those APIs which support it */
697 map
<string
, bufferlist
> uattrs
;
698 ret
= rgw_get_user_attrs_by_uid(store
, acct_acl_user
.uid
, uattrs
);
700 ret
= get_user_policy_from_attr(s
->cct
, store
, uattrs
, *s
->user_acl
);
702 if (-ENOENT
== ret
) {
703 /* In already existing clusters users won't have ACL. In such case
704 * assuming that only account owner has the rights seems to be
705 * reasonable. That allows to have only one verification logic.
706 * NOTE: there is small compatibility kludge for global, empty tenant:
707 * 1. if we try to reach an existing bucket, its owner is considered
709 * 2. otherwise account owner is identity stored in s->user->user_id. */
710 s
->user_acl
->create_default(acct_acl_user
.uid
,
711 acct_acl_user
.display_name
);
713 } else if (ret
< 0) {
714 ldpp_dout(s
, 0) << "NOTICE: couldn't get user attrs for handling ACL "
715 "(user_id=" << s
->user
->user_id
<< ", ret=" << ret
<< ")" << dendl
;
719 // We don't need user policies in case of STS token returned by AssumeRole,
720 // hence the check for user type
721 if (! s
->user
->user_id
.empty() && s
->auth
.identity
->get_identity_type() != TYPE_ROLE
) {
723 map
<string
, bufferlist
> uattrs
;
724 if (ret
= rgw_get_user_attrs_by_uid(store
, s
->user
->user_id
, uattrs
); ! ret
) {
725 if (s
->iam_user_policies
.empty()) {
726 s
->iam_user_policies
= get_iam_user_policy_from_attr(s
->cct
, store
, uattrs
, s
->user
->user_id
.tenant
);
728 // This scenario can happen when a STS token has a policy, then we need to append other user policies
729 // to the existing ones. (e.g. token returned by GetSessionToken)
730 auto user_policies
= get_iam_user_policy_from_attr(s
->cct
, store
, uattrs
, s
->user
->user_id
.tenant
);
731 s
->iam_user_policies
.insert(s
->iam_user_policies
.end(), user_policies
.begin(), user_policies
.end());
738 } catch (const std::exception
& e
) {
739 lderr(s
->cct
) << "Error reading IAM User Policy: " << e
.what() << dendl
;
745 s
->iam_policy
= get_iam_policy_from_attr(s
->cct
, store
, s
->bucket_attrs
,
747 } catch (const std::exception
& e
) {
748 // Really this is a can't happen condition. We parse the policy
749 // when it's given to us, so perhaps we should abort or otherwise
750 // raise bloody murder.
751 ldpp_dout(s
, 0) << "Error reading IAM Policy: " << e
.what() << dendl
;
755 bool success
= store
->svc
.zone
->get_redirect_zone_endpoint(&s
->redirect_zone_endpoint
);
757 ldpp_dout(s
, 20) << "redirect_zone_endpoint=" << s
->redirect_zone_endpoint
<< dendl
;
764 * Get the AccessControlPolicy for a bucket or object off of disk.
765 * s: The req_state to draw information from.
766 * only_bucket: If true, reads the bucket ACL rather than the object ACL.
767 * Returns: 0 on success, -ERR# otherwise.
769 int rgw_build_object_policies(RGWRados
*store
, struct req_state
*s
,
774 if (!s
->object
.empty()) {
775 if (!s
->bucket_exists
) {
776 return -ERR_NO_SUCH_BUCKET
;
778 s
->object_acl
= std::make_unique
<RGWAccessControlPolicy
>(s
->cct
);
779 rgw_obj
obj(s
->bucket
, s
->object
);
781 store
->set_atomic(s
->obj_ctx
, obj
);
783 store
->set_prefetch_data(s
->obj_ctx
, obj
);
785 ret
= read_obj_policy(store
, s
, s
->bucket_info
, s
->bucket_attrs
,
786 s
->object_acl
.get(), nullptr, s
->iam_policy
, s
->bucket
,
793 void rgw_add_to_iam_environment(rgw::IAM::Environment
& e
, std::string_view key
, std::string_view val
){
794 // This variant just adds non empty key pairs to IAM env., values can be empty
795 // in certain cases like tagging
800 static int rgw_iam_add_tags_from_bl(struct req_state
* s
, bufferlist
& bl
){
803 auto bliter
= bl
.cbegin();
804 tagset
.decode(bliter
);
805 } catch (buffer::error
& err
) {
806 ldpp_dout(s
, 0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl
;
810 for (const auto& tag
: tagset
.get_tags()){
811 rgw_add_to_iam_environment(s
->env
, "s3:ExistingObjectTag/" + tag
.first
, tag
.second
);
816 static int rgw_iam_add_existing_objtags(RGWRados
* store
, struct req_state
* s
, rgw_obj
& obj
, std::uint64_t action
){
817 map
<string
, bufferlist
> attrs
;
818 store
->set_atomic(s
->obj_ctx
, obj
);
819 int op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
822 auto tags
= attrs
.find(RGW_ATTR_TAGS
);
823 if (tags
!= attrs
.end()){
824 return rgw_iam_add_tags_from_bl(s
, tags
->second
);
829 static void rgw_add_grant_to_iam_environment(rgw::IAM::Environment
& e
, struct req_state
*s
){
831 using header_pair_t
= std::pair
<const char*, const char*>;
832 static const std::initializer_list
<header_pair_t
> acl_header_conditionals
{
833 {"HTTP_X_AMZ_GRANT_READ", "s3:x-amz-grant-read"},
834 {"HTTP_X_AMZ_GRANT_WRITE", "s3:x-amz-grant-write"},
835 {"HTTP_X_AMZ_GRANT_READ_ACP", "s3:x-amz-grant-read-acp"},
836 {"HTTP_X_AMZ_GRANT_WRITE_ACP", "s3:x-amz-grant-write-acp"},
837 {"HTTP_X_AMZ_GRANT_FULL_CONTROL", "s3:x-amz-grant-full-control"}
840 if (s
->has_acl_header
){
841 for (const auto& c
: acl_header_conditionals
){
842 auto hdr
= s
->info
.env
->get(c
.first
);
850 void rgw_build_iam_environment(RGWRados
* store
,
853 const auto& m
= s
->info
.env
->get_map();
854 auto t
= ceph::real_clock::now();
855 s
->env
.emplace("aws:CurrentTime", std::to_string(ceph::real_clock::to_time_t(t
)));
856 s
->env
.emplace("aws:EpochTime", ceph::to_iso_8601(t
));
857 // TODO: This is fine for now, but once we have STS we'll need to
858 // look and see. Also this won't work with the IdentityApplier
859 // model, since we need to know the actual credential.
860 s
->env
.emplace("aws:PrincipalType", "User");
862 auto i
= m
.find("HTTP_REFERER");
864 s
->env
.emplace("aws:Referer", i
->second
);
867 if (rgw_transport_is_secure(s
->cct
, *s
->info
.env
)) {
868 s
->env
.emplace("aws:SecureTransport", "true");
871 const auto remote_addr_param
= s
->cct
->_conf
->rgw_remote_addr_param
;
872 if (remote_addr_param
.length()) {
873 i
= m
.find(remote_addr_param
);
875 i
= m
.find("REMOTE_ADDR");
878 const string
* ip
= &(i
->second
);
880 if (remote_addr_param
== "HTTP_X_FORWARDED_FOR") {
881 const auto comma
= ip
->find(',');
882 if (comma
!= string::npos
) {
883 temp
.assign(*ip
, 0, comma
);
887 s
->env
.emplace("aws:SourceIp", *ip
);
890 i
= m
.find("HTTP_USER_AGENT"); {
892 s
->env
.emplace("aws:UserAgent", i
->second
);
896 // What to do about aws::userid? One can have multiple access
897 // keys so that isn't really suitable. Do we have a durable
898 // identifier that can persist through name changes?
899 s
->env
.emplace("aws:username", s
->user
->user_id
.id
);
902 i
= m
.find("HTTP_X_AMZ_SECURITY_TOKEN");
904 s
->env
.emplace("sts:authentication", "true");
906 s
->env
.emplace("sts:authentication", "false");
910 void rgw_bucket_object_pre_exec(struct req_state
*s
)
915 dump_bucket_from_state(s
);
918 // So! Now and then when we try to update bucket information, the
919 // bucket has changed during the course of the operation. (Or we have
920 // a cache consistency problem that Watch/Notify isn't ruling out
923 // When this happens, we need to update the bucket info and try
924 // again. We have, however, to try the right *part* again. We can't
925 // simply re-send, since that will obliterate the previous update.
927 // Thus, callers of this function should include everything that
928 // merges information to be changed into the bucket information as
929 // well as the call to set it.
931 // The called function must return an integer, negative on error. In
932 // general, they should just return op_ret.
935 int retry_raced_bucket_write(RGWRados
* g
, req_state
* s
, const F
& f
) {
937 for (auto i
= 0u; i
< 15u && r
== -ECANCELED
; ++i
) {
938 r
= g
->try_refresh_bucket_info(s
->bucket_info
, nullptr,
949 int RGWGetObj::verify_permission()
951 obj
= rgw_obj(s
->bucket
, s
->object
);
952 store
->set_atomic(s
->obj_ctx
, obj
);
954 store
->set_prefetch_data(s
->obj_ctx
, obj
);
957 if (torrent
.get_flag()) {
958 if (obj
.key
.instance
.empty()) {
959 action
= rgw::IAM::s3GetObjectTorrent
;
961 action
= rgw::IAM::s3GetObjectVersionTorrent
;
964 if (obj
.key
.instance
.empty()) {
965 action
= rgw::IAM::s3GetObject
;
967 action
= rgw::IAM::s3GetObjectVersion
;
969 if (s
->iam_policy
&& s
->iam_policy
->has_partial_conditional(S3_EXISTING_OBJTAG
))
970 rgw_iam_add_existing_objtags(store
, s
, obj
, action
);
971 if (! s
->iam_user_policies
.empty()) {
972 for (auto& user_policy
: s
->iam_user_policies
) {
973 if (user_policy
.has_partial_conditional(S3_EXISTING_OBJTAG
))
974 rgw_iam_add_existing_objtags(store
, s
, obj
, action
);
979 if (!verify_object_permission(this, s
, action
)) {
983 if (s
->bucket_info
.obj_lock_enabled()) {
984 get_retention
= verify_object_permission(this, s
, rgw::IAM::s3GetObjectRetention
);
985 get_legal_hold
= verify_object_permission(this, s
, rgw::IAM::s3GetObjectLegalHold
);
992 int RGWOp::verify_op_mask()
994 uint32_t required_mask
= op_mask();
996 ldpp_dout(this, 20) << "required_mask= " << required_mask
997 << " user.op_mask=" << s
->user
->op_mask
<< dendl
;
999 if ((s
->user
->op_mask
& required_mask
) != required_mask
) {
1003 if (!s
->system_request
&& (required_mask
& RGW_OP_TYPE_MODIFY
) && !store
->svc
.zone
->zone_is_writeable()) {
1004 ldpp_dout(this, 5) << "NOTICE: modify request to a read-only zone by a "
1005 "non-system user, permission denied" << dendl
;
1012 int RGWGetObjTags::verify_permission()
1014 auto iam_action
= s
->object
.instance
.empty()?
1015 rgw::IAM::s3GetObjectTagging
:
1016 rgw::IAM::s3GetObjectVersionTagging
;
1017 // TODO since we are parsing the bl now anyway, we probably change
1018 // the send_response function to accept RGWObjTag instead of a bl
1019 if (s
->iam_policy
&& s
->iam_policy
->has_partial_conditional(S3_EXISTING_OBJTAG
)){
1020 rgw_obj obj
= rgw_obj(s
->bucket
, s
->object
);
1021 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
1023 if (! s
->iam_user_policies
.empty()) {
1024 for (auto& user_policy
: s
->iam_user_policies
) {
1025 if (user_policy
.has_partial_conditional(S3_EXISTING_OBJTAG
)) {
1026 rgw_obj obj
= rgw_obj(s
->bucket
, s
->object
);
1027 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
1031 if (!verify_object_permission(this, s
,iam_action
))
1037 void RGWGetObjTags::pre_exec()
1039 rgw_bucket_object_pre_exec(s
);
1042 void RGWGetObjTags::execute()
1045 map
<string
,bufferlist
> attrs
;
1047 obj
= rgw_obj(s
->bucket
, s
->object
);
1049 store
->set_atomic(s
->obj_ctx
, obj
);
1051 op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
1053 ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << obj
1054 << " ret=" << op_ret
<< dendl
;
1058 auto tags
= attrs
.find(RGW_ATTR_TAGS
);
1059 if(tags
!= attrs
.end()){
1061 tags_bl
.append(tags
->second
);
1063 send_response_data(tags_bl
);
1066 int RGWPutObjTags::verify_permission()
1068 auto iam_action
= s
->object
.instance
.empty() ?
1069 rgw::IAM::s3PutObjectTagging
:
1070 rgw::IAM::s3PutObjectVersionTagging
;
1072 if(s
->iam_policy
&& s
->iam_policy
->has_partial_conditional(S3_EXISTING_OBJTAG
)){
1073 auto obj
= rgw_obj(s
->bucket
, s
->object
);
1074 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
1076 if (! s
->iam_user_policies
.empty()) {
1077 for (auto& user_policy
: s
->iam_user_policies
) {
1078 if (user_policy
.has_partial_conditional(S3_EXISTING_OBJTAG
)) {
1079 rgw_obj obj
= rgw_obj(s
->bucket
, s
->object
);
1080 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
1084 if (!verify_object_permission(this, s
,iam_action
))
1089 void RGWPutObjTags::execute()
1091 op_ret
= get_params();
1095 if (s
->object
.empty()){
1096 op_ret
= -EINVAL
; // we only support tagging on existing objects
1101 obj
= rgw_obj(s
->bucket
, s
->object
);
1102 store
->set_atomic(s
->obj_ctx
, obj
);
1103 op_ret
= modify_obj_attr(store
, s
, obj
, RGW_ATTR_TAGS
, tags_bl
);
1104 if (op_ret
== -ECANCELED
){
1105 op_ret
= -ERR_TAG_CONFLICT
;
1109 void RGWDeleteObjTags::pre_exec()
1111 rgw_bucket_object_pre_exec(s
);
1115 int RGWDeleteObjTags::verify_permission()
1117 if (!s
->object
.empty()) {
1118 auto iam_action
= s
->object
.instance
.empty() ?
1119 rgw::IAM::s3DeleteObjectTagging
:
1120 rgw::IAM::s3DeleteObjectVersionTagging
;
1122 if (s
->iam_policy
&& s
->iam_policy
->has_partial_conditional(S3_EXISTING_OBJTAG
)){
1123 auto obj
= rgw_obj(s
->bucket
, s
->object
);
1124 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
1126 if (! s
->iam_user_policies
.empty()) {
1127 for (auto& user_policy
: s
->iam_user_policies
) {
1128 if (user_policy
.has_partial_conditional(S3_EXISTING_OBJTAG
)) {
1129 auto obj
= rgw_obj(s
->bucket
, s
->object
);
1130 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
1134 if (!verify_object_permission(this, s
, iam_action
))
1140 void RGWDeleteObjTags::execute()
1142 if (s
->object
.empty())
1146 obj
= rgw_obj(s
->bucket
, s
->object
);
1147 store
->set_atomic(s
->obj_ctx
, obj
);
1148 map
<string
, bufferlist
> attrs
;
1149 map
<string
, bufferlist
> rmattr
;
1151 rmattr
[RGW_ATTR_TAGS
] = bl
;
1152 op_ret
= store
->set_attrs(s
->obj_ctx
, s
->bucket_info
, obj
, attrs
, &rmattr
);
1155 int RGWOp::do_aws4_auth_completion()
1157 ldpp_dout(this, 5) << "NOTICE: call to do_aws4_auth_completion" << dendl
;
1158 if (s
->auth
.completer
) {
1159 if (!s
->auth
.completer
->complete()) {
1160 return -ERR_AMZ_CONTENT_SHA256_MISMATCH
;
1162 ldpp_dout(this, 10) << "v4 auth ok -- do_aws4_auth_completion" << dendl
;
1165 /* TODO(rzarzynski): yes, we're really called twice on PUTs. Only first
1166 * call passes, so we disable second one. This is old behaviour, sorry!
1167 * Plan for tomorrow: seek and destroy. */
1168 s
->auth
.completer
= nullptr;
1174 int RGWOp::init_quota()
1176 /* no quota enforcement for system requests */
1177 if (s
->system_request
)
1180 /* init quota related stuff */
1181 if (!(s
->user
->op_mask
& RGW_OP_TYPE_MODIFY
)) {
1185 /* only interested in object related ops */
1186 if (s
->object
.empty()) {
1190 RGWUserInfo owner_info
;
1193 if (s
->user
->user_id
== s
->bucket_owner
.get_id()) {
1196 int r
= rgw_get_user_info_by_uid(store
, s
->bucket_info
.owner
, owner_info
);
1199 uinfo
= &owner_info
;
1202 if (s
->bucket_info
.quota
.enabled
) {
1203 bucket_quota
= s
->bucket_info
.quota
;
1204 } else if (uinfo
->bucket_quota
.enabled
) {
1205 bucket_quota
= uinfo
->bucket_quota
;
1207 bucket_quota
= store
->svc
.quota
->get_bucket_quota();
1210 if (uinfo
->user_quota
.enabled
) {
1211 user_quota
= uinfo
->user_quota
;
1213 user_quota
= store
->svc
.quota
->get_user_quota();
1219 static bool validate_cors_rule_method(RGWCORSRule
*rule
, const char *req_meth
) {
1223 dout(5) << "req_meth is null" << dendl
;
1227 if (strcmp(req_meth
, "GET") == 0) flags
= RGW_CORS_GET
;
1228 else if (strcmp(req_meth
, "POST") == 0) flags
= RGW_CORS_POST
;
1229 else if (strcmp(req_meth
, "PUT") == 0) flags
= RGW_CORS_PUT
;
1230 else if (strcmp(req_meth
, "DELETE") == 0) flags
= RGW_CORS_DELETE
;
1231 else if (strcmp(req_meth
, "HEAD") == 0) flags
= RGW_CORS_HEAD
;
1233 if (rule
->get_allowed_methods() & flags
) {
1234 dout(10) << "Method " << req_meth
<< " is supported" << dendl
;
1236 dout(5) << "Method " << req_meth
<< " is not supported" << dendl
;
1243 static bool validate_cors_rule_header(RGWCORSRule
*rule
, const char *req_hdrs
) {
1245 vector
<string
> hdrs
;
1246 get_str_vec(req_hdrs
, hdrs
);
1247 for (const auto& hdr
: hdrs
) {
1248 if (!rule
->is_header_allowed(hdr
.c_str(), hdr
.length())) {
1249 dout(5) << "Header " << hdr
<< " is not registered in this rule" << dendl
;
1257 int RGWOp::read_bucket_cors()
1261 map
<string
, bufferlist
>::iterator aiter
= s
->bucket_attrs
.find(RGW_ATTR_CORS
);
1262 if (aiter
== s
->bucket_attrs
.end()) {
1263 ldpp_dout(this, 20) << "no CORS configuration attr found" << dendl
;
1265 return 0; /* no CORS configuration found */
1272 auto iter
= bl
.cbegin();
1274 bucket_cors
.decode(iter
);
1275 } catch (buffer::error
& err
) {
1276 ldpp_dout(this, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl
;
1279 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
1280 RGWCORSConfiguration_S3
*s3cors
= static_cast<RGWCORSConfiguration_S3
*>(&bucket_cors
);
1281 ldpp_dout(this, 15) << "Read RGWCORSConfiguration";
1282 s3cors
->to_xml(*_dout
);
1289 * If any of the header field-names is not a ASCII case-insensitive match for
1290 * any of the values in list of headers do not set any additional headers and
1291 * terminate this set of steps.
1293 static void get_cors_response_headers(RGWCORSRule
*rule
, const char *req_hdrs
, string
& hdrs
, string
& exp_hdrs
, unsigned *max_age
) {
1296 get_str_list(req_hdrs
, hl
);
1297 for(list
<string
>::iterator it
= hl
.begin(); it
!= hl
.end(); ++it
) {
1298 if (!rule
->is_header_allowed((*it
).c_str(), (*it
).length())) {
1299 dout(5) << "Header " << (*it
) << " is not registered in this rule" << dendl
;
1301 if (hdrs
.length() > 0) hdrs
.append(",");
1306 rule
->format_exp_headers(exp_hdrs
);
1307 *max_age
= rule
->get_max_age();
1311 * Generate the CORS header response
1313 * This is described in the CORS standard, section 6.2.
1315 bool RGWOp::generate_cors_headers(string
& origin
, string
& method
, string
& headers
, string
& exp_headers
, unsigned *max_age
)
1318 const char *orig
= s
->info
.env
->get("HTTP_ORIGIN");
1325 op_ret
= read_bucket_cors();
1331 ldpp_dout(this, 2) << "No CORS configuration set yet for this bucket" << dendl
;
1336 RGWCORSRule
*rule
= bucket_cors
.host_name_rule(orig
);
1341 * Set the Allowed-Origin header to a asterisk if this is allowed in the rule
1342 * and no Authorization was send by the client
1344 * The origin parameter specifies a URI that may access the resource. The browser must enforce this.
1345 * For requests without credentials, the server may specify "*" as a wildcard,
1346 * thereby allowing any origin to access the resource.
1348 const char *authorization
= s
->info
.env
->get("HTTP_AUTHORIZATION");
1349 if (!authorization
&& rule
->has_wildcard_origin())
1353 const char *req_meth
= s
->info
.env
->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
1355 req_meth
= s
->info
.method
;
1361 if (!validate_cors_rule_method(rule
, req_meth
)) {
1367 const char *req_hdrs
= s
->info
.env
->get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS");
1370 get_cors_response_headers(rule
, req_hdrs
, headers
, exp_headers
, max_age
);
1375 int RGWGetObj::read_user_manifest_part(rgw_bucket
& bucket
,
1376 const rgw_bucket_dir_entry
& ent
,
1377 RGWAccessControlPolicy
* const bucket_acl
,
1378 const boost::optional
<Policy
>& bucket_policy
,
1379 const off_t start_ofs
,
1380 const off_t end_ofs
,
1383 ldpp_dout(this, 20) << "user manifest obj=" << ent
.key
.name
1384 << "[" << ent
.key
.instance
<< "]" << dendl
;
1385 RGWGetObj_CB
cb(this);
1386 RGWGetObj_Filter
* filter
= &cb
;
1387 boost::optional
<RGWGetObj_Decompress
> decompress
;
1389 int64_t cur_ofs
= start_ofs
;
1390 int64_t cur_end
= end_ofs
;
1392 rgw_obj
part(bucket
, ent
.key
);
1394 map
<string
, bufferlist
> attrs
;
1397 RGWObjectCtx
obj_ctx(store
);
1398 RGWAccessControlPolicy
obj_policy(s
->cct
);
1400 ldpp_dout(this, 20) << "reading obj=" << part
<< " ofs=" << cur_ofs
1401 << " end=" << cur_end
<< dendl
;
1403 obj_ctx
.set_atomic(part
);
1404 store
->set_prefetch_data(&obj_ctx
, part
);
1406 RGWRados::Object
op_target(store
, s
->bucket_info
, obj_ctx
, part
);
1407 RGWRados::Object::Read
read_op(&op_target
);
1410 /* SLO etag is optional */
1411 read_op
.conds
.if_match
= ent
.meta
.etag
.c_str();
1413 read_op
.params
.attrs
= &attrs
;
1414 read_op
.params
.obj_size
= &obj_size
;
1416 op_ret
= read_op
.prepare();
1419 op_ret
= read_op
.range_to_ofs(ent
.meta
.accounted_size
, cur_ofs
, cur_end
);
1422 bool need_decompress
;
1423 op_ret
= rgw_compression_info_from_attrset(attrs
, need_decompress
, cs_info
);
1425 ldpp_dout(this, 0) << "ERROR: failed to decode compression info" << dendl
;
1429 if (need_decompress
)
1431 if (cs_info
.orig_size
!= ent
.meta
.accounted_size
) {
1432 // hmm.. something wrong, object not as expected, abort!
1433 ldpp_dout(this, 0) << "ERROR: expected cs_info.orig_size=" << cs_info
.orig_size
1434 << ", actual read size=" << ent
.meta
.size
<< dendl
;
1437 decompress
.emplace(s
->cct
, &cs_info
, partial_content
, filter
);
1438 filter
= &*decompress
;
1442 if (obj_size
!= ent
.meta
.size
) {
1443 // hmm.. something wrong, object not as expected, abort!
1444 ldpp_dout(this, 0) << "ERROR: expected obj_size=" << obj_size
1445 << ", actual read size=" << ent
.meta
.size
<< dendl
;
1450 op_ret
= rgw_policy_from_attrset(s
->cct
, attrs
, &obj_policy
);
1454 /* We can use global user_acl because LOs cannot have segments
1455 * stored inside different accounts. */
1456 if (s
->system_request
) {
1457 ldpp_dout(this, 2) << "overriding permissions due to system operation" << dendl
;
1458 } else if (s
->auth
.identity
->is_admin_of(s
->user
->user_id
)) {
1459 ldpp_dout(this, 2) << "overriding permissions due to admin operation" << dendl
;
1460 } else if (!verify_object_permission(this, s
, part
, s
->user_acl
.get(), bucket_acl
,
1461 &obj_policy
, bucket_policy
, s
->iam_user_policies
, action
)) {
1464 if (ent
.meta
.size
== 0) {
1468 perfcounter
->inc(l_rgw_get_b
, cur_end
- cur_ofs
);
1469 filter
->fixup_range(cur_ofs
, cur_end
);
1470 op_ret
= read_op
.iterate(cur_ofs
, cur_end
, filter
);
1472 op_ret
= filter
->flush();
1476 static int iterate_user_manifest_parts(CephContext
* const cct
,
1477 RGWRados
* const store
,
1480 RGWBucketInfo
*pbucket_info
,
1481 const string
& obj_prefix
,
1482 RGWAccessControlPolicy
* const bucket_acl
,
1483 const boost::optional
<Policy
>& bucket_policy
,
1484 uint64_t * const ptotal_len
,
1485 uint64_t * const pobj_size
,
1486 string
* const pobj_sum
,
1487 int (*cb
)(rgw_bucket
& bucket
,
1488 const rgw_bucket_dir_entry
& ent
,
1489 RGWAccessControlPolicy
* const bucket_acl
,
1490 const boost::optional
<Policy
>& bucket_policy
,
1495 void * const cb_param
)
1497 rgw_bucket
& bucket
= pbucket_info
->bucket
;
1498 uint64_t obj_ofs
= 0, len_count
= 0;
1499 bool found_start
= false, found_end
= false, handled_end
= false;
1502 vector
<rgw_bucket_dir_entry
> objs
;
1504 utime_t start_time
= ceph_clock_now();
1506 RGWRados::Bucket
target(store
, *pbucket_info
);
1507 RGWRados::Bucket::List
list_op(&target
);
1509 list_op
.params
.prefix
= obj_prefix
;
1510 list_op
.params
.delim
= delim
;
1514 #define MAX_LIST_OBJS 100
1515 int r
= list_op
.list_objects(MAX_LIST_OBJS
, &objs
, NULL
, &is_truncated
);
1520 for (rgw_bucket_dir_entry
& ent
: objs
) {
1521 const uint64_t cur_total_len
= obj_ofs
;
1522 const uint64_t obj_size
= ent
.meta
.accounted_size
;
1523 uint64_t start_ofs
= 0, end_ofs
= obj_size
;
1525 if ((ptotal_len
|| cb
) && !found_start
&& cur_total_len
+ obj_size
> (uint64_t)ofs
) {
1526 start_ofs
= ofs
- obj_ofs
;
1530 obj_ofs
+= obj_size
;
1532 etag_sum
.Update((const unsigned char *)ent
.meta
.etag
.c_str(),
1533 ent
.meta
.etag
.length());
1536 if ((ptotal_len
|| cb
) && !found_end
&& obj_ofs
> (uint64_t)end
) {
1537 end_ofs
= end
- cur_total_len
+ 1;
1541 perfcounter
->tinc(l_rgw_get_lat
,
1542 (ceph_clock_now() - start_time
));
1544 if (found_start
&& !handled_end
) {
1545 len_count
+= end_ofs
- start_ofs
;
1548 r
= cb(bucket
, ent
, bucket_acl
, bucket_policy
, start_ofs
, end_ofs
,
1549 cb_param
, false /* swift_slo */);
1556 handled_end
= found_end
;
1557 start_time
= ceph_clock_now();
1559 } while (is_truncated
);
1562 *ptotal_len
= len_count
;
1565 *pobj_size
= obj_ofs
;
1568 complete_etag(etag_sum
, pobj_sum
);
1574 struct rgw_slo_part
{
1575 RGWAccessControlPolicy
*bucket_acl
= nullptr;
1576 Policy
* bucket_policy
= nullptr;
1583 static int iterate_slo_parts(CephContext
*cct
,
1587 map
<uint64_t, rgw_slo_part
>& slo_parts
,
1588 int (*cb
)(rgw_bucket
& bucket
,
1589 const rgw_bucket_dir_entry
& ent
,
1590 RGWAccessControlPolicy
*bucket_acl
,
1591 const boost::optional
<Policy
>& bucket_policy
,
1598 bool found_start
= false, found_end
= false;
1600 if (slo_parts
.empty()) {
1604 utime_t start_time
= ceph_clock_now();
1606 map
<uint64_t, rgw_slo_part
>::iterator iter
= slo_parts
.upper_bound(ofs
);
1607 if (iter
!= slo_parts
.begin()) {
1611 uint64_t obj_ofs
= iter
->first
;
1613 for (; iter
!= slo_parts
.end() && !found_end
; ++iter
) {
1614 rgw_slo_part
& part
= iter
->second
;
1615 rgw_bucket_dir_entry ent
;
1617 ent
.key
.name
= part
.obj_name
;
1618 ent
.meta
.accounted_size
= ent
.meta
.size
= part
.size
;
1619 ent
.meta
.etag
= part
.etag
;
1621 uint64_t cur_total_len
= obj_ofs
;
1622 uint64_t start_ofs
= 0, end_ofs
= ent
.meta
.size
- 1;
1624 if (!found_start
&& cur_total_len
+ ent
.meta
.size
> (uint64_t)ofs
) {
1625 start_ofs
= ofs
- obj_ofs
;
1629 obj_ofs
+= ent
.meta
.size
;
1631 if (!found_end
&& obj_ofs
> (uint64_t)end
) {
1632 end_ofs
= end
- cur_total_len
;
1636 perfcounter
->tinc(l_rgw_get_lat
,
1637 (ceph_clock_now() - start_time
));
1641 dout(20) << "iterate_slo_parts()"
1642 << " obj=" << part
.obj_name
1643 << " start_ofs=" << start_ofs
1644 << " end_ofs=" << end_ofs
1647 // SLO is a Swift thing, and Swift has no knowledge of S3 Policies.
1648 int r
= cb(part
.bucket
, ent
, part
.bucket_acl
,
1649 (part
.bucket_policy
?
1650 boost::optional
<Policy
>(*part
.bucket_policy
) : none
),
1651 start_ofs
, end_ofs
, cb_param
, true /* swift_slo */);
1657 start_time
= ceph_clock_now();
1663 static int get_obj_user_manifest_iterate_cb(rgw_bucket
& bucket
,
1664 const rgw_bucket_dir_entry
& ent
,
1665 RGWAccessControlPolicy
* const bucket_acl
,
1666 const boost::optional
<Policy
>& bucket_policy
,
1667 const off_t start_ofs
,
1668 const off_t end_ofs
,
1670 bool swift_slo
= false)
1672 RGWGetObj
*op
= static_cast<RGWGetObj
*>(param
);
1673 return op
->read_user_manifest_part(
1674 bucket
, ent
, bucket_acl
, bucket_policy
, start_ofs
, end_ofs
, swift_slo
);
1677 int RGWGetObj::handle_user_manifest(const char *prefix
)
1679 const boost::string_view
prefix_view(prefix
);
1680 ldpp_dout(this, 2) << "RGWGetObj::handle_user_manifest() prefix="
1681 << prefix_view
<< dendl
;
1683 const size_t pos
= prefix_view
.find('/');
1684 if (pos
== string::npos
) {
1688 const std::string bucket_name
= url_decode(prefix_view
.substr(0, pos
));
1689 const std::string obj_prefix
= url_decode(prefix_view
.substr(pos
+ 1));
1693 RGWAccessControlPolicy
_bucket_acl(s
->cct
);
1694 RGWAccessControlPolicy
*bucket_acl
;
1695 boost::optional
<Policy
> _bucket_policy
;
1696 boost::optional
<Policy
>* bucket_policy
;
1697 RGWBucketInfo bucket_info
;
1698 RGWBucketInfo
*pbucket_info
;
1700 if (bucket_name
.compare(s
->bucket
.name
) != 0) {
1701 map
<string
, bufferlist
> bucket_attrs
;
1702 auto obj_ctx
= store
->svc
.sysobj
->init_obj_ctx();
1703 int r
= store
->get_bucket_info(obj_ctx
, s
->user
->user_id
.tenant
,
1704 bucket_name
, bucket_info
, NULL
,
1707 ldpp_dout(this, 0) << "could not get bucket info for bucket="
1708 << bucket_name
<< dendl
;
1711 bucket
= bucket_info
.bucket
;
1712 pbucket_info
= &bucket_info
;
1713 bucket_acl
= &_bucket_acl
;
1714 r
= read_bucket_policy(store
, s
, bucket_info
, bucket_attrs
, bucket_acl
, bucket
);
1716 ldpp_dout(this, 0) << "failed to read bucket policy" << dendl
;
1719 _bucket_policy
= get_iam_policy_from_attr(s
->cct
, store
, bucket_attrs
,
1720 bucket_info
.bucket
.tenant
);
1721 bucket_policy
= &_bucket_policy
;
1724 pbucket_info
= &s
->bucket_info
;
1725 bucket_acl
= s
->bucket_acl
.get();
1726 bucket_policy
= &s
->iam_policy
;
1729 /* dry run to find out:
1730 * - total length (of the parts we are going to send to client),
1731 * - overall DLO's content size,
1732 * - md5 sum of overall DLO's content (for etag of Swift API). */
1733 int r
= iterate_user_manifest_parts(s
->cct
, store
, ofs
, end
,
1734 pbucket_info
, obj_prefix
, bucket_acl
, *bucket_policy
,
1735 nullptr, &s
->obj_size
, &lo_etag
,
1736 nullptr /* cb */, nullptr /* cb arg */);
1741 r
= RGWRados::Object::Read::range_to_ofs(s
->obj_size
, ofs
, end
);
1746 r
= iterate_user_manifest_parts(s
->cct
, store
, ofs
, end
,
1747 pbucket_info
, obj_prefix
, bucket_acl
, *bucket_policy
,
1748 &total_len
, nullptr, nullptr,
1756 send_response_data(bl
, 0, 0);
1760 r
= iterate_user_manifest_parts(s
->cct
, store
, ofs
, end
,
1761 pbucket_info
, obj_prefix
, bucket_acl
, *bucket_policy
,
1762 nullptr, nullptr, nullptr,
1763 get_obj_user_manifest_iterate_cb
, (void *)this);
1770 send_response_data(bl
, 0, 0);
1776 int RGWGetObj::handle_slo_manifest(bufferlist
& bl
)
1778 RGWSLOInfo slo_info
;
1779 auto bliter
= bl
.cbegin();
1781 decode(slo_info
, bliter
);
1782 } catch (buffer::error
& err
) {
1783 ldpp_dout(this, 0) << "ERROR: failed to decode slo manifest" << dendl
;
1786 ldpp_dout(this, 2) << "RGWGetObj::handle_slo_manifest()" << dendl
;
1788 vector
<RGWAccessControlPolicy
> allocated_acls
;
1789 map
<string
, pair
<RGWAccessControlPolicy
*, boost::optional
<Policy
>>> policies
;
1790 map
<string
, rgw_bucket
> buckets
;
1792 map
<uint64_t, rgw_slo_part
> slo_parts
;
1797 for (const auto& entry
: slo_info
.entries
) {
1798 const string
& path
= entry
.path
;
1800 /* If the path starts with slashes, strip them all. */
1801 const size_t pos_init
= path
.find_first_not_of('/');
1802 /* According to the documentation of std::string::find following check
1803 * is not necessary as we should get the std::string::npos propagation
1804 * here. This might be true with the accuracy to implementation's bugs.
1805 * See following question on SO:
1806 * http://stackoverflow.com/questions/1011790/why-does-stdstring-findtext-stdstringnpos-not-return-npos
1808 if (pos_init
== string::npos
) {
1812 const size_t pos_sep
= path
.find('/', pos_init
);
1813 if (pos_sep
== string::npos
) {
1817 string bucket_name
= path
.substr(pos_init
, pos_sep
- pos_init
);
1818 string obj_name
= path
.substr(pos_sep
+ 1);
1821 RGWAccessControlPolicy
*bucket_acl
;
1822 Policy
* bucket_policy
;
1824 if (bucket_name
.compare(s
->bucket
.name
) != 0) {
1825 const auto& piter
= policies
.find(bucket_name
);
1826 if (piter
!= policies
.end()) {
1827 bucket_acl
= piter
->second
.first
;
1828 bucket_policy
= piter
->second
.second
.get_ptr();
1829 bucket
= buckets
[bucket_name
];
1831 allocated_acls
.push_back(RGWAccessControlPolicy(s
->cct
));
1832 RGWAccessControlPolicy
& _bucket_acl
= allocated_acls
.back();
1834 RGWBucketInfo bucket_info
;
1835 map
<string
, bufferlist
> bucket_attrs
;
1836 auto obj_ctx
= store
->svc
.sysobj
->init_obj_ctx();
1837 int r
= store
->get_bucket_info(obj_ctx
, s
->user
->user_id
.tenant
,
1838 bucket_name
, bucket_info
, nullptr,
1841 ldpp_dout(this, 0) << "could not get bucket info for bucket="
1842 << bucket_name
<< dendl
;
1845 bucket
= bucket_info
.bucket
;
1846 bucket_acl
= &_bucket_acl
;
1847 r
= read_bucket_policy(store
, s
, bucket_info
, bucket_attrs
, bucket_acl
,
1850 ldpp_dout(this, 0) << "failed to read bucket ACL for bucket "
1854 auto _bucket_policy
= get_iam_policy_from_attr(
1855 s
->cct
, store
, bucket_attrs
, bucket_info
.bucket
.tenant
);
1856 bucket_policy
= _bucket_policy
.get_ptr();
1857 buckets
[bucket_name
] = bucket
;
1858 policies
[bucket_name
] = make_pair(bucket_acl
, _bucket_policy
);
1862 bucket_acl
= s
->bucket_acl
.get();
1863 bucket_policy
= s
->iam_policy
.get_ptr();
1867 part
.bucket_acl
= bucket_acl
;
1868 part
.bucket_policy
= bucket_policy
;
1869 part
.bucket
= bucket
;
1870 part
.obj_name
= obj_name
;
1871 part
.size
= entry
.size_bytes
;
1872 part
.etag
= entry
.etag
;
1873 ldpp_dout(this, 20) << "slo_part: bucket=" << part
.bucket
1874 << " obj=" << part
.obj_name
1875 << " size=" << part
.size
1876 << " etag=" << part
.etag
1879 etag_sum
.Update((const unsigned char *)entry
.etag
.c_str(),
1880 entry
.etag
.length());
1882 slo_parts
[total_len
] = part
;
1883 total_len
+= part
.size
;
1884 } /* foreach entry */
1886 complete_etag(etag_sum
, &lo_etag
);
1888 s
->obj_size
= slo_info
.total_size
;
1889 ldpp_dout(this, 20) << "s->obj_size=" << s
->obj_size
<< dendl
;
1891 int r
= RGWRados::Object::Read::range_to_ofs(total_len
, ofs
, end
);
1896 total_len
= end
- ofs
+ 1;
1897 ldpp_dout(this, 20) << "Requested: ofs=" << ofs
1899 << " total=" << total_len
1902 r
= iterate_slo_parts(s
->cct
, store
, ofs
, end
, slo_parts
,
1903 get_obj_user_manifest_iterate_cb
, (void *)this);
1911 int RGWGetObj::get_data_cb(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
)
1913 /* garbage collection related handling */
1914 utime_t start_time
= ceph_clock_now();
1915 if (start_time
> gc_invalidate_time
) {
1916 int r
= store
->defer_gc(s
->obj_ctx
, s
->bucket_info
, obj
);
1918 ldpp_dout(this, 0) << "WARNING: could not defer gc entry for obj" << dendl
;
1920 gc_invalidate_time
= start_time
;
1921 gc_invalidate_time
+= (s
->cct
->_conf
->rgw_gc_obj_min_wait
/ 2);
1923 return send_response_data(bl
, bl_ofs
, bl_len
);
1926 bool RGWGetObj::prefetch_data()
1928 /* HEAD request, stop prefetch*/
1933 bool prefetch_first_chunk
= true;
1934 range_str
= s
->info
.env
->get("HTTP_RANGE");
1937 int r
= parse_range();
1938 /* error on parsing the range, stop prefetch and will fail in execute() */
1940 return false; /* range_parsed==false */
1942 /* range get goes to shadow objects, stop prefetch */
1943 if (ofs
>= s
->cct
->_conf
->rgw_max_chunk_size
) {
1944 prefetch_first_chunk
= false;
1948 return get_data
&& prefetch_first_chunk
;
1951 void RGWGetObj::pre_exec()
1953 rgw_bucket_object_pre_exec(s
);
1956 static bool object_is_expired(map
<string
, bufferlist
>& attrs
) {
1957 map
<string
, bufferlist
>::iterator iter
= attrs
.find(RGW_ATTR_DELETE_AT
);
1958 if (iter
!= attrs
.end()) {
1961 decode(delete_at
, iter
->second
);
1962 } catch (buffer::error
& err
) {
1963 dout(0) << "ERROR: " << __func__
<< ": failed to decode " RGW_ATTR_DELETE_AT
" attr" << dendl
;
1967 if (delete_at
<= ceph_clock_now() && !delete_at
.is_zero()) {
1975 void RGWGetObj::execute()
1978 gc_invalidate_time
= ceph_clock_now();
1979 gc_invalidate_time
+= (s
->cct
->_conf
->rgw_gc_obj_min_wait
/ 2);
1981 bool need_decompress
;
1982 int64_t ofs_x
, end_x
;
1984 RGWGetObj_CB
cb(this);
1985 RGWGetObj_Filter
* filter
= (RGWGetObj_Filter
*)&cb
;
1986 boost::optional
<RGWGetObj_Decompress
> decompress
;
1987 std::unique_ptr
<RGWGetObj_Filter
> decrypt
;
1988 map
<string
, bufferlist
>::iterator attr_iter
;
1990 perfcounter
->inc(l_rgw_get
);
1992 RGWRados::Object
op_target(store
, s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
1993 RGWRados::Object::Read
read_op(&op_target
);
1995 op_ret
= get_params();
1999 op_ret
= init_common();
2003 read_op
.conds
.mod_ptr
= mod_ptr
;
2004 read_op
.conds
.unmod_ptr
= unmod_ptr
;
2005 read_op
.conds
.high_precision_time
= s
->system_request
; /* system request need to use high precision time */
2006 read_op
.conds
.mod_zone_id
= mod_zone_id
;
2007 read_op
.conds
.mod_pg_ver
= mod_pg_ver
;
2008 read_op
.conds
.if_match
= if_match
;
2009 read_op
.conds
.if_nomatch
= if_nomatch
;
2010 read_op
.params
.attrs
= &attrs
;
2011 read_op
.params
.lastmod
= &lastmod
;
2012 read_op
.params
.obj_size
= &s
->obj_size
;
2014 op_ret
= read_op
.prepare();
2017 version_id
= read_op
.state
.obj
.key
.instance
;
2019 /* STAT ops don't need data, and do no i/o */
2020 if (get_type() == RGW_OP_STAT_OBJ
) {
2024 /* start gettorrent */
2025 if (torrent
.get_flag())
2027 attr_iter
= attrs
.find(RGW_ATTR_CRYPT_MODE
);
2028 if (attr_iter
!= attrs
.end() && attr_iter
->second
.to_str() == "SSE-C-AES256") {
2029 ldpp_dout(this, 0) << "ERROR: torrents are not supported for objects "
2030 "encrypted with SSE-C" << dendl
;
2034 torrent
.init(s
, store
);
2035 op_ret
= torrent
.get_torrent_file(read_op
, total_len
, bl
, obj
);
2038 ldpp_dout(this, 0) << "ERROR: failed to get_torrent_file ret= " << op_ret
2042 op_ret
= send_response_data(bl
, 0, total_len
);
2045 ldpp_dout(this, 0) << "ERROR: failed to send_response_data ret= " << op_ret
<< dendl
;
2050 /* end gettorrent */
2052 op_ret
= rgw_compression_info_from_attrset(attrs
, need_decompress
, cs_info
);
2054 ldpp_dout(s
, 0) << "ERROR: failed to decode compression info, cannot decompress" << dendl
;
2057 if (need_decompress
) {
2058 s
->obj_size
= cs_info
.orig_size
;
2059 decompress
.emplace(s
->cct
, &cs_info
, partial_content
, filter
);
2060 filter
= &*decompress
;
2063 attr_iter
= attrs
.find(RGW_ATTR_USER_MANIFEST
);
2064 if (attr_iter
!= attrs
.end() && !skip_manifest
) {
2065 op_ret
= handle_user_manifest(attr_iter
->second
.c_str());
2067 ldpp_dout(this, 0) << "ERROR: failed to handle user manifest ret="
2074 attr_iter
= attrs
.find(RGW_ATTR_SLO_MANIFEST
);
2075 if (attr_iter
!= attrs
.end() && !skip_manifest
) {
2077 op_ret
= handle_slo_manifest(attr_iter
->second
);
2079 ldpp_dout(this, 0) << "ERROR: failed to handle slo manifest ret=" << op_ret
2086 // for range requests with obj size 0
2087 if (range_str
&& !(s
->obj_size
)) {
2093 op_ret
= read_op
.range_to_ofs(s
->obj_size
, ofs
, end
);
2096 total_len
= (ofs
<= end
? end
+ 1 - ofs
: 0);
2098 /* Check whether the object has expired. Swift API documentation
2099 * stands that we should return 404 Not Found in such case. */
2100 if (need_object_expiration() && object_is_expired(attrs
)) {
2107 attr_iter
= attrs
.find(RGW_ATTR_MANIFEST
);
2108 op_ret
= this->get_decrypt_filter(&decrypt
, filter
,
2109 attr_iter
!= attrs
.end() ? &(attr_iter
->second
) : nullptr);
2110 if (decrypt
!= nullptr) {
2111 filter
= decrypt
.get();
2117 if (!get_data
|| ofs
> end
) {
2118 send_response_data(bl
, 0, 0);
2122 perfcounter
->inc(l_rgw_get_b
, end
- ofs
);
2126 filter
->fixup_range(ofs_x
, end_x
);
2127 op_ret
= read_op
.iterate(ofs_x
, end_x
, filter
);
2130 op_ret
= filter
->flush();
2132 perfcounter
->tinc(l_rgw_get_lat
, s
->time_elapsed());
2137 op_ret
= send_response_data(bl
, 0, 0);
2144 send_response_data_error();
2147 int RGWGetObj::init_common()
2150 /* range parsed error when prefetch */
2151 if (!range_parsed
) {
2152 int r
= parse_range();
2158 if (parse_time(if_mod
, &mod_time
) < 0)
2160 mod_ptr
= &mod_time
;
2164 if (parse_time(if_unmod
, &unmod_time
) < 0)
2166 unmod_ptr
= &unmod_time
;
2172 int RGWListBuckets::verify_permission()
2174 rgw::Partition partition
= rgw::Partition::aws
;
2175 rgw::Service service
= rgw::Service::s3
;
2177 if (!verify_user_permission(this, s
, ARN(partition
, service
, "", s
->user
->user_id
.tenant
, "*"), rgw::IAM::s3ListAllMyBuckets
)) {
2184 int RGWGetUsage::verify_permission()
2186 if (s
->auth
.identity
->is_anonymous()) {
2193 void RGWListBuckets::execute()
2196 bool started
= false;
2197 uint64_t total_count
= 0;
2199 const uint64_t max_buckets
= s
->cct
->_conf
->rgw_list_buckets_max_chunk
;
2201 op_ret
= get_params();
2206 if (supports_account_metadata()) {
2207 op_ret
= rgw_get_user_attrs_by_uid(store
, s
->user
->user_id
, attrs
);
2213 is_truncated
= false;
2215 RGWUserBuckets buckets
;
2216 uint64_t read_count
;
2218 read_count
= min(limit
- total_count
, max_buckets
);
2220 read_count
= max_buckets
;
2223 op_ret
= rgw_read_user_buckets(store
, s
->user
->user_id
, buckets
,
2224 marker
, end_marker
, read_count
,
2225 should_get_stats(), &is_truncated
,
2228 /* hmm.. something wrong here.. the user was authenticated, so it
2230 ldpp_dout(this, 10) << "WARNING: failed on rgw_get_user_buckets uid="
2231 << s
->user
->user_id
<< dendl
;
2235 /* We need to have stats for all our policies - even if a given policy
2236 * isn't actually used in a given account. In such situation its usage
2237 * stats would be simply full of zeros. */
2238 for (const auto& policy
: store
->svc
.zone
->get_zonegroup().placement_targets
) {
2239 policies_stats
.emplace(policy
.second
.name
,
2240 decltype(policies_stats
)::mapped_type());
2243 std::map
<std::string
, RGWBucketEnt
>& m
= buckets
.get_buckets();
2244 for (const auto& kv
: m
) {
2245 const auto& bucket
= kv
.second
;
2247 global_stats
.bytes_used
+= bucket
.size
;
2248 global_stats
.bytes_used_rounded
+= bucket
.size_rounded
;
2249 global_stats
.objects_count
+= bucket
.count
;
2251 /* operator[] still can create a new entry for storage policy seen
2252 * for first time. */
2253 auto& policy_stats
= policies_stats
[bucket
.placement_rule
.to_str()];
2254 policy_stats
.bytes_used
+= bucket
.size
;
2255 policy_stats
.bytes_used_rounded
+= bucket
.size_rounded
;
2256 policy_stats
.buckets_count
++;
2257 policy_stats
.objects_count
+= bucket
.count
;
2259 global_stats
.buckets_count
+= m
.size();
2260 total_count
+= m
.size();
2262 done
= (m
.size() < read_count
|| (limit
>= 0 && total_count
>= (uint64_t)limit
));
2265 send_response_begin(buckets
.count() > 0);
2270 map
<string
, RGWBucketEnt
>::reverse_iterator riter
= m
.rbegin();
2271 marker
= riter
->first
;
2273 handle_listing_chunk(std::move(buckets
));
2275 } while (is_truncated
&& !done
);
2279 send_response_begin(false);
2281 send_response_end();
2284 void RGWGetUsage::execute()
2286 uint64_t start_epoch
= 0;
2287 uint64_t end_epoch
= (uint64_t)-1;
2288 op_ret
= get_params();
2292 if (!start_date
.empty()) {
2293 op_ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
2295 ldpp_dout(this, 0) << "ERROR: failed to parse start date" << dendl
;
2300 if (!end_date
.empty()) {
2301 op_ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
2303 ldpp_dout(this, 0) << "ERROR: failed to parse end date" << dendl
;
2308 uint32_t max_entries
= 1000;
2310 bool is_truncated
= true;
2312 RGWUsageIter usage_iter
;
2314 while (is_truncated
) {
2315 op_ret
= store
->read_usage(s
->user
->user_id
, s
->bucket_name
, start_epoch
, end_epoch
, max_entries
,
2316 &is_truncated
, usage_iter
, usage
);
2318 if (op_ret
== -ENOENT
) {
2320 is_truncated
= false;
2328 op_ret
= rgw_user_sync_all_stats(store
, s
->user
->user_id
);
2330 ldpp_dout(this, 0) << "ERROR: failed to sync user stats" << dendl
;
2334 op_ret
= rgw_user_get_all_buckets_stats(store
, s
->user
->user_id
, buckets_usage
);
2336 ldpp_dout(this, 0) << "ERROR: failed to get user's buckets stats" << dendl
;
2340 string user_str
= s
->user
->user_id
.to_str();
2341 op_ret
= store
->cls_user_get_header(user_str
, &header
);
2343 ldpp_dout(this, 0) << "ERROR: can't read user header" << dendl
;
2350 int RGWStatAccount::verify_permission()
2352 if (!verify_user_permission_no_policy(this, s
, RGW_PERM_READ
)) {
2359 void RGWStatAccount::execute()
2362 bool is_truncated
= false;
2363 uint64_t max_buckets
= s
->cct
->_conf
->rgw_list_buckets_max_chunk
;
2366 RGWUserBuckets buckets
;
2368 op_ret
= rgw_read_user_buckets(store
, s
->user
->user_id
, buckets
, marker
,
2369 string(), max_buckets
, true, &is_truncated
);
2371 /* hmm.. something wrong here.. the user was authenticated, so it
2373 ldpp_dout(this, 10) << "WARNING: failed on rgw_get_user_buckets uid="
2374 << s
->user
->user_id
<< dendl
;
2377 /* We need to have stats for all our policies - even if a given policy
2378 * isn't actually used in a given account. In such situation its usage
2379 * stats would be simply full of zeros. */
2380 for (const auto& policy
: store
->svc
.zone
->get_zonegroup().placement_targets
) {
2381 policies_stats
.emplace(policy
.second
.name
,
2382 decltype(policies_stats
)::mapped_type());
2385 std::map
<std::string
, RGWBucketEnt
>& m
= buckets
.get_buckets();
2386 for (const auto& kv
: m
) {
2387 const auto& bucket
= kv
.second
;
2389 global_stats
.bytes_used
+= bucket
.size
;
2390 global_stats
.bytes_used_rounded
+= bucket
.size_rounded
;
2391 global_stats
.objects_count
+= bucket
.count
;
2393 /* operator[] still can create a new entry for storage policy seen
2394 * for first time. */
2395 auto& policy_stats
= policies_stats
[bucket
.placement_rule
.to_str()];
2396 policy_stats
.bytes_used
+= bucket
.size
;
2397 policy_stats
.bytes_used_rounded
+= bucket
.size_rounded
;
2398 policy_stats
.buckets_count
++;
2399 policy_stats
.objects_count
+= bucket
.count
;
2401 global_stats
.buckets_count
+= m
.size();
2404 } while (is_truncated
);
2407 int RGWGetBucketVersioning::verify_permission()
2409 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketVersioning
);
2412 void RGWGetBucketVersioning::pre_exec()
2414 rgw_bucket_object_pre_exec(s
);
2417 void RGWGetBucketVersioning::execute()
2419 versioned
= s
->bucket_info
.versioned();
2420 versioning_enabled
= s
->bucket_info
.versioning_enabled();
2421 mfa_enabled
= s
->bucket_info
.mfa_enabled();
2424 int RGWSetBucketVersioning::verify_permission()
2426 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3PutBucketVersioning
);
2429 void RGWSetBucketVersioning::pre_exec()
2431 rgw_bucket_object_pre_exec(s
);
2434 void RGWSetBucketVersioning::execute()
2436 op_ret
= get_params();
2440 if (s
->bucket_info
.obj_lock_enabled() && versioning_status
!= VersioningEnabled
) {
2441 op_ret
= -ERR_INVALID_BUCKET_STATE
;
2445 bool cur_mfa_status
= (s
->bucket_info
.flags
& BUCKET_MFA_ENABLED
) != 0;
2447 mfa_set_status
&= (mfa_status
!= cur_mfa_status
);
2449 if (mfa_set_status
&&
2451 op_ret
= -ERR_MFA_REQUIRED
;
2455 if (!store
->svc
.zone
->is_meta_master()) {
2456 op_ret
= forward_request_to_master(s
, NULL
, store
, in_data
, nullptr);
2458 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
2463 bool modified
= mfa_set_status
;
2465 op_ret
= retry_raced_bucket_write(store
, s
, [&] {
2466 if (mfa_set_status
) {
2468 s
->bucket_info
.flags
|= BUCKET_MFA_ENABLED
;
2470 s
->bucket_info
.flags
&= ~BUCKET_MFA_ENABLED
;
2474 if (versioning_status
== VersioningEnabled
) {
2475 s
->bucket_info
.flags
|= BUCKET_VERSIONED
;
2476 s
->bucket_info
.flags
&= ~BUCKET_VERSIONS_SUSPENDED
;
2478 } else if (versioning_status
== VersioningSuspended
) {
2479 s
->bucket_info
.flags
|= (BUCKET_VERSIONED
| BUCKET_VERSIONS_SUSPENDED
);
2484 return store
->put_bucket_instance_info(s
->bucket_info
, false, real_time(),
2493 ldpp_dout(this, 0) << "NOTICE: put_bucket_info on bucket=" << s
->bucket
.name
2494 << " returned err=" << op_ret
<< dendl
;
2499 int RGWGetBucketWebsite::verify_permission()
2501 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketWebsite
);
2504 void RGWGetBucketWebsite::pre_exec()
2506 rgw_bucket_object_pre_exec(s
);
2509 void RGWGetBucketWebsite::execute()
2511 if (!s
->bucket_info
.has_website
) {
2512 op_ret
= -ERR_NO_SUCH_WEBSITE_CONFIGURATION
;
2516 int RGWSetBucketWebsite::verify_permission()
2518 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3PutBucketWebsite
);
2521 void RGWSetBucketWebsite::pre_exec()
2523 rgw_bucket_object_pre_exec(s
);
2526 void RGWSetBucketWebsite::execute()
2528 op_ret
= get_params();
2533 if (!store
->svc
.zone
->is_meta_master()) {
2534 op_ret
= forward_request_to_master(s
, NULL
, store
, in_data
, nullptr);
2536 ldpp_dout(this, 0) << " forward_request_to_master returned ret=" << op_ret
<< dendl
;
2541 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
2542 s
->bucket_info
.has_website
= true;
2543 s
->bucket_info
.website_conf
= website_conf
;
2544 op_ret
= store
->put_bucket_instance_info(s
->bucket_info
, false,
2545 real_time(), &s
->bucket_attrs
);
2550 ldpp_dout(this, 0) << "NOTICE: put_bucket_info on bucket=" << s
->bucket
.name
2551 << " returned err=" << op_ret
<< dendl
;
2556 int RGWDeleteBucketWebsite::verify_permission()
2558 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3DeleteBucketWebsite
);
2561 void RGWDeleteBucketWebsite::pre_exec()
2563 rgw_bucket_object_pre_exec(s
);
2566 void RGWDeleteBucketWebsite::execute()
2569 if (!store
->svc
.zone
->is_meta_master()) {
2571 op_ret
= forward_request_to_master(s
, nullptr, store
, in_data
, nullptr);
2573 ldpp_dout(this, 0) << "NOTICE: forward_to_master failed on bucket=" << s
->bucket
.name
2574 << "returned err=" << op_ret
<< dendl
;
2578 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
2579 s
->bucket_info
.has_website
= false;
2580 s
->bucket_info
.website_conf
= RGWBucketWebsiteConf();
2581 op_ret
= store
->put_bucket_instance_info(s
->bucket_info
, false,
2582 real_time(), &s
->bucket_attrs
);
2586 ldpp_dout(this, 0) << "NOTICE: put_bucket_info on bucket=" << s
->bucket
.name
2587 << " returned err=" << op_ret
<< dendl
;
2592 int RGWStatBucket::verify_permission()
2594 // This (a HEAD request on a bucket) is governed by the s3:ListBucket permission.
2595 if (!verify_bucket_permission(this, s
, rgw::IAM::s3ListBucket
)) {
2602 void RGWStatBucket::pre_exec()
2604 rgw_bucket_object_pre_exec(s
);
2607 void RGWStatBucket::execute()
2609 if (!s
->bucket_exists
) {
2610 op_ret
= -ERR_NO_SUCH_BUCKET
;
2614 RGWUserBuckets buckets
;
2615 bucket
.bucket
= s
->bucket
;
2616 buckets
.add(bucket
);
2617 map
<string
, RGWBucketEnt
>& m
= buckets
.get_buckets();
2618 op_ret
= store
->update_containers_stats(m
);
2623 map
<string
, RGWBucketEnt
>::iterator iter
= m
.find(bucket
.bucket
.name
);
2624 if (iter
!= m
.end()) {
2625 bucket
= iter
->second
;
2632 int RGWListBucket::verify_permission()
2634 op_ret
= get_params();
2638 if (!prefix
.empty())
2639 s
->env
.emplace("s3:prefix", prefix
);
2641 if (!delimiter
.empty())
2642 s
->env
.emplace("s3:delimiter", delimiter
);
2644 s
->env
.emplace("s3:max-keys", std::to_string(max
));
2646 if (!verify_bucket_permission(this,
2649 rgw::IAM::s3ListBucketVersions
:
2650 rgw::IAM::s3ListBucket
)) {
2657 int RGWListBucket::parse_max_keys()
2659 // Bound max value of max-keys to configured value for security
2660 // Bound min value of max-keys to '0'
2661 // Some S3 clients explicitly send max-keys=0 to detect if the bucket is
2662 // empty without listing any items.
2663 return parse_value_and_bound(max_keys
, max
, 0,
2664 g_conf().get_val
<uint64_t>("rgw_max_listing_results"),
2668 void RGWListBucket::pre_exec()
2670 rgw_bucket_object_pre_exec(s
);
2673 void RGWListBucket::execute()
2675 if (!s
->bucket_exists
) {
2676 op_ret
= -ERR_NO_SUCH_BUCKET
;
2680 if (allow_unordered
&& !delimiter
.empty()) {
2681 ldpp_dout(this, 0) <<
2682 "ERROR: unordered bucket listing requested with a delimiter" << dendl
;
2687 if (need_container_stats()) {
2688 map
<string
, RGWBucketEnt
> m
;
2689 m
[s
->bucket
.name
] = RGWBucketEnt();
2690 m
.begin()->second
.bucket
= s
->bucket
;
2691 op_ret
= store
->update_containers_stats(m
);
2693 bucket
= m
.begin()->second
;
2697 RGWRados::Bucket
target(store
, s
->bucket_info
);
2698 if (shard_id
>= 0) {
2699 target
.set_shard_id(shard_id
);
2701 RGWRados::Bucket::List
list_op(&target
);
2703 list_op
.params
.prefix
= prefix
;
2704 list_op
.params
.delim
= delimiter
;
2705 list_op
.params
.marker
= marker
;
2706 list_op
.params
.end_marker
= end_marker
;
2707 list_op
.params
.list_versions
= list_versions
;
2708 list_op
.params
.allow_unordered
= allow_unordered
;
2710 op_ret
= list_op
.list_objects(max
, &objs
, &common_prefixes
, &is_truncated
);
2712 next_marker
= list_op
.get_next_marker();
2716 int RGWGetBucketLogging::verify_permission()
2718 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketLogging
);
2721 int RGWGetBucketLocation::verify_permission()
2723 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketLocation
);
2726 int RGWCreateBucket::verify_permission()
2728 /* This check is mostly needed for S3 that doesn't support account ACL.
2729 * Swift doesn't allow to delegate any permission to an anonymous user,
2730 * so it will become an early exit in such case. */
2731 if (s
->auth
.identity
->is_anonymous()) {
2736 bucket
.name
= s
->bucket_name
;
2737 bucket
.tenant
= s
->bucket_tenant
;
2738 ARN arn
= ARN(bucket
);
2739 if (!verify_user_permission(this, s
, arn
, rgw::IAM::s3CreateBucket
)) {
2743 if (s
->user
->user_id
.tenant
!= s
->bucket_tenant
) {
2744 ldpp_dout(this, 10) << "user cannot create a bucket in a different tenant"
2745 << " (user_id.tenant=" << s
->user
->user_id
.tenant
2746 << " requested=" << s
->bucket_tenant
<< ")"
2750 if (s
->user
->max_buckets
< 0) {
2754 if (s
->user
->max_buckets
) {
2755 RGWUserBuckets buckets
;
2757 bool is_truncated
= false;
2758 op_ret
= rgw_read_user_buckets(store
, s
->user
->user_id
, buckets
,
2759 marker
, string(), s
->user
->max_buckets
,
2760 false, &is_truncated
);
2765 if ((int)buckets
.count() >= s
->user
->max_buckets
) {
2766 return -ERR_TOO_MANY_BUCKETS
;
2773 static int forward_request_to_master(struct req_state
*s
, obj_version
*objv
,
2774 RGWRados
*store
, bufferlist
& in_data
,
2775 JSONParser
*jp
, req_info
*forward_info
)
2777 if (!store
->svc
.zone
->get_master_conn()) {
2778 ldpp_dout(s
, 0) << "rest connection is invalid" << dendl
;
2781 ldpp_dout(s
, 0) << "sending request to master zonegroup" << dendl
;
2782 bufferlist response
;
2783 string uid_str
= s
->user
->user_id
.to_str();
2784 #define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response
2785 int ret
= store
->svc
.zone
->get_master_conn()->forward(uid_str
, (forward_info
? *forward_info
: s
->info
),
2786 objv
, MAX_REST_RESPONSE
, &in_data
, &response
);
2790 ldpp_dout(s
, 20) << "response: " << response
.c_str() << dendl
;
2791 if (jp
&& !jp
->parse(response
.c_str(), response
.length())) {
2792 ldpp_dout(s
, 0) << "failed parsing response from master zonegroup" << dendl
;
2799 void RGWCreateBucket::pre_exec()
2801 rgw_bucket_object_pre_exec(s
);
2804 static void prepare_add_del_attrs(const map
<string
, bufferlist
>& orig_attrs
,
2805 map
<string
, bufferlist
>& out_attrs
,
2806 map
<string
, bufferlist
>& out_rmattrs
)
2808 for (const auto& kv
: orig_attrs
) {
2809 const string
& name
= kv
.first
;
2811 /* Check if the attr is user-defined metadata item. */
2812 if (name
.compare(0, sizeof(RGW_ATTR_META_PREFIX
) - 1,
2813 RGW_ATTR_META_PREFIX
) == 0) {
2814 /* For the objects all existing meta attrs have to be removed. */
2815 out_rmattrs
[name
] = kv
.second
;
2816 } else if (out_attrs
.find(name
) == std::end(out_attrs
)) {
2817 out_attrs
[name
] = kv
.second
;
2822 /* Fuse resource metadata basing on original attributes in @orig_attrs, set
2823 * of _custom_ attribute names to remove in @rmattr_names and attributes in
2824 * @out_attrs. Place results in @out_attrs.
2826 * NOTE: it's supposed that all special attrs already present in @out_attrs
2827 * will be preserved without any change. Special attributes are those which
2828 * names start with RGW_ATTR_META_PREFIX. They're complement to custom ones
2829 * used for X-Account-Meta-*, X-Container-Meta-*, X-Amz-Meta and so on. */
2830 static void prepare_add_del_attrs(const map
<string
, bufferlist
>& orig_attrs
,
2831 const set
<string
>& rmattr_names
,
2832 map
<string
, bufferlist
>& out_attrs
)
2834 for (const auto& kv
: orig_attrs
) {
2835 const string
& name
= kv
.first
;
2837 /* Check if the attr is user-defined metadata item. */
2838 if (name
.compare(0, strlen(RGW_ATTR_META_PREFIX
),
2839 RGW_ATTR_META_PREFIX
) == 0) {
2840 /* For the buckets all existing meta attrs are preserved,
2841 except those that are listed in rmattr_names. */
2842 if (rmattr_names
.find(name
) != std::end(rmattr_names
)) {
2843 const auto aiter
= out_attrs
.find(name
);
2845 if (aiter
!= std::end(out_attrs
)) {
2846 out_attrs
.erase(aiter
);
2849 /* emplace() won't alter the map if the key is already present.
2850 * This behaviour is fully intensional here. */
2851 out_attrs
.emplace(kv
);
2853 } else if (out_attrs
.find(name
) == std::end(out_attrs
)) {
2854 out_attrs
[name
] = kv
.second
;
2860 static void populate_with_generic_attrs(const req_state
* const s
,
2861 map
<string
, bufferlist
>& out_attrs
)
2863 for (const auto& kv
: s
->generic_attrs
) {
2864 bufferlist
& attrbl
= out_attrs
[kv
.first
];
2865 const string
& val
= kv
.second
;
2867 attrbl
.append(val
.c_str(), val
.size() + 1);
2872 static int filter_out_quota_info(std::map
<std::string
, bufferlist
>& add_attrs
,
2873 const std::set
<std::string
>& rmattr_names
,
2874 RGWQuotaInfo
& quota
,
2875 bool * quota_extracted
= nullptr)
2877 bool extracted
= false;
2879 /* Put new limit on max objects. */
2880 auto iter
= add_attrs
.find(RGW_ATTR_QUOTA_NOBJS
);
2882 if (std::end(add_attrs
) != iter
) {
2884 static_cast<int64_t>(strict_strtoll(iter
->second
.c_str(), 10, &err
));
2888 add_attrs
.erase(iter
);
2892 /* Put new limit on bucket (container) size. */
2893 iter
= add_attrs
.find(RGW_ATTR_QUOTA_MSIZE
);
2894 if (iter
!= add_attrs
.end()) {
2896 static_cast<int64_t>(strict_strtoll(iter
->second
.c_str(), 10, &err
));
2900 add_attrs
.erase(iter
);
2904 for (const auto& name
: rmattr_names
) {
2905 /* Remove limit on max objects. */
2906 if (name
.compare(RGW_ATTR_QUOTA_NOBJS
) == 0) {
2907 quota
.max_objects
= -1;
2911 /* Remove limit on max bucket size. */
2912 if (name
.compare(RGW_ATTR_QUOTA_MSIZE
) == 0) {
2913 quota
.max_size
= -1;
2918 /* Swift requries checking on raw usage instead of the 4 KiB rounded one. */
2919 quota
.check_on_raw
= true;
2920 quota
.enabled
= quota
.max_size
> 0 || quota
.max_objects
> 0;
2922 if (quota_extracted
) {
2923 *quota_extracted
= extracted
;
2930 static void filter_out_website(std::map
<std::string
, ceph::bufferlist
>& add_attrs
,
2931 const std::set
<std::string
>& rmattr_names
,
2932 RGWBucketWebsiteConf
& ws_conf
)
2936 /* Let's define a mapping between each custom attribute and the memory where
2937 * attribute's value should be stored. The memory location is expressed by
2938 * a non-const reference. */
2939 const auto mapping
= {
2940 std::make_pair(RGW_ATTR_WEB_INDEX
, std::ref(ws_conf
.index_doc_suffix
)),
2941 std::make_pair(RGW_ATTR_WEB_ERROR
, std::ref(ws_conf
.error_doc
)),
2942 std::make_pair(RGW_ATTR_WEB_LISTINGS
, std::ref(lstval
)),
2943 std::make_pair(RGW_ATTR_WEB_LIST_CSS
, std::ref(ws_conf
.listing_css_doc
)),
2944 std::make_pair(RGW_ATTR_SUBDIR_MARKER
, std::ref(ws_conf
.subdir_marker
))
2947 for (const auto& kv
: mapping
) {
2948 const char * const key
= kv
.first
;
2949 auto& target
= kv
.second
;
2951 auto iter
= add_attrs
.find(key
);
2953 if (std::end(add_attrs
) != iter
) {
2954 /* The "target" is a reference to ws_conf. */
2955 target
= iter
->second
.c_str();
2956 add_attrs
.erase(iter
);
2959 if (rmattr_names
.count(key
)) {
2960 target
= std::string();
2964 if (! lstval
.empty()) {
2965 ws_conf
.listing_enabled
= boost::algorithm::iequals(lstval
, "true");
2970 void RGWCreateBucket::execute()
2972 RGWAccessControlPolicy
old_policy(s
->cct
);
2974 buffer::list corsbl
;
2977 rgw_make_bucket_entry_name(s
->bucket_tenant
, s
->bucket_name
, bucket_name
);
2978 rgw_raw_obj
obj(store
->svc
.zone
->get_zone_params().domain_root
, bucket_name
);
2979 obj_version objv
, *pobjv
= NULL
;
2981 op_ret
= get_params();
2985 if (!relaxed_region_enforcement
&&
2986 !location_constraint
.empty() &&
2987 !store
->svc
.zone
->has_zonegroup_api(location_constraint
)) {
2988 ldpp_dout(this, 0) << "location constraint (" << location_constraint
<< ")"
2989 << " can't be found." << dendl
;
2990 op_ret
= -ERR_INVALID_LOCATION_CONSTRAINT
;
2991 s
->err
.message
= "The specified location-constraint is not valid";
2995 if (!relaxed_region_enforcement
&& !store
->svc
.zone
->get_zonegroup().is_master_zonegroup() && !location_constraint
.empty() &&
2996 store
->svc
.zone
->get_zonegroup().api_name
!= location_constraint
) {
2997 ldpp_dout(this, 0) << "location constraint (" << location_constraint
<< ")"
2998 << " doesn't match zonegroup" << " (" << store
->svc
.zone
->get_zonegroup().api_name
<< ")"
3000 op_ret
= -ERR_INVALID_LOCATION_CONSTRAINT
;
3001 s
->err
.message
= "The specified location-constraint is not valid";
3005 const auto& zonegroup
= store
->svc
.zone
->get_zonegroup();
3006 if (!placement_rule
.name
.empty() &&
3007 !zonegroup
.placement_targets
.count(placement_rule
.name
)) {
3008 ldpp_dout(this, 0) << "placement target (" << placement_rule
.name
<< ")"
3009 << " doesn't exist in the placement targets of zonegroup"
3010 << " (" << store
->svc
.zone
->get_zonegroup().api_name
<< ")" << dendl
;
3011 op_ret
= -ERR_INVALID_LOCATION_CONSTRAINT
;
3012 s
->err
.message
= "The specified placement target does not exist";
3016 /* we need to make sure we read bucket info, it's not read before for this
3017 * specific request */
3018 op_ret
= store
->get_bucket_info(*s
->sysobj_ctx
, s
->bucket_tenant
, s
->bucket_name
,
3019 s
->bucket_info
, nullptr, &s
->bucket_attrs
);
3020 if (op_ret
< 0 && op_ret
!= -ENOENT
)
3022 s
->bucket_exists
= (op_ret
!= -ENOENT
);
3024 s
->bucket_owner
.set_id(s
->user
->user_id
);
3025 s
->bucket_owner
.set_name(s
->user
->display_name
);
3026 if (s
->bucket_exists
) {
3027 int r
= rgw_op_get_bucket_policy_from_attr(s
->cct
, store
, s
->bucket_info
,
3028 s
->bucket_attrs
, &old_policy
);
3030 if (old_policy
.get_owner().get_id().compare(s
->user
->user_id
) != 0) {
3037 RGWBucketInfo master_info
;
3038 rgw_bucket
*pmaster_bucket
;
3039 uint32_t *pmaster_num_shards
;
3040 real_time creation_time
;
3042 if (!store
->svc
.zone
->is_meta_master()) {
3044 op_ret
= forward_request_to_master(s
, NULL
, store
, in_data
, &jp
);
3049 JSONDecoder::decode_json("entry_point_object_ver", ep_objv
, &jp
);
3050 JSONDecoder::decode_json("object_ver", objv
, &jp
);
3051 JSONDecoder::decode_json("bucket_info", master_info
, &jp
);
3052 ldpp_dout(this, 20) << "parsed: objv.tag=" << objv
.tag
<< " objv.ver=" << objv
.ver
<< dendl
;
3053 ldpp_dout(this, 20) << "got creation time: << " << master_info
.creation_time
<< dendl
;
3054 pmaster_bucket
= &master_info
.bucket
;
3055 creation_time
= master_info
.creation_time
;
3056 pmaster_num_shards
= &master_info
.num_shards
;
3058 obj_lock_enabled
= master_info
.obj_lock_enabled();
3060 pmaster_bucket
= NULL
;
3061 pmaster_num_shards
= NULL
;
3064 string zonegroup_id
;
3066 if (s
->system_request
) {
3067 zonegroup_id
= s
->info
.args
.get(RGW_SYS_PARAM_PREFIX
"zonegroup");
3068 if (zonegroup_id
.empty()) {
3069 zonegroup_id
= store
->svc
.zone
->get_zonegroup().get_id();
3072 zonegroup_id
= store
->svc
.zone
->get_zonegroup().get_id();
3075 if (s
->bucket_exists
) {
3076 rgw_placement_rule selected_placement_rule
;
3078 bucket
.tenant
= s
->bucket_tenant
;
3079 bucket
.name
= s
->bucket_name
;
3080 op_ret
= store
->svc
.zone
->select_bucket_placement(*(s
->user
), zonegroup_id
,
3082 &selected_placement_rule
, nullptr);
3083 if (selected_placement_rule
!= s
->bucket_info
.placement_rule
) {
3089 /* Encode special metadata first as we're using std::map::emplace under
3090 * the hood. This method will add the new items only if the map doesn't
3091 * contain such keys yet. */
3092 policy
.encode(aclbl
);
3093 emplace_attr(RGW_ATTR_ACL
, std::move(aclbl
));
3096 cors_config
.encode(corsbl
);
3097 emplace_attr(RGW_ATTR_CORS
, std::move(corsbl
));
3100 RGWQuotaInfo quota_info
;
3101 const RGWQuotaInfo
* pquota_info
= nullptr;
3102 if (need_metadata_upload()) {
3103 /* It's supposed that following functions WILL NOT change any special
3104 * attributes (like RGW_ATTR_ACL) if they are already present in attrs. */
3105 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
, false);
3109 prepare_add_del_attrs(s
->bucket_attrs
, rmattr_names
, attrs
);
3110 populate_with_generic_attrs(s
, attrs
);
3112 op_ret
= filter_out_quota_info(attrs
, rmattr_names
, quota_info
);
3116 pquota_info
= "a_info
;
3119 /* Web site of Swift API. */
3120 filter_out_website(attrs
, rmattr_names
, s
->bucket_info
.website_conf
);
3121 s
->bucket_info
.has_website
= !s
->bucket_info
.website_conf
.is_empty();
3124 s
->bucket
.tenant
= s
->bucket_tenant
; /* ignored if bucket exists */
3125 s
->bucket
.name
= s
->bucket_name
;
3127 /* Handle updates of the metadata for Swift's object versioning. */
3128 if (swift_ver_location
) {
3129 s
->bucket_info
.swift_ver_location
= *swift_ver_location
;
3130 s
->bucket_info
.swift_versioning
= (! swift_ver_location
->empty());
3132 if (obj_lock_enabled
) {
3133 info
.flags
= BUCKET_VERSIONED
| BUCKET_OBJ_LOCK_ENABLED
;
3137 op_ret
= store
->create_bucket(*(s
->user
), s
->bucket
, zonegroup_id
,
3138 placement_rule
, s
->bucket_info
.swift_ver_location
,
3140 info
, pobjv
, &ep_objv
, creation_time
,
3141 pmaster_bucket
, pmaster_num_shards
, true);
3142 /* continue if EEXIST and create_bucket will fail below. this way we can
3143 * recover from a partial create by retrying it. */
3144 ldpp_dout(this, 20) << "rgw_create_bucket returned ret=" << op_ret
<< " bucket=" << s
->bucket
<< dendl
;
3146 if (op_ret
&& op_ret
!= -EEXIST
)
3149 existed
= (op_ret
== -EEXIST
);
3152 /* bucket already existed, might have raced with another bucket creation, or
3153 * might be partial bucket creation that never completed. Read existing bucket
3154 * info, verify that the reported bucket owner is the current user.
3155 * If all is ok then update the user's list of buckets.
3156 * Otherwise inform client about a name conflict.
3158 if (info
.owner
.compare(s
->user
->user_id
) != 0) {
3162 s
->bucket
= info
.bucket
;
3165 op_ret
= rgw_link_bucket(store
, s
->user
->user_id
, s
->bucket
,
3166 info
.creation_time
, false);
3167 if (op_ret
&& !existed
&& op_ret
!= -EEXIST
) {
3168 /* if it exists (or previously existed), don't remove it! */
3169 op_ret
= rgw_unlink_bucket(store
, s
->user
->user_id
, s
->bucket
.tenant
,
3172 ldpp_dout(this, 0) << "WARNING: failed to unlink bucket: ret=" << op_ret
3175 } else if (op_ret
== -EEXIST
|| (op_ret
== 0 && existed
)) {
3176 op_ret
= -ERR_BUCKET_EXISTS
;
3179 if (need_metadata_upload() && existed
) {
3180 /* OK, it looks we lost race with another request. As it's required to
3181 * handle metadata fusion and upload, the whole operation becomes very
3182 * similar in nature to PutMetadataBucket. However, as the attrs may
3183 * changed in the meantime, we have to refresh. */
3186 RGWBucketInfo binfo
;
3187 map
<string
, bufferlist
> battrs
;
3189 op_ret
= store
->get_bucket_info(*s
->sysobj_ctx
, s
->bucket_tenant
, s
->bucket_name
,
3190 binfo
, nullptr, &battrs
);
3193 } else if (binfo
.owner
.compare(s
->user
->user_id
) != 0) {
3194 /* New bucket doesn't belong to the account we're operating on. */
3198 s
->bucket_info
= binfo
;
3199 s
->bucket_attrs
= battrs
;
3204 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
, false);
3208 prepare_add_del_attrs(s
->bucket_attrs
, rmattr_names
, attrs
);
3209 populate_with_generic_attrs(s
, attrs
);
3210 op_ret
= filter_out_quota_info(attrs
, rmattr_names
, s
->bucket_info
.quota
);
3215 /* Handle updates of the metadata for Swift's object versioning. */
3216 if (swift_ver_location
) {
3217 s
->bucket_info
.swift_ver_location
= *swift_ver_location
;
3218 s
->bucket_info
.swift_versioning
= (! swift_ver_location
->empty());
3221 /* Web site of Swift API. */
3222 filter_out_website(attrs
, rmattr_names
, s
->bucket_info
.website_conf
);
3223 s
->bucket_info
.has_website
= !s
->bucket_info
.website_conf
.is_empty();
3225 /* This will also set the quota on the bucket. */
3226 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
,
3227 &s
->bucket_info
.objv_tracker
);
3228 } while (op_ret
== -ECANCELED
&& tries
++ < 20);
3230 /* Restore the proper return code. */
3232 op_ret
= -ERR_BUCKET_EXISTS
;
3237 int RGWDeleteBucket::verify_permission()
3239 if (!verify_bucket_permission(this, s
, rgw::IAM::s3DeleteBucket
)) {
3246 void RGWDeleteBucket::pre_exec()
3248 rgw_bucket_object_pre_exec(s
);
3251 void RGWDeleteBucket::execute()
3253 if (s
->bucket_name
.empty()) {
3258 if (!s
->bucket_exists
) {
3259 ldpp_dout(this, 0) << "ERROR: bucket " << s
->bucket_name
<< " not found" << dendl
;
3260 op_ret
= -ERR_NO_SUCH_BUCKET
;
3263 RGWObjVersionTracker ot
;
3264 ot
.read_version
= s
->bucket_info
.ep_objv
;
3266 if (s
->system_request
) {
3267 string tag
= s
->info
.args
.get(RGW_SYS_PARAM_PREFIX
"tag");
3268 string ver_str
= s
->info
.args
.get(RGW_SYS_PARAM_PREFIX
"ver");
3270 ot
.read_version
.tag
= tag
;
3273 ver
= strict_strtol(ver_str
.c_str(), 10, &err
);
3275 ldpp_dout(this, 0) << "failed to parse ver param" << dendl
;
3279 ot
.read_version
.ver
= ver
;
3283 op_ret
= rgw_bucket_sync_user_stats(store
, s
->user
->user_id
, s
->bucket_info
);
3285 ldpp_dout(this, 1) << "WARNING: failed to sync user stats before bucket delete: op_ret= " << op_ret
<< dendl
;
3288 op_ret
= store
->check_bucket_empty(s
->bucket_info
);
3293 if (!store
->svc
.zone
->is_meta_master()) {
3295 op_ret
= forward_request_to_master(s
, &ot
.read_version
, store
, in_data
,
3298 if (op_ret
== -ENOENT
) {
3299 /* adjust error, we want to return with NoSuchBucket and not
3301 op_ret
= -ERR_NO_SUCH_BUCKET
;
3307 string prefix
, delimiter
;
3309 if (s
->prot_flags
& RGW_REST_SWIFT
) {
3311 path_args
= s
->info
.args
.get("path");
3312 if (!path_args
.empty()) {
3313 if (!delimiter
.empty() || !prefix
.empty()) {
3322 op_ret
= abort_bucket_multiparts(store
, s
->cct
, s
->bucket_info
, prefix
, delimiter
);
3328 op_ret
= store
->delete_bucket(s
->bucket_info
, ot
, false);
3330 if (op_ret
== -ECANCELED
) {
3331 // lost a race, either with mdlog sync or another delete bucket operation.
3332 // in either case, we've already called rgw_unlink_bucket()
3338 op_ret
= rgw_unlink_bucket(store
, s
->bucket_info
.owner
, s
->bucket
.tenant
,
3339 s
->bucket
.name
, false);
3341 ldpp_dout(this, 0) << "WARNING: failed to unlink bucket: ret=" << op_ret
3347 int RGWPutObj::verify_permission()
3349 if (! copy_source
.empty()) {
3351 RGWAccessControlPolicy
cs_acl(s
->cct
);
3352 boost::optional
<Policy
> policy
;
3353 map
<string
, bufferlist
> cs_attrs
;
3354 rgw_bucket
cs_bucket(copy_source_bucket_info
.bucket
);
3355 rgw_obj_key
cs_object(copy_source_object_name
, copy_source_version_id
);
3357 rgw_obj
obj(cs_bucket
, cs_object
);
3358 store
->set_atomic(s
->obj_ctx
, obj
);
3359 store
->set_prefetch_data(s
->obj_ctx
, obj
);
3361 /* check source object permissions */
3362 if (read_obj_policy(store
, s
, copy_source_bucket_info
, cs_attrs
, &cs_acl
, nullptr,
3363 policy
, cs_bucket
, cs_object
) < 0) {
3367 /* admin request overrides permission checks */
3368 if (! s
->auth
.identity
->is_admin_of(cs_acl
.get_owner().get_id())) {
3369 if (policy
|| ! s
->iam_user_policies
.empty()) {
3370 auto usr_policy_res
= Effect::Pass
;
3371 for (auto& user_policy
: s
->iam_user_policies
) {
3372 if (usr_policy_res
= user_policy
.eval(s
->env
, *s
->auth
.identity
,
3373 cs_object
.instance
.empty() ?
3374 rgw::IAM::s3GetObject
:
3375 rgw::IAM::s3GetObjectVersion
,
3376 rgw::ARN(obj
)); usr_policy_res
== Effect::Deny
)
3378 else if (usr_policy_res
== Effect::Allow
)
3381 rgw::IAM::Effect e
= Effect::Pass
;
3383 e
= policy
->eval(s
->env
, *s
->auth
.identity
,
3384 cs_object
.instance
.empty() ?
3385 rgw::IAM::s3GetObject
:
3386 rgw::IAM::s3GetObjectVersion
,
3389 if (e
== Effect::Deny
) {
3391 } else if (usr_policy_res
== Effect::Pass
&& e
== Effect::Pass
&&
3392 !cs_acl
.verify_permission(this, *s
->auth
.identity
, s
->perm_mask
,
3396 } else if (!cs_acl
.verify_permission(this, *s
->auth
.identity
, s
->perm_mask
,
3403 auto op_ret
= get_params();
3405 ldpp_dout(this, 20) << "get_params() returned ret=" << op_ret
<< dendl
;
3409 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
3410 rgw_add_grant_to_iam_environment(s
->env
, s
);
3412 rgw_add_to_iam_environment(s
->env
, "s3:x-amz-acl", s
->canned_acl
);
3414 if (obj_tags
!= nullptr && obj_tags
->count() > 0){
3415 auto tags
= obj_tags
->get_tags();
3416 for (const auto& kv
: tags
){
3417 rgw_add_to_iam_environment(s
->env
, "s3:RequestObjectTag/"+kv
.first
, kv
.second
);
3421 constexpr auto encrypt_attr
= "x-amz-server-side-encryption";
3422 constexpr auto s3_encrypt_attr
= "s3:x-amz-server-side-encryption";
3423 auto enc_header
= s
->info
.x_meta_map
.find(encrypt_attr
);
3424 if (enc_header
!= s
->info
.x_meta_map
.end()){
3425 rgw_add_to_iam_environment(s
->env
, s3_encrypt_attr
, enc_header
->second
);
3428 constexpr auto kms_attr
= "x-amz-server-side-encryption-aws-kms-key-id";
3429 constexpr auto s3_kms_attr
= "s3:x-amz-server-side-encryption-aws-kms-key-id";
3430 auto kms_header
= s
->info
.x_meta_map
.find(kms_attr
);
3431 if (kms_header
!= s
->info
.x_meta_map
.end()){
3432 rgw_add_to_iam_environment(s
->env
, s3_kms_attr
, kms_header
->second
);
3435 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
3437 rgw::IAM::s3PutObject
,
3438 rgw_obj(s
->bucket
, s
->object
));
3439 if (usr_policy_res
== Effect::Deny
)
3442 rgw::IAM::Effect e
= Effect::Pass
;
3443 if (s
->iam_policy
) {
3444 e
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
3445 rgw::IAM::s3PutObject
,
3446 rgw_obj(s
->bucket
, s
->object
));
3448 if (e
== Effect::Allow
) {
3450 } else if (e
== Effect::Deny
) {
3452 } else if (usr_policy_res
== Effect::Allow
) {
3457 if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
3465 void RGWPutObj::pre_exec()
3467 rgw_bucket_object_pre_exec(s
);
3470 class RGWPutObj_CB
: public RGWGetObj_Filter
3474 explicit RGWPutObj_CB(RGWPutObj
*_op
) : op(_op
) {}
3475 ~RGWPutObj_CB() override
{}
3477 int handle_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
) override
{
3478 return op
->get_data_cb(bl
, bl_ofs
, bl_len
);
3482 int RGWPutObj::get_data_cb(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
)
3485 bl
.copy(bl_ofs
, bl_len
, bl_tmp
);
3487 bl_aux
.append(bl_tmp
);
3492 int RGWPutObj::get_data(const off_t fst
, const off_t lst
, bufferlist
& bl
)
3494 RGWPutObj_CB
cb(this);
3495 RGWGetObj_Filter
* filter
= &cb
;
3496 boost::optional
<RGWGetObj_Decompress
> decompress
;
3497 std::unique_ptr
<RGWGetObj_Filter
> decrypt
;
3498 RGWCompressionInfo cs_info
;
3499 map
<string
, bufferlist
> attrs
;
3500 map
<string
, bufferlist
>::iterator attr_iter
;
3504 int64_t new_ofs
, new_end
;
3509 rgw_obj_key
obj_key(copy_source_object_name
, copy_source_version_id
);
3510 rgw_obj
obj(copy_source_bucket_info
.bucket
, obj_key
);
3512 RGWRados::Object
op_target(store
, copy_source_bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
3513 RGWRados::Object::Read
read_op(&op_target
);
3514 read_op
.params
.obj_size
= &obj_size
;
3515 read_op
.params
.attrs
= &attrs
;
3517 ret
= read_op
.prepare();
3521 bool need_decompress
;
3522 op_ret
= rgw_compression_info_from_attrset(attrs
, need_decompress
, cs_info
);
3524 ldpp_dout(s
, 0) << "ERROR: failed to decode compression info" << dendl
;
3528 bool partial_content
= true;
3529 if (need_decompress
)
3531 obj_size
= cs_info
.orig_size
;
3532 decompress
.emplace(s
->cct
, &cs_info
, partial_content
, filter
);
3533 filter
= &*decompress
;
3536 attr_iter
= attrs
.find(RGW_ATTR_MANIFEST
);
3537 op_ret
= this->get_decrypt_filter(&decrypt
,
3540 attr_iter
!= attrs
.end() ? &(attr_iter
->second
) : nullptr);
3541 if (decrypt
!= nullptr) {
3542 filter
= decrypt
.get();
3548 ret
= read_op
.range_to_ofs(obj_size
, new_ofs
, new_end
);
3552 filter
->fixup_range(new_ofs
, new_end
);
3553 ret
= read_op
.iterate(new_ofs
, new_end
, filter
);
3556 ret
= filter
->flush();
3558 bl
.claim_append(bl_aux
);
3563 // special handling for compression type = "random" with multipart uploads
3564 static CompressorRef
get_compressor_plugin(const req_state
*s
,
3565 const std::string
& compression_type
)
3567 if (compression_type
!= "random") {
3568 return Compressor::create(s
->cct
, compression_type
);
3571 bool is_multipart
{false};
3572 const auto& upload_id
= s
->info
.args
.get("uploadId", &is_multipart
);
3574 if (!is_multipart
) {
3575 return Compressor::create(s
->cct
, compression_type
);
3578 // use a hash of the multipart upload id so all parts use the same plugin
3579 const auto alg
= std::hash
<std::string
>{}(upload_id
) % Compressor::COMP_ALG_LAST
;
3580 if (alg
== Compressor::COMP_ALG_NONE
) {
3583 return Compressor::create(s
->cct
, alg
);
3586 void RGWPutObj::execute()
3588 char supplied_md5_bin
[CEPH_CRYPTO_MD5_DIGESTSIZE
+ 1];
3589 char supplied_md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
3590 char calc_md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
3591 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
3593 bufferlist bl
, aclbl
, bs
;
3599 bool need_calc_md5
= (dlo_manifest
== NULL
) && (slo_info
== NULL
);
3600 perfcounter
->inc(l_rgw_put
);
3601 // report latency on return
3602 auto put_lat
= make_scope_guard([&] {
3603 perfcounter
->tinc(l_rgw_put_lat
, s
->time_elapsed());
3607 if (s
->object
.empty()) {
3611 if (!s
->bucket_exists
) {
3612 op_ret
= -ERR_NO_SUCH_BUCKET
;
3617 op_ret
= get_system_versioning_params(s
, &olh_epoch
, &version_id
);
3619 ldpp_dout(this, 20) << "get_system_versioning_params() returned ret="
3624 if (supplied_md5_b64
) {
3625 need_calc_md5
= true;
3627 ldpp_dout(this, 15) << "supplied_md5_b64=" << supplied_md5_b64
<< dendl
;
3628 op_ret
= ceph_unarmor(supplied_md5_bin
, &supplied_md5_bin
[CEPH_CRYPTO_MD5_DIGESTSIZE
+ 1],
3629 supplied_md5_b64
, supplied_md5_b64
+ strlen(supplied_md5_b64
));
3630 ldpp_dout(this, 15) << "ceph_armor ret=" << op_ret
<< dendl
;
3631 if (op_ret
!= CEPH_CRYPTO_MD5_DIGESTSIZE
) {
3632 op_ret
= -ERR_INVALID_DIGEST
;
3636 buf_to_hex((const unsigned char *)supplied_md5_bin
, CEPH_CRYPTO_MD5_DIGESTSIZE
, supplied_md5
);
3637 ldpp_dout(this, 15) << "supplied_md5=" << supplied_md5
<< dendl
;
3640 if (!chunked_upload
) { /* with chunked upload we don't know how big is the upload.
3641 we also check sizes at the end anyway */
3642 op_ret
= store
->check_quota(s
->bucket_owner
.get_id(), s
->bucket
,
3643 user_quota
, bucket_quota
, s
->content_length
);
3645 ldpp_dout(this, 20) << "check_quota() returned ret=" << op_ret
<< dendl
;
3648 op_ret
= store
->check_bucket_shards(s
->bucket_info
, s
->bucket
, bucket_quota
);
3650 ldpp_dout(this, 20) << "check_bucket_shards() returned ret=" << op_ret
<< dendl
;
3655 if (supplied_etag
) {
3656 strncpy(supplied_md5
, supplied_etag
, sizeof(supplied_md5
) - 1);
3657 supplied_md5
[sizeof(supplied_md5
) - 1] = '\0';
3660 const bool multipart
= !multipart_upload_id
.empty();
3661 auto& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
3662 rgw_obj obj
{s
->bucket
, s
->object
};
3664 /* Handle object versioning of Swift API. */
3666 op_ret
= store
->swift_versioning_copy(obj_ctx
,
3667 s
->bucket_owner
.get_id(),
3675 // create the object processor
3676 rgw::AioThrottle
aio(store
->ctx()->_conf
->rgw_put_obj_min_window_size
);
3677 using namespace rgw::putobj
;
3678 constexpr auto max_processor_size
= std::max({sizeof(MultipartObjectProcessor
),
3679 sizeof(AtomicObjectProcessor
),
3680 sizeof(AppendObjectProcessor
)});
3681 ceph::static_ptr
<ObjectProcessor
, max_processor_size
> processor
;
3683 rgw_placement_rule
*pdest_placement
;
3685 multipart_upload_info upload_info
;
3687 RGWMPObj
mp(s
->object
.name
, multipart_upload_id
);
3689 op_ret
= get_multipart_info(store
, s
, mp
.get_meta(), nullptr, nullptr, &upload_info
);
3691 if (op_ret
!= -ENOENT
) {
3692 ldpp_dout(this, 0) << "ERROR: get_multipart_info returned " << op_ret
<< ": " << cpp_strerror(-op_ret
) << dendl
;
3693 } else {// -ENOENT: raced with upload complete/cancel, no need to spam log
3694 ldpp_dout(this, 20) << "failed to get multipart info (returned " << op_ret
<< ": " << cpp_strerror(-op_ret
) << "): probably raced with upload complete / cancel" << dendl
;
3698 pdest_placement
= &upload_info
.dest_placement
;
3699 ldpp_dout(this, 20) << "dest_placement for part=" << upload_info
.dest_placement
<< dendl
;
3700 processor
.emplace
<MultipartObjectProcessor
>(
3701 &aio
, store
, s
->bucket_info
, pdest_placement
,
3702 s
->owner
.get_id(), obj_ctx
, obj
,
3703 multipart_upload_id
, multipart_part_num
, multipart_part_str
);
3705 if (s
->bucket_info
.versioned()) {
3706 op_ret
= -ERR_INVALID_BUCKET_STATE
;
3709 pdest_placement
= &s
->dest_placement
;
3710 processor
.emplace
<AppendObjectProcessor
>(
3711 &aio
, store
, s
->bucket_info
, pdest_placement
, s
->bucket_owner
.get_id(),obj_ctx
, obj
,
3712 s
->req_id
, position
, &cur_accounted_size
);
3714 if (s
->bucket_info
.versioning_enabled()) {
3715 if (!version_id
.empty()) {
3716 obj
.key
.set_instance(version_id
);
3718 store
->gen_rand_obj_instance_name(&obj
);
3719 version_id
= obj
.key
.instance
;
3722 pdest_placement
= &s
->dest_placement
;
3723 processor
.emplace
<AtomicObjectProcessor
>(
3724 &aio
, store
, s
->bucket_info
, pdest_placement
,
3725 s
->bucket_owner
.get_id(), obj_ctx
, obj
, olh_epoch
, s
->req_id
);
3728 op_ret
= processor
->prepare();
3730 ldpp_dout(this, 20) << "processor->prepare() returned ret=" << op_ret
3735 if ((! copy_source
.empty()) && !copy_source_range
) {
3736 rgw_obj_key
obj_key(copy_source_object_name
, copy_source_version_id
);
3737 rgw_obj
obj(copy_source_bucket_info
.bucket
, obj_key
.name
);
3739 RGWObjState
*astate
;
3740 op_ret
= store
->get_obj_state(&obj_ctx
, copy_source_bucket_info
, obj
,
3741 &astate
, true, false);
3743 ldpp_dout(this, 0) << "ERROR: get copy source obj state returned with error" << op_ret
<< dendl
;
3746 if (!astate
->exists
){
3750 lst
= astate
->accounted_size
- 1;
3752 lst
= copy_source_range_lst
;
3755 fst
= copy_source_range_fst
;
3757 // no filters by default
3758 DataProcessor
*filter
= processor
.get();
3760 const auto& compression_type
= store
->svc
.zone
->get_zone_params().get_compression_type(*pdest_placement
);
3761 CompressorRef plugin
;
3762 boost::optional
<RGWPutObj_Compress
> compressor
;
3764 std::unique_ptr
<DataProcessor
> encrypt
;
3766 if (!append
) { // compression and encryption only apply to full object uploads
3767 op_ret
= get_encrypt_filter(&encrypt
, filter
);
3771 if (encrypt
!= nullptr) {
3773 } else if (compression_type
!= "none") {
3774 plugin
= get_compressor_plugin(s
, compression_type
);
3776 ldpp_dout(this, 1) << "Cannot load plugin for compression type "
3777 << compression_type
<< dendl
;
3779 compressor
.emplace(s
->cct
, plugin
, filter
);
3780 filter
= &*compressor
;
3784 tracepoint(rgw_op
, before_data_transfer
, s
->req_id
.c_str());
3789 if (copy_source
.empty()) {
3790 len
= get_data(data
);
3792 uint64_t cur_lst
= min(fst
+ s
->cct
->_conf
->rgw_max_chunk_size
- 1, lst
);
3793 op_ret
= get_data(fst
, cur_lst
, data
);
3796 len
= data
.length();
3797 s
->content_length
+= len
;
3802 ldpp_dout(this, 20) << "get_data() returned ret=" << op_ret
<< dendl
;
3804 } else if (len
== 0) {
3808 if (need_calc_md5
) {
3809 hash
.Update((const unsigned char *)data
.c_str(), data
.length());
3812 /* update torrrent */
3813 torrent
.update(data
);
3815 op_ret
= filter
->process(std::move(data
), ofs
);
3817 ldpp_dout(this, 20) << "processor->process() returned ret="
3824 tracepoint(rgw_op
, after_data_transfer
, s
->req_id
.c_str(), ofs
);
3826 // flush any data in filters
3827 op_ret
= filter
->process({}, ofs
);
3832 if (!chunked_upload
&& ofs
!= s
->content_length
) {
3833 op_ret
= -ERR_REQUEST_TIMEOUT
;
3838 perfcounter
->inc(l_rgw_put_b
, s
->obj_size
);
3840 op_ret
= do_aws4_auth_completion();
3845 op_ret
= store
->check_quota(s
->bucket_owner
.get_id(), s
->bucket
,
3846 user_quota
, bucket_quota
, s
->obj_size
);
3848 ldpp_dout(this, 20) << "second check_quota() returned op_ret=" << op_ret
<< dendl
;
3852 op_ret
= store
->check_bucket_shards(s
->bucket_info
, s
->bucket
, bucket_quota
);
3854 ldpp_dout(this, 20) << "check_bucket_shards() returned ret=" << op_ret
<< dendl
;
3860 if (compressor
&& compressor
->is_compressed()) {
3862 RGWCompressionInfo cs_info
;
3863 cs_info
.compression_type
= plugin
->get_type_name();
3864 cs_info
.orig_size
= s
->obj_size
;
3865 cs_info
.blocks
= move(compressor
->get_compression_blocks());
3866 encode(cs_info
, tmp
);
3867 attrs
[RGW_ATTR_COMPRESSION
] = tmp
;
3868 ldpp_dout(this, 20) << "storing " << RGW_ATTR_COMPRESSION
3869 << " with type=" << cs_info
.compression_type
3870 << ", orig_size=" << cs_info
.orig_size
3871 << ", blocks=" << cs_info
.blocks
.size() << dendl
;
3874 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, calc_md5
);
3878 if (supplied_md5_b64
&& strcmp(calc_md5
, supplied_md5
)) {
3879 op_ret
= -ERR_BAD_DIGEST
;
3883 policy
.encode(aclbl
);
3884 emplace_attr(RGW_ATTR_ACL
, std::move(aclbl
));
3887 op_ret
= encode_dlo_manifest_attr(dlo_manifest
, attrs
);
3889 ldpp_dout(this, 0) << "bad user manifest: " << dlo_manifest
<< dendl
;
3895 bufferlist manifest_bl
;
3896 encode(*slo_info
, manifest_bl
);
3897 emplace_attr(RGW_ATTR_SLO_MANIFEST
, std::move(manifest_bl
));
3900 if (supplied_etag
&& etag
.compare(supplied_etag
) != 0) {
3901 op_ret
= -ERR_UNPROCESSABLE_ENTITY
;
3904 bl
.append(etag
.c_str(), etag
.size());
3905 emplace_attr(RGW_ATTR_ETAG
, std::move(bl
));
3907 populate_with_generic_attrs(s
, attrs
);
3908 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
);
3912 encode_delete_at_attr(delete_at
, attrs
);
3913 encode_obj_tags_attr(obj_tags
.get(), attrs
);
3915 /* Add a custom metadata to expose the information whether an object
3916 * is an SLO or not. Appending the attribute must be performed AFTER
3917 * processing any input from user in order to prohibit overwriting. */
3919 bufferlist slo_userindicator_bl
;
3920 slo_userindicator_bl
.append("True", 4);
3921 emplace_attr(RGW_ATTR_SLO_UINDICATOR
, std::move(slo_userindicator_bl
));
3923 if (obj_legal_hold
) {
3924 bufferlist obj_legal_hold_bl
;
3925 obj_legal_hold
->encode(obj_legal_hold_bl
);
3926 emplace_attr(RGW_ATTR_OBJECT_LEGAL_HOLD
, std::move(obj_legal_hold_bl
));
3928 if (obj_retention
) {
3929 bufferlist obj_retention_bl
;
3930 obj_retention
->encode(obj_retention_bl
);
3931 emplace_attr(RGW_ATTR_OBJECT_RETENTION
, std::move(obj_retention_bl
));
3934 tracepoint(rgw_op
, processor_complete_enter
, s
->req_id
.c_str());
3935 op_ret
= processor
->complete(s
->obj_size
, etag
, &mtime
, real_time(), attrs
,
3936 (delete_at
? *delete_at
: real_time()), if_match
, if_nomatch
,
3937 (user_data
.empty() ? nullptr : &user_data
), nullptr, nullptr);
3938 tracepoint(rgw_op
, processor_complete_exit
, s
->req_id
.c_str());
3940 /* produce torrent */
3941 if (s
->cct
->_conf
->rgw_torrent_flag
&& (ofs
== torrent
.get_data_len()))
3943 torrent
.init(s
, store
);
3944 torrent
.set_create_date(mtime
);
3945 op_ret
= torrent
.complete();
3948 ldpp_dout(this, 0) << "ERROR: torrent.handle_data() returned " << op_ret
<< dendl
;
3953 // send request to notification manager
3954 const auto ret
= rgw::notify::publish(s
, mtime
, etag
, rgw::notify::ObjectCreatedPut
, store
);
3956 ldpp_dout(this, 5) << "WARNING: publishing notification failed, with error: " << ret
<< dendl
;
3957 // TODO: we should have conf to make send a blocking coroutine and reply with error in case sending failed
3958 // this should be global conf (probably returnign a different handler)
3959 // so we don't need to read the configured values before we perform it
3963 int RGWPostObj::verify_permission()
3968 void RGWPostObj::pre_exec()
3970 rgw_bucket_object_pre_exec(s
);
3973 void RGWPostObj::execute()
3975 boost::optional
<RGWPutObj_Compress
> compressor
;
3976 CompressorRef plugin
;
3977 char supplied_md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
3979 /* Read in the data from the POST form. */
3980 op_ret
= get_params();
3985 op_ret
= verify_params();
3990 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
3991 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
3993 rgw::IAM::s3PutObject
,
3994 rgw_obj(s
->bucket
, s
->object
));
3995 if (usr_policy_res
== Effect::Deny
) {
4000 rgw::IAM::Effect e
= Effect::Pass
;
4001 if (s
->iam_policy
) {
4002 e
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
4003 rgw::IAM::s3PutObject
,
4004 rgw_obj(s
->bucket
, s
->object
));
4006 if (e
== Effect::Deny
) {
4009 } else if (usr_policy_res
== Effect::Pass
&& e
== Effect::Pass
&& !verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
4013 } else if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
4018 /* Start iteration over data fields. It's necessary as Swift's FormPost
4019 * is capable to handle multiple files in single form. */
4021 char calc_md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
4022 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
4024 ceph::buffer::list bl
, aclbl
;
4027 op_ret
= store
->check_quota(s
->bucket_owner
.get_id(),
4036 op_ret
= store
->check_bucket_shards(s
->bucket_info
, s
->bucket
, bucket_quota
);
4041 if (supplied_md5_b64
) {
4042 char supplied_md5_bin
[CEPH_CRYPTO_MD5_DIGESTSIZE
+ 1];
4043 ldpp_dout(this, 15) << "supplied_md5_b64=" << supplied_md5_b64
<< dendl
;
4044 op_ret
= ceph_unarmor(supplied_md5_bin
, &supplied_md5_bin
[CEPH_CRYPTO_MD5_DIGESTSIZE
+ 1],
4045 supplied_md5_b64
, supplied_md5_b64
+ strlen(supplied_md5_b64
));
4046 ldpp_dout(this, 15) << "ceph_armor ret=" << op_ret
<< dendl
;
4047 if (op_ret
!= CEPH_CRYPTO_MD5_DIGESTSIZE
) {
4048 op_ret
= -ERR_INVALID_DIGEST
;
4052 buf_to_hex((const unsigned char *)supplied_md5_bin
, CEPH_CRYPTO_MD5_DIGESTSIZE
, supplied_md5
);
4053 ldpp_dout(this, 15) << "supplied_md5=" << supplied_md5
<< dendl
;
4056 rgw_obj
obj(s
->bucket
, get_current_filename());
4057 if (s
->bucket_info
.versioning_enabled()) {
4058 store
->gen_rand_obj_instance_name(&obj
);
4061 rgw::AioThrottle
aio(s
->cct
->_conf
->rgw_put_obj_min_window_size
);
4063 using namespace rgw::putobj
;
4064 AtomicObjectProcessor
processor(&aio
, store
, s
->bucket_info
,
4066 s
->bucket_owner
.get_id(),
4067 *static_cast<RGWObjectCtx
*>(s
->obj_ctx
),
4069 op_ret
= processor
.prepare();
4074 /* No filters by default. */
4075 DataProcessor
*filter
= &processor
;
4077 std::unique_ptr
<DataProcessor
> encrypt
;
4078 op_ret
= get_encrypt_filter(&encrypt
, filter
);
4082 if (encrypt
!= nullptr) {
4083 filter
= encrypt
.get();
4085 const auto& compression_type
= store
->svc
.zone
->get_zone_params().get_compression_type(
4087 if (compression_type
!= "none") {
4088 plugin
= Compressor::create(s
->cct
, compression_type
);
4090 ldpp_dout(this, 1) << "Cannot load plugin for compression type "
4091 << compression_type
<< dendl
;
4093 compressor
.emplace(s
->cct
, plugin
, filter
);
4094 filter
= &*compressor
;
4101 ceph::bufferlist data
;
4102 len
= get_data(data
, again
);
4113 hash
.Update((const unsigned char *)data
.c_str(), data
.length());
4114 op_ret
= filter
->process(std::move(data
), ofs
);
4118 if (ofs
> max_len
) {
4119 op_ret
= -ERR_TOO_LARGE
;
4125 op_ret
= filter
->process({}, ofs
);
4130 if (len
< min_len
) {
4131 op_ret
= -ERR_TOO_SMALL
;
4138 op_ret
= store
->check_quota(s
->bucket_owner
.get_id(), s
->bucket
,
4139 user_quota
, bucket_quota
, s
->obj_size
);
4144 op_ret
= store
->check_bucket_shards(s
->bucket_info
, s
->bucket
, bucket_quota
);
4150 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, calc_md5
);
4154 if (supplied_md5_b64
&& strcmp(calc_md5
, supplied_md5
)) {
4155 op_ret
= -ERR_BAD_DIGEST
;
4159 bl
.append(etag
.c_str(), etag
.size());
4160 emplace_attr(RGW_ATTR_ETAG
, std::move(bl
));
4162 policy
.encode(aclbl
);
4163 emplace_attr(RGW_ATTR_ACL
, std::move(aclbl
));
4165 const std::string content_type
= get_current_content_type();
4166 if (! content_type
.empty()) {
4167 ceph::bufferlist ct_bl
;
4168 ct_bl
.append(content_type
.c_str(), content_type
.size() + 1);
4169 emplace_attr(RGW_ATTR_CONTENT_TYPE
, std::move(ct_bl
));
4172 if (compressor
&& compressor
->is_compressed()) {
4173 ceph::bufferlist tmp
;
4174 RGWCompressionInfo cs_info
;
4175 cs_info
.compression_type
= plugin
->get_type_name();
4176 cs_info
.orig_size
= s
->obj_size
;
4177 cs_info
.blocks
= move(compressor
->get_compression_blocks());
4178 encode(cs_info
, tmp
);
4179 emplace_attr(RGW_ATTR_COMPRESSION
, std::move(tmp
));
4182 op_ret
= processor
.complete(s
->obj_size
, etag
, nullptr, real_time(), attrs
,
4183 (delete_at
? *delete_at
: real_time()),
4184 nullptr, nullptr, nullptr, nullptr, nullptr);
4188 } while (is_next_file_to_upload());
4190 const auto ret
= rgw::notify::publish(s
, ceph::real_clock::now(), etag
, rgw::notify::ObjectCreatedPost
, store
);
4192 ldpp_dout(this, 5) << "WARNING: publishing notification failed, with error: " << ret
<< dendl
;
4193 // TODO: we should have conf to make send a blocking coroutine and reply with error in case sending failed
4194 // this should be global conf (probably returnign a different handler)
4195 // so we don't need to read the configured values before we perform it
4200 void RGWPutMetadataAccount::filter_out_temp_url(map
<string
, bufferlist
>& add_attrs
,
4201 const set
<string
>& rmattr_names
,
4202 map
<int, string
>& temp_url_keys
)
4204 map
<string
, bufferlist
>::iterator iter
;
4206 iter
= add_attrs
.find(RGW_ATTR_TEMPURL_KEY1
);
4207 if (iter
!= add_attrs
.end()) {
4208 temp_url_keys
[0] = iter
->second
.c_str();
4209 add_attrs
.erase(iter
);
4212 iter
= add_attrs
.find(RGW_ATTR_TEMPURL_KEY2
);
4213 if (iter
!= add_attrs
.end()) {
4214 temp_url_keys
[1] = iter
->second
.c_str();
4215 add_attrs
.erase(iter
);
4218 for (const string
& name
: rmattr_names
) {
4219 if (name
.compare(RGW_ATTR_TEMPURL_KEY1
) == 0) {
4220 temp_url_keys
[0] = string();
4222 if (name
.compare(RGW_ATTR_TEMPURL_KEY2
) == 0) {
4223 temp_url_keys
[1] = string();
4228 int RGWPutMetadataAccount::init_processing()
4230 /* First, go to the base class. At the time of writing the method was
4231 * responsible only for initializing the quota. This isn't necessary
4232 * here as we are touching metadata only. I'm putting this call only
4233 * for the future. */
4234 op_ret
= RGWOp::init_processing();
4239 op_ret
= get_params();
4244 op_ret
= rgw_get_user_attrs_by_uid(store
, s
->user
->user_id
, orig_attrs
,
4252 policy
.encode(acl_bl
);
4253 attrs
.emplace(RGW_ATTR_ACL
, std::move(acl_bl
));
4256 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
, false);
4260 prepare_add_del_attrs(orig_attrs
, rmattr_names
, attrs
);
4261 populate_with_generic_attrs(s
, attrs
);
4263 /* Try extract the TempURL-related stuff now to allow verify_permission
4264 * evaluate whether we need FULL_CONTROL or not. */
4265 filter_out_temp_url(attrs
, rmattr_names
, temp_url_keys
);
4267 /* The same with quota except a client needs to be reseller admin. */
4268 op_ret
= filter_out_quota_info(attrs
, rmattr_names
, new_quota
,
4269 &new_quota_extracted
);
4277 int RGWPutMetadataAccount::verify_permission()
4279 if (s
->auth
.identity
->is_anonymous()) {
4283 if (!verify_user_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
4287 /* Altering TempURL keys requires FULL_CONTROL. */
4288 if (!temp_url_keys
.empty() && s
->perm_mask
!= RGW_PERM_FULL_CONTROL
) {
4292 /* We are failing this intensionally to allow system user/reseller admin
4293 * override in rgw_process.cc. This is the way to specify a given RGWOp
4294 * expect extra privileges. */
4295 if (new_quota_extracted
) {
4302 void RGWPutMetadataAccount::execute()
4304 /* Params have been extracted earlier. See init_processing(). */
4305 RGWUserInfo new_uinfo
;
4306 op_ret
= rgw_get_user_info_by_uid(store
, s
->user
->user_id
, new_uinfo
,
4312 /* Handle the TempURL-related stuff. */
4313 if (!temp_url_keys
.empty()) {
4314 for (auto& pair
: temp_url_keys
) {
4315 new_uinfo
.temp_url_keys
[pair
.first
] = std::move(pair
.second
);
4319 /* Handle the quota extracted at the verify_permission step. */
4320 if (new_quota_extracted
) {
4321 new_uinfo
.user_quota
= std::move(new_quota
);
4324 /* We are passing here the current (old) user info to allow the function
4325 * optimize-out some operations. */
4326 op_ret
= rgw_store_user_info(store
, new_uinfo
, s
->user
,
4327 &acct_op_tracker
, real_time(), false, &attrs
);
4330 int RGWPutMetadataBucket::verify_permission()
4332 if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
4339 void RGWPutMetadataBucket::pre_exec()
4341 rgw_bucket_object_pre_exec(s
);
4344 void RGWPutMetadataBucket::execute()
4346 op_ret
= get_params();
4351 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
, false);
4356 if (!placement_rule
.empty() &&
4357 placement_rule
!= s
->bucket_info
.placement_rule
) {
4362 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
4363 /* Encode special metadata first as we're using std::map::emplace under
4364 * the hood. This method will add the new items only if the map doesn't
4365 * contain such keys yet. */
4367 if (s
->dialect
.compare("swift") == 0) {
4369 static_cast<RGWAccessControlPolicy_SWIFT
*>(s
->bucket_acl
.get());
4370 auto new_policy
= static_cast<RGWAccessControlPolicy_SWIFT
*>(&policy
);
4371 new_policy
->filter_merge(policy_rw_mask
, old_policy
);
4372 policy
= *new_policy
;
4376 emplace_attr(RGW_ATTR_ACL
, std::move(bl
));
4381 cors_config
.encode(bl
);
4382 emplace_attr(RGW_ATTR_CORS
, std::move(bl
));
4385 /* It's supposed that following functions WILL NOT change any
4386 * special attributes (like RGW_ATTR_ACL) if they are already
4387 * present in attrs. */
4388 prepare_add_del_attrs(s
->bucket_attrs
, rmattr_names
, attrs
);
4389 populate_with_generic_attrs(s
, attrs
);
4391 /* According to the Swift's behaviour and its container_quota
4392 * WSGI middleware implementation: anyone with write permissions
4393 * is able to set the bucket quota. This stays in contrast to
4394 * account quotas that can be set only by clients holding
4395 * reseller admin privileges. */
4396 op_ret
= filter_out_quota_info(attrs
, rmattr_names
, s
->bucket_info
.quota
);
4401 if (swift_ver_location
) {
4402 s
->bucket_info
.swift_ver_location
= *swift_ver_location
;
4403 s
->bucket_info
.swift_versioning
= (!swift_ver_location
->empty());
4406 /* Web site of Swift API. */
4407 filter_out_website(attrs
, rmattr_names
, s
->bucket_info
.website_conf
);
4408 s
->bucket_info
.has_website
= !s
->bucket_info
.website_conf
.is_empty();
4410 /* Setting attributes also stores the provided bucket info. Due
4411 * to this fact, the new quota settings can be serialized with
4413 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
,
4414 &s
->bucket_info
.objv_tracker
);
4419 int RGWPutMetadataObject::verify_permission()
4421 // This looks to be something specific to Swift. We could add
4422 // operations like swift:PutMetadataObject to the Policy Engine.
4423 if (!verify_object_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
4430 void RGWPutMetadataObject::pre_exec()
4432 rgw_bucket_object_pre_exec(s
);
4435 void RGWPutMetadataObject::execute()
4437 rgw_obj
obj(s
->bucket
, s
->object
);
4439 map
<string
, bufferlist
> attrs
, orig_attrs
, rmattrs
;
4441 store
->set_atomic(s
->obj_ctx
, obj
);
4443 op_ret
= get_params();
4448 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
);
4453 /* check if obj exists, read orig attrs */
4454 op_ret
= get_obj_attrs(store
, s
, obj
, orig_attrs
, &target_obj
);
4459 /* Check whether the object has expired. Swift API documentation
4460 * stands that we should return 404 Not Found in such case. */
4461 if (need_object_expiration() && object_is_expired(orig_attrs
)) {
4466 /* Filter currently existing attributes. */
4467 prepare_add_del_attrs(orig_attrs
, attrs
, rmattrs
);
4468 populate_with_generic_attrs(s
, attrs
);
4469 encode_delete_at_attr(delete_at
, attrs
);
4472 op_ret
= encode_dlo_manifest_attr(dlo_manifest
, attrs
);
4474 ldpp_dout(this, 0) << "bad user manifest: " << dlo_manifest
<< dendl
;
4479 op_ret
= store
->set_attrs(s
->obj_ctx
, s
->bucket_info
, target_obj
, attrs
, &rmattrs
);
4482 int RGWDeleteObj::handle_slo_manifest(bufferlist
& bl
)
4484 RGWSLOInfo slo_info
;
4485 auto bliter
= bl
.cbegin();
4487 decode(slo_info
, bliter
);
4488 } catch (buffer::error
& err
) {
4489 ldpp_dout(this, 0) << "ERROR: failed to decode slo manifest" << dendl
;
4494 deleter
= std::unique_ptr
<RGWBulkDelete::Deleter
>(\
4495 new RGWBulkDelete::Deleter(this, store
, s
));
4496 } catch (const std::bad_alloc
&) {
4500 list
<RGWBulkDelete::acct_path_t
> items
;
4501 for (const auto& iter
: slo_info
.entries
) {
4502 const string
& path_str
= iter
.path
;
4504 const size_t sep_pos
= path_str
.find('/', 1 /* skip first slash */);
4505 if (boost::string_view::npos
== sep_pos
) {
4509 RGWBulkDelete::acct_path_t path
;
4511 path
.bucket_name
= url_decode(path_str
.substr(1, sep_pos
- 1));
4512 path
.obj_key
= url_decode(path_str
.substr(sep_pos
+ 1));
4514 items
.push_back(path
);
4517 /* Request removal of the manifest object itself. */
4518 RGWBulkDelete::acct_path_t path
;
4519 path
.bucket_name
= s
->bucket_name
;
4520 path
.obj_key
= s
->object
;
4521 items
.push_back(path
);
4523 int ret
= deleter
->delete_chunk(items
);
4531 int RGWDeleteObj::verify_permission()
4533 int op_ret
= get_params();
4537 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
4538 if (s
->bucket_info
.obj_lock_enabled() && bypass_governance_mode
) {
4539 auto r
= eval_user_policies(s
->iam_user_policies
, s
->env
, boost::none
,
4540 rgw::IAM::s3BypassGovernanceRetention
, ARN(s
->bucket
, s
->object
.name
));
4541 if (r
== Effect::Deny
) {
4542 bypass_perm
= false;
4543 } else if (r
== Effect::Pass
&& s
->iam_policy
) {
4544 r
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
, rgw::IAM::s3BypassGovernanceRetention
,
4545 ARN(s
->bucket
, s
->object
.name
));
4546 if (r
== Effect::Deny
) {
4547 bypass_perm
= false;
4551 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
4553 s
->object
.instance
.empty() ?
4554 rgw::IAM::s3DeleteObject
:
4555 rgw::IAM::s3DeleteObjectVersion
,
4556 ARN(s
->bucket
, s
->object
.name
));
4557 if (usr_policy_res
== Effect::Deny
) {
4561 rgw::IAM::Effect r
= Effect::Pass
;
4562 if (s
->iam_policy
) {
4563 r
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
4564 s
->object
.instance
.empty() ?
4565 rgw::IAM::s3DeleteObject
:
4566 rgw::IAM::s3DeleteObjectVersion
,
4567 ARN(s
->bucket
, s
->object
.name
));
4569 if (r
== Effect::Allow
)
4571 else if (r
== Effect::Deny
)
4573 else if (usr_policy_res
== Effect::Allow
)
4577 if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
4581 if (s
->bucket_info
.mfa_enabled() &&
4582 !s
->object
.instance
.empty() &&
4584 ldpp_dout(this, 5) << "NOTICE: object delete request with a versioned object, mfa auth not provided" << dendl
;
4585 return -ERR_MFA_REQUIRED
;
4591 void RGWDeleteObj::pre_exec()
4593 rgw_bucket_object_pre_exec(s
);
4596 void RGWDeleteObj::execute()
4598 if (!s
->bucket_exists
) {
4599 op_ret
= -ERR_NO_SUCH_BUCKET
;
4603 rgw_obj
obj(s
->bucket
, s
->object
);
4604 map
<string
, bufferlist
> attrs
;
4606 bool check_obj_lock
= obj
.key
.have_instance() && s
->bucket_info
.obj_lock_enabled();
4608 if (!s
->object
.empty()) {
4609 if (need_object_expiration() || multipart_delete
) {
4610 /* check if obj exists, read orig attrs */
4611 op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
4617 if (check_obj_lock
) {
4618 /* check if obj exists, read orig attrs */
4619 op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
4621 if (op_ret
== -ENOENT
) {
4622 /* object maybe delete_marker, skip check_obj_lock*/
4623 check_obj_lock
= false;
4630 if (check_obj_lock
) {
4631 auto aiter
= attrs
.find(RGW_ATTR_OBJECT_RETENTION
);
4632 if (aiter
!= attrs
.end()) {
4633 RGWObjectRetention obj_retention
;
4635 decode(obj_retention
, aiter
->second
);
4636 } catch (buffer::error
& err
) {
4637 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl
;
4641 if (ceph::real_clock::to_time_t(obj_retention
.get_retain_until_date()) > ceph_clock_now()) {
4642 if (obj_retention
.get_mode().compare("GOVERNANCE") != 0 || !bypass_perm
|| !bypass_governance_mode
) {
4648 aiter
= attrs
.find(RGW_ATTR_OBJECT_LEGAL_HOLD
);
4649 if (aiter
!= attrs
.end()) {
4650 RGWObjectLegalHold obj_legal_hold
;
4652 decode(obj_legal_hold
, aiter
->second
);
4653 } catch (buffer::error
& err
) {
4654 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectLegalHold" << dendl
;
4658 if (obj_legal_hold
.is_enabled()) {
4665 if (multipart_delete
) {
4666 const auto slo_attr
= attrs
.find(RGW_ATTR_SLO_MANIFEST
);
4668 if (slo_attr
!= attrs
.end()) {
4669 op_ret
= handle_slo_manifest(slo_attr
->second
);
4671 ldpp_dout(this, 0) << "ERROR: failed to handle slo manifest ret=" << op_ret
<< dendl
;
4674 op_ret
= -ERR_NOT_SLO_MANIFEST
;
4680 RGWObjectCtx
*obj_ctx
= static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
4681 obj_ctx
->set_atomic(obj
);
4683 bool ver_restored
= false;
4684 op_ret
= store
->swift_versioning_restore(*s
->sysobj_ctx
, *obj_ctx
, s
->bucket_owner
.get_id(),
4685 s
->bucket_info
, obj
, ver_restored
);
4690 if (!ver_restored
) {
4691 /* Swift's versioning mechanism hasn't found any previous version of
4692 * the object that could be restored. This means we should proceed
4693 * with the regular delete path. */
4694 RGWRados::Object
del_target(store
, s
->bucket_info
, *obj_ctx
, obj
);
4695 RGWRados::Object::Delete
del_op(&del_target
);
4697 op_ret
= get_system_versioning_params(s
, &del_op
.params
.olh_epoch
,
4698 &del_op
.params
.marker_version_id
);
4703 del_op
.params
.bucket_owner
= s
->bucket_owner
.get_id();
4704 del_op
.params
.versioning_status
= s
->bucket_info
.versioning_status();
4705 del_op
.params
.obj_owner
= s
->owner
;
4706 del_op
.params
.unmod_since
= unmod_since
;
4707 del_op
.params
.high_precision_time
= s
->system_request
; /* system request uses high precision time */
4709 op_ret
= del_op
.delete_obj();
4711 delete_marker
= del_op
.result
.delete_marker
;
4712 version_id
= del_op
.result
.version_id
;
4715 /* Check whether the object has expired. Swift API documentation
4716 * stands that we should return 404 Not Found in such case. */
4717 if (need_object_expiration() && object_is_expired(attrs
)) {
4723 if (op_ret
== -ECANCELED
) {
4726 if (op_ret
== -ERR_PRECONDITION_FAILED
&& no_precondition_error
) {
4733 const auto ret
= rgw::notify::publish(s
, ceph::real_clock::now(), attrs
[RGW_ATTR_ETAG
].to_str(),
4734 delete_marker
&& s
->object
.instance
.empty() ? rgw::notify::ObjectRemovedDeleteMarkerCreated
: rgw::notify::ObjectRemovedDelete
,
4737 ldpp_dout(this, 5) << "WARNING: publishing notification failed, with error: " << ret
<< dendl
;
4738 // TODO: we should have conf to make send a blocking coroutine and reply with error in case sending failed
4739 // this should be global conf (probably returnign a different handler)
4740 // so we don't need to read the configured values before we perform it
4744 bool RGWCopyObj::parse_copy_location(const boost::string_view
& url_src
,
4745 string
& bucket_name
,
4748 boost::string_view name_str
;
4749 boost::string_view params_str
;
4751 // search for ? before url-decoding so we don't accidentally match %3F
4752 size_t pos
= url_src
.find('?');
4753 if (pos
== string::npos
) {
4756 name_str
= url_src
.substr(0, pos
);
4757 params_str
= url_src
.substr(pos
+ 1);
4760 boost::string_view dec_src
{name_str
};
4761 if (dec_src
[0] == '/')
4762 dec_src
.remove_prefix(1);
4764 pos
= dec_src
.find('/');
4765 if (pos
== string::npos
)
4768 bucket_name
= url_decode(dec_src
.substr(0, pos
));
4769 key
.name
= url_decode(dec_src
.substr(pos
+ 1));
4771 if (key
.name
.empty()) {
4775 if (! params_str
.empty()) {
4777 args
.set(params_str
.to_string());
4780 key
.instance
= args
.get("versionId", NULL
);
4786 int RGWCopyObj::verify_permission()
4788 RGWAccessControlPolicy
src_acl(s
->cct
);
4789 boost::optional
<Policy
> src_policy
;
4790 op_ret
= get_params();
4794 op_ret
= get_system_versioning_params(s
, &olh_epoch
, &version_id
);
4798 map
<string
, bufferlist
> src_attrs
;
4800 if (s
->bucket_instance_id
.empty()) {
4801 op_ret
= store
->get_bucket_info(*s
->sysobj_ctx
, src_tenant_name
, src_bucket_name
, src_bucket_info
, NULL
, &src_attrs
);
4803 /* will only happen in intra region sync where the source and dest bucket is the same */
4804 op_ret
= store
->get_bucket_instance_info(*s
->sysobj_ctx
, s
->bucket_instance_id
, src_bucket_info
, NULL
, &src_attrs
);
4807 if (op_ret
== -ENOENT
) {
4808 op_ret
= -ERR_NO_SUCH_BUCKET
;
4813 src_bucket
= src_bucket_info
.bucket
;
4815 /* get buckets info (source and dest) */
4816 if (s
->local_source
&& source_zone
.empty()) {
4817 rgw_obj
src_obj(src_bucket
, src_object
);
4818 store
->set_atomic(s
->obj_ctx
, src_obj
);
4819 store
->set_prefetch_data(s
->obj_ctx
, src_obj
);
4821 rgw_placement_rule src_placement
;
4823 /* check source object permissions */
4824 op_ret
= read_obj_policy(store
, s
, src_bucket_info
, src_attrs
, &src_acl
, &src_placement
.storage_class
,
4825 src_policy
, src_bucket
, src_object
);
4830 /* follow up on previous checks that required reading source object head */
4831 if (need_to_check_storage_class
) {
4832 src_placement
.inherit_from(src_bucket_info
.placement_rule
);
4834 op_ret
= check_storage_class(src_placement
);
4840 /* admin request overrides permission checks */
4841 if (!s
->auth
.identity
->is_admin_of(src_acl
.get_owner().get_id())) {
4843 auto e
= src_policy
->eval(s
->env
, *s
->auth
.identity
,
4844 src_object
.instance
.empty() ?
4845 rgw::IAM::s3GetObject
:
4846 rgw::IAM::s3GetObjectVersion
,
4848 if (e
== Effect::Deny
) {
4850 } else if (e
== Effect::Pass
&&
4851 !src_acl
.verify_permission(this, *s
->auth
.identity
, s
->perm_mask
,
4855 } else if (!src_acl
.verify_permission(this, *s
->auth
.identity
,
4863 RGWAccessControlPolicy
dest_bucket_policy(s
->cct
);
4864 map
<string
, bufferlist
> dest_attrs
;
4866 if (src_bucket_name
.compare(dest_bucket_name
) == 0) { /* will only happen if s->local_source
4867 or intra region sync */
4868 dest_bucket_info
= src_bucket_info
;
4869 dest_attrs
= src_attrs
;
4871 op_ret
= store
->get_bucket_info(*s
->sysobj_ctx
, dest_tenant_name
, dest_bucket_name
,
4872 dest_bucket_info
, nullptr, &dest_attrs
);
4874 if (op_ret
== -ENOENT
) {
4875 op_ret
= -ERR_NO_SUCH_BUCKET
;
4881 dest_bucket
= dest_bucket_info
.bucket
;
4883 rgw_obj
dest_obj(dest_bucket
, dest_object
);
4884 store
->set_atomic(s
->obj_ctx
, dest_obj
);
4886 /* check dest bucket permissions */
4887 op_ret
= read_bucket_policy(store
, s
, dest_bucket_info
, dest_attrs
,
4888 &dest_bucket_policy
, dest_bucket
);
4892 auto dest_iam_policy
= get_iam_policy_from_attr(s
->cct
, store
, dest_attrs
, dest_bucket
.tenant
);
4893 /* admin request overrides permission checks */
4894 if (! s
->auth
.identity
->is_admin_of(dest_policy
.get_owner().get_id())){
4895 if (dest_iam_policy
!= boost::none
) {
4896 rgw_add_to_iam_environment(s
->env
, "s3:x-amz-copy-source", copy_source
);
4898 rgw_add_to_iam_environment(s
->env
, "s3:x-amz-metadata-directive",
4901 auto e
= dest_iam_policy
->eval(s
->env
, *s
->auth
.identity
,
4902 rgw::IAM::s3PutObject
,
4904 if (e
== Effect::Deny
) {
4906 } else if (e
== Effect::Pass
&&
4907 ! dest_bucket_policy
.verify_permission(this,
4914 } else if (! dest_bucket_policy
.verify_permission(this, *s
->auth
.identity
, s
->perm_mask
,
4919 op_ret
= init_dest_policy();
4928 int RGWCopyObj::init_common()
4931 if (parse_time(if_mod
, &mod_time
) < 0) {
4935 mod_ptr
= &mod_time
;
4939 if (parse_time(if_unmod
, &unmod_time
) < 0) {
4943 unmod_ptr
= &unmod_time
;
4947 dest_policy
.encode(aclbl
);
4948 emplace_attr(RGW_ATTR_ACL
, std::move(aclbl
));
4950 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
);
4954 populate_with_generic_attrs(s
, attrs
);
4959 static void copy_obj_progress_cb(off_t ofs
, void *param
)
4961 RGWCopyObj
*op
= static_cast<RGWCopyObj
*>(param
);
4962 op
->progress_cb(ofs
);
4965 void RGWCopyObj::progress_cb(off_t ofs
)
4967 if (!s
->cct
->_conf
->rgw_copy_obj_progress
)
4970 if (ofs
- last_ofs
< s
->cct
->_conf
->rgw_copy_obj_progress_every_bytes
)
4973 send_partial_response(ofs
);
4978 void RGWCopyObj::pre_exec()
4980 rgw_bucket_object_pre_exec(s
);
4983 void RGWCopyObj::execute()
4985 if (init_common() < 0)
4988 rgw_obj
src_obj(src_bucket
, src_object
);
4989 rgw_obj
dst_obj(dest_bucket
, dest_object
);
4991 RGWObjectCtx
& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
4992 if ( ! version_id
.empty()) {
4993 dst_obj
.key
.set_instance(version_id
);
4994 } else if (dest_bucket_info
.versioning_enabled()) {
4995 store
->gen_rand_obj_instance_name(&dst_obj
);
4998 obj_ctx
.set_atomic(src_obj
);
4999 obj_ctx
.set_atomic(dst_obj
);
5001 encode_delete_at_attr(delete_at
, attrs
);
5003 bool high_precision_time
= (s
->system_request
);
5005 /* Handle object versioning of Swift API. In case of copying to remote this
5006 * should fail gently (op_ret == 0) as the dst_obj will not exist here. */
5007 op_ret
= store
->swift_versioning_copy(obj_ctx
,
5008 dest_bucket_info
.owner
,
5015 op_ret
= store
->copy_obj(obj_ctx
,
5028 high_precision_time
,
5033 attrs
, RGWObjCategory::Main
,
5035 (delete_at
? *delete_at
: real_time()),
5036 (version_id
.empty() ? NULL
: &version_id
),
5037 &s
->req_id
, /* use req_id as tag */
5039 copy_obj_progress_cb
, (void *)this
5042 const auto ret
= rgw::notify::publish(s
, mtime
, etag
, rgw::notify::ObjectCreatedCopy
, store
);
5044 ldpp_dout(this, 5) << "WARNING: publishing notification failed, with error: " << ret
<< dendl
;
5045 // TODO: we should have conf to make send a blocking coroutine and reply with error in case sending failed
5046 // this should be global conf (probably returnign a different handler)
5047 // so we don't need to read the configured values before we perform it
5051 int RGWGetACLs::verify_permission()
5054 if (!s
->object
.empty()) {
5055 auto iam_action
= s
->object
.instance
.empty() ?
5056 rgw::IAM::s3GetObjectAcl
:
5057 rgw::IAM::s3GetObjectVersionAcl
;
5059 if (s
->iam_policy
&& s
->iam_policy
->has_partial_conditional(S3_EXISTING_OBJTAG
)){
5060 rgw_obj obj
= rgw_obj(s
->bucket
, s
->object
);
5061 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
5063 if (! s
->iam_user_policies
.empty()) {
5064 for (auto& user_policy
: s
->iam_user_policies
) {
5065 if (user_policy
.has_partial_conditional(S3_EXISTING_OBJTAG
)) {
5066 rgw_obj obj
= rgw_obj(s
->bucket
, s
->object
);
5067 rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
5071 perm
= verify_object_permission(this, s
, iam_action
);
5073 if (!s
->bucket_exists
) {
5074 return -ERR_NO_SUCH_BUCKET
;
5076 perm
= verify_bucket_permission(this, s
, rgw::IAM::s3GetBucketAcl
);
5084 void RGWGetACLs::pre_exec()
5086 rgw_bucket_object_pre_exec(s
);
5089 void RGWGetACLs::execute()
5092 RGWAccessControlPolicy
* const acl
= \
5093 (!s
->object
.empty() ? s
->object_acl
.get() : s
->bucket_acl
.get());
5094 RGWAccessControlPolicy_S3
* const s3policy
= \
5095 static_cast<RGWAccessControlPolicy_S3
*>(acl
);
5096 s3policy
->to_xml(ss
);
5102 int RGWPutACLs::verify_permission()
5106 rgw_add_to_iam_environment(s
->env
, "s3:x-amz-acl", s
->canned_acl
);
5108 rgw_add_grant_to_iam_environment(s
->env
, s
);
5109 if (!s
->object
.empty()) {
5110 auto iam_action
= s
->object
.instance
.empty() ? rgw::IAM::s3PutObjectAcl
: rgw::IAM::s3PutObjectVersionAcl
;
5111 auto obj
= rgw_obj(s
->bucket
, s
->object
);
5112 op_ret
= rgw_iam_add_existing_objtags(store
, s
, obj
, iam_action
);
5113 perm
= verify_object_permission(this, s
, iam_action
);
5115 perm
= verify_bucket_permission(this, s
, rgw::IAM::s3PutBucketAcl
);
5123 int RGWGetLC::verify_permission()
5126 perm
= verify_bucket_permission(this, s
, rgw::IAM::s3GetLifecycleConfiguration
);
5133 int RGWPutLC::verify_permission()
5136 perm
= verify_bucket_permission(this, s
, rgw::IAM::s3PutLifecycleConfiguration
);
5143 int RGWDeleteLC::verify_permission()
5146 perm
= verify_bucket_permission(this, s
, rgw::IAM::s3PutLifecycleConfiguration
);
5153 void RGWPutACLs::pre_exec()
5155 rgw_bucket_object_pre_exec(s
);
5158 void RGWGetLC::pre_exec()
5160 rgw_bucket_object_pre_exec(s
);
5163 void RGWPutLC::pre_exec()
5165 rgw_bucket_object_pre_exec(s
);
5168 void RGWDeleteLC::pre_exec()
5170 rgw_bucket_object_pre_exec(s
);
5173 void RGWPutACLs::execute()
5177 RGWAccessControlPolicy_S3
*policy
= NULL
;
5178 RGWACLXMLParser_S3
parser(s
->cct
);
5179 RGWAccessControlPolicy_S3
new_policy(s
->cct
);
5183 op_ret
= 0; /* XXX redundant? */
5185 if (!parser
.init()) {
5191 RGWAccessControlPolicy
* const existing_policy
= \
5192 (s
->object
.empty() ? s
->bucket_acl
.get() : s
->object_acl
.get());
5194 owner
= existing_policy
->get_owner();
5196 op_ret
= get_params();
5198 if (op_ret
== -ERANGE
) {
5199 ldpp_dout(this, 4) << "The size of request xml data is larger than the max limitation, data size = "
5200 << s
->length
<< dendl
;
5201 op_ret
= -ERR_MALFORMED_XML
;
5202 s
->err
.message
= "The XML you provided was larger than the maximum " +
5203 std::to_string(s
->cct
->_conf
->rgw_max_put_param_size
) +
5209 char* buf
= data
.c_str();
5210 ldpp_dout(this, 15) << "read len=" << data
.length() << " data=" << (buf
? buf
: "") << dendl
;
5212 if (!s
->canned_acl
.empty() && data
.length() > 0) {
5217 if (!s
->canned_acl
.empty() || s
->has_acl_header
) {
5218 op_ret
= get_policy_from_state(store
, s
, ss
);
5223 data
.append(ss
.str());
5226 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
5230 policy
= static_cast<RGWAccessControlPolicy_S3
*>(parser
.find_first("AccessControlPolicy"));
5236 const RGWAccessControlList
& req_acl
= policy
->get_acl();
5237 const multimap
<string
, ACLGrant
>& req_grant_map
= req_acl
.get_grant_map();
5238 #define ACL_GRANTS_MAX_NUM 100
5239 int max_num
= s
->cct
->_conf
->rgw_acl_grants_max_num
;
5241 max_num
= ACL_GRANTS_MAX_NUM
;
5244 int grants_num
= req_grant_map
.size();
5245 if (grants_num
> max_num
) {
5246 ldpp_dout(this, 4) << "An acl can have up to " << max_num
5247 << " grants, request acl grants num: " << grants_num
<< dendl
;
5248 op_ret
= -ERR_MALFORMED_ACL_ERROR
;
5249 s
->err
.message
= "The request is rejected, because the acl grants number you requested is larger than the maximum "
5250 + std::to_string(max_num
)
5251 + " grants allowed in an acl.";
5255 // forward bucket acl requests to meta master zone
5256 if (s
->object
.empty() && !store
->svc
.zone
->is_meta_master()) {
5258 // include acl data unless it was generated from a canned_acl
5259 if (s
->canned_acl
.empty()) {
5260 in_data
.append(data
);
5262 op_ret
= forward_request_to_master(s
, NULL
, store
, in_data
, NULL
);
5264 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
5269 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
5270 ldpp_dout(this, 15) << "Old AccessControlPolicy";
5271 policy
->to_xml(*_dout
);
5275 op_ret
= policy
->rebuild(store
, &owner
, new_policy
);
5279 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
5280 ldpp_dout(this, 15) << "New AccessControlPolicy:";
5281 new_policy
.to_xml(*_dout
);
5285 new_policy
.encode(bl
);
5286 map
<string
, bufferlist
> attrs
;
5288 if (!s
->object
.empty()) {
5289 obj
= rgw_obj(s
->bucket
, s
->object
);
5290 store
->set_atomic(s
->obj_ctx
, obj
);
5291 //if instance is empty, we should modify the latest object
5292 op_ret
= modify_obj_attr(store
, s
, obj
, RGW_ATTR_ACL
, bl
);
5294 attrs
= s
->bucket_attrs
;
5295 attrs
[RGW_ATTR_ACL
] = bl
;
5296 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
, &s
->bucket_info
.objv_tracker
);
5298 if (op_ret
== -ECANCELED
) {
5299 op_ret
= 0; /* lost a race, but it's ok because acls are immutable */
5303 void RGWPutLC::execute()
5307 RGWLifecycleConfiguration_S3
config(s
->cct
);
5308 RGWXMLParser parser
;
5309 RGWLifecycleConfiguration_S3
new_config(s
->cct
);
5311 content_md5
= s
->info
.env
->get("HTTP_CONTENT_MD5");
5312 if (content_md5
== nullptr) {
5313 op_ret
= -ERR_INVALID_REQUEST
;
5314 s
->err
.message
= "Missing required header for this request: Content-MD5";
5315 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
5319 std::string content_md5_bin
;
5321 content_md5_bin
= rgw::from_base64(boost::string_view(content_md5
));
5323 s
->err
.message
= "Request header Content-MD5 contains character "
5324 "that is not base64 encoded.";
5325 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
5326 op_ret
= -ERR_BAD_DIGEST
;
5330 if (!parser
.init()) {
5335 op_ret
= get_params();
5339 char* buf
= data
.c_str();
5340 ldpp_dout(this, 15) << "read len=" << data
.length() << " data=" << (buf
? buf
: "") << dendl
;
5343 unsigned char data_hash_res
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
5344 data_hash
.Update(reinterpret_cast<const unsigned char*>(buf
), data
.length());
5345 data_hash
.Final(data_hash_res
);
5347 if (memcmp(data_hash_res
, content_md5_bin
.c_str(), CEPH_CRYPTO_MD5_DIGESTSIZE
) != 0) {
5348 op_ret
= -ERR_BAD_DIGEST
;
5349 s
->err
.message
= "The Content-MD5 you specified did not match what we received.";
5350 ldpp_dout(this, 5) << s
->err
.message
5351 << " Specified content md5: " << content_md5
5352 << ", calculated content md5: " << data_hash_res
5357 if (!parser
.parse(buf
, data
.length(), 1)) {
5358 op_ret
= -ERR_MALFORMED_XML
;
5363 RGWXMLDecoder::decode_xml("LifecycleConfiguration", config
, &parser
);
5364 } catch (RGWXMLDecoder::err
& err
) {
5365 ldpp_dout(this, 5) << "Bad lifecycle configuration: " << err
<< dendl
;
5366 op_ret
= -ERR_MALFORMED_XML
;
5370 op_ret
= config
.rebuild(store
, new_config
);
5374 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
5376 new_config
.dump_xml(&xf
);
5379 ldpp_dout(this, 15) << "New LifecycleConfiguration:" << ss
.str() << dendl
;
5382 if (!store
->svc
.zone
->is_meta_master()) {
5383 op_ret
= forward_request_to_master(s
, nullptr, store
, data
, nullptr);
5385 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
5390 op_ret
= store
->get_lc()->set_bucket_config(s
->bucket_info
, s
->bucket_attrs
, &new_config
);
5397 void RGWDeleteLC::execute()
5399 if (!store
->svc
.zone
->is_meta_master()) {
5401 op_ret
= forward_request_to_master(s
, nullptr, store
, data
, nullptr);
5403 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
5407 map
<string
, bufferlist
> attrs
= s
->bucket_attrs
;
5408 attrs
.erase(RGW_ATTR_LC
);
5409 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
,
5410 &s
->bucket_info
.objv_tracker
);
5412 ldpp_dout(this, 0) << "RGWLC::RGWDeleteLC() failed to set attrs on bucket="
5413 << s
->bucket
.name
<< " returned err=" << op_ret
<< dendl
;
5417 op_ret
= store
->get_lc()->remove_bucket_config(s
->bucket_info
, s
->bucket_attrs
);
5424 int RGWGetCORS::verify_permission()
5426 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketCORS
);
5429 void RGWGetCORS::execute()
5431 op_ret
= read_bucket_cors();
5436 ldpp_dout(this, 2) << "No CORS configuration set yet for this bucket" << dendl
;
5437 op_ret
= -ERR_NO_CORS_FOUND
;
5442 int RGWPutCORS::verify_permission()
5444 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3PutBucketCORS
);
5447 void RGWPutCORS::execute()
5451 op_ret
= get_params();
5455 if (!store
->svc
.zone
->is_meta_master()) {
5456 op_ret
= forward_request_to_master(s
, NULL
, store
, in_data
, nullptr);
5458 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
5463 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
5464 map
<string
, bufferlist
> attrs
= s
->bucket_attrs
;
5465 attrs
[RGW_ATTR_CORS
] = cors_bl
;
5466 return rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
, &s
->bucket_info
.objv_tracker
);
5470 int RGWDeleteCORS::verify_permission()
5472 // No separate delete permission
5473 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3PutBucketCORS
);
5476 void RGWDeleteCORS::execute()
5478 if (!store
->svc
.zone
->is_meta_master()) {
5480 op_ret
= forward_request_to_master(s
, nullptr, store
, data
, nullptr);
5482 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
5487 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
5488 op_ret
= read_bucket_cors();
5493 ldpp_dout(this, 2) << "No CORS configuration set yet for this bucket" << dendl
;
5498 map
<string
, bufferlist
> attrs
= s
->bucket_attrs
;
5499 attrs
.erase(RGW_ATTR_CORS
);
5500 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
,
5501 &s
->bucket_info
.objv_tracker
);
5503 ldpp_dout(this, 0) << "RGWLC::RGWDeleteCORS() failed to set attrs on bucket=" << s
->bucket
.name
5504 << " returned err=" << op_ret
<< dendl
;
5510 void RGWOptionsCORS::get_response_params(string
& hdrs
, string
& exp_hdrs
, unsigned *max_age
) {
5511 get_cors_response_headers(rule
, req_hdrs
, hdrs
, exp_hdrs
, max_age
);
5514 int RGWOptionsCORS::validate_cors_request(RGWCORSConfiguration
*cc
) {
5515 rule
= cc
->host_name_rule(origin
);
5517 ldpp_dout(this, 10) << "There is no cors rule present for " << origin
<< dendl
;
5521 if (!validate_cors_rule_method(rule
, req_meth
)) {
5525 if (!validate_cors_rule_header(rule
, req_hdrs
)) {
5532 void RGWOptionsCORS::execute()
5534 op_ret
= read_bucket_cors();
5538 origin
= s
->info
.env
->get("HTTP_ORIGIN");
5540 ldpp_dout(this, 0) << "Missing mandatory Origin header" << dendl
;
5544 req_meth
= s
->info
.env
->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
5546 ldpp_dout(this, 0) << "Missing mandatory Access-control-request-method header" << dendl
;
5551 ldpp_dout(this, 2) << "No CORS configuration set yet for this bucket" << dendl
;
5555 req_hdrs
= s
->info
.env
->get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS");
5556 op_ret
= validate_cors_request(&bucket_cors
);
5558 origin
= req_meth
= NULL
;
5564 int RGWGetRequestPayment::verify_permission()
5566 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketRequestPayment
);
5569 void RGWGetRequestPayment::pre_exec()
5571 rgw_bucket_object_pre_exec(s
);
5574 void RGWGetRequestPayment::execute()
5576 requester_pays
= s
->bucket_info
.requester_pays
;
5579 int RGWSetRequestPayment::verify_permission()
5581 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3PutBucketRequestPayment
);
5584 void RGWSetRequestPayment::pre_exec()
5586 rgw_bucket_object_pre_exec(s
);
5589 void RGWSetRequestPayment::execute()
5592 if (!store
->svc
.zone
->is_meta_master()) {
5593 op_ret
= forward_request_to_master(s
, nullptr, store
, in_data
, nullptr);
5595 ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
5600 op_ret
= get_params();
5605 s
->bucket_info
.requester_pays
= requester_pays
;
5606 op_ret
= store
->put_bucket_instance_info(s
->bucket_info
, false, real_time(),
5609 ldpp_dout(this, 0) << "NOTICE: put_bucket_info on bucket=" << s
->bucket
.name
5610 << " returned err=" << op_ret
<< dendl
;
5615 int RGWInitMultipart::verify_permission()
5617 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
5618 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
5620 rgw::IAM::s3PutObject
,
5621 rgw_obj(s
->bucket
, s
->object
));
5622 if (usr_policy_res
== Effect::Deny
) {
5626 rgw::IAM::Effect e
= Effect::Pass
;
5627 if (s
->iam_policy
) {
5628 e
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
5629 rgw::IAM::s3PutObject
,
5630 rgw_obj(s
->bucket
, s
->object
));
5632 if (e
== Effect::Allow
) {
5634 } else if (e
== Effect::Deny
) {
5636 } else if (usr_policy_res
== Effect::Allow
) {
5641 if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
5648 void RGWInitMultipart::pre_exec()
5650 rgw_bucket_object_pre_exec(s
);
5653 void RGWInitMultipart::execute()
5656 map
<string
, bufferlist
> attrs
;
5659 if (get_params() < 0)
5662 if (s
->object
.empty())
5665 policy
.encode(aclbl
);
5666 attrs
[RGW_ATTR_ACL
] = aclbl
;
5668 populate_with_generic_attrs(s
, attrs
);
5670 /* select encryption mode */
5671 op_ret
= prepare_encryption(attrs
);
5675 op_ret
= rgw_get_request_metadata(s
->cct
, s
->info
, attrs
);
5682 gen_rand_alphanumeric(s
->cct
, buf
, sizeof(buf
) - 1);
5683 upload_id
= MULTIPART_UPLOAD_ID_PREFIX
; /* v2 upload id */
5684 upload_id
.append(buf
);
5686 string tmp_obj_name
;
5687 RGWMPObj
mp(s
->object
.name
, upload_id
);
5688 tmp_obj_name
= mp
.get_meta();
5690 obj
.init_ns(s
->bucket
, tmp_obj_name
, mp_ns
);
5691 // the meta object will be indexed with 0 size, we c
5692 obj
.set_in_extra_data(true);
5693 obj
.index_hash_source
= s
->object
.name
;
5695 RGWRados::Object
op_target(store
, s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
5696 op_target
.set_versioning_disabled(true); /* no versioning for multipart meta */
5698 RGWRados::Object::Write
obj_op(&op_target
);
5700 obj_op
.meta
.owner
= s
->owner
.get_id();
5701 obj_op
.meta
.category
= RGWObjCategory::MultiMeta
;
5702 obj_op
.meta
.flags
= PUT_OBJ_CREATE_EXCL
;
5704 multipart_upload_info upload_info
;
5705 upload_info
.dest_placement
= s
->dest_placement
;
5708 encode(upload_info
, bl
);
5709 obj_op
.meta
.data
= &bl
;
5711 op_ret
= obj_op
.write_meta(bl
.length(), 0, attrs
);
5712 } while (op_ret
== -EEXIST
);
5714 const auto ret
= rgw::notify::publish(s
, ceph::real_clock::now(), attrs
[RGW_ATTR_ETAG
].to_str(), rgw::notify::ObjectCreatedPost
, store
);
5716 ldpp_dout(this, 5) << "WARNING: publishing notification failed, with error: " << ret
<< dendl
;
5717 // TODO: we should have conf to make send a blocking coroutine and reply with error in case sending failed
5718 // this should be global conf (probably returnign a different handler)
5719 // so we don't need to read the configured values before we perform it
5723 int RGWCompleteMultipart::verify_permission()
5725 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
5726 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
5728 rgw::IAM::s3PutObject
,
5729 rgw_obj(s
->bucket
, s
->object
));
5730 if (usr_policy_res
== Effect::Deny
) {
5734 rgw::IAM::Effect e
= Effect::Pass
;
5735 if (s
->iam_policy
) {
5736 e
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
5737 rgw::IAM::s3PutObject
,
5738 rgw_obj(s
->bucket
, s
->object
));
5740 if (e
== Effect::Allow
) {
5742 } else if (e
== Effect::Deny
) {
5744 } else if (usr_policy_res
== Effect::Allow
) {
5749 if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
5756 void RGWCompleteMultipart::pre_exec()
5758 rgw_bucket_object_pre_exec(s
);
5761 void RGWCompleteMultipart::execute()
5763 RGWMultiCompleteUpload
*parts
;
5764 map
<int, string
>::iterator iter
;
5765 RGWMultiXMLParser parser
;
5767 map
<uint32_t, RGWUploadPartInfo
> obj_parts
;
5768 map
<uint32_t, RGWUploadPartInfo
>::iterator obj_iter
;
5769 map
<string
, bufferlist
> attrs
;
5772 char final_etag
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
5773 char final_etag_str
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 16];
5778 RGWObjManifest manifest
;
5779 uint64_t olh_epoch
= 0;
5781 op_ret
= get_params();
5784 op_ret
= get_system_versioning_params(s
, &olh_epoch
, &version_id
);
5789 if (!data
.length()) {
5790 op_ret
= -ERR_MALFORMED_XML
;
5794 if (!parser
.init()) {
5799 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
5800 op_ret
= -ERR_MALFORMED_XML
;
5804 parts
= static_cast<RGWMultiCompleteUpload
*>(parser
.find_first("CompleteMultipartUpload"));
5805 if (!parts
|| parts
->parts
.empty()) {
5806 op_ret
= -ERR_MALFORMED_XML
;
5810 if ((int)parts
->parts
.size() >
5811 s
->cct
->_conf
->rgw_multipart_part_upload_limit
) {
5816 mp
.init(s
->object
.name
, upload_id
);
5817 meta_oid
= mp
.get_meta();
5819 int total_parts
= 0;
5820 int handled_parts
= 0;
5821 int max_parts
= 1000;
5824 RGWCompressionInfo cs_info
;
5825 bool compressed
= false;
5826 uint64_t accounted_size
= 0;
5828 uint64_t min_part_size
= s
->cct
->_conf
->rgw_multipart_min_part_size
;
5830 list
<rgw_obj_index_key
> remove_objs
; /* objects to be removed from index listing */
5832 bool versioned_object
= s
->bucket_info
.versioning_enabled();
5834 iter
= parts
->parts
.begin();
5836 meta_obj
.init_ns(s
->bucket
, meta_oid
, mp_ns
);
5837 meta_obj
.set_in_extra_data(true);
5838 meta_obj
.index_hash_source
= s
->object
.name
;
5840 /*take a cls lock on meta_obj to prevent racing completions (or retries)
5841 from deleting the parts*/
5843 rgw_raw_obj raw_obj
;
5844 int max_lock_secs_mp
=
5845 s
->cct
->_conf
.get_val
<int64_t>("rgw_mp_lock_max_time");
5846 utime_t
dur(max_lock_secs_mp
, 0);
5848 store
->obj_to_raw((s
->bucket_info
).placement_rule
, meta_obj
, &raw_obj
);
5849 store
->get_obj_data_pool((s
->bucket_info
).placement_rule
,
5850 meta_obj
,&meta_pool
);
5851 store
->open_pool_ctx(meta_pool
, serializer
.ioctx
, true);
5853 op_ret
= serializer
.try_lock(raw_obj
.oid
, dur
);
5855 ldpp_dout(this, 0) << "failed to acquire lock" << dendl
;
5856 op_ret
= -ERR_INTERNAL_ERROR
;
5857 s
->err
.message
= "This multipart completion is already in progress";
5861 op_ret
= get_obj_attrs(store
, s
, meta_obj
, attrs
);
5864 ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << meta_obj
5865 << " ret=" << op_ret
<< dendl
;
5870 op_ret
= list_multipart_parts(store
, s
, upload_id
, meta_oid
, max_parts
,
5871 marker
, obj_parts
, &marker
, &truncated
);
5872 if (op_ret
== -ENOENT
) {
5873 op_ret
= -ERR_NO_SUCH_UPLOAD
;
5878 total_parts
+= obj_parts
.size();
5879 if (!truncated
&& total_parts
!= (int)parts
->parts
.size()) {
5880 ldpp_dout(this, 0) << "NOTICE: total parts mismatch: have: " << total_parts
5881 << " expected: " << parts
->parts
.size() << dendl
;
5882 op_ret
= -ERR_INVALID_PART
;
5886 for (obj_iter
= obj_parts
.begin(); iter
!= parts
->parts
.end() && obj_iter
!= obj_parts
.end(); ++iter
, ++obj_iter
, ++handled_parts
) {
5887 uint64_t part_size
= obj_iter
->second
.accounted_size
;
5888 if (handled_parts
< (int)parts
->parts
.size() - 1 &&
5889 part_size
< min_part_size
) {
5890 op_ret
= -ERR_TOO_SMALL
;
5894 char petag
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
5895 if (iter
->first
!= (int)obj_iter
->first
) {
5896 ldpp_dout(this, 0) << "NOTICE: parts num mismatch: next requested: "
5897 << iter
->first
<< " next uploaded: "
5898 << obj_iter
->first
<< dendl
;
5899 op_ret
= -ERR_INVALID_PART
;
5902 string part_etag
= rgw_string_unquote(iter
->second
);
5903 if (part_etag
.compare(obj_iter
->second
.etag
) != 0) {
5904 ldpp_dout(this, 0) << "NOTICE: etag mismatch: part: " << iter
->first
5905 << " etag: " << iter
->second
<< dendl
;
5906 op_ret
= -ERR_INVALID_PART
;
5910 hex_to_buf(obj_iter
->second
.etag
.c_str(), petag
,
5911 CEPH_CRYPTO_MD5_DIGESTSIZE
);
5912 hash
.Update((const unsigned char *)petag
, sizeof(petag
));
5914 RGWUploadPartInfo
& obj_part
= obj_iter
->second
;
5916 /* update manifest for part */
5917 string oid
= mp
.get_part(obj_iter
->second
.num
);
5919 src_obj
.init_ns(s
->bucket
, oid
, mp_ns
);
5921 if (obj_part
.manifest
.empty()) {
5922 ldpp_dout(this, 0) << "ERROR: empty manifest for object part: obj="
5923 << src_obj
<< dendl
;
5924 op_ret
= -ERR_INVALID_PART
;
5927 manifest
.append(obj_part
.manifest
, store
->svc
.zone
);
5930 bool part_compressed
= (obj_part
.cs_info
.compression_type
!= "none");
5931 if ((obj_iter
!= obj_parts
.begin()) &&
5932 ((part_compressed
!= compressed
) ||
5933 (cs_info
.compression_type
!= obj_part
.cs_info
.compression_type
))) {
5934 ldpp_dout(this, 0) << "ERROR: compression type was changed during multipart upload ("
5935 << cs_info
.compression_type
<< ">>" << obj_part
.cs_info
.compression_type
<< ")" << dendl
;
5936 op_ret
= -ERR_INVALID_PART
;
5940 if (part_compressed
) {
5941 int64_t new_ofs
; // offset in compression data for new part
5942 if (cs_info
.blocks
.size() > 0)
5943 new_ofs
= cs_info
.blocks
.back().new_ofs
+ cs_info
.blocks
.back().len
;
5946 for (const auto& block
: obj_part
.cs_info
.blocks
) {
5947 compression_block cb
;
5948 cb
.old_ofs
= block
.old_ofs
+ cs_info
.orig_size
;
5949 cb
.new_ofs
= new_ofs
;
5951 cs_info
.blocks
.push_back(cb
);
5952 new_ofs
= cb
.new_ofs
+ cb
.len
;
5955 cs_info
.compression_type
= obj_part
.cs_info
.compression_type
;
5956 cs_info
.orig_size
+= obj_part
.cs_info
.orig_size
;
5960 rgw_obj_index_key remove_key
;
5961 src_obj
.key
.get_index_key(&remove_key
);
5963 remove_objs
.push_back(remove_key
);
5965 ofs
+= obj_part
.size
;
5966 accounted_size
+= obj_part
.accounted_size
;
5968 } while (truncated
);
5969 hash
.Final((unsigned char *)final_etag
);
5971 buf_to_hex((unsigned char *)final_etag
, sizeof(final_etag
), final_etag_str
);
5972 snprintf(&final_etag_str
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2], sizeof(final_etag_str
) - CEPH_CRYPTO_MD5_DIGESTSIZE
* 2,
5973 "-%lld", (long long)parts
->parts
.size());
5974 etag
= final_etag_str
;
5975 ldpp_dout(this, 10) << "calculated etag: " << final_etag_str
<< dendl
;
5977 etag_bl
.append(final_etag_str
, strlen(final_etag_str
));
5979 attrs
[RGW_ATTR_ETAG
] = etag_bl
;
5982 // write compression attribute to full object
5984 encode(cs_info
, tmp
);
5985 attrs
[RGW_ATTR_COMPRESSION
] = tmp
;
5988 target_obj
.init(s
->bucket
, s
->object
.name
);
5989 if (versioned_object
) {
5990 if (!version_id
.empty()) {
5991 target_obj
.key
.set_instance(version_id
);
5993 store
->gen_rand_obj_instance_name(&target_obj
);
5994 version_id
= target_obj
.key
.get_instance();
5998 RGWObjectCtx
& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
6000 obj_ctx
.set_atomic(target_obj
);
6002 RGWRados::Object
op_target(store
, s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), target_obj
);
6003 RGWRados::Object::Write
obj_op(&op_target
);
6005 obj_op
.meta
.manifest
= &manifest
;
6006 obj_op
.meta
.remove_objs
= &remove_objs
;
6008 obj_op
.meta
.ptag
= &s
->req_id
; /* use req_id as operation tag */
6009 obj_op
.meta
.owner
= s
->owner
.get_id();
6010 obj_op
.meta
.flags
= PUT_OBJ_CREATE
;
6011 obj_op
.meta
.modify_tail
= true;
6012 obj_op
.meta
.completeMultipart
= true;
6013 obj_op
.meta
.olh_epoch
= olh_epoch
;
6014 op_ret
= obj_op
.write_meta(ofs
, accounted_size
, attrs
);
6018 // remove the upload obj
6019 int r
= store
->delete_obj(*static_cast<RGWObjectCtx
*>(s
->obj_ctx
),
6020 s
->bucket_info
, meta_obj
, 0);
6022 /* serializer's exclusive lock is released */
6023 serializer
.clear_locked();
6025 ldpp_dout(this, 0) << "WARNING: failed to remove object " << meta_obj
<< dendl
;
6028 const auto ret
= rgw::notify::publish(s
, ceph::real_clock::now(), etag
, rgw::notify::ObjectCreatedCompleteMultipartUpload
, store
);
6030 ldpp_dout(this, 5) << "WARNING: publishing notification failed, with error: " << ret
<< dendl
;
6031 // TODO: we should have conf to make send a blocking coroutine and reply with error in case sending failed
6032 // this should be global conf (probably returnign a different handler)
6033 // so we don't need to read the configured values before we perform it
6037 int RGWCompleteMultipart::MPSerializer::try_lock(
6038 const std::string
& _oid
,
6043 lock
.set_duration(dur
);
6044 lock
.lock_exclusive(&op
);
6045 int ret
= ioctx
.operate(oid
, &op
);
6052 void RGWCompleteMultipart::complete()
6054 /* release exclusive lock iff not already */
6055 if (unlikely(serializer
.locked
)) {
6056 int r
= serializer
.unlock();
6058 ldpp_dout(this, 0) << "WARNING: failed to unlock " << serializer
.oid
<< dendl
;
6064 int RGWAbortMultipart::verify_permission()
6066 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
6067 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
6069 rgw::IAM::s3AbortMultipartUpload
,
6070 rgw_obj(s
->bucket
, s
->object
));
6071 if (usr_policy_res
== Effect::Deny
) {
6075 rgw::IAM::Effect e
= Effect::Pass
;
6076 if (s
->iam_policy
) {
6077 e
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
6078 rgw::IAM::s3AbortMultipartUpload
,
6079 rgw_obj(s
->bucket
, s
->object
));
6081 if (e
== Effect::Allow
) {
6083 } else if (e
== Effect::Deny
) {
6085 } else if (usr_policy_res
== Effect::Allow
)
6089 if (!verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
6096 void RGWAbortMultipart::pre_exec()
6098 rgw_bucket_object_pre_exec(s
);
6101 void RGWAbortMultipart::execute()
6106 upload_id
= s
->info
.args
.get("uploadId");
6110 if (upload_id
.empty() || s
->object
.empty())
6113 mp
.init(s
->object
.name
, upload_id
);
6114 meta_oid
= mp
.get_meta();
6116 op_ret
= get_multipart_info(store
, s
, meta_oid
, nullptr, nullptr, nullptr);
6120 RGWObjectCtx
*obj_ctx
= static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
6121 op_ret
= abort_multipart_upload(store
, s
->cct
, obj_ctx
, s
->bucket_info
, mp
);
6124 int RGWListMultipart::verify_permission()
6126 if (!verify_object_permission(this, s
, rgw::IAM::s3ListMultipartUploadParts
))
6132 void RGWListMultipart::pre_exec()
6134 rgw_bucket_object_pre_exec(s
);
6137 void RGWListMultipart::execute()
6142 op_ret
= get_params();
6146 mp
.init(s
->object
.name
, upload_id
);
6147 meta_oid
= mp
.get_meta();
6149 op_ret
= get_multipart_info(store
, s
, meta_oid
, &policy
, nullptr, nullptr);
6153 op_ret
= list_multipart_parts(store
, s
, upload_id
, meta_oid
, max_parts
,
6154 marker
, parts
, NULL
, &truncated
);
6157 int RGWListBucketMultiparts::verify_permission()
6159 if (!verify_bucket_permission(this,
6161 rgw::IAM::s3ListBucketMultipartUploads
))
6167 void RGWListBucketMultiparts::pre_exec()
6169 rgw_bucket_object_pre_exec(s
);
6172 void RGWListBucketMultiparts::execute()
6174 vector
<rgw_bucket_dir_entry
> objs
;
6177 op_ret
= get_params();
6181 if (s
->prot_flags
& RGW_REST_SWIFT
) {
6183 path_args
= s
->info
.args
.get("path");
6184 if (!path_args
.empty()) {
6185 if (!delimiter
.empty() || !prefix
.empty()) {
6193 marker_meta
= marker
.get_meta();
6195 op_ret
= list_bucket_multiparts(store
, s
->bucket_info
, prefix
, marker_meta
, delimiter
,
6196 max_uploads
, &objs
, &common_prefixes
, &is_truncated
);
6201 if (!objs
.empty()) {
6202 vector
<rgw_bucket_dir_entry
>::iterator iter
;
6203 RGWMultipartUploadEntry entry
;
6204 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
6205 rgw_obj_key
key(iter
->key
);
6206 if (!entry
.mp
.from_meta(key
.name
))
6209 uploads
.push_back(entry
);
6211 next_marker
= entry
;
6215 void RGWGetHealthCheck::execute()
6217 if (!g_conf()->rgw_healthcheck_disabling_path
.empty() &&
6218 (::access(g_conf()->rgw_healthcheck_disabling_path
.c_str(), F_OK
) == 0)) {
6219 /* Disabling path specified & existent in the filesystem. */
6220 op_ret
= -ERR_SERVICE_UNAVAILABLE
; /* 503 */
6222 op_ret
= 0; /* 200 OK */
6226 int RGWDeleteMultiObj::verify_permission()
6228 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
6229 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
6231 s
->object
.instance
.empty() ?
6232 rgw::IAM::s3DeleteObject
:
6233 rgw::IAM::s3DeleteObjectVersion
,
6235 if (usr_policy_res
== Effect::Deny
) {
6239 rgw::IAM::Effect r
= Effect::Pass
;
6240 if (s
->iam_policy
) {
6241 r
= s
->iam_policy
->eval(s
->env
, *s
->auth
.identity
,
6242 s
->object
.instance
.empty() ?
6243 rgw::IAM::s3DeleteObject
:
6244 rgw::IAM::s3DeleteObjectVersion
,
6247 if (r
== Effect::Allow
)
6249 else if (r
== Effect::Deny
)
6251 else if (usr_policy_res
== Effect::Allow
)
6255 acl_allowed
= verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
);
6262 void RGWDeleteMultiObj::pre_exec()
6264 rgw_bucket_object_pre_exec(s
);
6267 void RGWDeleteMultiObj::execute()
6269 RGWMultiDelDelete
*multi_delete
;
6270 vector
<rgw_obj_key
>::iterator iter
;
6271 RGWMultiDelXMLParser parser
;
6272 RGWObjectCtx
*obj_ctx
= static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
6275 op_ret
= get_params();
6286 if (!parser
.init()) {
6291 if (!parser
.parse(buf
, data
.length(), 1)) {
6296 multi_delete
= static_cast<RGWMultiDelDelete
*>(parser
.find_first("Delete"));
6297 if (!multi_delete
) {
6301 #define DELETE_MULTI_OBJ_MAX_NUM 1000
6302 int max_num
= s
->cct
->_conf
->rgw_delete_multi_obj_max_num
;
6304 max_num
= DELETE_MULTI_OBJ_MAX_NUM
;
6306 int multi_delete_object_num
= multi_delete
->objects
.size();
6307 if (multi_delete_object_num
> max_num
) {
6308 op_ret
= -ERR_MALFORMED_XML
;
6313 if (multi_delete
->is_quiet())
6316 if (s
->bucket_info
.mfa_enabled()) {
6317 bool has_versioned
= false;
6318 for (auto i
: multi_delete
->objects
) {
6319 if (!i
.instance
.empty()) {
6320 has_versioned
= true;
6324 if (has_versioned
&& !s
->mfa_verified
) {
6325 ldpp_dout(this, 5) << "NOTICE: multi-object delete request with a versioned object, mfa auth not provided" << dendl
;
6326 op_ret
= -ERR_MFA_REQUIRED
;
6332 if (multi_delete
->objects
.empty()) {
6336 for (iter
= multi_delete
->objects
.begin();
6337 iter
!= multi_delete
->objects
.end();
6339 rgw_obj
obj(bucket
, *iter
);
6340 if (s
->iam_policy
|| ! s
->iam_user_policies
.empty()) {
6341 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
6343 iter
->instance
.empty() ?
6344 rgw::IAM::s3DeleteObject
:
6345 rgw::IAM::s3DeleteObjectVersion
,
6347 if (usr_policy_res
== Effect::Deny
) {
6348 send_partial_response(*iter
, false, "", -EACCES
);
6352 rgw::IAM::Effect e
= Effect::Pass
;
6353 if (s
->iam_policy
) {
6354 e
= s
->iam_policy
->eval(s
->env
,
6356 iter
->instance
.empty() ?
6357 rgw::IAM::s3DeleteObject
:
6358 rgw::IAM::s3DeleteObjectVersion
,
6361 if ((e
== Effect::Deny
) ||
6362 (usr_policy_res
== Effect::Pass
&& e
== Effect::Pass
&& !acl_allowed
)) {
6363 send_partial_response(*iter
, false, "", -EACCES
);
6368 obj_ctx
->set_atomic(obj
);
6370 RGWRados::Object
del_target(store
, s
->bucket_info
, *obj_ctx
, obj
);
6371 RGWRados::Object::Delete
del_op(&del_target
);
6373 del_op
.params
.bucket_owner
= s
->bucket_owner
.get_id();
6374 del_op
.params
.versioning_status
= s
->bucket_info
.versioning_status();
6375 del_op
.params
.obj_owner
= s
->owner
;
6377 op_ret
= del_op
.delete_obj();
6378 if (op_ret
== -ENOENT
) {
6382 send_partial_response(*iter
, del_op
.result
.delete_marker
,
6383 del_op
.result
.version_id
, op_ret
);
6386 /* set the return code to zero, errors at this point will be
6387 dumped to the response */
6391 // will likely segfault if begin_response() has not been called
6401 bool RGWBulkDelete::Deleter::verify_permission(RGWBucketInfo
& binfo
,
6402 map
<string
, bufferlist
>& battrs
,
6403 ACLOwner
& bucket_owner
/* out */)
6405 RGWAccessControlPolicy
bacl(store
->ctx());
6406 int ret
= read_bucket_policy(store
, s
, binfo
, battrs
, &bacl
, binfo
.bucket
);
6411 auto policy
= get_iam_policy_from_attr(s
->cct
, store
, battrs
, binfo
.bucket
.tenant
);
6413 bucket_owner
= bacl
.get_owner();
6415 /* We can use global user_acl because each BulkDelete request is allowed
6416 * to work on entities from a single account only. */
6417 return verify_bucket_permission(dpp
, s
, binfo
.bucket
, s
->user_acl
.get(),
6418 &bacl
, policy
, s
->iam_user_policies
, rgw::IAM::s3DeleteBucket
);
6421 bool RGWBulkDelete::Deleter::delete_single(const acct_path_t
& path
)
6423 auto& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
6425 RGWBucketInfo binfo
;
6426 map
<string
, bufferlist
> battrs
;
6429 int ret
= store
->get_bucket_info(*s
->sysobj_ctx
, s
->user
->user_id
.tenant
,
6430 path
.bucket_name
, binfo
, nullptr,
6436 if (!verify_permission(binfo
, battrs
, bowner
)) {
6441 if (!path
.obj_key
.empty()) {
6442 rgw_obj
obj(binfo
.bucket
, path
.obj_key
);
6443 obj_ctx
.set_atomic(obj
);
6445 RGWRados::Object
del_target(store
, binfo
, obj_ctx
, obj
);
6446 RGWRados::Object::Delete
del_op(&del_target
);
6448 del_op
.params
.bucket_owner
= binfo
.owner
;
6449 del_op
.params
.versioning_status
= binfo
.versioning_status();
6450 del_op
.params
.obj_owner
= bowner
;
6452 ret
= del_op
.delete_obj();
6457 RGWObjVersionTracker ot
;
6458 ot
.read_version
= binfo
.ep_objv
;
6460 ret
= store
->delete_bucket(binfo
, ot
);
6462 ret
= rgw_unlink_bucket(store
, binfo
.owner
, binfo
.bucket
.tenant
,
6463 binfo
.bucket
.name
, false);
6465 ldpp_dout(s
, 0) << "WARNING: failed to unlink bucket: ret=" << ret
<< dendl
;
6472 if (!store
->svc
.zone
->is_meta_master()) {
6474 ret
= forward_request_to_master(s
, &ot
.read_version
, store
, in_data
,
6477 if (ret
== -ENOENT
) {
6478 /* adjust error, we want to return with NoSuchBucket and not
6480 ret
= -ERR_NO_SUCH_BUCKET
;
6492 if (-ENOENT
== ret
) {
6493 ldpp_dout(s
, 20) << "cannot find bucket = " << path
.bucket_name
<< dendl
;
6496 ldpp_dout(s
, 20) << "cannot get bucket info, ret = " << ret
<< dendl
;
6498 fail_desc_t failed_item
= {
6502 failures
.push_back(failed_item
);
6507 ldpp_dout(s
, 20) << "wrong auth for " << path
<< dendl
;
6509 fail_desc_t failed_item
= {
6513 failures
.push_back(failed_item
);
6518 if (-ENOENT
== ret
) {
6519 ldpp_dout(s
, 20) << "cannot find entry " << path
<< dendl
;
6522 fail_desc_t failed_item
= {
6526 failures
.push_back(failed_item
);
6531 bool RGWBulkDelete::Deleter::delete_chunk(const std::list
<acct_path_t
>& paths
)
6533 ldpp_dout(s
, 20) << "in delete_chunk" << dendl
;
6534 for (auto path
: paths
) {
6535 ldpp_dout(s
, 20) << "bulk deleting path: " << path
<< dendl
;
6536 delete_single(path
);
6542 int RGWBulkDelete::verify_permission()
6547 void RGWBulkDelete::pre_exec()
6549 rgw_bucket_object_pre_exec(s
);
6552 void RGWBulkDelete::execute()
6554 deleter
= std::unique_ptr
<Deleter
>(new Deleter(this, store
, s
));
6556 bool is_truncated
= false;
6558 list
<RGWBulkDelete::acct_path_t
> items
;
6560 int ret
= get_data(items
, &is_truncated
);
6565 ret
= deleter
->delete_chunk(items
);
6566 } while (!op_ret
&& is_truncated
);
6572 constexpr std::array
<int, 2> RGWBulkUploadOp::terminal_errors
;
6574 int RGWBulkUploadOp::verify_permission()
6576 if (s
->auth
.identity
->is_anonymous()) {
6580 if (! verify_user_permission_no_policy(this, s
, RGW_PERM_WRITE
)) {
6584 if (s
->user
->user_id
.tenant
!= s
->bucket_tenant
) {
6585 ldpp_dout(this, 10) << "user cannot create a bucket in a different tenant"
6586 << " (user_id.tenant=" << s
->user
->user_id
.tenant
6587 << " requested=" << s
->bucket_tenant
<< ")" << dendl
;
6591 if (s
->user
->max_buckets
< 0) {
6598 void RGWBulkUploadOp::pre_exec()
6600 rgw_bucket_object_pre_exec(s
);
6603 boost::optional
<std::pair
<std::string
, rgw_obj_key
>>
6604 RGWBulkUploadOp::parse_path(const boost::string_ref
& path
)
6606 /* We need to skip all slashes at the beginning in order to preserve
6607 * compliance with Swift. */
6608 const size_t start_pos
= path
.find_first_not_of('/');
6610 if (boost::string_ref::npos
!= start_pos
) {
6611 /* Seperator is the first slash after the leading ones. */
6612 const size_t sep_pos
= path
.substr(start_pos
).find('/');
6614 if (boost::string_ref::npos
!= sep_pos
) {
6615 const auto bucket_name
= path
.substr(start_pos
, sep_pos
- start_pos
);
6616 const auto obj_name
= path
.substr(sep_pos
+ 1);
6618 return std::make_pair(bucket_name
.to_string(),
6619 rgw_obj_key(obj_name
.to_string()));
6621 /* It's guaranteed here that bucket name is at least one character
6622 * long and is different than slash. */
6623 return std::make_pair(path
.substr(start_pos
).to_string(),
6631 std::pair
<std::string
, std::string
>
6632 RGWBulkUploadOp::handle_upload_path(struct req_state
*s
)
6634 std::string bucket_path
, file_prefix
;
6635 if (! s
->init_state
.url_bucket
.empty()) {
6636 file_prefix
= bucket_path
= s
->init_state
.url_bucket
+ "/";
6637 if (! s
->object
.empty()) {
6638 std::string
& object_name
= s
->object
.name
;
6640 /* As rgw_obj_key::empty() already verified emptiness of s->object.name,
6641 * we can safely examine its last element. */
6642 if (object_name
.back() == '/') {
6643 file_prefix
.append(object_name
);
6645 file_prefix
.append(object_name
).append("/");
6649 return std::make_pair(bucket_path
, file_prefix
);
6652 int RGWBulkUploadOp::handle_dir_verify_permission()
6654 if (s
->user
->max_buckets
> 0) {
6655 RGWUserBuckets buckets
;
6657 bool is_truncated
= false;
6658 op_ret
= rgw_read_user_buckets(store
, s
->user
->user_id
, buckets
,
6659 marker
, std::string(), s
->user
->max_buckets
,
6660 false, &is_truncated
);
6665 if (buckets
.count() >= static_cast<size_t>(s
->user
->max_buckets
)) {
6666 return -ERR_TOO_MANY_BUCKETS
;
6673 static void forward_req_info(CephContext
*cct
, req_info
& info
, const std::string
& bucket_name
)
6675 /* the request of container or object level will contain bucket name.
6676 * only at account level need to append the bucket name */
6677 if (info
.script_uri
.find(bucket_name
) != std::string::npos
) {
6681 ldout(cct
, 20) << "append the bucket: "<< bucket_name
<< " to req_info" << dendl
;
6682 info
.script_uri
.append("/").append(bucket_name
);
6683 info
.request_uri_aws4
= info
.request_uri
= info
.script_uri
;
6684 info
.effective_uri
= "/" + bucket_name
;
6687 void RGWBulkUploadOp::init(RGWRados
* const store
,
6688 struct req_state
* const s
,
6689 RGWHandler
* const h
)
6691 RGWOp::init(store
, s
, h
);
6692 dir_ctx
.emplace(store
->svc
.sysobj
->init_obj_ctx());
6695 int RGWBulkUploadOp::handle_dir(const boost::string_ref path
)
6697 ldpp_dout(this, 20) << "got directory=" << path
<< dendl
;
6699 op_ret
= handle_dir_verify_permission();
6704 std::string bucket_name
;
6705 rgw_obj_key object_junk
;
6706 std::tie(bucket_name
, object_junk
) = *parse_path(path
);
6708 rgw_raw_obj
obj(store
->svc
.zone
->get_zone_params().domain_root
,
6709 rgw_make_bucket_entry_name(s
->bucket_tenant
, bucket_name
));
6711 /* we need to make sure we read bucket info, it's not read before for this
6712 * specific request */
6713 RGWBucketInfo binfo
;
6714 std::map
<std::string
, ceph::bufferlist
> battrs
;
6715 op_ret
= store
->get_bucket_info(*dir_ctx
, s
->bucket_tenant
, bucket_name
,
6716 binfo
, nullptr, &battrs
);
6717 if (op_ret
< 0 && op_ret
!= -ENOENT
) {
6720 const bool bucket_exists
= (op_ret
!= -ENOENT
);
6722 if (bucket_exists
) {
6723 RGWAccessControlPolicy
old_policy(s
->cct
);
6724 int r
= rgw_op_get_bucket_policy_from_attr(s
->cct
, store
, binfo
,
6725 battrs
, &old_policy
);
6727 if (old_policy
.get_owner().get_id().compare(s
->user
->user_id
) != 0) {
6734 RGWBucketInfo master_info
;
6735 rgw_bucket
*pmaster_bucket
= nullptr;
6736 uint32_t *pmaster_num_shards
= nullptr;
6737 real_time creation_time
;
6738 obj_version objv
, ep_objv
, *pobjv
= nullptr;
6740 if (! store
->svc
.zone
->is_meta_master()) {
6742 ceph::bufferlist in_data
;
6743 req_info info
= s
->info
;
6744 forward_req_info(s
->cct
, info
, bucket_name
);
6745 op_ret
= forward_request_to_master(s
, nullptr, store
, in_data
, &jp
, &info
);
6750 JSONDecoder::decode_json("entry_point_object_ver", ep_objv
, &jp
);
6751 JSONDecoder::decode_json("object_ver", objv
, &jp
);
6752 JSONDecoder::decode_json("bucket_info", master_info
, &jp
);
6754 ldpp_dout(this, 20) << "parsed: objv.tag=" << objv
.tag
<< " objv.ver=" << objv
.ver
<< dendl
;
6755 ldpp_dout(this, 20) << "got creation_time="<< master_info
.creation_time
<< dendl
;
6757 pmaster_bucket
= &master_info
.bucket
;
6758 creation_time
= master_info
.creation_time
;
6759 pmaster_num_shards
= &master_info
.num_shards
;
6762 pmaster_bucket
= nullptr;
6763 pmaster_num_shards
= nullptr;
6766 rgw_placement_rule
placement_rule(binfo
.placement_rule
, s
->info
.storage_class
);
6768 if (bucket_exists
) {
6769 rgw_placement_rule selected_placement_rule
;
6771 bucket
.tenant
= s
->bucket_tenant
;
6772 bucket
.name
= s
->bucket_name
;
6773 op_ret
= store
->svc
.zone
->select_bucket_placement(*(s
->user
),
6774 store
->svc
.zone
->get_zonegroup().get_id(),
6776 &selected_placement_rule
,
6778 if (selected_placement_rule
!= binfo
.placement_rule
) {
6780 ldpp_dout(this, 20) << "non-coherent placement rule" << dendl
;
6785 /* Create metadata: ACLs. */
6786 std::map
<std::string
, ceph::bufferlist
> attrs
;
6787 RGWAccessControlPolicy policy
;
6788 policy
.create_default(s
->user
->user_id
, s
->user
->display_name
);
6789 ceph::bufferlist aclbl
;
6790 policy
.encode(aclbl
);
6791 attrs
.emplace(RGW_ATTR_ACL
, std::move(aclbl
));
6793 RGWQuotaInfo quota_info
;
6794 const RGWQuotaInfo
* pquota_info
= nullptr;
6797 bucket
.tenant
= s
->bucket_tenant
; /* ignored if bucket exists */
6798 bucket
.name
= bucket_name
;
6801 RGWBucketInfo out_info
;
6802 op_ret
= store
->create_bucket(*(s
->user
),
6804 store
->svc
.zone
->get_zonegroup().get_id(),
6805 placement_rule
, binfo
.swift_ver_location
,
6807 out_info
, pobjv
, &ep_objv
, creation_time
,
6808 pmaster_bucket
, pmaster_num_shards
, true);
6809 /* continue if EEXIST and create_bucket will fail below. this way we can
6810 * recover from a partial create by retrying it. */
6811 ldpp_dout(this, 20) << "rgw_create_bucket returned ret=" << op_ret
6812 << ", bucket=" << bucket
<< dendl
;
6814 if (op_ret
&& op_ret
!= -EEXIST
) {
6818 const bool existed
= (op_ret
== -EEXIST
);
6820 /* bucket already existed, might have raced with another bucket creation, or
6821 * might be partial bucket creation that never completed. Read existing bucket
6822 * info, verify that the reported bucket owner is the current user.
6823 * If all is ok then update the user's list of buckets.
6824 * Otherwise inform client about a name conflict.
6826 if (out_info
.owner
.compare(s
->user
->user_id
) != 0) {
6828 ldpp_dout(this, 20) << "conflicting bucket name" << dendl
;
6831 bucket
= out_info
.bucket
;
6834 op_ret
= rgw_link_bucket(store
, s
->user
->user_id
, bucket
,
6835 out_info
.creation_time
, false);
6836 if (op_ret
&& !existed
&& op_ret
!= -EEXIST
) {
6837 /* if it exists (or previously existed), don't remove it! */
6838 op_ret
= rgw_unlink_bucket(store
, s
->user
->user_id
,
6839 bucket
.tenant
, bucket
.name
);
6841 ldpp_dout(this, 0) << "WARNING: failed to unlink bucket: ret=" << op_ret
<< dendl
;
6843 } else if (op_ret
== -EEXIST
|| (op_ret
== 0 && existed
)) {
6844 ldpp_dout(this, 20) << "containers already exists" << dendl
;
6845 op_ret
= -ERR_BUCKET_EXISTS
;
6852 bool RGWBulkUploadOp::handle_file_verify_permission(RGWBucketInfo
& binfo
,
6854 std::map
<std::string
, ceph::bufferlist
>& battrs
,
6855 ACLOwner
& bucket_owner
/* out */)
6857 RGWAccessControlPolicy
bacl(store
->ctx());
6858 op_ret
= read_bucket_policy(store
, s
, binfo
, battrs
, &bacl
, binfo
.bucket
);
6860 ldpp_dout(this, 20) << "cannot read_policy() for bucket" << dendl
;
6864 auto policy
= get_iam_policy_from_attr(s
->cct
, store
, battrs
, binfo
.bucket
.tenant
);
6866 bucket_owner
= bacl
.get_owner();
6867 if (policy
|| ! s
->iam_user_policies
.empty()) {
6868 auto usr_policy_res
= eval_user_policies(s
->iam_user_policies
, s
->env
,
6870 rgw::IAM::s3PutObject
, obj
);
6871 if (usr_policy_res
== Effect::Deny
) {
6874 auto e
= policy
->eval(s
->env
, *s
->auth
.identity
,
6875 rgw::IAM::s3PutObject
, obj
);
6876 if (e
== Effect::Allow
) {
6878 } else if (e
== Effect::Deny
) {
6880 } else if (usr_policy_res
== Effect::Allow
) {
6885 return verify_bucket_permission_no_policy(this, s
, s
->user_acl
.get(),
6886 &bacl
, RGW_PERM_WRITE
);
6889 int RGWBulkUploadOp::handle_file(const boost::string_ref path
,
6891 AlignedStreamGetter
& body
)
6894 ldpp_dout(this, 20) << "got file=" << path
<< ", size=" << size
<< dendl
;
6896 if (size
> static_cast<size_t>(s
->cct
->_conf
->rgw_max_put_size
)) {
6897 op_ret
= -ERR_TOO_LARGE
;
6901 std::string bucket_name
;
6903 std::tie(bucket_name
, object
) = *parse_path(path
);
6905 auto& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
6906 RGWBucketInfo binfo
;
6907 std::map
<std::string
, ceph::bufferlist
> battrs
;
6909 op_ret
= store
->get_bucket_info(*s
->sysobj_ctx
, s
->user
->user_id
.tenant
,
6910 bucket_name
, binfo
, nullptr, &battrs
);
6911 if (op_ret
== -ENOENT
) {
6912 ldpp_dout(this, 20) << "non existent directory=" << bucket_name
<< dendl
;
6913 } else if (op_ret
< 0) {
6917 if (! handle_file_verify_permission(binfo
,
6918 rgw_obj(binfo
.bucket
, object
),
6920 ldpp_dout(this, 20) << "object creation unauthorized" << dendl
;
6925 op_ret
= store
->check_quota(bowner
.get_id(), binfo
.bucket
,
6926 user_quota
, bucket_quota
, size
);
6931 op_ret
= store
->check_bucket_shards(s
->bucket_info
, s
->bucket
, bucket_quota
);
6936 rgw_obj
obj(binfo
.bucket
, object
);
6937 if (s
->bucket_info
.versioning_enabled()) {
6938 store
->gen_rand_obj_instance_name(&obj
);
6941 rgw_placement_rule dest_placement
= s
->dest_placement
;
6942 dest_placement
.inherit_from(binfo
.placement_rule
);
6944 rgw::AioThrottle
aio(store
->ctx()->_conf
->rgw_put_obj_min_window_size
);
6946 using namespace rgw::putobj
;
6948 AtomicObjectProcessor
processor(&aio
, store
, binfo
, &s
->dest_placement
, bowner
.get_id(),
6949 obj_ctx
, obj
, 0, s
->req_id
);
6951 op_ret
= processor
.prepare();
6953 ldpp_dout(this, 20) << "cannot prepare processor due to ret=" << op_ret
<< dendl
;
6957 /* No filters by default. */
6958 DataProcessor
*filter
= &processor
;
6960 const auto& compression_type
= store
->svc
.zone
->get_zone_params().get_compression_type(
6962 CompressorRef plugin
;
6963 boost::optional
<RGWPutObj_Compress
> compressor
;
6964 if (compression_type
!= "none") {
6965 plugin
= Compressor::create(s
->cct
, compression_type
);
6967 ldpp_dout(this, 1) << "Cannot load plugin for rgw_compression_type "
6968 << compression_type
<< dendl
;
6970 compressor
.emplace(s
->cct
, plugin
, filter
);
6971 filter
= &*compressor
;
6975 /* Upload file content. */
6980 ceph::bufferlist data
;
6981 len
= body
.get_at_most(s
->cct
->_conf
->rgw_max_chunk_size
, data
);
6983 ldpp_dout(this, 20) << "body=" << data
.c_str() << dendl
;
6987 } else if (len
> 0) {
6988 hash
.Update((const unsigned char *)data
.c_str(), data
.length());
6989 op_ret
= filter
->process(std::move(data
), ofs
);
6991 ldpp_dout(this, 20) << "filter->process() returned ret=" << op_ret
<< dendl
;
7001 op_ret
= filter
->process({}, ofs
);
7007 ldpp_dout(this, 10) << "real file size different from declared" << dendl
;
7012 op_ret
= store
->check_quota(bowner
.get_id(), binfo
.bucket
,
7013 user_quota
, bucket_quota
, size
);
7015 ldpp_dout(this, 20) << "quota exceeded for path=" << path
<< dendl
;
7019 op_ret
= store
->check_bucket_shards(s
->bucket_info
, s
->bucket
, bucket_quota
);
7024 char calc_md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
7025 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
7027 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, calc_md5
);
7029 /* Create metadata: ETAG. */
7030 std::map
<std::string
, ceph::bufferlist
> attrs
;
7031 std::string etag
= calc_md5
;
7032 ceph::bufferlist etag_bl
;
7033 etag_bl
.append(etag
.c_str(), etag
.size() + 1);
7034 attrs
.emplace(RGW_ATTR_ETAG
, std::move(etag_bl
));
7036 /* Create metadata: ACLs. */
7037 RGWAccessControlPolicy policy
;
7038 policy
.create_default(s
->user
->user_id
, s
->user
->display_name
);
7039 ceph::bufferlist aclbl
;
7040 policy
.encode(aclbl
);
7041 attrs
.emplace(RGW_ATTR_ACL
, std::move(aclbl
));
7043 /* Create metadata: compression info. */
7044 if (compressor
&& compressor
->is_compressed()) {
7045 ceph::bufferlist tmp
;
7046 RGWCompressionInfo cs_info
;
7047 cs_info
.compression_type
= plugin
->get_type_name();
7048 cs_info
.orig_size
= s
->obj_size
;
7049 cs_info
.blocks
= std::move(compressor
->get_compression_blocks());
7050 encode(cs_info
, tmp
);
7051 attrs
.emplace(RGW_ATTR_COMPRESSION
, std::move(tmp
));
7054 /* Complete the transaction. */
7055 op_ret
= processor
.complete(size
, etag
, nullptr, ceph::real_time(),
7056 attrs
, ceph::real_time() /* delete_at */,
7057 nullptr, nullptr, nullptr, nullptr, nullptr);
7059 ldpp_dout(this, 20) << "processor::complete returned op_ret=" << op_ret
<< dendl
;
7065 void RGWBulkUploadOp::execute()
7067 ceph::bufferlist
buffer(64 * 1024);
7069 ldpp_dout(this, 20) << "start" << dendl
;
7071 /* Create an instance of stream-abstracting class. Having this indirection
7072 * allows for easy introduction of decompressors like gzip and bzip2. */
7073 auto stream
= create_stream();
7078 /* Handling the $UPLOAD_PATH accordingly to the Swift's Bulk middleware. See:
7079 * https://github.com/openstack/swift/blob/2.13.0/swift/common/middleware/bulk.py#L31-L41 */
7080 std::string bucket_path
, file_prefix
;
7081 std::tie(bucket_path
, file_prefix
) = handle_upload_path(s
);
7083 auto status
= rgw::tar::StatusIndicator::create();
7085 op_ret
= stream
->get_exactly(rgw::tar::BLOCK_SIZE
, buffer
);
7087 ldpp_dout(this, 2) << "cannot read header" << dendl
;
7091 /* We need to re-interpret the buffer as a TAR block. Exactly two blocks
7092 * must be tracked to detect out end-of-archive. It occurs when both of
7093 * them are empty (zeroed). Tracing this particular inter-block dependency
7094 * is responsibility of the rgw::tar::StatusIndicator class. */
7095 boost::optional
<rgw::tar::HeaderView
> header
;
7096 std::tie(status
, header
) = rgw::tar::interpret_block(status
, buffer
);
7098 if (! status
.empty() && header
) {
7099 /* This specific block isn't empty (entirely zeroed), so we can parse
7100 * it as a TAR header and dispatch. At the moment we do support only
7101 * regular files and directories. Everything else (symlinks, devices)
7102 * will be ignored but won't cease the whole upload. */
7103 switch (header
->get_filetype()) {
7104 case rgw::tar::FileType::NORMAL_FILE
: {
7105 ldpp_dout(this, 2) << "handling regular file" << dendl
;
7107 boost::string_ref filename
= bucket_path
.empty() ? header
->get_filename() : \
7108 file_prefix
+ header
->get_filename().to_string();
7109 auto body
= AlignedStreamGetter(0, header
->get_filesize(),
7110 rgw::tar::BLOCK_SIZE
, *stream
);
7111 op_ret
= handle_file(filename
,
7112 header
->get_filesize(),
7115 /* Only regular files counts. */
7118 failures
.emplace_back(op_ret
, filename
.to_string());
7122 case rgw::tar::FileType::DIRECTORY
: {
7123 ldpp_dout(this, 2) << "handling regular directory" << dendl
;
7125 boost::string_ref dirname
= bucket_path
.empty() ? header
->get_filename() : bucket_path
;
7126 op_ret
= handle_dir(dirname
);
7127 if (op_ret
< 0 && op_ret
!= -ERR_BUCKET_EXISTS
) {
7128 failures
.emplace_back(op_ret
, dirname
.to_string());
7133 /* Not recognized. Skip. */
7139 /* In case of any problems with sub-request authorization Swift simply
7140 * terminates whole upload immediately. */
7141 if (boost::algorithm::contains(std::initializer_list
<int>{ op_ret
},
7143 ldpp_dout(this, 2) << "terminating due to ret=" << op_ret
<< dendl
;
7147 ldpp_dout(this, 2) << "an empty block" << dendl
;
7152 } while (! status
.eof());
7157 RGWBulkUploadOp::AlignedStreamGetter::~AlignedStreamGetter()
7159 const size_t aligned_legnth
= length
+ (-length
% alignment
);
7160 ceph::bufferlist junk
;
7162 DecoratedStreamGetter::get_exactly(aligned_legnth
- position
, junk
);
7165 ssize_t
RGWBulkUploadOp::AlignedStreamGetter::get_at_most(const size_t want
,
7166 ceph::bufferlist
& dst
)
7168 const size_t max_to_read
= std::min(want
, length
- position
);
7169 const auto len
= DecoratedStreamGetter::get_at_most(max_to_read
, dst
);
7176 ssize_t
RGWBulkUploadOp::AlignedStreamGetter::get_exactly(const size_t want
,
7177 ceph::bufferlist
& dst
)
7179 const auto len
= DecoratedStreamGetter::get_exactly(want
, dst
);
7186 int RGWSetAttrs::verify_permission()
7188 // This looks to be part of the RGW-NFS machinery and has no S3 or
7189 // Swift equivalent.
7191 if (!s
->object
.empty()) {
7192 perm
= verify_object_permission_no_policy(this, s
, RGW_PERM_WRITE
);
7194 perm
= verify_bucket_permission_no_policy(this, s
, RGW_PERM_WRITE
);
7202 void RGWSetAttrs::pre_exec()
7204 rgw_bucket_object_pre_exec(s
);
7207 void RGWSetAttrs::execute()
7209 op_ret
= get_params();
7213 rgw_obj
obj(s
->bucket
, s
->object
);
7215 if (!s
->object
.empty()) {
7216 store
->set_atomic(s
->obj_ctx
, obj
);
7217 op_ret
= store
->set_attrs(s
->obj_ctx
, s
->bucket_info
, obj
, attrs
, nullptr);
7219 for (auto& iter
: attrs
) {
7220 s
->bucket_attrs
[iter
.first
] = std::move(iter
.second
);
7222 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, s
->bucket_attrs
,
7223 &s
->bucket_info
.objv_tracker
);
7227 void RGWGetObjLayout::pre_exec()
7229 rgw_bucket_object_pre_exec(s
);
7232 void RGWGetObjLayout::execute()
7234 rgw_obj
obj(s
->bucket
, s
->object
);
7235 RGWRados::Object
target(store
,
7237 *static_cast<RGWObjectCtx
*>(s
->obj_ctx
),
7238 rgw_obj(s
->bucket
, s
->object
));
7239 RGWRados::Object::Read
stat_op(&target
);
7241 op_ret
= stat_op
.prepare();
7246 head_obj
= stat_op
.state
.head_obj
;
7248 op_ret
= target
.get_manifest(&manifest
);
7252 int RGWConfigBucketMetaSearch::verify_permission()
7254 if (!s
->auth
.identity
->is_owner_of(s
->bucket_owner
.get_id())) {
7261 void RGWConfigBucketMetaSearch::pre_exec()
7263 rgw_bucket_object_pre_exec(s
);
7266 void RGWConfigBucketMetaSearch::execute()
7268 op_ret
= get_params();
7270 ldpp_dout(this, 20) << "NOTICE: get_params() returned ret=" << op_ret
<< dendl
;
7274 s
->bucket_info
.mdsearch_config
= mdsearch_config
;
7276 op_ret
= store
->put_bucket_instance_info(s
->bucket_info
, false, real_time(), &s
->bucket_attrs
);
7278 ldpp_dout(this, 0) << "NOTICE: put_bucket_info on bucket=" << s
->bucket
.name
7279 << " returned err=" << op_ret
<< dendl
;
7284 int RGWGetBucketMetaSearch::verify_permission()
7286 if (!s
->auth
.identity
->is_owner_of(s
->bucket_owner
.get_id())) {
7293 void RGWGetBucketMetaSearch::pre_exec()
7295 rgw_bucket_object_pre_exec(s
);
7298 int RGWDelBucketMetaSearch::verify_permission()
7300 if (!s
->auth
.identity
->is_owner_of(s
->bucket_owner
.get_id())) {
7307 void RGWDelBucketMetaSearch::pre_exec()
7309 rgw_bucket_object_pre_exec(s
);
7312 void RGWDelBucketMetaSearch::execute()
7314 s
->bucket_info
.mdsearch_config
.clear();
7316 op_ret
= store
->put_bucket_instance_info(s
->bucket_info
, false, real_time(), &s
->bucket_attrs
);
7318 ldpp_dout(this, 0) << "NOTICE: put_bucket_info on bucket=" << s
->bucket
.name
7319 << " returned err=" << op_ret
<< dendl
;
7325 RGWHandler::~RGWHandler()
7329 int RGWHandler::init(RGWRados
*_store
,
7330 struct req_state
*_s
,
7331 rgw::io::BasicClient
*cio
)
7339 int RGWHandler::do_init_permissions()
7341 int ret
= rgw_build_bucket_policies(store
, s
);
7343 ldpp_dout(s
, 10) << "init_permissions on " << s
->bucket
7344 << " failed, ret=" << ret
<< dendl
;
7345 return ret
==-ENODATA
? -EACCES
: ret
;
7348 rgw_build_iam_environment(store
, s
);
7352 int RGWHandler::do_read_permissions(RGWOp
*op
, bool only_bucket
)
7355 /* already read bucket info */
7358 int ret
= rgw_build_object_policies(store
, s
, op
->prefetch_data());
7361 ldpp_dout(op
, 10) << "read_permissions on " << s
->bucket
<< ":"
7362 << s
->object
<< " only_bucket=" << only_bucket
7363 << " ret=" << ret
<< dendl
;
7364 if (ret
== -ENODATA
)
7371 int RGWOp::error_handler(int err_no
, string
*error_content
) {
7372 return dialect_handler
->error_handler(err_no
, error_content
);
7375 int RGWHandler::error_handler(int err_no
, string
*error_content
) {
7376 // This is the do-nothing error handler
7380 std::ostream
& RGWOp::gen_prefix(std::ostream
& out
) const
7382 // append <dialect>:<op name> to the prefix
7383 return s
->gen_prefix(out
) << s
->dialect
<< ':' << name() << ' ';
7386 void RGWDefaultResponseOp::send_response() {
7388 set_req_state_err(s
, op_ret
);
7394 void RGWPutBucketPolicy::send_response()
7397 set_req_state_err(s
, op_ret
);
7403 int RGWPutBucketPolicy::verify_permission()
7405 if (!verify_bucket_permission(this, s
, rgw::IAM::s3PutBucketPolicy
)) {
7412 int RGWPutBucketPolicy::get_params()
7414 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
7415 // At some point when I have more time I want to make a version of
7416 // rgw_rest_read_all_input that doesn't use malloc.
7417 std::tie(op_ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
7419 // And throws exceptions.
7423 void RGWPutBucketPolicy::execute()
7425 op_ret
= get_params();
7430 if (!store
->svc
.zone
->is_meta_master()) {
7431 op_ret
= forward_request_to_master(s
, NULL
, store
, data
, nullptr);
7433 ldpp_dout(this, 20) << "forward_request_to_master returned ret=" << op_ret
<< dendl
;
7439 const Policy
p(s
->cct
, s
->bucket_tenant
, data
);
7440 op_ret
= retry_raced_bucket_write(store
, s
, [&p
, this] {
7441 auto attrs
= s
->bucket_attrs
;
7442 attrs
[RGW_ATTR_IAM_POLICY
].clear();
7443 attrs
[RGW_ATTR_IAM_POLICY
].append(p
.text
);
7444 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
,
7445 &s
->bucket_info
.objv_tracker
);
7448 } catch (rgw::IAM::PolicyParseException
& e
) {
7449 ldpp_dout(this, 20) << "failed to parse policy: " << e
.what() << dendl
;
7454 void RGWGetBucketPolicy::send_response()
7457 set_req_state_err(s
, op_ret
);
7460 end_header(s
, this, "application/json");
7461 dump_body(s
, policy
);
7464 int RGWGetBucketPolicy::verify_permission()
7466 if (!verify_bucket_permission(this, s
, rgw::IAM::s3GetBucketPolicy
)) {
7473 void RGWGetBucketPolicy::execute()
7475 auto attrs
= s
->bucket_attrs
;
7476 map
<string
, bufferlist
>::iterator aiter
= attrs
.find(RGW_ATTR_IAM_POLICY
);
7477 if (aiter
== attrs
.end()) {
7478 ldpp_dout(this, 0) << "can't find bucket IAM POLICY attr bucket_name = "
7479 << s
->bucket_name
<< dendl
;
7480 op_ret
= -ERR_NO_SUCH_BUCKET_POLICY
;
7481 s
->err
.message
= "The bucket policy does not exist";
7484 policy
= attrs
[RGW_ATTR_IAM_POLICY
];
7486 if (policy
.length() == 0) {
7487 ldpp_dout(this, 10) << "The bucket policy does not exist, bucket: "
7488 << s
->bucket_name
<< dendl
;
7489 op_ret
= -ERR_NO_SUCH_BUCKET_POLICY
;
7490 s
->err
.message
= "The bucket policy does not exist";
7496 void RGWDeleteBucketPolicy::send_response()
7499 set_req_state_err(s
, op_ret
);
7505 int RGWDeleteBucketPolicy::verify_permission()
7507 if (!verify_bucket_permission(this, s
, rgw::IAM::s3DeleteBucketPolicy
)) {
7514 void RGWDeleteBucketPolicy::execute()
7516 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
7517 auto attrs
= s
->bucket_attrs
;
7518 attrs
.erase(RGW_ATTR_IAM_POLICY
);
7519 op_ret
= rgw_bucket_set_attrs(store
, s
->bucket_info
, attrs
,
7520 &s
->bucket_info
.objv_tracker
);
7525 void RGWPutBucketObjectLock::pre_exec()
7527 rgw_bucket_object_pre_exec(s
);
7530 int RGWPutBucketObjectLock::verify_permission()
7532 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3PutBucketObjectLockConfiguration
);
7535 void RGWPutBucketObjectLock::execute()
7537 if (!s
->bucket_info
.obj_lock_enabled()) {
7538 ldpp_dout(this, 0) << "ERROR: object Lock configuration cannot be enabled on existing buckets" << dendl
;
7539 op_ret
= -ERR_INVALID_BUCKET_STATE
;
7543 RGWXMLDecoder::XMLParser parser
;
7544 if (!parser
.init()) {
7545 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
7549 op_ret
= get_params();
7553 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
7554 op_ret
= -ERR_MALFORMED_XML
;
7559 RGWXMLDecoder::decode_xml("ObjectLockConfiguration", obj_lock
, &parser
, true);
7560 } catch (RGWXMLDecoder::err
& err
) {
7561 ldout(s
->cct
, 5) << "unexpected xml:" << err
<< dendl
;
7562 op_ret
= -ERR_MALFORMED_XML
;
7565 if (obj_lock
.has_rule() && !obj_lock
.retention_period_valid()) {
7566 ldpp_dout(this, 0) << "ERROR: retention period must be a positive integer value" << dendl
;
7567 op_ret
= -ERR_INVALID_RETENTION_PERIOD
;
7571 if (!store
->svc
.zone
->is_meta_master()) {
7572 op_ret
= forward_request_to_master(s
, NULL
, store
, data
, nullptr);
7574 ldout(s
->cct
, 20) << __func__
<< "forward_request_to_master returned ret=" << op_ret
<< dendl
;
7579 op_ret
= retry_raced_bucket_write(store
, s
, [this] {
7580 s
->bucket_info
.obj_lock
= obj_lock
;
7581 op_ret
= store
->put_bucket_instance_info(s
->bucket_info
, false,
7582 real_time(), &s
->bucket_attrs
);
7588 void RGWGetBucketObjectLock::pre_exec()
7590 rgw_bucket_object_pre_exec(s
);
7593 int RGWGetBucketObjectLock::verify_permission()
7595 return verify_bucket_owner_or_policy(s
, rgw::IAM::s3GetBucketObjectLockConfiguration
);
7598 void RGWGetBucketObjectLock::execute()
7600 if (!s
->bucket_info
.obj_lock_enabled()) {
7601 op_ret
= -ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION
;
7606 int RGWPutObjRetention::verify_permission()
7608 if (!verify_object_permission(this, s
, rgw::IAM::s3PutObjectRetention
)) {
7611 op_ret
= get_params();
7615 if (bypass_governance_mode
) {
7616 bypass_perm
= verify_object_permission(this, s
, rgw::IAM::s3BypassGovernanceRetention
);
7621 void RGWPutObjRetention::pre_exec()
7623 rgw_bucket_object_pre_exec(s
);
7626 void RGWPutObjRetention::execute()
7628 if (!s
->bucket_info
.obj_lock_enabled()) {
7629 ldpp_dout(this, 0) << "ERROR: object retention can't be set if bucket object lock not configured" << dendl
;
7630 op_ret
= -ERR_INVALID_REQUEST
;
7634 RGWXMLDecoder::XMLParser parser
;
7635 if (!parser
.init()) {
7636 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
7641 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
7642 op_ret
= -ERR_MALFORMED_XML
;
7647 RGWXMLDecoder::decode_xml("Retention", obj_retention
, &parser
, true);
7648 } catch (RGWXMLDecoder::err
& err
) {
7649 ldpp_dout(this, 5) << "unexpected xml:" << err
<< dendl
;
7650 op_ret
= -ERR_MALFORMED_XML
;
7654 if (ceph::real_clock::to_time_t(obj_retention
.get_retain_until_date()) < ceph_clock_now()) {
7655 ldpp_dout(this, 0) << "ERROR: the retain until date must be in the future" << dendl
;
7660 obj_retention
.encode(bl
);
7661 rgw_obj
obj(s
->bucket
, s
->object
);
7663 //check old retention
7664 map
<string
, bufferlist
> attrs
;
7665 op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
7667 ldpp_dout(this, 0) << "ERROR: get obj attr error"<< dendl
;
7670 auto aiter
= attrs
.find(RGW_ATTR_OBJECT_RETENTION
);
7671 if (aiter
!= attrs
.end()) {
7672 RGWObjectRetention old_obj_retention
;
7674 decode(old_obj_retention
, aiter
->second
);
7675 } catch (buffer::error
& err
) {
7676 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl
;
7680 if (ceph::real_clock::to_time_t(obj_retention
.get_retain_until_date()) < ceph::real_clock::to_time_t(old_obj_retention
.get_retain_until_date())) {
7681 if (old_obj_retention
.get_mode().compare("GOVERNANCE") != 0 || !bypass_perm
|| !bypass_governance_mode
) {
7688 op_ret
= modify_obj_attr(store
, s
, obj
, RGW_ATTR_OBJECT_RETENTION
, bl
);
7693 int RGWGetObjRetention::verify_permission()
7695 if (!verify_object_permission(this, s
, rgw::IAM::s3GetObjectRetention
)) {
7701 void RGWGetObjRetention::pre_exec()
7703 rgw_bucket_object_pre_exec(s
);
7706 void RGWGetObjRetention::execute()
7708 if (!s
->bucket_info
.obj_lock_enabled()) {
7709 ldpp_dout(this, 0) << "ERROR: bucket object lock not configured" << dendl
;
7710 op_ret
= -ERR_INVALID_REQUEST
;
7713 rgw_obj
obj(s
->bucket
, s
->object
);
7714 map
<string
, bufferlist
> attrs
;
7715 op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
7717 ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << obj
7718 << " ret=" << op_ret
<< dendl
;
7721 auto aiter
= attrs
.find(RGW_ATTR_OBJECT_RETENTION
);
7722 if (aiter
== attrs
.end()) {
7723 op_ret
= -ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION
;
7727 bufferlist::const_iterator iter
{&aiter
->second
};
7729 obj_retention
.decode(iter
);
7730 } catch (const buffer::error
& e
) {
7731 ldout(s
->cct
, 0) << __func__
<< "decode object retention config failed" << dendl
;
7738 int RGWPutObjLegalHold::verify_permission()
7740 if (!verify_object_permission(this, s
, rgw::IAM::s3PutObjectLegalHold
)) {
7746 void RGWPutObjLegalHold::pre_exec()
7748 rgw_bucket_object_pre_exec(s
);
7751 void RGWPutObjLegalHold::execute() {
7752 if (!s
->bucket_info
.obj_lock_enabled()) {
7753 ldpp_dout(this, 0) << "ERROR: object legal hold can't be set if bucket object lock not configured" << dendl
;
7754 op_ret
= -ERR_INVALID_REQUEST
;
7758 RGWXMLDecoder::XMLParser parser
;
7759 if (!parser
.init()) {
7760 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
7765 op_ret
= get_params();
7769 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
7770 op_ret
= -ERR_MALFORMED_XML
;
7775 RGWXMLDecoder::decode_xml("LegalHold", obj_legal_hold
, &parser
, true);
7776 } catch (RGWXMLDecoder::err
&err
) {
7777 ldout(s
->cct
, 5) << "unexpected xml:" << err
<< dendl
;
7778 op_ret
= -ERR_MALFORMED_XML
;
7782 obj_legal_hold
.encode(bl
);
7783 rgw_obj
obj(s
->bucket
, s
->object
);
7784 //if instance is empty, we should modify the latest object
7785 op_ret
= modify_obj_attr(store
, s
, obj
, RGW_ATTR_OBJECT_LEGAL_HOLD
, bl
);
7789 int RGWGetObjLegalHold::verify_permission()
7791 if (!verify_object_permission(this, s
, rgw::IAM::s3GetObjectLegalHold
)) {
7797 void RGWGetObjLegalHold::pre_exec()
7799 rgw_bucket_object_pre_exec(s
);
7802 void RGWGetObjLegalHold::execute()
7804 if (!s
->bucket_info
.obj_lock_enabled()) {
7805 ldpp_dout(this, 0) << "ERROR: bucket object lock not configured" << dendl
;
7806 op_ret
= -ERR_INVALID_REQUEST
;
7809 rgw_obj
obj(s
->bucket
, s
->object
);
7810 map
<string
, bufferlist
> attrs
;
7811 op_ret
= get_obj_attrs(store
, s
, obj
, attrs
);
7813 ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << obj
7814 << " ret=" << op_ret
<< dendl
;
7817 auto aiter
= attrs
.find(RGW_ATTR_OBJECT_LEGAL_HOLD
);
7818 if (aiter
== attrs
.end()) {
7819 op_ret
= -ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION
;
7823 bufferlist::const_iterator iter
{&aiter
->second
};
7825 obj_legal_hold
.decode(iter
);
7826 } catch (const buffer::error
& e
) {
7827 ldout(s
->cct
, 0) << __func__
<< "decode object legal hold config failed" << dendl
;
7834 void RGWGetClusterStat::execute()
7836 op_ret
= this->store
->get_rados_handle()->cluster_stat(stats_op
);