1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
8 #include "common/ceph_crypto.h"
9 #include "common/Formatter.h"
10 #include "common/utf8.h"
11 #include "common/ceph_json.h"
12 #include "common/safe_io.h"
13 #include "common/errno.h"
14 #include "auth/Crypto.h"
15 #include <boost/algorithm/string.hpp>
16 #include <boost/algorithm/string/replace.hpp>
17 #include <boost/utility/string_view.hpp>
18 #include <boost/tokenizer.hpp>
20 #include <liboath/oath.h>
23 #include "rgw_rest_s3.h"
24 #include "rgw_rest_s3website.h"
25 #include "rgw_rest_pubsub.h"
26 #include "rgw_auth_s3.h"
28 #include "rgw_policy_s3.h"
31 #include "rgw_cors_s3.h"
32 #include "rgw_tag_s3.h"
34 #include "rgw_client_io.h"
36 #include "rgw_keystone.h"
37 #include "rgw_auth_keystone.h"
38 #include "rgw_auth_registry.h"
40 #include "rgw_es_query.h"
42 #include <typeinfo> // for 'typeid'
45 #include "rgw_token.h"
46 #include "rgw_rest_role.h"
47 #include "rgw_crypt.h"
48 #include "rgw_crypt_sanitize.h"
49 #include "rgw_rest_user_policy.h"
51 #include "rgw_bucket_sync.h"
53 #include "services/svc_zone.h"
54 #include "services/svc_cls.h"
56 #include "include/ceph_assert.h"
58 #include "rgw_rest_sts.h"
59 #include "rgw_rest_iam.h"
62 #define dout_context g_ceph_context
63 #define dout_subsys ceph_subsys_rgw
66 using namespace ceph::crypto
;
70 void list_all_buckets_start(struct req_state
*s
)
72 s
->formatter
->open_array_section_in_ns("ListAllMyBucketsResult", XMLNS_AWS_S3
);
75 void list_all_buckets_end(struct req_state
*s
)
77 s
->formatter
->close_section();
80 void dump_bucket(struct req_state
*s
, rgw::sal::RGWBucket
& obj
)
82 s
->formatter
->open_object_section("Bucket");
83 s
->formatter
->dump_string("Name", obj
.get_name());
84 dump_time(s
, "CreationDate", &obj
.get_creation_time());
85 s
->formatter
->close_section();
88 void rgw_get_errno_s3(rgw_http_error
*e
, int err_no
)
90 rgw_http_errors::const_iterator r
= rgw_http_s3_errors
.find(err_no
);
92 if (r
!= rgw_http_s3_errors
.end()) {
93 e
->http_ret
= r
->second
.first
;
94 e
->s3_code
= r
->second
.second
;
97 e
->s3_code
= "UnknownError";
101 static inline std::string
get_s3_expiration_header(
103 const ceph::real_time
& mtime
)
105 return rgw::lc::s3_expiration_header(
106 s
, s
->object
, s
->tagset
, mtime
, s
->bucket_attrs
);
109 struct response_attr_param
{
111 const char *http_attr
;
114 static struct response_attr_param resp_attr_params
[] = {
115 {"response-content-type", "Content-Type"},
116 {"response-content-language", "Content-Language"},
117 {"response-expires", "Expires"},
118 {"response-cache-control", "Cache-Control"},
119 {"response-content-disposition", "Content-Disposition"},
120 {"response-content-encoding", "Content-Encoding"},
124 int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
) {
125 map
<string
, bufferlist
>::iterator iter
;
126 iter
= attrs
.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION
);
127 if (iter
!= attrs
.end()) {
128 bufferlist
&bl
= iter
->second
;
129 s
->redirect
= bl
.c_str();
130 s
->err
.http_ret
= 301;
131 ldpp_dout(this, 20) << __CEPH_ASSERT_FUNCTION
<< " redirecting per x-amz-website-redirect-location=" << s
->redirect
<< dendl
;
132 op_ret
= -ERR_WEBSITE_REDIRECT
;
133 set_req_state_err(s
, op_ret
);
135 dump_content_length(s
, 0);
136 dump_redirect(s
, s
->redirect
);
140 return RGWGetObj_ObjStore_S3::send_response_data(bl
, bl_ofs
, bl_len
);
144 int RGWGetObj_ObjStore_S3Website::send_response_data_error()
146 return RGWGetObj_ObjStore_S3::send_response_data_error();
149 int RGWGetObj_ObjStore_S3::get_params()
151 // for multisite sync requests, only read the slo manifest itself, rather than
152 // all of the data from its parts. the parts will sync as separate objects
153 skip_manifest
= s
->info
.args
.exists(RGW_SYS_PARAM_PREFIX
"sync-manifest");
155 // multisite sync requests should fetch encrypted data, along with the
156 // attributes needed to support decryption on the other zone
157 if (s
->system_request
) {
158 skip_decrypt
= s
->info
.args
.exists(RGW_SYS_PARAM_PREFIX
"skip-decrypt");
161 return RGWGetObj_ObjStore::get_params();
164 int RGWGetObj_ObjStore_S3::send_response_data_error()
167 return send_response_data(bl
, 0 , 0);
171 int decode_attr_bl_single_value(map
<string
, bufferlist
>& attrs
, const char *attr_name
, T
*result
, T def_val
)
173 map
<string
, bufferlist
>::iterator iter
= attrs
.find(attr_name
);
174 if (iter
== attrs
.end()) {
178 bufferlist
& bl
= iter
->second
;
179 if (bl
.length() == 0) {
183 auto bliter
= bl
.cbegin();
185 decode(*result
, bliter
);
186 } catch (buffer::error
& err
) {
192 inline bool str_has_cntrl(const std::string s
) {
193 return std::any_of(s
.begin(), s
.end(), ::iscntrl
);
196 inline bool str_has_cntrl(const char* s
) {
198 return str_has_cntrl(_s
);
201 int RGWGetObj_ObjStore_S3::send_response_data(bufferlist
& bl
, off_t bl_ofs
,
204 const char *content_type
= NULL
;
205 string content_type_str
;
206 map
<string
, string
> response_attrs
;
207 map
<string
, string
>::iterator riter
;
208 bufferlist metadata_bl
;
210 string expires
= get_s3_expiration_header(s
, lastmod
);
215 if (custom_http_ret
) {
216 set_req_state_err(s
, 0);
217 dump_errno(s
, custom_http_ret
);
219 set_req_state_err(s
, (partial_content
&& !op_ret
) ? STATUS_PARTIAL_CONTENT
228 dump_range(s
, start
, end
, s
->obj_size
);
230 if (s
->system_request
&&
231 s
->info
.args
.exists(RGW_SYS_PARAM_PREFIX
"prepend-metadata")) {
233 dump_header(s
, "Rgwx-Object-Size", (long long)total_len
);
237 * in this case, we're not returning the object's content, only the prepended
243 /* JSON encode object metadata */
245 jf
.open_object_section("obj_metadata");
246 encode_json("attrs", attrs
, &jf
);
248 encode_json("mtime", ut
, &jf
);
252 metadata_bl
.append(ss
.str());
253 dump_header(s
, "Rgwx-Embedded-Metadata-Len", metadata_bl
.length());
254 total_len
+= metadata_bl
.length();
257 if (s
->system_request
&& !real_clock::is_zero(lastmod
)) {
258 /* we end up dumping mtime in two different methods, a bit redundant */
259 dump_epoch_header(s
, "Rgwx-Mtime", lastmod
);
261 int r
= decode_attr_bl_single_value(attrs
, RGW_ATTR_PG_VER
, &pg_ver
, (uint64_t)0);
263 ldpp_dout(this, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl
;
265 dump_header(s
, "Rgwx-Obj-PG-Ver", pg_ver
);
267 uint32_t source_zone_short_id
= 0;
268 r
= decode_attr_bl_single_value(attrs
, RGW_ATTR_SOURCE_ZONE
, &source_zone_short_id
, (uint32_t)0);
270 ldpp_dout(this, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl
;
272 if (source_zone_short_id
!= 0) {
273 dump_header(s
, "Rgwx-Source-Zone-Short-Id", source_zone_short_id
);
277 for (auto &it
: crypt_http_responses
)
278 dump_header(s
, it
.first
, it
.second
);
280 dump_content_length(s
, total_len
);
281 dump_last_modified(s
, lastmod
);
282 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
283 dump_header_if_nonempty(s
, "x-amz-expiration", expires
);
285 if (attrs
.find(RGW_ATTR_APPEND_PART_NUM
) != attrs
.end()) {
286 dump_header(s
, "x-rgw-object-type", "Appendable");
287 dump_header(s
, "x-rgw-next-append-position", s
->obj_size
);
289 dump_header(s
, "x-rgw-object-type", "Normal");
293 if (! lo_etag
.empty()) {
294 /* Handle etag of Swift API's large objects (DLO/SLO). It's entirerly
295 * legit to perform GET on them through S3 API. In such situation,
296 * a client should receive the composited content with corresponding
298 dump_etag(s
, lo_etag
);
300 auto iter
= attrs
.find(RGW_ATTR_ETAG
);
301 if (iter
!= attrs
.end()) {
302 dump_etag(s
, iter
->second
.to_str());
306 for (struct response_attr_param
*p
= resp_attr_params
; p
->param
; p
++) {
308 string val
= s
->info
.args
.get(p
->param
, &exists
);
310 /* reject unauthenticated response header manipulation, see
311 * https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html */
312 if (s
->auth
.identity
->is_anonymous()) {
313 return -ERR_INVALID_REQUEST
;
315 /* HTTP specification says no control characters should be present in
316 * header values: https://tools.ietf.org/html/rfc7230#section-3.2
317 * field-vchar = VCHAR / obs-text
319 * Failure to validate this permits a CRLF injection in HTTP headers,
320 * whereas S3 GetObject only permits specific headers.
322 if(str_has_cntrl(val
)) {
323 /* TODO: return a more distinct error in future;
324 * stating what the problem is */
325 return -ERR_INVALID_REQUEST
;
328 if (strcmp(p
->param
, "response-content-type") != 0) {
329 response_attrs
[p
->http_attr
] = val
;
331 content_type_str
= val
;
332 content_type
= content_type_str
.c_str();
337 for (auto iter
= attrs
.begin(); iter
!= attrs
.end(); ++iter
) {
338 const char *name
= iter
->first
.c_str();
339 map
<string
, string
>::iterator aiter
= rgw_to_http_attrs
.find(name
);
340 if (aiter
!= rgw_to_http_attrs
.end()) {
341 if (response_attrs
.count(aiter
->second
) == 0) {
342 /* Was not already overridden by a response param. */
344 size_t len
= iter
->second
.length();
345 string
s(iter
->second
.c_str(), len
);
346 while (len
&& !s
[len
- 1]) {
350 response_attrs
[aiter
->second
] = s
;
352 } else if (iter
->first
.compare(RGW_ATTR_CONTENT_TYPE
) == 0) {
353 /* Special handling for content_type. */
355 content_type_str
= rgw_bl_str(iter
->second
);
356 content_type
= content_type_str
.c_str();
358 } else if (strcmp(name
, RGW_ATTR_SLO_UINDICATOR
) == 0) {
359 // this attr has an extra length prefix from encode() in prior versions
360 dump_header(s
, "X-Object-Meta-Static-Large-Object", "True");
361 } else if (strncmp(name
, RGW_ATTR_META_PREFIX
,
362 sizeof(RGW_ATTR_META_PREFIX
)-1) == 0) {
363 /* User custom metadata. */
364 name
+= sizeof(RGW_ATTR_PREFIX
) - 1;
365 dump_header(s
, name
, iter
->second
);
366 } else if (iter
->first
.compare(RGW_ATTR_TAGS
) == 0) {
369 auto it
= iter
->second
.cbegin();
371 } catch (buffer::error
&err
) {
372 ldpp_dout(this,0) << "Error caught buffer::error couldn't decode TagSet " << dendl
;
374 dump_header(s
, RGW_AMZ_TAG_COUNT
, obj_tags
.count());
375 } else if (iter
->first
.compare(RGW_ATTR_OBJECT_RETENTION
) == 0 && get_retention
){
376 RGWObjectRetention retention
;
378 decode(retention
, iter
->second
);
379 dump_header(s
, "x-amz-object-lock-mode", retention
.get_mode());
380 dump_time_header(s
, "x-amz-object-lock-retain-until-date", retention
.get_retain_until_date());
381 } catch (buffer::error
& err
) {
382 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl
;
384 } else if (iter
->first
.compare(RGW_ATTR_OBJECT_LEGAL_HOLD
) == 0 && get_legal_hold
) {
385 RGWObjectLegalHold legal_hold
;
387 decode(legal_hold
, iter
->second
);
388 dump_header(s
, "x-amz-object-lock-legal-hold",legal_hold
.get_status());
389 } catch (buffer::error
& err
) {
390 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectLegalHold" << dendl
;
397 for (riter
= response_attrs
.begin(); riter
!= response_attrs
.end();
399 dump_header(s
, riter
->first
, riter
->second
);
402 if (op_ret
== -ERR_NOT_MODIFIED
) {
406 content_type
= "binary/octet-stream";
408 end_header(s
, this, content_type
);
411 if (metadata_bl
.length()) {
412 dump_body(s
, metadata_bl
);
417 if (get_data
&& !op_ret
) {
418 int r
= dump_body(s
, bl
.c_str() + bl_ofs
, bl_len
);
426 int RGWGetObj_ObjStore_S3::get_decrypt_filter(std::unique_ptr
<RGWGetObj_Filter
> *filter
, RGWGetObj_Filter
* cb
, bufferlist
* manifest_bl
)
428 if (skip_decrypt
) { // bypass decryption for multisite sync requests
433 std::unique_ptr
<BlockCrypt
> block_crypt
;
434 res
= rgw_s3_prepare_decrypt(s
, attrs
, &block_crypt
, crypt_http_responses
);
436 if (block_crypt
!= nullptr) {
437 auto f
= std::make_unique
<RGWGetObj_BlockDecrypt
>(s
->cct
, cb
, std::move(block_crypt
));
438 if (manifest_bl
!= nullptr) {
439 res
= f
->read_manifest(*manifest_bl
);
441 *filter
= std::move(f
);
449 void RGWGetObjTags_ObjStore_S3::send_response_data(bufferlist
& bl
)
452 end_header(s
, this, "application/xml");
455 s
->formatter
->open_object_section_in_ns("Tagging", XMLNS_AWS_S3
);
456 s
->formatter
->open_object_section("TagSet");
458 RGWObjTagSet_S3 tagset
;
459 auto iter
= bl
.cbegin();
462 } catch (buffer::error
& err
) {
463 ldpp_dout(this,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl
;
467 tagset
.dump_xml(s
->formatter
);
469 s
->formatter
->close_section();
470 s
->formatter
->close_section();
471 rgw_flush_formatter_and_reset(s
, s
->formatter
);
475 int RGWPutObjTags_ObjStore_S3::get_params()
483 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
487 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
492 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
493 return -ERR_MALFORMED_XML
;
496 RGWObjTagging_S3 tagging
;
499 RGWXMLDecoder::decode_xml("Tagging", tagging
, &parser
);
500 } catch (RGWXMLDecoder::err
& err
) {
501 ldpp_dout(this, 5) << "Malformed tagging request: " << err
<< dendl
;
502 return -ERR_MALFORMED_XML
;
506 r
= tagging
.rebuild(obj_tags
);
510 obj_tags
.encode(tags_bl
);
511 ldpp_dout(this, 20) << "Read " << obj_tags
.count() << "tags" << dendl
;
516 void RGWPutObjTags_ObjStore_S3::send_response()
519 set_req_state_err(s
, op_ret
);
521 end_header(s
, this, "application/xml");
526 void RGWDeleteObjTags_ObjStore_S3::send_response()
532 r
= STATUS_NO_CONTENT
;
534 set_req_state_err(s
, r
);
539 void RGWGetBucketTags_ObjStore_S3::send_response_data(bufferlist
& bl
)
542 set_req_state_err(s
, op_ret
);
544 end_header(s
, this, "application/xml");
548 s
->formatter
->open_object_section_in_ns("Tagging", XMLNS_AWS_S3
);
549 s
->formatter
->open_object_section("TagSet");
551 RGWObjTagSet_S3 tagset
;
552 auto iter
= bl
.cbegin();
555 } catch (buffer::error
& err
) {
556 ldout(s
->cct
,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl
;
560 tagset
.dump_xml(s
->formatter
);
562 s
->formatter
->close_section();
563 s
->formatter
->close_section();
564 rgw_flush_formatter_and_reset(s
, s
->formatter
);
568 int RGWPutBucketTags_ObjStore_S3::get_params()
576 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
580 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
585 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
586 return -ERR_MALFORMED_XML
;
589 RGWObjTagging_S3 tagging
;
591 RGWXMLDecoder::decode_xml("Tagging", tagging
, &parser
);
592 } catch (RGWXMLDecoder::err
& err
) {
594 ldout(s
->cct
, 5) << "Malformed tagging request: " << err
<< dendl
;
595 return -ERR_MALFORMED_XML
;
598 RGWObjTags
obj_tags(50); // A tag set can contain as many as 50 tags, or it can be empty.
599 r
= tagging
.rebuild(obj_tags
);
603 obj_tags
.encode(tags_bl
);
604 ldout(s
->cct
, 20) << "Read " << obj_tags
.count() << "tags" << dendl
;
606 // forward bucket tags requests to meta master zone
607 if (!store
->svc()->zone
->is_meta_master()) {
608 /* only need to keep this data around if we're not meta master */
609 in_data
= std::move(data
);
615 void RGWPutBucketTags_ObjStore_S3::send_response()
618 set_req_state_err(s
, op_ret
);
620 end_header(s
, this, "application/xml");
624 void RGWDeleteBucketTags_ObjStore_S3::send_response()
627 set_req_state_err(s
, op_ret
);
629 end_header(s
, this, "application/xml");
635 bool is_valid_status(const string
& s
) {
636 return (s
== "Enabled" ||
640 static string enabled_group_id
= "s3-bucket-replication:enabled";
641 static string disabled_group_id
= "s3-bucket-replication:disabled";
643 struct ReplicationConfiguration
{
647 struct DeleteMarkerReplication
{
650 void decode_xml(XMLObj
*obj
) {
651 RGWXMLDecoder::decode_xml("Status", status
, obj
);
654 void dump_xml(Formatter
*f
) const {
655 encode_xml("Status", status
, f
);
658 bool is_valid(CephContext
*cct
) const {
659 bool result
= is_valid_status(status
);
661 ldout(cct
, 5) << "NOTICE: bad status provided in DeleteMarkerReplication element (status=" << status
<< ")" << dendl
;
667 struct Source
{ /* rgw extension */
668 std::vector
<string
> zone_names
;
670 void decode_xml(XMLObj
*obj
) {
671 RGWXMLDecoder::decode_xml("Zone", zone_names
, obj
);
674 void dump_xml(Formatter
*f
) const {
675 encode_xml("Zone", zone_names
, f
);
680 struct AccessControlTranslation
{
683 void decode_xml(XMLObj
*obj
) {
684 RGWXMLDecoder::decode_xml("Owner", owner
, obj
);
686 void dump_xml(Formatter
*f
) const {
687 encode_xml("Owner", owner
, f
);
691 std::optional
<AccessControlTranslation
> acl_translation
;
692 std::optional
<string
> account
;
694 std::optional
<string
> storage_class
;
695 std::vector
<string
> zone_names
;
697 void decode_xml(XMLObj
*obj
) {
698 RGWXMLDecoder::decode_xml("AccessControlTranslation", acl_translation
, obj
);
699 RGWXMLDecoder::decode_xml("Account", account
, obj
);
700 if (account
&& account
->empty()) {
703 RGWXMLDecoder::decode_xml("Bucket", bucket
, obj
);
704 RGWXMLDecoder::decode_xml("StorageClass", storage_class
, obj
);
705 if (storage_class
&& storage_class
->empty()) {
706 storage_class
.reset();
708 RGWXMLDecoder::decode_xml("Zone", zone_names
, obj
); /* rgw extension */
711 void dump_xml(Formatter
*f
) const {
712 encode_xml("AccessControlTranslation", acl_translation
, f
);
713 encode_xml("Account", account
, f
);
714 encode_xml("Bucket", bucket
, f
);
715 encode_xml("StorageClass", storage_class
, f
);
716 encode_xml("Zone", zone_names
, f
);
726 return key
.empty() && value
.empty();
729 void decode_xml(XMLObj
*obj
) {
730 RGWXMLDecoder::decode_xml("Key", key
, obj
);
731 RGWXMLDecoder::decode_xml("Value", value
, obj
);
734 void dump_xml(Formatter
*f
) const {
735 encode_xml("Key", key
, f
);
736 encode_xml("Value", value
, f
);
741 std::optional
<string
> prefix
;
742 std::vector
<Tag
> tags
;
749 void decode_xml(XMLObj
*obj
) {
750 std::vector
<Tag
> _tags
;
751 RGWXMLDecoder::decode_xml("Prefix", prefix
, obj
);
752 if (prefix
&& prefix
->empty()) {
755 RGWXMLDecoder::decode_xml("Tag", _tags
, obj
);
756 for (auto& t
: _tags
) {
758 tags
.push_back(std::move(t
));
763 void dump_xml(Formatter
*f
) const {
764 encode_xml("Prefix", prefix
, f
);
765 encode_xml("Tag", tags
, f
);
769 std::optional
<string
> prefix
;
770 std::optional
<Tag
> tag
;
771 std::optional
<AndElements
> and_elements
;
774 return (!prefix
&& !tag
&& !and_elements
);
777 void decode_xml(XMLObj
*obj
) {
778 RGWXMLDecoder::decode_xml("Prefix", prefix
, obj
);
779 if (prefix
&& prefix
->empty()) {
782 RGWXMLDecoder::decode_xml("Tag", tag
, obj
);
783 if (tag
&& tag
->empty()) {
786 RGWXMLDecoder::decode_xml("And", and_elements
, obj
);
787 if (and_elements
&& and_elements
->empty()) {
788 and_elements
.reset();
792 void dump_xml(Formatter
*f
) const {
793 encode_xml("Prefix", prefix
, f
);
794 encode_xml("Tag", tag
, f
);
795 encode_xml("And", and_elements
, f
);
798 bool is_valid(CephContext
*cct
) const {
800 ldout(cct
, 5) << "NOTICE: both tag and prefix were provided in replication filter rule" << dendl
;
805 if (prefix
&& and_elements
->prefix
) {
806 ldout(cct
, 5) << "NOTICE: too many prefixes were provided in re" << dendl
;
813 int to_sync_pipe_filter(CephContext
*cct
,
814 rgw_sync_pipe_filter
*f
) const {
815 if (!is_valid(cct
)) {
822 f
->tags
.insert(rgw_sync_pipe_filter_tag(tag
->key
, tag
->value
));
826 if (and_elements
->prefix
) {
827 f
->prefix
= *and_elements
->prefix
;
829 for (auto& t
: and_elements
->tags
) {
830 f
->tags
.insert(rgw_sync_pipe_filter_tag(t
.key
, t
.value
));
836 void from_sync_pipe_filter(const rgw_sync_pipe_filter
& f
) {
837 if (f
.prefix
&& f
.tags
.empty()) {
842 and_elements
.emplace();
843 and_elements
->prefix
= f
.prefix
;
844 } else if (f
.tags
.size() == 1) {
845 auto iter
= f
.tags
.begin();
846 if (iter
== f
.tags
.end()) {
847 /* should never happen */
853 tag
->value
= t
.value
;
857 if (f
.tags
.empty()) {
862 and_elements
.emplace();
865 for (auto& t
: f
.tags
) {
866 auto& tag
= and_elements
->tags
.emplace_back();
873 set
<rgw_zone_id
> get_zone_ids_from_names(rgw::sal::RGWRadosStore
*store
,
874 const vector
<string
>& zone_names
) const {
875 set
<rgw_zone_id
> ids
;
877 for (auto& name
: zone_names
) {
879 if (store
->svc()->zone
->find_zone_id_by_name(name
, &id
)) {
880 ids
.insert(std::move(id
));
887 vector
<string
> get_zone_names_from_ids(rgw::sal::RGWRadosStore
*store
,
888 const set
<rgw_zone_id
>& zone_ids
) const {
889 vector
<string
> names
;
891 for (auto& id
: zone_ids
) {
893 if (store
->svc()->zone
->find_zone(id
, &zone
)) {
894 names
.emplace_back(zone
->name
);
901 std::optional
<DeleteMarkerReplication
> delete_marker_replication
;
902 std::optional
<Source
> source
;
903 Destination destination
;
904 std::optional
<Filter
> filter
;
909 void decode_xml(XMLObj
*obj
) {
910 RGWXMLDecoder::decode_xml("DeleteMarkerReplication", delete_marker_replication
, obj
);
911 RGWXMLDecoder::decode_xml("Source", source
, obj
);
912 RGWXMLDecoder::decode_xml("Destination", destination
, obj
);
913 RGWXMLDecoder::decode_xml("ID", id
, obj
);
915 std::optional
<string
> prefix
;
916 RGWXMLDecoder::decode_xml("Prefix", prefix
, obj
);
919 filter
->prefix
= prefix
;
923 RGWXMLDecoder::decode_xml("Filter", filter
, obj
);
925 /* don't want to have filter reset because it might have been initialized
926 * when decoding prefix
928 RGWXMLDecoder::decode_xml("Filter", *filter
, obj
);
931 RGWXMLDecoder::decode_xml("Priority", priority
, obj
);
932 RGWXMLDecoder::decode_xml("Status", status
, obj
);
935 void dump_xml(Formatter
*f
) const {
936 encode_xml("DeleteMarkerReplication", delete_marker_replication
, f
);
937 encode_xml("Source", source
, f
);
938 encode_xml("Destination", destination
, f
);
939 encode_xml("Filter", filter
, f
);
940 encode_xml("ID", id
, f
);
941 encode_xml("Priority", priority
, f
);
942 encode_xml("Status", status
, f
);
945 bool is_valid(CephContext
*cct
) const {
946 if (!is_valid_status(status
)) {
947 ldout(cct
, 5) << "NOTICE: bad status provided in rule (status=" << status
<< ")" << dendl
;
950 if ((filter
&& !filter
->is_valid(cct
)) ||
951 (delete_marker_replication
&& !delete_marker_replication
->is_valid(cct
))) {
957 int to_sync_policy_pipe(req_state
*s
, rgw::sal::RGWRadosStore
*store
,
958 rgw_sync_bucket_pipes
*pipe
,
959 bool *enabled
) const {
960 if (!is_valid(s
->cct
)) {
965 pipe
->params
.priority
= priority
;
967 const auto& user_id
= s
->user
->get_id();
969 rgw_bucket_key
dest_bk(user_id
.tenant
,
972 if (source
&& !source
->zone_names
.empty()) {
973 pipe
->source
.zones
= get_zone_ids_from_names(store
, source
->zone_names
);
975 pipe
->source
.set_all_zones(true);
977 if (!destination
.zone_names
.empty()) {
978 pipe
->dest
.zones
= get_zone_ids_from_names(store
, destination
.zone_names
);
980 pipe
->dest
.set_all_zones(true);
982 pipe
->dest
.bucket
.emplace(dest_bk
);
985 int r
= filter
->to_sync_pipe_filter(s
->cct
, &pipe
->params
.source
.filter
);
990 if (destination
.acl_translation
) {
992 u
.tenant
= user_id
.tenant
;
993 u
.from_str(destination
.acl_translation
->owner
); /* explicit tenant will override tenant,
994 otherwise will inherit it from s->user */
995 pipe
->params
.dest
.acl_translation
.emplace();
996 pipe
->params
.dest
.acl_translation
->owner
= u
;
998 pipe
->params
.dest
.storage_class
= destination
.storage_class
;
1000 *enabled
= (status
== "Enabled");
1002 pipe
->params
.mode
= rgw_sync_pipe_params::Mode::MODE_USER
;
1003 pipe
->params
.user
= user_id
.to_str();
1008 void from_sync_policy_pipe(rgw::sal::RGWRadosStore
*store
,
1009 const rgw_sync_bucket_pipes
& pipe
,
1012 status
= (enabled
? "Enabled" : "Disabled");
1013 priority
= pipe
.params
.priority
;
1015 if (pipe
.source
.all_zones
) {
1017 } else if (pipe
.source
.zones
) {
1019 source
->zone_names
= get_zone_names_from_ids(store
, *pipe
.source
.zones
);
1022 if (!pipe
.dest
.all_zones
&&
1024 destination
.zone_names
= get_zone_names_from_ids(store
, *pipe
.dest
.zones
);
1027 if (pipe
.params
.dest
.acl_translation
) {
1028 destination
.acl_translation
.emplace();
1029 destination
.acl_translation
->owner
= pipe
.params
.dest
.acl_translation
->owner
.to_str();
1032 if (pipe
.params
.dest
.storage_class
) {
1033 destination
.storage_class
= *pipe
.params
.dest
.storage_class
;
1036 if (pipe
.dest
.bucket
) {
1037 destination
.bucket
= pipe
.dest
.bucket
->get_key();
1041 filter
->from_sync_pipe_filter(pipe
.params
.source
.filter
);
1043 if (filter
->empty()) {
1049 std::vector
<Rule
> rules
;
1051 void decode_xml(XMLObj
*obj
) {
1052 RGWXMLDecoder::decode_xml("Role", role
, obj
);
1053 RGWXMLDecoder::decode_xml("Rule", rules
, obj
);
1056 void dump_xml(Formatter
*f
) const {
1057 encode_xml("Role", role
, f
);
1058 encode_xml("Rule", rules
, f
);
1061 int to_sync_policy_groups(req_state
*s
, rgw::sal::RGWRadosStore
*store
,
1062 vector
<rgw_sync_policy_group
> *result
) const {
1065 rgw_sync_policy_group
& enabled_group
= (*result
)[0];
1066 rgw_sync_policy_group
& disabled_group
= (*result
)[1];
1068 enabled_group
.id
= enabled_group_id
;
1069 enabled_group
.status
= rgw_sync_policy_group::Status::ENABLED
;
1070 disabled_group
.id
= disabled_group_id
;
1071 disabled_group
.status
= rgw_sync_policy_group::Status::ALLOWED
; /* not enabled, not forbidden */
1073 for (auto& rule
: rules
) {
1074 rgw_sync_bucket_pipes pipe
;
1076 int r
= rule
.to_sync_policy_pipe(s
, store
, &pipe
, &enabled
);
1078 ldout(s
->cct
, 5) << "NOTICE: failed to convert replication configuration into sync policy pipe (rule.id=" << rule
.id
<< "): " << cpp_strerror(-r
) << dendl
;
1083 enabled_group
.pipes
.emplace_back(std::move(pipe
));
1085 disabled_group
.pipes
.emplace_back(std::move(pipe
));
1091 void from_sync_policy_group(rgw::sal::RGWRadosStore
*store
,
1092 const rgw_sync_policy_group
& group
) {
1094 bool enabled
= (group
.status
== rgw_sync_policy_group::Status::ENABLED
);
1096 for (auto& pipe
: group
.pipes
) {
1097 auto& rule
= rules
.emplace_back();
1098 rule
.from_sync_policy_pipe(store
, pipe
, enabled
);
1105 void RGWGetBucketReplication_ObjStore_S3::send_response_data()
1108 set_req_state_err(s
, op_ret
);
1110 end_header(s
, this, "application/xml");
1113 ReplicationConfiguration conf
;
1115 if (s
->bucket_info
.sync_policy
) {
1116 auto policy
= s
->bucket_info
.sync_policy
;
1118 auto iter
= policy
->groups
.find(enabled_group_id
);
1119 if (iter
!= policy
->groups
.end()) {
1120 conf
.from_sync_policy_group(store
, iter
->second
);
1122 iter
= policy
->groups
.find(disabled_group_id
);
1123 if (iter
!= policy
->groups
.end()) {
1124 conf
.from_sync_policy_group(store
, iter
->second
);
1129 s
->formatter
->open_object_section_in_ns("ReplicationConfiguration", XMLNS_AWS_S3
);
1130 conf
.dump_xml(s
->formatter
);
1131 s
->formatter
->close_section();
1132 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1136 int RGWPutBucketReplication_ObjStore_S3::get_params()
1138 RGWXMLParser parser
;
1140 if (!parser
.init()){
1144 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
1148 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
1153 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
1154 return -ERR_MALFORMED_XML
;
1157 ReplicationConfiguration conf
;
1159 RGWXMLDecoder::decode_xml("ReplicationConfiguration", conf
, &parser
);
1160 } catch (RGWXMLDecoder::err
& err
) {
1162 ldout(s
->cct
, 5) << "Malformed tagging request: " << err
<< dendl
;
1163 return -ERR_MALFORMED_XML
;
1166 r
= conf
.to_sync_policy_groups(s
, store
, &sync_policy_groups
);
1171 // forward requests to meta master zone
1172 if (!store
->svc()->zone
->is_meta_master()) {
1173 /* only need to keep this data around if we're not meta master */
1174 in_data
= std::move(data
);
1180 void RGWPutBucketReplication_ObjStore_S3::send_response()
1183 set_req_state_err(s
, op_ret
);
1185 end_header(s
, this, "application/xml");
1189 void RGWDeleteBucketReplication_ObjStore_S3::update_sync_policy(rgw_sync_policy_info
*policy
)
1191 policy
->groups
.erase(enabled_group_id
);
1192 policy
->groups
.erase(disabled_group_id
);
1195 void RGWDeleteBucketReplication_ObjStore_S3::send_response()
1198 set_req_state_err(s
, op_ret
);
1200 end_header(s
, this, "application/xml");
1204 void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets
)
1207 set_req_state_err(s
, op_ret
);
1210 // Explicitly use chunked transfer encoding so that we can stream the result
1211 // to the user without having to wait for the full length of it.
1212 end_header(s
, NULL
, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1215 list_all_buckets_start(s
);
1216 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
1217 s
->formatter
->open_array_section("Buckets");
1222 void RGWListBuckets_ObjStore_S3::send_response_data(rgw::sal::RGWBucketList
& buckets
)
1227 map
<string
, rgw::sal::RGWBucket
*>& m
= buckets
.get_buckets();
1228 map
<string
, rgw::sal::RGWBucket
*>::iterator iter
;
1230 for (iter
= m
.begin(); iter
!= m
.end(); ++iter
) {
1231 rgw::sal::RGWBucket
* obj
= iter
->second
;
1232 dump_bucket(s
, *obj
);
1234 rgw_flush_formatter(s
, s
->formatter
);
1237 void RGWListBuckets_ObjStore_S3::send_response_end()
1240 s
->formatter
->close_section();
1241 list_all_buckets_end(s
);
1242 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1246 int RGWGetUsage_ObjStore_S3::get_params()
1248 start_date
= s
->info
.args
.get("start-date");
1249 end_date
= s
->info
.args
.get("end-date");
1253 static void dump_usage_categories_info(Formatter
*formatter
, const rgw_usage_log_entry
& entry
, map
<string
, bool> *categories
)
1255 formatter
->open_array_section("categories");
1256 map
<string
, rgw_usage_data
>::const_iterator uiter
;
1257 for (uiter
= entry
.usage_map
.begin(); uiter
!= entry
.usage_map
.end(); ++uiter
) {
1258 if (categories
&& !categories
->empty() && !categories
->count(uiter
->first
))
1260 const rgw_usage_data
& usage
= uiter
->second
;
1261 formatter
->open_object_section("Entry");
1262 encode_json("Category", uiter
->first
, formatter
);
1263 encode_json("BytesSent", usage
.bytes_sent
, formatter
);
1264 encode_json("BytesReceived", usage
.bytes_received
, formatter
);
1265 encode_json("Ops", usage
.ops
, formatter
);
1266 encode_json("SuccessfulOps", usage
.successful_ops
, formatter
);
1267 formatter
->close_section(); // Entry
1269 formatter
->close_section(); // Category
1272 static void dump_usage_bucket_info(Formatter
*formatter
, const std::string
& name
, const cls_user_bucket_entry
& entry
)
1274 formatter
->open_object_section("Entry");
1275 encode_json("Bucket", name
, formatter
);
1276 encode_json("Bytes", entry
.size
, formatter
);
1277 encode_json("Bytes_Rounded", entry
.size_rounded
, formatter
);
1278 formatter
->close_section(); // entry
1281 void RGWGetUsage_ObjStore_S3::send_response()
1284 set_req_state_err(s
, op_ret
);
1287 // Explicitly use chunked transfer encoding so that we can stream the result
1288 // to the user without having to wait for the full length of it.
1289 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1294 Formatter
*formatter
= s
->formatter
;
1296 bool user_section_open
= false;
1298 formatter
->open_object_section("Usage");
1299 if (show_log_entries
) {
1300 formatter
->open_array_section("Entries");
1302 map
<rgw_user_bucket
, rgw_usage_log_entry
>::iterator iter
;
1303 for (iter
= usage
.begin(); iter
!= usage
.end(); ++iter
) {
1304 const rgw_user_bucket
& ub
= iter
->first
;
1305 const rgw_usage_log_entry
& entry
= iter
->second
;
1307 if (show_log_entries
) {
1308 if (ub
.user
.compare(last_owner
) != 0) {
1309 if (user_section_open
) {
1310 formatter
->close_section();
1311 formatter
->close_section();
1313 formatter
->open_object_section("User");
1314 formatter
->dump_string("Owner", ub
.user
);
1315 formatter
->open_array_section("Buckets");
1316 user_section_open
= true;
1317 last_owner
= ub
.user
;
1319 formatter
->open_object_section("Bucket");
1320 formatter
->dump_string("Bucket", ub
.bucket
);
1321 utime_t
ut(entry
.epoch
, 0);
1322 ut
.gmtime(formatter
->dump_stream("Time"));
1323 formatter
->dump_int("Epoch", entry
.epoch
);
1324 dump_usage_categories_info(formatter
, entry
, &categories
);
1325 formatter
->close_section(); // bucket
1328 summary_map
[ub
.user
].aggregate(entry
, &categories
);
1331 if (show_log_entries
) {
1332 if (user_section_open
) {
1333 formatter
->close_section(); // buckets
1334 formatter
->close_section(); //user
1336 formatter
->close_section(); // entries
1340 formatter
->open_array_section("Summary");
1341 map
<string
, rgw_usage_log_entry
>::iterator siter
;
1342 for (siter
= summary_map
.begin(); siter
!= summary_map
.end(); ++siter
) {
1343 const rgw_usage_log_entry
& entry
= siter
->second
;
1344 formatter
->open_object_section("User");
1345 formatter
->dump_string("User", siter
->first
);
1346 dump_usage_categories_info(formatter
, entry
, &categories
);
1347 rgw_usage_data total_usage
;
1348 entry
.sum(total_usage
, categories
);
1349 formatter
->open_object_section("Total");
1350 encode_json("BytesSent", total_usage
.bytes_sent
, formatter
);
1351 encode_json("BytesReceived", total_usage
.bytes_received
, formatter
);
1352 encode_json("Ops", total_usage
.ops
, formatter
);
1353 encode_json("SuccessfulOps", total_usage
.successful_ops
, formatter
);
1354 formatter
->close_section(); // total
1355 formatter
->close_section(); // user
1358 if (s
->cct
->_conf
->rgw_rest_getusage_op_compat
) {
1359 formatter
->open_object_section("Stats");
1362 encode_json("TotalBytes", stats
.size
, formatter
);
1363 encode_json("TotalBytesRounded", stats
.size_rounded
, formatter
);
1364 encode_json("TotalEntries", stats
.num_objects
, formatter
);
1366 if (s
->cct
->_conf
->rgw_rest_getusage_op_compat
) {
1367 formatter
->close_section(); //Stats
1370 formatter
->close_section(); // summary
1373 formatter
->open_array_section("CapacityUsed");
1374 formatter
->open_object_section("User");
1375 formatter
->open_array_section("Buckets");
1376 for (const auto& biter
: buckets_usage
) {
1377 const cls_user_bucket_entry
& entry
= biter
.second
;
1378 dump_usage_bucket_info(formatter
, biter
.first
, entry
);
1380 formatter
->close_section(); // Buckets
1381 formatter
->close_section(); // User
1382 formatter
->close_section(); // CapacityUsed
1384 formatter
->close_section(); // usage
1385 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1388 int RGWListBucket_ObjStore_S3::get_common_params()
1390 list_versions
= s
->info
.args
.exists("versions");
1391 prefix
= s
->info
.args
.get("prefix");
1394 s
->info
.args
.get_bool("allow-unordered", &allow_unordered
, false);
1395 delimiter
= s
->info
.args
.get("delimiter");
1396 max_keys
= s
->info
.args
.get("max-keys");
1397 op_ret
= parse_max_keys();
1401 encoding_type
= s
->info
.args
.get("encoding-type");
1402 if (s
->system_request
) {
1403 s
->info
.args
.get_bool("objs-container", &objs_container
, false);
1404 const char *shard_id_str
= s
->info
.env
->get("HTTP_RGWX_SHARD_ID");
1407 shard_id
= strict_strtol(shard_id_str
, 10, &err
);
1409 ldout(s
->cct
, 5) << "bad shard id specified: " << shard_id_str
<< dendl
;
1413 shard_id
= s
->bucket_instance_shard_id
;
1419 int RGWListBucket_ObjStore_S3::get_params()
1421 int ret
= get_common_params();
1425 if (!list_versions
) {
1426 marker
= s
->info
.args
.get("marker");
1428 marker
.name
= s
->info
.args
.get("key-marker");
1429 marker
.instance
= s
->info
.args
.get("version-id-marker");
1434 int RGWListBucket_ObjStore_S3v2::get_params()
1436 int ret
= get_common_params();
1440 s
->info
.args
.get_bool("fetch-owner", &fetchOwner
, false);
1441 startAfter
= s
->info
.args
.get("start-after", &start_after_exist
);
1442 continuation_token
= s
->info
.args
.get("continuation-token", &continuation_token_exist
);
1443 if(!continuation_token_exist
) {
1444 marker
= startAfter
;
1446 marker
= continuation_token
;
1451 void RGWListBucket_ObjStore_S3::send_common_versioned_response()
1453 if (!s
->bucket_tenant
.empty()) {
1454 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
1456 s
->formatter
->dump_string("Name", s
->bucket_name
);
1457 s
->formatter
->dump_string("Prefix", prefix
);
1458 s
->formatter
->dump_int("MaxKeys", max
);
1459 if (!delimiter
.empty()) {
1460 s
->formatter
->dump_string("Delimiter", delimiter
);
1462 s
->formatter
->dump_string("IsTruncated", (max
&& is_truncated
? "true"
1465 if (!common_prefixes
.empty()) {
1466 map
<string
, bool>::iterator pref_iter
;
1467 for (pref_iter
= common_prefixes
.begin();
1468 pref_iter
!= common_prefixes
.end(); ++pref_iter
) {
1469 s
->formatter
->open_array_section("CommonPrefixes");
1471 s
->formatter
->dump_string("Prefix", url_encode(pref_iter
->first
, false));
1473 s
->formatter
->dump_string("Prefix", pref_iter
->first
);
1476 s
->formatter
->close_section();
1481 void RGWListBucket_ObjStore_S3::send_versioned_response()
1483 s
->formatter
->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3
);
1484 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1485 s
->formatter
->dump_string("EncodingType", "url");
1488 RGWListBucket_ObjStore_S3::send_common_versioned_response();
1489 s
->formatter
->dump_string("KeyMarker", marker
.name
);
1490 s
->formatter
->dump_string("VersionIdMarker", marker
.instance
);
1491 if (is_truncated
&& !next_marker
.empty()) {
1492 s
->formatter
->dump_string("NextKeyMarker", next_marker
.name
);
1493 if (next_marker
.instance
.empty()) {
1494 s
->formatter
->dump_string("NextVersionIdMarker", "null");
1497 s
->formatter
->dump_string("NextVersionIdMarker", next_marker
.instance
);
1502 if (objs_container
) {
1503 s
->formatter
->open_array_section("Entries");
1506 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1507 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1508 const char *section_name
= (iter
->is_delete_marker() ? "DeleteMarker"
1510 s
->formatter
->open_object_section(section_name
);
1511 if (objs_container
) {
1512 s
->formatter
->dump_bool("IsDeleteMarker", iter
->is_delete_marker());
1514 rgw_obj_key
key(iter
->key
);
1517 url_encode(key
.name
, key_name
);
1518 s
->formatter
->dump_string("Key", key_name
);
1521 s
->formatter
->dump_string("Key", key
.name
);
1523 string version_id
= key
.instance
;
1524 if (version_id
.empty()) {
1525 version_id
= "null";
1527 if (s
->system_request
) {
1528 if (iter
->versioned_epoch
> 0) {
1529 s
->formatter
->dump_int("VersionedEpoch", iter
->versioned_epoch
);
1531 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1532 utime_t
ut(iter
->meta
.mtime
);
1533 ut
.gmtime_nsec(s
->formatter
->dump_stream("RgwxMtime"));
1535 s
->formatter
->dump_string("VersionId", version_id
);
1536 s
->formatter
->dump_bool("IsLatest", iter
->is_current());
1537 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1538 if (!iter
->is_delete_marker()) {
1539 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1540 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1541 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1542 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1544 dump_owner(s
, rgw_user(iter
->meta
.owner
), iter
->meta
.owner_display_name
);
1545 if (iter
->meta
.appendable
) {
1546 s
->formatter
->dump_string("Type", "Appendable");
1548 s
->formatter
->dump_string("Type", "Normal");
1550 s
->formatter
->close_section(); // Version/DeleteMarker
1552 if (objs_container
) {
1553 s
->formatter
->close_section(); // Entries
1555 s
->formatter
->close_section(); // ListVersionsResult
1557 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1561 void RGWListBucket_ObjStore_S3::send_common_response()
1563 if (!s
->bucket_tenant
.empty()) {
1564 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
1566 s
->formatter
->dump_string("Name", s
->bucket_name
);
1567 s
->formatter
->dump_string("Prefix", prefix
);
1568 s
->formatter
->dump_int("MaxKeys", max
);
1569 if (!delimiter
.empty()) {
1570 s
->formatter
->dump_string("Delimiter", delimiter
);
1572 s
->formatter
->dump_string("IsTruncated", (max
&& is_truncated
? "true"
1575 if (!common_prefixes
.empty()) {
1576 map
<string
, bool>::iterator pref_iter
;
1577 for (pref_iter
= common_prefixes
.begin();
1578 pref_iter
!= common_prefixes
.end(); ++pref_iter
) {
1579 s
->formatter
->open_array_section("CommonPrefixes");
1581 s
->formatter
->dump_string("Prefix", url_encode(pref_iter
->first
, false));
1583 s
->formatter
->dump_string("Prefix", pref_iter
->first
);
1585 s
->formatter
->close_section();
1590 void RGWListBucket_ObjStore_S3::send_response()
1593 set_req_state_err(s
, op_ret
);
1597 // Explicitly use chunked transfer encoding so that we can stream the result
1598 // to the user without having to wait for the full length of it.
1599 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1604 if (list_versions
) {
1605 send_versioned_response();
1609 s
->formatter
->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3
);
1610 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1611 s
->formatter
->dump_string("EncodingType", "url");
1614 RGWListBucket_ObjStore_S3::send_common_response();
1616 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1617 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1618 rgw_obj_key
key(iter
->key
);
1619 s
->formatter
->open_array_section("Contents");
1622 url_encode(key
.name
, key_name
);
1623 s
->formatter
->dump_string("Key", key_name
);
1625 s
->formatter
->dump_string("Key", key
.name
);
1627 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1628 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1629 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1630 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1631 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1632 dump_owner(s
, rgw_user(iter
->meta
.owner
), iter
->meta
.owner_display_name
);
1633 if (s
->system_request
) {
1634 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1636 if (iter
->meta
.appendable
) {
1637 s
->formatter
->dump_string("Type", "Appendable");
1639 s
->formatter
->dump_string("Type", "Normal");
1641 s
->formatter
->close_section();
1644 s
->formatter
->dump_string("Marker", marker
.name
);
1645 if (is_truncated
&& !next_marker
.empty()) {
1646 s
->formatter
->dump_string("NextMarker", next_marker
.name
);
1648 s
->formatter
->close_section();
1649 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1652 void RGWListBucket_ObjStore_S3v2::send_versioned_response()
1654 s
->formatter
->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3
);
1655 RGWListBucket_ObjStore_S3v2::send_common_versioned_response();
1656 s
->formatter
->dump_string("KeyContinuationToken", marker
.name
);
1657 s
->formatter
->dump_string("VersionIdContinuationToken", marker
.instance
);
1658 if (is_truncated
&& !next_marker
.empty()) {
1659 s
->formatter
->dump_string("NextKeyContinuationToken", next_marker
.name
);
1660 s
->formatter
->dump_string("NextVersionIdContinuationToken", next_marker
.instance
);
1663 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1664 s
->formatter
->dump_string("EncodingType", "url");
1669 if (objs_container
) {
1670 s
->formatter
->open_array_section("Entries");
1673 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1674 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1675 const char *section_name
= (iter
->is_delete_marker() ? "DeleteContinuationToken"
1677 s
->formatter
->open_object_section(section_name
);
1678 if (objs_container
) {
1679 s
->formatter
->dump_bool("IsDeleteContinuationToken", iter
->is_delete_marker());
1681 rgw_obj_key
key(iter
->key
);
1684 url_encode(key
.name
, key_name
);
1685 s
->formatter
->dump_string("Key", key_name
);
1688 s
->formatter
->dump_string("Key", key
.name
);
1690 string version_id
= key
.instance
;
1691 if (version_id
.empty()) {
1692 version_id
= "null";
1694 if (s
->system_request
) {
1695 if (iter
->versioned_epoch
> 0) {
1696 s
->formatter
->dump_int("VersionedEpoch", iter
->versioned_epoch
);
1698 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1699 utime_t
ut(iter
->meta
.mtime
);
1700 ut
.gmtime_nsec(s
->formatter
->dump_stream("RgwxMtime"));
1702 s
->formatter
->dump_string("VersionId", version_id
);
1703 s
->formatter
->dump_bool("IsLatest", iter
->is_current());
1704 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1705 if (!iter
->is_delete_marker()) {
1706 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1707 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1708 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1709 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1711 if (fetchOwner
== true) {
1712 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
1714 s
->formatter
->close_section();
1718 if (objs_container
) {
1719 s
->formatter
->close_section();
1722 if (!common_prefixes
.empty()) {
1723 map
<string
, bool>::iterator pref_iter
;
1724 for (pref_iter
= common_prefixes
.begin();
1725 pref_iter
!= common_prefixes
.end(); ++pref_iter
) {
1726 s
->formatter
->open_array_section("CommonPrefixes");
1728 s
->formatter
->dump_string("Prefix", url_encode(pref_iter
->first
, false));
1730 s
->formatter
->dump_string("Prefix", pref_iter
->first
);
1733 s
->formatter
->dump_int("KeyCount",objs
.size());
1734 if (start_after_exist
) {
1735 s
->formatter
->dump_string("StartAfter", startAfter
);
1737 s
->formatter
->close_section();
1741 s
->formatter
->close_section();
1742 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1746 void RGWListBucket_ObjStore_S3v2::send_response()
1749 set_req_state_err(s
, op_ret
);
1753 // Explicitly use chunked transfer encoding so that we can stream the result
1754 // to the user without having to wait for the full length of it.
1755 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1760 if (list_versions
) {
1761 send_versioned_response();
1765 s
->formatter
->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3
);
1766 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1767 s
->formatter
->dump_string("EncodingType", "url");
1771 RGWListBucket_ObjStore_S3::send_common_response();
1773 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1774 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1775 rgw_obj_key
key(iter
->key
);
1776 s
->formatter
->open_array_section("Contents");
1779 url_encode(key
.name
, key_name
);
1780 s
->formatter
->dump_string("Key", key_name
);
1783 s
->formatter
->dump_string("Key", key
.name
);
1785 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1786 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1787 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1788 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1789 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1790 if (fetchOwner
== true) {
1791 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
1793 if (s
->system_request
) {
1794 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1796 if (iter
->meta
.appendable
) {
1797 s
->formatter
->dump_string("Type", "Appendable");
1799 s
->formatter
->dump_string("Type", "Normal");
1801 s
->formatter
->close_section();
1804 if (continuation_token_exist
) {
1805 s
->formatter
->dump_string("ContinuationToken", continuation_token
);
1807 if (is_truncated
&& !next_marker
.empty()) {
1808 s
->formatter
->dump_string("NextContinuationToken", next_marker
.name
);
1810 s
->formatter
->dump_int("KeyCount",objs
.size());
1811 if (start_after_exist
) {
1812 s
->formatter
->dump_string("StartAfter", startAfter
);
1814 s
->formatter
->close_section();
1815 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1818 void RGWGetBucketLogging_ObjStore_S3::send_response()
1821 end_header(s
, this, "application/xml");
1824 s
->formatter
->open_object_section_in_ns("BucketLoggingStatus", XMLNS_AWS_S3
);
1825 s
->formatter
->close_section();
1826 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1829 void RGWGetBucketLocation_ObjStore_S3::send_response()
1832 end_header(s
, this);
1835 RGWZoneGroup zonegroup
;
1838 int ret
= store
->svc()->zone
->get_zonegroup(s
->bucket_info
.zonegroup
, zonegroup
);
1840 api_name
= zonegroup
.api_name
;
1842 if (s
->bucket_info
.zonegroup
!= "default") {
1843 api_name
= s
->bucket_info
.zonegroup
;
1847 s
->formatter
->dump_format_ns("LocationConstraint", XMLNS_AWS_S3
,
1848 "%s", api_name
.c_str());
1849 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1852 void RGWGetBucketVersioning_ObjStore_S3::send_response()
1855 end_header(s
, this, "application/xml");
1858 s
->formatter
->open_object_section_in_ns("VersioningConfiguration", XMLNS_AWS_S3
);
1860 const char *status
= (versioning_enabled
? "Enabled" : "Suspended");
1861 s
->formatter
->dump_string("Status", status
);
1862 const char *mfa_status
= (mfa_enabled
? "Enabled" : "Disabled");
1863 s
->formatter
->dump_string("MfaDelete", mfa_status
);
1865 s
->formatter
->close_section();
1866 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1869 struct ver_config_status
{
1870 int status
{VersioningSuspended
};
1876 } mfa_status
{MFA_UNKNOWN
};
1879 void decode_xml(XMLObj
*obj
) {
1882 RGWXMLDecoder::decode_xml("Status", status_str
, obj
);
1883 if (status_str
== "Enabled") {
1884 status
= VersioningEnabled
;
1885 } else if (status_str
!= "Suspended") {
1886 status
= VersioningStatusInvalid
;
1890 if (RGWXMLDecoder::decode_xml("MfaDelete", mfa_str
, obj
)) {
1891 if (mfa_str
== "Enabled") {
1892 mfa_status
= MFA_ENABLED
;
1893 } else if (mfa_str
== "Disabled") {
1894 mfa_status
= MFA_DISABLED
;
1902 int RGWSetBucketVersioning_ObjStore_S3::get_params()
1907 rgw_rest_read_all_input(s
, s
->cct
->_conf
->rgw_max_put_param_size
, false);
1912 r
= do_aws4_auth_completion();
1917 RGWXMLDecoder::XMLParser parser
;
1918 if (!parser
.init()) {
1919 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
1923 char* buf
= data
.c_str();
1924 if (!parser
.parse(buf
, data
.length(), 1)) {
1925 ldpp_dout(this, 10) << "NOTICE: failed to parse data: " << buf
<< dendl
;
1930 ver_config_status status_conf
;
1932 if (!RGWXMLDecoder::decode_xml("VersioningConfiguration", status_conf
, &parser
)) {
1933 ldpp_dout(this, 10) << "NOTICE: bad versioning config input" << dendl
;
1937 if (!store
->svc()->zone
->is_meta_master()) {
1938 /* only need to keep this data around if we're not meta master */
1939 in_data
.append(data
);
1942 versioning_status
= status_conf
.status
;
1943 if (versioning_status
== VersioningStatusInvalid
) {
1947 if (status_conf
.mfa_status
!= ver_config_status::MFA_UNKNOWN
) {
1948 mfa_set_status
= true;
1949 switch (status_conf
.mfa_status
) {
1950 case ver_config_status::MFA_DISABLED
:
1953 case ver_config_status::MFA_ENABLED
:
1957 ldpp_dout(this, 0) << "ERROR: RGWSetBucketVersioning_ObjStore_S3::get_params(): unexpected switch case mfa_status=" << status_conf
.mfa_status
<< dendl
;
1960 } else if (status_conf
.retcode
< 0) {
1961 r
= status_conf
.retcode
;
1966 void RGWSetBucketVersioning_ObjStore_S3::send_response()
1969 set_req_state_err(s
, op_ret
);
1971 end_header(s
, this, "application/xml");
1974 int RGWSetBucketWebsite_ObjStore_S3::get_params()
1976 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
1980 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
1986 r
= do_aws4_auth_completion();
1991 in_data
.append(data
);
1993 RGWXMLDecoder::XMLParser parser
;
1994 if (!parser
.init()) {
1995 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
1999 char* buf
= data
.c_str();
2000 if (!parser
.parse(buf
, data
.length(), 1)) {
2001 ldpp_dout(this, 5) << "failed to parse xml: " << buf
<< dendl
;
2006 RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf
, &parser
, true);
2007 } catch (RGWXMLDecoder::err
& err
) {
2008 ldpp_dout(this, 5) << "unexpected xml: " << buf
<< dendl
;
2012 if (website_conf
.is_redirect_all
&& website_conf
.redirect_all
.hostname
.empty()) {
2013 s
->err
.message
= "A host name must be provided to redirect all requests (e.g. \"example.com\").";
2014 ldout(s
->cct
, 5) << s
->err
.message
<< dendl
;
2016 } else if (!website_conf
.is_redirect_all
&& !website_conf
.is_set_index_doc
) {
2017 s
->err
.message
= "A value for IndexDocument Suffix must be provided if RedirectAllRequestsTo is empty";
2018 ldout(s
->cct
, 5) << s
->err
.message
<< dendl
;
2020 } else if (!website_conf
.is_redirect_all
&& website_conf
.is_set_index_doc
&&
2021 website_conf
.index_doc_suffix
.empty()) {
2022 s
->err
.message
= "The IndexDocument Suffix is not well formed";
2023 ldout(s
->cct
, 5) << s
->err
.message
<< dendl
;
2027 #define WEBSITE_ROUTING_RULES_MAX_NUM 50
2028 int max_num
= s
->cct
->_conf
->rgw_website_routing_rules_max_num
;
2030 max_num
= WEBSITE_ROUTING_RULES_MAX_NUM
;
2032 int routing_rules_num
= website_conf
.routing_rules
.rules
.size();
2033 if (routing_rules_num
> max_num
) {
2034 ldpp_dout(this, 4) << "An website routing config can have up to "
2036 << " rules, request website routing rules num: "
2037 << routing_rules_num
<< dendl
;
2038 op_ret
= -ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR
;
2039 s
->err
.message
= std::to_string(routing_rules_num
) +" routing rules provided, the number of routing rules in a website configuration is limited to "
2040 + std::to_string(max_num
)
2042 return -ERR_INVALID_REQUEST
;
2048 void RGWSetBucketWebsite_ObjStore_S3::send_response()
2051 set_req_state_err(s
, op_ret
);
2053 end_header(s
, this, "application/xml");
2056 void RGWDeleteBucketWebsite_ObjStore_S3::send_response()
2059 op_ret
= STATUS_NO_CONTENT
;
2061 set_req_state_err(s
, op_ret
);
2063 end_header(s
, this, "application/xml");
2066 void RGWGetBucketWebsite_ObjStore_S3::send_response()
2069 set_req_state_err(s
, op_ret
);
2071 end_header(s
, this, "application/xml");
2078 RGWBucketWebsiteConf
& conf
= s
->bucket_info
.website_conf
;
2080 s
->formatter
->open_object_section_in_ns("WebsiteConfiguration", XMLNS_AWS_S3
);
2081 conf
.dump_xml(s
->formatter
);
2082 s
->formatter
->close_section(); // WebsiteConfiguration
2083 rgw_flush_formatter_and_reset(s
, s
->formatter
);
2086 static void dump_bucket_metadata(struct req_state
*s
, rgw::sal::RGWBucket
* bucket
)
2088 dump_header(s
, "X-RGW-Object-Count", static_cast<long long>(bucket
->get_count()));
2089 dump_header(s
, "X-RGW-Bytes-Used", static_cast<long long>(bucket
->get_size()));
2092 void RGWStatBucket_ObjStore_S3::send_response()
2095 dump_bucket_metadata(s
, bucket
);
2098 set_req_state_err(s
, op_ret
);
2101 end_header(s
, this);
2105 static int create_s3_policy(struct req_state
*s
, rgw::sal::RGWRadosStore
*store
,
2106 RGWAccessControlPolicy_S3
& s3policy
,
2109 if (s
->has_acl_header
) {
2110 if (!s
->canned_acl
.empty())
2111 return -ERR_INVALID_REQUEST
;
2113 return s3policy
.create_from_headers(store
->ctl()->user
, s
->info
.env
, owner
);
2116 return s3policy
.create_canned(owner
, s
->bucket_owner
, s
->canned_acl
);
2119 class RGWLocationConstraint
: public XMLObj
2122 RGWLocationConstraint() {}
2123 ~RGWLocationConstraint() override
{}
2124 bool xml_end(const char *el
) override
{
2128 location_constraint
= get_data();
2133 string location_constraint
;
2136 class RGWCreateBucketConfig
: public XMLObj
2139 RGWCreateBucketConfig() {}
2140 ~RGWCreateBucketConfig() override
{}
2143 class RGWCreateBucketParser
: public RGWXMLParser
2145 XMLObj
*alloc_obj(const char *el
) override
{
2150 RGWCreateBucketParser() {}
2151 ~RGWCreateBucketParser() override
{}
2153 bool get_location_constraint(string
& zone_group
) {
2154 XMLObj
*config
= find_first("CreateBucketConfiguration");
2158 XMLObj
*constraint
= config
->find_first("LocationConstraint");
2162 zone_group
= constraint
->get_data();
2168 int RGWCreateBucket_ObjStore_S3::get_params()
2170 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
2171 bool relaxed_names
= s
->cct
->_conf
->rgw_relaxed_s3_bucket_names
;
2173 int r
= valid_s3_bucket_name(s
->bucket_name
, relaxed_names
);
2177 r
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
2183 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
2187 std::tie(op_ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
2189 if ((op_ret
< 0) && (op_ret
!= -ERR_LENGTH_REQUIRED
))
2192 const int auth_ret
= do_aws4_auth_completion();
2197 in_data
.append(data
);
2199 if (data
.length()) {
2200 RGWCreateBucketParser parser
;
2202 if (!parser
.init()) {
2203 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
2207 char* buf
= data
.c_str();
2208 bool success
= parser
.parse(buf
, data
.length(), 1);
2209 ldpp_dout(this, 20) << "create bucket input data=" << buf
<< dendl
;
2212 ldpp_dout(this, 0) << "failed to parse input: " << buf
<< dendl
;
2216 if (!parser
.get_location_constraint(location_constraint
)) {
2217 ldpp_dout(this, 0) << "provided input did not specify location constraint correctly" << dendl
;
2221 ldpp_dout(this, 10) << "create bucket location constraint: "
2222 << location_constraint
<< dendl
;
2225 size_t pos
= location_constraint
.find(':');
2226 if (pos
!= string::npos
) {
2227 placement_rule
.init(location_constraint
.substr(pos
+ 1), s
->info
.storage_class
);
2228 location_constraint
= location_constraint
.substr(0, pos
);
2230 placement_rule
.storage_class
= s
->info
.storage_class
;
2232 auto iter
= s
->info
.x_meta_map
.find("x-amz-bucket-object-lock-enabled");
2233 if (iter
!= s
->info
.x_meta_map
.end()) {
2234 if (!boost::algorithm::iequals(iter
->second
, "true") && !boost::algorithm::iequals(iter
->second
, "false")) {
2237 obj_lock_enabled
= boost::algorithm::iequals(iter
->second
, "true");
2242 void RGWCreateBucket_ObjStore_S3::send_response()
2244 if (op_ret
== -ERR_BUCKET_EXISTS
)
2247 set_req_state_err(s
, op_ret
);
2254 if (s
->system_request
) {
2255 JSONFormatter f
; /* use json formatter for system requests output */
2257 f
.open_object_section("info");
2258 encode_json("entry_point_object_ver", ep_objv
, &f
);
2259 encode_json("object_ver", info
.objv_tracker
.read_version
, &f
);
2260 encode_json("bucket_info", info
, &f
);
2262 rgw_flush_formatter_and_reset(s
, &f
);
2266 void RGWDeleteBucket_ObjStore_S3::send_response()
2270 r
= STATUS_NO_CONTENT
;
2272 set_req_state_err(s
, r
);
2274 end_header(s
, this);
2277 static inline void map_qs_metadata(struct req_state
* s
)
2279 /* merge S3 valid user metadata from the query-string into
2280 * x_meta_map, which maps them to attributes */
2281 const auto& params
= const_cast<RGWHTTPArgs
&>(s
->info
.args
).get_params();
2282 for (const auto& elt
: params
) {
2283 std::string k
= boost::algorithm::to_lower_copy(elt
.first
);
2284 if (k
.find("x-amz-meta-") == /* offset */ 0) {
2285 rgw_add_amz_meta_header(s
->info
.x_meta_map
, k
, elt
.second
);
2290 int RGWPutObj_ObjStore_S3::get_params()
2293 return -ERR_LENGTH_REQUIRED
;
2295 map
<string
, bufferlist
> src_attrs
;
2301 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
2302 ret
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
2308 if_match
= s
->info
.env
->get("HTTP_IF_MATCH");
2309 if_nomatch
= s
->info
.env
->get("HTTP_IF_NONE_MATCH");
2310 copy_source
= url_decode(s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE", ""));
2311 copy_source_range
= s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE_RANGE");
2313 /* handle x-amz-copy-source */
2314 boost::string_view
cs_view(copy_source
);
2315 if (! cs_view
.empty()) {
2316 if (cs_view
[0] == '/')
2317 cs_view
.remove_prefix(1);
2318 copy_source_bucket_name
= cs_view
.to_string();
2319 pos
= copy_source_bucket_name
.find("/");
2320 if (pos
== std::string::npos
) {
2322 ldpp_dout(this, 5) << "x-amz-copy-source bad format" << dendl
;
2325 copy_source_object_name
=
2326 copy_source_bucket_name
.substr(pos
+ 1, copy_source_bucket_name
.size());
2327 copy_source_bucket_name
= copy_source_bucket_name
.substr(0, pos
);
2328 #define VERSION_ID_STR "?versionId="
2329 pos
= copy_source_object_name
.find(VERSION_ID_STR
);
2330 if (pos
== std::string::npos
) {
2331 copy_source_object_name
= url_decode(copy_source_object_name
);
2333 copy_source_version_id
=
2334 copy_source_object_name
.substr(pos
+ sizeof(VERSION_ID_STR
) - 1);
2335 copy_source_object_name
=
2336 url_decode(copy_source_object_name
.substr(0, pos
));
2338 pos
= copy_source_bucket_name
.find(":");
2339 if (pos
== std::string::npos
) {
2340 copy_source_tenant_name
= s
->src_tenant_name
;
2342 copy_source_tenant_name
= copy_source_bucket_name
.substr(0, pos
);
2343 copy_source_bucket_name
= copy_source_bucket_name
.substr(pos
+ 1, copy_source_bucket_name
.size());
2344 if (copy_source_bucket_name
.empty()) {
2346 ldpp_dout(this, 5) << "source bucket name is empty" << dendl
;
2350 ret
= store
->getRados()->get_bucket_info(store
->svc(),
2351 copy_source_tenant_name
,
2352 copy_source_bucket_name
,
2353 copy_source_bucket_info
,
2354 NULL
, s
->yield
, &src_attrs
);
2356 ldpp_dout(this, 5) << __func__
<< "(): get_bucket_info() returned ret=" << ret
<< dendl
;
2360 /* handle x-amz-copy-source-range */
2362 if (copy_source_range
) {
2363 string range
= copy_source_range
;
2364 pos
= range
.find("bytes=");
2365 if (pos
== std::string::npos
|| pos
!= 0) {
2367 ldpp_dout(this, 5) << "x-amz-copy-source-range bad format" << dendl
;
2370 /* 6 is the length of "bytes=" */
2371 range
= range
.substr(pos
+ 6);
2372 pos
= range
.find("-");
2373 if (pos
== std::string::npos
) {
2375 ldpp_dout(this, 5) << "x-amz-copy-source-range bad format" << dendl
;
2378 string first
= range
.substr(0, pos
);
2379 string last
= range
.substr(pos
+ 1);
2380 if (first
.find_first_not_of("0123456789") != std::string::npos
|| last
.find_first_not_of("0123456789") != std::string::npos
)
2382 ldpp_dout(this, 5) << "x-amz-copy-source-range bad format not an integer" << dendl
;
2386 copy_source_range_fst
= strtoull(first
.c_str(), NULL
, 10);
2387 copy_source_range_lst
= strtoull(last
.c_str(), NULL
, 10);
2388 if (copy_source_range_fst
> copy_source_range_lst
)
2391 ldpp_dout(this, 5) << "x-amz-copy-source-range bad format first number bigger than second" << dendl
;
2398 /* handle object tagging */
2399 auto tag_str
= s
->info
.env
->get("HTTP_X_AMZ_TAGGING");
2401 obj_tags
= std::make_unique
<RGWObjTags
>();
2402 ret
= obj_tags
->set_from_string(tag_str
);
2404 ldpp_dout(this,0) << "setting obj tags failed with " << ret
<< dendl
;
2405 if (ret
== -ERR_INVALID_TAG
){
2406 ret
= -EINVAL
; //s3 returns only -EINVAL for PUT requests
2413 //handle object lock
2414 auto obj_lock_mode_str
= s
->info
.env
->get("HTTP_X_AMZ_OBJECT_LOCK_MODE");
2415 auto obj_lock_date_str
= s
->info
.env
->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE");
2416 auto obj_legal_hold_str
= s
->info
.env
->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD");
2417 if (obj_lock_mode_str
&& obj_lock_date_str
) {
2418 boost::optional
<ceph::real_time
> date
= ceph::from_iso_8601(obj_lock_date_str
);
2419 if (boost::none
== date
|| ceph::real_clock::to_time_t(*date
) <= ceph_clock_now()) {
2421 ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl
;
2424 if (strcmp(obj_lock_mode_str
, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str
, "COMPLIANCE") != 0) {
2426 ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl
;
2429 obj_retention
= new RGWObjectRetention(obj_lock_mode_str
, *date
);
2430 } else if ((obj_lock_mode_str
&& !obj_lock_date_str
) || (!obj_lock_mode_str
&& obj_lock_date_str
)) {
2432 ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl
;
2435 if (obj_legal_hold_str
) {
2436 if (strcmp(obj_legal_hold_str
, "ON") != 0 && strcmp(obj_legal_hold_str
, "OFF") != 0) {
2438 ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl
;
2441 obj_legal_hold
= new RGWObjectLegalHold(obj_legal_hold_str
);
2443 if (!s
->bucket_info
.obj_lock_enabled() && (obj_retention
|| obj_legal_hold
)) {
2444 ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl
;
2445 ret
= -ERR_INVALID_REQUEST
;
2448 multipart_upload_id
= s
->info
.args
.get("uploadId");
2449 multipart_part_str
= s
->info
.args
.get("partNumber");
2450 if (!multipart_part_str
.empty()) {
2452 multipart_part_num
= strict_strtol(multipart_part_str
.c_str(), 10, &err
);
2454 ldpp_dout(s
, 10) << "bad part number: " << multipart_part_str
<< ": " << err
<< dendl
;
2457 } else if (!multipart_upload_id
.empty()) {
2458 ldpp_dout(s
, 10) << "part number with no multipart upload id" << dendl
;
2462 append
= s
->info
.args
.exists("append");
2464 string pos_str
= s
->info
.args
.get("position");
2465 if (pos_str
.empty()) {
2468 position
= strtoull(pos_str
.c_str(), NULL
, 10);
2472 return RGWPutObj_ObjStore::get_params();
2475 int RGWPutObj_ObjStore_S3::get_data(bufferlist
& bl
)
2477 const int ret
= RGWPutObj_ObjStore::get_data(bl
);
2479 const int ret_auth
= do_aws4_auth_completion();
2488 static int get_success_retcode(int code
)
2492 return STATUS_CREATED
;
2494 return STATUS_NO_CONTENT
;
2499 void RGWPutObj_ObjStore_S3::send_response()
2502 set_req_state_err(s
, op_ret
);
2505 if (s
->cct
->_conf
->rgw_s3_success_create_obj_status
) {
2506 op_ret
= get_success_retcode(
2507 s
->cct
->_conf
->rgw_s3_success_create_obj_status
);
2508 set_req_state_err(s
, op_ret
);
2511 string expires
= get_s3_expiration_header(s
, mtime
);
2513 if (copy_source
.empty()) {
2516 dump_content_length(s
, 0);
2517 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
2518 dump_header_if_nonempty(s
, "x-amz-expiration", expires
);
2519 for (auto &it
: crypt_http_responses
)
2520 dump_header(s
, it
.first
, it
.second
);
2523 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
2524 dump_header_if_nonempty(s
, "x-amz-expiration", expires
);
2525 end_header(s
, this, "application/xml");
2529 time_t secs
= (time_t)ut
.sec();
2530 gmtime_r(&secs
, &tmp
);
2531 char buf
[TIME_BUF_SIZE
];
2532 s
->formatter
->open_object_section_in_ns("CopyPartResult",
2533 "http://s3.amazonaws.com/doc/2006-03-01/");
2534 if (strftime(buf
, sizeof(buf
), "%Y-%m-%dT%T.000Z", &tmp
) > 0) {
2535 s
->formatter
->dump_string("LastModified", buf
);
2537 s
->formatter
->dump_string("ETag", etag
);
2538 s
->formatter
->close_section();
2539 rgw_flush_formatter_and_reset(s
, s
->formatter
);
2544 if (op_ret
== 0 || op_ret
== -ERR_POSITION_NOT_EQUAL_TO_LENGTH
) {
2545 dump_header(s
, "x-rgw-next-append-position", cur_accounted_size
);
2548 if (s
->system_request
&& !real_clock::is_zero(mtime
)) {
2549 dump_epoch_header(s
, "Rgwx-Mtime", mtime
);
2551 end_header(s
, this);
2554 static inline int get_obj_attrs(rgw::sal::RGWRadosStore
*store
, struct req_state
*s
, rgw_obj
& obj
, map
<string
, bufferlist
>& attrs
)
2556 RGWRados::Object
op_target(store
->getRados(), s
->bucket_info
, *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
2557 RGWRados::Object::Read
read_op(&op_target
);
2559 read_op
.params
.attrs
= &attrs
;
2561 return read_op
.prepare(s
->yield
);
2564 static inline void set_attr(map
<string
, bufferlist
>& attrs
, const char* key
, const std::string
& value
)
2568 attrs
.emplace(key
, std::move(bl
));
2571 static inline void set_attr(map
<string
, bufferlist
>& attrs
, const char* key
, const char* value
)
2575 attrs
.emplace(key
, std::move(bl
));
2578 int RGWPutObj_ObjStore_S3::get_decrypt_filter(
2579 std::unique_ptr
<RGWGetObj_Filter
>* filter
,
2580 RGWGetObj_Filter
* cb
,
2581 map
<string
, bufferlist
>& attrs
,
2582 bufferlist
* manifest_bl
)
2584 std::map
<std::string
, std::string
> crypt_http_responses_unused
;
2587 std::unique_ptr
<BlockCrypt
> block_crypt
;
2588 res
= rgw_s3_prepare_decrypt(s
, attrs
, &block_crypt
, crypt_http_responses_unused
);
2590 if (block_crypt
!= nullptr) {
2591 auto f
= std::unique_ptr
<RGWGetObj_BlockDecrypt
>(new RGWGetObj_BlockDecrypt(s
->cct
, cb
, std::move(block_crypt
)));
2592 //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
2594 if (manifest_bl
!= nullptr) {
2595 res
= f
->read_manifest(*manifest_bl
);
2597 *filter
= std::move(f
);
2606 int RGWPutObj_ObjStore_S3::get_encrypt_filter(
2607 std::unique_ptr
<rgw::putobj::DataProcessor
> *filter
,
2608 rgw::putobj::DataProcessor
*cb
)
2611 if (!multipart_upload_id
.empty()) {
2612 RGWMPObj
mp(s
->object
.name
, multipart_upload_id
);
2614 obj
.init_ns(s
->bucket
, mp
.get_meta(), RGW_OBJ_NS_MULTIPART
);
2615 obj
.set_in_extra_data(true);
2616 map
<string
, bufferlist
> xattrs
;
2617 res
= get_obj_attrs(store
, s
, obj
, xattrs
);
2619 std::unique_ptr
<BlockCrypt
> block_crypt
;
2620 /* We are adding to existing object.
2621 * We use crypto mode that configured as if we were decrypting. */
2622 res
= rgw_s3_prepare_decrypt(s
, xattrs
, &block_crypt
, crypt_http_responses
);
2623 if (res
== 0 && block_crypt
!= nullptr)
2624 filter
->reset(new RGWPutObj_BlockEncrypt(s
->cct
, cb
, std::move(block_crypt
)));
2626 /* it is ok, to not have encryption at all */
2630 std::unique_ptr
<BlockCrypt
> block_crypt
;
2631 res
= rgw_s3_prepare_encrypt(s
, attrs
, nullptr, &block_crypt
, crypt_http_responses
);
2632 if (res
== 0 && block_crypt
!= nullptr) {
2633 filter
->reset(new RGWPutObj_BlockEncrypt(s
->cct
, cb
, std::move(block_crypt
)));
2639 void RGWPostObj_ObjStore_S3::rebuild_key(string
& key
)
2641 static string var
= "${filename}";
2642 int pos
= key
.find(var
);
2646 string new_key
= key
.substr(0, pos
);
2647 new_key
.append(filename
);
2648 new_key
.append(key
.substr(pos
+ var
.size()));
2653 std::string
RGWPostObj_ObjStore_S3::get_current_filename() const
2655 return s
->object
.name
;
2658 std::string
RGWPostObj_ObjStore_S3::get_current_content_type() const
2660 return content_type
;
2663 int RGWPostObj_ObjStore_S3::get_params()
2665 op_ret
= RGWPostObj_ObjStore::get_params();
2672 ldpp_dout(this, 20) << "adding bucket to policy env: " << s
->bucket
.name
2674 env
.add_var("bucket", s
->bucket
.name
);
2678 struct post_form_part part
;
2679 int r
= read_form_part_header(&part
, done
);
2683 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 20>()) {
2684 ldpp_dout(this, 20) << "read part header -- part.name="
2685 << part
.name
<< dendl
;
2687 for (const auto& pair
: part
.fields
) {
2688 ldpp_dout(this, 20) << "field.name=" << pair
.first
<< dendl
;
2689 ldpp_dout(this, 20) << "field.val=" << pair
.second
.val
<< dendl
;
2690 ldpp_dout(this, 20) << "field.params:" << dendl
;
2692 for (const auto& param_pair
: pair
.second
.params
) {
2693 ldpp_dout(this, 20) << " " << param_pair
.first
2694 << " -> " << param_pair
.second
<< dendl
;
2699 if (done
) { /* unexpected here */
2700 err_msg
= "Malformed request";
2704 if (stringcasecmp(part
.name
, "file") == 0) { /* beginning of data transfer */
2705 struct post_part_field
& field
= part
.fields
["Content-Disposition"];
2706 map
<string
, string
>::iterator iter
= field
.params
.find("filename");
2707 if (iter
!= field
.params
.end()) {
2708 filename
= iter
->second
;
2710 parts
[part
.name
] = part
;
2715 uint64_t chunk_size
= s
->cct
->_conf
->rgw_max_chunk_size
;
2716 r
= read_data(part
.data
, chunk_size
, boundary
, done
);
2717 if (r
< 0 || !boundary
) {
2718 err_msg
= "Couldn't find boundary";
2721 parts
[part
.name
] = part
;
2722 string
part_str(part
.data
.c_str(), part
.data
.length());
2723 env
.add_var(part
.name
, part_str
);
2727 if (!part_str(parts
, "key", &object_str
)) {
2728 err_msg
= "Key not specified";
2732 s
->object
= rgw_obj_key(object_str
);
2734 rebuild_key(s
->object
.name
);
2736 if (s
->object
.empty()) {
2737 err_msg
= "Empty object name";
2741 env
.add_var("key", s
->object
.name
);
2743 part_str(parts
, "Content-Type", &content_type
);
2745 /* AWS permits POST without Content-Type: http://tracker.ceph.com/issues/20201 */
2746 if (! content_type
.empty()) {
2747 env
.add_var("Content-Type", content_type
);
2750 std::string storage_class
;
2751 part_str(parts
, "x-amz-storage-class", &storage_class
);
2753 if (! storage_class
.empty()) {
2754 s
->dest_placement
.storage_class
= storage_class
;
2755 if (!store
->svc()->zone
->get_zone_params().valid_placement(s
->dest_placement
)) {
2756 ldpp_dout(this, 0) << "NOTICE: invalid dest placement: " << s
->dest_placement
.to_str() << dendl
;
2757 err_msg
= "The storage class you specified is not valid";
2762 map
<string
, struct post_form_part
, ltstr_nocase
>::iterator piter
=
2763 parts
.upper_bound(RGW_AMZ_META_PREFIX
);
2764 for (; piter
!= parts
.end(); ++piter
) {
2765 string n
= piter
->first
;
2766 if (strncasecmp(n
.c_str(), RGW_AMZ_META_PREFIX
,
2767 sizeof(RGW_AMZ_META_PREFIX
) - 1) != 0)
2770 string attr_name
= RGW_ATTR_PREFIX
;
2771 attr_name
.append(n
);
2773 /* need to null terminate it */
2774 bufferlist
& data
= piter
->second
.data
;
2775 string str
= string(data
.c_str(), data
.length());
2778 attr_bl
.append(str
.c_str(), str
.size() + 1);
2780 attrs
[attr_name
] = attr_bl
;
2782 // TODO: refactor this and the above loop to share code
2783 piter
= parts
.find(RGW_AMZ_WEBSITE_REDIRECT_LOCATION
);
2784 if (piter
!= parts
.end()) {
2785 string n
= piter
->first
;
2786 string attr_name
= RGW_ATTR_PREFIX
;
2787 attr_name
.append(n
);
2788 /* need to null terminate it */
2789 bufferlist
& data
= piter
->second
.data
;
2790 string str
= string(data
.c_str(), data
.length());
2793 attr_bl
.append(str
.c_str(), str
.size() + 1);
2795 attrs
[attr_name
] = attr_bl
;
2798 int r
= get_policy();
2807 min_len
= post_policy
.min_length
;
2808 max_len
= post_policy
.max_length
;
2815 int RGWPostObj_ObjStore_S3::get_tags()
2818 if (part_str(parts
, "tagging", &tags_str
)) {
2819 RGWXMLParser parser
;
2820 if (!parser
.init()){
2821 ldpp_dout(this, 0) << "Couldn't init RGWObjTags XML parser" << dendl
;
2822 err_msg
= "Server couldn't process the request";
2823 return -EINVAL
; // TODO: This class of errors in rgw code should be a 5XX error
2825 if (!parser
.parse(tags_str
.c_str(), tags_str
.size(), 1)) {
2826 ldpp_dout(this,0 ) << "Invalid Tagging XML" << dendl
;
2827 err_msg
= "Invalid Tagging XML";
2831 RGWObjTagging_S3 tagging
;
2834 RGWXMLDecoder::decode_xml("Tagging", tagging
, &parser
);
2835 } catch (RGWXMLDecoder::err
& err
) {
2836 ldpp_dout(this, 5) << "Malformed tagging request: " << err
<< dendl
;
2840 RGWObjTags obj_tags
;
2841 int r
= tagging
.rebuild(obj_tags
);
2846 obj_tags
.encode(tags_bl
);
2847 ldpp_dout(this, 20) << "Read " << obj_tags
.count() << "tags" << dendl
;
2848 attrs
[RGW_ATTR_TAGS
] = tags_bl
;
2855 int RGWPostObj_ObjStore_S3::get_policy()
2857 if (part_bl(parts
, "policy", &s
->auth
.s3_postobj_creds
.encoded_policy
)) {
2858 bool aws4_auth
= false;
2860 /* x-amz-algorithm handling */
2861 using rgw::auth::s3::AWS4_HMAC_SHA256_STR
;
2862 if ((part_str(parts
, "x-amz-algorithm", &s
->auth
.s3_postobj_creds
.x_amz_algorithm
)) &&
2863 (s
->auth
.s3_postobj_creds
.x_amz_algorithm
== AWS4_HMAC_SHA256_STR
)) {
2864 ldpp_dout(this, 0) << "Signature verification algorithm AWS v4 (AWS4-HMAC-SHA256)" << dendl
;
2867 ldpp_dout(this, 0) << "Signature verification algorithm AWS v2" << dendl
;
2870 // check that the signature matches the encoded policy
2874 /* x-amz-credential handling */
2875 if (!part_str(parts
, "x-amz-credential",
2876 &s
->auth
.s3_postobj_creds
.x_amz_credential
)) {
2877 ldpp_dout(this, 0) << "No S3 aws4 credential found!" << dendl
;
2878 err_msg
= "Missing aws4 credential";
2882 /* x-amz-signature handling */
2883 if (!part_str(parts
, "x-amz-signature",
2884 &s
->auth
.s3_postobj_creds
.signature
)) {
2885 ldpp_dout(this, 0) << "No aws4 signature found!" << dendl
;
2886 err_msg
= "Missing aws4 signature";
2890 /* x-amz-date handling */
2891 std::string received_date_str
;
2892 if (!part_str(parts
, "x-amz-date", &received_date_str
)) {
2893 ldpp_dout(this, 0) << "No aws4 date found!" << dendl
;
2894 err_msg
= "Missing aws4 date";
2900 // check that the signature matches the encoded policy
2901 if (!part_str(parts
, "AWSAccessKeyId",
2902 &s
->auth
.s3_postobj_creds
.access_key
)) {
2903 ldpp_dout(this, 0) << "No S3 aws2 access key found!" << dendl
;
2904 err_msg
= "Missing aws2 access key";
2908 if (!part_str(parts
, "signature", &s
->auth
.s3_postobj_creds
.signature
)) {
2909 ldpp_dout(this, 0) << "No aws2 signature found!" << dendl
;
2910 err_msg
= "Missing aws2 signature";
2915 if (part_str(parts
, "x-amz-security-token", &s
->auth
.s3_postobj_creds
.x_amz_security_token
)) {
2916 if (s
->auth
.s3_postobj_creds
.x_amz_security_token
.size() == 0) {
2917 err_msg
= "Invalid token";
2922 /* FIXME: this is a makeshift solution. The browser upload authentication will be
2923 * handled by an instance of rgw::auth::Completer spawned in Handler's authorize()
2925 const int ret
= rgw::auth::Strategy::apply(this, auth_registry_ptr
->get_s3_post(), s
);
2929 /* Populate the owner info. */
2930 s
->owner
.set_id(s
->user
->get_id());
2931 s
->owner
.set_name(s
->user
->get_display_name());
2932 ldpp_dout(this, 20) << "Successful Signature Verification!" << dendl
;
2935 ceph::bufferlist decoded_policy
;
2937 decoded_policy
.decode_base64(s
->auth
.s3_postobj_creds
.encoded_policy
);
2938 } catch (buffer::error
& err
) {
2939 ldpp_dout(this, 0) << "failed to decode_base64 policy" << dendl
;
2940 err_msg
= "Could not decode policy";
2944 decoded_policy
.append('\0'); // NULL terminate
2945 ldpp_dout(this, 20) << "POST policy: " << decoded_policy
.c_str() << dendl
;
2948 int r
= post_policy
.from_json(decoded_policy
, err_msg
);
2950 if (err_msg
.empty()) {
2951 err_msg
= "Failed to parse policy";
2953 ldpp_dout(this, 0) << "failed to parse policy" << dendl
;
2959 post_policy
.set_var_checked("x-amz-signature");
2962 post_policy
.set_var_checked("AWSAccessKeyId");
2963 post_policy
.set_var_checked("signature");
2965 post_policy
.set_var_checked("policy");
2967 r
= post_policy
.check(&env
, err_msg
);
2969 if (err_msg
.empty()) {
2970 err_msg
= "Policy check failed";
2972 ldpp_dout(this, 0) << "policy check failed" << dendl
;
2977 ldpp_dout(this, 0) << "No attached policy found!" << dendl
;
2981 part_str(parts
, "acl", &canned_acl
);
2983 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
2984 ldpp_dout(this, 20) << "canned_acl=" << canned_acl
<< dendl
;
2985 if (s3policy
.create_canned(s
->owner
, s
->bucket_owner
, canned_acl
) < 0) {
2986 err_msg
= "Bad canned ACLs";
2995 int RGWPostObj_ObjStore_S3::complete_get_params()
2999 struct post_form_part part
;
3000 int r
= read_form_part_header(&part
, done
);
3005 ceph::bufferlist part_data
;
3007 uint64_t chunk_size
= s
->cct
->_conf
->rgw_max_chunk_size
;
3008 r
= read_data(part
.data
, chunk_size
, boundary
, done
);
3009 if (r
< 0 || !boundary
) {
3013 /* Just reading the data but not storing any results of that. */
3019 int RGWPostObj_ObjStore_S3::get_data(ceph::bufferlist
& bl
, bool& again
)
3024 const uint64_t chunk_size
= s
->cct
->_conf
->rgw_max_chunk_size
;
3025 int r
= read_data(bl
, chunk_size
, boundary
, done
);
3032 /* Reached end of data, let's drain the rest of the params */
3033 r
= complete_get_params();
3044 void RGWPostObj_ObjStore_S3::send_response()
3046 if (op_ret
== 0 && parts
.count("success_action_redirect")) {
3049 part_str(parts
, "success_action_redirect", &redirect
);
3054 string etag_str
= "\"";
3056 etag_str
.append(etag
);
3057 etag_str
.append("\"");
3061 url_encode(s
->bucket_tenant
, tenant
); /* surely overkill, but cheap */
3062 url_encode(s
->bucket_name
, bucket
);
3063 url_encode(s
->object
.name
, key
);
3064 url_encode(etag_str
, etag_url
);
3066 if (!s
->bucket_tenant
.empty()) {
3068 * What we really would like is to quaily the bucket name, so
3069 * that the client could simply copy it and paste into next request.
3070 * Unfortunately, in S3 we cannot know if the client will decide
3071 * to come through DNS, with "bucket.tenant" sytanx, or through
3072 * URL with "tenant\bucket" syntax. Therefore, we provide the
3073 * tenant separately.
3075 redirect
.append("?tenant=");
3076 redirect
.append(tenant
);
3077 redirect
.append("&bucket=");
3078 redirect
.append(bucket
);
3080 redirect
.append("?bucket=");
3081 redirect
.append(bucket
);
3083 redirect
.append("&key=");
3084 redirect
.append(key
);
3085 redirect
.append("&etag=");
3086 redirect
.append(etag_url
);
3088 int r
= check_utf8(redirect
.c_str(), redirect
.size());
3093 dump_redirect(s
, redirect
);
3094 op_ret
= STATUS_REDIRECT
;
3095 } else if (op_ret
== 0 && parts
.count("success_action_status")) {
3096 string status_string
;
3097 uint32_t status_int
;
3099 part_str(parts
, "success_action_status", &status_string
);
3101 int r
= stringtoul(status_string
, &status_int
);
3107 switch (status_int
) {
3111 op_ret
= STATUS_CREATED
;
3114 op_ret
= STATUS_NO_CONTENT
;
3117 } else if (! op_ret
) {
3118 op_ret
= STATUS_NO_CONTENT
;
3122 if (op_ret
== STATUS_CREATED
) {
3123 for (auto &it
: crypt_http_responses
)
3124 dump_header(s
, it
.first
, it
.second
);
3125 s
->formatter
->open_object_section("PostResponse");
3126 std::string base_uri
= compute_domain_uri(s
);
3127 if (!s
->bucket_tenant
.empty()){
3128 s
->formatter
->dump_format("Location", "%s/%s:%s/%s",
3130 url_encode(s
->bucket_tenant
).c_str(),
3131 url_encode(s
->bucket_name
).c_str(),
3132 url_encode(s
->object
.name
).c_str());
3133 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3135 s
->formatter
->dump_format("Location", "%s/%s/%s",
3137 url_encode(s
->bucket_name
).c_str(),
3138 url_encode(s
->object
.name
).c_str());
3140 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3141 s
->formatter
->dump_string("Key", s
->object
.name
);
3142 s
->formatter
->dump_string("ETag", etag
);
3143 s
->formatter
->close_section();
3145 s
->err
.message
= err_msg
;
3146 set_req_state_err(s
, op_ret
);
3149 dump_content_length(s
, s
->formatter
->get_len());
3151 end_header(s
, this);
3152 if (op_ret
!= STATUS_CREATED
)
3155 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3158 int RGWPostObj_ObjStore_S3::get_encrypt_filter(
3159 std::unique_ptr
<rgw::putobj::DataProcessor
> *filter
,
3160 rgw::putobj::DataProcessor
*cb
)
3162 std::unique_ptr
<BlockCrypt
> block_crypt
;
3163 int res
= rgw_s3_prepare_encrypt(s
, attrs
, &parts
, &block_crypt
,
3164 crypt_http_responses
);
3165 if (res
== 0 && block_crypt
!= nullptr) {
3166 filter
->reset(new RGWPutObj_BlockEncrypt(s
->cct
, cb
, std::move(block_crypt
)));
3171 int RGWDeleteObj_ObjStore_S3::get_params()
3173 const char *if_unmod
= s
->info
.env
->get("HTTP_X_AMZ_DELETE_IF_UNMODIFIED_SINCE");
3175 if (s
->system_request
) {
3176 s
->info
.args
.get_bool(RGW_SYS_PARAM_PREFIX
"no-precondition-error", &no_precondition_error
, false);
3180 std::string if_unmod_decoded
= url_decode(if_unmod
);
3183 if (utime_t::parse_date(if_unmod_decoded
, &epoch
, &nsec
) < 0) {
3184 ldpp_dout(this, 10) << "failed to parse time: " << if_unmod_decoded
<< dendl
;
3187 unmod_since
= utime_t(epoch
, nsec
).to_real_time();
3190 const char *bypass_gov_header
= s
->info
.env
->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
3191 if (bypass_gov_header
) {
3192 std::string bypass_gov_decoded
= url_decode(bypass_gov_header
);
3193 bypass_governance_mode
= boost::algorithm::iequals(bypass_gov_decoded
, "true");
3199 void RGWDeleteObj_ObjStore_S3::send_response()
3205 r
= STATUS_NO_CONTENT
;
3207 set_req_state_err(s
, r
);
3209 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
3210 if (delete_marker
) {
3211 dump_header(s
, "x-amz-delete-marker", "true");
3213 end_header(s
, this);
3216 int RGWCopyObj_ObjStore_S3::init_dest_policy()
3218 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
3220 /* build a policy for the target object */
3221 int r
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
3225 dest_policy
= s3policy
;
3230 int RGWCopyObj_ObjStore_S3::get_params()
3232 if_mod
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
3233 if_unmod
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
3234 if_match
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_MATCH");
3235 if_nomatch
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
3237 src_tenant_name
= s
->src_tenant_name
;
3238 src_bucket_name
= s
->src_bucket_name
;
3239 src_object
= s
->src_object
;
3240 dest_tenant_name
= s
->bucket
.tenant
;
3241 dest_bucket_name
= s
->bucket
.name
;
3242 dest_object
= s
->object
.name
;
3244 if (s
->system_request
) {
3245 source_zone
= s
->info
.args
.get(RGW_SYS_PARAM_PREFIX
"source-zone");
3246 s
->info
.args
.get_bool(RGW_SYS_PARAM_PREFIX
"copy-if-newer", ©_if_newer
, false);
3249 copy_source
= s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE");
3250 auto tmp_md_d
= s
->info
.env
->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
3252 if (strcasecmp(tmp_md_d
, "COPY") == 0) {
3253 attrs_mod
= RGWRados::ATTRSMOD_NONE
;
3254 } else if (strcasecmp(tmp_md_d
, "REPLACE") == 0) {
3255 attrs_mod
= RGWRados::ATTRSMOD_REPLACE
;
3256 } else if (!source_zone
.empty()) {
3257 attrs_mod
= RGWRados::ATTRSMOD_NONE
; // default for intra-zone_group copy
3259 s
->err
.message
= "Unknown metadata directive.";
3260 ldpp_dout(this, 0) << s
->err
.message
<< dendl
;
3263 md_directive
= tmp_md_d
;
3266 if (source_zone
.empty() &&
3267 (dest_tenant_name
.compare(src_tenant_name
) == 0) &&
3268 (dest_bucket_name
.compare(src_bucket_name
) == 0) &&
3269 (dest_object
.compare(src_object
.name
) == 0) &&
3270 src_object
.instance
.empty() &&
3271 (attrs_mod
!= RGWRados::ATTRSMOD_REPLACE
)) {
3272 need_to_check_storage_class
= true;
3278 int RGWCopyObj_ObjStore_S3::check_storage_class(const rgw_placement_rule
& src_placement
)
3280 if (src_placement
== s
->dest_placement
) {
3281 /* can only copy object into itself if replacing attrs */
3282 s
->err
.message
= "This copy request is illegal because it is trying to copy "
3283 "an object to itself without changing the object's metadata, "
3284 "storage class, website redirect location or encryption attributes.";
3285 ldpp_dout(this, 0) << s
->err
.message
<< dendl
;
3286 return -ERR_INVALID_REQUEST
;
3291 void RGWCopyObj_ObjStore_S3::send_partial_response(off_t ofs
)
3293 if (! sent_header
) {
3295 set_req_state_err(s
, op_ret
);
3298 // Explicitly use chunked transfer encoding so that we can stream the result
3299 // to the user without having to wait for the full length of it.
3300 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3303 s
->formatter
->open_object_section_in_ns("CopyObjectResult", XMLNS_AWS_S3
);
3307 /* Send progress field. Note that this diverge from the original S3
3308 * spec. We do this in order to keep connection alive.
3310 s
->formatter
->dump_int("Progress", (uint64_t)ofs
);
3312 rgw_flush_formatter(s
, s
->formatter
);
3315 void RGWCopyObj_ObjStore_S3::send_response()
3318 send_partial_response(0);
3321 dump_time(s
, "LastModified", &mtime
);
3322 if (!etag
.empty()) {
3323 s
->formatter
->dump_string("ETag", std::move(etag
));
3325 s
->formatter
->close_section();
3326 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3330 void RGWGetACLs_ObjStore_S3::send_response()
3333 set_req_state_err(s
, op_ret
);
3335 end_header(s
, this, "application/xml");
3337 rgw_flush_formatter(s
, s
->formatter
);
3341 int RGWPutACLs_ObjStore_S3::get_params()
3343 int ret
= RGWPutACLs_ObjStore::get_params();
3345 const int ret_auth
= do_aws4_auth_completion();
3350 /* a request body is not required an S3 PutACLs request--n.b.,
3351 * s->length is non-null iff a content length was parsed (the
3352 * ACP or canned ACL could be in any of 3 headers, don't worry
3353 * about that here) */
3354 if ((ret
== -ERR_LENGTH_REQUIRED
) &&
3362 int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::RGWRadosStore
*store
,
3363 struct req_state
*s
,
3366 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
3368 // bucket-* canned acls do not apply to bucket
3369 if (s
->object
.empty()) {
3370 if (s
->canned_acl
.find("bucket") != string::npos
)
3371 s
->canned_acl
.clear();
3374 int r
= create_s3_policy(s
, store
, s3policy
, owner
);
3378 s3policy
.to_xml(ss
);
3383 void RGWPutACLs_ObjStore_S3::send_response()
3386 set_req_state_err(s
, op_ret
);
3388 end_header(s
, this, "application/xml");
3392 void RGWGetLC_ObjStore_S3::execute()
3394 config
.set_ctx(s
->cct
);
3396 map
<string
, bufferlist
>::iterator aiter
= s
->bucket_attrs
.find(RGW_ATTR_LC
);
3397 if (aiter
== s
->bucket_attrs
.end()) {
3402 bufferlist::const_iterator iter
{&aiter
->second
};
3404 config
.decode(iter
);
3405 } catch (const buffer::error
& e
) {
3406 ldpp_dout(this, 0) << __func__
<< "decode life cycle config failed" << dendl
;
3412 void RGWGetLC_ObjStore_S3::send_response()
3415 if (op_ret
== -ENOENT
) {
3416 set_req_state_err(s
, ERR_NO_SUCH_LC
);
3418 set_req_state_err(s
, op_ret
);
3422 end_header(s
, this, "application/xml");
3428 encode_xml("LifecycleConfiguration", XMLNS_AWS_S3
, config
, s
->formatter
);
3429 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3432 void RGWPutLC_ObjStore_S3::send_response()
3435 set_req_state_err(s
, op_ret
);
3437 end_header(s
, this, "application/xml");
3441 void RGWDeleteLC_ObjStore_S3::send_response()
3444 op_ret
= STATUS_NO_CONTENT
;
3446 set_req_state_err(s
, op_ret
);
3449 end_header(s
, this, "application/xml");
3453 void RGWGetCORS_ObjStore_S3::send_response()
3456 if (op_ret
== -ENOENT
)
3457 set_req_state_err(s
, ERR_NO_SUCH_CORS_CONFIGURATION
);
3459 set_req_state_err(s
, op_ret
);
3462 end_header(s
, NULL
, "application/xml");
3466 RGWCORSConfiguration_S3
*s3cors
=
3467 static_cast<RGWCORSConfiguration_S3
*>(&bucket_cors
);
3476 int RGWPutCORS_ObjStore_S3::get_params()
3478 RGWCORSXMLParser_S3
parser(s
->cct
);
3479 RGWCORSConfiguration_S3
*cors_config
;
3481 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
3485 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
3490 r
= do_aws4_auth_completion();
3495 if (!parser
.init()) {
3499 char* buf
= data
.c_str();
3500 if (!buf
|| !parser
.parse(buf
, data
.length(), 1)) {
3501 return -ERR_MALFORMED_XML
;
3504 static_cast<RGWCORSConfiguration_S3
*>(parser
.find_first(
3505 "CORSConfiguration"));
3507 return -ERR_MALFORMED_XML
;
3510 #define CORS_RULES_MAX_NUM 100
3511 int max_num
= s
->cct
->_conf
->rgw_cors_rules_max_num
;
3513 max_num
= CORS_RULES_MAX_NUM
;
3515 int cors_rules_num
= cors_config
->get_rules().size();
3516 if (cors_rules_num
> max_num
) {
3517 ldpp_dout(this, 4) << "An cors config can have up to "
3519 << " rules, request cors rules num: "
3520 << cors_rules_num
<< dendl
;
3521 op_ret
= -ERR_INVALID_CORS_RULES_ERROR
;
3522 s
->err
.message
= "The number of CORS rules should not exceed allowed limit of "
3523 + std::to_string(max_num
) + " rules.";
3524 return -ERR_INVALID_REQUEST
;
3527 // forward bucket cors requests to meta master zone
3528 if (!store
->svc()->zone
->is_meta_master()) {
3529 /* only need to keep this data around if we're not meta master */
3530 in_data
.append(data
);
3533 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
3534 ldpp_dout(this, 15) << "CORSConfiguration";
3535 cors_config
->to_xml(*_dout
);
3539 cors_config
->encode(cors_bl
);
3544 void RGWPutCORS_ObjStore_S3::send_response()
3547 set_req_state_err(s
, op_ret
);
3549 end_header(s
, NULL
, "application/xml");
3553 void RGWDeleteCORS_ObjStore_S3::send_response()
3556 if (!r
|| r
== -ENOENT
)
3557 r
= STATUS_NO_CONTENT
;
3559 set_req_state_err(s
, r
);
3561 end_header(s
, NULL
);
3564 void RGWOptionsCORS_ObjStore_S3::send_response()
3566 string hdrs
, exp_hdrs
;
3567 uint32_t max_age
= CORS_MAX_AGE_INVALID
;
3568 /*EACCES means, there is no CORS registered yet for the bucket
3569 *ENOENT means, there is no match of the Origin in the list of CORSRule
3571 if (op_ret
== -ENOENT
)
3574 set_req_state_err(s
, op_ret
);
3576 end_header(s
, NULL
);
3579 get_response_params(hdrs
, exp_hdrs
, &max_age
);
3582 dump_access_control(s
, origin
, req_meth
, hdrs
.c_str(), exp_hdrs
.c_str(),
3584 end_header(s
, NULL
);
3587 void RGWGetRequestPayment_ObjStore_S3::send_response()
3590 end_header(s
, this, "application/xml");
3593 s
->formatter
->open_object_section_in_ns("RequestPaymentConfiguration", XMLNS_AWS_S3
);
3594 const char *payer
= requester_pays
? "Requester" : "BucketOwner";
3595 s
->formatter
->dump_string("Payer", payer
);
3596 s
->formatter
->close_section();
3597 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3600 class RGWSetRequestPaymentParser
: public RGWXMLParser
3602 XMLObj
*alloc_obj(const char *el
) override
{
3607 RGWSetRequestPaymentParser() {}
3608 ~RGWSetRequestPaymentParser() override
{}
3610 int get_request_payment_payer(bool *requester_pays
) {
3611 XMLObj
*config
= find_first("RequestPaymentConfiguration");
3615 *requester_pays
= false;
3617 XMLObj
*field
= config
->find_first("Payer");
3621 auto& s
= field
->get_data();
3623 if (stringcasecmp(s
, "Requester") == 0) {
3624 *requester_pays
= true;
3625 } else if (stringcasecmp(s
, "BucketOwner") != 0) {
3633 int RGWSetRequestPayment_ObjStore_S3::get_params()
3635 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
3638 std::tie(r
, in_data
) = rgw_rest_read_all_input(s
, max_size
, false);
3645 RGWSetRequestPaymentParser parser
;
3647 if (!parser
.init()) {
3648 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
3652 char* buf
= in_data
.c_str();
3653 if (!parser
.parse(buf
, in_data
.length(), 1)) {
3654 ldpp_dout(this, 10) << "failed to parse data: " << buf
<< dendl
;
3658 return parser
.get_request_payment_payer(&requester_pays
);
3661 void RGWSetRequestPayment_ObjStore_S3::send_response()
3664 set_req_state_err(s
, op_ret
);
3669 int RGWInitMultipart_ObjStore_S3::get_params()
3671 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
3672 op_ret
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
3681 void RGWInitMultipart_ObjStore_S3::send_response()
3684 set_req_state_err(s
, op_ret
);
3686 for (auto &it
: crypt_http_responses
)
3687 dump_header(s
, it
.first
, it
.second
);
3688 end_header(s
, this, "application/xml");
3691 s
->formatter
->open_object_section_in_ns("InitiateMultipartUploadResult", XMLNS_AWS_S3
);
3692 if (!s
->bucket_tenant
.empty())
3693 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3694 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3695 s
->formatter
->dump_string("Key", s
->object
.name
);
3696 s
->formatter
->dump_string("UploadId", upload_id
);
3697 s
->formatter
->close_section();
3698 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3702 int RGWInitMultipart_ObjStore_S3::prepare_encryption(map
<string
, bufferlist
>& attrs
)
3705 res
= rgw_s3_prepare_encrypt(s
, attrs
, nullptr, nullptr, crypt_http_responses
);
3709 int RGWCompleteMultipart_ObjStore_S3::get_params()
3711 int ret
= RGWCompleteMultipart_ObjStore::get_params();
3718 return do_aws4_auth_completion();
3721 void RGWCompleteMultipart_ObjStore_S3::send_response()
3724 set_req_state_err(s
, op_ret
);
3726 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
3727 end_header(s
, this, "application/xml");
3730 s
->formatter
->open_object_section_in_ns("CompleteMultipartUploadResult", XMLNS_AWS_S3
);
3731 std::string base_uri
= compute_domain_uri(s
);
3732 if (!s
->bucket_tenant
.empty()) {
3733 s
->formatter
->dump_format("Location", "%s/%s:%s/%s",
3735 s
->bucket_tenant
.c_str(),
3736 s
->bucket_name
.c_str(),
3737 s
->object
.name
.c_str()
3739 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3741 s
->formatter
->dump_format("Location", "%s/%s/%s",
3743 s
->bucket_name
.c_str(),
3744 s
->object
.name
.c_str()
3747 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3748 s
->formatter
->dump_string("Key", s
->object
.name
);
3749 s
->formatter
->dump_string("ETag", etag
);
3750 s
->formatter
->close_section();
3751 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3755 void RGWAbortMultipart_ObjStore_S3::send_response()
3759 r
= STATUS_NO_CONTENT
;
3761 set_req_state_err(s
, r
);
3763 end_header(s
, this);
3766 void RGWListMultipart_ObjStore_S3::send_response()
3769 set_req_state_err(s
, op_ret
);
3771 // Explicitly use chunked transfer encoding so that we can stream the result
3772 // to the user without having to wait for the full length of it.
3773 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3777 s
->formatter
->open_object_section_in_ns("ListPartsResult", XMLNS_AWS_S3
);
3778 map
<uint32_t, RGWUploadPartInfo
>::iterator iter
;
3779 map
<uint32_t, RGWUploadPartInfo
>::reverse_iterator test_iter
;
3782 iter
= parts
.begin();
3783 test_iter
= parts
.rbegin();
3784 if (test_iter
!= parts
.rend()) {
3785 cur_max
= test_iter
->first
;
3787 if (!s
->bucket_tenant
.empty())
3788 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3789 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3790 s
->formatter
->dump_string("Key", s
->object
.name
);
3791 s
->formatter
->dump_string("UploadId", upload_id
);
3792 s
->formatter
->dump_string("StorageClass", "STANDARD");
3793 s
->formatter
->dump_int("PartNumberMarker", marker
);
3794 s
->formatter
->dump_int("NextPartNumberMarker", cur_max
);
3795 s
->formatter
->dump_int("MaxParts", max_parts
);
3796 s
->formatter
->dump_string("IsTruncated", (truncated
? "true" : "false"));
3798 ACLOwner
& owner
= policy
.get_owner();
3799 dump_owner(s
, owner
.get_id(), owner
.get_display_name());
3801 for (; iter
!= parts
.end(); ++iter
) {
3802 RGWUploadPartInfo
& info
= iter
->second
;
3804 s
->formatter
->open_object_section("Part");
3806 dump_time(s
, "LastModified", &info
.modified
);
3808 s
->formatter
->dump_unsigned("PartNumber", info
.num
);
3809 s
->formatter
->dump_format("ETag", "\"%s\"", info
.etag
.c_str());
3810 s
->formatter
->dump_unsigned("Size", info
.accounted_size
);
3811 s
->formatter
->close_section();
3813 s
->formatter
->close_section();
3814 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3818 void RGWListBucketMultiparts_ObjStore_S3::send_response()
3821 set_req_state_err(s
, op_ret
);
3824 // Explicitly use chunked transfer encoding so that we can stream the result
3825 // to the user without having to wait for the full length of it.
3826 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3831 s
->formatter
->open_object_section_in_ns("ListMultipartUploadsResult", XMLNS_AWS_S3
);
3832 if (!s
->bucket_tenant
.empty())
3833 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3834 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3835 if (!prefix
.empty())
3836 s
->formatter
->dump_string("ListMultipartUploadsResult.Prefix", prefix
);
3837 const string
& key_marker
= marker
.get_key();
3838 if (!key_marker
.empty())
3839 s
->formatter
->dump_string("KeyMarker", key_marker
);
3840 const string
& upload_id_marker
= marker
.get_upload_id();
3841 if (!upload_id_marker
.empty())
3842 s
->formatter
->dump_string("UploadIdMarker", upload_id_marker
);
3843 string next_key
= next_marker
.mp
.get_key();
3844 if (!next_key
.empty())
3845 s
->formatter
->dump_string("NextKeyMarker", next_key
);
3846 string next_upload_id
= next_marker
.mp
.get_upload_id();
3847 if (!next_upload_id
.empty())
3848 s
->formatter
->dump_string("NextUploadIdMarker", next_upload_id
);
3849 s
->formatter
->dump_int("MaxUploads", max_uploads
);
3850 if (!delimiter
.empty())
3851 s
->formatter
->dump_string("Delimiter", delimiter
);
3852 s
->formatter
->dump_string("IsTruncated", (is_truncated
? "true" : "false"));
3855 vector
<RGWMultipartUploadEntry
>::iterator iter
;
3856 for (iter
= uploads
.begin(); iter
!= uploads
.end(); ++iter
) {
3857 RGWMPObj
& mp
= iter
->mp
;
3858 s
->formatter
->open_array_section("Upload");
3860 s
->formatter
->dump_string("Key", url_encode(mp
.get_key(), false));
3862 s
->formatter
->dump_string("Key", mp
.get_key());
3864 s
->formatter
->dump_string("UploadId", mp
.get_upload_id());
3865 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name(), "Initiator");
3866 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
3867 s
->formatter
->dump_string("StorageClass", "STANDARD");
3868 dump_time(s
, "Initiated", &iter
->obj
.meta
.mtime
);
3869 s
->formatter
->close_section();
3871 if (!common_prefixes
.empty()) {
3872 s
->formatter
->open_array_section("CommonPrefixes");
3873 for (const auto& kv
: common_prefixes
) {
3875 s
->formatter
->dump_string("CommonPrefixes.Prefix",
3876 url_encode(kv
.first
, false));
3878 s
->formatter
->dump_string("CommonPrefixes.Prefix", kv
.first
);
3881 s
->formatter
->close_section();
3884 s
->formatter
->close_section();
3885 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3888 int RGWDeleteMultiObj_ObjStore_S3::get_params()
3890 int ret
= RGWDeleteMultiObj_ObjStore::get_params();
3895 return do_aws4_auth_completion();
3898 void RGWDeleteMultiObj_ObjStore_S3::send_status()
3900 if (! status_dumped
) {
3902 set_req_state_err(s
, op_ret
);
3904 status_dumped
= true;
3908 void RGWDeleteMultiObj_ObjStore_S3::begin_response()
3911 if (!status_dumped
) {
3916 // Explicitly use chunked transfer encoding so that we can stream the result
3917 // to the user without having to wait for the full length of it.
3918 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3919 s
->formatter
->open_object_section_in_ns("DeleteResult", XMLNS_AWS_S3
);
3921 rgw_flush_formatter(s
, s
->formatter
);
3924 void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key
& key
,
3926 const string
& marker_version_id
, int ret
)
3929 if (ret
== 0 && !quiet
) {
3930 s
->formatter
->open_object_section("Deleted");
3931 s
->formatter
->dump_string("Key", key
.name
);
3932 if (!key
.instance
.empty()) {
3933 s
->formatter
->dump_string("VersionId", key
.instance
);
3935 if (delete_marker
) {
3936 s
->formatter
->dump_bool("DeleteMarker", true);
3937 s
->formatter
->dump_string("DeleteMarkerVersionId", marker_version_id
);
3939 s
->formatter
->close_section();
3940 } else if (ret
< 0) {
3941 struct rgw_http_error r
;
3944 s
->formatter
->open_object_section("Error");
3947 rgw_get_errno_s3(&r
, err_no
);
3949 s
->formatter
->dump_string("Key", key
.name
);
3950 s
->formatter
->dump_string("VersionId", key
.instance
);
3951 s
->formatter
->dump_string("Code", r
.s3_code
);
3952 s
->formatter
->dump_string("Message", r
.s3_code
);
3953 s
->formatter
->close_section();
3956 rgw_flush_formatter(s
, s
->formatter
);
3960 void RGWDeleteMultiObj_ObjStore_S3::end_response()
3963 s
->formatter
->close_section();
3964 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3967 void RGWGetObjLayout_ObjStore_S3::send_response()
3970 set_req_state_err(s
, op_ret
);
3972 end_header(s
, this, "application/json");
3980 f
.open_object_section("result");
3981 ::encode_json("head", head_obj
, &f
);
3982 ::encode_json("manifest", *manifest
, &f
);
3983 f
.open_array_section("data_location");
3984 for (auto miter
= manifest
->obj_begin(); miter
!= manifest
->obj_end(); ++miter
) {
3985 f
.open_object_section("obj");
3986 rgw_raw_obj raw_loc
= miter
.get_location().get_raw_obj(store
->getRados());
3987 uint64_t ofs
= miter
.get_ofs();
3988 uint64_t left
= manifest
->get_obj_size() - ofs
;
3989 ::encode_json("ofs", miter
.get_ofs(), &f
);
3990 ::encode_json("loc", raw_loc
, &f
);
3991 ::encode_json("loc_ofs", miter
.location_ofs(), &f
);
3992 uint64_t loc_size
= miter
.get_stripe_size();
3993 if (loc_size
> left
) {
3996 ::encode_json("loc_size", loc_size
, &f
);
3998 rgw_flush_formatter(s
, &f
);
4002 rgw_flush_formatter(s
, &f
);
4005 int RGWConfigBucketMetaSearch_ObjStore_S3::get_params()
4007 auto iter
= s
->info
.x_meta_map
.find("x-amz-meta-search");
4008 if (iter
== s
->info
.x_meta_map
.end()) {
4009 s
->err
.message
= "X-Rgw-Meta-Search header not provided";
4010 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4014 list
<string
> expressions
;
4015 get_str_list(iter
->second
, ",", expressions
);
4017 for (auto& expression
: expressions
) {
4018 vector
<string
> args
;
4019 get_str_vec(expression
, ";", args
);
4022 s
->err
.message
= "invalid empty expression";
4023 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4026 if (args
.size() > 2) {
4027 s
->err
.message
= string("invalid expression: ") + expression
;
4028 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4032 string key
= boost::algorithm::to_lower_copy(rgw_trim_whitespace(args
[0]));
4034 if (args
.size() > 1) {
4035 val
= boost::algorithm::to_lower_copy(rgw_trim_whitespace(args
[1]));
4038 if (!boost::algorithm::starts_with(key
, RGW_AMZ_META_PREFIX
)) {
4039 s
->err
.message
= string("invalid expression, key must start with '" RGW_AMZ_META_PREFIX
"' : ") + expression
;
4040 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4044 key
= key
.substr(sizeof(RGW_AMZ_META_PREFIX
) - 1);
4046 ESEntityTypeMap::EntityType entity_type
;
4048 if (val
.empty() || val
== "str" || val
== "string") {
4049 entity_type
= ESEntityTypeMap::ES_ENTITY_STR
;
4050 } else if (val
== "int" || val
== "integer") {
4051 entity_type
= ESEntityTypeMap::ES_ENTITY_INT
;
4052 } else if (val
== "date" || val
== "datetime") {
4053 entity_type
= ESEntityTypeMap::ES_ENTITY_DATE
;
4055 s
->err
.message
= string("invalid entity type: ") + val
;
4056 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4060 mdsearch_config
[key
] = entity_type
;
4066 void RGWConfigBucketMetaSearch_ObjStore_S3::send_response()
4069 set_req_state_err(s
, op_ret
);
4071 end_header(s
, this);
4074 void RGWGetBucketMetaSearch_ObjStore_S3::send_response()
4077 set_req_state_err(s
, op_ret
);
4079 end_header(s
, NULL
, "application/xml");
4081 Formatter
*f
= s
->formatter
;
4082 f
->open_array_section("GetBucketMetaSearchResult");
4083 for (auto& e
: s
->bucket_info
.mdsearch_config
) {
4084 f
->open_object_section("Entry");
4085 string k
= string("x-amz-meta-") + e
.first
;
4086 f
->dump_string("Key", k
.c_str());
4089 case ESEntityTypeMap::ES_ENTITY_INT
:
4092 case ESEntityTypeMap::ES_ENTITY_DATE
:
4098 f
->dump_string("Type", type
);
4102 rgw_flush_formatter(s
, f
);
4105 void RGWDelBucketMetaSearch_ObjStore_S3::send_response()
4108 set_req_state_err(s
, op_ret
);
4110 end_header(s
, this);
4113 void RGWPutBucketObjectLock_ObjStore_S3::send_response()
4116 set_req_state_err(s
, op_ret
);
4122 void RGWGetBucketObjectLock_ObjStore_S3::send_response()
4125 set_req_state_err(s
, op_ret
);
4128 end_header(s
, this, "application/xml");
4134 encode_xml("ObjectLockConfiguration", s
->bucket_info
.obj_lock
, s
->formatter
);
4135 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4139 int RGWPutObjRetention_ObjStore_S3::get_params()
4141 const char *bypass_gov_header
= s
->info
.env
->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
4142 if (bypass_gov_header
) {
4143 std::string bypass_gov_decoded
= url_decode(bypass_gov_header
);
4144 bypass_governance_mode
= boost::algorithm::iequals(bypass_gov_decoded
, "true");
4147 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
4148 std::tie(op_ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
4152 void RGWPutObjRetention_ObjStore_S3::send_response()
4155 set_req_state_err(s
, op_ret
);
4161 void RGWGetObjRetention_ObjStore_S3::send_response()
4164 set_req_state_err(s
, op_ret
);
4167 end_header(s
, this, "application/xml");
4173 encode_xml("Retention", obj_retention
, s
->formatter
);
4174 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4177 void RGWPutObjLegalHold_ObjStore_S3::send_response()
4180 set_req_state_err(s
, op_ret
);
4186 void RGWGetObjLegalHold_ObjStore_S3::send_response()
4189 set_req_state_err(s
, op_ret
);
4192 end_header(s
, this, "application/xml");
4198 encode_xml("LegalHold", obj_legal_hold
, s
->formatter
);
4199 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4202 void RGWGetBucketPolicyStatus_ObjStore_S3::send_response()
4205 set_req_state_err(s
, op_ret
);
4208 end_header(s
, this, "application/xml");
4211 s
->formatter
->open_object_section_in_ns("PolicyStatus", XMLNS_AWS_S3
);
4212 // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETPolicyStatus.html
4213 // mentions TRUE and FALSE, but boto/aws official clients seem to want lower
4214 // case which is returned by AWS as well; so let's be bug to bug compatible
4216 s
->formatter
->dump_bool("IsPublic", isPublic
);
4217 s
->formatter
->close_section();
4218 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4222 void RGWPutBucketPublicAccessBlock_ObjStore_S3::send_response()
4225 set_req_state_err(s
, op_ret
);
4231 void RGWGetBucketPublicAccessBlock_ObjStore_S3::send_response()
4234 set_req_state_err(s
, op_ret
);
4237 end_header(s
, this, "application/xml");
4240 access_conf
.dump_xml(s
->formatter
);
4241 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4244 RGWOp
*RGWHandler_REST_Service_S3::op_get()
4246 if (is_usage_op()) {
4247 return new RGWGetUsage_ObjStore_S3
;
4249 return new RGWListBuckets_ObjStore_S3
;
4253 RGWOp
*RGWHandler_REST_Service_S3::op_head()
4255 return new RGWListBuckets_ObjStore_S3
;
4258 RGWOp
*RGWHandler_REST_Service_S3::op_post()
4260 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
4264 std::tie(ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
4269 const auto post_body
= data
.to_str();
4272 RGWHandler_REST_STS
sts_handler(auth_registry
, post_body
);
4273 sts_handler
.init(store
, s
, s
->cio
);
4274 auto op
= sts_handler
.get_op();
4281 RGWHandler_REST_IAM
iam_handler(auth_registry
, post_body
);
4282 iam_handler
.init(store
, s
, s
->cio
);
4283 auto op
= iam_handler
.get_op();
4290 RGWHandler_REST_PSTopic_AWS
topic_handler(auth_registry
, post_body
);
4291 topic_handler
.init(store
, s
, s
->cio
);
4292 auto op
= topic_handler
.get_op();
4301 RGWOp
*RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data
) const
4306 s
->info
.args
.get_int("list-type", &list_type
, 1);
4307 switch (list_type
) {
4309 return new RGWListBucket_ObjStore_S3
;
4311 return new RGWListBucket_ObjStore_S3v2
;
4313 ldpp_dout(s
, 5) << __func__
<< ": unsupported list-type " << list_type
<< dendl
;
4314 return new RGWListBucket_ObjStore_S3
;
4317 return new RGWStatBucket_ObjStore_S3
;
4321 RGWOp
*RGWHandler_REST_Bucket_S3::op_get()
4323 if (s
->info
.args
.sub_resource_exists("logging"))
4324 return new RGWGetBucketLogging_ObjStore_S3
;
4326 if (s
->info
.args
.sub_resource_exists("location"))
4327 return new RGWGetBucketLocation_ObjStore_S3
;
4329 if (s
->info
.args
.sub_resource_exists("versioning"))
4330 return new RGWGetBucketVersioning_ObjStore_S3
;
4332 if (s
->info
.args
.sub_resource_exists("website")) {
4333 if (!s
->cct
->_conf
->rgw_enable_static_website
) {
4336 return new RGWGetBucketWebsite_ObjStore_S3
;
4339 if (s
->info
.args
.exists("mdsearch")) {
4340 return new RGWGetBucketMetaSearch_ObjStore_S3
;
4344 return new RGWGetACLs_ObjStore_S3
;
4345 } else if (is_cors_op()) {
4346 return new RGWGetCORS_ObjStore_S3
;
4347 } else if (is_request_payment_op()) {
4348 return new RGWGetRequestPayment_ObjStore_S3
;
4349 } else if (s
->info
.args
.exists("uploads")) {
4350 return new RGWListBucketMultiparts_ObjStore_S3
;
4351 } else if(is_lc_op()) {
4352 return new RGWGetLC_ObjStore_S3
;
4353 } else if(is_policy_op()) {
4354 return new RGWGetBucketPolicy
;
4355 } else if (is_tagging_op()) {
4356 return new RGWGetBucketTags_ObjStore_S3
;
4357 } else if (is_object_lock_op()) {
4358 return new RGWGetBucketObjectLock_ObjStore_S3
;
4359 } else if (is_notification_op()) {
4360 return RGWHandler_REST_PSNotifs_S3::create_get_op();
4361 } else if (is_replication_op()) {
4362 return new RGWGetBucketReplication_ObjStore_S3
;
4363 } else if (is_policy_status_op()) {
4364 return new RGWGetBucketPolicyStatus_ObjStore_S3
;
4365 } else if (is_block_public_access_op()) {
4366 return new RGWGetBucketPublicAccessBlock_ObjStore_S3
;
4368 return get_obj_op(true);
4371 RGWOp
*RGWHandler_REST_Bucket_S3::op_head()
4374 return new RGWGetACLs_ObjStore_S3
;
4375 } else if (s
->info
.args
.exists("uploads")) {
4376 return new RGWListBucketMultiparts_ObjStore_S3
;
4378 return get_obj_op(false);
4381 RGWOp
*RGWHandler_REST_Bucket_S3::op_put()
4383 if (s
->info
.args
.sub_resource_exists("logging"))
4385 if (s
->info
.args
.sub_resource_exists("versioning"))
4386 return new RGWSetBucketVersioning_ObjStore_S3
;
4387 if (s
->info
.args
.sub_resource_exists("website")) {
4388 if (!s
->cct
->_conf
->rgw_enable_static_website
) {
4391 return new RGWSetBucketWebsite_ObjStore_S3
;
4393 if (is_tagging_op()) {
4394 return new RGWPutBucketTags_ObjStore_S3
;
4395 } else if (is_acl_op()) {
4396 return new RGWPutACLs_ObjStore_S3
;
4397 } else if (is_cors_op()) {
4398 return new RGWPutCORS_ObjStore_S3
;
4399 } else if (is_request_payment_op()) {
4400 return new RGWSetRequestPayment_ObjStore_S3
;
4401 } else if(is_lc_op()) {
4402 return new RGWPutLC_ObjStore_S3
;
4403 } else if(is_policy_op()) {
4404 return new RGWPutBucketPolicy
;
4405 } else if (is_object_lock_op()) {
4406 return new RGWPutBucketObjectLock_ObjStore_S3
;
4407 } else if (is_notification_op()) {
4408 return RGWHandler_REST_PSNotifs_S3::create_put_op();
4409 } else if (is_replication_op()) {
4410 auto sync_policy_handler
= store
->svc()->zone
->get_sync_policy_handler(nullopt
);
4411 if (!sync_policy_handler
||
4412 sync_policy_handler
->is_legacy_config()) {
4416 return new RGWPutBucketReplication_ObjStore_S3
;
4417 } else if (is_block_public_access_op()) {
4418 return new RGWPutBucketPublicAccessBlock_ObjStore_S3
;
4420 return new RGWCreateBucket_ObjStore_S3
;
4423 RGWOp
*RGWHandler_REST_Bucket_S3::op_delete()
4425 if (is_tagging_op()) {
4426 return new RGWDeleteBucketTags_ObjStore_S3
;
4427 } else if (is_cors_op()) {
4428 return new RGWDeleteCORS_ObjStore_S3
;
4429 } else if(is_lc_op()) {
4430 return new RGWDeleteLC_ObjStore_S3
;
4431 } else if(is_policy_op()) {
4432 return new RGWDeleteBucketPolicy
;
4433 } else if (is_notification_op()) {
4434 return RGWHandler_REST_PSNotifs_S3::create_delete_op();
4435 } else if (is_replication_op()) {
4436 return new RGWDeleteBucketReplication_ObjStore_S3
;
4437 } else if (is_block_public_access_op()) {
4438 return new RGWDeleteBucketPublicAccessBlock
;
4441 if (s
->info
.args
.sub_resource_exists("website")) {
4442 if (!s
->cct
->_conf
->rgw_enable_static_website
) {
4445 return new RGWDeleteBucketWebsite_ObjStore_S3
;
4448 if (s
->info
.args
.exists("mdsearch")) {
4449 return new RGWDelBucketMetaSearch_ObjStore_S3
;
4452 return new RGWDeleteBucket_ObjStore_S3
;
4455 RGWOp
*RGWHandler_REST_Bucket_S3::op_post()
4457 if (s
->info
.args
.exists("delete")) {
4458 return new RGWDeleteMultiObj_ObjStore_S3
;
4461 if (s
->info
.args
.exists("mdsearch")) {
4462 return new RGWConfigBucketMetaSearch_ObjStore_S3
;
4465 return new RGWPostObj_ObjStore_S3
;
4468 RGWOp
*RGWHandler_REST_Bucket_S3::op_options()
4470 return new RGWOptionsCORS_ObjStore_S3
;
4473 RGWOp
*RGWHandler_REST_Obj_S3::get_obj_op(bool get_data
)
4475 RGWGetObj_ObjStore_S3
*get_obj_op
= new RGWGetObj_ObjStore_S3
;
4476 get_obj_op
->set_get_data(get_data
);
4480 RGWOp
*RGWHandler_REST_Obj_S3::op_get()
4483 return new RGWGetACLs_ObjStore_S3
;
4484 } else if (s
->info
.args
.exists("uploadId")) {
4485 return new RGWListMultipart_ObjStore_S3
;
4486 } else if (s
->info
.args
.exists("layout")) {
4487 return new RGWGetObjLayout_ObjStore_S3
;
4488 } else if (is_tagging_op()) {
4489 return new RGWGetObjTags_ObjStore_S3
;
4490 } else if (is_obj_retention_op()) {
4491 return new RGWGetObjRetention_ObjStore_S3
;
4492 } else if (is_obj_legal_hold_op()) {
4493 return new RGWGetObjLegalHold_ObjStore_S3
;
4495 return get_obj_op(true);
4498 RGWOp
*RGWHandler_REST_Obj_S3::op_head()
4501 return new RGWGetACLs_ObjStore_S3
;
4502 } else if (s
->info
.args
.exists("uploadId")) {
4503 return new RGWListMultipart_ObjStore_S3
;
4505 return get_obj_op(false);
4508 RGWOp
*RGWHandler_REST_Obj_S3::op_put()
4511 return new RGWPutACLs_ObjStore_S3
;
4512 } else if (is_tagging_op()) {
4513 return new RGWPutObjTags_ObjStore_S3
;
4514 } else if (is_obj_retention_op()) {
4515 return new RGWPutObjRetention_ObjStore_S3
;
4516 } else if (is_obj_legal_hold_op()) {
4517 return new RGWPutObjLegalHold_ObjStore_S3
;
4520 if (s
->init_state
.src_bucket
.empty())
4521 return new RGWPutObj_ObjStore_S3
;
4523 return new RGWCopyObj_ObjStore_S3
;
4526 RGWOp
*RGWHandler_REST_Obj_S3::op_delete()
4528 if (is_tagging_op()) {
4529 return new RGWDeleteObjTags_ObjStore_S3
;
4531 string upload_id
= s
->info
.args
.get("uploadId");
4533 if (upload_id
.empty())
4534 return new RGWDeleteObj_ObjStore_S3
;
4536 return new RGWAbortMultipart_ObjStore_S3
;
4539 RGWOp
*RGWHandler_REST_Obj_S3::op_post()
4541 if (s
->info
.args
.exists("uploadId"))
4542 return new RGWCompleteMultipart_ObjStore_S3
;
4544 if (s
->info
.args
.exists("uploads"))
4545 return new RGWInitMultipart_ObjStore_S3
;
4547 return new RGWPostObj_ObjStore_S3
;
4550 RGWOp
*RGWHandler_REST_Obj_S3::op_options()
4552 return new RGWOptionsCORS_ObjStore_S3
;
4555 int RGWHandler_REST_S3::init_from_header(struct req_state
* s
,
4556 int default_formatter
,
4557 bool configurable_format
)
4562 const char *req_name
= s
->relative_uri
.c_str();
4565 if (*req_name
== '?') {
4568 p
= s
->info
.request_params
.c_str();
4571 s
->info
.args
.set(p
);
4572 s
->info
.args
.parse();
4574 /* must be called after the args parsing */
4575 int ret
= allocate_formatter(s
, default_formatter
, configurable_format
);
4579 if (*req_name
!= '/')
4588 int pos
= req
.find('/');
4590 first
= req
.substr(0, pos
);
4596 * XXX The intent of the check for empty is apparently to let the bucket
4597 * name from DNS to be set ahead. However, we currently take the DNS
4598 * bucket and re-insert it into URL in rgw_rest.cc:RGWREST::preprocess().
4599 * So, this check is meaningless.
4601 * Rather than dropping this, the code needs to be changed into putting
4602 * the bucket (and its tenant) from DNS and Host: header (HTTP_HOST)
4603 * into req_status.bucket_name directly.
4605 if (s
->init_state
.url_bucket
.empty()) {
4606 // Save bucket to tide us over until token is parsed.
4607 s
->init_state
.url_bucket
= first
;
4609 string encoded_obj_str
= req
.substr(pos
+1);
4610 s
->object
= rgw_obj_key(encoded_obj_str
, s
->info
.args
.get("versionId"));
4613 s
->object
= rgw_obj_key(req_name
, s
->info
.args
.get("versionId"));
4618 static int verify_mfa(rgw::sal::RGWRadosStore
*store
, RGWUserInfo
*user
, const string
& mfa_str
, bool *verified
, const DoutPrefixProvider
*dpp
)
4620 vector
<string
> params
;
4621 get_str_vec(mfa_str
, " ", params
);
4623 if (params
.size() != 2) {
4624 ldpp_dout(dpp
, 5) << "NOTICE: invalid mfa string provided: " << mfa_str
<< dendl
;
4628 string
& serial
= params
[0];
4629 string
& pin
= params
[1];
4631 auto i
= user
->mfa_ids
.find(serial
);
4632 if (i
== user
->mfa_ids
.end()) {
4633 ldpp_dout(dpp
, 5) << "NOTICE: user does not have mfa device with serial=" << serial
<< dendl
;
4637 int ret
= store
->svc()->cls
->mfa
.check_mfa(user
->user_id
, serial
, pin
, null_yield
);
4639 ldpp_dout(dpp
, 20) << "NOTICE: failed to check MFA, serial=" << serial
<< dendl
;
4648 int RGWHandler_REST_S3::postauth_init()
4650 struct req_init_state
*t
= &s
->init_state
;
4652 rgw_parse_url_bucket(t
->url_bucket
, s
->user
->get_tenant(),
4653 s
->bucket_tenant
, s
->bucket_name
);
4655 dout(10) << "s->object=" << (!s
->object
.empty() ? s
->object
: rgw_obj_key("<NULL>"))
4656 << " s->bucket=" << rgw_make_bucket_entry_name(s
->bucket_tenant
, s
->bucket_name
) << dendl
;
4659 ret
= rgw_validate_tenant_name(s
->bucket_tenant
);
4662 if (!s
->bucket_name
.empty()) {
4663 ret
= validate_object_name(s
->object
.name
);
4668 if (!t
->src_bucket
.empty()) {
4669 rgw_parse_url_bucket(t
->src_bucket
, s
->user
->get_tenant(),
4670 s
->src_tenant_name
, s
->src_bucket_name
);
4671 ret
= rgw_validate_tenant_name(s
->src_tenant_name
);
4676 const char *mfa
= s
->info
.env
->get("HTTP_X_AMZ_MFA");
4678 ret
= verify_mfa(store
, &s
->user
->get_info(), string(mfa
), &s
->mfa_verified
, s
);
4684 int RGWHandler_REST_S3::init(rgw::sal::RGWRadosStore
*store
, struct req_state
*s
,
4685 rgw::io::BasicClient
*cio
)
4691 ret
= rgw_validate_tenant_name(s
->bucket_tenant
);
4694 if (!s
->bucket_name
.empty()) {
4695 ret
= validate_object_name(s
->object
.name
);
4700 const char *cacl
= s
->info
.env
->get("HTTP_X_AMZ_ACL");
4702 s
->canned_acl
= cacl
;
4704 s
->has_acl_header
= s
->info
.env
->exists_prefix("HTTP_X_AMZ_GRANT");
4706 const char *copy_source
= s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE");
4708 (! s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE_RANGE")) &&
4709 (! s
->info
.args
.exists("uploadId"))) {
4711 ret
= RGWCopyObj::parse_copy_location(copy_source
,
4712 s
->init_state
.src_bucket
,
4715 ldpp_dout(s
, 0) << "failed to parse copy location" << dendl
;
4716 return -EINVAL
; // XXX why not -ERR_INVALID_BUCKET_NAME or -ERR_BAD_URL?
4720 const char *sc
= s
->info
.env
->get("HTTP_X_AMZ_STORAGE_CLASS");
4722 s
->info
.storage_class
= sc
;
4725 return RGWHandler_REST::init(store
, s
, cio
);
4728 int RGWHandler_REST_S3::authorize(const DoutPrefixProvider
*dpp
)
4730 if (s
->info
.args
.exists("Action") && s
->info
.args
.get("Action") == "AssumeRoleWithWebIdentity") {
4731 return RGW_Auth_STS::authorize(dpp
, store
, auth_registry
, s
);
4733 return RGW_Auth_S3::authorize(dpp
, store
, auth_registry
, s
);
4736 enum class AwsVersion
{
4742 enum class AwsRoute
{
4748 static inline std::pair
<AwsVersion
, AwsRoute
>
4749 discover_aws_flavour(const req_info
& info
)
4751 using rgw::auth::s3::AWS4_HMAC_SHA256_STR
;
4753 AwsVersion version
= AwsVersion::UNKNOWN
;
4754 AwsRoute route
= AwsRoute::UNKNOWN
;
4756 const char* http_auth
= info
.env
->get("HTTP_AUTHORIZATION");
4757 if (http_auth
&& http_auth
[0]) {
4758 /* Authorization in Header */
4759 route
= AwsRoute::HEADERS
;
4761 if (!strncmp(http_auth
, AWS4_HMAC_SHA256_STR
,
4762 strlen(AWS4_HMAC_SHA256_STR
))) {
4764 version
= AwsVersion::V4
;
4765 } else if (!strncmp(http_auth
, "AWS ", 4)) {
4767 version
= AwsVersion::V2
;
4770 route
= AwsRoute::QUERY_STRING
;
4772 if (info
.args
.get("X-Amz-Algorithm") == AWS4_HMAC_SHA256_STR
) {
4774 version
= AwsVersion::V4
;
4775 } else if (!info
.args
.get("AWSAccessKeyId").empty()) {
4777 version
= AwsVersion::V2
;
4781 return std::make_pair(version
, route
);
4785 * verify that a signed request comes from the keyholder
4786 * by checking the signature against our locally-computed version
4788 * it tries AWS v4 before AWS v2
4790 int RGW_Auth_S3::authorize(const DoutPrefixProvider
*dpp
,
4791 rgw::sal::RGWRadosStore
* const store
,
4792 const rgw::auth::StrategyRegistry
& auth_registry
,
4793 struct req_state
* const s
)
4796 /* neither keystone and rados enabled; warn and exit! */
4797 if (!store
->ctx()->_conf
->rgw_s3_auth_use_rados
&&
4798 !store
->ctx()->_conf
->rgw_s3_auth_use_keystone
&&
4799 !store
->ctx()->_conf
->rgw_s3_auth_use_ldap
) {
4800 ldpp_dout(dpp
, 0) << "WARNING: no authorization backend enabled! Users will never authenticate." << dendl
;
4804 const auto ret
= rgw::auth::Strategy::apply(dpp
, auth_registry
.get_s3_main(), s
);
4806 /* Populate the owner info. */
4807 s
->owner
.set_id(s
->user
->get_id());
4808 s
->owner
.set_name(s
->user
->get_display_name());
4813 int RGWHandler_Auth_S3::init(rgw::sal::RGWRadosStore
*store
, struct req_state
*state
,
4814 rgw::io::BasicClient
*cio
)
4816 int ret
= RGWHandler_REST_S3::init_from_header(state
, RGW_FORMAT_JSON
,
4821 return RGWHandler_REST::init(store
, state
, cio
);
4824 RGWHandler_REST
* RGWRESTMgr_S3::get_handler(struct req_state
* const s
,
4825 const rgw::auth::StrategyRegistry
& auth_registry
,
4826 const std::string
& frontend_prefix
)
4828 bool is_s3website
= enable_s3website
&& (s
->prot_flags
& RGW_REST_WEBSITE
);
4830 RGWHandler_REST_S3::init_from_header(s
,
4831 is_s3website
? RGW_FORMAT_HTML
:
4832 RGW_FORMAT_XML
, true);
4836 RGWHandler_REST
* handler
;
4837 // TODO: Make this more readable
4839 if (s
->init_state
.url_bucket
.empty()) {
4840 handler
= new RGWHandler_REST_Service_S3Website(auth_registry
);
4841 } else if (s
->object
.empty()) {
4842 handler
= new RGWHandler_REST_Bucket_S3Website(auth_registry
);
4844 handler
= new RGWHandler_REST_Obj_S3Website(auth_registry
);
4847 if (s
->init_state
.url_bucket
.empty()) {
4848 handler
= new RGWHandler_REST_Service_S3(auth_registry
, enable_sts
, enable_iam
, enable_pubsub
);
4849 } else if (s
->object
.empty()) {
4850 handler
= new RGWHandler_REST_Bucket_S3(auth_registry
, enable_pubsub
);
4852 handler
= new RGWHandler_REST_Obj_S3(auth_registry
);
4856 ldpp_dout(s
, 20) << __func__
<< " handler=" << typeid(*handler
).name()
4861 bool RGWHandler_REST_S3Website::web_dir() const {
4862 std::string subdir_name
= url_decode(s
->object
.name
);
4864 if (subdir_name
.empty()) {
4866 } else if (subdir_name
.back() == '/') {
4867 subdir_name
.pop_back();
4870 rgw_obj
obj(s
->bucket
, subdir_name
);
4872 RGWObjectCtx
& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
4873 obj_ctx
.set_atomic(obj
);
4874 obj_ctx
.set_prefetch_data(obj
);
4876 RGWObjState
* state
= nullptr;
4877 if (store
->getRados()->get_obj_state(&obj_ctx
, s
->bucket_info
, obj
, &state
, false, s
->yield
) < 0) {
4880 if (! state
->exists
) {
4883 return state
->exists
;
4886 int RGWHandler_REST_S3Website::init(rgw::sal::RGWRadosStore
*store
, req_state
*s
,
4887 rgw::io::BasicClient
* cio
)
4889 // save the original object name before retarget() replaces it with the
4890 // result of get_effective_key(). the error_handler() needs the original
4891 // object name for redirect handling
4892 original_object_name
= s
->object
.name
;
4894 return RGWHandler_REST_S3::init(store
, s
, cio
);
4897 int RGWHandler_REST_S3Website::retarget(RGWOp
* op
, RGWOp
** new_op
) {
4899 ldpp_dout(s
, 10) << __func__
<< " Starting retarget" << dendl
;
4901 if (!(s
->prot_flags
& RGW_REST_WEBSITE
))
4904 int ret
= store
->getRados()->get_bucket_info(store
->svc(), s
->bucket_tenant
,
4905 s
->bucket_name
, s
->bucket_info
, NULL
,
4906 s
->yield
, &s
->bucket_attrs
);
4908 // TODO-FUTURE: if the bucket does not exist, maybe expose it here?
4909 return -ERR_NO_SUCH_BUCKET
;
4911 if (!s
->bucket_info
.has_website
) {
4912 // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here
4913 return -ERR_NO_SUCH_WEBSITE_CONFIGURATION
;
4916 rgw_obj_key new_obj
;
4917 bool get_res
= s
->bucket_info
.website_conf
.get_effective_key(s
->object
.name
, &new_obj
.name
, web_dir());
4919 s
->err
.message
= "The IndexDocument Suffix is not configurated or not well formed!";
4920 ldpp_dout(s
, 5) << s
->err
.message
<< dendl
;
4924 ldpp_dout(s
, 10) << "retarget get_effective_key " << s
->object
<< " -> "
4925 << new_obj
<< dendl
;
4927 RGWBWRoutingRule rrule
;
4928 bool should_redirect
=
4929 s
->bucket_info
.website_conf
.should_redirect(new_obj
.name
, 0, &rrule
);
4931 if (should_redirect
) {
4932 const string
& hostname
= s
->info
.env
->get("HTTP_HOST", "");
4933 const string
& protocol
=
4934 (s
->info
.env
->get("SERVER_PORT_SECURE") ? "https" : "http");
4935 int redirect_code
= 0;
4936 rrule
.apply_rule(protocol
, hostname
, s
->object
.name
, &s
->redirect
,
4938 // APply a custom HTTP response code
4939 if (redirect_code
> 0)
4940 s
->err
.http_ret
= redirect_code
; // Apply a custom HTTP response code
4941 ldpp_dout(s
, 10) << "retarget redirect code=" << redirect_code
4942 << " proto+host:" << protocol
<< "://" << hostname
4943 << " -> " << s
->redirect
<< dendl
;
4944 return -ERR_WEBSITE_REDIRECT
;
4948 * FIXME: if s->object != new_obj, drop op and create a new op to handle
4949 * operation. Or remove this comment if it's not applicable anymore
4952 s
->object
= new_obj
;
4957 RGWOp
* RGWHandler_REST_S3Website::op_get()
4959 return get_obj_op(true);
4962 RGWOp
* RGWHandler_REST_S3Website::op_head()
4964 return get_obj_op(false);
4967 int RGWHandler_REST_S3Website::serve_errordoc(int http_ret
, const string
& errordoc_key
) {
4969 s
->formatter
->reset(); /* Try to throw it all away */
4971 std::shared_ptr
<RGWGetObj_ObjStore_S3Website
> getop( static_cast<RGWGetObj_ObjStore_S3Website
*>(op_get()));
4972 if (getop
.get() == NULL
) {
4973 return -1; // Trigger double error handler
4975 getop
->init(store
, s
, this);
4976 getop
->range_str
= NULL
;
4977 getop
->if_mod
= NULL
;
4978 getop
->if_unmod
= NULL
;
4979 getop
->if_match
= NULL
;
4980 getop
->if_nomatch
= NULL
;
4981 s
->object
= errordoc_key
;
4983 ret
= init_permissions(getop
.get());
4985 ldpp_dout(s
, 20) << "serve_errordoc failed, init_permissions ret=" << ret
<< dendl
;
4986 return -1; // Trigger double error handler
4989 ret
= read_permissions(getop
.get());
4991 ldpp_dout(s
, 20) << "serve_errordoc failed, read_permissions ret=" << ret
<< dendl
;
4992 return -1; // Trigger double error handler
4996 getop
->set_custom_http_response(http_ret
);
4999 ret
= getop
->init_processing();
5001 ldpp_dout(s
, 20) << "serve_errordoc failed, init_processing ret=" << ret
<< dendl
;
5002 return -1; // Trigger double error handler
5005 ret
= getop
->verify_op_mask();
5007 ldpp_dout(s
, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret
<< dendl
;
5008 return -1; // Trigger double error handler
5011 ret
= getop
->verify_permission();
5013 ldpp_dout(s
, 20) << "serve_errordoc failed, verify_permission ret=" << ret
<< dendl
;
5014 return -1; // Trigger double error handler
5017 ret
= getop
->verify_params();
5019 ldpp_dout(s
, 20) << "serve_errordoc failed, verify_params ret=" << ret
<< dendl
;
5020 return -1; // Trigger double error handler
5023 // No going back now
5026 * FIXME Missing headers:
5027 * With a working errordoc, the s3 error fields are rendered as HTTP headers,
5028 * x-amz-error-code: NoSuchKey
5029 * x-amz-error-message: The specified key does not exist.
5030 * x-amz-error-detail-Key: foo
5038 int RGWHandler_REST_S3Website::error_handler(int err_no
,
5039 string
* error_content
) {
5040 int new_err_no
= -1;
5041 rgw_http_errors::const_iterator r
= rgw_http_s3_errors
.find(err_no
> 0 ? err_no
: -err_no
);
5042 int http_error_code
= -1;
5044 if (r
!= rgw_http_s3_errors
.end()) {
5045 http_error_code
= r
->second
.first
;
5047 ldpp_dout(s
, 10) << "RGWHandler_REST_S3Website::error_handler err_no=" << err_no
<< " http_ret=" << http_error_code
<< dendl
;
5049 RGWBWRoutingRule rrule
;
5050 bool should_redirect
=
5051 s
->bucket_info
.website_conf
.should_redirect(original_object_name
,
5052 http_error_code
, &rrule
);
5054 if (should_redirect
) {
5055 const string
& hostname
= s
->info
.env
->get("HTTP_HOST", "");
5056 const string
& protocol
=
5057 (s
->info
.env
->get("SERVER_PORT_SECURE") ? "https" : "http");
5058 int redirect_code
= 0;
5059 rrule
.apply_rule(protocol
, hostname
, original_object_name
,
5060 &s
->redirect
, &redirect_code
);
5061 // Apply a custom HTTP response code
5062 if (redirect_code
> 0)
5063 s
->err
.http_ret
= redirect_code
; // Apply a custom HTTP response code
5064 ldpp_dout(s
, 10) << "error handler redirect code=" << redirect_code
5065 << " proto+host:" << protocol
<< "://" << hostname
5066 << " -> " << s
->redirect
<< dendl
;
5067 return -ERR_WEBSITE_REDIRECT
;
5068 } else if (err_no
== -ERR_WEBSITE_REDIRECT
) {
5069 // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block
5070 // Do NOT fire the ErrorDoc handler
5071 } else if (!s
->bucket_info
.website_conf
.error_doc
.empty()) {
5072 /* This serves an entire page!
5073 On success, it will return zero, and no further content should be sent to the socket
5074 On failure, we need the double-error handler
5076 new_err_no
= RGWHandler_REST_S3Website::serve_errordoc(http_error_code
, s
->bucket_info
.website_conf
.error_doc
);
5077 if (new_err_no
&& new_err_no
!= -1) {
5078 err_no
= new_err_no
;
5081 ldpp_dout(s
, 20) << "No special error handling today!" << dendl
;
5087 RGWOp
* RGWHandler_REST_Obj_S3Website::get_obj_op(bool get_data
)
5089 /** If we are in website mode, then it is explicitly impossible to run GET or
5090 * HEAD on the actual directory. We must convert the request to run on the
5091 * suffix object instead!
5093 RGWGetObj_ObjStore_S3Website
* op
= new RGWGetObj_ObjStore_S3Website
;
5094 op
->set_get_data(get_data
);
5098 RGWOp
* RGWHandler_REST_Bucket_S3Website::get_obj_op(bool get_data
)
5100 /** If we are in website mode, then it is explicitly impossible to run GET or
5101 * HEAD on the actual directory. We must convert the request to run on the
5102 * suffix object instead!
5104 RGWGetObj_ObjStore_S3Website
* op
= new RGWGetObj_ObjStore_S3Website
;
5105 op
->set_get_data(get_data
);
5109 RGWOp
* RGWHandler_REST_Service_S3Website::get_obj_op(bool get_data
)
5111 /** If we are in website mode, then it is explicitly impossible to run GET or
5112 * HEAD on the actual directory. We must convert the request to run on the
5113 * suffix object instead!
5115 RGWGetObj_ObjStore_S3Website
* op
= new RGWGetObj_ObjStore_S3Website
;
5116 op
->set_get_data(get_data
);
5121 namespace rgw::auth::s3
{
5123 static rgw::auth::Completer::cmplptr_t
5124 null_completer_factory(const boost::optional
<std::string
>& secret_key
)
5130 AWSEngine::VersionAbstractor::auth_data_t
5131 AWSGeneralAbstractor::get_auth_data(const req_state
* const s
) const
5135 std::tie(version
, route
) = discover_aws_flavour(s
->info
);
5137 if (version
== AwsVersion::V2
) {
5138 return get_auth_data_v2(s
);
5139 } else if (version
== AwsVersion::V4
) {
5140 return get_auth_data_v4(s
, route
== AwsRoute::QUERY_STRING
);
5142 /* FIXME(rzarzynski): handle anon user. */
5147 boost::optional
<std::string
>
5148 AWSGeneralAbstractor::get_v4_canonical_headers(
5149 const req_info
& info
,
5150 const boost::string_view
& signedheaders
,
5151 const bool using_qs
) const
5153 return rgw::auth::s3::get_v4_canonical_headers(info
, signedheaders
,
5157 AWSEngine::VersionAbstractor::auth_data_t
5158 AWSGeneralAbstractor::get_auth_data_v4(const req_state
* const s
,
5159 const bool using_qs
) const
5161 boost::string_view access_key_id
;
5162 boost::string_view signed_hdrs
;
5164 boost::string_view date
;
5165 boost::string_view credential_scope
;
5166 boost::string_view client_signature
;
5167 boost::string_view session_token
;
5169 int ret
= rgw::auth::s3::parse_v4_credentials(s
->info
,
5181 /* craft canonical headers */
5182 boost::optional
<std::string
> canonical_headers
= \
5183 get_v4_canonical_headers(s
->info
, signed_hdrs
, using_qs
);
5184 if (canonical_headers
) {
5185 using sanitize
= rgw::crypt_sanitize::log_content
;
5186 ldpp_dout(s
, 10) << "canonical headers format = "
5187 << sanitize
{*canonical_headers
} << dendl
;
5192 bool is_non_s3_op
= false;
5193 if (s
->op_type
== RGW_STS_GET_SESSION_TOKEN
||
5194 s
->op_type
== RGW_STS_ASSUME_ROLE
||
5195 s
->op_type
== RGW_STS_ASSUME_ROLE_WEB_IDENTITY
||
5196 s
->op_type
== RGW_OP_CREATE_ROLE
||
5197 s
->op_type
== RGW_OP_DELETE_ROLE
||
5198 s
->op_type
== RGW_OP_GET_ROLE
||
5199 s
->op_type
== RGW_OP_MODIFY_ROLE
||
5200 s
->op_type
== RGW_OP_LIST_ROLES
||
5201 s
->op_type
== RGW_OP_PUT_ROLE_POLICY
||
5202 s
->op_type
== RGW_OP_GET_ROLE_POLICY
||
5203 s
->op_type
== RGW_OP_LIST_ROLE_POLICIES
||
5204 s
->op_type
== RGW_OP_DELETE_ROLE_POLICY
||
5205 s
->op_type
== RGW_OP_PUT_USER_POLICY
||
5206 s
->op_type
== RGW_OP_GET_USER_POLICY
||
5207 s
->op_type
== RGW_OP_LIST_USER_POLICIES
||
5208 s
->op_type
== RGW_OP_DELETE_USER_POLICY
) {
5209 is_non_s3_op
= true;
5212 const char* exp_payload_hash
= nullptr;
5213 string payload_hash
;
5215 //For non s3 ops, we need to calculate the payload hash
5216 payload_hash
= s
->info
.args
.get("PayloadHash");
5217 exp_payload_hash
= payload_hash
.c_str();
5219 /* Get the expected hash. */
5220 exp_payload_hash
= rgw::auth::s3::get_v4_exp_payload_hash(s
->info
);
5223 /* Craft canonical URI. Using std::move later so let it be non-const. */
5224 auto canonical_uri
= rgw::auth::s3::get_v4_canonical_uri(s
->info
);
5226 /* Craft canonical query string. std::moving later so non-const here. */
5227 auto canonical_qs
= rgw::auth::s3::get_v4_canonical_qs(s
->info
, using_qs
);
5229 /* Craft canonical request. */
5230 auto canonical_req_hash
= \
5231 rgw::auth::s3::get_v4_canon_req_hash(s
->cct
,
5233 std::move(canonical_uri
),
5234 std::move(canonical_qs
),
5235 std::move(*canonical_headers
),
5239 auto string_to_sign
= \
5240 rgw::auth::s3::get_v4_string_to_sign(s
->cct
,
5241 AWS4_HMAC_SHA256_STR
,
5244 std::move(canonical_req_hash
));
5246 const auto sig_factory
= std::bind(rgw::auth::s3::get_v4_signature
,
5248 std::placeholders::_1
,
5249 std::placeholders::_2
,
5250 std::placeholders::_3
);
5252 /* Requests authenticated with the Query Parameters are treated as unsigned.
5253 * From "Authenticating Requests: Using Query Parameters (AWS Signature
5256 * You don't include a payload hash in the Canonical Request, because
5257 * when you create a presigned URL, you don't know the payload content
5258 * because the URL is used to upload an arbitrary payload. Instead, you
5259 * use a constant string UNSIGNED-PAYLOAD.
5261 * This means we have absolutely no business in spawning completer. Both
5262 * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
5263 * by default. We don't need to change that. */
5264 if (is_v4_payload_unsigned(exp_payload_hash
) || is_v4_payload_empty(s
) || is_non_s3_op
) {
5269 std::move(string_to_sign
),
5271 null_completer_factory
5274 /* We're going to handle a signed payload. Be aware that even empty HTTP
5275 * body (no payload) requires verification:
5277 * The x-amz-content-sha256 header is required for all AWS Signature
5278 * Version 4 requests. It provides a hash of the request payload. If
5279 * there is no payload, you must provide the hash of an empty string. */
5280 if (!is_v4_payload_streamed(exp_payload_hash
)) {
5281 ldpp_dout(s
, 10) << "delaying v4 auth" << dendl
;
5283 /* payload in a single chunk */
5286 case RGW_OP_CREATE_BUCKET
:
5287 case RGW_OP_PUT_OBJ
:
5288 case RGW_OP_PUT_ACLS
:
5289 case RGW_OP_PUT_CORS
:
5290 case RGW_OP_INIT_MULTIPART
: // in case that Init Multipart uses CHUNK encoding
5291 case RGW_OP_COMPLETE_MULTIPART
:
5292 case RGW_OP_SET_BUCKET_VERSIONING
:
5293 case RGW_OP_DELETE_MULTI_OBJ
:
5294 case RGW_OP_ADMIN_SET_METADATA
:
5295 case RGW_OP_SET_BUCKET_WEBSITE
:
5296 case RGW_OP_PUT_BUCKET_POLICY
:
5297 case RGW_OP_PUT_OBJ_TAGGING
:
5298 case RGW_OP_PUT_BUCKET_TAGGING
:
5299 case RGW_OP_PUT_BUCKET_REPLICATION
:
5301 case RGW_OP_SET_REQUEST_PAYMENT
:
5302 case RGW_OP_PUBSUB_NOTIF_CREATE
:
5303 case RGW_OP_PUT_BUCKET_OBJ_LOCK
:
5304 case RGW_OP_PUT_OBJ_RETENTION
:
5305 case RGW_OP_PUT_OBJ_LEGAL_HOLD
:
5306 case RGW_STS_GET_SESSION_TOKEN
:
5307 case RGW_STS_ASSUME_ROLE
:
5308 case RGW_OP_PUT_BUCKET_PUBLIC_ACCESS_BLOCK
:
5309 case RGW_OP_GET_BUCKET_PUBLIC_ACCESS_BLOCK
:
5310 case RGW_OP_DELETE_BUCKET_PUBLIC_ACCESS_BLOCK
:
5313 dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl
;
5314 throw -ERR_NOT_IMPLEMENTED
;
5317 const auto cmpl_factory
= std::bind(AWSv4ComplSingle::create
,
5319 std::placeholders::_1
);
5324 std::move(string_to_sign
),
5329 /* IMHO "streamed" doesn't fit too good here. I would prefer to call
5330 * it "chunked" but let's be coherent with Amazon's terminology. */
5332 dout(10) << "body content detected in multiple chunks" << dendl
;
5334 /* payload in multiple chunks */
5338 case RGW_OP_PUT_OBJ
:
5341 dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl
;
5342 throw -ERR_NOT_IMPLEMENTED
;
5345 dout(10) << "aws4 seed signature ok... delaying v4 auth" << dendl
;
5347 /* In the case of streamed payload client sets the x-amz-content-sha256
5348 * to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" but uses "UNSIGNED-PAYLOAD"
5349 * when constructing the Canonical Request. */
5351 /* In the case of single-chunk upload client set the header's value is
5352 * coherent with the one used for Canonical Request crafting. */
5354 /* In the case of query string-based authentication there should be no
5355 * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
5357 const auto cmpl_factory
= std::bind(AWSv4ComplMulti::create
,
5362 std::placeholders::_1
);
5367 std::move(string_to_sign
),
5376 boost::optional
<std::string
>
5377 AWSGeneralBoto2Abstractor::get_v4_canonical_headers(
5378 const req_info
& info
,
5379 const boost::string_view
& signedheaders
,
5380 const bool using_qs
) const
5382 return rgw::auth::s3::get_v4_canonical_headers(info
, signedheaders
,
5387 AWSEngine::VersionAbstractor::auth_data_t
5388 AWSGeneralAbstractor::get_auth_data_v2(const req_state
* const s
) const
5390 boost::string_view access_key_id
;
5391 boost::string_view signature
;
5392 boost::string_view session_token
;
5395 const char* http_auth
= s
->info
.env
->get("HTTP_AUTHORIZATION");
5396 if (! http_auth
|| http_auth
[0] == '\0') {
5397 /* Credentials are provided in query string. We also need to verify
5398 * the "Expires" parameter now. */
5399 access_key_id
= s
->info
.args
.get("AWSAccessKeyId");
5400 signature
= s
->info
.args
.get("Signature");
5403 boost::string_view expires
= s
->info
.args
.get("Expires");
5404 if (expires
.empty()) {
5408 /* It looks we have the guarantee that expires is a null-terminated,
5409 * and thus string_view::data() can be safely used. */
5410 const time_t exp
= atoll(expires
.data());
5417 if (s
->info
.args
.exists("X-Amz-Security-Token")) {
5418 session_token
= s
->info
.args
.get("X-Amz-Security-Token");
5419 if (session_token
.size() == 0) {
5425 /* The "Authorization" HTTP header is being used. */
5426 const boost::string_view
auth_str(http_auth
+ strlen("AWS "));
5427 const size_t pos
= auth_str
.rfind(':');
5428 if (pos
!= boost::string_view::npos
) {
5429 access_key_id
= auth_str
.substr(0, pos
);
5430 signature
= auth_str
.substr(pos
+ 1);
5433 if (s
->info
.env
->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
5434 session_token
= s
->info
.env
->get("HTTP_X_AMZ_SECURITY_TOKEN");
5435 if (session_token
.size() == 0) {
5441 /* Let's canonize the HTTP headers that are covered by the AWS auth v2. */
5442 std::string string_to_sign
;
5443 utime_t header_time
;
5444 if (! rgw_create_s3_canonical_header(s
->info
, &header_time
, string_to_sign
,
5446 ldpp_dout(s
, 10) << "failed to create the canonized auth header\n"
5447 << rgw::crypt_sanitize::auth
{s
,string_to_sign
} << dendl
;
5451 ldpp_dout(s
, 10) << "string_to_sign:\n"
5452 << rgw::crypt_sanitize::auth
{s
,string_to_sign
} << dendl
;
5454 if (!qsr
&& !is_time_skew_ok(header_time
)) {
5455 throw -ERR_REQUEST_TIME_SKEWED
;
5459 std::move(access_key_id
),
5460 std::move(signature
),
5461 std::move(session_token
),
5462 std::move(string_to_sign
),
5463 rgw::auth::s3::get_v2_signature
,
5464 null_completer_factory
5469 AWSEngine::VersionAbstractor::auth_data_t
5470 AWSBrowserUploadAbstractor::get_auth_data_v2(const req_state
* const s
) const
5473 s
->auth
.s3_postobj_creds
.access_key
,
5474 s
->auth
.s3_postobj_creds
.signature
,
5475 s
->auth
.s3_postobj_creds
.x_amz_security_token
,
5476 s
->auth
.s3_postobj_creds
.encoded_policy
.to_str(),
5477 rgw::auth::s3::get_v2_signature
,
5478 null_completer_factory
5482 AWSEngine::VersionAbstractor::auth_data_t
5483 AWSBrowserUploadAbstractor::get_auth_data_v4(const req_state
* const s
) const
5485 const boost::string_view credential
= s
->auth
.s3_postobj_creds
.x_amz_credential
;
5487 /* grab access key id */
5488 const size_t pos
= credential
.find("/");
5489 const boost::string_view access_key_id
= credential
.substr(0, pos
);
5490 dout(10) << "access key id = " << access_key_id
<< dendl
;
5492 /* grab credential scope */
5493 const boost::string_view credential_scope
= credential
.substr(pos
+ 1);
5494 dout(10) << "credential scope = " << credential_scope
<< dendl
;
5496 const auto sig_factory
= std::bind(rgw::auth::s3::get_v4_signature
,
5498 std::placeholders::_1
,
5499 std::placeholders::_2
,
5500 std::placeholders::_3
);
5504 s
->auth
.s3_postobj_creds
.signature
,
5505 s
->auth
.s3_postobj_creds
.x_amz_security_token
,
5506 s
->auth
.s3_postobj_creds
.encoded_policy
.to_str(),
5508 null_completer_factory
5512 AWSEngine::VersionAbstractor::auth_data_t
5513 AWSBrowserUploadAbstractor::get_auth_data(const req_state
* const s
) const
5515 if (s
->auth
.s3_postobj_creds
.x_amz_algorithm
== AWS4_HMAC_SHA256_STR
) {
5516 ldpp_dout(s
, 0) << "Signature verification algorithm AWS v4"
5517 << " (AWS4-HMAC-SHA256)" << dendl
;
5518 return get_auth_data_v4(s
);
5520 ldpp_dout(s
, 0) << "Signature verification algorithm AWS v2" << dendl
;
5521 return get_auth_data_v2(s
);
5526 AWSEngine::authenticate(const DoutPrefixProvider
* dpp
, const req_state
* const s
) const
5528 /* Small reminder: an ver_abstractor is allowed to throw! */
5529 const auto auth_data
= ver_abstractor
.get_auth_data(s
);
5531 if (auth_data
.access_key_id
.empty() || auth_data
.client_signature
.empty()) {
5532 return result_t::deny(-EINVAL
);
5534 return authenticate(dpp
,
5535 auth_data
.access_key_id
,
5536 auth_data
.client_signature
,
5537 auth_data
.session_token
,
5538 auth_data
.string_to_sign
,
5539 auth_data
.signature_factory
,
5540 auth_data
.completer_factory
,
5545 } // namespace rgw::auth::s3
5547 rgw::LDAPHelper
* rgw::auth::s3::LDAPEngine::ldh
= nullptr;
5548 std::mutex
rgw::auth::s3::LDAPEngine::mtx
;
5550 void rgw::auth::s3::LDAPEngine::init(CephContext
* const cct
)
5552 if (! cct
->_conf
->rgw_s3_auth_use_ldap
||
5553 cct
->_conf
->rgw_ldap_uri
.empty()) {
5558 std::lock_guard
<std::mutex
> lck(mtx
);
5560 const string
& ldap_uri
= cct
->_conf
->rgw_ldap_uri
;
5561 const string
& ldap_binddn
= cct
->_conf
->rgw_ldap_binddn
;
5562 const string
& ldap_searchdn
= cct
->_conf
->rgw_ldap_searchdn
;
5563 const string
& ldap_searchfilter
= cct
->_conf
->rgw_ldap_searchfilter
;
5564 const string
& ldap_dnattr
= cct
->_conf
->rgw_ldap_dnattr
;
5565 std::string ldap_bindpw
= parse_rgw_ldap_bindpw(cct
);
5567 ldh
= new rgw::LDAPHelper(ldap_uri
, ldap_binddn
, ldap_bindpw
,
5568 ldap_searchdn
, ldap_searchfilter
, ldap_dnattr
);
5576 bool rgw::auth::s3::LDAPEngine::valid() {
5577 std::lock_guard
<std::mutex
> lck(mtx
);
5581 rgw::auth::RemoteApplier::acl_strategy_t
5582 rgw::auth::s3::LDAPEngine::get_acl_strategy() const
5584 //This is based on the assumption that the default acl strategy in
5585 // get_perms_from_aclspec, will take care. Extra acl spec is not required.
5589 rgw::auth::RemoteApplier::AuthInfo
5590 rgw::auth::s3::LDAPEngine::get_creds_info(const rgw::RGWToken
& token
) const noexcept
5592 /* The short form of "using" can't be used here -- we're aliasing a class'
5594 using acct_privilege_t
= \
5595 rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t
;
5597 return rgw::auth::RemoteApplier::AuthInfo
{
5600 RGW_PERM_FULL_CONTROL
,
5601 acct_privilege_t::IS_PLAIN_ACCT
,
5606 rgw::auth::Engine::result_t
5607 rgw::auth::s3::LDAPEngine::authenticate(
5608 const DoutPrefixProvider
* dpp
,
5609 const boost::string_view
& access_key_id
,
5610 const boost::string_view
& signature
,
5611 const boost::string_view
& session_token
,
5612 const string_to_sign_t
& string_to_sign
,
5613 const signature_factory_t
&,
5614 const completer_factory_t
& completer_factory
,
5615 const req_state
* const s
) const
5617 /* boost filters and/or string_ref may throw on invalid input */
5618 rgw::RGWToken base64_token
;
5620 base64_token
= rgw::from_base64(access_key_id
);
5622 base64_token
= std::string("");
5625 if (! base64_token
.valid()) {
5626 return result_t::deny();
5629 //TODO: Uncomment, when we have a migration plan in place.
5630 //Check if a user of type other than 'ldap' is already present, if yes, then
5632 /*RGWUserInfo user_info;
5633 user_info.user_id = base64_token.id;
5634 if (rgw_get_user_info_by_uid(store, user_info.user_id, user_info) >= 0) {
5635 if (user_info.type != TYPE_LDAP) {
5636 ldpp_dout(dpp, 10) << "ERROR: User id of type: " << user_info.type << " is already present" << dendl;
5641 if (ldh
->auth(base64_token
.id
, base64_token
.key
) != 0) {
5642 return result_t::deny(-ERR_INVALID_ACCESS_KEY
);
5645 auto apl
= apl_factory
->create_apl_remote(cct
, s
, get_acl_strategy(),
5646 get_creds_info(base64_token
));
5647 return result_t::grant(std::move(apl
), completer_factory(boost::none
));
5648 } /* rgw::auth::s3::LDAPEngine::authenticate */
5650 void rgw::auth::s3::LDAPEngine::shutdown() {
5658 rgw::auth::Engine::result_t
5659 rgw::auth::s3::LocalEngine::authenticate(
5660 const DoutPrefixProvider
* dpp
,
5661 const boost::string_view
& _access_key_id
,
5662 const boost::string_view
& signature
,
5663 const boost::string_view
& session_token
,
5664 const string_to_sign_t
& string_to_sign
,
5665 const signature_factory_t
& signature_factory
,
5666 const completer_factory_t
& completer_factory
,
5667 const req_state
* const s
) const
5669 /* get the user info */
5670 RGWUserInfo user_info
;
5671 /* TODO(rzarzynski): we need to have string-view taking variant. */
5672 const std::string access_key_id
= _access_key_id
.to_string();
5673 if (rgw_get_user_info_by_access_key(ctl
->user
, access_key_id
, user_info
) < 0) {
5674 ldpp_dout(dpp
, 5) << "error reading user info, uid=" << access_key_id
5675 << " can't authenticate" << dendl
;
5676 return result_t::deny(-ERR_INVALID_ACCESS_KEY
);
5678 //TODO: Uncomment, when we have a migration plan in place.
5680 if (s->user->type != TYPE_RGW) {
5681 ldpp_dout(dpp, 10) << "ERROR: User id of type: " << s->user->type
5682 << " is present" << dendl;
5687 const auto iter
= user_info
.access_keys
.find(access_key_id
);
5688 if (iter
== std::end(user_info
.access_keys
)) {
5689 ldpp_dout(dpp
, 0) << "ERROR: access key not encoded in user info" << dendl
;
5690 return result_t::deny(-EPERM
);
5692 const RGWAccessKey
& k
= iter
->second
;
5694 const VersionAbstractor::server_signature_t server_signature
= \
5695 signature_factory(cct
, k
.key
, string_to_sign
);
5696 auto compare
= signature
.compare(server_signature
);
5698 ldpp_dout(dpp
, 15) << "string_to_sign="
5699 << rgw::crypt_sanitize::log_content
{string_to_sign
}
5701 ldpp_dout(dpp
, 15) << "server signature=" << server_signature
<< dendl
;
5702 ldpp_dout(dpp
, 15) << "client signature=" << signature
<< dendl
;
5703 ldpp_dout(dpp
, 15) << "compare=" << compare
<< dendl
;
5706 return result_t::deny(-ERR_SIGNATURE_NO_MATCH
);
5709 auto apl
= apl_factory
->create_apl_local(cct
, s
, user_info
, k
.subuser
, boost::none
);
5710 return result_t::grant(std::move(apl
), completer_factory(k
.key
));
5713 rgw::auth::RemoteApplier::AuthInfo
5714 rgw::auth::s3::STSEngine::get_creds_info(const STS::SessionToken
& token
) const noexcept
5716 using acct_privilege_t
= \
5717 rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t
;
5719 return rgw::auth::RemoteApplier::AuthInfo
{
5723 (token
.is_admin
) ? acct_privilege_t::IS_ADMIN_ACCT
: acct_privilege_t::IS_PLAIN_ACCT
,
5729 rgw::auth::s3::STSEngine::get_session_token(const DoutPrefixProvider
* dpp
, const boost::string_view
& session_token
,
5730 STS::SessionToken
& token
) const
5732 string decodedSessionToken
;
5734 decodedSessionToken
= rgw::from_base64(session_token
);
5736 ldpp_dout(dpp
, 0) << "ERROR: Invalid session token, not base64 encoded." << dendl
;
5740 auto* cryptohandler
= cct
->get_crypto_handler(CEPH_CRYPTO_AES
);
5741 if (! cryptohandler
) {
5744 string secret_s
= cct
->_conf
->rgw_sts_key
;
5745 buffer::ptr
secret(secret_s
.c_str(), secret_s
.length());
5747 if (ret
= cryptohandler
->validate_secret(secret
); ret
< 0) {
5748 ldpp_dout(dpp
, 0) << "ERROR: Invalid secret key" << dendl
;
5752 auto* keyhandler
= cryptohandler
->get_key_handler(secret
, error
);
5758 string decrypted_str
;
5759 buffer::list en_input
, dec_output
;
5760 en_input
= buffer::list::static_from_string(decodedSessionToken
);
5762 ret
= keyhandler
->decrypt(en_input
, dec_output
, &error
);
5764 ldpp_dout(dpp
, 0) << "ERROR: Decryption failed: " << error
<< dendl
;
5768 dec_output
.append('\0');
5769 auto iter
= dec_output
.cbegin();
5770 decode(token
, iter
);
5771 } catch (const buffer::error
& e
) {
5772 ldout(cct
, 0) << "ERROR: decode SessionToken failed: " << error
<< dendl
;
5779 rgw::auth::Engine::result_t
5780 rgw::auth::s3::STSEngine::authenticate(
5781 const DoutPrefixProvider
* dpp
,
5782 const boost::string_view
& _access_key_id
,
5783 const boost::string_view
& signature
,
5784 const boost::string_view
& session_token
,
5785 const string_to_sign_t
& string_to_sign
,
5786 const signature_factory_t
& signature_factory
,
5787 const completer_factory_t
& completer_factory
,
5788 const req_state
* const s
) const
5790 if (! s
->info
.args
.exists("X-Amz-Security-Token") &&
5791 ! s
->info
.env
->exists("HTTP_X_AMZ_SECURITY_TOKEN") &&
5792 s
->auth
.s3_postobj_creds
.x_amz_security_token
.empty()) {
5793 return result_t::deny();
5796 STS::SessionToken token
;
5797 if (int ret
= get_session_token(dpp
, session_token
, token
); ret
< 0) {
5798 return result_t::reject(ret
);
5801 //Check if access key is not the same passed in by client
5802 if (token
.access_key_id
!= _access_key_id
) {
5803 ldpp_dout(dpp
, 0) << "Invalid access key" << dendl
;
5804 return result_t::reject(-EPERM
);
5806 //Check if the token has expired
5807 if (! token
.expiration
.empty()) {
5808 std::string expiration
= token
.expiration
;
5809 if (! expiration
.empty()) {
5810 boost::optional
<real_clock::time_point
> exp
= ceph::from_iso_8601(expiration
, false);
5812 real_clock::time_point now
= real_clock::now();
5814 ldpp_dout(dpp
, 0) << "ERROR: Token expired" << dendl
;
5815 return result_t::reject(-EPERM
);
5818 ldpp_dout(dpp
, 0) << "ERROR: Invalid expiration: " << expiration
<< dendl
;
5819 return result_t::reject(-EPERM
);
5823 //Check for signature mismatch
5824 const VersionAbstractor::server_signature_t server_signature
= \
5825 signature_factory(cct
, token
.secret_access_key
, string_to_sign
);
5826 auto compare
= signature
.compare(server_signature
);
5828 ldpp_dout(dpp
, 15) << "string_to_sign="
5829 << rgw::crypt_sanitize::log_content
{string_to_sign
}
5831 ldpp_dout(dpp
, 15) << "server signature=" << server_signature
<< dendl
;
5832 ldpp_dout(dpp
, 15) << "client signature=" << signature
<< dendl
;
5833 ldpp_dout(dpp
, 15) << "compare=" << compare
<< dendl
;
5836 return result_t::reject(-ERR_SIGNATURE_NO_MATCH
);
5839 // Get all the authorization info
5840 RGWUserInfo user_info
;
5842 vector
<string
> role_policies
;
5844 if (! token
.roleId
.empty()) {
5845 RGWRole
role(s
->cct
, ctl
, token
.roleId
);
5846 if (role
.get_by_id() < 0) {
5847 return result_t::deny(-EPERM
);
5849 vector
<string
> role_policy_names
= role
.get_role_policy_names();
5850 for (auto& policy_name
: role_policy_names
) {
5852 if (int ret
= role
.get_role_policy(policy_name
, perm_policy
); ret
== 0) {
5853 role_policies
.push_back(std::move(perm_policy
));
5856 if (! token
.policy
.empty()) {
5857 role_policies
.push_back(std::move(token
.policy
));
5859 // This is mostly needed to assign the owner of a bucket during its creation
5860 user_id
= token
.user
;
5861 role_name
= role
.get_name();
5864 if (! token
.user
.empty() && token
.acct_type
!= TYPE_ROLE
) {
5866 int ret
= rgw_get_user_info_by_uid(ctl
->user
, token
.user
, user_info
, NULL
);
5868 ldpp_dout(dpp
, 5) << "ERROR: failed reading user info: uid=" << token
.user
<< dendl
;
5869 return result_t::reject(-EPERM
);
5873 if (token
.acct_type
== TYPE_KEYSTONE
|| token
.acct_type
== TYPE_LDAP
) {
5874 auto apl
= remote_apl_factory
->create_apl_remote(cct
, s
, get_acl_strategy(),
5875 get_creds_info(token
));
5876 return result_t::grant(std::move(apl
), completer_factory(boost::none
));
5877 } else if (token
.acct_type
== TYPE_ROLE
) {
5878 auto apl
= role_apl_factory
->create_apl_role(cct
, s
, role_name
, user_id
, role_policies
);
5879 return result_t::grant(std::move(apl
), completer_factory(token
.secret_access_key
));
5880 } else { // This is for all local users of type TYPE_RGW or TYPE_NONE
5882 auto apl
= local_apl_factory
->create_apl_local(cct
, s
, user_info
, subuser
, token
.perm_mask
);
5883 return result_t::grant(std::move(apl
), completer_factory(token
.secret_access_key
));
5887 bool rgw::auth::s3::S3AnonymousEngine::is_applicable(
5890 if (s
->op
== OP_OPTIONS
) {
5896 std::tie(version
, route
) = discover_aws_flavour(s
->info
);
5898 return route
== AwsRoute::QUERY_STRING
&& version
== AwsVersion::UNKNOWN
;