1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
9 #include "common/ceph_crypto.h"
10 #include "common/split.h"
11 #include "common/Formatter.h"
12 #include "common/utf8.h"
13 #include "common/ceph_json.h"
14 #include "common/safe_io.h"
15 #include "common/errno.h"
16 #include "auth/Crypto.h"
17 #include <boost/algorithm/string.hpp>
18 #include <boost/algorithm/string/replace.hpp>
19 #include <boost/tokenizer.hpp>
20 #define BOOST_BIND_GLOBAL_PLACEHOLDERS
21 #include <s3select/include/s3select.h>
22 #undef BOOST_BIND_GLOBAL_PLACEHOLDERS
24 #include <liboath/oath.h>
27 #include "rgw_rest_s3.h"
28 #include "rgw_rest_s3website.h"
29 #include "rgw_rest_pubsub.h"
30 #include "rgw_auth_s3.h"
32 #include "rgw_policy_s3.h"
35 #include "rgw_cors_s3.h"
36 #include "rgw_tag_s3.h"
38 #include "rgw_client_io.h"
40 #include "rgw_keystone.h"
41 #include "rgw_auth_keystone.h"
42 #include "rgw_auth_registry.h"
44 #include "rgw_es_query.h"
46 #include <typeinfo> // for 'typeid'
49 #include "rgw_token.h"
50 #include "rgw_rest_role.h"
51 #include "rgw_crypt.h"
52 #include "rgw_crypt_sanitize.h"
53 #include "rgw_rest_user_policy.h"
55 #include "rgw_bucket_sync.h"
57 #include "services/svc_zone.h"
58 #include "services/svc_cls.h"
60 #include "include/ceph_assert.h"
62 #include "rgw_rest_sts.h"
63 #include "rgw_rest_iam.h"
65 #include "rgw_sal_rados.h"
67 #define dout_context g_ceph_context
68 #define dout_subsys ceph_subsys_rgw
71 using namespace ceph::crypto
;
75 void list_all_buckets_start(struct req_state
*s
)
77 s
->formatter
->open_array_section_in_ns("ListAllMyBucketsResult", XMLNS_AWS_S3
);
80 void list_all_buckets_end(struct req_state
*s
)
82 s
->formatter
->close_section();
85 void dump_bucket(struct req_state
*s
, rgw::sal::RGWBucket
& obj
)
87 s
->formatter
->open_object_section("Bucket");
88 s
->formatter
->dump_string("Name", obj
.get_name());
89 dump_time(s
, "CreationDate", &obj
.get_creation_time());
90 s
->formatter
->close_section();
93 void rgw_get_errno_s3(rgw_http_error
*e
, int err_no
)
95 rgw_http_errors::const_iterator r
= rgw_http_s3_errors
.find(err_no
);
97 if (r
!= rgw_http_s3_errors
.end()) {
98 e
->http_ret
= r
->second
.first
;
99 e
->s3_code
= r
->second
.second
;
102 e
->s3_code
= "UnknownError";
106 static inline std::string
get_s3_expiration_header(
108 const ceph::real_time
& mtime
)
110 return rgw::lc::s3_expiration_header(
111 s
, s
->object
->get_key(), s
->tagset
, mtime
, s
->bucket_attrs
);
114 static inline bool get_s3_multipart_abort_header(
115 struct req_state
* s
, const ceph::real_time
& mtime
,
116 ceph::real_time
& date
, std::string
& rule_id
)
118 return rgw::lc::s3_multipart_abort_header(
119 s
, s
->object
->get_key(), mtime
, s
->bucket_attrs
, date
, rule_id
);
122 struct response_attr_param
{
124 const char *http_attr
;
127 static struct response_attr_param resp_attr_params
[] = {
128 {"response-content-type", "Content-Type"},
129 {"response-content-language", "Content-Language"},
130 {"response-expires", "Expires"},
131 {"response-cache-control", "Cache-Control"},
132 {"response-content-disposition", "Content-Disposition"},
133 {"response-content-encoding", "Content-Encoding"},
137 int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
) {
138 map
<string
, bufferlist
>::iterator iter
;
139 iter
= attrs
.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION
);
140 if (iter
!= attrs
.end()) {
141 bufferlist
&bl
= iter
->second
;
142 s
->redirect
= bl
.c_str();
143 s
->err
.http_ret
= 301;
144 ldpp_dout(this, 20) << __CEPH_ASSERT_FUNCTION
<< " redirecting per x-amz-website-redirect-location=" << s
->redirect
<< dendl
;
145 op_ret
= -ERR_WEBSITE_REDIRECT
;
146 set_req_state_err(s
, op_ret
);
148 dump_content_length(s
, 0);
149 dump_redirect(s
, s
->redirect
);
153 return RGWGetObj_ObjStore_S3::send_response_data(bl
, bl_ofs
, bl_len
);
157 int RGWGetObj_ObjStore_S3Website::send_response_data_error(optional_yield y
)
159 return RGWGetObj_ObjStore_S3::send_response_data_error(y
);
162 int RGWGetObj_ObjStore_S3::get_params(optional_yield y
)
164 // for multisite sync requests, only read the slo manifest itself, rather than
165 // all of the data from its parts. the parts will sync as separate objects
166 skip_manifest
= s
->info
.args
.exists(RGW_SYS_PARAM_PREFIX
"sync-manifest");
168 // multisite sync requests should fetch encrypted data, along with the
169 // attributes needed to support decryption on the other zone
170 if (s
->system_request
) {
171 skip_decrypt
= s
->info
.args
.exists(RGW_SYS_PARAM_PREFIX
"skip-decrypt");
174 return RGWGetObj_ObjStore::get_params(y
);
177 int RGWGetObj_ObjStore_S3::send_response_data_error(optional_yield y
)
180 return send_response_data(bl
, 0 , 0);
184 int decode_attr_bl_single_value(map
<string
, bufferlist
>& attrs
, const char *attr_name
, T
*result
, T def_val
)
186 map
<string
, bufferlist
>::iterator iter
= attrs
.find(attr_name
);
187 if (iter
== attrs
.end()) {
191 bufferlist
& bl
= iter
->second
;
192 if (bl
.length() == 0) {
196 auto bliter
= bl
.cbegin();
198 decode(*result
, bliter
);
199 } catch (buffer::error
& err
) {
205 inline bool str_has_cntrl(const std::string s
) {
206 return std::any_of(s
.begin(), s
.end(), ::iscntrl
);
209 inline bool str_has_cntrl(const char* s
) {
211 return str_has_cntrl(_s
);
214 int RGWGetObj_ObjStore_S3::send_response_data(bufferlist
& bl
, off_t bl_ofs
,
217 const char *content_type
= NULL
;
218 string content_type_str
;
219 map
<string
, string
> response_attrs
;
220 map
<string
, string
>::iterator riter
;
221 bufferlist metadata_bl
;
223 string expires
= get_s3_expiration_header(s
, lastmod
);
228 if (custom_http_ret
) {
229 set_req_state_err(s
, 0);
230 dump_errno(s
, custom_http_ret
);
232 set_req_state_err(s
, (partial_content
&& !op_ret
) ? STATUS_PARTIAL_CONTENT
241 dump_range(s
, start
, end
, s
->obj_size
);
243 if (s
->system_request
&&
244 s
->info
.args
.exists(RGW_SYS_PARAM_PREFIX
"prepend-metadata")) {
246 dump_header(s
, "Rgwx-Object-Size", (long long)total_len
);
250 * in this case, we're not returning the object's content, only the prepended
256 /* JSON encode object metadata */
258 jf
.open_object_section("obj_metadata");
259 encode_json("attrs", attrs
, &jf
);
261 encode_json("mtime", ut
, &jf
);
265 metadata_bl
.append(ss
.str());
266 dump_header(s
, "Rgwx-Embedded-Metadata-Len", metadata_bl
.length());
267 total_len
+= metadata_bl
.length();
270 if (s
->system_request
&& !real_clock::is_zero(lastmod
)) {
271 /* we end up dumping mtime in two different methods, a bit redundant */
272 dump_epoch_header(s
, "Rgwx-Mtime", lastmod
);
274 int r
= decode_attr_bl_single_value(attrs
, RGW_ATTR_PG_VER
, &pg_ver
, (uint64_t)0);
276 ldpp_dout(this, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl
;
278 dump_header(s
, "Rgwx-Obj-PG-Ver", pg_ver
);
280 uint32_t source_zone_short_id
= 0;
281 r
= decode_attr_bl_single_value(attrs
, RGW_ATTR_SOURCE_ZONE
, &source_zone_short_id
, (uint32_t)0);
283 ldpp_dout(this, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl
;
285 if (source_zone_short_id
!= 0) {
286 dump_header(s
, "Rgwx-Source-Zone-Short-Id", source_zone_short_id
);
290 for (auto &it
: crypt_http_responses
)
291 dump_header(s
, it
.first
, it
.second
);
293 dump_content_length(s
, total_len
);
294 dump_last_modified(s
, lastmod
);
295 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
296 dump_header_if_nonempty(s
, "x-amz-expiration", expires
);
298 if (attrs
.find(RGW_ATTR_APPEND_PART_NUM
) != attrs
.end()) {
299 dump_header(s
, "x-rgw-object-type", "Appendable");
300 dump_header(s
, "x-rgw-next-append-position", s
->obj_size
);
302 dump_header(s
, "x-rgw-object-type", "Normal");
306 if (! lo_etag
.empty()) {
307 /* Handle etag of Swift API's large objects (DLO/SLO). It's entirerly
308 * legit to perform GET on them through S3 API. In such situation,
309 * a client should receive the composited content with corresponding
311 dump_etag(s
, lo_etag
);
313 auto iter
= attrs
.find(RGW_ATTR_ETAG
);
314 if (iter
!= attrs
.end()) {
315 dump_etag(s
, iter
->second
.to_str());
319 for (struct response_attr_param
*p
= resp_attr_params
; p
->param
; p
++) {
321 string val
= s
->info
.args
.get(p
->param
, &exists
);
323 /* reject unauthenticated response header manipulation, see
324 * https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html */
325 if (s
->auth
.identity
->is_anonymous()) {
326 return -ERR_INVALID_REQUEST
;
328 /* HTTP specification says no control characters should be present in
329 * header values: https://tools.ietf.org/html/rfc7230#section-3.2
330 * field-vchar = VCHAR / obs-text
332 * Failure to validate this permits a CRLF injection in HTTP headers,
333 * whereas S3 GetObject only permits specific headers.
335 if(str_has_cntrl(val
)) {
336 /* TODO: return a more distinct error in future;
337 * stating what the problem is */
338 return -ERR_INVALID_REQUEST
;
341 if (strcmp(p
->param
, "response-content-type") != 0) {
342 response_attrs
[p
->http_attr
] = val
;
344 content_type_str
= val
;
345 content_type
= content_type_str
.c_str();
350 for (auto iter
= attrs
.begin(); iter
!= attrs
.end(); ++iter
) {
351 const char *name
= iter
->first
.c_str();
352 map
<string
, string
>::iterator aiter
= rgw_to_http_attrs
.find(name
);
353 if (aiter
!= rgw_to_http_attrs
.end()) {
354 if (response_attrs
.count(aiter
->second
) == 0) {
355 /* Was not already overridden by a response param. */
357 size_t len
= iter
->second
.length();
358 string
s(iter
->second
.c_str(), len
);
359 while (len
&& !s
[len
- 1]) {
363 response_attrs
[aiter
->second
] = s
;
365 } else if (iter
->first
.compare(RGW_ATTR_CONTENT_TYPE
) == 0) {
366 /* Special handling for content_type. */
368 content_type_str
= rgw_bl_str(iter
->second
);
369 content_type
= content_type_str
.c_str();
371 } else if (strcmp(name
, RGW_ATTR_SLO_UINDICATOR
) == 0) {
372 // this attr has an extra length prefix from encode() in prior versions
373 dump_header(s
, "X-Object-Meta-Static-Large-Object", "True");
374 } else if (strncmp(name
, RGW_ATTR_META_PREFIX
,
375 sizeof(RGW_ATTR_META_PREFIX
)-1) == 0) {
376 /* User custom metadata. */
377 name
+= sizeof(RGW_ATTR_PREFIX
) - 1;
378 dump_header(s
, name
, iter
->second
);
379 } else if (iter
->first
.compare(RGW_ATTR_TAGS
) == 0) {
382 auto it
= iter
->second
.cbegin();
384 } catch (buffer::error
&err
) {
385 ldpp_dout(this,0) << "Error caught buffer::error couldn't decode TagSet " << dendl
;
387 dump_header(s
, RGW_AMZ_TAG_COUNT
, obj_tags
.count());
388 } else if (iter
->first
.compare(RGW_ATTR_OBJECT_RETENTION
) == 0 && get_retention
){
389 RGWObjectRetention retention
;
391 decode(retention
, iter
->second
);
392 dump_header(s
, "x-amz-object-lock-mode", retention
.get_mode());
393 dump_time_header(s
, "x-amz-object-lock-retain-until-date", retention
.get_retain_until_date());
394 } catch (buffer::error
& err
) {
395 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl
;
397 } else if (iter
->first
.compare(RGW_ATTR_OBJECT_LEGAL_HOLD
) == 0 && get_legal_hold
) {
398 RGWObjectLegalHold legal_hold
;
400 decode(legal_hold
, iter
->second
);
401 dump_header(s
, "x-amz-object-lock-legal-hold",legal_hold
.get_status());
402 } catch (buffer::error
& err
) {
403 ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectLegalHold" << dendl
;
410 for (riter
= response_attrs
.begin(); riter
!= response_attrs
.end();
412 dump_header(s
, riter
->first
, riter
->second
);
415 if (op_ret
== -ERR_NOT_MODIFIED
) {
419 content_type
= "binary/octet-stream";
421 end_header(s
, this, content_type
);
424 if (metadata_bl
.length()) {
425 dump_body(s
, metadata_bl
);
430 if (get_data
&& !op_ret
) {
431 int r
= dump_body(s
, bl
.c_str() + bl_ofs
, bl_len
);
439 int RGWGetObj_ObjStore_S3::get_decrypt_filter(std::unique_ptr
<RGWGetObj_Filter
> *filter
, RGWGetObj_Filter
* cb
, bufferlist
* manifest_bl
)
441 if (skip_decrypt
) { // bypass decryption for multisite sync requests
446 std::unique_ptr
<BlockCrypt
> block_crypt
;
447 res
= rgw_s3_prepare_decrypt(s
, attrs
, &block_crypt
, crypt_http_responses
);
449 if (block_crypt
!= nullptr) {
450 auto f
= std::make_unique
<RGWGetObj_BlockDecrypt
>(s
->cct
, cb
, std::move(block_crypt
));
451 if (manifest_bl
!= nullptr) {
452 res
= f
->read_manifest(this, *manifest_bl
);
454 *filter
= std::move(f
);
461 int RGWGetObj_ObjStore_S3::verify_requester(const rgw::auth::StrategyRegistry
& auth_registry
, optional_yield y
)
464 ret
= RGWOp::verify_requester(auth_registry
, y
);
465 if(!s
->user
->get_caps().check_cap("amz-cache", RGW_CAP_READ
) && !ret
&& s
->info
.env
->exists("HTTP_X_AMZ_CACHE"))
466 ret
= override_range_hdr(auth_registry
, y
);
470 int RGWGetObj_ObjStore_S3::override_range_hdr(const rgw::auth::StrategyRegistry
& auth_registry
, optional_yield y
)
473 ldpp_dout(this, 10) << "cache override headers" << dendl
;
474 RGWEnv
* rgw_env
= const_cast<RGWEnv
*>(s
->info
.env
);
475 const char* backup_range
= rgw_env
->get("HTTP_RANGE");
476 const char hdrs_split
[2] = {(char)178,'\0'};
477 const char kv_split
[2] = {(char)177,'\0'};
478 const char* cache_hdr
= rgw_env
->get("HTTP_X_AMZ_CACHE");
479 for (std::string_view hdr
: ceph::split(cache_hdr
, hdrs_split
)) {
480 auto kv
= ceph::split(hdr
, kv_split
);
482 if (std::distance(k
, kv
.end()) != 2) {
485 auto v
= std::next(k
);
486 std::string key
= "HTTP_";
488 boost::replace_all(key
, "-", "_");
489 rgw_env
->set(std::move(key
), std::string(*v
));
490 ldpp_dout(this, 10) << "after splitting cache kv key: " << key
<< " " << rgw_env
->get(key
.c_str()) << dendl
;
492 ret
= RGWOp::verify_requester(auth_registry
, y
);
493 if(!ret
&& backup_range
) {
494 rgw_env
->set("HTTP_RANGE",backup_range
);
496 rgw_env
->remove("HTTP_RANGE");
502 void RGWGetObjTags_ObjStore_S3::send_response_data(bufferlist
& bl
)
505 end_header(s
, this, "application/xml");
508 s
->formatter
->open_object_section_in_ns("Tagging", XMLNS_AWS_S3
);
509 s
->formatter
->open_object_section("TagSet");
511 RGWObjTagSet_S3 tagset
;
512 auto iter
= bl
.cbegin();
515 } catch (buffer::error
& err
) {
516 ldpp_dout(this,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl
;
520 tagset
.dump_xml(s
->formatter
);
522 s
->formatter
->close_section();
523 s
->formatter
->close_section();
524 rgw_flush_formatter_and_reset(s
, s
->formatter
);
528 int RGWPutObjTags_ObjStore_S3::get_params(optional_yield y
)
536 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
540 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
545 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
546 return -ERR_MALFORMED_XML
;
549 RGWObjTagging_S3 tagging
;
552 RGWXMLDecoder::decode_xml("Tagging", tagging
, &parser
);
553 } catch (RGWXMLDecoder::err
& err
) {
554 ldpp_dout(this, 5) << "Malformed tagging request: " << err
<< dendl
;
555 return -ERR_MALFORMED_XML
;
559 r
= tagging
.rebuild(obj_tags
);
563 obj_tags
.encode(tags_bl
);
564 ldpp_dout(this, 20) << "Read " << obj_tags
.count() << "tags" << dendl
;
569 void RGWPutObjTags_ObjStore_S3::send_response()
572 set_req_state_err(s
, op_ret
);
574 end_header(s
, this, "application/xml");
579 void RGWDeleteObjTags_ObjStore_S3::send_response()
585 r
= STATUS_NO_CONTENT
;
587 set_req_state_err(s
, r
);
592 void RGWGetBucketTags_ObjStore_S3::send_response_data(bufferlist
& bl
)
595 set_req_state_err(s
, op_ret
);
597 end_header(s
, this, "application/xml");
601 s
->formatter
->open_object_section_in_ns("Tagging", XMLNS_AWS_S3
);
602 s
->formatter
->open_object_section("TagSet");
604 RGWObjTagSet_S3 tagset
;
605 auto iter
= bl
.cbegin();
608 } catch (buffer::error
& err
) {
609 ldpp_dout(this,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl
;
613 tagset
.dump_xml(s
->formatter
);
615 s
->formatter
->close_section();
616 s
->formatter
->close_section();
617 rgw_flush_formatter_and_reset(s
, s
->formatter
);
621 int RGWPutBucketTags_ObjStore_S3::get_params(const DoutPrefixProvider
*dpp
, optional_yield y
)
629 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
633 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
638 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
639 return -ERR_MALFORMED_XML
;
642 RGWObjTagging_S3 tagging
;
644 RGWXMLDecoder::decode_xml("Tagging", tagging
, &parser
);
645 } catch (RGWXMLDecoder::err
& err
) {
647 ldpp_dout(dpp
, 5) << "Malformed tagging request: " << err
<< dendl
;
648 return -ERR_MALFORMED_XML
;
651 RGWObjTags
obj_tags(50); // A tag set can contain as many as 50 tags, or it can be empty.
652 r
= tagging
.rebuild(obj_tags
);
656 obj_tags
.encode(tags_bl
);
657 ldpp_dout(dpp
, 20) << "Read " << obj_tags
.count() << "tags" << dendl
;
659 // forward bucket tags requests to meta master zone
660 if (!store
->svc()->zone
->is_meta_master()) {
661 /* only need to keep this data around if we're not meta master */
662 in_data
= std::move(data
);
668 void RGWPutBucketTags_ObjStore_S3::send_response()
671 set_req_state_err(s
, op_ret
);
673 end_header(s
, this, "application/xml");
677 void RGWDeleteBucketTags_ObjStore_S3::send_response()
680 set_req_state_err(s
, op_ret
);
682 end_header(s
, this, "application/xml");
688 bool is_valid_status(const string
& s
) {
689 return (s
== "Enabled" ||
693 static string enabled_group_id
= "s3-bucket-replication:enabled";
694 static string disabled_group_id
= "s3-bucket-replication:disabled";
696 struct ReplicationConfiguration
{
700 struct DeleteMarkerReplication
{
703 void decode_xml(XMLObj
*obj
) {
704 RGWXMLDecoder::decode_xml("Status", status
, obj
);
707 void dump_xml(Formatter
*f
) const {
708 encode_xml("Status", status
, f
);
711 bool is_valid(CephContext
*cct
) const {
712 bool result
= is_valid_status(status
);
714 ldout(cct
, 5) << "NOTICE: bad status provided in DeleteMarkerReplication element (status=" << status
<< ")" << dendl
;
720 struct Source
{ /* rgw extension */
721 std::vector
<string
> zone_names
;
723 void decode_xml(XMLObj
*obj
) {
724 RGWXMLDecoder::decode_xml("Zone", zone_names
, obj
);
727 void dump_xml(Formatter
*f
) const {
728 encode_xml("Zone", zone_names
, f
);
733 struct AccessControlTranslation
{
736 void decode_xml(XMLObj
*obj
) {
737 RGWXMLDecoder::decode_xml("Owner", owner
, obj
);
739 void dump_xml(Formatter
*f
) const {
740 encode_xml("Owner", owner
, f
);
744 std::optional
<AccessControlTranslation
> acl_translation
;
745 std::optional
<string
> account
;
747 std::optional
<string
> storage_class
;
748 std::vector
<string
> zone_names
;
750 void decode_xml(XMLObj
*obj
) {
751 RGWXMLDecoder::decode_xml("AccessControlTranslation", acl_translation
, obj
);
752 RGWXMLDecoder::decode_xml("Account", account
, obj
);
753 if (account
&& account
->empty()) {
756 RGWXMLDecoder::decode_xml("Bucket", bucket
, obj
);
757 RGWXMLDecoder::decode_xml("StorageClass", storage_class
, obj
);
758 if (storage_class
&& storage_class
->empty()) {
759 storage_class
.reset();
761 RGWXMLDecoder::decode_xml("Zone", zone_names
, obj
); /* rgw extension */
764 void dump_xml(Formatter
*f
) const {
765 encode_xml("AccessControlTranslation", acl_translation
, f
);
766 encode_xml("Account", account
, f
);
767 encode_xml("Bucket", bucket
, f
);
768 encode_xml("StorageClass", storage_class
, f
);
769 encode_xml("Zone", zone_names
, f
);
779 return key
.empty() && value
.empty();
782 void decode_xml(XMLObj
*obj
) {
783 RGWXMLDecoder::decode_xml("Key", key
, obj
);
784 RGWXMLDecoder::decode_xml("Value", value
, obj
);
787 void dump_xml(Formatter
*f
) const {
788 encode_xml("Key", key
, f
);
789 encode_xml("Value", value
, f
);
794 std::optional
<string
> prefix
;
795 std::vector
<Tag
> tags
;
802 void decode_xml(XMLObj
*obj
) {
803 std::vector
<Tag
> _tags
;
804 RGWXMLDecoder::decode_xml("Prefix", prefix
, obj
);
805 if (prefix
&& prefix
->empty()) {
808 RGWXMLDecoder::decode_xml("Tag", _tags
, obj
);
809 for (auto& t
: _tags
) {
811 tags
.push_back(std::move(t
));
816 void dump_xml(Formatter
*f
) const {
817 encode_xml("Prefix", prefix
, f
);
818 encode_xml("Tag", tags
, f
);
822 std::optional
<string
> prefix
;
823 std::optional
<Tag
> tag
;
824 std::optional
<AndElements
> and_elements
;
827 return (!prefix
&& !tag
&& !and_elements
);
830 void decode_xml(XMLObj
*obj
) {
831 RGWXMLDecoder::decode_xml("Prefix", prefix
, obj
);
832 if (prefix
&& prefix
->empty()) {
835 RGWXMLDecoder::decode_xml("Tag", tag
, obj
);
836 if (tag
&& tag
->empty()) {
839 RGWXMLDecoder::decode_xml("And", and_elements
, obj
);
840 if (and_elements
&& and_elements
->empty()) {
841 and_elements
.reset();
845 void dump_xml(Formatter
*f
) const {
846 encode_xml("Prefix", prefix
, f
);
847 encode_xml("Tag", tag
, f
);
848 encode_xml("And", and_elements
, f
);
851 bool is_valid(CephContext
*cct
) const {
853 ldout(cct
, 5) << "NOTICE: both tag and prefix were provided in replication filter rule" << dendl
;
858 if (prefix
&& and_elements
->prefix
) {
859 ldout(cct
, 5) << "NOTICE: too many prefixes were provided in re" << dendl
;
866 int to_sync_pipe_filter(CephContext
*cct
,
867 rgw_sync_pipe_filter
*f
) const {
868 if (!is_valid(cct
)) {
875 f
->tags
.insert(rgw_sync_pipe_filter_tag(tag
->key
, tag
->value
));
879 if (and_elements
->prefix
) {
880 f
->prefix
= *and_elements
->prefix
;
882 for (auto& t
: and_elements
->tags
) {
883 f
->tags
.insert(rgw_sync_pipe_filter_tag(t
.key
, t
.value
));
889 void from_sync_pipe_filter(const rgw_sync_pipe_filter
& f
) {
890 if (f
.prefix
&& f
.tags
.empty()) {
895 and_elements
.emplace();
896 and_elements
->prefix
= f
.prefix
;
897 } else if (f
.tags
.size() == 1) {
898 auto iter
= f
.tags
.begin();
899 if (iter
== f
.tags
.end()) {
900 /* should never happen */
906 tag
->value
= t
.value
;
910 if (f
.tags
.empty()) {
915 and_elements
.emplace();
918 for (auto& t
: f
.tags
) {
919 auto& tag
= and_elements
->tags
.emplace_back();
926 set
<rgw_zone_id
> get_zone_ids_from_names(rgw::sal::RGWRadosStore
*store
,
927 const vector
<string
>& zone_names
) const {
928 set
<rgw_zone_id
> ids
;
930 for (auto& name
: zone_names
) {
932 if (store
->svc()->zone
->find_zone_id_by_name(name
, &id
)) {
933 ids
.insert(std::move(id
));
940 vector
<string
> get_zone_names_from_ids(rgw::sal::RGWRadosStore
*store
,
941 const set
<rgw_zone_id
>& zone_ids
) const {
942 vector
<string
> names
;
944 for (auto& id
: zone_ids
) {
946 if (store
->svc()->zone
->find_zone(id
, &zone
)) {
947 names
.emplace_back(zone
->name
);
954 std::optional
<DeleteMarkerReplication
> delete_marker_replication
;
955 std::optional
<Source
> source
;
956 Destination destination
;
957 std::optional
<Filter
> filter
;
962 void decode_xml(XMLObj
*obj
) {
963 RGWXMLDecoder::decode_xml("DeleteMarkerReplication", delete_marker_replication
, obj
);
964 RGWXMLDecoder::decode_xml("Source", source
, obj
);
965 RGWXMLDecoder::decode_xml("Destination", destination
, obj
);
966 RGWXMLDecoder::decode_xml("ID", id
, obj
);
968 std::optional
<string
> prefix
;
969 RGWXMLDecoder::decode_xml("Prefix", prefix
, obj
);
972 filter
->prefix
= prefix
;
976 RGWXMLDecoder::decode_xml("Filter", filter
, obj
);
978 /* don't want to have filter reset because it might have been initialized
979 * when decoding prefix
981 RGWXMLDecoder::decode_xml("Filter", *filter
, obj
);
984 RGWXMLDecoder::decode_xml("Priority", priority
, obj
);
985 RGWXMLDecoder::decode_xml("Status", status
, obj
);
988 void dump_xml(Formatter
*f
) const {
989 encode_xml("DeleteMarkerReplication", delete_marker_replication
, f
);
990 encode_xml("Source", source
, f
);
991 encode_xml("Destination", destination
, f
);
992 encode_xml("Filter", filter
, f
);
993 encode_xml("ID", id
, f
);
994 encode_xml("Priority", priority
, f
);
995 encode_xml("Status", status
, f
);
998 bool is_valid(CephContext
*cct
) const {
999 if (!is_valid_status(status
)) {
1000 ldout(cct
, 5) << "NOTICE: bad status provided in rule (status=" << status
<< ")" << dendl
;
1003 if ((filter
&& !filter
->is_valid(cct
)) ||
1004 (delete_marker_replication
&& !delete_marker_replication
->is_valid(cct
))) {
1010 int to_sync_policy_pipe(req_state
*s
, rgw::sal::RGWRadosStore
*store
,
1011 rgw_sync_bucket_pipes
*pipe
,
1012 bool *enabled
) const {
1013 if (!is_valid(s
->cct
)) {
1018 pipe
->params
.priority
= priority
;
1020 const auto& user_id
= s
->user
->get_id();
1022 rgw_bucket_key
dest_bk(user_id
.tenant
,
1023 destination
.bucket
);
1025 if (source
&& !source
->zone_names
.empty()) {
1026 pipe
->source
.zones
= get_zone_ids_from_names(store
, source
->zone_names
);
1028 pipe
->source
.set_all_zones(true);
1030 if (!destination
.zone_names
.empty()) {
1031 pipe
->dest
.zones
= get_zone_ids_from_names(store
, destination
.zone_names
);
1033 pipe
->dest
.set_all_zones(true);
1035 pipe
->dest
.bucket
.emplace(dest_bk
);
1038 int r
= filter
->to_sync_pipe_filter(s
->cct
, &pipe
->params
.source
.filter
);
1043 if (destination
.acl_translation
) {
1045 u
.tenant
= user_id
.tenant
;
1046 u
.from_str(destination
.acl_translation
->owner
); /* explicit tenant will override tenant,
1047 otherwise will inherit it from s->user */
1048 pipe
->params
.dest
.acl_translation
.emplace();
1049 pipe
->params
.dest
.acl_translation
->owner
= u
;
1051 pipe
->params
.dest
.storage_class
= destination
.storage_class
;
1053 *enabled
= (status
== "Enabled");
1055 pipe
->params
.mode
= rgw_sync_pipe_params::Mode::MODE_USER
;
1056 pipe
->params
.user
= user_id
.to_str();
1061 void from_sync_policy_pipe(rgw::sal::RGWRadosStore
*store
,
1062 const rgw_sync_bucket_pipes
& pipe
,
1065 status
= (enabled
? "Enabled" : "Disabled");
1066 priority
= pipe
.params
.priority
;
1068 if (pipe
.source
.all_zones
) {
1070 } else if (pipe
.source
.zones
) {
1072 source
->zone_names
= get_zone_names_from_ids(store
, *pipe
.source
.zones
);
1075 if (!pipe
.dest
.all_zones
&&
1077 destination
.zone_names
= get_zone_names_from_ids(store
, *pipe
.dest
.zones
);
1080 if (pipe
.params
.dest
.acl_translation
) {
1081 destination
.acl_translation
.emplace();
1082 destination
.acl_translation
->owner
= pipe
.params
.dest
.acl_translation
->owner
.to_str();
1085 if (pipe
.params
.dest
.storage_class
) {
1086 destination
.storage_class
= *pipe
.params
.dest
.storage_class
;
1089 if (pipe
.dest
.bucket
) {
1090 destination
.bucket
= pipe
.dest
.bucket
->get_key();
1094 filter
->from_sync_pipe_filter(pipe
.params
.source
.filter
);
1096 if (filter
->empty()) {
1102 std::vector
<Rule
> rules
;
1104 void decode_xml(XMLObj
*obj
) {
1105 RGWXMLDecoder::decode_xml("Role", role
, obj
);
1106 RGWXMLDecoder::decode_xml("Rule", rules
, obj
);
1109 void dump_xml(Formatter
*f
) const {
1110 encode_xml("Role", role
, f
);
1111 encode_xml("Rule", rules
, f
);
1114 int to_sync_policy_groups(req_state
*s
, rgw::sal::RGWRadosStore
*store
,
1115 vector
<rgw_sync_policy_group
> *result
) const {
1118 rgw_sync_policy_group
& enabled_group
= (*result
)[0];
1119 rgw_sync_policy_group
& disabled_group
= (*result
)[1];
1121 enabled_group
.id
= enabled_group_id
;
1122 enabled_group
.status
= rgw_sync_policy_group::Status::ENABLED
;
1123 disabled_group
.id
= disabled_group_id
;
1124 disabled_group
.status
= rgw_sync_policy_group::Status::ALLOWED
; /* not enabled, not forbidden */
1126 for (auto& rule
: rules
) {
1127 rgw_sync_bucket_pipes pipe
;
1129 int r
= rule
.to_sync_policy_pipe(s
, store
, &pipe
, &enabled
);
1131 ldpp_dout(s
, 5) << "NOTICE: failed to convert replication configuration into sync policy pipe (rule.id=" << rule
.id
<< "): " << cpp_strerror(-r
) << dendl
;
1136 enabled_group
.pipes
.emplace_back(std::move(pipe
));
1138 disabled_group
.pipes
.emplace_back(std::move(pipe
));
1144 void from_sync_policy_group(rgw::sal::RGWRadosStore
*store
,
1145 const rgw_sync_policy_group
& group
) {
1147 bool enabled
= (group
.status
== rgw_sync_policy_group::Status::ENABLED
);
1149 for (auto& pipe
: group
.pipes
) {
1150 auto& rule
= rules
.emplace_back();
1151 rule
.from_sync_policy_pipe(store
, pipe
, enabled
);
1158 void RGWGetBucketReplication_ObjStore_S3::send_response_data()
1161 set_req_state_err(s
, op_ret
);
1163 end_header(s
, this, "application/xml");
1166 ReplicationConfiguration conf
;
1168 if (s
->bucket
->get_info().sync_policy
) {
1169 auto policy
= s
->bucket
->get_info().sync_policy
;
1171 auto iter
= policy
->groups
.find(enabled_group_id
);
1172 if (iter
!= policy
->groups
.end()) {
1173 conf
.from_sync_policy_group(store
, iter
->second
);
1175 iter
= policy
->groups
.find(disabled_group_id
);
1176 if (iter
!= policy
->groups
.end()) {
1177 conf
.from_sync_policy_group(store
, iter
->second
);
1182 s
->formatter
->open_object_section_in_ns("ReplicationConfiguration", XMLNS_AWS_S3
);
1183 conf
.dump_xml(s
->formatter
);
1184 s
->formatter
->close_section();
1185 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1189 int RGWPutBucketReplication_ObjStore_S3::get_params(optional_yield y
)
1191 RGWXMLParser parser
;
1193 if (!parser
.init()){
1197 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
1201 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
1206 if (!parser
.parse(data
.c_str(), data
.length(), 1)) {
1207 return -ERR_MALFORMED_XML
;
1210 ReplicationConfiguration conf
;
1212 RGWXMLDecoder::decode_xml("ReplicationConfiguration", conf
, &parser
);
1213 } catch (RGWXMLDecoder::err
& err
) {
1215 ldpp_dout(this, 5) << "Malformed tagging request: " << err
<< dendl
;
1216 return -ERR_MALFORMED_XML
;
1219 r
= conf
.to_sync_policy_groups(s
, store
, &sync_policy_groups
);
1224 // forward requests to meta master zone
1225 if (!store
->svc()->zone
->is_meta_master()) {
1226 /* only need to keep this data around if we're not meta master */
1227 in_data
= std::move(data
);
1233 void RGWPutBucketReplication_ObjStore_S3::send_response()
1236 set_req_state_err(s
, op_ret
);
1238 end_header(s
, this, "application/xml");
1242 void RGWDeleteBucketReplication_ObjStore_S3::update_sync_policy(rgw_sync_policy_info
*policy
)
1244 policy
->groups
.erase(enabled_group_id
);
1245 policy
->groups
.erase(disabled_group_id
);
1248 void RGWDeleteBucketReplication_ObjStore_S3::send_response()
1251 set_req_state_err(s
, op_ret
);
1253 end_header(s
, this, "application/xml");
1257 void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets
)
1260 set_req_state_err(s
, op_ret
);
1263 // Explicitly use chunked transfer encoding so that we can stream the result
1264 // to the user without having to wait for the full length of it.
1265 end_header(s
, NULL
, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1268 list_all_buckets_start(s
);
1269 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
1270 s
->formatter
->open_array_section("Buckets");
1275 void RGWListBuckets_ObjStore_S3::send_response_data(rgw::sal::RGWBucketList
& buckets
)
1280 auto& m
= buckets
.get_buckets();
1282 for (auto iter
= m
.begin(); iter
!= m
.end(); ++iter
) {
1283 auto& bucket
= iter
->second
;
1284 dump_bucket(s
, *bucket
);
1286 rgw_flush_formatter(s
, s
->formatter
);
1289 void RGWListBuckets_ObjStore_S3::send_response_end()
1292 s
->formatter
->close_section();
1293 list_all_buckets_end(s
);
1294 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1298 int RGWGetUsage_ObjStore_S3::get_params(optional_yield y
)
1300 start_date
= s
->info
.args
.get("start-date");
1301 end_date
= s
->info
.args
.get("end-date");
1305 static void dump_usage_categories_info(Formatter
*formatter
, const rgw_usage_log_entry
& entry
, map
<string
, bool> *categories
)
1307 formatter
->open_array_section("categories");
1308 map
<string
, rgw_usage_data
>::const_iterator uiter
;
1309 for (uiter
= entry
.usage_map
.begin(); uiter
!= entry
.usage_map
.end(); ++uiter
) {
1310 if (categories
&& !categories
->empty() && !categories
->count(uiter
->first
))
1312 const rgw_usage_data
& usage
= uiter
->second
;
1313 formatter
->open_object_section("Entry");
1314 encode_json("Category", uiter
->first
, formatter
);
1315 encode_json("BytesSent", usage
.bytes_sent
, formatter
);
1316 encode_json("BytesReceived", usage
.bytes_received
, formatter
);
1317 encode_json("Ops", usage
.ops
, formatter
);
1318 encode_json("SuccessfulOps", usage
.successful_ops
, formatter
);
1319 formatter
->close_section(); // Entry
1321 formatter
->close_section(); // Category
1324 static void dump_usage_bucket_info(Formatter
*formatter
, const std::string
& name
, const cls_user_bucket_entry
& entry
)
1326 formatter
->open_object_section("Entry");
1327 encode_json("Bucket", name
, formatter
);
1328 encode_json("Bytes", entry
.size
, formatter
);
1329 encode_json("Bytes_Rounded", entry
.size_rounded
, formatter
);
1330 formatter
->close_section(); // entry
1333 void RGWGetUsage_ObjStore_S3::send_response()
1336 set_req_state_err(s
, op_ret
);
1339 // Explicitly use chunked transfer encoding so that we can stream the result
1340 // to the user without having to wait for the full length of it.
1341 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1346 Formatter
*formatter
= s
->formatter
;
1348 bool user_section_open
= false;
1350 formatter
->open_object_section("Usage");
1351 if (show_log_entries
) {
1352 formatter
->open_array_section("Entries");
1354 map
<rgw_user_bucket
, rgw_usage_log_entry
>::iterator iter
;
1355 for (iter
= usage
.begin(); iter
!= usage
.end(); ++iter
) {
1356 const rgw_user_bucket
& ub
= iter
->first
;
1357 const rgw_usage_log_entry
& entry
= iter
->second
;
1359 if (show_log_entries
) {
1360 if (ub
.user
.compare(last_owner
) != 0) {
1361 if (user_section_open
) {
1362 formatter
->close_section();
1363 formatter
->close_section();
1365 formatter
->open_object_section("User");
1366 formatter
->dump_string("Owner", ub
.user
);
1367 formatter
->open_array_section("Buckets");
1368 user_section_open
= true;
1369 last_owner
= ub
.user
;
1371 formatter
->open_object_section("Bucket");
1372 formatter
->dump_string("Bucket", ub
.bucket
);
1373 utime_t
ut(entry
.epoch
, 0);
1374 ut
.gmtime(formatter
->dump_stream("Time"));
1375 formatter
->dump_int("Epoch", entry
.epoch
);
1376 dump_usage_categories_info(formatter
, entry
, &categories
);
1377 formatter
->close_section(); // bucket
1380 summary_map
[ub
.user
].aggregate(entry
, &categories
);
1383 if (show_log_entries
) {
1384 if (user_section_open
) {
1385 formatter
->close_section(); // buckets
1386 formatter
->close_section(); //user
1388 formatter
->close_section(); // entries
1392 formatter
->open_array_section("Summary");
1393 map
<string
, rgw_usage_log_entry
>::iterator siter
;
1394 for (siter
= summary_map
.begin(); siter
!= summary_map
.end(); ++siter
) {
1395 const rgw_usage_log_entry
& entry
= siter
->second
;
1396 formatter
->open_object_section("User");
1397 formatter
->dump_string("User", siter
->first
);
1398 dump_usage_categories_info(formatter
, entry
, &categories
);
1399 rgw_usage_data total_usage
;
1400 entry
.sum(total_usage
, categories
);
1401 formatter
->open_object_section("Total");
1402 encode_json("BytesSent", total_usage
.bytes_sent
, formatter
);
1403 encode_json("BytesReceived", total_usage
.bytes_received
, formatter
);
1404 encode_json("Ops", total_usage
.ops
, formatter
);
1405 encode_json("SuccessfulOps", total_usage
.successful_ops
, formatter
);
1406 formatter
->close_section(); // total
1407 formatter
->close_section(); // user
1410 if (s
->cct
->_conf
->rgw_rest_getusage_op_compat
) {
1411 formatter
->open_object_section("Stats");
1414 // send info about quota config
1415 auto user_info
= s
->user
->get_info();
1416 encode_json("QuotaMaxBytes", user_info
.user_quota
.max_size
, formatter
);
1417 encode_json("QuotaMaxBuckets", user_info
.max_buckets
, formatter
);
1418 encode_json("QuotaMaxObjCount", user_info
.user_quota
.max_objects
, formatter
);
1419 encode_json("QuotaMaxBytesPerBucket", user_info
.bucket_quota
.max_objects
, formatter
);
1420 encode_json("QuotaMaxObjCountPerBucket", user_info
.bucket_quota
.max_size
, formatter
);
1421 // send info about user's capacity utilization
1422 encode_json("TotalBytes", stats
.size
, formatter
);
1423 encode_json("TotalBytesRounded", stats
.size_rounded
, formatter
);
1424 encode_json("TotalEntries", stats
.num_objects
, formatter
);
1426 if (s
->cct
->_conf
->rgw_rest_getusage_op_compat
) {
1427 formatter
->close_section(); //Stats
1430 formatter
->close_section(); // summary
1433 formatter
->open_array_section("CapacityUsed");
1434 formatter
->open_object_section("User");
1435 formatter
->open_array_section("Buckets");
1436 for (const auto& biter
: buckets_usage
) {
1437 const cls_user_bucket_entry
& entry
= biter
.second
;
1438 dump_usage_bucket_info(formatter
, biter
.first
, entry
);
1440 formatter
->close_section(); // Buckets
1441 formatter
->close_section(); // User
1442 formatter
->close_section(); // CapacityUsed
1444 formatter
->close_section(); // usage
1445 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1448 int RGWListBucket_ObjStore_S3::get_common_params()
1450 list_versions
= s
->info
.args
.exists("versions");
1451 prefix
= s
->info
.args
.get("prefix");
1454 s
->info
.args
.get_bool("allow-unordered", &allow_unordered
, false);
1455 delimiter
= s
->info
.args
.get("delimiter");
1456 max_keys
= s
->info
.args
.get("max-keys");
1457 op_ret
= parse_max_keys();
1461 encoding_type
= s
->info
.args
.get("encoding-type");
1462 if (s
->system_request
) {
1463 s
->info
.args
.get_bool("objs-container", &objs_container
, false);
1464 const char *shard_id_str
= s
->info
.env
->get("HTTP_RGWX_SHARD_ID");
1467 shard_id
= strict_strtol(shard_id_str
, 10, &err
);
1469 ldpp_dout(this, 5) << "bad shard id specified: " << shard_id_str
<< dendl
;
1473 shard_id
= s
->bucket_instance_shard_id
;
1479 int RGWListBucket_ObjStore_S3::get_params(optional_yield y
)
1481 int ret
= get_common_params();
1485 if (!list_versions
) {
1486 marker
= s
->info
.args
.get("marker");
1488 marker
.name
= s
->info
.args
.get("key-marker");
1489 marker
.instance
= s
->info
.args
.get("version-id-marker");
1494 int RGWListBucket_ObjStore_S3v2::get_params(optional_yield y
)
1496 int ret
= get_common_params();
1500 s
->info
.args
.get_bool("fetch-owner", &fetchOwner
, false);
1501 startAfter
= s
->info
.args
.get("start-after", &start_after_exist
);
1502 continuation_token
= s
->info
.args
.get("continuation-token", &continuation_token_exist
);
1503 if(!continuation_token_exist
) {
1504 marker
= startAfter
;
1506 marker
= continuation_token
;
1511 void RGWListBucket_ObjStore_S3::send_common_versioned_response()
1513 if (!s
->bucket_tenant
.empty()) {
1514 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
1516 s
->formatter
->dump_string("Name", s
->bucket_name
);
1517 s
->formatter
->dump_string("Prefix", prefix
);
1518 s
->formatter
->dump_int("MaxKeys", max
);
1519 if (!delimiter
.empty()) {
1520 s
->formatter
->dump_string("Delimiter", delimiter
);
1522 s
->formatter
->dump_string("IsTruncated", (max
&& is_truncated
? "true"
1525 if (!common_prefixes
.empty()) {
1526 map
<string
, bool>::iterator pref_iter
;
1527 for (pref_iter
= common_prefixes
.begin();
1528 pref_iter
!= common_prefixes
.end(); ++pref_iter
) {
1529 s
->formatter
->open_array_section("CommonPrefixes");
1531 s
->formatter
->dump_string("Prefix", url_encode(pref_iter
->first
, false));
1533 s
->formatter
->dump_string("Prefix", pref_iter
->first
);
1536 s
->formatter
->close_section();
1541 void RGWListBucket_ObjStore_S3::send_versioned_response()
1543 s
->formatter
->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3
);
1544 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1545 s
->formatter
->dump_string("EncodingType", "url");
1548 RGWListBucket_ObjStore_S3::send_common_versioned_response();
1549 s
->formatter
->dump_string("KeyMarker", marker
.name
);
1550 s
->formatter
->dump_string("VersionIdMarker", marker
.instance
);
1551 if (is_truncated
&& !next_marker
.empty()) {
1552 s
->formatter
->dump_string("NextKeyMarker", next_marker
.name
);
1553 if (next_marker
.instance
.empty()) {
1554 s
->formatter
->dump_string("NextVersionIdMarker", "null");
1557 s
->formatter
->dump_string("NextVersionIdMarker", next_marker
.instance
);
1562 if (objs_container
) {
1563 s
->formatter
->open_array_section("Entries");
1566 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1567 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1568 const char *section_name
= (iter
->is_delete_marker() ? "DeleteMarker"
1570 s
->formatter
->open_object_section(section_name
);
1571 if (objs_container
) {
1572 s
->formatter
->dump_bool("IsDeleteMarker", iter
->is_delete_marker());
1574 rgw_obj_key
key(iter
->key
);
1577 url_encode(key
.name
, key_name
);
1578 s
->formatter
->dump_string("Key", key_name
);
1581 s
->formatter
->dump_string("Key", key
.name
);
1583 string version_id
= key
.instance
;
1584 if (version_id
.empty()) {
1585 version_id
= "null";
1587 if (s
->system_request
) {
1588 if (iter
->versioned_epoch
> 0) {
1589 s
->formatter
->dump_int("VersionedEpoch", iter
->versioned_epoch
);
1591 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1592 utime_t
ut(iter
->meta
.mtime
);
1593 ut
.gmtime_nsec(s
->formatter
->dump_stream("RgwxMtime"));
1595 s
->formatter
->dump_string("VersionId", version_id
);
1596 s
->formatter
->dump_bool("IsLatest", iter
->is_current());
1597 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1598 if (!iter
->is_delete_marker()) {
1599 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1600 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1601 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1602 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1604 dump_owner(s
, rgw_user(iter
->meta
.owner
), iter
->meta
.owner_display_name
);
1605 if (iter
->meta
.appendable
) {
1606 s
->formatter
->dump_string("Type", "Appendable");
1608 s
->formatter
->dump_string("Type", "Normal");
1610 s
->formatter
->close_section(); // Version/DeleteMarker
1612 if (objs_container
) {
1613 s
->formatter
->close_section(); // Entries
1615 s
->formatter
->close_section(); // ListVersionsResult
1617 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1621 void RGWListBucket_ObjStore_S3::send_common_response()
1623 if (!s
->bucket_tenant
.empty()) {
1624 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
1626 s
->formatter
->dump_string("Name", s
->bucket_name
);
1627 s
->formatter
->dump_string("Prefix", prefix
);
1628 s
->formatter
->dump_int("MaxKeys", max
);
1629 if (!delimiter
.empty()) {
1630 s
->formatter
->dump_string("Delimiter", delimiter
);
1632 s
->formatter
->dump_string("IsTruncated", (max
&& is_truncated
? "true"
1635 if (!common_prefixes
.empty()) {
1636 map
<string
, bool>::iterator pref_iter
;
1637 for (pref_iter
= common_prefixes
.begin();
1638 pref_iter
!= common_prefixes
.end(); ++pref_iter
) {
1639 s
->formatter
->open_array_section("CommonPrefixes");
1641 s
->formatter
->dump_string("Prefix", url_encode(pref_iter
->first
, false));
1643 s
->formatter
->dump_string("Prefix", pref_iter
->first
);
1645 s
->formatter
->close_section();
1650 void RGWListBucket_ObjStore_S3::send_response()
1653 set_req_state_err(s
, op_ret
);
1657 // Explicitly use chunked transfer encoding so that we can stream the result
1658 // to the user without having to wait for the full length of it.
1659 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1664 if (list_versions
) {
1665 send_versioned_response();
1669 s
->formatter
->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3
);
1670 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1671 s
->formatter
->dump_string("EncodingType", "url");
1674 RGWListBucket_ObjStore_S3::send_common_response();
1676 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1677 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1678 rgw_obj_key
key(iter
->key
);
1679 s
->formatter
->open_array_section("Contents");
1682 url_encode(key
.name
, key_name
);
1683 s
->formatter
->dump_string("Key", key_name
);
1685 s
->formatter
->dump_string("Key", key
.name
);
1687 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1688 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1689 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1690 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1691 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1692 dump_owner(s
, rgw_user(iter
->meta
.owner
), iter
->meta
.owner_display_name
);
1693 if (s
->system_request
) {
1694 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1696 if (iter
->meta
.appendable
) {
1697 s
->formatter
->dump_string("Type", "Appendable");
1699 s
->formatter
->dump_string("Type", "Normal");
1701 s
->formatter
->close_section();
1704 s
->formatter
->dump_string("Marker", marker
.name
);
1705 if (is_truncated
&& !next_marker
.empty()) {
1706 s
->formatter
->dump_string("NextMarker", next_marker
.name
);
1708 s
->formatter
->close_section();
1709 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1712 void RGWListBucket_ObjStore_S3v2::send_versioned_response()
1714 s
->formatter
->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3
);
1715 RGWListBucket_ObjStore_S3v2::send_common_versioned_response();
1716 s
->formatter
->dump_string("KeyContinuationToken", marker
.name
);
1717 s
->formatter
->dump_string("VersionIdContinuationToken", marker
.instance
);
1718 if (is_truncated
&& !next_marker
.empty()) {
1719 s
->formatter
->dump_string("NextKeyContinuationToken", next_marker
.name
);
1720 s
->formatter
->dump_string("NextVersionIdContinuationToken", next_marker
.instance
);
1723 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1724 s
->formatter
->dump_string("EncodingType", "url");
1729 if (objs_container
) {
1730 s
->formatter
->open_array_section("Entries");
1733 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1734 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1735 const char *section_name
= (iter
->is_delete_marker() ? "DeleteContinuationToken"
1737 s
->formatter
->open_object_section(section_name
);
1738 if (objs_container
) {
1739 s
->formatter
->dump_bool("IsDeleteContinuationToken", iter
->is_delete_marker());
1741 rgw_obj_key
key(iter
->key
);
1744 url_encode(key
.name
, key_name
);
1745 s
->formatter
->dump_string("Key", key_name
);
1748 s
->formatter
->dump_string("Key", key
.name
);
1750 string version_id
= key
.instance
;
1751 if (version_id
.empty()) {
1752 version_id
= "null";
1754 if (s
->system_request
) {
1755 if (iter
->versioned_epoch
> 0) {
1756 s
->formatter
->dump_int("VersionedEpoch", iter
->versioned_epoch
);
1758 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1759 utime_t
ut(iter
->meta
.mtime
);
1760 ut
.gmtime_nsec(s
->formatter
->dump_stream("RgwxMtime"));
1762 s
->formatter
->dump_string("VersionId", version_id
);
1763 s
->formatter
->dump_bool("IsLatest", iter
->is_current());
1764 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1765 if (!iter
->is_delete_marker()) {
1766 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1767 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1768 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1769 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1771 if (fetchOwner
== true) {
1772 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
1774 s
->formatter
->close_section();
1778 if (objs_container
) {
1779 s
->formatter
->close_section();
1782 if (!common_prefixes
.empty()) {
1783 map
<string
, bool>::iterator pref_iter
;
1784 for (pref_iter
= common_prefixes
.begin();
1785 pref_iter
!= common_prefixes
.end(); ++pref_iter
) {
1786 s
->formatter
->open_array_section("CommonPrefixes");
1788 s
->formatter
->dump_string("Prefix", url_encode(pref_iter
->first
, false));
1790 s
->formatter
->dump_string("Prefix", pref_iter
->first
);
1793 s
->formatter
->dump_int("KeyCount",objs
.size());
1794 if (start_after_exist
) {
1795 s
->formatter
->dump_string("StartAfter", startAfter
);
1797 s
->formatter
->close_section();
1801 s
->formatter
->close_section();
1802 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1806 void RGWListBucket_ObjStore_S3v2::send_response()
1809 set_req_state_err(s
, op_ret
);
1813 // Explicitly use chunked transfer encoding so that we can stream the result
1814 // to the user without having to wait for the full length of it.
1815 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
1820 if (list_versions
) {
1821 send_versioned_response();
1825 s
->formatter
->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3
);
1826 if (strcasecmp(encoding_type
.c_str(), "url") == 0) {
1827 s
->formatter
->dump_string("EncodingType", "url");
1831 RGWListBucket_ObjStore_S3::send_common_response();
1833 vector
<rgw_bucket_dir_entry
>::iterator iter
;
1834 for (iter
= objs
.begin(); iter
!= objs
.end(); ++iter
) {
1835 rgw_obj_key
key(iter
->key
);
1836 s
->formatter
->open_array_section("Contents");
1839 url_encode(key
.name
, key_name
);
1840 s
->formatter
->dump_string("Key", key_name
);
1843 s
->formatter
->dump_string("Key", key
.name
);
1845 dump_time(s
, "LastModified", &iter
->meta
.mtime
);
1846 s
->formatter
->dump_format("ETag", "\"%s\"", iter
->meta
.etag
.c_str());
1847 s
->formatter
->dump_int("Size", iter
->meta
.accounted_size
);
1848 auto& storage_class
= rgw_placement_rule::get_canonical_storage_class(iter
->meta
.storage_class
);
1849 s
->formatter
->dump_string("StorageClass", storage_class
.c_str());
1850 if (fetchOwner
== true) {
1851 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
1853 if (s
->system_request
) {
1854 s
->formatter
->dump_string("RgwxTag", iter
->tag
);
1856 if (iter
->meta
.appendable
) {
1857 s
->formatter
->dump_string("Type", "Appendable");
1859 s
->formatter
->dump_string("Type", "Normal");
1861 s
->formatter
->close_section();
1864 if (continuation_token_exist
) {
1865 s
->formatter
->dump_string("ContinuationToken", continuation_token
);
1867 if (is_truncated
&& !next_marker
.empty()) {
1868 s
->formatter
->dump_string("NextContinuationToken", next_marker
.name
);
1870 s
->formatter
->dump_int("KeyCount", objs
.size() + common_prefixes
.size());
1871 if (start_after_exist
) {
1872 s
->formatter
->dump_string("StartAfter", startAfter
);
1874 s
->formatter
->close_section();
1875 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1878 void RGWGetBucketLogging_ObjStore_S3::send_response()
1881 end_header(s
, this, "application/xml");
1884 s
->formatter
->open_object_section_in_ns("BucketLoggingStatus", XMLNS_AWS_S3
);
1885 s
->formatter
->close_section();
1886 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1889 void RGWGetBucketLocation_ObjStore_S3::send_response()
1892 end_header(s
, this);
1895 RGWZoneGroup zonegroup
;
1898 int ret
= store
->svc()->zone
->get_zonegroup(s
->bucket
->get_info().zonegroup
, zonegroup
);
1900 api_name
= zonegroup
.api_name
;
1902 if (s
->bucket
->get_info().zonegroup
!= "default") {
1903 api_name
= s
->bucket
->get_info().zonegroup
;
1907 s
->formatter
->dump_format_ns("LocationConstraint", XMLNS_AWS_S3
,
1908 "%s", api_name
.c_str());
1909 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1912 void RGWGetBucketVersioning_ObjStore_S3::send_response()
1915 set_req_state_err(s
, op_ret
);
1917 end_header(s
, this, "application/xml");
1920 s
->formatter
->open_object_section_in_ns("VersioningConfiguration", XMLNS_AWS_S3
);
1922 const char *status
= (versioning_enabled
? "Enabled" : "Suspended");
1923 s
->formatter
->dump_string("Status", status
);
1924 const char *mfa_status
= (mfa_enabled
? "Enabled" : "Disabled");
1925 s
->formatter
->dump_string("MfaDelete", mfa_status
);
1927 s
->formatter
->close_section();
1928 rgw_flush_formatter_and_reset(s
, s
->formatter
);
1931 struct ver_config_status
{
1932 int status
{VersioningSuspended
};
1938 } mfa_status
{MFA_UNKNOWN
};
1941 void decode_xml(XMLObj
*obj
) {
1944 RGWXMLDecoder::decode_xml("Status", status_str
, obj
);
1945 if (status_str
== "Enabled") {
1946 status
= VersioningEnabled
;
1947 } else if (status_str
!= "Suspended") {
1948 status
= VersioningStatusInvalid
;
1952 if (RGWXMLDecoder::decode_xml("MfaDelete", mfa_str
, obj
)) {
1953 if (mfa_str
== "Enabled") {
1954 mfa_status
= MFA_ENABLED
;
1955 } else if (mfa_str
== "Disabled") {
1956 mfa_status
= MFA_DISABLED
;
1964 int RGWSetBucketVersioning_ObjStore_S3::get_params(optional_yield y
)
1969 rgw_rest_read_all_input(s
, s
->cct
->_conf
->rgw_max_put_param_size
, false);
1974 r
= do_aws4_auth_completion();
1979 RGWXMLDecoder::XMLParser parser
;
1980 if (!parser
.init()) {
1981 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
1985 char* buf
= data
.c_str();
1986 if (!parser
.parse(buf
, data
.length(), 1)) {
1987 ldpp_dout(this, 10) << "NOTICE: failed to parse data: " << buf
<< dendl
;
1992 ver_config_status status_conf
;
1994 if (!RGWXMLDecoder::decode_xml("VersioningConfiguration", status_conf
, &parser
)) {
1995 ldpp_dout(this, 10) << "NOTICE: bad versioning config input" << dendl
;
1999 if (!store
->svc()->zone
->is_meta_master()) {
2000 /* only need to keep this data around if we're not meta master */
2001 in_data
.append(data
);
2004 versioning_status
= status_conf
.status
;
2005 if (versioning_status
== VersioningStatusInvalid
) {
2009 if (status_conf
.mfa_status
!= ver_config_status::MFA_UNKNOWN
) {
2010 mfa_set_status
= true;
2011 switch (status_conf
.mfa_status
) {
2012 case ver_config_status::MFA_DISABLED
:
2015 case ver_config_status::MFA_ENABLED
:
2019 ldpp_dout(this, 0) << "ERROR: RGWSetBucketVersioning_ObjStore_S3::get_params(optional_yield y): unexpected switch case mfa_status=" << status_conf
.mfa_status
<< dendl
;
2022 } else if (status_conf
.retcode
< 0) {
2023 r
= status_conf
.retcode
;
2028 void RGWSetBucketVersioning_ObjStore_S3::send_response()
2031 set_req_state_err(s
, op_ret
);
2033 end_header(s
, this, "application/xml");
2036 int RGWSetBucketWebsite_ObjStore_S3::get_params(optional_yield y
)
2038 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
2042 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
2048 r
= do_aws4_auth_completion();
2053 in_data
.append(data
);
2055 RGWXMLDecoder::XMLParser parser
;
2056 if (!parser
.init()) {
2057 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
2061 char* buf
= data
.c_str();
2062 if (!parser
.parse(buf
, data
.length(), 1)) {
2063 ldpp_dout(this, 5) << "failed to parse xml: " << buf
<< dendl
;
2068 RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf
, &parser
, true);
2069 } catch (RGWXMLDecoder::err
& err
) {
2070 ldpp_dout(this, 5) << "unexpected xml: " << buf
<< dendl
;
2074 if (website_conf
.is_redirect_all
&& website_conf
.redirect_all
.hostname
.empty()) {
2075 s
->err
.message
= "A host name must be provided to redirect all requests (e.g. \"example.com\").";
2076 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
2078 } else if (!website_conf
.is_redirect_all
&& !website_conf
.is_set_index_doc
) {
2079 s
->err
.message
= "A value for IndexDocument Suffix must be provided if RedirectAllRequestsTo is empty";
2080 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
2082 } else if (!website_conf
.is_redirect_all
&& website_conf
.is_set_index_doc
&&
2083 website_conf
.index_doc_suffix
.empty()) {
2084 s
->err
.message
= "The IndexDocument Suffix is not well formed";
2085 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
2089 #define WEBSITE_ROUTING_RULES_MAX_NUM 50
2090 int max_num
= s
->cct
->_conf
->rgw_website_routing_rules_max_num
;
2092 max_num
= WEBSITE_ROUTING_RULES_MAX_NUM
;
2094 int routing_rules_num
= website_conf
.routing_rules
.rules
.size();
2095 if (routing_rules_num
> max_num
) {
2096 ldpp_dout(this, 4) << "An website routing config can have up to "
2098 << " rules, request website routing rules num: "
2099 << routing_rules_num
<< dendl
;
2100 op_ret
= -ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR
;
2101 s
->err
.message
= std::to_string(routing_rules_num
) +" routing rules provided, the number of routing rules in a website configuration is limited to "
2102 + std::to_string(max_num
)
2104 return -ERR_INVALID_REQUEST
;
2110 void RGWSetBucketWebsite_ObjStore_S3::send_response()
2113 set_req_state_err(s
, op_ret
);
2115 end_header(s
, this, "application/xml");
2118 void RGWDeleteBucketWebsite_ObjStore_S3::send_response()
2121 op_ret
= STATUS_NO_CONTENT
;
2123 set_req_state_err(s
, op_ret
);
2125 end_header(s
, this, "application/xml");
2128 void RGWGetBucketWebsite_ObjStore_S3::send_response()
2131 set_req_state_err(s
, op_ret
);
2133 end_header(s
, this, "application/xml");
2140 RGWBucketWebsiteConf
& conf
= s
->bucket
->get_info().website_conf
;
2142 s
->formatter
->open_object_section_in_ns("WebsiteConfiguration", XMLNS_AWS_S3
);
2143 conf
.dump_xml(s
->formatter
);
2144 s
->formatter
->close_section(); // WebsiteConfiguration
2145 rgw_flush_formatter_and_reset(s
, s
->formatter
);
2148 static void dump_bucket_metadata(struct req_state
*s
, rgw::sal::RGWBucket
* bucket
)
2150 dump_header(s
, "X-RGW-Object-Count", static_cast<long long>(bucket
->get_count()));
2151 dump_header(s
, "X-RGW-Bytes-Used", static_cast<long long>(bucket
->get_size()));
2152 // only bucket's owner is allowed to get the quota settings of the account
2153 if (bucket
->is_owner(s
->user
.get())) {
2154 auto user_info
= s
->user
->get_info();
2155 dump_header(s
, "X-RGW-Quota-User-Size", static_cast<long long>(user_info
.user_quota
.max_size
));
2156 dump_header(s
, "X-RGW-Quota-User-Objects", static_cast<long long>(user_info
.user_quota
.max_objects
));
2157 dump_header(s
, "X-RGW-Quota-Max-Buckets", static_cast<long long>(user_info
.max_buckets
));
2158 dump_header(s
, "X-RGW-Quota-Bucket-Size", static_cast<long long>(user_info
.bucket_quota
.max_size
));
2159 dump_header(s
, "X-RGW-Quota-Bucket-Objects", static_cast<long long>(user_info
.bucket_quota
.max_objects
));
2163 void RGWStatBucket_ObjStore_S3::send_response()
2166 dump_bucket_metadata(s
, bucket
.get());
2169 set_req_state_err(s
, op_ret
);
2172 end_header(s
, this);
2176 static int create_s3_policy(struct req_state
*s
, rgw::sal::RGWRadosStore
*store
,
2177 RGWAccessControlPolicy_S3
& s3policy
,
2180 if (s
->has_acl_header
) {
2181 if (!s
->canned_acl
.empty())
2182 return -ERR_INVALID_REQUEST
;
2184 return s3policy
.create_from_headers(s
, store
->ctl()->user
, s
->info
.env
, owner
);
2187 return s3policy
.create_canned(owner
, s
->bucket_owner
, s
->canned_acl
);
2190 class RGWLocationConstraint
: public XMLObj
2193 RGWLocationConstraint() {}
2194 ~RGWLocationConstraint() override
{}
2195 bool xml_end(const char *el
) override
{
2199 location_constraint
= get_data();
2204 string location_constraint
;
2207 class RGWCreateBucketConfig
: public XMLObj
2210 RGWCreateBucketConfig() {}
2211 ~RGWCreateBucketConfig() override
{}
2214 class RGWCreateBucketParser
: public RGWXMLParser
2216 XMLObj
*alloc_obj(const char *el
) override
{
2221 RGWCreateBucketParser() {}
2222 ~RGWCreateBucketParser() override
{}
2224 bool get_location_constraint(string
& zone_group
) {
2225 XMLObj
*config
= find_first("CreateBucketConfiguration");
2229 XMLObj
*constraint
= config
->find_first("LocationConstraint");
2233 zone_group
= constraint
->get_data();
2239 int RGWCreateBucket_ObjStore_S3::get_params(optional_yield y
)
2241 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
2242 bool relaxed_names
= s
->cct
->_conf
->rgw_relaxed_s3_bucket_names
;
2244 int r
= valid_s3_bucket_name(s
->bucket_name
, relaxed_names
);
2248 r
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
2254 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
2258 std::tie(op_ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
2260 if ((op_ret
< 0) && (op_ret
!= -ERR_LENGTH_REQUIRED
))
2263 const int auth_ret
= do_aws4_auth_completion();
2268 in_data
.append(data
);
2270 if (data
.length()) {
2271 RGWCreateBucketParser parser
;
2273 if (!parser
.init()) {
2274 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
2278 char* buf
= data
.c_str();
2279 bool success
= parser
.parse(buf
, data
.length(), 1);
2280 ldpp_dout(this, 20) << "create bucket input data=" << buf
<< dendl
;
2283 ldpp_dout(this, 0) << "failed to parse input: " << buf
<< dendl
;
2287 if (!parser
.get_location_constraint(location_constraint
)) {
2288 ldpp_dout(this, 0) << "provided input did not specify location constraint correctly" << dendl
;
2292 ldpp_dout(this, 10) << "create bucket location constraint: "
2293 << location_constraint
<< dendl
;
2296 size_t pos
= location_constraint
.find(':');
2297 if (pos
!= string::npos
) {
2298 placement_rule
.init(location_constraint
.substr(pos
+ 1), s
->info
.storage_class
);
2299 location_constraint
= location_constraint
.substr(0, pos
);
2301 placement_rule
.storage_class
= s
->info
.storage_class
;
2303 auto iter
= s
->info
.x_meta_map
.find("x-amz-bucket-object-lock-enabled");
2304 if (iter
!= s
->info
.x_meta_map
.end()) {
2305 if (!boost::algorithm::iequals(iter
->second
, "true") && !boost::algorithm::iequals(iter
->second
, "false")) {
2308 obj_lock_enabled
= boost::algorithm::iequals(iter
->second
, "true");
2313 void RGWCreateBucket_ObjStore_S3::send_response()
2315 if (op_ret
== -ERR_BUCKET_EXISTS
)
2318 set_req_state_err(s
, op_ret
);
2325 if (s
->system_request
) {
2326 JSONFormatter f
; /* use json formatter for system requests output */
2328 f
.open_object_section("info");
2329 encode_json("entry_point_object_ver", ep_objv
, &f
);
2330 encode_json("object_ver", info
.objv_tracker
.read_version
, &f
);
2331 encode_json("bucket_info", info
, &f
);
2333 rgw_flush_formatter_and_reset(s
, &f
);
2337 void RGWDeleteBucket_ObjStore_S3::send_response()
2341 r
= STATUS_NO_CONTENT
;
2343 set_req_state_err(s
, r
);
2345 end_header(s
, this);
2348 static inline void map_qs_metadata(struct req_state
* s
)
2350 /* merge S3 valid user metadata from the query-string into
2351 * x_meta_map, which maps them to attributes */
2352 const auto& params
= const_cast<RGWHTTPArgs
&>(s
->info
.args
).get_params();
2353 for (const auto& elt
: params
) {
2354 std::string k
= boost::algorithm::to_lower_copy(elt
.first
);
2355 if (k
.find("x-amz-meta-") == /* offset */ 0) {
2356 rgw_add_amz_meta_header(s
->info
.x_meta_map
, k
, elt
.second
);
2361 int RGWPutObj_ObjStore_S3::get_params(optional_yield y
)
2364 return -ERR_LENGTH_REQUIRED
;
2370 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
2371 ret
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
2377 if_match
= s
->info
.env
->get("HTTP_IF_MATCH");
2378 if_nomatch
= s
->info
.env
->get("HTTP_IF_NONE_MATCH");
2380 /* handle object tagging */
2381 auto tag_str
= s
->info
.env
->get("HTTP_X_AMZ_TAGGING");
2383 obj_tags
= std::make_unique
<RGWObjTags
>();
2384 ret
= obj_tags
->set_from_string(tag_str
);
2386 ldpp_dout(this,0) << "setting obj tags failed with " << ret
<< dendl
;
2387 if (ret
== -ERR_INVALID_TAG
){
2388 ret
= -EINVAL
; //s3 returns only -EINVAL for PUT requests
2395 //handle object lock
2396 auto obj_lock_mode_str
= s
->info
.env
->get("HTTP_X_AMZ_OBJECT_LOCK_MODE");
2397 auto obj_lock_date_str
= s
->info
.env
->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE");
2398 auto obj_legal_hold_str
= s
->info
.env
->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD");
2399 if (obj_lock_mode_str
&& obj_lock_date_str
) {
2400 boost::optional
<ceph::real_time
> date
= ceph::from_iso_8601(obj_lock_date_str
);
2401 if (boost::none
== date
|| ceph::real_clock::to_time_t(*date
) <= ceph_clock_now()) {
2403 ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl
;
2406 if (strcmp(obj_lock_mode_str
, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str
, "COMPLIANCE") != 0) {
2408 ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl
;
2411 obj_retention
= new RGWObjectRetention(obj_lock_mode_str
, *date
);
2412 } else if ((obj_lock_mode_str
&& !obj_lock_date_str
) || (!obj_lock_mode_str
&& obj_lock_date_str
)) {
2414 ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl
;
2417 if (obj_legal_hold_str
) {
2418 if (strcmp(obj_legal_hold_str
, "ON") != 0 && strcmp(obj_legal_hold_str
, "OFF") != 0) {
2420 ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl
;
2423 obj_legal_hold
= new RGWObjectLegalHold(obj_legal_hold_str
);
2425 if (!s
->bucket
->get_info().obj_lock_enabled() && (obj_retention
|| obj_legal_hold
)) {
2426 ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl
;
2427 ret
= -ERR_INVALID_REQUEST
;
2430 multipart_upload_id
= s
->info
.args
.get("uploadId");
2431 multipart_part_str
= s
->info
.args
.get("partNumber");
2432 if (!multipart_part_str
.empty()) {
2434 multipart_part_num
= strict_strtol(multipart_part_str
.c_str(), 10, &err
);
2436 ldpp_dout(s
, 10) << "bad part number: " << multipart_part_str
<< ": " << err
<< dendl
;
2439 } else if (!multipart_upload_id
.empty()) {
2440 ldpp_dout(s
, 10) << "part number with no multipart upload id" << dendl
;
2444 append
= s
->info
.args
.exists("append");
2446 string pos_str
= s
->info
.args
.get("position");
2448 long long pos_tmp
= strict_strtoll(pos_str
.c_str(), 10, &err
);
2450 ldpp_dout(s
, 10) << "bad position: " << pos_str
<< ": " << err
<< dendl
;
2452 } else if (pos_tmp
< 0) {
2453 ldpp_dout(s
, 10) << "bad position: " << pos_str
<< ": " << "position shouldn't be negative" << dendl
;
2456 position
= uint64_t(pos_tmp
);
2459 return RGWPutObj_ObjStore::get_params(y
);
2462 int RGWPutObj_ObjStore_S3::get_data(bufferlist
& bl
)
2464 const int ret
= RGWPutObj_ObjStore::get_data(bl
);
2466 const int ret_auth
= do_aws4_auth_completion();
2475 static int get_success_retcode(int code
)
2479 return STATUS_CREATED
;
2481 return STATUS_NO_CONTENT
;
2486 void RGWPutObj_ObjStore_S3::send_response()
2489 set_req_state_err(s
, op_ret
);
2492 if (s
->cct
->_conf
->rgw_s3_success_create_obj_status
) {
2493 op_ret
= get_success_retcode(
2494 s
->cct
->_conf
->rgw_s3_success_create_obj_status
);
2495 set_req_state_err(s
, op_ret
);
2498 string expires
= get_s3_expiration_header(s
, mtime
);
2500 if (copy_source
.empty()) {
2503 dump_content_length(s
, 0);
2504 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
2505 dump_header_if_nonempty(s
, "x-amz-expiration", expires
);
2506 for (auto &it
: crypt_http_responses
)
2507 dump_header(s
, it
.first
, it
.second
);
2510 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
2511 dump_header_if_nonempty(s
, "x-amz-expiration", expires
);
2512 end_header(s
, this, "application/xml");
2516 time_t secs
= (time_t)ut
.sec();
2517 gmtime_r(&secs
, &tmp
);
2518 char buf
[TIME_BUF_SIZE
];
2519 s
->formatter
->open_object_section_in_ns("CopyPartResult",
2520 "http://s3.amazonaws.com/doc/2006-03-01/");
2521 if (strftime(buf
, sizeof(buf
), "%Y-%m-%dT%T.000Z", &tmp
) > 0) {
2522 s
->formatter
->dump_string("LastModified", buf
);
2524 s
->formatter
->dump_string("ETag", etag
);
2525 s
->formatter
->close_section();
2526 rgw_flush_formatter_and_reset(s
, s
->formatter
);
2531 if (op_ret
== 0 || op_ret
== -ERR_POSITION_NOT_EQUAL_TO_LENGTH
) {
2532 dump_header(s
, "x-rgw-next-append-position", cur_accounted_size
);
2535 if (s
->system_request
&& !real_clock::is_zero(mtime
)) {
2536 dump_epoch_header(s
, "Rgwx-Mtime", mtime
);
2538 end_header(s
, this);
2541 static inline int get_obj_attrs(rgw::sal::RGWRadosStore
*store
, struct req_state
*s
, rgw_obj
& obj
, map
<string
, bufferlist
>& attrs
)
2543 RGWRados::Object
op_target(store
->getRados(), s
->bucket
->get_info(), *static_cast<RGWObjectCtx
*>(s
->obj_ctx
), obj
);
2544 RGWRados::Object::Read
read_op(&op_target
);
2546 read_op
.params
.attrs
= &attrs
;
2548 return read_op
.prepare(s
->yield
, s
);
2551 static inline void set_attr(map
<string
, bufferlist
>& attrs
, const char* key
, const std::string
& value
)
2555 attrs
.emplace(key
, std::move(bl
));
2558 static inline void set_attr(map
<string
, bufferlist
>& attrs
, const char* key
, const char* value
)
2562 attrs
.emplace(key
, std::move(bl
));
2565 int RGWPutObj_ObjStore_S3::get_decrypt_filter(
2566 std::unique_ptr
<RGWGetObj_Filter
>* filter
,
2567 RGWGetObj_Filter
* cb
,
2568 map
<string
, bufferlist
>& attrs
,
2569 bufferlist
* manifest_bl
)
2571 std::map
<std::string
, std::string
> crypt_http_responses_unused
;
2574 std::unique_ptr
<BlockCrypt
> block_crypt
;
2575 res
= rgw_s3_prepare_decrypt(s
, attrs
, &block_crypt
, crypt_http_responses_unused
);
2577 if (block_crypt
!= nullptr) {
2578 auto f
= std::unique_ptr
<RGWGetObj_BlockDecrypt
>(new RGWGetObj_BlockDecrypt(s
->cct
, cb
, std::move(block_crypt
)));
2579 //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
2581 if (manifest_bl
!= nullptr) {
2582 res
= f
->read_manifest(this, *manifest_bl
);
2584 *filter
= std::move(f
);
2593 int RGWPutObj_ObjStore_S3::get_encrypt_filter(
2594 std::unique_ptr
<rgw::putobj::DataProcessor
> *filter
,
2595 rgw::putobj::DataProcessor
*cb
)
2598 if (!multipart_upload_id
.empty()) {
2599 RGWMPObj
mp(s
->object
->get_name(), multipart_upload_id
);
2601 obj
.init_ns(s
->bucket
->get_key(), mp
.get_meta(), RGW_OBJ_NS_MULTIPART
);
2602 obj
.set_in_extra_data(true);
2603 map
<string
, bufferlist
> xattrs
;
2604 res
= get_obj_attrs(store
, s
, obj
, xattrs
);
2606 std::unique_ptr
<BlockCrypt
> block_crypt
;
2607 /* We are adding to existing object.
2608 * We use crypto mode that configured as if we were decrypting. */
2609 res
= rgw_s3_prepare_decrypt(s
, xattrs
, &block_crypt
, crypt_http_responses
);
2610 if (res
== 0 && block_crypt
!= nullptr)
2611 filter
->reset(new RGWPutObj_BlockEncrypt(s
->cct
, cb
, std::move(block_crypt
)));
2613 /* it is ok, to not have encryption at all */
2617 std::unique_ptr
<BlockCrypt
> block_crypt
;
2618 res
= rgw_s3_prepare_encrypt(s
, attrs
, nullptr, &block_crypt
, crypt_http_responses
);
2619 if (res
== 0 && block_crypt
!= nullptr) {
2620 filter
->reset(new RGWPutObj_BlockEncrypt(s
->cct
, cb
, std::move(block_crypt
)));
2626 void RGWPostObj_ObjStore_S3::rebuild_key(rgw::sal::RGWObject
* obj
)
2628 string key
= obj
->get_name();
2629 static string var
= "${filename}";
2630 int pos
= key
.find(var
);
2634 string new_key
= key
.substr(0, pos
);
2635 new_key
.append(filename
);
2636 new_key
.append(key
.substr(pos
+ var
.size()));
2638 obj
->set_key(new_key
);
2641 std::string
RGWPostObj_ObjStore_S3::get_current_filename() const
2643 return s
->object
->get_name();
2646 std::string
RGWPostObj_ObjStore_S3::get_current_content_type() const
2648 return content_type
;
2651 int RGWPostObj_ObjStore_S3::get_params(optional_yield y
)
2653 op_ret
= RGWPostObj_ObjStore::get_params(y
);
2660 ldpp_dout(this, 20) << "adding bucket to policy env: " << s
->bucket
->get_name()
2662 env
.add_var("bucket", s
->bucket
->get_name());
2666 struct post_form_part part
;
2667 int r
= read_form_part_header(&part
, done
);
2671 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 20>()) {
2672 ldpp_dout(this, 20) << "read part header -- part.name="
2673 << part
.name
<< dendl
;
2675 for (const auto& pair
: part
.fields
) {
2676 ldpp_dout(this, 20) << "field.name=" << pair
.first
<< dendl
;
2677 ldpp_dout(this, 20) << "field.val=" << pair
.second
.val
<< dendl
;
2678 ldpp_dout(this, 20) << "field.params:" << dendl
;
2680 for (const auto& param_pair
: pair
.second
.params
) {
2681 ldpp_dout(this, 20) << " " << param_pair
.first
2682 << " -> " << param_pair
.second
<< dendl
;
2687 if (done
) { /* unexpected here */
2688 err_msg
= "Malformed request";
2692 if (stringcasecmp(part
.name
, "file") == 0) { /* beginning of data transfer */
2693 struct post_part_field
& field
= part
.fields
["Content-Disposition"];
2694 map
<string
, string
>::iterator iter
= field
.params
.find("filename");
2695 if (iter
!= field
.params
.end()) {
2696 filename
= iter
->second
;
2698 parts
[part
.name
] = part
;
2703 uint64_t chunk_size
= s
->cct
->_conf
->rgw_max_chunk_size
;
2704 r
= read_data(part
.data
, chunk_size
, boundary
, done
);
2705 if (r
< 0 || !boundary
) {
2706 err_msg
= "Couldn't find boundary";
2709 parts
[part
.name
] = part
;
2710 string
part_str(part
.data
.c_str(), part
.data
.length());
2711 env
.add_var(part
.name
, part_str
);
2715 if (!part_str(parts
, "key", &object_str
)) {
2716 err_msg
= "Key not specified";
2720 s
->object
= store
->get_object(rgw_obj_key(object_str
));
2722 rebuild_key(s
->object
.get());
2724 if (rgw::sal::RGWObject::empty(s
->object
.get())) {
2725 err_msg
= "Empty object name";
2729 env
.add_var("key", s
->object
->get_name());
2731 part_str(parts
, "Content-Type", &content_type
);
2733 /* AWS permits POST without Content-Type: http://tracker.ceph.com/issues/20201 */
2734 if (! content_type
.empty()) {
2735 env
.add_var("Content-Type", content_type
);
2738 std::string storage_class
;
2739 part_str(parts
, "x-amz-storage-class", &storage_class
);
2741 if (! storage_class
.empty()) {
2742 s
->dest_placement
.storage_class
= storage_class
;
2743 if (!store
->svc()->zone
->get_zone_params().valid_placement(s
->dest_placement
)) {
2744 ldpp_dout(this, 0) << "NOTICE: invalid dest placement: " << s
->dest_placement
.to_str() << dendl
;
2745 err_msg
= "The storage class you specified is not valid";
2750 map
<string
, struct post_form_part
, ltstr_nocase
>::iterator piter
=
2751 parts
.upper_bound(RGW_AMZ_META_PREFIX
);
2752 for (; piter
!= parts
.end(); ++piter
) {
2753 string n
= piter
->first
;
2754 if (strncasecmp(n
.c_str(), RGW_AMZ_META_PREFIX
,
2755 sizeof(RGW_AMZ_META_PREFIX
) - 1) != 0)
2758 string attr_name
= RGW_ATTR_PREFIX
;
2759 attr_name
.append(n
);
2761 /* need to null terminate it */
2762 bufferlist
& data
= piter
->second
.data
;
2763 string str
= string(data
.c_str(), data
.length());
2766 attr_bl
.append(str
.c_str(), str
.size() + 1);
2768 attrs
[attr_name
] = attr_bl
;
2770 // TODO: refactor this and the above loop to share code
2771 piter
= parts
.find(RGW_AMZ_WEBSITE_REDIRECT_LOCATION
);
2772 if (piter
!= parts
.end()) {
2773 string n
= piter
->first
;
2774 string attr_name
= RGW_ATTR_PREFIX
;
2775 attr_name
.append(n
);
2776 /* need to null terminate it */
2777 bufferlist
& data
= piter
->second
.data
;
2778 string str
= string(data
.c_str(), data
.length());
2781 attr_bl
.append(str
.c_str(), str
.size() + 1);
2783 attrs
[attr_name
] = attr_bl
;
2786 int r
= get_policy(y
);
2795 min_len
= post_policy
.min_length
;
2796 max_len
= post_policy
.max_length
;
2803 int RGWPostObj_ObjStore_S3::get_tags()
2806 if (part_str(parts
, "tagging", &tags_str
)) {
2807 RGWXMLParser parser
;
2808 if (!parser
.init()){
2809 ldpp_dout(this, 0) << "Couldn't init RGWObjTags XML parser" << dendl
;
2810 err_msg
= "Server couldn't process the request";
2811 return -EINVAL
; // TODO: This class of errors in rgw code should be a 5XX error
2813 if (!parser
.parse(tags_str
.c_str(), tags_str
.size(), 1)) {
2814 ldpp_dout(this,0 ) << "Invalid Tagging XML" << dendl
;
2815 err_msg
= "Invalid Tagging XML";
2819 RGWObjTagging_S3 tagging
;
2822 RGWXMLDecoder::decode_xml("Tagging", tagging
, &parser
);
2823 } catch (RGWXMLDecoder::err
& err
) {
2824 ldpp_dout(this, 5) << "Malformed tagging request: " << err
<< dendl
;
2828 RGWObjTags obj_tags
;
2829 int r
= tagging
.rebuild(obj_tags
);
2834 obj_tags
.encode(tags_bl
);
2835 ldpp_dout(this, 20) << "Read " << obj_tags
.count() << "tags" << dendl
;
2836 attrs
[RGW_ATTR_TAGS
] = tags_bl
;
2843 int RGWPostObj_ObjStore_S3::get_policy(optional_yield y
)
2845 if (part_bl(parts
, "policy", &s
->auth
.s3_postobj_creds
.encoded_policy
)) {
2846 bool aws4_auth
= false;
2848 /* x-amz-algorithm handling */
2849 using rgw::auth::s3::AWS4_HMAC_SHA256_STR
;
2850 if ((part_str(parts
, "x-amz-algorithm", &s
->auth
.s3_postobj_creds
.x_amz_algorithm
)) &&
2851 (s
->auth
.s3_postobj_creds
.x_amz_algorithm
== AWS4_HMAC_SHA256_STR
)) {
2852 ldpp_dout(this, 0) << "Signature verification algorithm AWS v4 (AWS4-HMAC-SHA256)" << dendl
;
2855 ldpp_dout(this, 0) << "Signature verification algorithm AWS v2" << dendl
;
2858 // check that the signature matches the encoded policy
2862 /* x-amz-credential handling */
2863 if (!part_str(parts
, "x-amz-credential",
2864 &s
->auth
.s3_postobj_creds
.x_amz_credential
)) {
2865 ldpp_dout(this, 0) << "No S3 aws4 credential found!" << dendl
;
2866 err_msg
= "Missing aws4 credential";
2870 /* x-amz-signature handling */
2871 if (!part_str(parts
, "x-amz-signature",
2872 &s
->auth
.s3_postobj_creds
.signature
)) {
2873 ldpp_dout(this, 0) << "No aws4 signature found!" << dendl
;
2874 err_msg
= "Missing aws4 signature";
2878 /* x-amz-date handling */
2879 std::string received_date_str
;
2880 if (!part_str(parts
, "x-amz-date", &received_date_str
)) {
2881 ldpp_dout(this, 0) << "No aws4 date found!" << dendl
;
2882 err_msg
= "Missing aws4 date";
2888 // check that the signature matches the encoded policy
2889 if (!part_str(parts
, "AWSAccessKeyId",
2890 &s
->auth
.s3_postobj_creds
.access_key
)) {
2891 ldpp_dout(this, 0) << "No S3 aws2 access key found!" << dendl
;
2892 err_msg
= "Missing aws2 access key";
2896 if (!part_str(parts
, "signature", &s
->auth
.s3_postobj_creds
.signature
)) {
2897 ldpp_dout(this, 0) << "No aws2 signature found!" << dendl
;
2898 err_msg
= "Missing aws2 signature";
2903 if (part_str(parts
, "x-amz-security-token", &s
->auth
.s3_postobj_creds
.x_amz_security_token
)) {
2904 if (s
->auth
.s3_postobj_creds
.x_amz_security_token
.size() == 0) {
2905 err_msg
= "Invalid token";
2910 /* FIXME: this is a makeshift solution. The browser upload authentication will be
2911 * handled by an instance of rgw::auth::Completer spawned in Handler's authorize()
2913 const int ret
= rgw::auth::Strategy::apply(this, auth_registry_ptr
->get_s3_post(), s
, y
);
2917 /* Populate the owner info. */
2918 s
->owner
.set_id(s
->user
->get_id());
2919 s
->owner
.set_name(s
->user
->get_display_name());
2920 ldpp_dout(this, 20) << "Successful Signature Verification!" << dendl
;
2923 ceph::bufferlist decoded_policy
;
2925 decoded_policy
.decode_base64(s
->auth
.s3_postobj_creds
.encoded_policy
);
2926 } catch (buffer::error
& err
) {
2927 ldpp_dout(this, 0) << "failed to decode_base64 policy" << dendl
;
2928 err_msg
= "Could not decode policy";
2932 decoded_policy
.append('\0'); // NULL terminate
2933 ldpp_dout(this, 20) << "POST policy: " << decoded_policy
.c_str() << dendl
;
2936 int r
= post_policy
.from_json(decoded_policy
, err_msg
);
2938 if (err_msg
.empty()) {
2939 err_msg
= "Failed to parse policy";
2941 ldpp_dout(this, 0) << "failed to parse policy" << dendl
;
2947 post_policy
.set_var_checked("x-amz-signature");
2950 post_policy
.set_var_checked("AWSAccessKeyId");
2951 post_policy
.set_var_checked("signature");
2953 post_policy
.set_var_checked("policy");
2955 r
= post_policy
.check(&env
, err_msg
);
2957 if (err_msg
.empty()) {
2958 err_msg
= "Policy check failed";
2960 ldpp_dout(this, 0) << "policy check failed" << dendl
;
2965 ldpp_dout(this, 0) << "No attached policy found!" << dendl
;
2969 part_str(parts
, "acl", &canned_acl
);
2971 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
2972 ldpp_dout(this, 20) << "canned_acl=" << canned_acl
<< dendl
;
2973 if (s3policy
.create_canned(s
->owner
, s
->bucket_owner
, canned_acl
) < 0) {
2974 err_msg
= "Bad canned ACLs";
2983 int RGWPostObj_ObjStore_S3::complete_get_params()
2987 struct post_form_part part
;
2988 int r
= read_form_part_header(&part
, done
);
2993 ceph::bufferlist part_data
;
2995 uint64_t chunk_size
= s
->cct
->_conf
->rgw_max_chunk_size
;
2996 r
= read_data(part
.data
, chunk_size
, boundary
, done
);
2997 if (r
< 0 || !boundary
) {
3001 /* Just reading the data but not storing any results of that. */
3007 int RGWPostObj_ObjStore_S3::get_data(ceph::bufferlist
& bl
, bool& again
)
3012 const uint64_t chunk_size
= s
->cct
->_conf
->rgw_max_chunk_size
;
3013 int r
= read_data(bl
, chunk_size
, boundary
, done
);
3020 /* Reached end of data, let's drain the rest of the params */
3021 r
= complete_get_params();
3032 void RGWPostObj_ObjStore_S3::send_response()
3034 if (op_ret
== 0 && parts
.count("success_action_redirect")) {
3037 part_str(parts
, "success_action_redirect", &redirect
);
3042 string etag_str
= "\"";
3044 etag_str
.append(etag
);
3045 etag_str
.append("\"");
3049 url_encode(s
->bucket_tenant
, tenant
); /* surely overkill, but cheap */
3050 url_encode(s
->bucket_name
, bucket
);
3051 url_encode(s
->object
->get_name(), key
);
3052 url_encode(etag_str
, etag_url
);
3054 if (!s
->bucket_tenant
.empty()) {
3056 * What we really would like is to quaily the bucket name, so
3057 * that the client could simply copy it and paste into next request.
3058 * Unfortunately, in S3 we cannot know if the client will decide
3059 * to come through DNS, with "bucket.tenant" sytanx, or through
3060 * URL with "tenant\bucket" syntax. Therefore, we provide the
3061 * tenant separately.
3063 redirect
.append("?tenant=");
3064 redirect
.append(tenant
);
3065 redirect
.append("&bucket=");
3066 redirect
.append(bucket
);
3068 redirect
.append("?bucket=");
3069 redirect
.append(bucket
);
3071 redirect
.append("&key=");
3072 redirect
.append(key
);
3073 redirect
.append("&etag=");
3074 redirect
.append(etag_url
);
3076 int r
= check_utf8(redirect
.c_str(), redirect
.size());
3081 dump_redirect(s
, redirect
);
3082 op_ret
= STATUS_REDIRECT
;
3083 } else if (op_ret
== 0 && parts
.count("success_action_status")) {
3084 string status_string
;
3085 uint32_t status_int
;
3087 part_str(parts
, "success_action_status", &status_string
);
3089 int r
= stringtoul(status_string
, &status_int
);
3095 switch (status_int
) {
3099 op_ret
= STATUS_CREATED
;
3102 op_ret
= STATUS_NO_CONTENT
;
3105 } else if (! op_ret
) {
3106 op_ret
= STATUS_NO_CONTENT
;
3110 if (op_ret
== STATUS_CREATED
) {
3111 for (auto &it
: crypt_http_responses
)
3112 dump_header(s
, it
.first
, it
.second
);
3113 s
->formatter
->open_object_section("PostResponse");
3114 std::string base_uri
= compute_domain_uri(s
);
3115 if (!s
->bucket_tenant
.empty()){
3116 s
->formatter
->dump_format("Location", "%s/%s:%s/%s",
3118 url_encode(s
->bucket_tenant
).c_str(),
3119 url_encode(s
->bucket_name
).c_str(),
3120 url_encode(s
->object
->get_name()).c_str());
3121 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3123 s
->formatter
->dump_format("Location", "%s/%s/%s",
3125 url_encode(s
->bucket_name
).c_str(),
3126 url_encode(s
->object
->get_name()).c_str());
3128 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3129 s
->formatter
->dump_string("Key", s
->object
->get_name());
3130 s
->formatter
->dump_string("ETag", etag
);
3131 s
->formatter
->close_section();
3133 s
->err
.message
= err_msg
;
3134 set_req_state_err(s
, op_ret
);
3137 dump_content_length(s
, s
->formatter
->get_len());
3139 end_header(s
, this);
3140 if (op_ret
!= STATUS_CREATED
)
3143 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3146 int RGWPostObj_ObjStore_S3::get_encrypt_filter(
3147 std::unique_ptr
<rgw::putobj::DataProcessor
> *filter
,
3148 rgw::putobj::DataProcessor
*cb
)
3150 std::unique_ptr
<BlockCrypt
> block_crypt
;
3151 int res
= rgw_s3_prepare_encrypt(s
, attrs
, &parts
, &block_crypt
,
3152 crypt_http_responses
);
3153 if (res
== 0 && block_crypt
!= nullptr) {
3154 filter
->reset(new RGWPutObj_BlockEncrypt(s
->cct
, cb
, std::move(block_crypt
)));
3159 int RGWDeleteObj_ObjStore_S3::get_params(optional_yield y
)
3161 const char *if_unmod
= s
->info
.env
->get("HTTP_X_AMZ_DELETE_IF_UNMODIFIED_SINCE");
3163 if (s
->system_request
) {
3164 s
->info
.args
.get_bool(RGW_SYS_PARAM_PREFIX
"no-precondition-error", &no_precondition_error
, false);
3168 std::string if_unmod_decoded
= url_decode(if_unmod
);
3171 if (utime_t::parse_date(if_unmod_decoded
, &epoch
, &nsec
) < 0) {
3172 ldpp_dout(this, 10) << "failed to parse time: " << if_unmod_decoded
<< dendl
;
3175 unmod_since
= utime_t(epoch
, nsec
).to_real_time();
3178 const char *bypass_gov_header
= s
->info
.env
->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
3179 if (bypass_gov_header
) {
3180 std::string bypass_gov_decoded
= url_decode(bypass_gov_header
);
3181 bypass_governance_mode
= boost::algorithm::iequals(bypass_gov_decoded
, "true");
3187 void RGWDeleteObj_ObjStore_S3::send_response()
3193 r
= STATUS_NO_CONTENT
;
3195 set_req_state_err(s
, r
);
3197 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
3198 if (delete_marker
) {
3199 dump_header(s
, "x-amz-delete-marker", "true");
3201 end_header(s
, this);
3204 int RGWCopyObj_ObjStore_S3::init_dest_policy()
3206 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
3208 /* build a policy for the target object */
3209 int r
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
3213 dest_policy
= s3policy
;
3218 int RGWCopyObj_ObjStore_S3::get_params(optional_yield y
)
3220 if_mod
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
3221 if_unmod
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
3222 if_match
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_MATCH");
3223 if_nomatch
= s
->info
.env
->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
3225 src_tenant_name
= s
->src_tenant_name
;
3226 src_bucket_name
= s
->src_bucket_name
;
3227 src_object
= s
->src_object
->clone();
3228 dest_tenant_name
= s
->bucket
->get_tenant();
3229 dest_bucket_name
= s
->bucket
->get_name();
3230 dest_obj_name
= s
->object
->get_name();
3232 if (s
->system_request
) {
3233 source_zone
= s
->info
.args
.get(RGW_SYS_PARAM_PREFIX
"source-zone");
3234 s
->info
.args
.get_bool(RGW_SYS_PARAM_PREFIX
"copy-if-newer", ©_if_newer
, false);
3237 copy_source
= s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE");
3238 auto tmp_md_d
= s
->info
.env
->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
3240 if (strcasecmp(tmp_md_d
, "COPY") == 0) {
3241 attrs_mod
= rgw::sal::ATTRSMOD_NONE
;
3242 } else if (strcasecmp(tmp_md_d
, "REPLACE") == 0) {
3243 attrs_mod
= rgw::sal::ATTRSMOD_REPLACE
;
3244 } else if (!source_zone
.empty()) {
3245 attrs_mod
= rgw::sal::ATTRSMOD_NONE
; // default for intra-zone_group copy
3247 s
->err
.message
= "Unknown metadata directive.";
3248 ldpp_dout(this, 0) << s
->err
.message
<< dendl
;
3251 md_directive
= tmp_md_d
;
3254 if (source_zone
.empty() &&
3255 (dest_tenant_name
.compare(src_tenant_name
) == 0) &&
3256 (dest_bucket_name
.compare(src_bucket_name
) == 0) &&
3257 (dest_obj_name
.compare(src_object
->get_name()) == 0) &&
3258 src_object
->get_instance().empty() &&
3259 (attrs_mod
!= rgw::sal::ATTRSMOD_REPLACE
)) {
3260 need_to_check_storage_class
= true;
3266 int RGWCopyObj_ObjStore_S3::check_storage_class(const rgw_placement_rule
& src_placement
)
3268 if (src_placement
== s
->dest_placement
) {
3269 /* can only copy object into itself if replacing attrs */
3270 s
->err
.message
= "This copy request is illegal because it is trying to copy "
3271 "an object to itself without changing the object's metadata, "
3272 "storage class, website redirect location or encryption attributes.";
3273 ldpp_dout(this, 0) << s
->err
.message
<< dendl
;
3274 return -ERR_INVALID_REQUEST
;
3279 void RGWCopyObj_ObjStore_S3::send_partial_response(off_t ofs
)
3281 if (! sent_header
) {
3283 set_req_state_err(s
, op_ret
);
3286 // Explicitly use chunked transfer encoding so that we can stream the result
3287 // to the user without having to wait for the full length of it.
3288 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3291 s
->formatter
->open_object_section_in_ns("CopyObjectResult", XMLNS_AWS_S3
);
3295 /* Send progress field. Note that this diverge from the original S3
3296 * spec. We do this in order to keep connection alive.
3298 s
->formatter
->dump_int("Progress", (uint64_t)ofs
);
3300 rgw_flush_formatter(s
, s
->formatter
);
3303 void RGWCopyObj_ObjStore_S3::send_response()
3306 send_partial_response(0);
3309 dump_time(s
, "LastModified", &mtime
);
3310 if (!etag
.empty()) {
3311 s
->formatter
->dump_string("ETag", std::move(etag
));
3313 s
->formatter
->close_section();
3314 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3318 void RGWGetACLs_ObjStore_S3::send_response()
3321 set_req_state_err(s
, op_ret
);
3323 end_header(s
, this, "application/xml");
3325 rgw_flush_formatter(s
, s
->formatter
);
3329 int RGWPutACLs_ObjStore_S3::get_params(optional_yield y
)
3331 int ret
= RGWPutACLs_ObjStore::get_params(y
);
3333 const int ret_auth
= do_aws4_auth_completion();
3338 /* a request body is not required an S3 PutACLs request--n.b.,
3339 * s->length is non-null iff a content length was parsed (the
3340 * ACP or canned ACL could be in any of 3 headers, don't worry
3341 * about that here) */
3342 if ((ret
== -ERR_LENGTH_REQUIRED
) &&
3350 int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::RGWRadosStore
*store
,
3351 struct req_state
*s
,
3354 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
3356 // bucket-* canned acls do not apply to bucket
3357 if (rgw::sal::RGWObject::empty(s
->object
.get())) {
3358 if (s
->canned_acl
.find("bucket") != string::npos
)
3359 s
->canned_acl
.clear();
3362 int r
= create_s3_policy(s
, store
, s3policy
, owner
);
3366 s3policy
.to_xml(ss
);
3371 void RGWPutACLs_ObjStore_S3::send_response()
3374 set_req_state_err(s
, op_ret
);
3376 end_header(s
, this, "application/xml");
3380 void RGWGetLC_ObjStore_S3::execute(optional_yield y
)
3382 config
.set_ctx(s
->cct
);
3384 map
<string
, bufferlist
>::iterator aiter
= s
->bucket_attrs
.find(RGW_ATTR_LC
);
3385 if (aiter
== s
->bucket_attrs
.end()) {
3390 bufferlist::const_iterator iter
{&aiter
->second
};
3392 config
.decode(iter
);
3393 } catch (const buffer::error
& e
) {
3394 ldpp_dout(this, 0) << __func__
<< "decode life cycle config failed" << dendl
;
3400 void RGWGetLC_ObjStore_S3::send_response()
3403 if (op_ret
== -ENOENT
) {
3404 set_req_state_err(s
, ERR_NO_SUCH_LC
);
3406 set_req_state_err(s
, op_ret
);
3410 end_header(s
, this, "application/xml");
3416 encode_xml("LifecycleConfiguration", XMLNS_AWS_S3
, config
, s
->formatter
);
3417 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3420 void RGWPutLC_ObjStore_S3::send_response()
3423 set_req_state_err(s
, op_ret
);
3425 end_header(s
, this, "application/xml");
3429 void RGWDeleteLC_ObjStore_S3::send_response()
3432 op_ret
= STATUS_NO_CONTENT
;
3434 set_req_state_err(s
, op_ret
);
3437 end_header(s
, this, "application/xml");
3441 void RGWGetCORS_ObjStore_S3::send_response()
3444 if (op_ret
== -ENOENT
)
3445 set_req_state_err(s
, ERR_NO_SUCH_CORS_CONFIGURATION
);
3447 set_req_state_err(s
, op_ret
);
3450 end_header(s
, NULL
, "application/xml");
3454 RGWCORSConfiguration_S3
*s3cors
=
3455 static_cast<RGWCORSConfiguration_S3
*>(&bucket_cors
);
3464 int RGWPutCORS_ObjStore_S3::get_params(optional_yield y
)
3466 RGWCORSXMLParser_S3
parser(this, s
->cct
);
3467 RGWCORSConfiguration_S3
*cors_config
;
3469 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
3473 std::tie(r
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
3478 r
= do_aws4_auth_completion();
3483 if (!parser
.init()) {
3487 char* buf
= data
.c_str();
3488 if (!buf
|| !parser
.parse(buf
, data
.length(), 1)) {
3489 return -ERR_MALFORMED_XML
;
3492 static_cast<RGWCORSConfiguration_S3
*>(parser
.find_first(
3493 "CORSConfiguration"));
3495 return -ERR_MALFORMED_XML
;
3498 #define CORS_RULES_MAX_NUM 100
3499 int max_num
= s
->cct
->_conf
->rgw_cors_rules_max_num
;
3501 max_num
= CORS_RULES_MAX_NUM
;
3503 int cors_rules_num
= cors_config
->get_rules().size();
3504 if (cors_rules_num
> max_num
) {
3505 ldpp_dout(this, 4) << "An cors config can have up to "
3507 << " rules, request cors rules num: "
3508 << cors_rules_num
<< dendl
;
3509 op_ret
= -ERR_INVALID_CORS_RULES_ERROR
;
3510 s
->err
.message
= "The number of CORS rules should not exceed allowed limit of "
3511 + std::to_string(max_num
) + " rules.";
3512 return -ERR_INVALID_REQUEST
;
3515 // forward bucket cors requests to meta master zone
3516 if (!store
->svc()->zone
->is_meta_master()) {
3517 /* only need to keep this data around if we're not meta master */
3518 in_data
.append(data
);
3521 if (s
->cct
->_conf
->subsys
.should_gather
<ceph_subsys_rgw
, 15>()) {
3522 ldpp_dout(this, 15) << "CORSConfiguration";
3523 cors_config
->to_xml(*_dout
);
3527 cors_config
->encode(cors_bl
);
3532 void RGWPutCORS_ObjStore_S3::send_response()
3535 set_req_state_err(s
, op_ret
);
3537 end_header(s
, NULL
, "application/xml");
3541 void RGWDeleteCORS_ObjStore_S3::send_response()
3544 if (!r
|| r
== -ENOENT
)
3545 r
= STATUS_NO_CONTENT
;
3547 set_req_state_err(s
, r
);
3549 end_header(s
, NULL
);
3552 void RGWOptionsCORS_ObjStore_S3::send_response()
3554 string hdrs
, exp_hdrs
;
3555 uint32_t max_age
= CORS_MAX_AGE_INVALID
;
3556 /*EACCES means, there is no CORS registered yet for the bucket
3557 *ENOENT means, there is no match of the Origin in the list of CORSRule
3559 if (op_ret
== -ENOENT
)
3562 set_req_state_err(s
, op_ret
);
3564 end_header(s
, NULL
);
3567 get_response_params(hdrs
, exp_hdrs
, &max_age
);
3570 dump_access_control(s
, origin
, req_meth
, hdrs
.c_str(), exp_hdrs
.c_str(),
3572 end_header(s
, NULL
);
3575 void RGWGetRequestPayment_ObjStore_S3::send_response()
3578 end_header(s
, this, "application/xml");
3581 s
->formatter
->open_object_section_in_ns("RequestPaymentConfiguration", XMLNS_AWS_S3
);
3582 const char *payer
= requester_pays
? "Requester" : "BucketOwner";
3583 s
->formatter
->dump_string("Payer", payer
);
3584 s
->formatter
->close_section();
3585 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3588 class RGWSetRequestPaymentParser
: public RGWXMLParser
3590 XMLObj
*alloc_obj(const char *el
) override
{
3595 RGWSetRequestPaymentParser() {}
3596 ~RGWSetRequestPaymentParser() override
{}
3598 int get_request_payment_payer(bool *requester_pays
) {
3599 XMLObj
*config
= find_first("RequestPaymentConfiguration");
3603 *requester_pays
= false;
3605 XMLObj
*field
= config
->find_first("Payer");
3609 auto& s
= field
->get_data();
3611 if (stringcasecmp(s
, "Requester") == 0) {
3612 *requester_pays
= true;
3613 } else if (stringcasecmp(s
, "BucketOwner") != 0) {
3621 int RGWSetRequestPayment_ObjStore_S3::get_params(optional_yield y
)
3623 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
3626 std::tie(r
, in_data
) = rgw_rest_read_all_input(s
, max_size
, false);
3633 RGWSetRequestPaymentParser parser
;
3635 if (!parser
.init()) {
3636 ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl
;
3640 char* buf
= in_data
.c_str();
3641 if (!parser
.parse(buf
, in_data
.length(), 1)) {
3642 ldpp_dout(this, 10) << "failed to parse data: " << buf
<< dendl
;
3646 return parser
.get_request_payment_payer(&requester_pays
);
3649 void RGWSetRequestPayment_ObjStore_S3::send_response()
3652 set_req_state_err(s
, op_ret
);
3657 int RGWInitMultipart_ObjStore_S3::get_params(optional_yield y
)
3659 RGWAccessControlPolicy_S3
s3policy(s
->cct
);
3660 op_ret
= create_s3_policy(s
, store
, s3policy
, s
->owner
);
3669 void RGWInitMultipart_ObjStore_S3::send_response()
3672 set_req_state_err(s
, op_ret
);
3674 for (auto &it
: crypt_http_responses
)
3675 dump_header(s
, it
.first
, it
.second
);
3676 ceph::real_time abort_date
;
3678 bool exist_multipart_abort
= get_s3_multipart_abort_header(s
, mtime
, abort_date
, rule_id
);
3679 if (exist_multipart_abort
) {
3680 dump_time_header(s
, "x-amz-abort-date", abort_date
);
3681 dump_header_if_nonempty(s
, "x-amz-abort-rule-id", rule_id
);
3683 end_header(s
, this, "application/xml");
3686 s
->formatter
->open_object_section_in_ns("InitiateMultipartUploadResult", XMLNS_AWS_S3
);
3687 if (!s
->bucket_tenant
.empty())
3688 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3689 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3690 s
->formatter
->dump_string("Key", s
->object
->get_name());
3691 s
->formatter
->dump_string("UploadId", upload_id
);
3692 s
->formatter
->close_section();
3693 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3697 int RGWInitMultipart_ObjStore_S3::prepare_encryption(map
<string
, bufferlist
>& attrs
)
3700 res
= rgw_s3_prepare_encrypt(s
, attrs
, nullptr, nullptr, crypt_http_responses
);
3704 int RGWCompleteMultipart_ObjStore_S3::get_params(optional_yield y
)
3706 int ret
= RGWCompleteMultipart_ObjStore::get_params(y
);
3713 return do_aws4_auth_completion();
3716 void RGWCompleteMultipart_ObjStore_S3::send_response()
3719 set_req_state_err(s
, op_ret
);
3721 dump_header_if_nonempty(s
, "x-amz-version-id", version_id
);
3722 end_header(s
, this, "application/xml");
3725 s
->formatter
->open_object_section_in_ns("CompleteMultipartUploadResult", XMLNS_AWS_S3
);
3726 std::string base_uri
= compute_domain_uri(s
);
3727 if (!s
->bucket_tenant
.empty()) {
3728 s
->formatter
->dump_format("Location", "%s/%s:%s/%s",
3730 s
->bucket_tenant
.c_str(),
3731 s
->bucket_name
.c_str(),
3732 s
->object
->get_name().c_str()
3734 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3736 s
->formatter
->dump_format("Location", "%s/%s/%s",
3738 s
->bucket_name
.c_str(),
3739 s
->object
->get_name().c_str()
3742 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3743 s
->formatter
->dump_string("Key", s
->object
->get_name());
3744 s
->formatter
->dump_string("ETag", etag
);
3745 s
->formatter
->close_section();
3746 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3750 void RGWAbortMultipart_ObjStore_S3::send_response()
3754 r
= STATUS_NO_CONTENT
;
3756 set_req_state_err(s
, r
);
3758 end_header(s
, this);
3761 void RGWListMultipart_ObjStore_S3::send_response()
3764 set_req_state_err(s
, op_ret
);
3766 // Explicitly use chunked transfer encoding so that we can stream the result
3767 // to the user without having to wait for the full length of it.
3768 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3772 s
->formatter
->open_object_section_in_ns("ListPartsResult", XMLNS_AWS_S3
);
3773 map
<uint32_t, RGWUploadPartInfo
>::iterator iter
;
3774 map
<uint32_t, RGWUploadPartInfo
>::reverse_iterator test_iter
;
3777 iter
= parts
.begin();
3778 test_iter
= parts
.rbegin();
3779 if (test_iter
!= parts
.rend()) {
3780 cur_max
= test_iter
->first
;
3782 if (!s
->bucket_tenant
.empty())
3783 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3784 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3785 s
->formatter
->dump_string("Key", s
->object
->get_name());
3786 s
->formatter
->dump_string("UploadId", upload_id
);
3787 s
->formatter
->dump_string("StorageClass", "STANDARD");
3788 s
->formatter
->dump_int("PartNumberMarker", marker
);
3789 s
->formatter
->dump_int("NextPartNumberMarker", cur_max
);
3790 s
->formatter
->dump_int("MaxParts", max_parts
);
3791 s
->formatter
->dump_string("IsTruncated", (truncated
? "true" : "false"));
3793 ACLOwner
& owner
= policy
.get_owner();
3794 dump_owner(s
, owner
.get_id(), owner
.get_display_name());
3796 for (; iter
!= parts
.end(); ++iter
) {
3797 RGWUploadPartInfo
& info
= iter
->second
;
3799 s
->formatter
->open_object_section("Part");
3801 dump_time(s
, "LastModified", &info
.modified
);
3803 s
->formatter
->dump_unsigned("PartNumber", info
.num
);
3804 s
->formatter
->dump_format("ETag", "\"%s\"", info
.etag
.c_str());
3805 s
->formatter
->dump_unsigned("Size", info
.accounted_size
);
3806 s
->formatter
->close_section();
3808 s
->formatter
->close_section();
3809 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3813 void RGWListBucketMultiparts_ObjStore_S3::send_response()
3816 set_req_state_err(s
, op_ret
);
3819 // Explicitly use chunked transfer encoding so that we can stream the result
3820 // to the user without having to wait for the full length of it.
3821 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3826 s
->formatter
->open_object_section_in_ns("ListMultipartUploadsResult", XMLNS_AWS_S3
);
3827 if (!s
->bucket_tenant
.empty())
3828 s
->formatter
->dump_string("Tenant", s
->bucket_tenant
);
3829 s
->formatter
->dump_string("Bucket", s
->bucket_name
);
3830 if (!prefix
.empty())
3831 s
->formatter
->dump_string("ListMultipartUploadsResult.Prefix", prefix
);
3832 const string
& key_marker
= marker
.get_key();
3833 if (!key_marker
.empty())
3834 s
->formatter
->dump_string("KeyMarker", key_marker
);
3835 const string
& upload_id_marker
= marker
.get_upload_id();
3836 if (!upload_id_marker
.empty())
3837 s
->formatter
->dump_string("UploadIdMarker", upload_id_marker
);
3838 string next_key
= next_marker
.mp
.get_key();
3839 if (!next_key
.empty())
3840 s
->formatter
->dump_string("NextKeyMarker", next_key
);
3841 string next_upload_id
= next_marker
.mp
.get_upload_id();
3842 if (!next_upload_id
.empty())
3843 s
->formatter
->dump_string("NextUploadIdMarker", next_upload_id
);
3844 s
->formatter
->dump_int("MaxUploads", max_uploads
);
3845 if (!delimiter
.empty())
3846 s
->formatter
->dump_string("Delimiter", delimiter
);
3847 s
->formatter
->dump_string("IsTruncated", (is_truncated
? "true" : "false"));
3850 vector
<RGWMultipartUploadEntry
>::iterator iter
;
3851 for (iter
= uploads
.begin(); iter
!= uploads
.end(); ++iter
) {
3852 RGWMPObj
& mp
= iter
->mp
;
3853 s
->formatter
->open_array_section("Upload");
3855 s
->formatter
->dump_string("Key", url_encode(mp
.get_key(), false));
3857 s
->formatter
->dump_string("Key", mp
.get_key());
3859 s
->formatter
->dump_string("UploadId", mp
.get_upload_id());
3860 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name(), "Initiator");
3861 dump_owner(s
, s
->user
->get_id(), s
->user
->get_display_name());
3862 s
->formatter
->dump_string("StorageClass", "STANDARD");
3863 dump_time(s
, "Initiated", &iter
->obj
.meta
.mtime
);
3864 s
->formatter
->close_section();
3866 if (!common_prefixes
.empty()) {
3867 s
->formatter
->open_array_section("CommonPrefixes");
3868 for (const auto& kv
: common_prefixes
) {
3870 s
->formatter
->dump_string("CommonPrefixes.Prefix",
3871 url_encode(kv
.first
, false));
3873 s
->formatter
->dump_string("CommonPrefixes.Prefix", kv
.first
);
3876 s
->formatter
->close_section();
3879 s
->formatter
->close_section();
3880 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3883 int RGWDeleteMultiObj_ObjStore_S3::get_params(optional_yield y
)
3885 int ret
= RGWDeleteMultiObj_ObjStore::get_params(y
);
3890 const char *bypass_gov_header
= s
->info
.env
->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
3891 if (bypass_gov_header
) {
3892 std::string bypass_gov_decoded
= url_decode(bypass_gov_header
);
3893 bypass_governance_mode
= boost::algorithm::iequals(bypass_gov_decoded
, "true");
3896 return do_aws4_auth_completion();
3899 void RGWDeleteMultiObj_ObjStore_S3::send_status()
3901 if (! status_dumped
) {
3903 set_req_state_err(s
, op_ret
);
3905 status_dumped
= true;
3909 void RGWDeleteMultiObj_ObjStore_S3::begin_response()
3912 if (!status_dumped
) {
3917 // Explicitly use chunked transfer encoding so that we can stream the result
3918 // to the user without having to wait for the full length of it.
3919 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
3920 s
->formatter
->open_object_section_in_ns("DeleteResult", XMLNS_AWS_S3
);
3922 rgw_flush_formatter(s
, s
->formatter
);
3925 void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key
& key
,
3927 const string
& marker_version_id
, int ret
)
3930 if (ret
== 0 && !quiet
) {
3931 s
->formatter
->open_object_section("Deleted");
3932 s
->formatter
->dump_string("Key", key
.name
);
3933 if (!key
.instance
.empty()) {
3934 s
->formatter
->dump_string("VersionId", key
.instance
);
3936 if (delete_marker
) {
3937 s
->formatter
->dump_bool("DeleteMarker", true);
3938 s
->formatter
->dump_string("DeleteMarkerVersionId", marker_version_id
);
3940 s
->formatter
->close_section();
3941 } else if (ret
< 0) {
3942 struct rgw_http_error r
;
3945 s
->formatter
->open_object_section("Error");
3948 rgw_get_errno_s3(&r
, err_no
);
3950 s
->formatter
->dump_string("Key", key
.name
);
3951 s
->formatter
->dump_string("VersionId", key
.instance
);
3952 s
->formatter
->dump_string("Code", r
.s3_code
);
3953 s
->formatter
->dump_string("Message", r
.s3_code
);
3954 s
->formatter
->close_section();
3957 rgw_flush_formatter(s
, s
->formatter
);
3961 void RGWDeleteMultiObj_ObjStore_S3::end_response()
3964 s
->formatter
->close_section();
3965 rgw_flush_formatter_and_reset(s
, s
->formatter
);
3968 void RGWGetObjLayout_ObjStore_S3::send_response()
3971 set_req_state_err(s
, op_ret
);
3973 end_header(s
, this, "application/json");
3981 f
.open_object_section("result");
3982 ::encode_json("head", head_obj
, &f
);
3983 ::encode_json("manifest", *manifest
, &f
);
3984 f
.open_array_section("data_location");
3985 for (auto miter
= manifest
->obj_begin(this); miter
!= manifest
->obj_end(this); ++miter
) {
3986 f
.open_object_section("obj");
3987 rgw_raw_obj raw_loc
= miter
.get_location().get_raw_obj(store
);
3988 uint64_t ofs
= miter
.get_ofs();
3989 uint64_t left
= manifest
->get_obj_size() - ofs
;
3990 ::encode_json("ofs", miter
.get_ofs(), &f
);
3991 ::encode_json("loc", raw_loc
, &f
);
3992 ::encode_json("loc_ofs", miter
.location_ofs(), &f
);
3993 uint64_t loc_size
= miter
.get_stripe_size();
3994 if (loc_size
> left
) {
3997 ::encode_json("loc_size", loc_size
, &f
);
3999 rgw_flush_formatter(s
, &f
);
4003 rgw_flush_formatter(s
, &f
);
4006 int RGWConfigBucketMetaSearch_ObjStore_S3::get_params(optional_yield y
)
4008 auto iter
= s
->info
.x_meta_map
.find("x-amz-meta-search");
4009 if (iter
== s
->info
.x_meta_map
.end()) {
4010 s
->err
.message
= "X-Rgw-Meta-Search header not provided";
4011 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4015 list
<string
> expressions
;
4016 get_str_list(iter
->second
, ",", expressions
);
4018 for (auto& expression
: expressions
) {
4019 vector
<string
> args
;
4020 get_str_vec(expression
, ";", args
);
4023 s
->err
.message
= "invalid empty expression";
4024 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4027 if (args
.size() > 2) {
4028 s
->err
.message
= string("invalid expression: ") + expression
;
4029 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4033 string key
= boost::algorithm::to_lower_copy(rgw_trim_whitespace(args
[0]));
4035 if (args
.size() > 1) {
4036 val
= boost::algorithm::to_lower_copy(rgw_trim_whitespace(args
[1]));
4039 if (!boost::algorithm::starts_with(key
, RGW_AMZ_META_PREFIX
)) {
4040 s
->err
.message
= string("invalid expression, key must start with '" RGW_AMZ_META_PREFIX
"' : ") + expression
;
4041 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4045 key
= key
.substr(sizeof(RGW_AMZ_META_PREFIX
) - 1);
4047 ESEntityTypeMap::EntityType entity_type
;
4049 if (val
.empty() || val
== "str" || val
== "string") {
4050 entity_type
= ESEntityTypeMap::ES_ENTITY_STR
;
4051 } else if (val
== "int" || val
== "integer") {
4052 entity_type
= ESEntityTypeMap::ES_ENTITY_INT
;
4053 } else if (val
== "date" || val
== "datetime") {
4054 entity_type
= ESEntityTypeMap::ES_ENTITY_DATE
;
4056 s
->err
.message
= string("invalid entity type: ") + val
;
4057 ldpp_dout(this, 5) << s
->err
.message
<< dendl
;
4061 mdsearch_config
[key
] = entity_type
;
4067 void RGWConfigBucketMetaSearch_ObjStore_S3::send_response()
4070 set_req_state_err(s
, op_ret
);
4072 end_header(s
, this);
4075 void RGWGetBucketMetaSearch_ObjStore_S3::send_response()
4078 set_req_state_err(s
, op_ret
);
4080 end_header(s
, NULL
, "application/xml");
4082 Formatter
*f
= s
->formatter
;
4083 f
->open_array_section("GetBucketMetaSearchResult");
4084 for (auto& e
: s
->bucket
->get_info().mdsearch_config
) {
4085 f
->open_object_section("Entry");
4086 string k
= string("x-amz-meta-") + e
.first
;
4087 f
->dump_string("Key", k
.c_str());
4090 case ESEntityTypeMap::ES_ENTITY_INT
:
4093 case ESEntityTypeMap::ES_ENTITY_DATE
:
4099 f
->dump_string("Type", type
);
4103 rgw_flush_formatter(s
, f
);
4106 void RGWDelBucketMetaSearch_ObjStore_S3::send_response()
4109 set_req_state_err(s
, op_ret
);
4111 end_header(s
, this);
4114 void RGWPutBucketObjectLock_ObjStore_S3::send_response()
4117 set_req_state_err(s
, op_ret
);
4123 void RGWGetBucketObjectLock_ObjStore_S3::send_response()
4126 set_req_state_err(s
, op_ret
);
4129 end_header(s
, this, "application/xml");
4135 encode_xml("ObjectLockConfiguration", s
->bucket
->get_info().obj_lock
, s
->formatter
);
4136 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4140 int RGWPutObjRetention_ObjStore_S3::get_params(optional_yield y
)
4142 const char *bypass_gov_header
= s
->info
.env
->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
4143 if (bypass_gov_header
) {
4144 std::string bypass_gov_decoded
= url_decode(bypass_gov_header
);
4145 bypass_governance_mode
= boost::algorithm::iequals(bypass_gov_decoded
, "true");
4148 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
4149 std::tie(op_ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
4153 void RGWPutObjRetention_ObjStore_S3::send_response()
4156 set_req_state_err(s
, op_ret
);
4162 void RGWGetObjRetention_ObjStore_S3::send_response()
4165 set_req_state_err(s
, op_ret
);
4168 end_header(s
, this, "application/xml");
4174 encode_xml("Retention", obj_retention
, s
->formatter
);
4175 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4178 void RGWPutObjLegalHold_ObjStore_S3::send_response()
4181 set_req_state_err(s
, op_ret
);
4187 void RGWGetObjLegalHold_ObjStore_S3::send_response()
4190 set_req_state_err(s
, op_ret
);
4193 end_header(s
, this, "application/xml");
4199 encode_xml("LegalHold", obj_legal_hold
, s
->formatter
);
4200 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4203 void RGWGetBucketPolicyStatus_ObjStore_S3::send_response()
4206 set_req_state_err(s
, op_ret
);
4209 end_header(s
, this, "application/xml");
4212 s
->formatter
->open_object_section_in_ns("PolicyStatus", XMLNS_AWS_S3
);
4213 // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETPolicyStatus.html
4214 // mentions TRUE and FALSE, but boto/aws official clients seem to want lower
4215 // case which is returned by AWS as well; so let's be bug to bug compatible
4217 s
->formatter
->dump_bool("IsPublic", isPublic
);
4218 s
->formatter
->close_section();
4219 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4223 void RGWPutBucketPublicAccessBlock_ObjStore_S3::send_response()
4226 set_req_state_err(s
, op_ret
);
4232 void RGWGetBucketPublicAccessBlock_ObjStore_S3::send_response()
4235 set_req_state_err(s
, op_ret
);
4238 end_header(s
, this, "application/xml");
4241 access_conf
.dump_xml(s
->formatter
);
4242 rgw_flush_formatter_and_reset(s
, s
->formatter
);
4245 RGWOp
*RGWHandler_REST_Service_S3::op_get()
4247 if (is_usage_op()) {
4248 return new RGWGetUsage_ObjStore_S3
;
4250 return new RGWListBuckets_ObjStore_S3
;
4254 RGWOp
*RGWHandler_REST_Service_S3::op_head()
4256 return new RGWListBuckets_ObjStore_S3
;
4259 RGWOp
*RGWHandler_REST_Service_S3::op_post()
4261 const auto max_size
= s
->cct
->_conf
->rgw_max_put_param_size
;
4265 std::tie(ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
4270 const auto post_body
= data
.to_str();
4273 RGWHandler_REST_STS
sts_handler(auth_registry
, post_body
);
4274 sts_handler
.init(store
, s
, s
->cio
);
4275 auto op
= sts_handler
.get_op();
4282 RGWHandler_REST_IAM
iam_handler(auth_registry
, post_body
);
4283 iam_handler
.init(store
, s
, s
->cio
);
4284 auto op
= iam_handler
.get_op();
4291 RGWHandler_REST_PSTopic_AWS
topic_handler(auth_registry
, post_body
);
4292 topic_handler
.init(store
, s
, s
->cio
);
4293 auto op
= topic_handler
.get_op();
4302 RGWOp
*RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data
) const
4307 s
->info
.args
.get_int("list-type", &list_type
, 1);
4308 switch (list_type
) {
4310 return new RGWListBucket_ObjStore_S3
;
4312 return new RGWListBucket_ObjStore_S3v2
;
4314 ldpp_dout(s
, 5) << __func__
<< ": unsupported list-type " << list_type
<< dendl
;
4315 return new RGWListBucket_ObjStore_S3
;
4318 return new RGWStatBucket_ObjStore_S3
;
4322 RGWOp
*RGWHandler_REST_Bucket_S3::op_get()
4324 if (s
->info
.args
.sub_resource_exists("encryption"))
4327 if (s
->info
.args
.sub_resource_exists("logging"))
4328 return new RGWGetBucketLogging_ObjStore_S3
;
4330 if (s
->info
.args
.sub_resource_exists("location"))
4331 return new RGWGetBucketLocation_ObjStore_S3
;
4333 if (s
->info
.args
.sub_resource_exists("versioning"))
4334 return new RGWGetBucketVersioning_ObjStore_S3
;
4336 if (s
->info
.args
.sub_resource_exists("website")) {
4337 if (!s
->cct
->_conf
->rgw_enable_static_website
) {
4340 return new RGWGetBucketWebsite_ObjStore_S3
;
4343 if (s
->info
.args
.exists("mdsearch")) {
4344 return new RGWGetBucketMetaSearch_ObjStore_S3
;
4348 return new RGWGetACLs_ObjStore_S3
;
4349 } else if (is_cors_op()) {
4350 return new RGWGetCORS_ObjStore_S3
;
4351 } else if (is_request_payment_op()) {
4352 return new RGWGetRequestPayment_ObjStore_S3
;
4353 } else if (s
->info
.args
.exists("uploads")) {
4354 return new RGWListBucketMultiparts_ObjStore_S3
;
4355 } else if(is_lc_op()) {
4356 return new RGWGetLC_ObjStore_S3
;
4357 } else if(is_policy_op()) {
4358 return new RGWGetBucketPolicy
;
4359 } else if (is_tagging_op()) {
4360 return new RGWGetBucketTags_ObjStore_S3
;
4361 } else if (is_object_lock_op()) {
4362 return new RGWGetBucketObjectLock_ObjStore_S3
;
4363 } else if (is_notification_op()) {
4364 return RGWHandler_REST_PSNotifs_S3::create_get_op();
4365 } else if (is_replication_op()) {
4366 return new RGWGetBucketReplication_ObjStore_S3
;
4367 } else if (is_policy_status_op()) {
4368 return new RGWGetBucketPolicyStatus_ObjStore_S3
;
4369 } else if (is_block_public_access_op()) {
4370 return new RGWGetBucketPublicAccessBlock_ObjStore_S3
;
4372 return get_obj_op(true);
4375 RGWOp
*RGWHandler_REST_Bucket_S3::op_head()
4378 return new RGWGetACLs_ObjStore_S3
;
4379 } else if (s
->info
.args
.exists("uploads")) {
4380 return new RGWListBucketMultiparts_ObjStore_S3
;
4382 return get_obj_op(false);
4385 RGWOp
*RGWHandler_REST_Bucket_S3::op_put()
4387 if (s
->info
.args
.sub_resource_exists("logging") ||
4388 s
->info
.args
.sub_resource_exists("encryption"))
4390 if (s
->info
.args
.sub_resource_exists("versioning"))
4391 return new RGWSetBucketVersioning_ObjStore_S3
;
4392 if (s
->info
.args
.sub_resource_exists("website")) {
4393 if (!s
->cct
->_conf
->rgw_enable_static_website
) {
4396 return new RGWSetBucketWebsite_ObjStore_S3
;
4398 if (is_tagging_op()) {
4399 return new RGWPutBucketTags_ObjStore_S3
;
4400 } else if (is_acl_op()) {
4401 return new RGWPutACLs_ObjStore_S3
;
4402 } else if (is_cors_op()) {
4403 return new RGWPutCORS_ObjStore_S3
;
4404 } else if (is_request_payment_op()) {
4405 return new RGWSetRequestPayment_ObjStore_S3
;
4406 } else if(is_lc_op()) {
4407 return new RGWPutLC_ObjStore_S3
;
4408 } else if(is_policy_op()) {
4409 return new RGWPutBucketPolicy
;
4410 } else if (is_object_lock_op()) {
4411 return new RGWPutBucketObjectLock_ObjStore_S3
;
4412 } else if (is_notification_op()) {
4413 return RGWHandler_REST_PSNotifs_S3::create_put_op();
4414 } else if (is_replication_op()) {
4415 auto sync_policy_handler
= store
->svc()->zone
->get_sync_policy_handler(nullopt
);
4416 if (!sync_policy_handler
||
4417 sync_policy_handler
->is_legacy_config()) {
4421 return new RGWPutBucketReplication_ObjStore_S3
;
4422 } else if (is_block_public_access_op()) {
4423 return new RGWPutBucketPublicAccessBlock_ObjStore_S3
;
4425 return new RGWCreateBucket_ObjStore_S3
;
4428 RGWOp
*RGWHandler_REST_Bucket_S3::op_delete()
4430 if (s
->info
.args
.sub_resource_exists("logging") ||
4431 s
->info
.args
.sub_resource_exists("encryption"))
4434 if (is_tagging_op()) {
4435 return new RGWDeleteBucketTags_ObjStore_S3
;
4436 } else if (is_cors_op()) {
4437 return new RGWDeleteCORS_ObjStore_S3
;
4438 } else if(is_lc_op()) {
4439 return new RGWDeleteLC_ObjStore_S3
;
4440 } else if(is_policy_op()) {
4441 return new RGWDeleteBucketPolicy
;
4442 } else if (is_notification_op()) {
4443 return RGWHandler_REST_PSNotifs_S3::create_delete_op();
4444 } else if (is_replication_op()) {
4445 return new RGWDeleteBucketReplication_ObjStore_S3
;
4446 } else if (is_block_public_access_op()) {
4447 return new RGWDeleteBucketPublicAccessBlock
;
4450 if (s
->info
.args
.sub_resource_exists("website")) {
4451 if (!s
->cct
->_conf
->rgw_enable_static_website
) {
4454 return new RGWDeleteBucketWebsite_ObjStore_S3
;
4457 if (s
->info
.args
.exists("mdsearch")) {
4458 return new RGWDelBucketMetaSearch_ObjStore_S3
;
4461 return new RGWDeleteBucket_ObjStore_S3
;
4464 RGWOp
*RGWHandler_REST_Bucket_S3::op_post()
4466 if (s
->info
.args
.exists("delete")) {
4467 return new RGWDeleteMultiObj_ObjStore_S3
;
4470 if (s
->info
.args
.exists("mdsearch")) {
4471 return new RGWConfigBucketMetaSearch_ObjStore_S3
;
4474 return new RGWPostObj_ObjStore_S3
;
4477 RGWOp
*RGWHandler_REST_Bucket_S3::op_options()
4479 return new RGWOptionsCORS_ObjStore_S3
;
4482 RGWOp
*RGWHandler_REST_Obj_S3::get_obj_op(bool get_data
)
4484 RGWGetObj_ObjStore_S3
*get_obj_op
= new RGWGetObj_ObjStore_S3
;
4485 get_obj_op
->set_get_data(get_data
);
4489 RGWOp
*RGWHandler_REST_Obj_S3::op_get()
4492 return new RGWGetACLs_ObjStore_S3
;
4493 } else if (s
->info
.args
.exists("uploadId")) {
4494 return new RGWListMultipart_ObjStore_S3
;
4495 } else if (s
->info
.args
.exists("layout")) {
4496 return new RGWGetObjLayout_ObjStore_S3
;
4497 } else if (is_tagging_op()) {
4498 return new RGWGetObjTags_ObjStore_S3
;
4499 } else if (is_obj_retention_op()) {
4500 return new RGWGetObjRetention_ObjStore_S3
;
4501 } else if (is_obj_legal_hold_op()) {
4502 return new RGWGetObjLegalHold_ObjStore_S3
;
4504 return get_obj_op(true);
4507 RGWOp
*RGWHandler_REST_Obj_S3::op_head()
4510 return new RGWGetACLs_ObjStore_S3
;
4511 } else if (s
->info
.args
.exists("uploadId")) {
4512 return new RGWListMultipart_ObjStore_S3
;
4514 return get_obj_op(false);
4517 RGWOp
*RGWHandler_REST_Obj_S3::op_put()
4520 return new RGWPutACLs_ObjStore_S3
;
4521 } else if (is_tagging_op()) {
4522 return new RGWPutObjTags_ObjStore_S3
;
4523 } else if (is_obj_retention_op()) {
4524 return new RGWPutObjRetention_ObjStore_S3
;
4525 } else if (is_obj_legal_hold_op()) {
4526 return new RGWPutObjLegalHold_ObjStore_S3
;
4529 if (s
->init_state
.src_bucket
.empty())
4530 return new RGWPutObj_ObjStore_S3
;
4532 return new RGWCopyObj_ObjStore_S3
;
4535 RGWOp
*RGWHandler_REST_Obj_S3::op_delete()
4537 if (is_tagging_op()) {
4538 return new RGWDeleteObjTags_ObjStore_S3
;
4540 string upload_id
= s
->info
.args
.get("uploadId");
4542 if (upload_id
.empty())
4543 return new RGWDeleteObj_ObjStore_S3
;
4545 return new RGWAbortMultipart_ObjStore_S3
;
4548 RGWOp
*RGWHandler_REST_Obj_S3::op_post()
4550 if (s
->info
.args
.exists("uploadId"))
4551 return new RGWCompleteMultipart_ObjStore_S3
;
4553 if (s
->info
.args
.exists("uploads"))
4554 return new RGWInitMultipart_ObjStore_S3
;
4557 return new RGWSelectObj_ObjStore_S3
;
4559 return new RGWPostObj_ObjStore_S3
;
4562 RGWOp
*RGWHandler_REST_Obj_S3::op_options()
4564 return new RGWOptionsCORS_ObjStore_S3
;
4567 int RGWHandler_REST_S3::init_from_header(rgw::sal::RGWRadosStore
*store
,
4568 struct req_state
* s
,
4569 int default_formatter
,
4570 bool configurable_format
)
4575 const char *req_name
= s
->relative_uri
.c_str();
4578 if (*req_name
== '?') {
4581 p
= s
->info
.request_params
.c_str();
4584 s
->info
.args
.set(p
);
4585 s
->info
.args
.parse(s
);
4587 /* must be called after the args parsing */
4588 int ret
= allocate_formatter(s
, default_formatter
, configurable_format
);
4592 if (*req_name
!= '/')
4601 int pos
= req
.find('/');
4603 first
= req
.substr(0, pos
);
4609 * XXX The intent of the check for empty is apparently to let the bucket
4610 * name from DNS to be set ahead. However, we currently take the DNS
4611 * bucket and re-insert it into URL in rgw_rest.cc:RGWREST::preprocess().
4612 * So, this check is meaningless.
4614 * Rather than dropping this, the code needs to be changed into putting
4615 * the bucket (and its tenant) from DNS and Host: header (HTTP_HOST)
4616 * into req_status.bucket_name directly.
4618 if (s
->init_state
.url_bucket
.empty()) {
4619 // Save bucket to tide us over until token is parsed.
4620 s
->init_state
.url_bucket
= first
;
4621 string encoded_obj_str
;
4623 encoded_obj_str
= req
.substr(pos
+1);
4626 if (!encoded_obj_str
.empty()) {
4628 s
->object
= s
->bucket
->get_object(rgw_obj_key(encoded_obj_str
, s
->info
.args
.get("versionId")));
4630 s
->object
= store
->get_object(rgw_obj_key(encoded_obj_str
, s
->info
.args
.get("versionId")));
4635 s
->object
= s
->bucket
->get_object(rgw_obj_key(req_name
, s
->info
.args
.get("versionId")));
4637 s
->object
= store
->get_object(rgw_obj_key(req_name
, s
->info
.args
.get("versionId")));
4643 static int verify_mfa(rgw::sal::RGWRadosStore
*store
, RGWUserInfo
*user
,
4644 const string
& mfa_str
, bool *verified
, const DoutPrefixProvider
*dpp
, optional_yield y
)
4646 vector
<string
> params
;
4647 get_str_vec(mfa_str
, " ", params
);
4649 if (params
.size() != 2) {
4650 ldpp_dout(dpp
, 5) << "NOTICE: invalid mfa string provided: " << mfa_str
<< dendl
;
4654 string
& serial
= params
[0];
4655 string
& pin
= params
[1];
4657 auto i
= user
->mfa_ids
.find(serial
);
4658 if (i
== user
->mfa_ids
.end()) {
4659 ldpp_dout(dpp
, 5) << "NOTICE: user does not have mfa device with serial=" << serial
<< dendl
;
4663 int ret
= store
->svc()->cls
->mfa
.check_mfa(dpp
, user
->user_id
, serial
, pin
, y
);
4665 ldpp_dout(dpp
, 20) << "NOTICE: failed to check MFA, serial=" << serial
<< dendl
;
4674 int RGWHandler_REST_S3::postauth_init(optional_yield y
)
4676 struct req_init_state
*t
= &s
->init_state
;
4678 rgw_parse_url_bucket(t
->url_bucket
, s
->user
->get_tenant(),
4679 s
->bucket_tenant
, s
->bucket_name
);
4681 if (s
->auth
.identity
->get_identity_type() == TYPE_ROLE
) {
4682 s
->bucket_tenant
= s
->auth
.identity
->get_role_tenant();
4685 ldpp_dout(s
, 10) << "s->object=" << s
->object
4686 << " s->bucket=" << rgw_make_bucket_entry_name(s
->bucket_tenant
, s
->bucket_name
) << dendl
;
4689 ret
= rgw_validate_tenant_name(s
->bucket_tenant
);
4692 if (!s
->bucket_name
.empty() && !rgw::sal::RGWObject::empty(s
->object
.get())) {
4693 ret
= validate_object_name(s
->object
->get_name());
4698 if (!t
->src_bucket
.empty()) {
4699 rgw_parse_url_bucket(t
->src_bucket
, s
->user
->get_tenant(),
4700 s
->src_tenant_name
, s
->src_bucket_name
);
4701 ret
= rgw_validate_tenant_name(s
->src_tenant_name
);
4706 const char *mfa
= s
->info
.env
->get("HTTP_X_AMZ_MFA");
4708 ret
= verify_mfa(store
, &s
->user
->get_info(), string(mfa
), &s
->mfa_verified
, s
, y
);
4714 int RGWHandler_REST_S3::init(rgw::sal::RGWRadosStore
*store
, struct req_state
*s
,
4715 rgw::io::BasicClient
*cio
)
4721 ret
= rgw_validate_tenant_name(s
->bucket_tenant
);
4724 if (!s
->bucket_name
.empty()) {
4725 ret
= validate_object_name(s
->object
->get_name());
4730 const char *cacl
= s
->info
.env
->get("HTTP_X_AMZ_ACL");
4732 s
->canned_acl
= cacl
;
4734 s
->has_acl_header
= s
->info
.env
->exists_prefix("HTTP_X_AMZ_GRANT");
4736 const char *copy_source
= s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE");
4738 (! s
->info
.env
->get("HTTP_X_AMZ_COPY_SOURCE_RANGE")) &&
4739 (! s
->info
.args
.exists("uploadId"))) {
4742 ret
= RGWCopyObj::parse_copy_location(copy_source
,
4743 s
->init_state
.src_bucket
,
4747 ldpp_dout(s
, 0) << "failed to parse copy location" << dendl
;
4748 return -EINVAL
; // XXX why not -ERR_INVALID_BUCKET_NAME or -ERR_BAD_URL?
4750 s
->src_object
= store
->get_object(key
);
4753 const char *sc
= s
->info
.env
->get("HTTP_X_AMZ_STORAGE_CLASS");
4755 s
->info
.storage_class
= sc
;
4758 return RGWHandler_REST::init(store
, s
, cio
);
4761 int RGWHandler_REST_S3::authorize(const DoutPrefixProvider
*dpp
, optional_yield y
)
4763 if (s
->info
.args
.exists("Action") && s
->info
.args
.get("Action") == "AssumeRoleWithWebIdentity") {
4764 return RGW_Auth_STS::authorize(dpp
, store
, auth_registry
, s
, y
);
4766 return RGW_Auth_S3::authorize(dpp
, store
, auth_registry
, s
, y
);
4769 enum class AwsVersion
{
4775 enum class AwsRoute
{
4781 static inline std::pair
<AwsVersion
, AwsRoute
>
4782 discover_aws_flavour(const req_info
& info
)
4784 using rgw::auth::s3::AWS4_HMAC_SHA256_STR
;
4786 AwsVersion version
= AwsVersion::UNKNOWN
;
4787 AwsRoute route
= AwsRoute::UNKNOWN
;
4789 const char* http_auth
= info
.env
->get("HTTP_AUTHORIZATION");
4790 if (http_auth
&& http_auth
[0]) {
4791 /* Authorization in Header */
4792 route
= AwsRoute::HEADERS
;
4794 if (!strncmp(http_auth
, AWS4_HMAC_SHA256_STR
,
4795 strlen(AWS4_HMAC_SHA256_STR
))) {
4797 version
= AwsVersion::V4
;
4798 } else if (!strncmp(http_auth
, "AWS ", 4)) {
4800 version
= AwsVersion::V2
;
4803 route
= AwsRoute::QUERY_STRING
;
4805 if (info
.args
.get("x-amz-algorithm") == AWS4_HMAC_SHA256_STR
) {
4807 version
= AwsVersion::V4
;
4808 } else if (!info
.args
.get("AWSAccessKeyId").empty()) {
4810 version
= AwsVersion::V2
;
4814 return std::make_pair(version
, route
);
4818 * verify that a signed request comes from the keyholder
4819 * by checking the signature against our locally-computed version
4821 * it tries AWS v4 before AWS v2
4823 int RGW_Auth_S3::authorize(const DoutPrefixProvider
*dpp
,
4824 rgw::sal::RGWRadosStore
* const store
,
4825 const rgw::auth::StrategyRegistry
& auth_registry
,
4826 struct req_state
* const s
, optional_yield y
)
4829 /* neither keystone and rados enabled; warn and exit! */
4830 if (!store
->ctx()->_conf
->rgw_s3_auth_use_rados
&&
4831 !store
->ctx()->_conf
->rgw_s3_auth_use_keystone
&&
4832 !store
->ctx()->_conf
->rgw_s3_auth_use_ldap
) {
4833 ldpp_dout(dpp
, 0) << "WARNING: no authorization backend enabled! Users will never authenticate." << dendl
;
4837 const auto ret
= rgw::auth::Strategy::apply(dpp
, auth_registry
.get_s3_main(), s
, y
);
4839 /* Populate the owner info. */
4840 s
->owner
.set_id(s
->user
->get_id());
4841 s
->owner
.set_name(s
->user
->get_display_name());
4846 int RGWHandler_Auth_S3::init(rgw::sal::RGWRadosStore
*store
, struct req_state
*state
,
4847 rgw::io::BasicClient
*cio
)
4849 int ret
= RGWHandler_REST_S3::init_from_header(store
, state
, RGW_FORMAT_JSON
, true);
4853 return RGWHandler_REST::init(store
, state
, cio
);
4856 RGWHandler_REST
* RGWRESTMgr_S3::get_handler(rgw::sal::RGWRadosStore
*store
,
4857 struct req_state
* const s
,
4858 const rgw::auth::StrategyRegistry
& auth_registry
,
4859 const std::string
& frontend_prefix
)
4861 bool is_s3website
= enable_s3website
&& (s
->prot_flags
& RGW_REST_WEBSITE
);
4863 RGWHandler_REST_S3::init_from_header(store
, s
,
4864 is_s3website
? RGW_FORMAT_HTML
:
4865 RGW_FORMAT_XML
, true);
4869 RGWHandler_REST
* handler
;
4870 // TODO: Make this more readable
4872 if (s
->init_state
.url_bucket
.empty()) {
4873 handler
= new RGWHandler_REST_Service_S3Website(auth_registry
);
4874 } else if (rgw::sal::RGWObject::empty(s
->object
.get())) {
4875 handler
= new RGWHandler_REST_Bucket_S3Website(auth_registry
);
4877 handler
= new RGWHandler_REST_Obj_S3Website(auth_registry
);
4880 if (s
->init_state
.url_bucket
.empty()) {
4881 handler
= new RGWHandler_REST_Service_S3(auth_registry
, enable_sts
, enable_iam
, enable_pubsub
);
4882 } else if (rgw::sal::RGWObject::empty(s
->object
.get())) {
4883 handler
= new RGWHandler_REST_Bucket_S3(auth_registry
, enable_pubsub
);
4885 handler
= new RGWHandler_REST_Obj_S3(auth_registry
);
4889 ldpp_dout(s
, 20) << __func__
<< " handler=" << typeid(*handler
).name()
4894 bool RGWHandler_REST_S3Website::web_dir() const {
4895 std::string subdir_name
;
4896 if (!rgw::sal::RGWObject::empty(s
->object
.get())) {
4897 subdir_name
= url_decode(s
->object
->get_name());
4900 if (subdir_name
.empty()) {
4902 } else if (subdir_name
.back() == '/' && subdir_name
.size() > 1) {
4903 subdir_name
.pop_back();
4906 rgw_obj
obj(s
->bucket
->get_key(), subdir_name
);
4908 RGWObjectCtx
& obj_ctx
= *static_cast<RGWObjectCtx
*>(s
->obj_ctx
);
4909 obj_ctx
.set_atomic(obj
);
4910 obj_ctx
.set_prefetch_data(obj
);
4912 RGWObjState
* state
= nullptr;
4913 if (store
->getRados()->get_obj_state(s
, &obj_ctx
, s
->bucket
->get_info(), obj
, &state
, false, s
->yield
) < 0) {
4916 if (! state
->exists
) {
4919 return state
->exists
;
4922 int RGWHandler_REST_S3Website::init(rgw::sal::RGWRadosStore
*store
, req_state
*s
,
4923 rgw::io::BasicClient
* cio
)
4925 // save the original object name before retarget() replaces it with the
4926 // result of get_effective_key(). the error_handler() needs the original
4927 // object name for redirect handling
4928 if (!rgw::sal::RGWObject::empty(s
->object
.get())) {
4929 original_object_name
= s
->object
->get_name();
4931 original_object_name
= "";
4934 return RGWHandler_REST_S3::init(store
, s
, cio
);
4937 int RGWHandler_REST_S3Website::retarget(RGWOp
* op
, RGWOp
** new_op
, optional_yield y
) {
4939 ldpp_dout(s
, 10) << __func__
<< " Starting retarget" << dendl
;
4941 if (!(s
->prot_flags
& RGW_REST_WEBSITE
))
4944 int ret
= store
->get_bucket(s
, nullptr, s
->bucket_tenant
, s
->bucket_name
, &s
->bucket
, y
);
4946 // TODO-FUTURE: if the bucket does not exist, maybe expose it here?
4947 return -ERR_NO_SUCH_BUCKET
;
4950 s
->bucket_attrs
= s
->bucket
->get_attrs();
4952 if (!s
->bucket
->get_info().has_website
) {
4953 // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here
4954 return -ERR_NO_SUCH_WEBSITE_CONFIGURATION
;
4957 rgw_obj_key new_obj
;
4959 if (!rgw::sal::RGWObject::empty(s
->object
.get())) {
4960 key_name
= s
->object
->get_name();
4962 bool get_res
= s
->bucket
->get_info().website_conf
.get_effective_key(key_name
, &new_obj
.name
, web_dir());
4964 s
->err
.message
= "The IndexDocument Suffix is not configurated or not well formed!";
4965 ldpp_dout(s
, 5) << s
->err
.message
<< dendl
;
4969 ldpp_dout(s
, 10) << "retarget get_effective_key " << s
->object
<< " -> "
4970 << new_obj
<< dendl
;
4972 RGWBWRoutingRule rrule
;
4973 bool should_redirect
=
4974 s
->bucket
->get_info().website_conf
.should_redirect(new_obj
.name
, 0, &rrule
);
4976 if (should_redirect
) {
4977 const string
& hostname
= s
->info
.env
->get("HTTP_HOST", "");
4978 const string
& protocol
=
4979 (s
->info
.env
->get("SERVER_PORT_SECURE") ? "https" : "http");
4980 int redirect_code
= 0;
4981 rrule
.apply_rule(protocol
, hostname
, key_name
, &s
->redirect
,
4983 // APply a custom HTTP response code
4984 if (redirect_code
> 0)
4985 s
->err
.http_ret
= redirect_code
; // Apply a custom HTTP response code
4986 ldpp_dout(s
, 10) << "retarget redirect code=" << redirect_code
4987 << " proto+host:" << protocol
<< "://" << hostname
4988 << " -> " << s
->redirect
<< dendl
;
4989 return -ERR_WEBSITE_REDIRECT
;
4993 * FIXME: if s->object != new_obj, drop op and create a new op to handle
4994 * operation. Or remove this comment if it's not applicable anymore
4997 s
->object
= store
->get_object(new_obj
);
4998 s
->object
->set_bucket(s
->bucket
.get());
5003 RGWOp
* RGWHandler_REST_S3Website::op_get()
5005 return get_obj_op(true);
5008 RGWOp
* RGWHandler_REST_S3Website::op_head()
5010 return get_obj_op(false);
5013 int RGWHandler_REST_S3Website::serve_errordoc(const DoutPrefixProvider
*dpp
, int http_ret
, const string
& errordoc_key
, optional_yield y
) {
5015 s
->formatter
->reset(); /* Try to throw it all away */
5017 std::shared_ptr
<RGWGetObj_ObjStore_S3Website
> getop( static_cast<RGWGetObj_ObjStore_S3Website
*>(op_get()));
5018 if (getop
.get() == NULL
) {
5019 return -1; // Trigger double error handler
5021 getop
->init(store
, s
, this);
5022 getop
->range_str
= NULL
;
5023 getop
->if_mod
= NULL
;
5024 getop
->if_unmod
= NULL
;
5025 getop
->if_match
= NULL
;
5026 getop
->if_nomatch
= NULL
;
5027 s
->object
= store
->get_object(errordoc_key
);
5029 ret
= init_permissions(getop
.get(), y
);
5031 ldpp_dout(s
, 20) << "serve_errordoc failed, init_permissions ret=" << ret
<< dendl
;
5032 return -1; // Trigger double error handler
5035 ret
= read_permissions(getop
.get(), y
);
5037 ldpp_dout(s
, 20) << "serve_errordoc failed, read_permissions ret=" << ret
<< dendl
;
5038 return -1; // Trigger double error handler
5042 getop
->set_custom_http_response(http_ret
);
5045 ret
= getop
->init_processing(y
);
5047 ldpp_dout(s
, 20) << "serve_errordoc failed, init_processing ret=" << ret
<< dendl
;
5048 return -1; // Trigger double error handler
5051 ret
= getop
->verify_op_mask();
5053 ldpp_dout(s
, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret
<< dendl
;
5054 return -1; // Trigger double error handler
5057 ret
= getop
->verify_permission(y
);
5059 ldpp_dout(s
, 20) << "serve_errordoc failed, verify_permission ret=" << ret
<< dendl
;
5060 return -1; // Trigger double error handler
5063 ret
= getop
->verify_params();
5065 ldpp_dout(s
, 20) << "serve_errordoc failed, verify_params ret=" << ret
<< dendl
;
5066 return -1; // Trigger double error handler
5069 // No going back now
5072 * FIXME Missing headers:
5073 * With a working errordoc, the s3 error fields are rendered as HTTP headers,
5074 * x-amz-error-code: NoSuchKey
5075 * x-amz-error-message: The specified key does not exist.
5076 * x-amz-error-detail-Key: foo
5083 int RGWHandler_REST_S3Website::error_handler(int err_no
,
5084 string
* error_content
,
5086 int new_err_no
= -1;
5087 rgw_http_errors::const_iterator r
= rgw_http_s3_errors
.find(err_no
> 0 ? err_no
: -err_no
);
5088 int http_error_code
= -1;
5090 if (r
!= rgw_http_s3_errors
.end()) {
5091 http_error_code
= r
->second
.first
;
5093 ldpp_dout(s
, 10) << "RGWHandler_REST_S3Website::error_handler err_no=" << err_no
<< " http_ret=" << http_error_code
<< dendl
;
5095 RGWBWRoutingRule rrule
;
5096 bool have_bucket
= !rgw::sal::RGWBucket::empty(s
->bucket
.get());
5097 bool should_redirect
= false;
5100 s
->bucket
->get_info().website_conf
.should_redirect(original_object_name
,
5101 http_error_code
, &rrule
);
5104 if (should_redirect
) {
5105 const string
& hostname
= s
->info
.env
->get("HTTP_HOST", "");
5106 const string
& protocol
=
5107 (s
->info
.env
->get("SERVER_PORT_SECURE") ? "https" : "http");
5108 int redirect_code
= 0;
5109 rrule
.apply_rule(protocol
, hostname
, original_object_name
,
5110 &s
->redirect
, &redirect_code
);
5111 // Apply a custom HTTP response code
5112 if (redirect_code
> 0)
5113 s
->err
.http_ret
= redirect_code
; // Apply a custom HTTP response code
5114 ldpp_dout(s
, 10) << "error handler redirect code=" << redirect_code
5115 << " proto+host:" << protocol
<< "://" << hostname
5116 << " -> " << s
->redirect
<< dendl
;
5117 return -ERR_WEBSITE_REDIRECT
;
5118 } else if (err_no
== -ERR_WEBSITE_REDIRECT
) {
5119 // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block
5120 // Do NOT fire the ErrorDoc handler
5121 } else if (have_bucket
&& !s
->bucket
->get_info().website_conf
.error_doc
.empty()) {
5122 /* This serves an entire page!
5123 On success, it will return zero, and no further content should be sent to the socket
5124 On failure, we need the double-error handler
5126 new_err_no
= RGWHandler_REST_S3Website::serve_errordoc(s
, http_error_code
, s
->bucket
->get_info().website_conf
.error_doc
, y
);
5127 if (new_err_no
!= -1) {
5128 err_no
= new_err_no
;
5131 ldpp_dout(s
, 20) << "No special error handling today!" << dendl
;
5137 RGWOp
* RGWHandler_REST_Obj_S3Website::get_obj_op(bool get_data
)
5139 /** If we are in website mode, then it is explicitly impossible to run GET or
5140 * HEAD on the actual directory. We must convert the request to run on the
5141 * suffix object instead!
5143 RGWGetObj_ObjStore_S3Website
* op
= new RGWGetObj_ObjStore_S3Website
;
5144 op
->set_get_data(get_data
);
5148 RGWOp
* RGWHandler_REST_Bucket_S3Website::get_obj_op(bool get_data
)
5150 /** If we are in website mode, then it is explicitly impossible to run GET or
5151 * HEAD on the actual directory. We must convert the request to run on the
5152 * suffix object instead!
5154 RGWGetObj_ObjStore_S3Website
* op
= new RGWGetObj_ObjStore_S3Website
;
5155 op
->set_get_data(get_data
);
5159 RGWOp
* RGWHandler_REST_Service_S3Website::get_obj_op(bool get_data
)
5161 /** If we are in website mode, then it is explicitly impossible to run GET or
5162 * HEAD on the actual directory. We must convert the request to run on the
5163 * suffix object instead!
5165 RGWGetObj_ObjStore_S3Website
* op
= new RGWGetObj_ObjStore_S3Website
;
5166 op
->set_get_data(get_data
);
5171 namespace rgw::auth::s3
{
5173 static rgw::auth::Completer::cmplptr_t
5174 null_completer_factory(const boost::optional
<std::string
>& secret_key
)
5180 AWSEngine::VersionAbstractor::auth_data_t
5181 AWSGeneralAbstractor::get_auth_data(const req_state
* const s
) const
5185 std::tie(version
, route
) = discover_aws_flavour(s
->info
);
5187 if (version
== AwsVersion::V2
) {
5188 return get_auth_data_v2(s
);
5189 } else if (version
== AwsVersion::V4
) {
5190 return get_auth_data_v4(s
, route
== AwsRoute::QUERY_STRING
);
5192 /* FIXME(rzarzynski): handle anon user. */
5197 boost::optional
<std::string
>
5198 AWSGeneralAbstractor::get_v4_canonical_headers(
5199 const req_info
& info
,
5200 const std::string_view
& signedheaders
,
5201 const bool using_qs
) const
5203 return rgw::auth::s3::get_v4_canonical_headers(info
, signedheaders
,
5207 AWSEngine::VersionAbstractor::auth_data_t
5208 AWSGeneralAbstractor::get_auth_data_v4(const req_state
* const s
,
5209 const bool using_qs
) const
5211 std::string_view access_key_id
;
5212 std::string_view signed_hdrs
;
5214 std::string_view date
;
5215 std::string_view credential_scope
;
5216 std::string_view client_signature
;
5217 std::string_view session_token
;
5219 int ret
= rgw::auth::s3::parse_v4_credentials(s
->info
,
5232 /* craft canonical headers */
5233 boost::optional
<std::string
> canonical_headers
= \
5234 get_v4_canonical_headers(s
->info
, signed_hdrs
, using_qs
);
5235 if (canonical_headers
) {
5236 using sanitize
= rgw::crypt_sanitize::log_content
;
5237 ldpp_dout(s
, 10) << "canonical headers format = "
5238 << sanitize
{*canonical_headers
} << dendl
;
5243 bool is_non_s3_op
= false;
5244 if (s
->op_type
== RGW_STS_GET_SESSION_TOKEN
||
5245 s
->op_type
== RGW_STS_ASSUME_ROLE
||
5246 s
->op_type
== RGW_STS_ASSUME_ROLE_WEB_IDENTITY
||
5247 s
->op_type
== RGW_OP_CREATE_ROLE
||
5248 s
->op_type
== RGW_OP_DELETE_ROLE
||
5249 s
->op_type
== RGW_OP_GET_ROLE
||
5250 s
->op_type
== RGW_OP_MODIFY_ROLE
||
5251 s
->op_type
== RGW_OP_LIST_ROLES
||
5252 s
->op_type
== RGW_OP_PUT_ROLE_POLICY
||
5253 s
->op_type
== RGW_OP_GET_ROLE_POLICY
||
5254 s
->op_type
== RGW_OP_LIST_ROLE_POLICIES
||
5255 s
->op_type
== RGW_OP_DELETE_ROLE_POLICY
||
5256 s
->op_type
== RGW_OP_PUT_USER_POLICY
||
5257 s
->op_type
== RGW_OP_GET_USER_POLICY
||
5258 s
->op_type
== RGW_OP_LIST_USER_POLICIES
||
5259 s
->op_type
== RGW_OP_DELETE_USER_POLICY
||
5260 s
->op_type
== RGW_OP_CREATE_OIDC_PROVIDER
||
5261 s
->op_type
== RGW_OP_DELETE_OIDC_PROVIDER
||
5262 s
->op_type
== RGW_OP_GET_OIDC_PROVIDER
||
5263 s
->op_type
== RGW_OP_LIST_OIDC_PROVIDERS
) {
5264 is_non_s3_op
= true;
5267 const char* exp_payload_hash
= nullptr;
5268 string payload_hash
;
5270 //For non s3 ops, we need to calculate the payload hash
5271 payload_hash
= s
->info
.args
.get("PayloadHash");
5272 exp_payload_hash
= payload_hash
.c_str();
5274 /* Get the expected hash. */
5275 exp_payload_hash
= rgw::auth::s3::get_v4_exp_payload_hash(s
->info
);
5278 /* Craft canonical URI. Using std::move later so let it be non-const. */
5279 auto canonical_uri
= rgw::auth::s3::get_v4_canonical_uri(s
->info
);
5281 /* Craft canonical query string. std::moving later so non-const here. */
5282 auto canonical_qs
= rgw::auth::s3::get_v4_canonical_qs(s
->info
, using_qs
);
5284 /* Craft canonical request. */
5285 auto canonical_req_hash
= \
5286 rgw::auth::s3::get_v4_canon_req_hash(s
->cct
,
5288 std::move(canonical_uri
),
5289 std::move(canonical_qs
),
5290 std::move(*canonical_headers
),
5295 auto string_to_sign
= \
5296 rgw::auth::s3::get_v4_string_to_sign(s
->cct
,
5297 AWS4_HMAC_SHA256_STR
,
5300 std::move(canonical_req_hash
),
5303 const auto sig_factory
= std::bind(rgw::auth::s3::get_v4_signature
,
5305 std::placeholders::_1
,
5306 std::placeholders::_2
,
5307 std::placeholders::_3
,
5310 /* Requests authenticated with the Query Parameters are treated as unsigned.
5311 * From "Authenticating Requests: Using Query Parameters (AWS Signature
5314 * You don't include a payload hash in the Canonical Request, because
5315 * when you create a presigned URL, you don't know the payload content
5316 * because the URL is used to upload an arbitrary payload. Instead, you
5317 * use a constant string UNSIGNED-PAYLOAD.
5319 * This means we have absolutely no business in spawning completer. Both
5320 * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
5321 * by default. We don't need to change that. */
5322 if (is_v4_payload_unsigned(exp_payload_hash
) || is_v4_payload_empty(s
) || is_non_s3_op
) {
5327 std::move(string_to_sign
),
5329 null_completer_factory
5332 /* We're going to handle a signed payload. Be aware that even empty HTTP
5333 * body (no payload) requires verification:
5335 * The x-amz-content-sha256 header is required for all AWS Signature
5336 * Version 4 requests. It provides a hash of the request payload. If
5337 * there is no payload, you must provide the hash of an empty string. */
5338 if (!is_v4_payload_streamed(exp_payload_hash
)) {
5339 ldpp_dout(s
, 10) << "delaying v4 auth" << dendl
;
5341 /* payload in a single chunk */
5344 case RGW_OP_CREATE_BUCKET
:
5345 case RGW_OP_PUT_OBJ
:
5346 case RGW_OP_PUT_ACLS
:
5347 case RGW_OP_PUT_CORS
:
5348 case RGW_OP_INIT_MULTIPART
: // in case that Init Multipart uses CHUNK encoding
5349 case RGW_OP_COMPLETE_MULTIPART
:
5350 case RGW_OP_SET_BUCKET_VERSIONING
:
5351 case RGW_OP_DELETE_MULTI_OBJ
:
5352 case RGW_OP_ADMIN_SET_METADATA
:
5353 case RGW_OP_SET_BUCKET_WEBSITE
:
5354 case RGW_OP_PUT_BUCKET_POLICY
:
5355 case RGW_OP_PUT_OBJ_TAGGING
:
5356 case RGW_OP_PUT_BUCKET_TAGGING
:
5357 case RGW_OP_PUT_BUCKET_REPLICATION
:
5359 case RGW_OP_SET_REQUEST_PAYMENT
:
5360 case RGW_OP_PUBSUB_NOTIF_CREATE
:
5361 case RGW_OP_PUT_BUCKET_OBJ_LOCK
:
5362 case RGW_OP_PUT_OBJ_RETENTION
:
5363 case RGW_OP_PUT_OBJ_LEGAL_HOLD
:
5364 case RGW_STS_GET_SESSION_TOKEN
:
5365 case RGW_STS_ASSUME_ROLE
:
5366 case RGW_OP_PUT_BUCKET_PUBLIC_ACCESS_BLOCK
:
5367 case RGW_OP_GET_BUCKET_PUBLIC_ACCESS_BLOCK
:
5368 case RGW_OP_DELETE_BUCKET_PUBLIC_ACCESS_BLOCK
:
5369 case RGW_OP_GET_OBJ
://s3select its post-method(payload contain the query) , the request is get-object
5372 ldpp_dout(s
, 10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl
;
5373 throw -ERR_NOT_IMPLEMENTED
;
5376 const auto cmpl_factory
= std::bind(AWSv4ComplSingle::create
,
5378 std::placeholders::_1
);
5383 std::move(string_to_sign
),
5388 /* IMHO "streamed" doesn't fit too good here. I would prefer to call
5389 * it "chunked" but let's be coherent with Amazon's terminology. */
5391 ldpp_dout(s
, 10) << "body content detected in multiple chunks" << dendl
;
5393 /* payload in multiple chunks */
5397 case RGW_OP_PUT_OBJ
:
5400 ldpp_dout(s
, 10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl
;
5401 throw -ERR_NOT_IMPLEMENTED
;
5404 ldpp_dout(s
, 10) << "aws4 seed signature ok... delaying v4 auth" << dendl
;
5406 /* In the case of streamed payload client sets the x-amz-content-sha256
5407 * to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" but uses "UNSIGNED-PAYLOAD"
5408 * when constructing the Canonical Request. */
5410 /* In the case of single-chunk upload client set the header's value is
5411 * coherent with the one used for Canonical Request crafting. */
5413 /* In the case of query string-based authentication there should be no
5414 * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
5416 const auto cmpl_factory
= std::bind(AWSv4ComplMulti::create
,
5421 std::placeholders::_1
);
5426 std::move(string_to_sign
),
5435 boost::optional
<std::string
>
5436 AWSGeneralBoto2Abstractor::get_v4_canonical_headers(
5437 const req_info
& info
,
5438 const std::string_view
& signedheaders
,
5439 const bool using_qs
) const
5441 return rgw::auth::s3::get_v4_canonical_headers(info
, signedheaders
,
5446 AWSEngine::VersionAbstractor::auth_data_t
5447 AWSGeneralAbstractor::get_auth_data_v2(const req_state
* const s
) const
5449 std::string_view access_key_id
;
5450 std::string_view signature
;
5451 std::string_view session_token
;
5454 const char* http_auth
= s
->info
.env
->get("HTTP_AUTHORIZATION");
5455 if (! http_auth
|| http_auth
[0] == '\0') {
5456 /* Credentials are provided in query string. We also need to verify
5457 * the "Expires" parameter now. */
5458 access_key_id
= s
->info
.args
.get("AWSAccessKeyId");
5459 signature
= s
->info
.args
.get("Signature");
5462 std::string_view expires
= s
->info
.args
.get("Expires");
5463 if (expires
.empty()) {
5467 /* It looks we have the guarantee that expires is a null-terminated,
5468 * and thus string_view::data() can be safely used. */
5469 const time_t exp
= atoll(expires
.data());
5476 if (s
->info
.args
.exists("x-amz-security-token")) {
5477 session_token
= s
->info
.args
.get("x-amz-security-token");
5478 if (session_token
.size() == 0) {
5484 /* The "Authorization" HTTP header is being used. */
5485 const std::string_view
auth_str(http_auth
+ strlen("AWS "));
5486 const size_t pos
= auth_str
.rfind(':');
5487 if (pos
!= std::string_view::npos
) {
5488 access_key_id
= auth_str
.substr(0, pos
);
5489 signature
= auth_str
.substr(pos
+ 1);
5492 if (s
->info
.env
->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
5493 session_token
= s
->info
.env
->get("HTTP_X_AMZ_SECURITY_TOKEN");
5494 if (session_token
.size() == 0) {
5500 /* Let's canonize the HTTP headers that are covered by the AWS auth v2. */
5501 std::string string_to_sign
;
5502 utime_t header_time
;
5503 if (! rgw_create_s3_canonical_header(s
, s
->info
, &header_time
, string_to_sign
,
5505 ldpp_dout(s
, 10) << "failed to create the canonized auth header\n"
5506 << rgw::crypt_sanitize::auth
{s
,string_to_sign
} << dendl
;
5510 ldpp_dout(s
, 10) << "string_to_sign:\n"
5511 << rgw::crypt_sanitize::auth
{s
,string_to_sign
} << dendl
;
5513 if (!qsr
&& !is_time_skew_ok(header_time
)) {
5514 throw -ERR_REQUEST_TIME_SKEWED
;
5518 std::move(access_key_id
),
5519 std::move(signature
),
5520 std::move(session_token
),
5521 std::move(string_to_sign
),
5522 rgw::auth::s3::get_v2_signature
,
5523 null_completer_factory
5528 AWSEngine::VersionAbstractor::auth_data_t
5529 AWSBrowserUploadAbstractor::get_auth_data_v2(const req_state
* const s
) const
5532 s
->auth
.s3_postobj_creds
.access_key
,
5533 s
->auth
.s3_postobj_creds
.signature
,
5534 s
->auth
.s3_postobj_creds
.x_amz_security_token
,
5535 s
->auth
.s3_postobj_creds
.encoded_policy
.to_str(),
5536 rgw::auth::s3::get_v2_signature
,
5537 null_completer_factory
5541 AWSEngine::VersionAbstractor::auth_data_t
5542 AWSBrowserUploadAbstractor::get_auth_data_v4(const req_state
* const s
) const
5544 const std::string_view credential
= s
->auth
.s3_postobj_creds
.x_amz_credential
;
5546 /* grab access key id */
5547 const size_t pos
= credential
.find("/");
5548 const std::string_view access_key_id
= credential
.substr(0, pos
);
5549 ldpp_dout(s
, 10) << "access key id = " << access_key_id
<< dendl
;
5551 /* grab credential scope */
5552 const std::string_view credential_scope
= credential
.substr(pos
+ 1);
5553 ldpp_dout(s
, 10) << "credential scope = " << credential_scope
<< dendl
;
5555 const auto sig_factory
= std::bind(rgw::auth::s3::get_v4_signature
,
5557 std::placeholders::_1
,
5558 std::placeholders::_2
,
5559 std::placeholders::_3
,
5564 s
->auth
.s3_postobj_creds
.signature
,
5565 s
->auth
.s3_postobj_creds
.x_amz_security_token
,
5566 s
->auth
.s3_postobj_creds
.encoded_policy
.to_str(),
5568 null_completer_factory
5572 AWSEngine::VersionAbstractor::auth_data_t
5573 AWSBrowserUploadAbstractor::get_auth_data(const req_state
* const s
) const
5575 if (s
->auth
.s3_postobj_creds
.x_amz_algorithm
== AWS4_HMAC_SHA256_STR
) {
5576 ldpp_dout(s
, 0) << "Signature verification algorithm AWS v4"
5577 << " (AWS4-HMAC-SHA256)" << dendl
;
5578 return get_auth_data_v4(s
);
5580 ldpp_dout(s
, 0) << "Signature verification algorithm AWS v2" << dendl
;
5581 return get_auth_data_v2(s
);
5586 AWSEngine::authenticate(const DoutPrefixProvider
* dpp
, const req_state
* const s
, optional_yield y
) const
5588 /* Small reminder: an ver_abstractor is allowed to throw! */
5589 const auto auth_data
= ver_abstractor
.get_auth_data(s
);
5591 if (auth_data
.access_key_id
.empty() || auth_data
.client_signature
.empty()) {
5592 return result_t::deny(-EINVAL
);
5594 return authenticate(dpp
,
5595 auth_data
.access_key_id
,
5596 auth_data
.client_signature
,
5597 auth_data
.session_token
,
5598 auth_data
.string_to_sign
,
5599 auth_data
.signature_factory
,
5600 auth_data
.completer_factory
,
5605 } // namespace rgw::auth::s3
5607 rgw::LDAPHelper
* rgw::auth::s3::LDAPEngine::ldh
= nullptr;
5608 std::mutex
rgw::auth::s3::LDAPEngine::mtx
;
5610 void rgw::auth::s3::LDAPEngine::init(CephContext
* const cct
)
5612 if (! cct
->_conf
->rgw_s3_auth_use_ldap
||
5613 cct
->_conf
->rgw_ldap_uri
.empty()) {
5618 std::lock_guard
<std::mutex
> lck(mtx
);
5620 const string
& ldap_uri
= cct
->_conf
->rgw_ldap_uri
;
5621 const string
& ldap_binddn
= cct
->_conf
->rgw_ldap_binddn
;
5622 const string
& ldap_searchdn
= cct
->_conf
->rgw_ldap_searchdn
;
5623 const string
& ldap_searchfilter
= cct
->_conf
->rgw_ldap_searchfilter
;
5624 const string
& ldap_dnattr
= cct
->_conf
->rgw_ldap_dnattr
;
5625 std::string ldap_bindpw
= parse_rgw_ldap_bindpw(cct
);
5627 ldh
= new rgw::LDAPHelper(ldap_uri
, ldap_binddn
, ldap_bindpw
,
5628 ldap_searchdn
, ldap_searchfilter
, ldap_dnattr
);
5636 bool rgw::auth::s3::LDAPEngine::valid() {
5637 std::lock_guard
<std::mutex
> lck(mtx
);
5641 rgw::auth::RemoteApplier::acl_strategy_t
5642 rgw::auth::s3::LDAPEngine::get_acl_strategy() const
5644 //This is based on the assumption that the default acl strategy in
5645 // get_perms_from_aclspec, will take care. Extra acl spec is not required.
5649 rgw::auth::RemoteApplier::AuthInfo
5650 rgw::auth::s3::LDAPEngine::get_creds_info(const rgw::RGWToken
& token
) const noexcept
5652 /* The short form of "using" can't be used here -- we're aliasing a class'
5654 using acct_privilege_t
= \
5655 rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t
;
5657 return rgw::auth::RemoteApplier::AuthInfo
{
5660 RGW_PERM_FULL_CONTROL
,
5661 acct_privilege_t::IS_PLAIN_ACCT
,
5666 rgw::auth::Engine::result_t
5667 rgw::auth::s3::LDAPEngine::authenticate(
5668 const DoutPrefixProvider
* dpp
,
5669 const std::string_view
& access_key_id
,
5670 const std::string_view
& signature
,
5671 const std::string_view
& session_token
,
5672 const string_to_sign_t
& string_to_sign
,
5673 const signature_factory_t
&,
5674 const completer_factory_t
& completer_factory
,
5675 const req_state
* const s
,
5676 optional_yield y
) const
5678 /* boost filters and/or string_ref may throw on invalid input */
5679 rgw::RGWToken base64_token
;
5681 base64_token
= rgw::from_base64(access_key_id
);
5683 base64_token
= std::string("");
5686 if (! base64_token
.valid()) {
5687 return result_t::deny();
5690 //TODO: Uncomment, when we have a migration plan in place.
5691 //Check if a user of type other than 'ldap' is already present, if yes, then
5693 /*RGWUserInfo user_info;
5694 user_info.user_id = base64_token.id;
5695 if (rgw_get_user_info_by_uid(store, user_info.user_id, user_info) >= 0) {
5696 if (user_info.type != TYPE_LDAP) {
5697 ldpp_dout(dpp, 10) << "ERROR: User id of type: " << user_info.type << " is already present" << dendl;
5702 if (ldh
->auth(base64_token
.id
, base64_token
.key
) != 0) {
5703 return result_t::deny(-ERR_INVALID_ACCESS_KEY
);
5706 auto apl
= apl_factory
->create_apl_remote(cct
, s
, get_acl_strategy(),
5707 get_creds_info(base64_token
));
5708 return result_t::grant(std::move(apl
), completer_factory(boost::none
));
5709 } /* rgw::auth::s3::LDAPEngine::authenticate */
5711 void rgw::auth::s3::LDAPEngine::shutdown() {
5719 rgw::auth::Engine::result_t
5720 rgw::auth::s3::LocalEngine::authenticate(
5721 const DoutPrefixProvider
* dpp
,
5722 const std::string_view
& _access_key_id
,
5723 const std::string_view
& signature
,
5724 const std::string_view
& session_token
,
5725 const string_to_sign_t
& string_to_sign
,
5726 const signature_factory_t
& signature_factory
,
5727 const completer_factory_t
& completer_factory
,
5728 const req_state
* const s
,
5729 optional_yield y
) const
5731 /* get the user info */
5732 RGWUserInfo user_info
;
5733 /* TODO(rzarzynski): we need to have string-view taking variant. */
5734 const std::string
access_key_id(_access_key_id
);
5735 if (rgw_get_user_info_by_access_key(dpp
, ctl
->user
, access_key_id
, user_info
, y
) < 0) {
5736 ldpp_dout(dpp
, 5) << "error reading user info, uid=" << access_key_id
5737 << " can't authenticate" << dendl
;
5738 return result_t::deny(-ERR_INVALID_ACCESS_KEY
);
5740 //TODO: Uncomment, when we have a migration plan in place.
5742 if (s->user->type != TYPE_RGW) {
5743 ldpp_dout(dpp, 10) << "ERROR: User id of type: " << s->user->type
5744 << " is present" << dendl;
5749 const auto iter
= user_info
.access_keys
.find(access_key_id
);
5750 if (iter
== std::end(user_info
.access_keys
)) {
5751 ldpp_dout(dpp
, 0) << "ERROR: access key not encoded in user info" << dendl
;
5752 return result_t::deny(-EPERM
);
5754 const RGWAccessKey
& k
= iter
->second
;
5756 const VersionAbstractor::server_signature_t server_signature
= \
5757 signature_factory(cct
, k
.key
, string_to_sign
);
5758 auto compare
= signature
.compare(server_signature
);
5760 ldpp_dout(dpp
, 15) << "string_to_sign="
5761 << rgw::crypt_sanitize::log_content
{string_to_sign
}
5763 ldpp_dout(dpp
, 15) << "server signature=" << server_signature
<< dendl
;
5764 ldpp_dout(dpp
, 15) << "client signature=" << signature
<< dendl
;
5765 ldpp_dout(dpp
, 15) << "compare=" << compare
<< dendl
;
5768 return result_t::deny(-ERR_SIGNATURE_NO_MATCH
);
5771 auto apl
= apl_factory
->create_apl_local(cct
, s
, user_info
, k
.subuser
, boost::none
);
5772 return result_t::grant(std::move(apl
), completer_factory(k
.key
));
5775 rgw::auth::RemoteApplier::AuthInfo
5776 rgw::auth::s3::STSEngine::get_creds_info(const STS::SessionToken
& token
) const noexcept
5778 using acct_privilege_t
= \
5779 rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t
;
5781 return rgw::auth::RemoteApplier::AuthInfo
{
5785 (token
.is_admin
) ? acct_privilege_t::IS_ADMIN_ACCT
: acct_privilege_t::IS_PLAIN_ACCT
,
5791 rgw::auth::s3::STSEngine::get_session_token(const DoutPrefixProvider
* dpp
, const std::string_view
& session_token
,
5792 STS::SessionToken
& token
) const
5794 string decodedSessionToken
;
5796 decodedSessionToken
= rgw::from_base64(session_token
);
5798 ldpp_dout(dpp
, 0) << "ERROR: Invalid session token, not base64 encoded." << dendl
;
5802 auto* cryptohandler
= cct
->get_crypto_handler(CEPH_CRYPTO_AES
);
5803 if (! cryptohandler
) {
5806 string secret_s
= cct
->_conf
->rgw_sts_key
;
5807 buffer::ptr
secret(secret_s
.c_str(), secret_s
.length());
5809 if (ret
= cryptohandler
->validate_secret(secret
); ret
< 0) {
5810 ldpp_dout(dpp
, 0) << "ERROR: Invalid secret key" << dendl
;
5814 auto* keyhandler
= cryptohandler
->get_key_handler(secret
, error
);
5820 string decrypted_str
;
5821 buffer::list en_input
, dec_output
;
5822 en_input
= buffer::list::static_from_string(decodedSessionToken
);
5824 ret
= keyhandler
->decrypt(en_input
, dec_output
, &error
);
5826 ldpp_dout(dpp
, 0) << "ERROR: Decryption failed: " << error
<< dendl
;
5830 dec_output
.append('\0');
5831 auto iter
= dec_output
.cbegin();
5832 decode(token
, iter
);
5833 } catch (const buffer::error
& e
) {
5834 ldpp_dout(dpp
, 0) << "ERROR: decode SessionToken failed: " << error
<< dendl
;
5841 rgw::auth::Engine::result_t
5842 rgw::auth::s3::STSEngine::authenticate(
5843 const DoutPrefixProvider
* dpp
,
5844 const std::string_view
& _access_key_id
,
5845 const std::string_view
& signature
,
5846 const std::string_view
& session_token
,
5847 const string_to_sign_t
& string_to_sign
,
5848 const signature_factory_t
& signature_factory
,
5849 const completer_factory_t
& completer_factory
,
5850 const req_state
* const s
,
5851 optional_yield y
) const
5853 if (! s
->info
.args
.exists("x-amz-security-token") &&
5854 ! s
->info
.env
->exists("HTTP_X_AMZ_SECURITY_TOKEN") &&
5855 s
->auth
.s3_postobj_creds
.x_amz_security_token
.empty()) {
5856 return result_t::deny();
5859 STS::SessionToken token
;
5860 if (int ret
= get_session_token(dpp
, session_token
, token
); ret
< 0) {
5861 return result_t::reject(ret
);
5864 //Check if access key is not the same passed in by client
5865 if (token
.access_key_id
!= _access_key_id
) {
5866 ldpp_dout(dpp
, 0) << "Invalid access key" << dendl
;
5867 return result_t::reject(-EPERM
);
5869 //Check if the token has expired
5870 if (! token
.expiration
.empty()) {
5871 std::string expiration
= token
.expiration
;
5872 if (! expiration
.empty()) {
5873 boost::optional
<real_clock::time_point
> exp
= ceph::from_iso_8601(expiration
, false);
5875 real_clock::time_point now
= real_clock::now();
5877 ldpp_dout(dpp
, 0) << "ERROR: Token expired" << dendl
;
5878 return result_t::reject(-EPERM
);
5881 ldpp_dout(dpp
, 0) << "ERROR: Invalid expiration: " << expiration
<< dendl
;
5882 return result_t::reject(-EPERM
);
5886 //Check for signature mismatch
5887 const VersionAbstractor::server_signature_t server_signature
= \
5888 signature_factory(cct
, token
.secret_access_key
, string_to_sign
);
5889 auto compare
= signature
.compare(server_signature
);
5891 ldpp_dout(dpp
, 15) << "string_to_sign="
5892 << rgw::crypt_sanitize::log_content
{string_to_sign
}
5894 ldpp_dout(dpp
, 15) << "server signature=" << server_signature
<< dendl
;
5895 ldpp_dout(dpp
, 15) << "client signature=" << signature
<< dendl
;
5896 ldpp_dout(dpp
, 15) << "compare=" << compare
<< dendl
;
5899 return result_t::reject(-ERR_SIGNATURE_NO_MATCH
);
5902 // Get all the authorization info
5903 RGWUserInfo user_info
;
5906 rgw::auth::RoleApplier::Role r
;
5907 if (! token
.roleId
.empty()) {
5908 RGWRole
role(s
->cct
, ctl
, token
.roleId
);
5909 if (role
.get_by_id(dpp
, y
) < 0) {
5910 return result_t::deny(-EPERM
);
5912 r
.id
= token
.roleId
;
5913 r
.name
= role
.get_name();
5914 r
.tenant
= role
.get_tenant();
5916 vector
<string
> role_policy_names
= role
.get_role_policy_names();
5917 for (auto& policy_name
: role_policy_names
) {
5919 if (int ret
= role
.get_role_policy(policy_name
, perm_policy
); ret
== 0) {
5920 r
.role_policies
.push_back(std::move(perm_policy
));
5923 // This is mostly needed to assign the owner of a bucket during its creation
5924 user_id
= token
.user
;
5927 if (! token
.user
.empty() && token
.acct_type
!= TYPE_ROLE
) {
5929 int ret
= rgw_get_user_info_by_uid(dpp
, ctl
->user
, token
.user
, user_info
, y
, NULL
);
5931 ldpp_dout(dpp
, 5) << "ERROR: failed reading user info: uid=" << token
.user
<< dendl
;
5932 return result_t::reject(-EPERM
);
5936 if (token
.acct_type
== TYPE_KEYSTONE
|| token
.acct_type
== TYPE_LDAP
) {
5937 auto apl
= remote_apl_factory
->create_apl_remote(cct
, s
, get_acl_strategy(),
5938 get_creds_info(token
));
5939 return result_t::grant(std::move(apl
), completer_factory(boost::none
));
5940 } else if (token
.acct_type
== TYPE_ROLE
) {
5941 auto apl
= role_apl_factory
->create_apl_role(cct
, s
, r
, user_id
, token
.policy
, token
.role_session
, token
.token_claims
, token
.issued_at
);
5942 return result_t::grant(std::move(apl
), completer_factory(token
.secret_access_key
));
5943 } else { // This is for all local users of type TYPE_RGW or TYPE_NONE
5945 auto apl
= local_apl_factory
->create_apl_local(cct
, s
, user_info
, subuser
, token
.perm_mask
);
5946 return result_t::grant(std::move(apl
), completer_factory(token
.secret_access_key
));
5950 bool rgw::auth::s3::S3AnonymousEngine::is_applicable(
5953 if (s
->op
== OP_OPTIONS
) {
5959 std::tie(version
, route
) = discover_aws_flavour(s
->info
);
5961 return route
== AwsRoute::QUERY_STRING
&& version
== AwsVersion::UNKNOWN
;
5965 using namespace s3selectEngine
;
5966 const char* RGWSelectObj_ObjStore_S3::header_name_str
[3] = {":event-type", ":content-type", ":message-type"};
5967 const char* RGWSelectObj_ObjStore_S3::header_value_str
[3] = {"Records", "application/octet-stream", "event"};
5969 RGWSelectObj_ObjStore_S3::RGWSelectObj_ObjStore_S3():
5970 s3select_syntax(std::make_unique
<s3selectEngine::s3select
>()),
5971 m_s3_csv_object(std::unique_ptr
<s3selectEngine::csv_object
>()),
5972 m_buff_header(std::make_unique
<char[]>(1000)),
5974 crc32(std::unique_ptr
<boost::crc_32_type
>())
5979 RGWSelectObj_ObjStore_S3::~RGWSelectObj_ObjStore_S3()
5983 int RGWSelectObj_ObjStore_S3::get_params(optional_yield y
)
5986 //retrieve s3-select query from payload
5989 int max_size
= 4096;
5990 std::tie(ret
, data
) = rgw_rest_read_all_input(s
, max_size
, false);
5992 ldpp_dout(this, 10) << "s3-select query: failed to retrieve query; ret = " << ret
<< dendl
;
5996 m_s3select_query
= data
.to_str();
5997 if (m_s3select_query
.length() > 0) {
5998 ldpp_dout(this, 10) << "s3-select query: " << m_s3select_query
<< dendl
;
6001 ldpp_dout(this, 10) << "s3-select query: failed to retrieve query;" << dendl
;
6005 int status
= handle_aws_cli_parameters(m_sql_query
);
6011 return RGWGetObj_ObjStore_S3::get_params(y
);
6014 void RGWSelectObj_ObjStore_S3::encode_short(char* buff
, uint16_t s
, int& i
)
6017 memcpy(buff
, &x
, sizeof(s
));
6021 void RGWSelectObj_ObjStore_S3::encode_int(char* buff
, u_int32_t s
, int& i
)
6023 u_int32_t x
= htonl(s
);
6024 memcpy(buff
, &x
, sizeof(s
));
6028 int RGWSelectObj_ObjStore_S3::create_header_records(char* buff
)
6032 //headers description(AWS)
6033 //[header-name-byte-length:1][header-name:variable-length][header-value-type:1][header-value:variable-length]
6036 buff
[i
++] = char(strlen(header_name_str
[EVENT_TYPE
]));
6037 memcpy(&buff
[i
], header_name_str
[EVENT_TYPE
], strlen(header_name_str
[EVENT_TYPE
]));
6038 i
+= strlen(header_name_str
[EVENT_TYPE
]);
6039 buff
[i
++] = char(7);
6040 encode_short(&buff
[i
], uint16_t(strlen(header_value_str
[RECORDS
])), i
);
6041 memcpy(&buff
[i
], header_value_str
[RECORDS
], strlen(header_value_str
[RECORDS
]));
6042 i
+= strlen(header_value_str
[RECORDS
]);
6045 buff
[i
++] = char(strlen(header_name_str
[CONTENT_TYPE
]));
6046 memcpy(&buff
[i
], header_name_str
[CONTENT_TYPE
], strlen(header_name_str
[CONTENT_TYPE
]));
6047 i
+= strlen(header_name_str
[CONTENT_TYPE
]);
6048 buff
[i
++] = char(7);
6049 encode_short(&buff
[i
], uint16_t(strlen(header_value_str
[OCTET_STREAM
])), i
);
6050 memcpy(&buff
[i
], header_value_str
[OCTET_STREAM
], strlen(header_value_str
[OCTET_STREAM
]));
6051 i
+= strlen(header_value_str
[OCTET_STREAM
]);
6054 buff
[i
++] = char(strlen(header_name_str
[MESSAGE_TYPE
]));
6055 memcpy(&buff
[i
], header_name_str
[MESSAGE_TYPE
], strlen(header_name_str
[MESSAGE_TYPE
]));
6056 i
+= strlen(header_name_str
[MESSAGE_TYPE
]);
6057 buff
[i
++] = char(7);
6058 encode_short(&buff
[i
], uint16_t(strlen(header_value_str
[EVENT
])), i
);
6059 memcpy(&buff
[i
], header_value_str
[EVENT
], strlen(header_value_str
[EVENT
]));
6060 i
+= strlen(header_value_str
[EVENT
]);
6065 int RGWSelectObj_ObjStore_S3::create_message(std::string
&out_string
, u_int32_t result_len
, u_int32_t header_len
)
6067 //message description(AWS):
6068 //[total-byte-length:4][header-byte-length:4][crc:4][headers:variable-length][payload:variable-length][crc:4]
6069 //s3select result is produced into m_result, the m_result is also the response-message, thus the attach headers and CRC
6070 //are created later to the produced SQL result, and actually wrapping the payload.
6072 u_int32_t total_byte_len
= 0;
6073 u_int32_t preload_crc
= 0;
6074 u_int32_t message_crc
= 0;
6076 char * buff
= out_string
.data();
6079 // the parameters are according to CRC-32 algorithm and its aligned with AWS-cli checksum
6080 crc32
= std::unique_ptr
<boost::crc_32_type
>(new boost::crc_optimal
<32, 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, true, true>);
6083 total_byte_len
= result_len
+ 16;//the total is greater in 4 bytes than current size
6085 encode_int(&buff
[i
], total_byte_len
, i
);//store sizes at the beginning of the buffer
6086 encode_int(&buff
[i
], header_len
, i
);
6089 *crc32
= std::for_each( buff
, buff
+ 8, *crc32
);//crc for starting 8 bytes
6090 preload_crc
= (*crc32
)();
6091 encode_int(&buff
[i
], preload_crc
, i
);
6093 i
+= result_len
;//advance to the end of payload.
6096 *crc32
= std::for_each( buff
, buff
+ i
, *crc32
);//crc for payload + checksum
6097 message_crc
= (*crc32
)();
6099 encode_int(out_encode
, message_crc
, i
);
6100 out_string
.append(out_encode
,sizeof(out_encode
));
6105 #define PAYLOAD_LINE "\n<Payload>\n<Records>\n<Payload>\n"
6106 #define END_PAYLOAD_LINE "\n</Payload></Records></Payload>"
6108 int RGWSelectObj_ObjStore_S3::run_s3select(const char* query
, const char* input
, size_t input_length
)
6111 csv_object::csv_defintions csv
;
6113 m_result
= "012345678901"; //12 positions for header-crc
6115 int header_size
= 0;
6117 if (m_s3_csv_object
==0) {
6118 s3select_syntax
->parse_query(query
);
6120 if (m_row_delimiter
.size()) {
6121 csv
.row_delimiter
= *m_row_delimiter
.c_str();
6124 if (m_column_delimiter
.size()) {
6125 csv
.column_delimiter
= *m_column_delimiter
.c_str();
6128 if (m_quot
.size()) {
6129 csv
.quot_char
= *m_quot
.c_str();
6132 if (m_escape_char
.size()) {
6133 csv
.escape_char
= *m_escape_char
.c_str();
6136 if(m_header_info
.compare("IGNORE")==0) {
6137 csv
.ignore_header_info
=true;
6139 else if(m_header_info
.compare("USE")==0) {
6140 csv
.use_header_info
=true;
6143 m_s3_csv_object
= std::unique_ptr
<s3selectEngine::csv_object
>(new s3selectEngine::csv_object(s3select_syntax
.get(), csv
));
6146 header_size
= create_header_records(m_buff_header
.get());
6147 m_result
.append(m_buff_header
.get(), header_size
);
6148 m_result
.append(PAYLOAD_LINE
);
6150 if (s3select_syntax
->get_error_description().empty() == false) {
6151 m_result
.append(s3select_syntax
->get_error_description());
6152 ldpp_dout(this, 10) << "s3-select query: failed to prase query; {" << s3select_syntax
->get_error_description() << "}"<< dendl
;
6156 status
= m_s3_csv_object
->run_s3select_on_stream(m_result
, input
, input_length
, s
->obj_size
);
6158 m_result
.append(m_s3_csv_object
->get_error_description());
6162 if (m_result
.size() > strlen(PAYLOAD_LINE
)) {
6163 m_result
.append(END_PAYLOAD_LINE
);
6164 int buff_len
= create_message(m_result
, m_result
.size() - 12, header_size
);
6165 s
->formatter
->write_bin_data(m_result
.data(), buff_len
);
6170 rgw_flush_formatter_and_reset(s
, s
->formatter
);
6175 int RGWSelectObj_ObjStore_S3::handle_aws_cli_parameters(std::string
& sql_query
)
6178 if(chunk_number
!=0) {
6184 if (m_s3select_query
.find(GT
) != std::string::npos
) {
6185 boost::replace_all(m_s3select_query
, GT
, ">");
6187 if (m_s3select_query
.find(LT
) != std::string::npos
) {
6188 boost::replace_all(m_s3select_query
, LT
, "<");
6191 //AWS cli s3select parameters
6192 extract_by_tag("Expression", sql_query
);
6193 extract_by_tag("FieldDelimiter", m_column_delimiter
);
6194 extract_by_tag("QuoteCharacter", m_quot
);
6195 extract_by_tag("RecordDelimiter", m_row_delimiter
);
6196 if (m_row_delimiter
.size()==0) {
6197 m_row_delimiter
='\n';
6200 extract_by_tag("QuoteEscapeCharacter", m_escape_char
);
6201 extract_by_tag("CompressionType", m_compression_type
);
6202 if (m_compression_type
.length()>0 && m_compression_type
.compare("NONE") != 0) {
6203 ldpp_dout(this, 10) << "RGW supports currently only NONE option for compression type" << dendl
;
6207 extract_by_tag("FileHeaderInfo", m_header_info
);
6212 int RGWSelectObj_ObjStore_S3::extract_by_tag(std::string tag_name
, std::string
& result
)
6215 size_t _qs
= m_s3select_query
.find("<" + tag_name
+ ">", 0) + tag_name
.size() + 2;
6216 if (_qs
== std::string::npos
) {
6219 size_t _qe
= m_s3select_query
.find("</" + tag_name
+ ">", _qs
);
6220 if (_qe
== std::string::npos
) {
6224 result
= m_s3select_query
.substr(_qs
, _qe
- _qs
);
6229 int RGWSelectObj_ObjStore_S3::send_response_data(bufferlist
& bl
, off_t ofs
, off_t len
)
6235 if (chunk_number
== 0) {
6237 set_req_state_err(s
, op_ret
);
6242 // Explicitly use chunked transfer encoding so that we can stream the result
6243 // to the user without having to wait for the full length of it.
6244 if (chunk_number
== 0) {
6245 end_header(s
, this, "application/xml", CHUNKED_TRANSFER_ENCODING
);
6249 for(auto& it
: bl
.buffers()) {
6250 status
= run_s3select(m_sql_query
.c_str(), &(it
)[0], it
.length());