1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
10 #include "common/armor.h"
11 #include "common/utf8.h"
12 #include "rgw_rest_s3.h"
13 #include "rgw_auth_s3.h"
14 #include "rgw_common.h"
15 #include "rgw_client_io.h"
17 #include "rgw_crypt_sanitize.h"
19 #include <boost/container/small_vector.hpp>
20 #include <boost/utility/string_view.hpp>
21 #include <boost/algorithm/string/trim_all.hpp>
23 #define dout_context g_ceph_context
24 #define dout_subsys ceph_subsys_rgw
26 static const auto signed_subresources
= {
37 "response-cache-control",
38 "response-content-disposition",
39 "response-content-encoding",
40 "response-content-language",
41 "response-content-type",
55 * ?get the canonical amazon-style header for something?
59 get_canon_amz_hdr(const std::map
<std::string
, std::string
>& meta_map
)
63 for (const auto& kv
: meta_map
) {
64 dest
.append(kv
.first
);
66 dest
.append(kv
.second
);
74 * ?get the canonical representation of the object's location
77 get_canon_resource(const char* const request_uri
,
78 const std::map
<std::string
, std::string
>& sub_resources
)
83 dest
.append(request_uri
);
87 for (const auto& subresource
: signed_subresources
) {
88 const auto iter
= sub_resources
.find(subresource
);
89 if (iter
== std::end(sub_resources
)) {
100 dest
.append(iter
->first
);
101 if (! iter
->second
.empty()) {
103 dest
.append(iter
->second
);
107 dout(10) << "get_canon_resource(): dest=" << dest
<< dendl
;
112 * get the header authentication information required to
113 * compute a request's signature
115 void rgw_create_s3_canonical_header(
116 const char* const method
,
117 const char* const content_md5
,
118 const char* const content_type
,
119 const char* const date
,
120 const std::map
<std::string
, std::string
>& meta_map
,
121 const std::map
<std::string
, std::string
>& qs_map
,
122 const char* const request_uri
,
123 const std::map
<std::string
, std::string
>& sub_resources
,
124 std::string
& dest_str
)
134 dest
.append(content_md5
);
139 dest
.append(content_type
);
148 dest
.append(get_canon_amz_hdr(meta_map
));
149 dest
.append(get_canon_amz_hdr(qs_map
));
150 dest
.append(get_canon_resource(request_uri
, sub_resources
));
155 static inline bool is_base64_for_content_md5(unsigned char c
) {
156 return (isalnum(c
) || isspace(c
) || (c
== '+') || (c
== '/') || (c
== '='));
159 static inline void get_v2_qs_map(const req_info
& info
,
160 std::map
<std::string
, std::string
>& qs_map
) {
161 const auto& params
= const_cast<RGWHTTPArgs
&>(info
.args
).get_params();
162 for (const auto& elt
: params
) {
163 std::string k
= boost::algorithm::to_lower_copy(elt
.first
);
164 if (k
.find("x-amz-meta-") == /* offset */ 0) {
165 add_amz_meta_header(qs_map
, k
, elt
.second
);
171 * get the header authentication information required to
172 * compute a request's signature
174 bool rgw_create_s3_canonical_header(const req_info
& info
,
175 utime_t
* const header_time
,
179 const char* const content_md5
= info
.env
->get("HTTP_CONTENT_MD5");
181 for (const char *p
= content_md5
; *p
; p
++) {
182 if (!is_base64_for_content_md5(*p
)) {
183 dout(0) << "NOTICE: bad content-md5 provided (not base64),"
184 << " aborting request p=" << *p
<< " " << (int)*p
<< dendl
;
190 const char *content_type
= info
.env
->get("CONTENT_TYPE");
193 std::map
<std::string
, std::string
> qs_map
;
196 get_v2_qs_map(info
, qs_map
); // handle qs metadata
197 date
= info
.args
.get("Expires");
199 const char *str
= info
.env
->get("HTTP_X_AMZ_DATE");
200 const char *req_date
= str
;
202 req_date
= info
.env
->get("HTTP_DATE");
204 dout(0) << "NOTICE: missing date for auth header" << dendl
;
212 if (!parse_rfc2616(req_date
, &t
)) {
213 dout(0) << "NOTICE: failed to parse date for auth header" << dendl
;
216 if (t
.tm_year
< 70) {
217 dout(0) << "NOTICE: bad date (predates epoch): " << req_date
<< dendl
;
220 *header_time
= utime_t(internal_timegm(&t
), 0);
224 const auto& meta_map
= info
.x_meta_map
;
225 const auto& sub_resources
= info
.args
.get_sub_resources();
227 std::string request_uri
;
228 if (info
.effective_uri
.empty()) {
229 request_uri
= info
.request_uri
;
231 request_uri
= info
.effective_uri
;
234 rgw_create_s3_canonical_header(info
.method
, content_md5
, content_type
,
235 date
.c_str(), meta_map
, qs_map
,
236 request_uri
.c_str(), sub_resources
, dest
);
245 bool is_time_skew_ok(time_t t
)
247 auto req_tp
= ceph::coarse_real_clock::from_time_t(t
);
248 auto cur_tp
= ceph::coarse_real_clock::now();
250 if (std::chrono::abs(cur_tp
- req_tp
) > RGW_AUTH_GRACE
) {
251 dout(10) << "NOTICE: request time skew too big." << dendl
;
252 using ceph::operator<<;
253 dout(10) << "req_tp=" << req_tp
<< ", cur_tp=" << cur_tp
<< dendl
;
260 static inline int parse_v4_query_string(const req_info
& info
, /* in */
261 boost::string_view
& credential
, /* out */
262 boost::string_view
& signedheaders
, /* out */
263 boost::string_view
& signature
, /* out */
264 boost::string_view
& date
, /* out */
265 boost::string_view
& sessiontoken
) /* out */
267 /* auth ships with req params ... */
269 /* look for required params */
270 credential
= info
.args
.get("X-Amz-Credential");
271 if (credential
.size() == 0) {
275 date
= info
.args
.get("X-Amz-Date");
277 if (!parse_iso8601(sview2cstr(date
).data(), &date_t
, nullptr, false)) {
281 boost::string_view expires
= info
.args
.get("X-Amz-Expires");
282 if (expires
.empty()) {
285 /* X-Amz-Expires provides the time period, in seconds, for which
286 the generated presigned URL is valid. The minimum value
287 you can set is 1, and the maximum is 604800 (seven days) */
288 time_t exp
= atoll(expires
.data());
289 if ((exp
< 1) || (exp
> 7*24*60*60)) {
290 dout(10) << "NOTICE: exp out of range, exp = " << exp
<< dendl
;
293 /* handle expiration in epoch time */
294 uint64_t req_sec
= (uint64_t)internal_timegm(&date_t
);
295 uint64_t now
= ceph_clock_now();
296 if (now
>= req_sec
+ exp
) {
297 dout(10) << "NOTICE: now = " << now
<< ", req_sec = " << req_sec
<< ", exp = " << exp
<< dendl
;
301 signedheaders
= info
.args
.get("X-Amz-SignedHeaders");
302 if (signedheaders
.size() == 0) {
306 signature
= info
.args
.get("X-Amz-Signature");
307 if (signature
.size() == 0) {
311 if (info
.args
.exists("X-Amz-Security-Token")) {
312 sessiontoken
= info
.args
.get("X-Amz-Security-Token");
313 if (sessiontoken
.size() == 0) {
321 static bool get_next_token(const boost::string_view
& s
,
323 const char* const delims
,
324 boost::string_view
& token
)
326 const size_t start
= s
.find_first_not_of(delims
, pos
);
327 if (start
== boost::string_view::npos
) {
332 size_t end
= s
.find_first_of(delims
, start
);
333 if (end
!= boost::string_view::npos
)
336 pos
= end
= s
.size();
339 token
= s
.substr(start
, end
- start
);
343 template<std::size_t ExpectedStrNum
>
344 boost::container::small_vector
<boost::string_view
, ExpectedStrNum
>
345 get_str_vec(const boost::string_view
& str
, const char* const delims
)
347 boost::container::small_vector
<boost::string_view
, ExpectedStrNum
> str_vec
;
350 boost::string_view token
;
351 while (pos
< str
.size()) {
352 if (get_next_token(str
, pos
, delims
, token
)) {
353 if (token
.size() > 0) {
354 str_vec
.push_back(token
);
362 template<std::size_t ExpectedStrNum
>
363 boost::container::small_vector
<boost::string_view
, ExpectedStrNum
>
364 get_str_vec(const boost::string_view
& str
)
366 const char delims
[] = ";,= \t";
367 return get_str_vec
<ExpectedStrNum
>(str
, delims
);
370 static inline int parse_v4_auth_header(const req_info
& info
, /* in */
371 boost::string_view
& credential
, /* out */
372 boost::string_view
& signedheaders
, /* out */
373 boost::string_view
& signature
, /* out */
374 boost::string_view
& date
, /* out */
375 boost::string_view
& sessiontoken
) /* out */
377 boost::string_view
input(info
.env
->get("HTTP_AUTHORIZATION", ""));
379 input
= input
.substr(::strlen(AWS4_HMAC_SHA256_STR
) + 1);
380 } catch (std::out_of_range
&) {
381 /* We should never ever run into this situation as the presence of
382 * AWS4_HMAC_SHA256_STR had been verified earlier. */
383 dout(10) << "credentials string is too short" << dendl
;
387 std::map
<boost::string_view
, boost::string_view
> kv
;
388 for (const auto& s
: get_str_vec
<4>(input
, ",")) {
389 const auto parsed_pair
= parse_key_value(s
);
391 kv
[parsed_pair
->first
] = parsed_pair
->second
;
393 dout(10) << "NOTICE: failed to parse auth header (s=" << s
<< ")"
399 static const std::array
<boost::string_view
, 3> required_keys
= {
405 /* Ensure that the presigned required keys are really there. */
406 for (const auto& k
: required_keys
) {
407 if (kv
.find(k
) == std::end(kv
)) {
408 dout(10) << "NOTICE: auth header missing key: " << k
<< dendl
;
413 credential
= kv
["Credential"];
414 signedheaders
= kv
["SignedHeaders"];
415 signature
= kv
["Signature"];
418 dout(10) << "v4 signature format = " << signature
<< dendl
;
420 /* ------------------------- handle x-amz-date header */
424 const char *d
= info
.env
->get("HTTP_X_AMZ_DATE");
426 if (!parse_iso8601(d
, &t
, NULL
, false)) {
427 dout(10) << "error reading date via http_x_amz_date" << dendl
;
432 if (!is_time_skew_ok(internal_timegm(&t
))) {
433 return -ERR_REQUEST_TIME_SKEWED
;
436 if (info
.env
->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
437 sessiontoken
= info
.env
->get("HTTP_X_AMZ_SECURITY_TOKEN");
443 int parse_v4_credentials(const req_info
& info
, /* in */
444 boost::string_view
& access_key_id
, /* out */
445 boost::string_view
& credential_scope
, /* out */
446 boost::string_view
& signedheaders
, /* out */
447 boost::string_view
& signature
, /* out */
448 boost::string_view
& date
, /* out */
449 boost::string_view
& session_token
, /* out */
450 const bool using_qs
) /* in */
452 boost::string_view credential
;
455 ret
= parse_v4_query_string(info
, credential
, signedheaders
,
456 signature
, date
, session_token
);
458 ret
= parse_v4_auth_header(info
, credential
, signedheaders
,
459 signature
, date
, session_token
);
466 /* access_key/YYYYMMDD/region/service/aws4_request */
467 dout(10) << "v4 credential format = " << credential
<< dendl
;
469 if (std::count(credential
.begin(), credential
.end(), '/') != 4) {
473 /* credential must end with 'aws4_request' */
474 if (credential
.find("aws4_request") == std::string::npos
) {
478 /* grab access key id */
479 const size_t pos
= credential
.find("/");
480 access_key_id
= credential
.substr(0, pos
);
481 dout(10) << "access key id = " << access_key_id
<< dendl
;
483 /* grab credential scope */
484 credential_scope
= credential
.substr(pos
+ 1);
485 dout(10) << "credential scope = " << credential_scope
<< dendl
;
490 std::string
get_v4_canonical_qs(const req_info
& info
, const bool using_qs
)
492 const std::string
*params
= &info
.request_params
;
493 std::string copy_params
;
494 if (params
->empty()) {
495 /* Optimize the typical flow. */
496 return std::string();
498 if (params
->find_first_of('+') != std::string::npos
) {
499 copy_params
= *params
;
500 boost::replace_all(copy_params
, "+", " ");
501 params
= ©_params
;
504 /* Handle case when query string exists. Step 3 described in: http://docs.
505 * aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html */
506 std::map
<std::string
, std::string
> canonical_qs_map
;
507 for (const auto& s
: get_str_vec
<5>(*params
, "&")) {
508 boost::string_view key
, val
;
509 const auto parsed_pair
= parse_key_value(s
);
511 std::tie(key
, val
) = *parsed_pair
;
513 /* Handling a parameter without any value (even the empty one). That's
514 * it, we've encountered something like "this_param&other_param=val"
515 * which is used by S3 for subresources. */
519 if (using_qs
&& key
== "X-Amz-Signature") {
520 /* Preserving the original behaviour of get_v4_canonical_qs() here. */
524 // while awsv4 specs ask for all slashes to be encoded, s3 itself is relaxed
525 // in its implementation allowing non-url-encoded slashes to be present in
526 // presigned urls for instance
527 canonical_qs_map
[aws4_uri_recode(key
, true)] = aws4_uri_recode(val
, true);
530 /* Thanks to the early exist we have the guarantee that canonical_qs_map has
531 * at least one element. */
532 auto iter
= std::begin(canonical_qs_map
);
533 std::string canonical_qs
;
534 canonical_qs
.append(iter
->first
)
535 .append("=", ::strlen("="))
536 .append(iter
->second
);
538 for (iter
++; iter
!= std::end(canonical_qs_map
); iter
++) {
539 canonical_qs
.append("&", ::strlen("&"))
541 .append("=", ::strlen("="))
542 .append(iter
->second
);
548 boost::optional
<std::string
>
549 get_v4_canonical_headers(const req_info
& info
,
550 const boost::string_view
& signedheaders
,
552 const bool force_boto2_compat
)
554 std::map
<boost::string_view
, std::string
> canonical_hdrs_map
;
555 for (const auto& token
: get_str_vec
<5>(signedheaders
, ";")) {
556 /* TODO(rzarzynski): we'd like to switch to sstring here but it should
557 * get push_back() and reserve() first. */
558 std::string token_env
= "HTTP_";
559 token_env
.reserve(token
.length() + std::strlen("HTTP_") + 1);
561 std::transform(std::begin(token
), std::end(token
),
562 std::back_inserter(token_env
), [](const int c
) {
563 return c
== '-' ? '_' : std::toupper(c
);
566 if (token_env
== "HTTP_CONTENT_LENGTH") {
567 token_env
= "CONTENT_LENGTH";
568 } else if (token_env
== "HTTP_CONTENT_TYPE") {
569 token_env
= "CONTENT_TYPE";
571 const char* const t
= info
.env
->get(token_env
.c_str());
573 dout(10) << "warning env var not available" << dendl
;
577 std::string
token_value(t
);
578 if (token_env
== "HTTP_CONTENT_MD5" &&
579 !std::all_of(std::begin(token_value
), std::end(token_value
),
580 is_base64_for_content_md5
)) {
581 dout(0) << "NOTICE: bad content-md5 provided (not base64)"
582 << ", aborting request" << dendl
;
586 if (force_boto2_compat
&& using_qs
&& token
== "host") {
587 boost::string_view port
= info
.env
->get("SERVER_PORT", "");
588 boost::string_view secure_port
= info
.env
->get("SERVER_PORT_SECURE", "");
590 if (!secure_port
.empty()) {
591 if (secure_port
!= "443")
592 token_value
.append(":", std::strlen(":"))
593 .append(secure_port
.data(), secure_port
.length());
594 } else if (!port
.empty()) {
596 token_value
.append(":", std::strlen(":"))
597 .append(port
.data(), port
.length());
601 canonical_hdrs_map
[token
] = rgw_trim_whitespace(token_value
);
604 std::string canonical_hdrs
;
605 for (const auto& header
: canonical_hdrs_map
) {
606 const boost::string_view
& name
= header
.first
;
607 std::string value
= header
.second
;
608 boost::trim_all
<std::string
>(value
);
610 canonical_hdrs
.append(name
.data(), name
.length())
611 .append(":", std::strlen(":"))
613 .append("\n", std::strlen("\n"));
616 return canonical_hdrs
;
620 * create canonical request for signature version 4
622 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
625 get_v4_canon_req_hash(CephContext
* cct
,
626 const boost::string_view
& http_verb
,
627 const std::string
& canonical_uri
,
628 const std::string
& canonical_qs
,
629 const std::string
& canonical_hdrs
,
630 const boost::string_view
& signed_hdrs
,
631 const boost::string_view
& request_payload_hash
)
633 ldout(cct
, 10) << "payload request hash = " << request_payload_hash
<< dendl
;
635 const auto canonical_req
= string_join_reserve("\n",
641 request_payload_hash
);
643 const auto canonical_req_hash
= calc_hash_sha256(canonical_req
);
645 using sanitize
= rgw::crypt_sanitize::log_content
;
646 ldout(cct
, 10) << "canonical request = " << sanitize
{canonical_req
} << dendl
;
647 ldout(cct
, 10) << "canonical request hash = "
648 << canonical_req_hash
<< dendl
;
650 return canonical_req_hash
;
654 * create string to sign for signature version 4
656 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
658 AWSEngine::VersionAbstractor::string_to_sign_t
659 get_v4_string_to_sign(CephContext
* const cct
,
660 const boost::string_view
& algorithm
,
661 const boost::string_view
& request_date
,
662 const boost::string_view
& credential_scope
,
663 const sha256_digest_t
& canonreq_hash
)
665 const auto hexed_cr_hash
= canonreq_hash
.to_str();
666 const boost::string_view
hexed_cr_hash_str(hexed_cr_hash
);
668 const auto string_to_sign
= string_join_reserve("\n",
674 ldout(cct
, 10) << "string to sign = "
675 << rgw::crypt_sanitize::log_content
{string_to_sign
}
678 return string_to_sign
;
682 static inline std::tuple
<boost::string_view
, /* date */
683 boost::string_view
, /* region */
684 boost::string_view
> /* service */
685 parse_cred_scope(boost::string_view credential_scope
)
688 size_t pos
= credential_scope
.find("/");
689 const auto date_cs
= credential_scope
.substr(0, pos
);
690 credential_scope
= credential_scope
.substr(pos
+ 1);
693 pos
= credential_scope
.find("/");
694 const auto region_cs
= credential_scope
.substr(0, pos
);
695 credential_scope
= credential_scope
.substr(pos
+ 1);
698 pos
= credential_scope
.find("/");
699 const auto service_cs
= credential_scope
.substr(0, pos
);
701 return std::make_tuple(date_cs
, region_cs
, service_cs
);
704 static inline std::vector
<unsigned char>
705 transform_secret_key(const boost::string_view
& secret_access_key
)
707 /* TODO(rzarzynski): switch to constexpr when C++14 becomes available. */
708 static const std::initializer_list
<unsigned char> AWS4
{ 'A', 'W', 'S', '4' };
710 /* boost::container::small_vector might be used here if someone wants to
711 * optimize out even more dynamic allocations. */
712 std::vector
<unsigned char> secret_key_utf8
;
713 secret_key_utf8
.reserve(AWS4
.size() + secret_access_key
.size());
714 secret_key_utf8
.assign(AWS4
);
716 for (const auto c
: secret_access_key
) {
717 std::array
<unsigned char, MAX_UTF8_SZ
> buf
;
718 const size_t n
= encode_utf8(c
, buf
.data());
719 secret_key_utf8
.insert(std::end(secret_key_utf8
),
720 std::begin(buf
), std::begin(buf
) + n
);
723 return secret_key_utf8
;
727 * calculate the SigningKey of AWS auth version 4
729 static sha256_digest_t
730 get_v4_signing_key(CephContext
* const cct
,
731 const boost::string_view
& credential_scope
,
732 const boost::string_view
& secret_access_key
)
734 boost::string_view date
, region
, service
;
735 std::tie(date
, region
, service
) = parse_cred_scope(credential_scope
);
737 const auto utfed_sec_key
= transform_secret_key(secret_access_key
);
738 const auto date_k
= calc_hmac_sha256(utfed_sec_key
, date
);
739 const auto region_k
= calc_hmac_sha256(date_k
, region
);
740 const auto service_k
= calc_hmac_sha256(region_k
, service
);
743 const auto signing_key
= calc_hmac_sha256(service_k
,
744 boost::string_view("aws4_request"));
746 ldout(cct
, 10) << "date_k = " << date_k
<< dendl
;
747 ldout(cct
, 10) << "region_k = " << region_k
<< dendl
;
748 ldout(cct
, 10) << "service_k = " << service_k
<< dendl
;
749 ldout(cct
, 10) << "signing_k = " << signing_key
<< dendl
;
755 * calculate the AWS signature version 4
757 * http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
759 * srv_signature_t is an alias over Ceph's basic_sstring. We're using
760 * it to keep everything within the stack boundaries instead of doing
761 * dynamic allocations.
763 AWSEngine::VersionAbstractor::server_signature_t
764 get_v4_signature(const boost::string_view
& credential_scope
,
765 CephContext
* const cct
,
766 const boost::string_view
& secret_key
,
767 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
)
769 auto signing_key
= get_v4_signing_key(cct
, credential_scope
, secret_key
);
771 /* The server-side generated digest for comparison. */
772 const auto digest
= calc_hmac_sha256(signing_key
, string_to_sign
);
774 /* TODO(rzarzynski): I would love to see our sstring having reserve() and
775 * the non-const data() variant like C++17's std::string. */
776 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
777 srv_signature_t
signature(srv_signature_t::initialized_later(),
779 buf_to_hex(digest
.v
, digest
.SIZE
, signature
.begin());
781 ldout(cct
, 10) << "generated signature = " << signature
<< dendl
;
786 AWSEngine::VersionAbstractor::server_signature_t
787 get_v2_signature(CephContext
* const cct
,
788 const std::string
& secret_key
,
789 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
)
791 if (secret_key
.empty()) {
795 const auto digest
= calc_hmac_sha1(secret_key
, string_to_sign
);
797 /* 64 is really enough */;
799 const int ret
= ceph_armor(std::begin(buf
),
800 std::begin(buf
) + 64,
801 reinterpret_cast<const char *>(digest
.v
),
802 reinterpret_cast<const char *>(digest
.v
+ digest
.SIZE
));
804 ldout(cct
, 10) << "ceph_armor failed" << dendl
;
808 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
809 return srv_signature_t(buf
, ret
);
813 bool AWSv4ComplMulti::ChunkMeta::is_new_chunk_in_stream(size_t stream_pos
) const
815 return stream_pos
>= (data_offset_in_stream
+ data_length
);
818 size_t AWSv4ComplMulti::ChunkMeta::get_data_size(size_t stream_pos
) const
820 if (stream_pos
> (data_offset_in_stream
+ data_length
)) {
821 /* Data in parsing_buf. */
824 return data_offset_in_stream
+ data_length
- stream_pos
;
829 /* AWSv4 completers begin. */
830 std::pair
<AWSv4ComplMulti::ChunkMeta
, size_t /* consumed */>
831 AWSv4ComplMulti::ChunkMeta::create_next(CephContext
* const cct
,
833 const char* const metabuf
,
834 const size_t metabuf_len
)
836 boost::string_ref
metastr(metabuf
, metabuf_len
);
838 const size_t semicolon_pos
= metastr
.find(";");
839 if (semicolon_pos
== boost::string_ref::npos
) {
840 ldout(cct
, 20) << "AWSv4ComplMulti cannot find the ';' separator"
842 throw rgw::io::Exception(EINVAL
, std::system_category());
845 char* data_field_end
;
846 /* strtoull ignores the "\r\n" sequence after each non-first chunk. */
847 const size_t data_length
= std::strtoull(metabuf
, &data_field_end
, 16);
848 if (data_length
== 0 && data_field_end
== metabuf
) {
849 ldout(cct
, 20) << "AWSv4ComplMulti: cannot parse the data size"
851 throw rgw::io::Exception(EINVAL
, std::system_category());
854 /* Parse the chunk_signature=... part. */
855 const auto signature_part
= metastr
.substr(semicolon_pos
+ 1);
856 const size_t eq_sign_pos
= signature_part
.find("=");
857 if (eq_sign_pos
== boost::string_ref::npos
) {
858 ldout(cct
, 20) << "AWSv4ComplMulti: cannot find the '=' separator"
860 throw rgw::io::Exception(EINVAL
, std::system_category());
863 /* OK, we have at least the beginning of a signature. */
864 const size_t data_sep_pos
= signature_part
.find("\r\n");
865 if (data_sep_pos
== boost::string_ref::npos
) {
866 ldout(cct
, 20) << "AWSv4ComplMulti: no new line at signature end"
868 throw rgw::io::Exception(EINVAL
, std::system_category());
871 const auto signature
= \
872 signature_part
.substr(eq_sign_pos
+ 1, data_sep_pos
- 1 - eq_sign_pos
);
873 if (signature
.length() != SIG_SIZE
) {
874 ldout(cct
, 20) << "AWSv4ComplMulti: signature.length() != 64"
876 throw rgw::io::Exception(EINVAL
, std::system_category());
879 const size_t data_starts_in_stream
= \
880 + semicolon_pos
+ strlen(";") + data_sep_pos
+ strlen("\r\n")
881 + old
.data_offset_in_stream
+ old
.data_length
;
883 ldout(cct
, 20) << "parsed new chunk; signature=" << signature
884 << ", data_length=" << data_length
885 << ", data_starts_in_stream=" << data_starts_in_stream
888 return std::make_pair(ChunkMeta(data_starts_in_stream
,
895 AWSv4ComplMulti::calc_chunk_signature(const std::string
& payload_hash
) const
897 const auto string_to_sign
= string_join_reserve("\n",
898 AWS4_HMAC_SHA256_PAYLOAD_STR
,
901 prev_chunk_signature
,
902 AWS4_EMPTY_PAYLOAD_HASH
,
905 ldout(cct
, 20) << "AWSv4ComplMulti: string_to_sign=\n" << string_to_sign
908 /* new chunk signature */
909 const auto sig
= calc_hmac_sha256(signing_key
, string_to_sign
);
910 /* FIXME(rzarzynski): std::string here is really unnecessary. */
915 bool AWSv4ComplMulti::is_signature_mismatched()
917 /* The validity of previous chunk can be verified only after getting meta-
918 * data of the next one. */
919 const auto payload_hash
= calc_hash_sha256_restart_stream(&sha256_hash
);
920 const auto calc_signature
= calc_chunk_signature(payload_hash
);
922 if (chunk_meta
.get_signature() != calc_signature
) {
923 ldout(cct
, 20) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
925 ldout(cct
, 20) << "AWSv4ComplMulti: declared signature="
926 << chunk_meta
.get_signature() << dendl
;
927 ldout(cct
, 20) << "AWSv4ComplMulti: calculated signature="
928 << calc_signature
<< dendl
;
932 prev_chunk_signature
= chunk_meta
.get_signature();
937 size_t AWSv4ComplMulti::recv_body(char* const buf
, const size_t buf_max
)
939 /* Buffer stores only parsed stream. Raw values reflect the stream
940 * we're getting from a client. */
943 if (chunk_meta
.is_new_chunk_in_stream(stream_pos
)) {
944 /* Verify signature of the previous chunk. We aren't doing that for new
945 * one as the procedure requires calculation of payload hash. This code
946 * won't be triggered for the last, zero-length chunk. Instead, is will
947 * be checked in the complete() method. */
948 if (stream_pos
>= ChunkMeta::META_MAX_SIZE
&& is_signature_mismatched()) {
949 throw rgw::io::Exception(ERR_SIGNATURE_NO_MATCH
, std::system_category());
952 /* We don't have metadata for this range. This means a new chunk, so we
953 * need to parse a fresh portion of the stream. Let's start. */
954 size_t to_extract
= parsing_buf
.capacity() - parsing_buf
.size();
956 const size_t orig_size
= parsing_buf
.size();
957 parsing_buf
.resize(parsing_buf
.size() + to_extract
);
958 const size_t received
= io_base_t::recv_body(parsing_buf
.data() + orig_size
,
960 parsing_buf
.resize(parsing_buf
.size() - (to_extract
- received
));
965 stream_pos
+= received
;
966 to_extract
-= received
;
967 } while (to_extract
> 0);
970 std::tie(chunk_meta
, consumed
) = \
971 ChunkMeta::create_next(cct
, std::move(chunk_meta
),
972 parsing_buf
.data(), parsing_buf
.size());
974 /* We can drop the bytes consumed during metadata parsing. The remainder
975 * can be chunk's data plus possibly beginning of next chunks' metadata. */
976 parsing_buf
.erase(std::begin(parsing_buf
),
977 std::begin(parsing_buf
) + consumed
);
980 size_t stream_pos_was
= stream_pos
- parsing_buf
.size();
982 size_t to_extract
= \
983 std::min(chunk_meta
.get_data_size(stream_pos_was
), buf_max
);
984 dout(30) << "AWSv4ComplMulti: stream_pos_was=" << stream_pos_was
<< ", to_extract=" << to_extract
<< dendl
;
986 /* It's quite probable we have a couple of real data bytes stored together
987 * with meta-data in the parsing_buf. We need to extract them and move to
988 * the final buffer. This is a trade-off between frontend's read overhead
990 if (to_extract
> 0 && parsing_buf
.size() > 0) {
991 const auto data_len
= std::min(to_extract
, parsing_buf
.size());
992 const auto data_end_iter
= std::begin(parsing_buf
) + data_len
;
993 dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
<< ", data_len=" << data_len
<< dendl
;
995 std::copy(std::begin(parsing_buf
), data_end_iter
, buf
);
996 parsing_buf
.erase(std::begin(parsing_buf
), data_end_iter
);
998 calc_hash_sha256_update_stream(sha256_hash
, buf
, data_len
);
1000 to_extract
-= data_len
;
1001 buf_pos
+= data_len
;
1004 /* Now we can do the bulk read directly from RestfulClient without any extra
1006 while (to_extract
> 0) {
1007 const size_t received
= io_base_t::recv_body(buf
+ buf_pos
, to_extract
);
1008 dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
<< ", received=" << received
<< dendl
;
1010 if (received
== 0) {
1014 calc_hash_sha256_update_stream(sha256_hash
, buf
+ buf_pos
, received
);
1016 buf_pos
+= received
;
1017 stream_pos
+= received
;
1018 to_extract
-= received
;
1021 dout(20) << "AWSv4ComplMulti: filled=" << buf_pos
<< dendl
;
1025 void AWSv4ComplMulti::modify_request_state(const DoutPrefixProvider
* dpp
, req_state
* const s_rw
)
1027 const char* const decoded_length
= \
1028 s_rw
->info
.env
->get("HTTP_X_AMZ_DECODED_CONTENT_LENGTH");
1030 if (!decoded_length
) {
1033 s_rw
->length
= decoded_length
;
1034 s_rw
->content_length
= parse_content_length(decoded_length
);
1036 if (s_rw
->content_length
< 0) {
1037 ldpp_dout(dpp
, 10) << "negative AWSv4's content length, aborting" << dendl
;
1042 /* Install the filter over rgw::io::RestfulClient. */
1043 AWS_AUTHv4_IO(s_rw
)->add_filter(
1044 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1047 bool AWSv4ComplMulti::complete()
1049 /* Now it's time to verify the signature of the last, zero-length chunk. */
1050 if (is_signature_mismatched()) {
1051 ldout(cct
, 10) << "ERROR: signature of last chunk does not match"
1059 rgw::auth::Completer::cmplptr_t
1060 AWSv4ComplMulti::create(const req_state
* const s
,
1061 boost::string_view date
,
1062 boost::string_view credential_scope
,
1063 boost::string_view seed_signature
,
1064 const boost::optional
<std::string
>& secret_key
)
1067 /* Some external authorizers (like Keystone) aren't fully compliant with
1068 * AWSv4. They do not provide the secret_key which is necessary to handle
1069 * the streamed upload. */
1070 throw -ERR_NOT_IMPLEMENTED
;
1073 const auto signing_key
= \
1074 rgw::auth::s3::get_v4_signing_key(s
->cct
, credential_scope
, *secret_key
);
1076 return std::make_shared
<AWSv4ComplMulti
>(s
,
1078 std::move(credential_scope
),
1079 std::move(seed_signature
),
1083 size_t AWSv4ComplSingle::recv_body(char* const buf
, const size_t max
)
1085 const auto received
= io_base_t::recv_body(buf
, max
);
1086 calc_hash_sha256_update_stream(sha256_hash
, buf
, received
);
1091 void AWSv4ComplSingle::modify_request_state(const DoutPrefixProvider
* dpp
, req_state
* const s_rw
)
1093 /* Install the filter over rgw::io::RestfulClient. */
1094 AWS_AUTHv4_IO(s_rw
)->add_filter(
1095 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1098 bool AWSv4ComplSingle::complete()
1100 /* The completer is only for the cases where signed payload has been
1101 * requested. It won't be used, for instance, during the query string-based
1102 * authentication. */
1103 const auto payload_hash
= calc_hash_sha256_close_stream(&sha256_hash
);
1105 /* Validate x-amz-sha256 */
1106 if (payload_hash
.compare(expected_request_payload_hash
) == 0) {
1109 ldout(cct
, 10) << "ERROR: x-amz-content-sha256 does not match"
1111 ldout(cct
, 10) << "ERROR: grab_aws4_sha256_hash()="
1112 << payload_hash
<< dendl
;
1113 ldout(cct
, 10) << "ERROR: expected_request_payload_hash="
1114 << expected_request_payload_hash
<< dendl
;
1119 AWSv4ComplSingle::AWSv4ComplSingle(const req_state
* const s
)
1120 : io_base_t(nullptr),
1122 expected_request_payload_hash(get_v4_exp_payload_hash(s
->info
)),
1123 sha256_hash(calc_hash_sha256_open_stream()) {
1126 rgw::auth::Completer::cmplptr_t
1127 AWSv4ComplSingle::create(const req_state
* const s
,
1128 const boost::optional
<std::string
>&)
1130 return std::make_shared
<AWSv4ComplSingle
>(s
);
1133 } /* namespace s3 */
1134 } /* namespace auth */
1135 } /* namespace rgw */