1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
10 #include "common/armor.h"
11 #include "common/utf8.h"
12 #include "rgw_auth_s3.h"
13 #include "rgw_common.h"
14 #include "rgw_client_io.h"
16 #include "rgw_crypt_sanitize.h"
18 #include <boost/container/small_vector.hpp>
19 #include <boost/utility/string_view.hpp>
21 #define dout_context g_ceph_context
22 #define dout_subsys ceph_subsys_rgw
24 static const auto signed_subresources
= {
35 "response-cache-control",
36 "response-content-disposition",
37 "response-content-encoding",
38 "response-content-language",
39 "response-content-type",
54 * ?get the canonical amazon-style header for something?
58 get_canon_amz_hdr(const std::map
<std::string
, std::string
>& meta_map
)
62 for (const auto& kv
: meta_map
) {
63 dest
.append(kv
.first
);
65 dest
.append(kv
.second
);
73 * ?get the canonical representation of the object's location
76 get_canon_resource(const char* const request_uri
,
77 const std::map
<std::string
, std::string
>& sub_resources
)
82 dest
.append(request_uri
);
86 for (const auto& subresource
: signed_subresources
) {
87 const auto iter
= sub_resources
.find(subresource
);
88 if (iter
== std::end(sub_resources
)) {
99 dest
.append(iter
->first
);
100 if (! iter
->second
.empty()) {
102 dest
.append(iter
->second
);
106 dout(10) << "get_canon_resource(): dest=" << dest
<< dendl
;
111 * get the header authentication information required to
112 * compute a request's signature
114 void rgw_create_s3_canonical_header(
115 const char* const method
,
116 const char* const content_md5
,
117 const char* const content_type
,
118 const char* const date
,
119 const std::map
<std::string
, std::string
>& meta_map
,
120 const char* const request_uri
,
121 const std::map
<std::string
, std::string
>& sub_resources
,
122 std::string
& dest_str
)
132 dest
.append(content_md5
);
137 dest
.append(content_type
);
146 dest
.append(get_canon_amz_hdr(meta_map
));
147 dest
.append(get_canon_resource(request_uri
, sub_resources
));
152 static inline bool is_base64_for_content_md5(unsigned char c
) {
153 return (isalnum(c
) || isspace(c
) || (c
== '+') || (c
== '/') || (c
== '='));
157 * get the header authentication information required to
158 * compute a request's signature
160 bool rgw_create_s3_canonical_header(const req_info
& info
,
161 utime_t
* const header_time
,
165 const char* const content_md5
= info
.env
->get("HTTP_CONTENT_MD5");
167 for (const char *p
= content_md5
; *p
; p
++) {
168 if (!is_base64_for_content_md5(*p
)) {
169 dout(0) << "NOTICE: bad content-md5 provided (not base64),"
170 << " aborting request p=" << *p
<< " " << (int)*p
<< dendl
;
176 const char *content_type
= info
.env
->get("CONTENT_TYPE");
180 date
= info
.args
.get("Expires");
182 const char *str
= info
.env
->get("HTTP_X_AMZ_DATE");
183 const char *req_date
= str
;
185 req_date
= info
.env
->get("HTTP_DATE");
187 dout(0) << "NOTICE: missing date for auth header" << dendl
;
195 if (!parse_rfc2616(req_date
, &t
)) {
196 dout(0) << "NOTICE: failed to parse date for auth header" << dendl
;
199 if (t
.tm_year
< 70) {
200 dout(0) << "NOTICE: bad date (predates epoch): " << req_date
<< dendl
;
203 *header_time
= utime_t(internal_timegm(&t
), 0);
207 const auto& meta_map
= info
.x_meta_map
;
208 const auto& sub_resources
= info
.args
.get_sub_resources();
210 std::string request_uri
;
211 if (info
.effective_uri
.empty()) {
212 request_uri
= info
.request_uri
;
214 request_uri
= info
.effective_uri
;
217 rgw_create_s3_canonical_header(info
.method
, content_md5
, content_type
,
218 date
.c_str(), meta_map
, request_uri
.c_str(),
219 sub_resources
, dest
);
228 /* FIXME(rzarzynski): duplicated from rgw_rest_s3.h. */
229 #define RGW_AUTH_GRACE_MINS 15
231 static inline int parse_v4_query_string(const req_info
& info
, /* in */
232 boost::string_view
& credential
, /* out */
233 boost::string_view
& signedheaders
, /* out */
234 boost::string_view
& signature
, /* out */
235 boost::string_view
& date
) /* out */
237 /* auth ships with req params ... */
239 /* look for required params */
240 credential
= info
.args
.get("X-Amz-Credential");
241 if (credential
.size() == 0) {
245 date
= info
.args
.get("X-Amz-Date");
247 if (!parse_iso8601(sview2cstr(date
).data(), &date_t
, nullptr, false)) {
251 /* Used for pre-signatured url, We shouldn't return -ERR_REQUEST_TIME_SKEWED
252 * when current time <= X-Amz-Expires */
255 uint64_t now_req
= 0;
256 uint64_t now
= ceph_clock_now();
258 boost::string_view expires
= info
.args
.get("X-Amz-Expires");
259 if (!expires
.empty()) {
260 /* X-Amz-Expires provides the time period, in seconds, for which
261 the generated presigned URL is valid. The minimum value
262 you can set is 1, and the maximum is 604800 (seven days) */
263 time_t exp
= atoll(expires
.data());
264 if ((exp
< 1) || (exp
> 7*24*60*60)) {
265 dout(10) << "NOTICE: exp out of range, exp = " << exp
<< dendl
;
268 /* handle expiration in epoch time */
269 now_req
= (uint64_t)internal_timegm(&date_t
);
270 if (now
>= now_req
+ exp
) {
271 dout(10) << "NOTICE: now = " << now
<< ", now_req = " << now_req
<< ", exp = " << exp
<< dendl
;
277 if ((now_req
< now
- RGW_AUTH_GRACE_MINS
* 60 ||
278 now_req
> now
+ RGW_AUTH_GRACE_MINS
* 60) && !qsr
) {
279 dout(10) << "NOTICE: request time skew too big." << dendl
;
280 dout(10) << "now_req = " << now_req
<< " now = " << now
281 << "; now - RGW_AUTH_GRACE_MINS="
282 << now
- RGW_AUTH_GRACE_MINS
* 60
283 << "; now + RGW_AUTH_GRACE_MINS="
284 << now
+ RGW_AUTH_GRACE_MINS
* 60 << dendl
;
285 return -ERR_REQUEST_TIME_SKEWED
;
288 signedheaders
= info
.args
.get("X-Amz-SignedHeaders");
289 if (signedheaders
.size() == 0) {
293 signature
= info
.args
.get("X-Amz-Signature");
294 if (signature
.size() == 0) {
302 static bool get_next_token(const boost::string_view
& s
,
304 const char* const delims
,
305 boost::string_view
& token
)
307 const size_t start
= s
.find_first_not_of(delims
, pos
);
308 if (start
== boost::string_view::npos
) {
313 size_t end
= s
.find_first_of(delims
, start
);
314 if (end
!= boost::string_view::npos
)
317 pos
= end
= s
.size();
320 token
= s
.substr(start
, end
- start
);
324 template<std::size_t ExpectedStrNum
>
325 boost::container::small_vector
<boost::string_view
, ExpectedStrNum
>
326 get_str_vec(const boost::string_view
& str
, const char* const delims
)
328 boost::container::small_vector
<boost::string_view
, ExpectedStrNum
> str_vec
;
331 boost::string_view token
;
332 while (pos
< str
.size()) {
333 if (get_next_token(str
, pos
, delims
, token
)) {
334 if (token
.size() > 0) {
335 str_vec
.push_back(token
);
343 template<std::size_t ExpectedStrNum
>
344 boost::container::small_vector
<boost::string_view
, ExpectedStrNum
>
345 get_str_vec(const boost::string_view
& str
)
347 const char delims
[] = ";,= \t";
348 return get_str_vec
<ExpectedStrNum
>(str
, delims
);
352 static inline int parse_v4_auth_header(const req_info
& info
, /* in */
353 boost::string_view
& credential
, /* out */
354 boost::string_view
& signedheaders
, /* out */
355 boost::string_view
& signature
, /* out */
356 boost::string_view
& date
) /* out */
358 boost::string_view
input(info
.env
->get("HTTP_AUTHORIZATION", ""));
360 input
= input
.substr(::strlen(AWS4_HMAC_SHA256_STR
) + 1);
361 } catch (std::out_of_range
&) {
362 /* We should never ever run into this situation as the presence of
363 * AWS4_HMAC_SHA256_STR had been verified earlier. */
364 dout(10) << "credentials string is too short" << dendl
;
368 std::map
<boost::string_view
, boost::string_view
> kv
;
369 for (const auto& s
: get_str_vec
<4>(input
, ",")) {
370 const auto parsed_pair
= parse_key_value(s
);
372 kv
[parsed_pair
->first
] = parsed_pair
->second
;
374 dout(10) << "NOTICE: failed to parse auth header (s=" << s
<< ")"
380 static const std::array
<boost::string_view
, 3> required_keys
= {
386 /* Ensure that the presigned required keys are really there. */
387 for (const auto& k
: required_keys
) {
388 if (kv
.find(k
) == std::end(kv
)) {
389 dout(10) << "NOTICE: auth header missing key: " << k
<< dendl
;
394 credential
= kv
["Credential"];
395 signedheaders
= kv
["SignedHeaders"];
396 signature
= kv
["Signature"];
399 dout(10) << "v4 signature format = " << signature
<< dendl
;
401 /* ------------------------- handle x-amz-date header */
405 const char *d
= info
.env
->get("HTTP_X_AMZ_DATE");
407 if (!parse_iso8601(d
, &t
, NULL
, false)) {
408 dout(10) << "error reading date via http_x_amz_date" << dendl
;
416 int parse_credentials(const req_info
& info
, /* in */
417 boost::string_view
& access_key_id
, /* out */
418 boost::string_view
& credential_scope
, /* out */
419 boost::string_view
& signedheaders
, /* out */
420 boost::string_view
& signature
, /* out */
421 boost::string_view
& date
, /* out */
422 bool& using_qs
) /* out */
424 const char* const http_auth
= info
.env
->get("HTTP_AUTHORIZATION");
425 using_qs
= http_auth
== nullptr || http_auth
[0] == '\0';
428 boost::string_view credential
;
430 ret
= parse_v4_query_string(info
, credential
, signedheaders
,
433 ret
= parse_v4_auth_header(info
, credential
, signedheaders
,
441 /* AKIAIVKTAZLOCF43WNQD/AAAAMMDD/region/host/aws4_request */
442 dout(10) << "v4 credential format = " << credential
<< dendl
;
444 if (std::count(credential
.begin(), credential
.end(), '/') != 4) {
448 /* credential must end with 'aws4_request' */
449 if (credential
.find("aws4_request") == std::string::npos
) {
453 /* grab access key id */
454 const size_t pos
= credential
.find("/");
455 access_key_id
= credential
.substr(0, pos
);
456 dout(10) << "access key id = " << access_key_id
<< dendl
;
458 /* grab credential scope */
459 credential_scope
= credential
.substr(pos
+ 1);
460 dout(10) << "credential scope = " << credential_scope
<< dendl
;
465 static inline bool char_needs_aws4_escaping(const char c
)
467 if ((c
>= 'a' && c
<= 'z') ||
468 (c
>= 'A' && c
<= 'Z') ||
469 (c
>= '0' && c
<= '9')) {
483 static inline std::string
aws4_uri_encode(const std::string
& src
)
487 for (const std::string::value_type c
: src
) {
488 if (char_needs_aws4_escaping(c
)) {
489 rgw_uri_escape_char(c
, result
);
498 static inline std::string
aws4_uri_recode(const boost::string_view
& src
)
500 std::string decoded
= url_decode(src
);
501 return aws4_uri_encode(decoded
);
504 std::string
get_v4_canonical_qs(const req_info
& info
, const bool using_qs
)
506 if (info
.request_params
.empty()) {
507 /* Optimize the typical flow. */
508 return std::string();
511 /* Handle case when query string exists. Step 3 described in: http://docs.
512 * aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html */
513 std::map
<std::string
, std::string
> canonical_qs_map
;
514 for (const auto& s
: get_str_vec
<5>(info
.request_params
, "&")) {
515 boost::string_view key
, val
;
516 const auto parsed_pair
= parse_key_value(s
);
518 std::tie(key
, val
) = *parsed_pair
;
520 /* Handling a parameter without any value (even the empty one). That's
521 * it, we've encountered something like "this_param&other_param=val"
522 * which is used by S3 for subresources. */
526 if (using_qs
&& key
== "X-Amz-Signature") {
527 /* Preserving the original behaviour of get_v4_canonical_qs() here. */
531 if (key
== "X-Amz-Credential") {
532 /* FIXME(rzarzynski): I can't find any comment in the previously linked
533 * Amazon's docs saying that X-Amz-Credential should be handled in this
535 canonical_qs_map
[key
.to_string()] = val
.to_string();
537 canonical_qs_map
[aws4_uri_recode(key
)] = aws4_uri_recode(val
);
541 /* Thanks to the early exist we have the guarantee that canonical_qs_map has
542 * at least one element. */
543 auto iter
= std::begin(canonical_qs_map
);
544 std::string canonical_qs
;
545 canonical_qs
.append(iter
->first
)
546 .append("=", ::strlen("="))
547 .append(iter
->second
);
549 for (iter
++; iter
!= std::end(canonical_qs_map
); iter
++) {
550 canonical_qs
.append("&", ::strlen("&"))
552 .append("=", ::strlen("="))
553 .append(iter
->second
);
559 boost::optional
<std::string
>
560 get_v4_canonical_headers(const req_info
& info
,
561 const boost::string_view
& signedheaders
,
563 const bool force_boto2_compat
)
565 std::map
<boost::string_view
, std::string
> canonical_hdrs_map
;
566 for (const auto& token
: get_str_vec
<5>(signedheaders
, ";")) {
567 /* TODO(rzarzynski): we'd like to switch to sstring here but it should
568 * get push_back() and reserve() first. */
569 std::string token_env
= "HTTP_";
570 token_env
.reserve(token
.length() + std::strlen("HTTP_") + 1);
572 std::transform(std::begin(token
), std::end(token
),
573 std::back_inserter(token_env
), [](const int c
) {
574 return c
== '-' ? '_' : std::toupper(c
);
577 if (token_env
== "HTTP_CONTENT_LENGTH") {
578 token_env
= "CONTENT_LENGTH";
579 } else if (token_env
== "HTTP_CONTENT_TYPE") {
580 token_env
= "CONTENT_TYPE";
582 const char* const t
= info
.env
->get(token_env
.c_str());
584 dout(10) << "warning env var not available" << dendl
;
588 std::string
token_value(t
);
589 if (token_env
== "HTTP_CONTENT_MD5" &&
590 !std::all_of(std::begin(token_value
), std::end(token_value
),
591 is_base64_for_content_md5
)) {
592 dout(0) << "NOTICE: bad content-md5 provided (not base64)"
593 << ", aborting request" << dendl
;
597 if (force_boto2_compat
&& using_qs
&& token
== "host") {
598 boost::string_view port
= info
.env
->get("SERVER_PORT", "");
599 boost::string_view secure_port
= info
.env
->get("SERVER_PORT_SECURE", "");
601 if (!secure_port
.empty()) {
602 if (secure_port
!= "443")
603 token_value
.append(":", std::strlen(":"))
604 .append(secure_port
.data(), secure_port
.length());
605 } else if (!port
.empty()) {
607 token_value
.append(":", std::strlen(":"))
608 .append(port
.data(), port
.length());
612 canonical_hdrs_map
[token
] = rgw_trim_whitespace(token_value
);
615 std::string canonical_hdrs
;
616 for (const auto& header
: canonical_hdrs_map
) {
617 const boost::string_view
& name
= header
.first
;
618 const std::string
& value
= header
.second
;
620 canonical_hdrs
.append(name
.data(), name
.length())
621 .append(":", std::strlen(":"))
623 .append("\n", std::strlen("\n"));
626 return canonical_hdrs
;
630 * create canonical request for signature version 4
632 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
635 get_v4_canon_req_hash(CephContext
* cct
,
636 const boost::string_view
& http_verb
,
637 const std::string
& canonical_uri
,
638 const std::string
& canonical_qs
,
639 const std::string
& canonical_hdrs
,
640 const boost::string_view
& signed_hdrs
,
641 const boost::string_view
& request_payload_hash
)
643 ldout(cct
, 10) << "payload request hash = " << request_payload_hash
<< dendl
;
645 const auto canonical_req
= string_join_reserve("\n",
651 request_payload_hash
);
653 const auto canonical_req_hash
= calc_hash_sha256(canonical_req
);
655 ldout(cct
, 10) << "canonical request = " << canonical_req
<< dendl
;
656 ldout(cct
, 10) << "canonical request hash = "
657 << buf_to_hex(canonical_req_hash
).data() << dendl
;
659 return canonical_req_hash
;
663 * create string to sign for signature version 4
665 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
667 AWSEngine::VersionAbstractor::string_to_sign_t
668 get_v4_string_to_sign(CephContext
* const cct
,
669 const boost::string_view
& algorithm
,
670 const boost::string_view
& request_date
,
671 const boost::string_view
& credential_scope
,
672 const sha256_digest_t
& canonreq_hash
)
674 const auto hexed_cr_hash
= buf_to_hex(canonreq_hash
);
675 const boost::string_view
hexed_cr_hash_str(hexed_cr_hash
.data(),
676 hexed_cr_hash
.size() - 1);
678 const auto string_to_sign
= string_join_reserve("\n",
684 ldout(cct
, 10) << "string to sign = "
685 << rgw::crypt_sanitize::log_content
{string_to_sign
}
688 return string_to_sign
;
692 static inline std::tuple
<boost::string_view
, /* date */
693 boost::string_view
, /* region */
694 boost::string_view
> /* service */
695 parse_cred_scope(boost::string_view credential_scope
)
698 size_t pos
= credential_scope
.find("/");
699 const auto date_cs
= credential_scope
.substr(0, pos
);
700 credential_scope
= credential_scope
.substr(pos
+ 1);
703 pos
= credential_scope
.find("/");
704 const auto region_cs
= credential_scope
.substr(0, pos
);
705 credential_scope
= credential_scope
.substr(pos
+ 1);
708 pos
= credential_scope
.find("/");
709 const auto service_cs
= credential_scope
.substr(0, pos
);
711 return std::make_tuple(date_cs
, region_cs
, service_cs
);
714 static inline std::vector
<unsigned char>
715 transform_secret_key(const boost::string_view
& secret_access_key
)
717 /* TODO(rzarzynski): switch to constexpr when C++14 becomes available. */
718 static const std::initializer_list
<unsigned char> AWS4
{ 'A', 'W', 'S', '4' };
720 /* boost::container::small_vector might be used here if someone wants to
721 * optimize out even more dynamic allocations. */
722 std::vector
<unsigned char> secret_key_utf8
;
723 secret_key_utf8
.reserve(AWS4
.size() + secret_access_key
.size());
724 secret_key_utf8
.assign(AWS4
);
726 for (const auto c
: secret_access_key
) {
727 std::array
<unsigned char, MAX_UTF8_SZ
> buf
;
728 const size_t n
= encode_utf8(c
, buf
.data());
729 secret_key_utf8
.insert(std::end(secret_key_utf8
),
730 std::begin(buf
), std::begin(buf
) + n
);
733 return secret_key_utf8
;
737 * calculate the SigningKey of AWS auth version 4
739 static sha256_digest_t
740 get_v4_signing_key(CephContext
* const cct
,
741 const boost::string_view
& credential_scope
,
742 const boost::string_view
& secret_access_key
)
744 boost::string_view date
, region
, service
;
745 std::tie(date
, region
, service
) = parse_cred_scope(credential_scope
);
747 const auto utfed_sec_key
= transform_secret_key(secret_access_key
);
748 const auto date_k
= calc_hmac_sha256(utfed_sec_key
, date
);
749 const auto region_k
= calc_hmac_sha256(date_k
, region
);
750 const auto service_k
= calc_hmac_sha256(region_k
, service
);
753 const auto signing_key
= calc_hmac_sha256(service_k
,
754 boost::string_view("aws4_request"));
756 ldout(cct
, 10) << "date_k = " << buf_to_hex(date_k
).data() << dendl
;
757 ldout(cct
, 10) << "region_k = " << buf_to_hex(region_k
).data() << dendl
;
758 ldout(cct
, 10) << "service_k = " << buf_to_hex(service_k
).data() << dendl
;
759 ldout(cct
, 10) << "signing_k = " << buf_to_hex(signing_key
).data() << dendl
;
765 * calculate the AWS signature version 4
767 * http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
769 * srv_signature_t is an alias over Ceph's basic_sstring. We're using
770 * it to keep everything within the stack boundaries instead of doing
771 * dynamic allocations.
773 AWSEngine::VersionAbstractor::server_signature_t
774 get_v4_signature(const boost::string_view
& credential_scope
,
775 CephContext
* const cct
,
776 const boost::string_view
& secret_key
,
777 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
)
779 auto signing_key
= get_v4_signing_key(cct
, credential_scope
, secret_key
);
781 /* The server-side generated digest for comparison. */
782 const auto digest
= calc_hmac_sha256(signing_key
, string_to_sign
);
784 /* TODO(rzarzynski): I would love to see our sstring having reserve() and
785 * the non-const data() variant like C++17's std::string. */
786 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
787 srv_signature_t
signature(srv_signature_t::initialized_later(),
789 buf_to_hex(digest
.data(), digest
.size(), signature
.begin());
791 ldout(cct
, 10) << "generated signature = " << signature
<< dendl
;
796 AWSEngine::VersionAbstractor::server_signature_t
797 get_v2_signature(CephContext
* const cct
,
798 const std::string
& secret_key
,
799 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
)
801 if (secret_key
.empty()) {
805 const auto digest
= calc_hmac_sha1(secret_key
, string_to_sign
);
807 /* 64 is really enough */;
809 const int ret
= ceph_armor(std::begin(buf
),
810 std::begin(buf
) + 64,
812 std::begin(digest
) + digest
.size());
814 ldout(cct
, 10) << "ceph_armor failed" << dendl
;
818 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
819 return srv_signature_t(buf
, ret
);
823 bool AWSv4ComplMulti::ChunkMeta::is_new_chunk_in_stream(size_t stream_pos
) const
825 return stream_pos
>= (data_offset_in_stream
+ data_length
);
828 size_t AWSv4ComplMulti::ChunkMeta::get_data_size(size_t stream_pos
) const
830 if (stream_pos
> (data_offset_in_stream
+ data_length
)) {
831 /* Data in parsing_buf. */
834 return data_offset_in_stream
+ data_length
- stream_pos
;
839 /* AWSv4 completers begin. */
840 std::pair
<AWSv4ComplMulti::ChunkMeta
, size_t /* consumed */>
841 AWSv4ComplMulti::ChunkMeta::create_next(CephContext
* const cct
,
843 const char* const metabuf
,
844 const size_t metabuf_len
)
846 boost::string_ref
metastr(metabuf
, metabuf_len
);
848 const size_t semicolon_pos
= metastr
.find(";");
849 if (semicolon_pos
== boost::string_ref::npos
) {
850 ldout(cct
, 20) << "AWSv4ComplMulti cannot find the ';' separator"
852 throw rgw::io::Exception(EINVAL
, std::system_category());
855 char* data_field_end
;
856 /* strtoull ignores the "\r\n" sequence after each non-first chunk. */
857 const size_t data_length
= std::strtoull(metabuf
, &data_field_end
, 16);
858 if (data_length
== 0 && data_field_end
== metabuf
) {
859 ldout(cct
, 20) << "AWSv4ComplMulti: cannot parse the data size"
861 throw rgw::io::Exception(EINVAL
, std::system_category());
864 /* Parse the chunk_signature=... part. */
865 const auto signature_part
= metastr
.substr(semicolon_pos
+ 1);
866 const size_t eq_sign_pos
= signature_part
.find("=");
867 if (eq_sign_pos
== boost::string_ref::npos
) {
868 ldout(cct
, 20) << "AWSv4ComplMulti: cannot find the '=' separator"
870 throw rgw::io::Exception(EINVAL
, std::system_category());
873 /* OK, we have at least the beginning of a signature. */
874 const size_t data_sep_pos
= signature_part
.find("\r\n");
875 if (data_sep_pos
== boost::string_ref::npos
) {
876 ldout(cct
, 20) << "AWSv4ComplMulti: no new line at signature end"
878 throw rgw::io::Exception(EINVAL
, std::system_category());
881 const auto signature
= \
882 signature_part
.substr(eq_sign_pos
+ 1, data_sep_pos
- 1 - eq_sign_pos
);
883 if (signature
.length() != SIG_SIZE
) {
884 ldout(cct
, 20) << "AWSv4ComplMulti: signature.length() != 64"
886 throw rgw::io::Exception(EINVAL
, std::system_category());
889 const size_t data_starts_in_stream
= \
890 + semicolon_pos
+ strlen(";") + data_sep_pos
+ strlen("\r\n")
891 + old
.data_offset_in_stream
+ old
.data_length
;
893 ldout(cct
, 20) << "parsed new chunk; signature=" << signature
894 << ", data_length=" << data_length
895 << ", data_starts_in_stream=" << data_starts_in_stream
898 return std::make_pair(ChunkMeta(data_starts_in_stream
,
905 AWSv4ComplMulti::calc_chunk_signature(const std::string
& payload_hash
) const
907 const auto string_to_sign
= string_join_reserve("\n",
908 AWS4_HMAC_SHA256_PAYLOAD_STR
,
911 prev_chunk_signature
,
912 AWS4_EMPTY_PAYLOAD_HASH
,
915 ldout(cct
, 20) << "AWSv4ComplMulti: string_to_sign=\n" << string_to_sign
918 /* new chunk signature */
919 const auto sighex
= buf_to_hex(calc_hmac_sha256(signing_key
,
921 /* FIXME(rzarzynski): std::string here is really unnecessary. */
922 return std::string(sighex
.data(), sighex
.size() - 1);
926 bool AWSv4ComplMulti::is_signature_mismatched()
928 /* The validity of previous chunk can be verified only after getting meta-
929 * data of the next one. */
930 const auto payload_hash
= calc_hash_sha256_restart_stream(&sha256_hash
);
931 const auto calc_signature
= calc_chunk_signature(payload_hash
);
933 if (chunk_meta
.get_signature() != calc_signature
) {
934 ldout(cct
, 20) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
936 ldout(cct
, 20) << "AWSv4ComplMulti: declared signature="
937 << chunk_meta
.get_signature() << dendl
;
938 ldout(cct
, 20) << "AWSv4ComplMulti: calculated signature="
939 << calc_signature
<< dendl
;
943 prev_chunk_signature
= chunk_meta
.get_signature();
948 size_t AWSv4ComplMulti::recv_body(char* const buf
, const size_t buf_max
)
950 /* Buffer stores only parsed stream. Raw values reflect the stream
951 * we're getting from a client. */
954 if (chunk_meta
.is_new_chunk_in_stream(stream_pos
)) {
955 /* Verify signature of the previous chunk. We aren't doing that for new
956 * one as the procedure requires calculation of payload hash. This code
957 * won't be triggered for the last, zero-length chunk. Instead, is will
958 * be checked in the complete() method. */
959 if (stream_pos
>= ChunkMeta::META_MAX_SIZE
&& is_signature_mismatched()) {
960 throw rgw::io::Exception(ERR_SIGNATURE_NO_MATCH
, std::system_category());
963 /* We don't have metadata for this range. This means a new chunk, so we
964 * need to parse a fresh portion of the stream. Let's start. */
965 size_t to_extract
= parsing_buf
.capacity() - parsing_buf
.size();
967 const size_t orig_size
= parsing_buf
.size();
968 parsing_buf
.resize(parsing_buf
.size() + to_extract
);
969 const size_t received
= io_base_t::recv_body(parsing_buf
.data() + orig_size
,
971 parsing_buf
.resize(parsing_buf
.size() - (to_extract
- received
));
976 stream_pos
+= received
;
977 to_extract
-= received
;
978 } while (to_extract
> 0);
981 std::tie(chunk_meta
, consumed
) = \
982 ChunkMeta::create_next(cct
, std::move(chunk_meta
),
983 parsing_buf
.data(), parsing_buf
.size());
985 /* We can drop the bytes consumed during metadata parsing. The remainder
986 * can be chunk's data plus possibly beginning of next chunks' metadata. */
987 parsing_buf
.erase(std::begin(parsing_buf
),
988 std::begin(parsing_buf
) + consumed
);
991 size_t to_extract
= \
992 std::min(chunk_meta
.get_data_size(stream_pos
), buf_max
);
994 /* It's quite probable we have a couple of real data bytes stored together
995 * with meta-data in the parsing_buf. We need to extract them and move to
996 * the final buffer. This is a trade-off between frontend's read overhead
998 if (to_extract
> 0 && parsing_buf
.size() > 0) {
999 const auto data_len
= std::min(to_extract
, parsing_buf
.size());
1000 const auto data_end_iter
= std::begin(parsing_buf
) + data_len
;
1002 std::copy(std::begin(parsing_buf
), data_end_iter
, buf
);
1003 parsing_buf
.erase(std::begin(parsing_buf
), data_end_iter
);
1005 calc_hash_sha256_update_stream(sha256_hash
, buf
, data_len
);
1007 to_extract
-= data_len
;
1008 buf_pos
+= data_len
;
1011 /* Now we can do the bulk read directly from RestfulClient without any extra
1013 while (to_extract
> 0) {
1014 const size_t received
= io_base_t::recv_body(buf
+ buf_pos
, to_extract
);
1016 if (received
== 0) {
1020 calc_hash_sha256_update_stream(sha256_hash
, buf
+ buf_pos
, received
);
1022 buf_pos
+= received
;
1023 stream_pos
+= received
;
1024 to_extract
-= received
;
1027 dout(20) << "AWSv4ComplMulti: filled=" << buf_pos
<< dendl
;
1031 void AWSv4ComplMulti::modify_request_state(req_state
* const s_rw
)
1033 const char* const decoded_length
= \
1034 s_rw
->info
.env
->get("HTTP_X_AMZ_DECODED_CONTENT_LENGTH");
1036 if (!decoded_length
) {
1039 s_rw
->length
= decoded_length
;
1040 s_rw
->content_length
= parse_content_length(decoded_length
);
1042 if (s_rw
->content_length
< 0) {
1043 ldout(cct
, 10) << "negative AWSv4's content length, aborting" << dendl
;
1048 /* Install the filter over rgw::io::RestfulClient. */
1049 AWS_AUTHv4_IO(s_rw
)->add_filter(
1050 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1053 bool AWSv4ComplMulti::complete()
1055 /* Now it's time to verify the signature of the last, zero-length chunk. */
1056 if (is_signature_mismatched()) {
1057 ldout(cct
, 10) << "ERROR: signature of last chunk does not match"
1065 rgw::auth::Completer::cmplptr_t
1066 AWSv4ComplMulti::create(const req_state
* const s
,
1067 boost::string_view date
,
1068 boost::string_view credential_scope
,
1069 boost::string_view seed_signature
,
1070 const boost::optional
<std::string
>& secret_key
)
1073 /* Some external authorizers (like Keystone) aren't fully compliant with
1074 * AWSv4. They do not provide the secret_key which is necessary to handle
1075 * the streamed upload. */
1076 throw -ERR_NOT_IMPLEMENTED
;
1079 const auto signing_key
= \
1080 rgw::auth::s3::get_v4_signing_key(s
->cct
, credential_scope
, *secret_key
);
1082 return std::make_shared
<AWSv4ComplMulti
>(s
,
1084 std::move(credential_scope
),
1085 std::move(seed_signature
),
1089 size_t AWSv4ComplSingle::recv_body(char* const buf
, const size_t max
)
1091 const auto received
= io_base_t::recv_body(buf
, max
);
1092 calc_hash_sha256_update_stream(sha256_hash
, buf
, received
);
1097 void AWSv4ComplSingle::modify_request_state(req_state
* const s_rw
)
1099 /* Install the filter over rgw::io::RestfulClient. */
1100 AWS_AUTHv4_IO(s_rw
)->add_filter(
1101 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1104 bool AWSv4ComplSingle::complete()
1106 /* The completer is only for the cases where signed payload has been
1107 * requested. It won't be used, for instance, during the query string-based
1108 * authentication. */
1109 const auto payload_hash
= calc_hash_sha256_close_stream(&sha256_hash
);
1111 /* Validate x-amz-sha256 */
1112 if (payload_hash
.compare(expected_request_payload_hash
) == 0) {
1115 ldout(cct
, 10) << "ERROR: x-amz-content-sha256 does not match"
1117 ldout(cct
, 10) << "ERROR: grab_aws4_sha256_hash()="
1118 << payload_hash
<< dendl
;
1119 ldout(cct
, 10) << "ERROR: expected_request_payload_hash="
1120 << expected_request_payload_hash
<< dendl
;
1125 AWSv4ComplSingle::AWSv4ComplSingle(const req_state
* const s
)
1126 : io_base_t(nullptr),
1128 expected_request_payload_hash(get_v4_exp_payload_hash(s
->info
)),
1129 sha256_hash(calc_hash_sha256_open_stream()) {
1132 rgw::auth::Completer::cmplptr_t
1133 AWSv4ComplSingle::create(const req_state
* const s
,
1134 const boost::optional
<std::string
>&)
1136 return std::make_shared
<AWSv4ComplSingle
>(s
);
1139 } /* namespace s3 */
1140 } /* namespace auth */
1141 } /* namespace rgw */