1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
11 #include "common/armor.h"
12 #include "common/utf8.h"
13 #include "rgw_rest_s3.h"
14 #include "rgw_auth_s3.h"
15 #include "rgw_common.h"
16 #include "rgw_client_io.h"
18 #include "rgw_crypt_sanitize.h"
20 #include <boost/container/small_vector.hpp>
21 #include <boost/algorithm/string.hpp>
22 #include <boost/algorithm/string/trim_all.hpp>
24 #define dout_context g_ceph_context
25 #define dout_subsys ceph_subsys_rgw
27 static const auto signed_subresources
= {
40 "response-cache-control",
41 "response-content-disposition",
42 "response-content-encoding",
43 "response-content-language",
44 "response-content-type",
58 * ?get the canonical amazon-style header for something?
62 get_canon_amz_hdr(const meta_map_t
& meta_map
)
66 for (const auto& kv
: meta_map
) {
67 dest
.append(kv
.first
);
69 dest
.append(kv
.second
);
77 * ?get the canonical representation of the object's location
80 get_canon_resource(const DoutPrefixProvider
*dpp
, const char* const request_uri
,
81 const std::map
<std::string
, std::string
>& sub_resources
)
86 dest
.append(request_uri
);
90 for (const auto& subresource
: signed_subresources
) {
91 const auto iter
= sub_resources
.find(subresource
);
92 if (iter
== std::end(sub_resources
)) {
103 dest
.append(iter
->first
);
104 if (! iter
->second
.empty()) {
106 dest
.append(iter
->second
);
110 ldpp_dout(dpp
, 10) << "get_canon_resource(): dest=" << dest
<< dendl
;
115 * get the header authentication information required to
116 * compute a request's signature
118 void rgw_create_s3_canonical_header(
119 const DoutPrefixProvider
*dpp
,
120 const char* const method
,
121 const char* const content_md5
,
122 const char* const content_type
,
123 const char* const date
,
124 const meta_map_t
& meta_map
,
125 const meta_map_t
& qs_map
,
126 const char* const request_uri
,
127 const std::map
<std::string
, std::string
>& sub_resources
,
128 std::string
& dest_str
)
138 dest
.append(content_md5
);
143 dest
.append(content_type
);
152 dest
.append(get_canon_amz_hdr(meta_map
));
153 dest
.append(get_canon_amz_hdr(qs_map
));
154 dest
.append(get_canon_resource(dpp
, request_uri
, sub_resources
));
159 static inline bool is_base64_for_content_md5(unsigned char c
) {
160 return (isalnum(c
) || isspace(c
) || (c
== '+') || (c
== '/') || (c
== '='));
163 static inline void get_v2_qs_map(const req_info
& info
,
164 meta_map_t
& qs_map
) {
165 const auto& params
= const_cast<RGWHTTPArgs
&>(info
.args
).get_params();
166 for (const auto& elt
: params
) {
167 std::string k
= boost::algorithm::to_lower_copy(elt
.first
);
168 if (k
.find("x-amz-meta-") == /* offset */ 0) {
169 rgw_add_amz_meta_header(qs_map
, k
, elt
.second
);
171 if (k
== "x-amz-security-token") {
172 qs_map
[k
] = elt
.second
;
178 * get the header authentication information required to
179 * compute a request's signature
181 bool rgw_create_s3_canonical_header(const DoutPrefixProvider
*dpp
,
182 const req_info
& info
,
183 utime_t
* const header_time
,
187 const char* const content_md5
= info
.env
->get("HTTP_CONTENT_MD5");
189 for (const char *p
= content_md5
; *p
; p
++) {
190 if (!is_base64_for_content_md5(*p
)) {
191 ldpp_dout(dpp
, 0) << "NOTICE: bad content-md5 provided (not base64),"
192 << " aborting request p=" << *p
<< " " << (int)*p
<< dendl
;
198 const char *content_type
= info
.env
->get("CONTENT_TYPE");
204 get_v2_qs_map(info
, qs_map
); // handle qs metadata
205 date
= info
.args
.get("Expires");
207 const char *str
= info
.env
->get("HTTP_X_AMZ_DATE");
208 const char *req_date
= str
;
210 req_date
= info
.env
->get("HTTP_DATE");
212 ldpp_dout(dpp
, 0) << "NOTICE: missing date for auth header" << dendl
;
220 if (!parse_rfc2616(req_date
, &t
)) {
221 ldpp_dout(dpp
, 0) << "NOTICE: failed to parse date for auth header" << dendl
;
224 if (t
.tm_year
< 70) {
225 ldpp_dout(dpp
, 0) << "NOTICE: bad date (predates epoch): " << req_date
<< dendl
;
228 *header_time
= utime_t(internal_timegm(&t
), 0);
229 *header_time
-= t
.tm_gmtoff
;
233 const auto& meta_map
= info
.x_meta_map
;
234 const auto& sub_resources
= info
.args
.get_sub_resources();
236 std::string request_uri
;
237 if (info
.effective_uri
.empty()) {
238 request_uri
= info
.request_uri
;
240 request_uri
= info
.effective_uri
;
243 rgw_create_s3_canonical_header(dpp
, info
.method
, content_md5
, content_type
,
244 date
.c_str(), meta_map
, qs_map
,
245 request_uri
.c_str(), sub_resources
, dest
);
250 namespace rgw::auth::s3
{
252 bool is_time_skew_ok(time_t t
)
254 auto req_tp
= ceph::coarse_real_clock::from_time_t(t
);
255 auto cur_tp
= ceph::coarse_real_clock::now();
257 if (std::chrono::abs(cur_tp
- req_tp
) > RGW_AUTH_GRACE
) {
258 dout(10) << "NOTICE: request time skew too big." << dendl
;
259 using ceph::operator<<;
260 dout(10) << "req_tp=" << req_tp
<< ", cur_tp=" << cur_tp
<< dendl
;
267 static inline int parse_v4_query_string(const req_info
& info
, /* in */
268 std::string_view
& credential
, /* out */
269 std::string_view
& signedheaders
, /* out */
270 std::string_view
& signature
, /* out */
271 std::string_view
& date
, /* out */
272 std::string_view
& sessiontoken
) /* out */
274 /* auth ships with req params ... */
276 /* look for required params */
277 credential
= info
.args
.get("x-amz-credential");
278 if (credential
.size() == 0) {
282 date
= info
.args
.get("x-amz-date");
284 if (!parse_iso8601(sview2cstr(date
).data(), &date_t
, nullptr, false)) {
288 std::string_view expires
= info
.args
.get("x-amz-expires");
289 if (expires
.empty()) {
292 /* X-Amz-Expires provides the time period, in seconds, for which
293 the generated presigned URL is valid. The minimum value
294 you can set is 1, and the maximum is 604800 (seven days) */
295 time_t exp
= atoll(expires
.data());
296 if ((exp
< 1) || (exp
> 7*24*60*60)) {
297 dout(10) << "NOTICE: exp out of range, exp = " << exp
<< dendl
;
300 /* handle expiration in epoch time */
301 uint64_t req_sec
= (uint64_t)internal_timegm(&date_t
);
302 uint64_t now
= ceph_clock_now();
303 if (now
>= req_sec
+ exp
) {
304 dout(10) << "NOTICE: now = " << now
<< ", req_sec = " << req_sec
<< ", exp = " << exp
<< dendl
;
308 signedheaders
= info
.args
.get("x-amz-signedheaders");
309 if (signedheaders
.size() == 0) {
313 signature
= info
.args
.get("x-amz-signature");
314 if (signature
.size() == 0) {
318 if (info
.args
.exists("x-amz-security-token")) {
319 sessiontoken
= info
.args
.get("x-amz-security-token");
320 if (sessiontoken
.size() == 0) {
328 static bool get_next_token(const std::string_view
& s
,
330 const char* const delims
,
331 std::string_view
& token
)
333 const size_t start
= s
.find_first_not_of(delims
, pos
);
334 if (start
== std::string_view::npos
) {
339 size_t end
= s
.find_first_of(delims
, start
);
340 if (end
!= std::string_view::npos
)
343 pos
= end
= s
.size();
346 token
= s
.substr(start
, end
- start
);
350 template<std::size_t ExpectedStrNum
>
351 boost::container::small_vector
<std::string_view
, ExpectedStrNum
>
352 get_str_vec(const std::string_view
& str
, const char* const delims
)
354 boost::container::small_vector
<std::string_view
, ExpectedStrNum
> str_vec
;
357 std::string_view token
;
358 while (pos
< str
.size()) {
359 if (get_next_token(str
, pos
, delims
, token
)) {
360 if (token
.size() > 0) {
361 str_vec
.push_back(token
);
369 template<std::size_t ExpectedStrNum
>
370 boost::container::small_vector
<std::string_view
, ExpectedStrNum
>
371 get_str_vec(const std::string_view
& str
)
373 const char delims
[] = ";,= \t";
374 return get_str_vec
<ExpectedStrNum
>(str
, delims
);
377 static inline int parse_v4_auth_header(const req_info
& info
, /* in */
378 std::string_view
& credential
, /* out */
379 std::string_view
& signedheaders
, /* out */
380 std::string_view
& signature
, /* out */
381 std::string_view
& date
, /* out */
382 std::string_view
& sessiontoken
, /* out */
383 const DoutPrefixProvider
*dpp
)
385 std::string_view
input(info
.env
->get("HTTP_AUTHORIZATION", ""));
387 input
= input
.substr(::strlen(AWS4_HMAC_SHA256_STR
) + 1);
388 } catch (std::out_of_range
&) {
389 /* We should never ever run into this situation as the presence of
390 * AWS4_HMAC_SHA256_STR had been verified earlier. */
391 ldpp_dout(dpp
, 10) << "credentials string is too short" << dendl
;
395 std::map
<std::string_view
, std::string_view
> kv
;
396 for (const auto& s
: get_str_vec
<4>(input
, ",")) {
397 const auto parsed_pair
= parse_key_value(s
);
399 kv
[parsed_pair
->first
] = parsed_pair
->second
;
401 ldpp_dout(dpp
, 10) << "NOTICE: failed to parse auth header (s=" << s
<< ")"
407 static const std::array
<std::string_view
, 3> required_keys
= {
413 /* Ensure that the presigned required keys are really there. */
414 for (const auto& k
: required_keys
) {
415 if (kv
.find(k
) == std::end(kv
)) {
416 ldpp_dout(dpp
, 10) << "NOTICE: auth header missing key: " << k
<< dendl
;
421 credential
= kv
["Credential"];
422 signedheaders
= kv
["SignedHeaders"];
423 signature
= kv
["Signature"];
426 ldpp_dout(dpp
, 10) << "v4 signature format = " << signature
<< dendl
;
428 /* ------------------------- handle x-amz-date header */
432 const char *d
= info
.env
->get("HTTP_X_AMZ_DATE");
434 if (!parse_iso8601(d
, &t
, NULL
, false)) {
435 ldpp_dout(dpp
, 10) << "error reading date via http_x_amz_date" << dendl
;
440 if (!is_time_skew_ok(internal_timegm(&t
))) {
441 return -ERR_REQUEST_TIME_SKEWED
;
444 if (info
.env
->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
445 sessiontoken
= info
.env
->get("HTTP_X_AMZ_SECURITY_TOKEN");
451 int parse_v4_credentials(const req_info
& info
, /* in */
452 std::string_view
& access_key_id
, /* out */
453 std::string_view
& credential_scope
, /* out */
454 std::string_view
& signedheaders
, /* out */
455 std::string_view
& signature
, /* out */
456 std::string_view
& date
, /* out */
457 std::string_view
& session_token
, /* out */
458 const bool using_qs
, /* in */
459 const DoutPrefixProvider
*dpp
)
461 std::string_view credential
;
464 ret
= parse_v4_query_string(info
, credential
, signedheaders
,
465 signature
, date
, session_token
);
467 ret
= parse_v4_auth_header(info
, credential
, signedheaders
,
468 signature
, date
, session_token
, dpp
);
475 /* access_key/YYYYMMDD/region/service/aws4_request */
476 ldpp_dout(dpp
, 10) << "v4 credential format = " << credential
<< dendl
;
478 if (std::count(credential
.begin(), credential
.end(), '/') != 4) {
482 /* credential must end with 'aws4_request' */
483 if (credential
.find("aws4_request") == std::string::npos
) {
487 /* grab access key id */
488 const size_t pos
= credential
.find("/");
489 access_key_id
= credential
.substr(0, pos
);
490 ldpp_dout(dpp
, 10) << "access key id = " << access_key_id
<< dendl
;
492 /* grab credential scope */
493 credential_scope
= credential
.substr(pos
+ 1);
494 ldpp_dout(dpp
, 10) << "credential scope = " << credential_scope
<< dendl
;
499 std::string
get_v4_canonical_qs(const req_info
& info
, const bool using_qs
)
501 const std::string
*params
= &info
.request_params
;
502 std::string copy_params
;
503 if (params
->empty()) {
504 /* Optimize the typical flow. */
505 return std::string();
507 if (params
->find_first_of('+') != std::string::npos
) {
508 copy_params
= *params
;
509 boost::replace_all(copy_params
, "+", "%20");
510 params
= ©_params
;
513 /* Handle case when query string exists. Step 3 described in: http://docs.
514 * aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html */
515 std::map
<std::string
, std::string
> canonical_qs_map
;
516 for (const auto& s
: get_str_vec
<5>(*params
, "&")) {
517 std::string_view key
, val
;
518 const auto parsed_pair
= parse_key_value(s
);
520 std::tie(key
, val
) = *parsed_pair
;
522 /* Handling a parameter without any value (even the empty one). That's
523 * it, we've encountered something like "this_param&other_param=val"
524 * which is used by S3 for subresources. */
528 if (using_qs
&& boost::iequals(key
, "X-Amz-Signature")) {
529 /* Preserving the original behaviour of get_v4_canonical_qs() here. */
533 // while awsv4 specs ask for all slashes to be encoded, s3 itself is relaxed
534 // in its implementation allowing non-url-encoded slashes to be present in
535 // presigned urls for instance
536 canonical_qs_map
[aws4_uri_recode(key
, true)] = aws4_uri_recode(val
, true);
539 /* Thanks to the early exist we have the guarantee that canonical_qs_map has
540 * at least one element. */
541 auto iter
= std::begin(canonical_qs_map
);
542 std::string canonical_qs
;
543 canonical_qs
.append(iter
->first
)
544 .append("=", ::strlen("="))
545 .append(iter
->second
);
547 for (iter
++; iter
!= std::end(canonical_qs_map
); iter
++) {
548 canonical_qs
.append("&", ::strlen("&"))
550 .append("=", ::strlen("="))
551 .append(iter
->second
);
557 boost::optional
<std::string
>
558 get_v4_canonical_headers(const req_info
& info
,
559 const std::string_view
& signedheaders
,
561 const bool force_boto2_compat
)
563 std::map
<std::string_view
, std::string
> canonical_hdrs_map
;
564 for (const auto& token
: get_str_vec
<5>(signedheaders
, ";")) {
565 /* TODO(rzarzynski): we'd like to switch to sstring here but it should
566 * get push_back() and reserve() first. */
567 std::string token_env
= "HTTP_";
568 token_env
.reserve(token
.length() + std::strlen("HTTP_") + 1);
570 std::transform(std::begin(token
), std::end(token
),
571 std::back_inserter(token_env
), [](const int c
) {
572 return c
== '-' ? '_' : std::toupper(c
);
575 if (token_env
== "HTTP_CONTENT_LENGTH") {
576 token_env
= "CONTENT_LENGTH";
577 } else if (token_env
== "HTTP_CONTENT_TYPE") {
578 token_env
= "CONTENT_TYPE";
580 const char* const t
= info
.env
->get(token_env
.c_str());
582 dout(10) << "warning env var not available " << token_env
.c_str() << dendl
;
586 std::string
token_value(t
);
587 if (token_env
== "HTTP_CONTENT_MD5" &&
588 !std::all_of(std::begin(token_value
), std::end(token_value
),
589 is_base64_for_content_md5
)) {
590 dout(0) << "NOTICE: bad content-md5 provided (not base64)"
591 << ", aborting request" << dendl
;
595 if (force_boto2_compat
&& using_qs
&& token
== "host") {
596 std::string_view port
= info
.env
->get("SERVER_PORT", "");
597 std::string_view secure_port
= info
.env
->get("SERVER_PORT_SECURE", "");
599 if (!secure_port
.empty()) {
600 if (secure_port
!= "443")
601 token_value
.append(":", std::strlen(":"))
602 .append(secure_port
.data(), secure_port
.length());
603 } else if (!port
.empty()) {
605 token_value
.append(":", std::strlen(":"))
606 .append(port
.data(), port
.length());
610 canonical_hdrs_map
[token
] = rgw_trim_whitespace(token_value
);
613 std::string canonical_hdrs
;
614 for (const auto& header
: canonical_hdrs_map
) {
615 const std::string_view
& name
= header
.first
;
616 std::string value
= header
.second
;
617 boost::trim_all
<std::string
>(value
);
619 canonical_hdrs
.append(name
.data(), name
.length())
620 .append(":", std::strlen(":"))
622 .append("\n", std::strlen("\n"));
625 return canonical_hdrs
;
629 * create canonical request for signature version 4
631 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
634 get_v4_canon_req_hash(CephContext
* cct
,
635 const std::string_view
& http_verb
,
636 const std::string
& canonical_uri
,
637 const std::string
& canonical_qs
,
638 const std::string
& canonical_hdrs
,
639 const std::string_view
& signed_hdrs
,
640 const std::string_view
& request_payload_hash
,
641 const DoutPrefixProvider
*dpp
)
643 ldpp_dout(dpp
, 10) << "payload request hash = " << request_payload_hash
<< dendl
;
645 const auto canonical_req
= string_join_reserve("\n",
651 request_payload_hash
);
653 const auto canonical_req_hash
= calc_hash_sha256(canonical_req
);
655 using sanitize
= rgw::crypt_sanitize::log_content
;
656 ldpp_dout(dpp
, 10) << "canonical request = " << sanitize
{canonical_req
} << dendl
;
657 ldpp_dout(dpp
, 10) << "canonical request hash = "
658 << canonical_req_hash
<< dendl
;
660 return canonical_req_hash
;
664 * create string to sign for signature version 4
666 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
668 AWSEngine::VersionAbstractor::string_to_sign_t
669 get_v4_string_to_sign(CephContext
* const cct
,
670 const std::string_view
& algorithm
,
671 const std::string_view
& request_date
,
672 const std::string_view
& credential_scope
,
673 const sha256_digest_t
& canonreq_hash
,
674 const DoutPrefixProvider
*dpp
)
676 const auto hexed_cr_hash
= canonreq_hash
.to_str();
677 const std::string_view
hexed_cr_hash_str(hexed_cr_hash
);
679 const auto string_to_sign
= string_join_reserve("\n",
685 ldpp_dout(dpp
, 10) << "string to sign = "
686 << rgw::crypt_sanitize::log_content
{string_to_sign
}
689 return string_to_sign
;
693 static inline std::tuple
<std::string_view
, /* date */
694 std::string_view
, /* region */
695 std::string_view
> /* service */
696 parse_cred_scope(std::string_view credential_scope
)
699 size_t pos
= credential_scope
.find("/");
700 const auto date_cs
= credential_scope
.substr(0, pos
);
701 credential_scope
= credential_scope
.substr(pos
+ 1);
704 pos
= credential_scope
.find("/");
705 const auto region_cs
= credential_scope
.substr(0, pos
);
706 credential_scope
= credential_scope
.substr(pos
+ 1);
709 pos
= credential_scope
.find("/");
710 const auto service_cs
= credential_scope
.substr(0, pos
);
712 return std::make_tuple(date_cs
, region_cs
, service_cs
);
715 static inline std::vector
<unsigned char>
716 transform_secret_key(const std::string_view
& secret_access_key
)
718 /* TODO(rzarzynski): switch to constexpr when C++14 becomes available. */
719 static const std::initializer_list
<unsigned char> AWS4
{ 'A', 'W', 'S', '4' };
721 /* boost::container::small_vector might be used here if someone wants to
722 * optimize out even more dynamic allocations. */
723 std::vector
<unsigned char> secret_key_utf8
;
724 secret_key_utf8
.reserve(AWS4
.size() + secret_access_key
.size());
725 secret_key_utf8
.assign(AWS4
);
727 for (const auto c
: secret_access_key
) {
728 std::array
<unsigned char, MAX_UTF8_SZ
> buf
;
729 const size_t n
= encode_utf8(c
, buf
.data());
730 secret_key_utf8
.insert(std::end(secret_key_utf8
),
731 std::begin(buf
), std::begin(buf
) + n
);
734 return secret_key_utf8
;
738 * calculate the SigningKey of AWS auth version 4
740 static sha256_digest_t
741 get_v4_signing_key(CephContext
* const cct
,
742 const std::string_view
& credential_scope
,
743 const std::string_view
& secret_access_key
,
744 const DoutPrefixProvider
*dpp
)
746 std::string_view date
, region
, service
;
747 std::tie(date
, region
, service
) = parse_cred_scope(credential_scope
);
749 const auto utfed_sec_key
= transform_secret_key(secret_access_key
);
750 const auto date_k
= calc_hmac_sha256(utfed_sec_key
, date
);
751 const auto region_k
= calc_hmac_sha256(date_k
, region
);
752 const auto service_k
= calc_hmac_sha256(region_k
, service
);
755 const auto signing_key
= calc_hmac_sha256(service_k
,
756 std::string_view("aws4_request"));
758 ldpp_dout(dpp
, 10) << "date_k = " << date_k
<< dendl
;
759 ldpp_dout(dpp
, 10) << "region_k = " << region_k
<< dendl
;
760 ldpp_dout(dpp
, 10) << "service_k = " << service_k
<< dendl
;
761 ldpp_dout(dpp
, 10) << "signing_k = " << signing_key
<< dendl
;
767 * calculate the AWS signature version 4
769 * http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
771 * srv_signature_t is an alias over Ceph's basic_sstring. We're using
772 * it to keep everything within the stack boundaries instead of doing
773 * dynamic allocations.
775 AWSEngine::VersionAbstractor::server_signature_t
776 get_v4_signature(const std::string_view
& credential_scope
,
777 CephContext
* const cct
,
778 const std::string_view
& secret_key
,
779 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
,
780 const DoutPrefixProvider
*dpp
)
782 auto signing_key
= get_v4_signing_key(cct
, credential_scope
, secret_key
, dpp
);
784 /* The server-side generated digest for comparison. */
785 const auto digest
= calc_hmac_sha256(signing_key
, string_to_sign
);
787 /* TODO(rzarzynski): I would love to see our sstring having reserve() and
788 * the non-const data() variant like C++17's std::string. */
789 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
790 srv_signature_t
signature(srv_signature_t::initialized_later(),
792 buf_to_hex(digest
.v
, digest
.SIZE
, signature
.begin());
794 ldpp_dout(dpp
, 10) << "generated signature = " << signature
<< dendl
;
799 AWSEngine::VersionAbstractor::server_signature_t
800 get_v2_signature(CephContext
* const cct
,
801 const std::string
& secret_key
,
802 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
)
804 if (secret_key
.empty()) {
808 const auto digest
= calc_hmac_sha1(secret_key
, string_to_sign
);
810 /* 64 is really enough */;
812 const int ret
= ceph_armor(std::begin(buf
),
813 std::begin(buf
) + 64,
814 reinterpret_cast<const char *>(digest
.v
),
815 reinterpret_cast<const char *>(digest
.v
+ digest
.SIZE
));
817 ldout(cct
, 10) << "ceph_armor failed" << dendl
;
821 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
822 return srv_signature_t(buf
, ret
);
826 bool AWSv4ComplMulti::ChunkMeta::is_new_chunk_in_stream(size_t stream_pos
) const
828 return stream_pos
>= (data_offset_in_stream
+ data_length
);
831 size_t AWSv4ComplMulti::ChunkMeta::get_data_size(size_t stream_pos
) const
833 if (stream_pos
> (data_offset_in_stream
+ data_length
)) {
834 /* Data in parsing_buf. */
837 return data_offset_in_stream
+ data_length
- stream_pos
;
842 /* AWSv4 completers begin. */
843 std::pair
<AWSv4ComplMulti::ChunkMeta
, size_t /* consumed */>
844 AWSv4ComplMulti::ChunkMeta::create_next(CephContext
* const cct
,
846 const char* const metabuf
,
847 const size_t metabuf_len
)
849 std::string_view
metastr(metabuf
, metabuf_len
);
851 const size_t semicolon_pos
= metastr
.find(";");
852 if (semicolon_pos
== std::string_view::npos
) {
853 ldout(cct
, 20) << "AWSv4ComplMulti cannot find the ';' separator"
855 throw rgw::io::Exception(EINVAL
, std::system_category());
858 char* data_field_end
;
859 /* strtoull ignores the "\r\n" sequence after each non-first chunk. */
860 const size_t data_length
= std::strtoull(metabuf
, &data_field_end
, 16);
861 if (data_length
== 0 && data_field_end
== metabuf
) {
862 ldout(cct
, 20) << "AWSv4ComplMulti: cannot parse the data size"
864 throw rgw::io::Exception(EINVAL
, std::system_category());
867 /* Parse the chunk_signature=... part. */
868 const auto signature_part
= metastr
.substr(semicolon_pos
+ 1);
869 const size_t eq_sign_pos
= signature_part
.find("=");
870 if (eq_sign_pos
== std::string_view::npos
) {
871 ldout(cct
, 20) << "AWSv4ComplMulti: cannot find the '=' separator"
873 throw rgw::io::Exception(EINVAL
, std::system_category());
876 /* OK, we have at least the beginning of a signature. */
877 const size_t data_sep_pos
= signature_part
.find("\r\n");
878 if (data_sep_pos
== std::string_view::npos
) {
879 ldout(cct
, 20) << "AWSv4ComplMulti: no new line at signature end"
881 throw rgw::io::Exception(EINVAL
, std::system_category());
884 const auto signature
= \
885 signature_part
.substr(eq_sign_pos
+ 1, data_sep_pos
- 1 - eq_sign_pos
);
886 if (signature
.length() != SIG_SIZE
) {
887 ldout(cct
, 20) << "AWSv4ComplMulti: signature.length() != 64"
889 throw rgw::io::Exception(EINVAL
, std::system_category());
892 const size_t data_starts_in_stream
= \
893 + semicolon_pos
+ strlen(";") + data_sep_pos
+ strlen("\r\n")
894 + old
.data_offset_in_stream
+ old
.data_length
;
896 ldout(cct
, 20) << "parsed new chunk; signature=" << signature
897 << ", data_length=" << data_length
898 << ", data_starts_in_stream=" << data_starts_in_stream
901 return std::make_pair(ChunkMeta(data_starts_in_stream
,
908 AWSv4ComplMulti::calc_chunk_signature(const std::string
& payload_hash
) const
910 const auto string_to_sign
= string_join_reserve("\n",
911 AWS4_HMAC_SHA256_PAYLOAD_STR
,
914 prev_chunk_signature
,
915 AWS4_EMPTY_PAYLOAD_HASH
,
918 ldout(cct
, 20) << "AWSv4ComplMulti: string_to_sign=\n" << string_to_sign
921 /* new chunk signature */
922 const auto sig
= calc_hmac_sha256(signing_key
, string_to_sign
);
923 /* FIXME(rzarzynski): std::string here is really unnecessary. */
928 bool AWSv4ComplMulti::is_signature_mismatched()
930 /* The validity of previous chunk can be verified only after getting meta-
931 * data of the next one. */
932 const auto payload_hash
= calc_hash_sha256_restart_stream(&sha256_hash
);
933 const auto calc_signature
= calc_chunk_signature(payload_hash
);
935 if (chunk_meta
.get_signature() != calc_signature
) {
936 ldout(cct
, 20) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
938 ldout(cct
, 20) << "AWSv4ComplMulti: declared signature="
939 << chunk_meta
.get_signature() << dendl
;
940 ldout(cct
, 20) << "AWSv4ComplMulti: calculated signature="
941 << calc_signature
<< dendl
;
945 prev_chunk_signature
= chunk_meta
.get_signature();
950 size_t AWSv4ComplMulti::recv_body(char* const buf
, const size_t buf_max
)
952 /* Buffer stores only parsed stream. Raw values reflect the stream
953 * we're getting from a client. */
956 if (chunk_meta
.is_new_chunk_in_stream(stream_pos
)) {
957 /* Verify signature of the previous chunk. We aren't doing that for new
958 * one as the procedure requires calculation of payload hash. This code
959 * won't be triggered for the last, zero-length chunk. Instead, is will
960 * be checked in the complete() method. */
961 if (stream_pos
>= ChunkMeta::META_MAX_SIZE
&& is_signature_mismatched()) {
962 throw rgw::io::Exception(ERR_SIGNATURE_NO_MATCH
, std::system_category());
965 /* We don't have metadata for this range. This means a new chunk, so we
966 * need to parse a fresh portion of the stream. Let's start. */
967 size_t to_extract
= parsing_buf
.capacity() - parsing_buf
.size();
969 const size_t orig_size
= parsing_buf
.size();
970 parsing_buf
.resize(parsing_buf
.size() + to_extract
);
971 const size_t received
= io_base_t::recv_body(parsing_buf
.data() + orig_size
,
973 parsing_buf
.resize(parsing_buf
.size() - (to_extract
- received
));
978 stream_pos
+= received
;
979 to_extract
-= received
;
980 } while (to_extract
> 0);
983 std::tie(chunk_meta
, consumed
) = \
984 ChunkMeta::create_next(cct
, std::move(chunk_meta
),
985 parsing_buf
.data(), parsing_buf
.size());
987 /* We can drop the bytes consumed during metadata parsing. The remainder
988 * can be chunk's data plus possibly beginning of next chunks' metadata. */
989 parsing_buf
.erase(std::begin(parsing_buf
),
990 std::begin(parsing_buf
) + consumed
);
993 size_t stream_pos_was
= stream_pos
- parsing_buf
.size();
995 size_t to_extract
= \
996 std::min(chunk_meta
.get_data_size(stream_pos_was
), buf_max
);
997 dout(30) << "AWSv4ComplMulti: stream_pos_was=" << stream_pos_was
<< ", to_extract=" << to_extract
<< dendl
;
999 /* It's quite probable we have a couple of real data bytes stored together
1000 * with meta-data in the parsing_buf. We need to extract them and move to
1001 * the final buffer. This is a trade-off between frontend's read overhead
1003 if (to_extract
> 0 && parsing_buf
.size() > 0) {
1004 const auto data_len
= std::min(to_extract
, parsing_buf
.size());
1005 const auto data_end_iter
= std::begin(parsing_buf
) + data_len
;
1006 dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
<< ", data_len=" << data_len
<< dendl
;
1008 std::copy(std::begin(parsing_buf
), data_end_iter
, buf
);
1009 parsing_buf
.erase(std::begin(parsing_buf
), data_end_iter
);
1011 calc_hash_sha256_update_stream(sha256_hash
, buf
, data_len
);
1013 to_extract
-= data_len
;
1014 buf_pos
+= data_len
;
1017 /* Now we can do the bulk read directly from RestfulClient without any extra
1019 while (to_extract
> 0) {
1020 const size_t received
= io_base_t::recv_body(buf
+ buf_pos
, to_extract
);
1021 dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
<< ", received=" << received
<< dendl
;
1023 if (received
== 0) {
1027 calc_hash_sha256_update_stream(sha256_hash
, buf
+ buf_pos
, received
);
1029 buf_pos
+= received
;
1030 stream_pos
+= received
;
1031 to_extract
-= received
;
1034 dout(20) << "AWSv4ComplMulti: filled=" << buf_pos
<< dendl
;
1038 void AWSv4ComplMulti::modify_request_state(const DoutPrefixProvider
* dpp
, req_state
* const s_rw
)
1040 const char* const decoded_length
= \
1041 s_rw
->info
.env
->get("HTTP_X_AMZ_DECODED_CONTENT_LENGTH");
1043 if (!decoded_length
) {
1046 s_rw
->length
= decoded_length
;
1047 s_rw
->content_length
= parse_content_length(decoded_length
);
1049 if (s_rw
->content_length
< 0) {
1050 ldpp_dout(dpp
, 10) << "negative AWSv4's content length, aborting" << dendl
;
1055 /* Install the filter over rgw::io::RestfulClient. */
1056 AWS_AUTHv4_IO(s_rw
)->add_filter(
1057 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1060 bool AWSv4ComplMulti::complete()
1062 /* Now it's time to verify the signature of the last, zero-length chunk. */
1063 if (is_signature_mismatched()) {
1064 ldout(cct
, 10) << "ERROR: signature of last chunk does not match"
1072 rgw::auth::Completer::cmplptr_t
1073 AWSv4ComplMulti::create(const req_state
* const s
,
1074 std::string_view date
,
1075 std::string_view credential_scope
,
1076 std::string_view seed_signature
,
1077 const boost::optional
<std::string
>& secret_key
)
1080 /* Some external authorizers (like Keystone) aren't fully compliant with
1081 * AWSv4. They do not provide the secret_key which is necessary to handle
1082 * the streamed upload. */
1083 throw -ERR_NOT_IMPLEMENTED
;
1086 const auto signing_key
= \
1087 rgw::auth::s3::get_v4_signing_key(s
->cct
, credential_scope
, *secret_key
, s
);
1089 return std::make_shared
<AWSv4ComplMulti
>(s
,
1091 std::move(credential_scope
),
1092 std::move(seed_signature
),
1096 size_t AWSv4ComplSingle::recv_body(char* const buf
, const size_t max
)
1098 const auto received
= io_base_t::recv_body(buf
, max
);
1099 calc_hash_sha256_update_stream(sha256_hash
, buf
, received
);
1104 void AWSv4ComplSingle::modify_request_state(const DoutPrefixProvider
* dpp
, req_state
* const s_rw
)
1106 /* Install the filter over rgw::io::RestfulClient. */
1107 AWS_AUTHv4_IO(s_rw
)->add_filter(
1108 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1111 bool AWSv4ComplSingle::complete()
1113 /* The completer is only for the cases where signed payload has been
1114 * requested. It won't be used, for instance, during the query string-based
1115 * authentication. */
1116 const auto payload_hash
= calc_hash_sha256_close_stream(&sha256_hash
);
1118 /* Validate x-amz-sha256 */
1119 if (payload_hash
.compare(expected_request_payload_hash
) == 0) {
1122 ldout(cct
, 10) << "ERROR: x-amz-content-sha256 does not match"
1124 ldout(cct
, 10) << "ERROR: grab_aws4_sha256_hash()="
1125 << payload_hash
<< dendl
;
1126 ldout(cct
, 10) << "ERROR: expected_request_payload_hash="
1127 << expected_request_payload_hash
<< dendl
;
1132 AWSv4ComplSingle::AWSv4ComplSingle(const req_state
* const s
)
1133 : io_base_t(nullptr),
1135 expected_request_payload_hash(get_v4_exp_payload_hash(s
->info
)),
1136 sha256_hash(calc_hash_sha256_open_stream()) {
1139 rgw::auth::Completer::cmplptr_t
1140 AWSv4ComplSingle::create(const req_state
* const s
,
1141 const boost::optional
<std::string
>&)
1143 return std::make_shared
<AWSv4ComplSingle
>(s
);
1146 } // namespace rgw::auth::s3