1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
11 #include "common/armor.h"
12 #include "common/utf8.h"
13 #include "rgw_rest_s3.h"
14 #include "rgw_auth_s3.h"
15 #include "rgw_common.h"
16 #include "rgw_client_io.h"
18 #include "rgw_crypt_sanitize.h"
20 #include <boost/container/small_vector.hpp>
21 #include <boost/algorithm/string.hpp>
22 #include <boost/algorithm/string/trim_all.hpp>
24 #define dout_context g_ceph_context
25 #define dout_subsys ceph_subsys_rgw
29 static const auto signed_subresources
= {
43 "response-cache-control",
44 "response-content-disposition",
45 "response-content-encoding",
46 "response-content-language",
47 "response-content-type",
61 * ?get the canonical amazon-style header for something?
65 get_canon_amz_hdr(const meta_map_t
& meta_map
)
69 for (const auto& kv
: meta_map
) {
70 dest
.append(kv
.first
);
72 dest
.append(kv
.second
);
80 * ?get the canonical representation of the object's location
83 get_canon_resource(const DoutPrefixProvider
*dpp
, const char* const request_uri
,
84 const std::map
<std::string
, std::string
>& sub_resources
)
89 dest
.append(request_uri
);
93 for (const auto& subresource
: signed_subresources
) {
94 const auto iter
= sub_resources
.find(subresource
);
95 if (iter
== std::end(sub_resources
)) {
106 dest
.append(iter
->first
);
107 if (! iter
->second
.empty()) {
109 dest
.append(iter
->second
);
113 ldpp_dout(dpp
, 10) << "get_canon_resource(): dest=" << dest
<< dendl
;
118 * get the header authentication information required to
119 * compute a request's signature
121 void rgw_create_s3_canonical_header(
122 const DoutPrefixProvider
*dpp
,
123 const char* const method
,
124 const char* const content_md5
,
125 const char* const content_type
,
126 const char* const date
,
127 const meta_map_t
& meta_map
,
128 const meta_map_t
& qs_map
,
129 const char* const request_uri
,
130 const std::map
<std::string
, std::string
>& sub_resources
,
131 std::string
& dest_str
)
141 dest
.append(content_md5
);
146 dest
.append(content_type
);
155 dest
.append(get_canon_amz_hdr(meta_map
));
156 dest
.append(get_canon_amz_hdr(qs_map
));
157 dest
.append(get_canon_resource(dpp
, request_uri
, sub_resources
));
162 static inline bool is_base64_for_content_md5(unsigned char c
) {
163 return (isalnum(c
) || isspace(c
) || (c
== '+') || (c
== '/') || (c
== '='));
166 static inline void get_v2_qs_map(const req_info
& info
,
167 meta_map_t
& qs_map
) {
168 const auto& params
= const_cast<RGWHTTPArgs
&>(info
.args
).get_params();
169 for (const auto& elt
: params
) {
170 std::string k
= boost::algorithm::to_lower_copy(elt
.first
);
171 if (k
.find("x-amz-meta-") == /* offset */ 0) {
172 rgw_add_amz_meta_header(qs_map
, k
, elt
.second
);
174 if (k
== "x-amz-security-token") {
175 qs_map
[k
] = elt
.second
;
181 * get the header authentication information required to
182 * compute a request's signature
184 bool rgw_create_s3_canonical_header(const DoutPrefixProvider
*dpp
,
185 const req_info
& info
,
186 utime_t
* const header_time
,
190 const char* const content_md5
= info
.env
->get("HTTP_CONTENT_MD5");
192 for (const char *p
= content_md5
; *p
; p
++) {
193 if (!is_base64_for_content_md5(*p
)) {
194 ldpp_dout(dpp
, 0) << "NOTICE: bad content-md5 provided (not base64),"
195 << " aborting request p=" << *p
<< " " << (int)*p
<< dendl
;
201 const char *content_type
= info
.env
->get("CONTENT_TYPE");
207 get_v2_qs_map(info
, qs_map
); // handle qs metadata
208 date
= info
.args
.get("Expires");
210 const char *str
= info
.env
->get("HTTP_X_AMZ_DATE");
211 const char *req_date
= str
;
213 req_date
= info
.env
->get("HTTP_DATE");
215 ldpp_dout(dpp
, 0) << "NOTICE: missing date for auth header" << dendl
;
224 if (!parse_rfc2616(req_date
, &t
) && !parse_iso8601(req_date
, &t
, &ns
, false)) {
225 ldpp_dout(dpp
, 0) << "NOTICE: failed to parse date <" << req_date
<< "> for auth header" << dendl
;
228 if (t
.tm_year
< 70) {
229 ldpp_dout(dpp
, 0) << "NOTICE: bad date (predates epoch): " << req_date
<< dendl
;
232 *header_time
= utime_t(internal_timegm(&t
), 0);
233 *header_time
-= t
.tm_gmtoff
;
237 const auto& meta_map
= info
.x_meta_map
;
238 const auto& sub_resources
= info
.args
.get_sub_resources();
240 std::string request_uri
;
241 if (info
.effective_uri
.empty()) {
242 request_uri
= info
.request_uri
;
244 request_uri
= info
.effective_uri
;
247 rgw_create_s3_canonical_header(dpp
, info
.method
, content_md5
, content_type
,
248 date
.c_str(), meta_map
, qs_map
,
249 request_uri
.c_str(), sub_resources
, dest
);
254 namespace rgw::auth::s3
{
256 bool is_time_skew_ok(time_t t
)
258 auto req_tp
= ceph::coarse_real_clock::from_time_t(t
);
259 auto cur_tp
= ceph::coarse_real_clock::now();
261 if (std::chrono::abs(cur_tp
- req_tp
) > RGW_AUTH_GRACE
) {
262 dout(10) << "NOTICE: request time skew too big." << dendl
;
263 using ceph::operator<<;
264 dout(10) << "req_tp=" << req_tp
<< ", cur_tp=" << cur_tp
<< dendl
;
271 static inline int parse_v4_query_string(const req_info
& info
, /* in */
272 std::string_view
& credential
, /* out */
273 std::string_view
& signedheaders
, /* out */
274 std::string_view
& signature
, /* out */
275 std::string_view
& date
, /* out */
276 std::string_view
& sessiontoken
) /* out */
278 /* auth ships with req params ... */
280 /* look for required params */
281 credential
= info
.args
.get("x-amz-credential");
282 if (credential
.size() == 0) {
286 date
= info
.args
.get("x-amz-date");
288 if (!parse_iso8601(sview2cstr(date
).data(), &date_t
, nullptr, false)) {
292 std::string_view expires
= info
.args
.get("x-amz-expires");
293 if (expires
.empty()) {
296 /* X-Amz-Expires provides the time period, in seconds, for which
297 the generated presigned URL is valid. The minimum value
298 you can set is 1, and the maximum is 604800 (seven days) */
299 time_t exp
= atoll(expires
.data());
300 if ((exp
< 1) || (exp
> 7*24*60*60)) {
301 dout(10) << "NOTICE: exp out of range, exp = " << exp
<< dendl
;
304 /* handle expiration in epoch time */
305 uint64_t req_sec
= (uint64_t)internal_timegm(&date_t
);
306 uint64_t now
= ceph_clock_now();
307 if (now
>= req_sec
+ exp
) {
308 dout(10) << "NOTICE: now = " << now
<< ", req_sec = " << req_sec
<< ", exp = " << exp
<< dendl
;
312 signedheaders
= info
.args
.get("x-amz-signedheaders");
313 if (signedheaders
.size() == 0) {
317 signature
= info
.args
.get("x-amz-signature");
318 if (signature
.size() == 0) {
322 if (info
.args
.exists("x-amz-security-token")) {
323 sessiontoken
= info
.args
.get("x-amz-security-token");
324 if (sessiontoken
.size() == 0) {
332 static bool get_next_token(const std::string_view
& s
,
334 const char* const delims
,
335 std::string_view
& token
)
337 const size_t start
= s
.find_first_not_of(delims
, pos
);
338 if (start
== std::string_view::npos
) {
343 size_t end
= s
.find_first_of(delims
, start
);
344 if (end
!= std::string_view::npos
)
347 pos
= end
= s
.size();
350 token
= s
.substr(start
, end
- start
);
354 template<std::size_t ExpectedStrNum
>
355 boost::container::small_vector
<std::string_view
, ExpectedStrNum
>
356 get_str_vec(const std::string_view
& str
, const char* const delims
)
358 boost::container::small_vector
<std::string_view
, ExpectedStrNum
> str_vec
;
361 std::string_view token
;
362 while (pos
< str
.size()) {
363 if (get_next_token(str
, pos
, delims
, token
)) {
364 if (token
.size() > 0) {
365 str_vec
.push_back(token
);
373 template<std::size_t ExpectedStrNum
>
374 boost::container::small_vector
<std::string_view
, ExpectedStrNum
>
375 get_str_vec(const std::string_view
& str
)
377 const char delims
[] = ";,= \t";
378 return get_str_vec
<ExpectedStrNum
>(str
, delims
);
381 static inline int parse_v4_auth_header(const req_info
& info
, /* in */
382 std::string_view
& credential
, /* out */
383 std::string_view
& signedheaders
, /* out */
384 std::string_view
& signature
, /* out */
385 std::string_view
& date
, /* out */
386 std::string_view
& sessiontoken
, /* out */
387 const DoutPrefixProvider
*dpp
)
389 std::string_view
input(info
.env
->get("HTTP_AUTHORIZATION", ""));
391 input
= input
.substr(::strlen(AWS4_HMAC_SHA256_STR
) + 1);
392 } catch (std::out_of_range
&) {
393 /* We should never ever run into this situation as the presence of
394 * AWS4_HMAC_SHA256_STR had been verified earlier. */
395 ldpp_dout(dpp
, 10) << "credentials string is too short" << dendl
;
399 std::map
<std::string_view
, std::string_view
> kv
;
400 for (const auto& s
: get_str_vec
<4>(input
, ",")) {
401 const auto parsed_pair
= parse_key_value(s
);
403 kv
[parsed_pair
->first
] = parsed_pair
->second
;
405 ldpp_dout(dpp
, 10) << "NOTICE: failed to parse auth header (s=" << s
<< ")"
411 static const std::array
<std::string_view
, 3> required_keys
= {
417 /* Ensure that the presigned required keys are really there. */
418 for (const auto& k
: required_keys
) {
419 if (kv
.find(k
) == std::end(kv
)) {
420 ldpp_dout(dpp
, 10) << "NOTICE: auth header missing key: " << k
<< dendl
;
425 credential
= kv
["Credential"];
426 signedheaders
= kv
["SignedHeaders"];
427 signature
= kv
["Signature"];
430 ldpp_dout(dpp
, 10) << "v4 signature format = " << signature
<< dendl
;
432 /* ------------------------- handle x-amz-date header */
436 const char *d
= info
.env
->get("HTTP_X_AMZ_DATE");
439 if (unlikely(d
== NULL
)) {
440 d
= info
.env
->get("HTTP_DATE");
442 if (!d
|| !parse_iso8601(d
, &t
, NULL
, false)) {
443 ldpp_dout(dpp
, 10) << "error reading date via http_x_amz_date and http_date" << dendl
;
448 if (!is_time_skew_ok(internal_timegm(&t
))) {
449 return -ERR_REQUEST_TIME_SKEWED
;
452 auto token
= info
.env
->get_optional("HTTP_X_AMZ_SECURITY_TOKEN");
454 sessiontoken
= *token
;
460 bool is_non_s3_op(RGWOpType op_type
)
462 if (op_type
== RGW_STS_GET_SESSION_TOKEN
||
463 op_type
== RGW_STS_ASSUME_ROLE
||
464 op_type
== RGW_STS_ASSUME_ROLE_WEB_IDENTITY
||
465 op_type
== RGW_OP_CREATE_ROLE
||
466 op_type
== RGW_OP_DELETE_ROLE
||
467 op_type
== RGW_OP_GET_ROLE
||
468 op_type
== RGW_OP_MODIFY_ROLE_TRUST_POLICY
||
469 op_type
== RGW_OP_LIST_ROLES
||
470 op_type
== RGW_OP_PUT_ROLE_POLICY
||
471 op_type
== RGW_OP_GET_ROLE_POLICY
||
472 op_type
== RGW_OP_LIST_ROLE_POLICIES
||
473 op_type
== RGW_OP_DELETE_ROLE_POLICY
||
474 op_type
== RGW_OP_PUT_USER_POLICY
||
475 op_type
== RGW_OP_GET_USER_POLICY
||
476 op_type
== RGW_OP_LIST_USER_POLICIES
||
477 op_type
== RGW_OP_DELETE_USER_POLICY
||
478 op_type
== RGW_OP_CREATE_OIDC_PROVIDER
||
479 op_type
== RGW_OP_DELETE_OIDC_PROVIDER
||
480 op_type
== RGW_OP_GET_OIDC_PROVIDER
||
481 op_type
== RGW_OP_LIST_OIDC_PROVIDERS
||
482 op_type
== RGW_OP_PUBSUB_TOPIC_CREATE
||
483 op_type
== RGW_OP_PUBSUB_TOPICS_LIST
||
484 op_type
== RGW_OP_PUBSUB_TOPIC_GET
||
485 op_type
== RGW_OP_PUBSUB_TOPIC_DELETE
||
486 op_type
== RGW_OP_TAG_ROLE
||
487 op_type
== RGW_OP_LIST_ROLE_TAGS
||
488 op_type
== RGW_OP_UNTAG_ROLE
||
489 op_type
== RGW_OP_UPDATE_ROLE
) {
495 int parse_v4_credentials(const req_info
& info
, /* in */
496 std::string_view
& access_key_id
, /* out */
497 std::string_view
& credential_scope
, /* out */
498 std::string_view
& signedheaders
, /* out */
499 std::string_view
& signature
, /* out */
500 std::string_view
& date
, /* out */
501 std::string_view
& session_token
, /* out */
502 const bool using_qs
, /* in */
503 const DoutPrefixProvider
*dpp
)
505 std::string_view credential
;
508 ret
= parse_v4_query_string(info
, credential
, signedheaders
,
509 signature
, date
, session_token
);
511 ret
= parse_v4_auth_header(info
, credential
, signedheaders
,
512 signature
, date
, session_token
, dpp
);
519 /* access_key/YYYYMMDD/region/service/aws4_request */
520 ldpp_dout(dpp
, 10) << "v4 credential format = " << credential
<< dendl
;
522 if (std::count(credential
.begin(), credential
.end(), '/') != 4) {
526 /* credential must end with 'aws4_request' */
527 if (credential
.find("aws4_request") == std::string::npos
) {
531 /* grab access key id */
532 const size_t pos
= credential
.find("/");
533 access_key_id
= credential
.substr(0, pos
);
534 ldpp_dout(dpp
, 10) << "access key id = " << access_key_id
<< dendl
;
536 /* grab credential scope */
537 credential_scope
= credential
.substr(pos
+ 1);
538 ldpp_dout(dpp
, 10) << "credential scope = " << credential_scope
<< dendl
;
543 string
gen_v4_scope(const ceph::real_time
& timestamp
,
544 const string
& region
,
545 const string
& service
)
548 auto sec
= real_clock::to_time_t(timestamp
);
553 auto year
= 1900 + bt
.tm_year
;
554 auto mon
= bt
.tm_mon
+ 1;
555 auto day
= bt
.tm_mday
;
557 return fmt::format(FMT_STRING("{:d}{:02d}{:02d}/{:s}/{:s}/aws4_request"),
558 year
, mon
, day
, region
, service
);
561 std::string
get_v4_canonical_qs(const req_info
& info
, const bool using_qs
)
563 const std::string
*params
= &info
.request_params
;
564 std::string copy_params
;
565 if (params
->empty()) {
566 /* Optimize the typical flow. */
567 return std::string();
569 if (params
->find_first_of('+') != std::string::npos
) {
570 copy_params
= *params
;
571 boost::replace_all(copy_params
, "+", "%20");
572 params
= ©_params
;
575 /* Handle case when query string exists. Step 3 described in: http://docs.
576 * aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html */
577 std::map
<std::string
, std::string
> canonical_qs_map
;
578 for (const auto& s
: get_str_vec
<5>(*params
, "&")) {
579 std::string_view key
, val
;
580 const auto parsed_pair
= parse_key_value(s
);
582 std::tie(key
, val
) = *parsed_pair
;
584 /* Handling a parameter without any value (even the empty one). That's
585 * it, we've encountered something like "this_param&other_param=val"
586 * which is used by S3 for subresources. */
590 if (using_qs
&& boost::iequals(key
, "X-Amz-Signature")) {
591 /* Preserving the original behaviour of get_v4_canonical_qs() here. */
595 // while awsv4 specs ask for all slashes to be encoded, s3 itself is relaxed
596 // in its implementation allowing non-url-encoded slashes to be present in
597 // presigned urls for instance
598 canonical_qs_map
[aws4_uri_recode(key
, true)] = aws4_uri_recode(val
, true);
601 /* Thanks to the early exist we have the guarantee that canonical_qs_map has
602 * at least one element. */
603 auto iter
= std::begin(canonical_qs_map
);
604 std::string canonical_qs
;
605 canonical_qs
.append(iter
->first
)
606 .append("=", ::strlen("="))
607 .append(iter
->second
);
609 for (iter
++; iter
!= std::end(canonical_qs_map
); iter
++) {
610 canonical_qs
.append("&", ::strlen("&"))
612 .append("=", ::strlen("="))
613 .append(iter
->second
);
619 static void add_v4_canonical_params_from_map(const map
<string
, string
>& m
,
620 std::map
<string
, string
> *result
,
623 for (auto& entry
: m
) {
624 const auto& key
= entry
.first
;
625 if (key
.empty() || (is_non_s3_op
&& key
== "PayloadHash")) {
629 (*result
)[aws4_uri_recode(key
, true)] = aws4_uri_recode(entry
.second
, true);
633 std::string
gen_v4_canonical_qs(const req_info
& info
, bool is_non_s3_op
)
635 std::map
<std::string
, std::string
> canonical_qs_map
;
637 add_v4_canonical_params_from_map(info
.args
.get_params(), &canonical_qs_map
, is_non_s3_op
);
638 add_v4_canonical_params_from_map(info
.args
.get_sys_params(), &canonical_qs_map
, false);
640 if (canonical_qs_map
.empty()) {
644 /* Thanks to the early exit we have the guarantee that canonical_qs_map has
645 * at least one element. */
646 auto iter
= std::begin(canonical_qs_map
);
647 std::string canonical_qs
;
648 canonical_qs
.append(iter
->first
)
649 .append("=", ::strlen("="))
650 .append(iter
->second
);
652 for (iter
++; iter
!= std::end(canonical_qs_map
); iter
++) {
653 canonical_qs
.append("&", ::strlen("&"))
655 .append("=", ::strlen("="))
656 .append(iter
->second
);
662 std::string
get_v4_canonical_method(const req_state
* s
)
664 /* If this is a OPTIONS request we need to compute the v4 signature for the
665 * intended HTTP method and not the OPTIONS request itself. */
666 if (s
->op_type
== RGW_OP_OPTIONS_CORS
) {
667 const char *cors_method
= s
->info
.env
->get("HTTP_ACCESS_CONTROL_REQUEST_METHOD");
670 /* Validate request method passed in access-control-request-method is valid. */
671 auto cors_flags
= get_cors_method_flags(cors_method
);
673 ldpp_dout(s
, 1) << "invalid access-control-request-method header = "
674 << cors_method
<< dendl
;
678 ldpp_dout(s
, 10) << "canonical req method = " << cors_method
679 << ", due to access-control-request-method header" << dendl
;
682 ldpp_dout(s
, 1) << "invalid http options req missing "
683 << "access-control-request-method header" << dendl
;
688 return s
->info
.method
;
691 boost::optional
<std::string
>
692 get_v4_canonical_headers(const req_info
& info
,
693 const std::string_view
& signedheaders
,
695 const bool force_boto2_compat
)
697 std::map
<std::string_view
, std::string
> canonical_hdrs_map
;
698 for (const auto& token
: get_str_vec
<5>(signedheaders
, ";")) {
699 /* TODO(rzarzynski): we'd like to switch to sstring here but it should
700 * get push_back() and reserve() first. */
701 std::string token_env
= "HTTP_";
702 token_env
.reserve(token
.length() + std::strlen("HTTP_") + 1);
704 std::transform(std::begin(token
), std::end(token
),
705 std::back_inserter(token_env
), [](const int c
) {
706 return c
== '-' ? '_' : c
== '_' ? '-' : std::toupper(c
);
709 if (token_env
== "HTTP_CONTENT_LENGTH") {
710 token_env
= "CONTENT_LENGTH";
711 } else if (token_env
== "HTTP_CONTENT_TYPE") {
712 token_env
= "CONTENT_TYPE";
714 const char* const t
= info
.env
->get(token_env
.c_str());
716 dout(10) << "warning env var not available " << token_env
.c_str() << dendl
;
720 std::string
token_value(t
);
721 if (token_env
== "HTTP_CONTENT_MD5" &&
722 !std::all_of(std::begin(token_value
), std::end(token_value
),
723 is_base64_for_content_md5
)) {
724 dout(0) << "NOTICE: bad content-md5 provided (not base64)"
725 << ", aborting request" << dendl
;
729 if (force_boto2_compat
&& using_qs
&& token
== "host") {
730 std::string_view port
= info
.env
->get("SERVER_PORT", "");
731 std::string_view secure_port
= info
.env
->get("SERVER_PORT_SECURE", "");
733 if (!secure_port
.empty()) {
734 if (secure_port
!= "443")
735 token_value
.append(":", std::strlen(":"))
736 .append(secure_port
.data(), secure_port
.length());
737 } else if (!port
.empty()) {
739 token_value
.append(":", std::strlen(":"))
740 .append(port
.data(), port
.length());
744 canonical_hdrs_map
[token
] = rgw_trim_whitespace(token_value
);
747 std::string canonical_hdrs
;
748 for (const auto& header
: canonical_hdrs_map
) {
749 const std::string_view
& name
= header
.first
;
750 std::string value
= header
.second
;
751 boost::trim_all
<std::string
>(value
);
753 canonical_hdrs
.append(name
.data(), name
.length())
754 .append(":", std::strlen(":"))
756 .append("\n", std::strlen("\n"));
758 return canonical_hdrs
;
761 static void handle_header(const string
& header
, const string
& val
,
762 std::map
<std::string
, std::string
> *canonical_hdrs_map
)
764 /* TODO(rzarzynski): we'd like to switch to sstring here but it should
765 * get push_back() and reserve() first. */
768 token
.reserve(header
.length());
770 if (header
== "HTTP_CONTENT_LENGTH") {
771 token
= "content-length";
772 } else if (header
== "HTTP_CONTENT_TYPE") {
773 token
= "content-type";
775 auto start
= std::begin(header
);
776 if (boost::algorithm::starts_with(header
, "HTTP_")) {
777 start
+= 5; /* len("HTTP_") */
780 std::transform(start
, std::end(header
),
781 std::back_inserter(token
), [](const int c
) {
782 return c
== '_' ? '-' : std::tolower(c
);
786 (*canonical_hdrs_map
)[token
] = rgw_trim_whitespace(val
);
789 std::string
gen_v4_canonical_headers(const req_info
& info
,
790 const map
<string
, string
>& extra_headers
,
793 std::map
<std::string
, std::string
> canonical_hdrs_map
;
794 for (auto& entry
: info
.env
->get_map()) {
795 handle_header(entry
.first
, entry
.second
, &canonical_hdrs_map
);
797 for (auto& entry
: extra_headers
) {
798 handle_header(entry
.first
, entry
.second
, &canonical_hdrs_map
);
801 std::string canonical_hdrs
;
802 signed_hdrs
->clear();
803 for (const auto& header
: canonical_hdrs_map
) {
804 const auto& name
= header
.first
;
805 std::string value
= header
.second
;
806 boost::trim_all
<std::string
>(value
);
808 if (!signed_hdrs
->empty()) {
809 signed_hdrs
->append(";");
811 signed_hdrs
->append(name
);
813 canonical_hdrs
.append(name
.data(), name
.length())
814 .append(":", std::strlen(":"))
816 .append("\n", std::strlen("\n"));
819 return canonical_hdrs
;
823 * create canonical request for signature version 4
825 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
828 get_v4_canon_req_hash(CephContext
* cct
,
829 const std::string_view
& http_verb
,
830 const std::string
& canonical_uri
,
831 const std::string
& canonical_qs
,
832 const std::string
& canonical_hdrs
,
833 const std::string_view
& signed_hdrs
,
834 const std::string_view
& request_payload_hash
,
835 const DoutPrefixProvider
*dpp
)
837 ldpp_dout(dpp
, 10) << "payload request hash = " << request_payload_hash
<< dendl
;
839 const auto canonical_req
= string_join_reserve("\n",
845 request_payload_hash
);
847 const auto canonical_req_hash
= calc_hash_sha256(canonical_req
);
849 using sanitize
= rgw::crypt_sanitize::log_content
;
850 ldpp_dout(dpp
, 10) << "canonical request = " << sanitize
{canonical_req
} << dendl
;
851 ldpp_dout(dpp
, 10) << "canonical request hash = "
852 << canonical_req_hash
<< dendl
;
854 return canonical_req_hash
;
858 * create string to sign for signature version 4
860 * http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
862 AWSEngine::VersionAbstractor::string_to_sign_t
863 get_v4_string_to_sign(CephContext
* const cct
,
864 const std::string_view
& algorithm
,
865 const std::string_view
& request_date
,
866 const std::string_view
& credential_scope
,
867 const sha256_digest_t
& canonreq_hash
,
868 const DoutPrefixProvider
*dpp
)
870 const auto hexed_cr_hash
= canonreq_hash
.to_str();
871 const std::string_view
hexed_cr_hash_str(hexed_cr_hash
);
873 const auto string_to_sign
= string_join_reserve("\n",
879 ldpp_dout(dpp
, 10) << "string to sign = "
880 << rgw::crypt_sanitize::log_content
{string_to_sign
}
883 return string_to_sign
;
887 static inline std::tuple
<std::string_view
, /* date */
888 std::string_view
, /* region */
889 std::string_view
> /* service */
890 parse_cred_scope(std::string_view credential_scope
)
893 size_t pos
= credential_scope
.find("/");
894 const auto date_cs
= credential_scope
.substr(0, pos
);
895 credential_scope
= credential_scope
.substr(pos
+ 1);
898 pos
= credential_scope
.find("/");
899 const auto region_cs
= credential_scope
.substr(0, pos
);
900 credential_scope
= credential_scope
.substr(pos
+ 1);
903 pos
= credential_scope
.find("/");
904 const auto service_cs
= credential_scope
.substr(0, pos
);
906 return std::make_tuple(date_cs
, region_cs
, service_cs
);
909 static inline std::vector
<unsigned char>
910 transform_secret_key(const std::string_view
& secret_access_key
)
912 /* TODO(rzarzynski): switch to constexpr when C++14 becomes available. */
913 static const std::initializer_list
<unsigned char> AWS4
{ 'A', 'W', 'S', '4' };
915 /* boost::container::small_vector might be used here if someone wants to
916 * optimize out even more dynamic allocations. */
917 std::vector
<unsigned char> secret_key_utf8
;
918 secret_key_utf8
.reserve(AWS4
.size() + secret_access_key
.size());
919 secret_key_utf8
.assign(AWS4
);
921 for (const auto c
: secret_access_key
) {
922 std::array
<unsigned char, MAX_UTF8_SZ
> buf
;
923 const size_t n
= encode_utf8(c
, buf
.data());
924 secret_key_utf8
.insert(std::end(secret_key_utf8
),
925 std::begin(buf
), std::begin(buf
) + n
);
928 return secret_key_utf8
;
932 * calculate the SigningKey of AWS auth version 4
934 static sha256_digest_t
935 get_v4_signing_key(CephContext
* const cct
,
936 const std::string_view
& credential_scope
,
937 const std::string_view
& secret_access_key
,
938 const DoutPrefixProvider
*dpp
)
940 std::string_view date
, region
, service
;
941 std::tie(date
, region
, service
) = parse_cred_scope(credential_scope
);
943 const auto utfed_sec_key
= transform_secret_key(secret_access_key
);
944 const auto date_k
= calc_hmac_sha256(utfed_sec_key
, date
);
945 const auto region_k
= calc_hmac_sha256(date_k
, region
);
946 const auto service_k
= calc_hmac_sha256(region_k
, service
);
949 const auto signing_key
= calc_hmac_sha256(service_k
,
950 std::string_view("aws4_request"));
952 ldpp_dout(dpp
, 10) << "date_k = " << date_k
<< dendl
;
953 ldpp_dout(dpp
, 10) << "region_k = " << region_k
<< dendl
;
954 ldpp_dout(dpp
, 10) << "service_k = " << service_k
<< dendl
;
955 ldpp_dout(dpp
, 10) << "signing_k = " << signing_key
<< dendl
;
961 * calculate the AWS signature version 4
963 * http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
965 * srv_signature_t is an alias over Ceph's basic_sstring. We're using
966 * it to keep everything within the stack boundaries instead of doing
967 * dynamic allocations.
969 AWSEngine::VersionAbstractor::server_signature_t
970 get_v4_signature(const std::string_view
& credential_scope
,
971 CephContext
* const cct
,
972 const std::string_view
& secret_key
,
973 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
,
974 const DoutPrefixProvider
*dpp
)
976 auto signing_key
= get_v4_signing_key(cct
, credential_scope
, secret_key
, dpp
);
978 /* The server-side generated digest for comparison. */
979 const auto digest
= calc_hmac_sha256(signing_key
, string_to_sign
);
981 /* TODO(rzarzynski): I would love to see our sstring having reserve() and
982 * the non-const data() variant like C++17's std::string. */
983 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
984 srv_signature_t
signature(srv_signature_t::initialized_later(),
986 buf_to_hex(digest
.v
, digest
.SIZE
, signature
.begin());
988 ldpp_dout(dpp
, 10) << "generated signature = " << signature
<< dendl
;
993 AWSEngine::VersionAbstractor::server_signature_t
994 get_v2_signature(CephContext
* const cct
,
995 const std::string
& secret_key
,
996 const AWSEngine::VersionAbstractor::string_to_sign_t
& string_to_sign
)
998 if (secret_key
.empty()) {
1002 const auto digest
= calc_hmac_sha1(secret_key
, string_to_sign
);
1004 /* 64 is really enough */;
1006 const int ret
= ceph_armor(std::begin(buf
),
1007 std::begin(buf
) + 64,
1008 reinterpret_cast<const char *>(digest
.v
),
1009 reinterpret_cast<const char *>(digest
.v
+ digest
.SIZE
));
1011 ldout(cct
, 10) << "ceph_armor failed" << dendl
;
1015 using srv_signature_t
= AWSEngine::VersionAbstractor::server_signature_t
;
1016 return srv_signature_t(buf
, ret
);
1020 bool AWSv4ComplMulti::ChunkMeta::is_new_chunk_in_stream(size_t stream_pos
) const
1022 return stream_pos
>= (data_offset_in_stream
+ data_length
);
1025 size_t AWSv4ComplMulti::ChunkMeta::get_data_size(size_t stream_pos
) const
1027 if (stream_pos
> (data_offset_in_stream
+ data_length
)) {
1028 /* Data in parsing_buf. */
1031 return data_offset_in_stream
+ data_length
- stream_pos
;
1036 /* AWSv4 completers begin. */
1037 std::pair
<AWSv4ComplMulti::ChunkMeta
, size_t /* consumed */>
1038 AWSv4ComplMulti::ChunkMeta::create_next(CephContext
* const cct
,
1040 const char* const metabuf
,
1041 const size_t metabuf_len
)
1043 std::string_view
metastr(metabuf
, metabuf_len
);
1045 const size_t semicolon_pos
= metastr
.find(";");
1046 if (semicolon_pos
== std::string_view::npos
) {
1047 ldout(cct
, 20) << "AWSv4ComplMulti cannot find the ';' separator"
1049 throw rgw::io::Exception(EINVAL
, std::system_category());
1052 char* data_field_end
;
1053 /* strtoull ignores the "\r\n" sequence after each non-first chunk. */
1054 const size_t data_length
= std::strtoull(metabuf
, &data_field_end
, 16);
1055 if (data_length
== 0 && data_field_end
== metabuf
) {
1056 ldout(cct
, 20) << "AWSv4ComplMulti: cannot parse the data size"
1058 throw rgw::io::Exception(EINVAL
, std::system_category());
1061 /* Parse the chunk_signature=... part. */
1062 const auto signature_part
= metastr
.substr(semicolon_pos
+ 1);
1063 const size_t eq_sign_pos
= signature_part
.find("=");
1064 if (eq_sign_pos
== std::string_view::npos
) {
1065 ldout(cct
, 20) << "AWSv4ComplMulti: cannot find the '=' separator"
1067 throw rgw::io::Exception(EINVAL
, std::system_category());
1070 /* OK, we have at least the beginning of a signature. */
1071 const size_t data_sep_pos
= signature_part
.find("\r\n");
1072 if (data_sep_pos
== std::string_view::npos
) {
1073 ldout(cct
, 20) << "AWSv4ComplMulti: no new line at signature end"
1075 throw rgw::io::Exception(EINVAL
, std::system_category());
1078 const auto signature
= \
1079 signature_part
.substr(eq_sign_pos
+ 1, data_sep_pos
- 1 - eq_sign_pos
);
1080 if (signature
.length() != SIG_SIZE
) {
1081 ldout(cct
, 20) << "AWSv4ComplMulti: signature.length() != 64"
1083 throw rgw::io::Exception(EINVAL
, std::system_category());
1086 const size_t data_starts_in_stream
= \
1087 + semicolon_pos
+ strlen(";") + data_sep_pos
+ strlen("\r\n")
1088 + old
.data_offset_in_stream
+ old
.data_length
;
1090 ldout(cct
, 20) << "parsed new chunk; signature=" << signature
1091 << ", data_length=" << data_length
1092 << ", data_starts_in_stream=" << data_starts_in_stream
1095 return std::make_pair(ChunkMeta(data_starts_in_stream
,
1098 semicolon_pos
+ 83);
1102 AWSv4ComplMulti::calc_chunk_signature(const std::string
& payload_hash
) const
1104 const auto string_to_sign
= string_join_reserve("\n",
1105 AWS4_HMAC_SHA256_PAYLOAD_STR
,
1108 prev_chunk_signature
,
1109 AWS4_EMPTY_PAYLOAD_HASH
,
1112 ldout(cct
, 20) << "AWSv4ComplMulti: string_to_sign=\n" << string_to_sign
1115 /* new chunk signature */
1116 const auto sig
= calc_hmac_sha256(signing_key
, string_to_sign
);
1117 /* FIXME(rzarzynski): std::string here is really unnecessary. */
1118 return sig
.to_str();
1122 bool AWSv4ComplMulti::is_signature_mismatched()
1124 /* The validity of previous chunk can be verified only after getting meta-
1125 * data of the next one. */
1126 const auto payload_hash
= calc_hash_sha256_restart_stream(&sha256_hash
);
1127 const auto calc_signature
= calc_chunk_signature(payload_hash
);
1129 if (chunk_meta
.get_signature() != calc_signature
) {
1130 ldout(cct
, 20) << "AWSv4ComplMulti: ERROR: chunk signature mismatch"
1132 ldout(cct
, 20) << "AWSv4ComplMulti: declared signature="
1133 << chunk_meta
.get_signature() << dendl
;
1134 ldout(cct
, 20) << "AWSv4ComplMulti: calculated signature="
1135 << calc_signature
<< dendl
;
1139 prev_chunk_signature
= chunk_meta
.get_signature();
1144 size_t AWSv4ComplMulti::recv_chunk(char* const buf
, const size_t buf_max
, bool& eof
)
1146 /* Buffer stores only parsed stream. Raw values reflect the stream
1147 * we're getting from a client. */
1150 if (chunk_meta
.is_new_chunk_in_stream(stream_pos
)) {
1151 /* Verify signature of the previous chunk. We aren't doing that for new
1152 * one as the procedure requires calculation of payload hash. This code
1153 * won't be triggered for the last, zero-length chunk. Instead, is will
1154 * be checked in the complete() method. */
1155 if (stream_pos
>= ChunkMeta::META_MAX_SIZE
&& is_signature_mismatched()) {
1156 throw rgw::io::Exception(ERR_SIGNATURE_NO_MATCH
, std::system_category());
1159 /* We don't have metadata for this range. This means a new chunk, so we
1160 * need to parse a fresh portion of the stream. Let's start. */
1161 size_t to_extract
= parsing_buf
.capacity() - parsing_buf
.size();
1163 const size_t orig_size
= parsing_buf
.size();
1164 parsing_buf
.resize(parsing_buf
.size() + to_extract
);
1165 const size_t received
= io_base_t::recv_body(parsing_buf
.data() + orig_size
,
1167 parsing_buf
.resize(parsing_buf
.size() - (to_extract
- received
));
1168 if (received
== 0) {
1173 stream_pos
+= received
;
1174 to_extract
-= received
;
1175 } while (to_extract
> 0);
1178 std::tie(chunk_meta
, consumed
) = \
1179 ChunkMeta::create_next(cct
, std::move(chunk_meta
),
1180 parsing_buf
.data(), parsing_buf
.size());
1182 /* We can drop the bytes consumed during metadata parsing. The remainder
1183 * can be chunk's data plus possibly beginning of next chunks' metadata. */
1184 parsing_buf
.erase(std::begin(parsing_buf
),
1185 std::begin(parsing_buf
) + consumed
);
1188 size_t stream_pos_was
= stream_pos
- parsing_buf
.size();
1190 size_t to_extract
= \
1191 std::min(chunk_meta
.get_data_size(stream_pos_was
), buf_max
);
1192 dout(30) << "AWSv4ComplMulti: stream_pos_was=" << stream_pos_was
<< ", to_extract=" << to_extract
<< dendl
;
1194 /* It's quite probable we have a couple of real data bytes stored together
1195 * with meta-data in the parsing_buf. We need to extract them and move to
1196 * the final buffer. This is a trade-off between frontend's read overhead
1198 if (to_extract
> 0 && parsing_buf
.size() > 0) {
1199 const auto data_len
= std::min(to_extract
, parsing_buf
.size());
1200 const auto data_end_iter
= std::begin(parsing_buf
) + data_len
;
1201 dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
<< ", data_len=" << data_len
<< dendl
;
1203 std::copy(std::begin(parsing_buf
), data_end_iter
, buf
);
1204 parsing_buf
.erase(std::begin(parsing_buf
), data_end_iter
);
1206 calc_hash_sha256_update_stream(sha256_hash
, buf
, data_len
);
1208 to_extract
-= data_len
;
1209 buf_pos
+= data_len
;
1212 /* Now we can do the bulk read directly from RestfulClient without any extra
1214 while (to_extract
> 0) {
1215 const size_t received
= io_base_t::recv_body(buf
+ buf_pos
, to_extract
);
1216 dout(30) << "AWSv4ComplMulti: to_extract=" << to_extract
<< ", received=" << received
<< dendl
;
1218 if (received
== 0) {
1223 calc_hash_sha256_update_stream(sha256_hash
, buf
+ buf_pos
, received
);
1225 buf_pos
+= received
;
1226 stream_pos
+= received
;
1227 to_extract
-= received
;
1230 dout(20) << "AWSv4ComplMulti: filled=" << buf_pos
<< dendl
;
1234 size_t AWSv4ComplMulti::recv_body(char* const buf
, const size_t buf_max
)
1239 while (total
< buf_max
&& !eof
) {
1240 const size_t received
= recv_chunk(buf
+ total
, buf_max
- total
, eof
);
1243 dout(20) << "AWSv4ComplMulti: received=" << total
<< dendl
;
1247 void AWSv4ComplMulti::modify_request_state(const DoutPrefixProvider
* dpp
, req_state
* const s_rw
)
1249 const char* const decoded_length
= \
1250 s_rw
->info
.env
->get("HTTP_X_AMZ_DECODED_CONTENT_LENGTH");
1252 if (!decoded_length
) {
1255 s_rw
->length
= decoded_length
;
1256 s_rw
->content_length
= parse_content_length(decoded_length
);
1258 if (s_rw
->content_length
< 0) {
1259 ldpp_dout(dpp
, 10) << "negative AWSv4's content length, aborting" << dendl
;
1264 /* Install the filter over rgw::io::RestfulClient. */
1265 AWS_AUTHv4_IO(s_rw
)->add_filter(
1266 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1269 bool AWSv4ComplMulti::complete()
1271 /* Now it's time to verify the signature of the last, zero-length chunk. */
1272 if (is_signature_mismatched()) {
1273 ldout(cct
, 10) << "ERROR: signature of last chunk does not match"
1281 rgw::auth::Completer::cmplptr_t
1282 AWSv4ComplMulti::create(const req_state
* const s
,
1283 std::string_view date
,
1284 std::string_view credential_scope
,
1285 std::string_view seed_signature
,
1286 const boost::optional
<std::string
>& secret_key
)
1289 /* Some external authorizers (like Keystone) aren't fully compliant with
1290 * AWSv4. They do not provide the secret_key which is necessary to handle
1291 * the streamed upload. */
1292 throw -ERR_NOT_IMPLEMENTED
;
1295 const auto signing_key
= \
1296 rgw::auth::s3::get_v4_signing_key(s
->cct
, credential_scope
, *secret_key
, s
);
1298 return std::make_shared
<AWSv4ComplMulti
>(s
,
1300 std::move(credential_scope
),
1301 std::move(seed_signature
),
1305 size_t AWSv4ComplSingle::recv_body(char* const buf
, const size_t max
)
1307 const auto received
= io_base_t::recv_body(buf
, max
);
1308 calc_hash_sha256_update_stream(sha256_hash
, buf
, received
);
1313 void AWSv4ComplSingle::modify_request_state(const DoutPrefixProvider
* dpp
, req_state
* const s_rw
)
1315 /* Install the filter over rgw::io::RestfulClient. */
1316 AWS_AUTHv4_IO(s_rw
)->add_filter(
1317 std::static_pointer_cast
<io_base_t
>(shared_from_this()));
1320 bool AWSv4ComplSingle::complete()
1322 /* The completer is only for the cases where signed payload has been
1323 * requested. It won't be used, for instance, during the query string-based
1324 * authentication. */
1325 const auto payload_hash
= calc_hash_sha256_close_stream(&sha256_hash
);
1327 /* Validate x-amz-sha256 */
1328 if (payload_hash
.compare(expected_request_payload_hash
) == 0) {
1331 ldout(cct
, 10) << "ERROR: x-amz-content-sha256 does not match"
1333 ldout(cct
, 10) << "ERROR: grab_aws4_sha256_hash()="
1334 << payload_hash
<< dendl
;
1335 ldout(cct
, 10) << "ERROR: expected_request_payload_hash="
1336 << expected_request_payload_hash
<< dendl
;
1341 AWSv4ComplSingle::AWSv4ComplSingle(const req_state
* const s
)
1342 : io_base_t(nullptr),
1344 expected_request_payload_hash(get_v4_exp_payload_hash(s
->info
)),
1345 sha256_hash(calc_hash_sha256_open_stream()) {
1348 rgw::auth::Completer::cmplptr_t
1349 AWSv4ComplSingle::create(const req_state
* const s
,
1350 const boost::optional
<std::string
>&)
1352 return std::make_shared
<AWSv4ComplSingle
>(s
);
1355 } // namespace rgw::auth::s3