1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "rgw_common.h"
5 #include "rgw_rest_client.h"
6 #include "rgw_auth_s3.h"
7 #include "rgw_http_errors.h"
10 #include "common/ceph_crypto_cms.h"
11 #include "common/armor.h"
12 #include "common/strtol.h"
13 #include "include/str_list.h"
14 #include "rgw_crypt_sanitize.h"
16 #define dout_context g_ceph_context
17 #define dout_subsys ceph_subsys_rgw
19 int RGWRESTSimpleRequest::get_status()
21 int retcode
= get_req_retcode();
28 int RGWRESTSimpleRequest::handle_header(const string
& name
, const string
& val
)
30 if (name
== "CONTENT_LENGTH") {
32 long len
= strict_strtol(val
.c_str(), 10, &err
);
34 ldout(cct
, 0) << "ERROR: failed converting content length (" << val
<< ") to int " << dendl
;
44 int RGWRESTSimpleRequest::receive_header(void *ptr
, size_t len
)
48 char *s
= (char *)ptr
, *end
= (char *)ptr
+ len
;
50 ldout(cct
, 10) << "receive_http_header" << dendl
;
59 ldout(cct
, 10) << "received header:" << line
<< dendl
;
60 // TODO: fill whatever data required here
62 char *tok
= strsep(&l
, " \t:");
67 if (strcmp(tok
, "HTTP") == 0 || strncmp(tok
, "HTTP/", 5) == 0) {
68 http_status
= atoi(l
);
69 if (http_status
== 100) /* 100-continue response */
71 status
= rgw_http_error_to_errno(http_status
);
73 /* convert header field name to upper case */
77 for (i
= 0; i
< len
&& *src
; ++i
, ++src
) {
83 buf
[i
] = toupper(*src
);
88 int r
= handle_header(buf
, l
);
100 static void get_new_date_str(string
& date_str
)
102 date_str
= rgw_to_asctime(ceph_clock_now());
105 int RGWRESTSimpleRequest::execute(RGWAccessKey
& key
, const char *method
, const char *resource
)
107 string new_url
= url
;
108 string new_resource
= resource
;
110 if (new_url
[new_url
.size() - 1] == '/' && resource
[0] == '/') {
111 new_url
= new_url
.substr(0, new_url
.size() - 1);
112 } else if (resource
[0] != '/') {
114 new_resource
.append(resource
);
116 new_url
.append(new_resource
);
119 get_new_date_str(date_str
);
120 headers
.push_back(pair
<string
, string
>("HTTP_DATE", date_str
));
122 string canonical_header
;
123 map
<string
, string
> meta_map
;
124 map
<string
, string
> sub_resources
;
125 rgw_create_s3_canonical_header(method
, NULL
, NULL
, date_str
.c_str(),
126 meta_map
, new_url
.c_str(), sub_resources
,
131 digest
= rgw::auth::s3::get_v2_signature(cct
, key
.key
, canonical_header
);
136 string auth_hdr
= "AWS " + key
.id
+ ":" + digest
;
138 ldout(cct
, 15) << "generated auth header: " << auth_hdr
<< dendl
;
140 headers
.push_back(pair
<string
, string
>("AUTHORIZATION", auth_hdr
));
141 int r
= process(method
, new_url
.c_str());
148 int RGWRESTSimpleRequest::send_data(void *ptr
, size_t len
)
153 if (len
> send_iter
->get_remaining())
154 len
= send_iter
->get_remaining();
156 send_iter
->copy(len
, (char *)ptr
);
161 int RGWRESTSimpleRequest::receive_data(void *ptr
, size_t len
)
163 size_t cp_len
, left_len
;
165 left_len
= max_response
> response
.length() ? (max_response
- response
.length()) : 0;
167 return 0; /* don't read extra data */
169 cp_len
= (len
> left_len
) ? left_len
: len
;
170 bufferptr
p((char *)ptr
, cp_len
);
178 void RGWRESTSimpleRequest::append_param(string
& dest
, const string
& name
, const string
& val
)
186 url_encode(name
, url_name
);
187 dest
.append(url_name
);
191 url_encode(val
, url_val
);
193 dest
.append(url_val
);
197 void RGWRESTSimpleRequest::get_params_str(map
<string
, string
>& extra_args
, string
& dest
)
199 map
<string
, string
>::iterator miter
;
200 for (miter
= extra_args
.begin(); miter
!= extra_args
.end(); ++miter
) {
201 append_param(dest
, miter
->first
, miter
->second
);
203 param_vec_t::iterator iter
;
204 for (iter
= params
.begin(); iter
!= params
.end(); ++iter
) {
205 append_param(dest
, iter
->first
, iter
->second
);
209 int RGWRESTSimpleRequest::sign_request(RGWAccessKey
& key
, RGWEnv
& env
, req_info
& info
)
211 /* don't sign if no key is provided */
212 if (key
.key
.empty()) {
216 if (cct
->_conf
->subsys
.should_gather(ceph_subsys_rgw
, 20)) {
217 for (const auto& i
: env
.get_map()) {
218 ldout(cct
, 20) << "> " << i
.first
<< " -> " << rgw::crypt_sanitize::x_meta_map
{i
.first
, i
.second
} << dendl
;
222 string canonical_header
;
223 if (!rgw_create_s3_canonical_header(info
, NULL
, canonical_header
, false)) {
224 ldout(cct
, 0) << "failed to create canonical s3 header" << dendl
;
228 ldout(cct
, 10) << "generated canonical header: " << canonical_header
<< dendl
;
232 digest
= rgw::auth::s3::get_v2_signature(cct
, key
.key
, canonical_header
);
237 string auth_hdr
= "AWS " + key
.id
+ ":" + digest
;
238 ldout(cct
, 15) << "generated auth header: " << auth_hdr
<< dendl
;
240 env
.set("AUTHORIZATION", auth_hdr
);
245 int RGWRESTSimpleRequest::forward_request(RGWAccessKey
& key
, req_info
& info
, size_t max_response
, bufferlist
*inbl
, bufferlist
*outbl
)
249 get_new_date_str(date_str
);
252 req_info
new_info(cct
, &new_env
);
253 new_info
.rebuild_from(info
);
255 new_env
.set("HTTP_DATE", date_str
.c_str());
257 int ret
= sign_request(key
, new_env
, new_info
);
259 ldout(cct
, 0) << "ERROR: failed to sign request" << dendl
;
263 for (const auto& kv
: new_env
.get_map()) {
264 headers
.emplace_back(kv
);
267 map
<string
, string
>& meta_map
= new_info
.x_meta_map
;
268 for (const auto& kv
: meta_map
) {
269 headers
.emplace_back(kv
);
273 get_params_str(info
.args
.get_params(), params_str
);
275 string new_url
= url
;
276 string
& resource
= new_info
.request_uri
;
277 string new_resource
= resource
;
278 if (new_url
[new_url
.size() - 1] == '/' && resource
[0] == '/') {
279 new_url
= new_url
.substr(0, new_url
.size() - 1);
280 } else if (resource
[0] != '/') {
282 new_resource
.append(resource
);
284 new_url
.append(new_resource
+ params_str
);
286 bufferlist::iterator bliter
;
289 bliter
= inbl
->begin();
292 set_send_length(inbl
->length());
295 int r
= process(new_info
.method
, new_url
.c_str());
298 // curl_easy has errored, generally means the service is not available
299 r
= -ERR_SERVICE_UNAVAILABLE
;
304 response
.append((char)0); /* NULL terminate response */
307 outbl
->claim(response
);
313 class RGWRESTStreamOutCB
: public RGWGetDataCB
{
314 RGWRESTStreamWriteRequest
*req
;
316 explicit RGWRESTStreamOutCB(RGWRESTStreamWriteRequest
*_req
) : req(_req
) {}
317 int handle_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
) override
; /* callback for object iteration when sending data */
320 int RGWRESTStreamOutCB::handle_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
)
322 dout(20) << "RGWRESTStreamOutCB::handle_data bl.length()=" << bl
.length() << " bl_ofs=" << bl_ofs
<< " bl_len=" << bl_len
<< dendl
;
323 if (!bl_ofs
&& bl_len
== bl
.length()) {
324 return req
->add_output_data(bl
);
327 bufferptr
bp(bl
.c_str() + bl_ofs
, bl_len
);
329 new_bl
.push_back(bp
);
331 return req
->add_output_data(new_bl
);
334 RGWRESTStreamWriteRequest::~RGWRESTStreamWriteRequest()
339 int RGWRESTStreamWriteRequest::add_output_data(bufferlist
& bl
)
347 pending_send
.push_back(bl
);
351 return http_manager
.process_requests(false, &done
);
354 static void grants_by_type_add_one_grant(map
<int, string
>& grants_by_type
, int perm
, ACLGrant
& grant
)
356 string
& s
= grants_by_type
[perm
];
362 ACLGranteeType
& type
= grant
.get_type();
363 switch (type
.get_type()) {
367 case ACL_TYPE_EMAIL_USER
:
368 id_type_str
= "emailAddress";
375 s
.append(id_type_str
+ "=\"" + id
.to_str() + "\"");
378 struct grant_type_to_header
{
383 struct grant_type_to_header grants_headers_def
[] = {
384 { RGW_PERM_FULL_CONTROL
, "x-amz-grant-full-control"},
385 { RGW_PERM_READ
, "x-amz-grant-read"},
386 { RGW_PERM_WRITE
, "x-amz-grant-write"},
387 { RGW_PERM_READ_ACP
, "x-amz-grant-read-acp"},
388 { RGW_PERM_WRITE_ACP
, "x-amz-grant-write-acp"},
392 static bool grants_by_type_check_perm(map
<int, string
>& grants_by_type
, int perm
, ACLGrant
& grant
, int check_perm
)
394 if ((perm
& check_perm
) == perm
) {
395 grants_by_type_add_one_grant(grants_by_type
, check_perm
, grant
);
401 static void grants_by_type_add_perm(map
<int, string
>& grants_by_type
, int perm
, ACLGrant
& grant
)
403 struct grant_type_to_header
*t
;
405 for (t
= grants_headers_def
; t
->header
; t
++) {
406 if (grants_by_type_check_perm(grants_by_type
, perm
, grant
, t
->type
))
411 static void add_grants_headers(map
<int, string
>& grants
, RGWEnv
& env
, map
<string
, string
>& meta_map
)
413 struct grant_type_to_header
*t
;
415 for (t
= grants_headers_def
; t
->header
; t
++) {
416 map
<int, string
>::iterator iter
= grants
.find(t
->type
);
417 if (iter
!= grants
.end()) {
418 env
.set(t
->header
,iter
->second
);
419 meta_map
[t
->header
] = iter
->second
;
424 int RGWRESTStreamWriteRequest::put_obj_init(RGWAccessKey
& key
, rgw_obj
& obj
, uint64_t obj_size
, map
<string
, bufferlist
>& attrs
)
426 string resource
= obj
.bucket
.name
+ "/" + obj
.get_oid();
427 string new_url
= url
;
428 if (new_url
[new_url
.size() - 1] != '/')
432 get_new_date_str(date_str
);
435 req_info
new_info(cct
, &new_env
);
438 map
<string
, string
>& args
= new_info
.args
.get_params();
439 get_params_str(args
, params_str
);
441 new_url
.append(resource
+ params_str
);
443 new_env
.set("HTTP_DATE", date_str
.c_str());
445 new_info
.method
= "PUT";
447 new_info
.script_uri
= "/";
448 new_info
.script_uri
.append(resource
);
449 new_info
.request_uri
= new_info
.script_uri
;
451 /* merge send headers */
452 for (auto& attr
: attrs
) {
453 bufferlist
& bl
= attr
.second
;
454 const string
& name
= attr
.first
;
455 string val
= bl
.c_str();
456 if (name
.compare(0, sizeof(RGW_ATTR_META_PREFIX
) - 1, RGW_ATTR_META_PREFIX
) == 0) {
457 string header_name
= RGW_AMZ_META_PREFIX
;
458 header_name
.append(name
.substr(sizeof(RGW_ATTR_META_PREFIX
) - 1));
459 new_env
.set(header_name
, val
);
460 new_info
.x_meta_map
[header_name
] = val
;
463 RGWAccessControlPolicy policy
;
464 int ret
= rgw_policy_from_attrset(cct
, attrs
, &policy
);
466 ldout(cct
, 0) << "ERROR: couldn't get policy ret=" << ret
<< dendl
;
470 /* update acl headers */
471 RGWAccessControlList
& acl
= policy
.get_acl();
472 multimap
<string
, ACLGrant
>& grant_map
= acl
.get_grant_map();
473 multimap
<string
, ACLGrant
>::iterator giter
;
474 map
<int, string
> grants_by_type
;
475 for (giter
= grant_map
.begin(); giter
!= grant_map
.end(); ++giter
) {
476 ACLGrant
& grant
= giter
->second
;
477 ACLPermission
& perm
= grant
.get_permission();
478 grants_by_type_add_perm(grants_by_type
, perm
.get_permissions(), grant
);
480 add_grants_headers(grants_by_type
, new_env
, new_info
.x_meta_map
);
481 ret
= sign_request(key
, new_env
, new_info
);
483 ldout(cct
, 0) << "ERROR: failed to sign request" << dendl
;
487 for (const auto& kv
: new_env
.get_map()) {
488 headers
.emplace_back(kv
);
491 cb
= new RGWRESTStreamOutCB(this);
493 set_send_length(obj_size
);
495 int r
= http_manager
.add_request(this, new_info
.method
, new_url
.c_str());
502 int RGWRESTStreamWriteRequest::send_data(void *ptr
, size_t len
)
506 dout(20) << "RGWRESTStreamWriteRequest::send_data()" << dendl
;
508 if (pending_send
.empty() || status
< 0) {
513 list
<bufferlist
>::iterator iter
= pending_send
.begin();
514 while (iter
!= pending_send
.end() && len
> 0) {
515 bufferlist
& bl
= *iter
;
517 list
<bufferlist
>::iterator next_iter
= iter
;
521 uint64_t send_len
= min(len
, (size_t)bl
.length());
523 memcpy(ptr
, bl
.c_str(), send_len
);
525 ptr
= (char *)ptr
+ send_len
;
532 if (bl
.length() > send_len
) {
533 bufferptr
bp(bl
.c_str() + send_len
, bl
.length() - send_len
);
536 pending_send
.pop_front(); /* need to do this after we copy data from bl */
537 if (new_bl
.length()) {
538 pending_send
.push_front(new_bl
);
548 void set_str_from_headers(map
<string
, string
>& out_headers
, const string
& header_name
, string
& str
)
550 map
<string
, string
>::iterator iter
= out_headers
.find(header_name
);
551 if (iter
!= out_headers
.end()) {
558 static int parse_rgwx_mtime(CephContext
*cct
, const string
& s
, ceph::real_time
*rt
)
563 get_str_vec(s
, ".", vec
);
569 long secs
= strict_strtol(vec
[0].c_str(), 10, &err
);
572 ldout(cct
, 0) << "ERROR: failed converting mtime (" << s
<< ") to real_time " << dendl
;
576 if (vec
.size() > 1) {
577 nsecs
= strict_strtol(vec
[1].c_str(), 10, &err
);
579 ldout(cct
, 0) << "ERROR: failed converting mtime (" << s
<< ") to real_time " << dendl
;
584 *rt
= utime_t(secs
, nsecs
).to_real_time();
589 int RGWRESTStreamWriteRequest::complete(string
& etag
, real_time
*mtime
)
591 int ret
= http_manager
.complete_requests();
595 set_str_from_headers(out_headers
, "ETAG", etag
);
599 set_str_from_headers(out_headers
, "RGWX_MTIME", mtime_str
);
601 ret
= parse_rgwx_mtime(cct
, mtime_str
, mtime
);
609 int RGWRESTStreamRWRequest::send_request(RGWAccessKey
& key
, map
<string
, string
>& extra_headers
, rgw_obj
& obj
, RGWHTTPManager
*mgr
)
611 string urlsafe_bucket
, urlsafe_object
;
612 url_encode(obj
.bucket
.get_key(':', 0), urlsafe_bucket
);
613 url_encode(obj
.key
.name
, urlsafe_object
);
614 string resource
= urlsafe_bucket
+ "/" + urlsafe_object
;
616 return send_request(&key
, extra_headers
, resource
, nullptr, mgr
);
619 int RGWRESTStreamRWRequest::send_request(RGWAccessKey
*key
, map
<string
, string
>& extra_headers
, const string
& resource
,
620 bufferlist
*send_data
, RGWHTTPManager
*mgr
)
622 string new_url
= url
;
623 if (new_url
[new_url
.size() - 1] != '/')
627 get_new_date_str(date_str
);
630 req_info
new_info(cct
, &new_env
);
633 map
<string
, string
>& args
= new_info
.args
.get_params();
634 get_params_str(args
, params_str
);
636 /* merge params with extra args so that we can sign correctly */
637 for (param_vec_t::iterator iter
= params
.begin(); iter
!= params
.end(); ++iter
) {
638 new_info
.args
.append(iter
->first
, iter
->second
);
642 if (resource
[0] == '/') {
643 new_resource
= resource
.substr(1);
645 new_resource
= resource
;
648 new_url
.append(new_resource
+ params_str
);
650 new_env
.set("HTTP_DATE", date_str
.c_str());
652 for (map
<string
, string
>::iterator iter
= extra_headers
.begin();
653 iter
!= extra_headers
.end(); ++iter
) {
654 new_env
.set(iter
->first
.c_str(), iter
->second
.c_str());
657 new_info
.method
= method
;
659 new_info
.script_uri
= "/";
660 new_info
.script_uri
.append(new_resource
);
661 new_info
.request_uri
= new_info
.script_uri
;
663 new_info
.init_meta_info(NULL
);
666 int ret
= sign_request(*key
, new_env
, new_info
);
668 ldout(cct
, 0) << "ERROR: failed to sign request" << dendl
;
673 for (const auto& kv
: new_env
.get_map()) {
674 headers
.emplace_back(kv
);
677 bool send_data_hint
= false;
679 outbl
.claim(*send_data
);
680 send_data_hint
= true;
683 RGWHTTPManager
*pmanager
= &http_manager
;
688 int r
= pmanager
->add_request(this, new_info
.method
, new_url
.c_str(), send_data_hint
);
693 r
= pmanager
->complete_requests();
701 int RGWRESTStreamRWRequest::complete_request(string
& etag
, real_time
*mtime
, uint64_t *psize
, map
<string
, string
>& attrs
)
703 set_str_from_headers(out_headers
, "ETAG", etag
);
707 set_str_from_headers(out_headers
, "RGWX_MTIME", mtime_str
);
708 if (!mtime_str
.empty()) {
709 int ret
= parse_rgwx_mtime(cct
, mtime_str
, mtime
);
714 *mtime
= real_time();
719 set_str_from_headers(out_headers
, "RGWX_OBJECT_SIZE", size_str
);
721 *psize
= strict_strtoll(size_str
.c_str(), 10, &err
);
723 ldout(cct
, 0) << "ERROR: failed parsing embedded metadata object size (" << size_str
<< ") to int " << dendl
;
729 map
<string
, string
>::iterator iter
;
730 for (iter
= out_headers
.begin(); iter
!= out_headers
.end(); ++iter
) {
731 const string
& attr_name
= iter
->first
;
732 if (attr_name
.compare(0, sizeof(RGW_HTTP_RGWX_ATTR_PREFIX
) - 1, RGW_HTTP_RGWX_ATTR_PREFIX
) == 0) {
733 string name
= attr_name
.substr(sizeof(RGW_HTTP_RGWX_ATTR_PREFIX
) - 1);
734 const char *src
= name
.c_str();
735 char buf
[name
.size() + 1];
737 for (; *src
; ++src
, ++dest
) {
743 *dest
= tolower(*src
);
747 attrs
[buf
] = iter
->second
;
753 int RGWRESTStreamRWRequest::handle_header(const string
& name
, const string
& val
)
755 if (name
== "RGWX_EMBEDDED_METADATA_LEN") {
757 long len
= strict_strtol(val
.c_str(), 10, &err
);
759 ldout(cct
, 0) << "ERROR: failed converting embedded metadata len (" << val
<< ") to int " << dendl
;
763 cb
->set_extra_data_len(len
);
768 int RGWRESTStreamRWRequest::receive_data(void *ptr
, size_t len
)
770 bufferptr
bp((const char *)ptr
, len
);
773 int ret
= cb
->handle_data(bl
, ofs
, len
);
780 int RGWRESTStreamRWRequest::send_data(void *ptr
, size_t len
)
782 if (outbl
.length() == 0) {
786 uint64_t send_size
= min(len
, (size_t)(outbl
.length() - write_ofs
));
788 memcpy(ptr
, outbl
.c_str() + write_ofs
, send_size
);
789 write_ofs
+= send_size
;
794 class StreamIntoBufferlist
: public RGWGetDataCB
{
797 StreamIntoBufferlist(bufferlist
& _bl
) : bl(_bl
) {}
798 int handle_data(bufferlist
& inbl
, off_t bl_ofs
, off_t bl_len
) override
{
799 bl
.claim_append(inbl
);