ceph/src/rgw/rgw_rest_conn.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #pragma once
   5
   6 #include "rgw_rest_client.h"
   7 #include "common/ceph_json.h"
   8 #include "common/RefCountedObj.h"
   9 #include "include/common_fwd.h"
  10
  11 #include <atomic>
  12
  13 class RGWSI_Zone;
  14
  15 template <class T>
  16 static int parse_decode_json(T& t, bufferlist& bl)
  17 {
  18   JSONParser p;
  19   if (!p.parse(bl.c_str(), bl.length())) {
  20     return -EINVAL;
  21   }
  22
  23   try {
  24     decode_json_obj(t, &p);
  25   } catch (JSONDecoder::err& e) {
  26     return -EINVAL;
  27   }
  28   return 0;
  29 }
  30
  31 struct rgw_http_param_pair {
  32   const char *key;
  33   const char *val;
  34 };
  35
  36 // append a null-terminated rgw_http_param_pair list into a list of string pairs
  37 inline void append_param_list(param_vec_t& params, const rgw_http_param_pair* pp)
  38 {
  39   while (pp && pp->key) {
  40     string k = pp->key;
  41     string v = (pp->val ? pp->val : "");
  42     params.emplace_back(make_pair(std::move(k), std::move(v)));
  43     ++pp;
  44   }
  45 }
  46
  47 // copy a null-terminated rgw_http_param_pair list into a list of string pairs
  48 inline param_vec_t make_param_list(const rgw_http_param_pair* pp)
  49 {
  50   param_vec_t params;
  51   append_param_list(params, pp);
  52   return params;
  53 }
  54
  55 inline param_vec_t make_param_list(const map<string, string> *pp)
  56 {
  57   param_vec_t params;
  58   if (!pp) {
  59     return params;
  60   }
  61   for (auto iter : *pp) {
  62     params.emplace_back(make_pair(iter.first, iter.second));
  63   }
  64   return params;
  65 }
  66
  67 class RGWRESTConn
  68 {
  69   CephContext *cct;
  70   vector<string> endpoints;
  71   RGWAccessKey key;
  72   string self_zone_group;
  73   string remote_id;
  74   HostStyle host_style;
  75   std::atomic<int64_t> counter = { 0 };
  76
  77 public:
  78
  79   RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc, const string& _remote_id, const list<string>& endpoints, HostStyle _host_style = PathStyle);
  80   RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc, const string& _remote_id, const list<string>& endpoints, RGWAccessKey _cred, HostStyle _host_style = PathStyle);
  81
  82   // custom move needed for atomic
  83   RGWRESTConn(RGWRESTConn&& other);
  84   RGWRESTConn& operator=(RGWRESTConn&& other);
  85   virtual ~RGWRESTConn() = default;
  86
  87   int get_url(string& endpoint);
  88   string get_url();
  89   const string& get_self_zonegroup() {
  90     return self_zone_group;
  91   }
  92   const string& get_remote_id() {
  93     return remote_id;
  94   }
  95   RGWAccessKey& get_key() {
  96     return key;
  97   }
  98
  99   HostStyle get_host_style() {
 100     return host_style;
 101   }
 102
 103   CephContext *get_ctx() {
 104     return cct;
 105   }
 106   size_t get_endpoint_count() const { return endpoints.size(); }
 107
 108   virtual void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup);
 109
 110   /* sync request */
 111   int forward(const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 112
 113
 114   /* async requests */
 115   int put_obj_send_init(rgw::sal::RGWObject* obj, const rgw_http_param_pair *extra_params, RGWRESTStreamS3PutObj **req);
 116   int put_obj_async(const rgw_user& uid, rgw::sal::RGWObject* obj, uint64_t obj_size,
 117                     map<string, bufferlist>& attrs, bool send, RGWRESTStreamS3PutObj **req);
 118   int complete_request(RGWRESTStreamS3PutObj *req, string& etag,
 119                        ceph::real_time *mtime, optional_yield y);
 120
 121   struct get_obj_params {
 122     rgw_user uid;
 123     req_info *info{nullptr};
 124     const ceph::real_time *mod_ptr{nullptr};
 125     const ceph::real_time *unmod_ptr{nullptr};
 126     bool high_precision_time{true};
 127
 128     string etag;
 129
 130     uint32_t mod_zone_id{0};
 131     uint64_t mod_pg_ver{0};
 132
 133     bool prepend_metadata{false};
 134     bool get_op{false};
 135     bool rgwx_stat{false};
 136     bool sync_manifest{false};
 137
 138     bool skip_decrypt{true};
 139     RGWHTTPStreamRWRequest::ReceiveCB *cb{nullptr};
 140
 141     bool range_is_set{false};
 142     uint64_t range_start{0};
 143     uint64_t range_end{0};
 144   };
 145
 146   int get_obj(const rgw::sal::RGWObject* obj, const get_obj_params& params, bool send, RGWRESTStreamRWRequest **req);
 147
 148   int get_obj(const rgw_user& uid, req_info *info /* optional */, const rgw::sal::RGWObject* obj,
 149               const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr,
 150               uint32_t mod_zone_id, uint64_t mod_pg_ver,
 151               bool prepend_metadata, bool get_op, bool rgwx_stat, bool sync_manifest,
 152               bool skip_decrypt, bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req);
 153   int complete_request(RGWRESTStreamRWRequest *req,
 154                        string *etag,
 155                        ceph::real_time *mtime,
 156                        uint64_t *psize,
 157                        map<string, string> *pattrs,
 158                        map<string, string> *pheaders,
 159                        optional_yield y);
 160
 161   int get_resource(const string& resource,
 162                    param_vec_t *extra_params,
 163                    map<string, string>* extra_headers,
 164                    bufferlist& bl,
 165                    bufferlist *send_data,
 166                    RGWHTTPManager *mgr,
 167                    optional_yield y);
 168
 169   template <class T>
 170   int get_json_resource(const string& resource, param_vec_t *params,
 171                         bufferlist *in_data, optional_yield y, T& t);
 172   template <class T>
 173   int get_json_resource(const string& resource, param_vec_t *params,
 174                         optional_yield y, T& t);
 175   template <class T>
 176   int get_json_resource(const string& resource, const rgw_http_param_pair *pp,
 177                         optional_yield y, T& t);
 178
 179 private:
 180   void populate_zonegroup(param_vec_t& params, const string& zonegroup) {
 181     if (!zonegroup.empty()) {
 182       params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
 183     }
 184   }
 185   void populate_uid(param_vec_t& params, const rgw_user *uid) {
 186     if (uid) {
 187       string uid_str = uid->to_str();
 188       if (!uid->empty()){
 189         params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
 190       }
 191     }
 192   }
 193 };
 194
 195 class S3RESTConn : public RGWRESTConn {
 196
 197 public:
 198
 199   S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const string& _remote_id, const list<string>& endpoints, HostStyle _host_style = PathStyle) :
 200     RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _host_style) {}
 201
 202   S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const string& _remote_id, const list<string>& endpoints, RGWAccessKey _cred, HostStyle _host_style = PathStyle):
 203     RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _cred, _host_style) {}
 204   ~S3RESTConn() override = default;
 205
 206   void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup) override {
 207     // do not populate any params in S3 REST Connection.
 208     return;
 209   }
 210 };
 211
 212
 213 template<class T>
 214 int RGWRESTConn::get_json_resource(const string& resource, param_vec_t *params,
 215                                    bufferlist *in_data, optional_yield y, T& t)
 216 {
 217   bufferlist bl;
 218   int ret = get_resource(resource, params, nullptr, bl, in_data, nullptr, y);
 219   if (ret < 0) {
 220     return ret;
 221   }
 222
 223   ret = parse_decode_json(t, bl);
 224   if (ret < 0) {
 225     return ret;
 226   }
 227
 228   return 0;
 229 }
 230
 231 template<class T>
 232 int RGWRESTConn::get_json_resource(const string& resource, param_vec_t *params,
 233                                    optional_yield y, T& t)
 234 {
 235   return get_json_resource(resource, params, nullptr, y, t);
 236 }
 237
 238 template<class T>
 239 int RGWRESTConn::get_json_resource(const string& resource, const rgw_http_param_pair *pp,
 240                                    optional_yield y, T& t)
 241 {
 242   param_vec_t params = make_param_list(pp);
 243   return get_json_resource(resource, &params, y, t);
 244 }
 245
 246 class RGWStreamIntoBufferlist : public RGWHTTPStreamRWRequest::ReceiveCB {
 247   bufferlist& bl;
 248 public:
 249   explicit RGWStreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {}
 250   int handle_data(bufferlist& inbl, bool *pause) override {
 251     bl.claim_append(inbl);
 252     return inbl.length();
 253   }
 254 };
 255
 256 class RGWRESTReadResource : public RefCountedObject, public RGWIOProvider {
 257   CephContext *cct;
 258   RGWRESTConn *conn;
 259   string resource;
 260   param_vec_t params;
 261   map<string, string> headers;
 262   bufferlist bl;
 263   RGWStreamIntoBufferlist cb;
 264
 265   RGWHTTPManager *mgr;
 266   RGWRESTStreamReadRequest req;
 267
 268   void init_common(param_vec_t *extra_headers);
 269
 270 public:
 271   RGWRESTReadResource(RGWRESTConn *_conn,
 272                       const string& _resource,
 273                       const rgw_http_param_pair *pp,
 274                       param_vec_t *extra_headers,
 275                       RGWHTTPManager *_mgr);
 276
 277   RGWRESTReadResource(RGWRESTConn *_conn,
 278                       const string& _resource,
 279                       param_vec_t& _params,
 280                       param_vec_t *extra_headers,
 281                       RGWHTTPManager *_mgr);
 282   ~RGWRESTReadResource() = default;
 283
 284   rgw_io_id get_io_id(int io_type) {
 285     return req.get_io_id(io_type);
 286   }
 287
 288   void set_io_user_info(void *user_info) override {
 289     req.set_io_user_info(user_info);
 290   }
 291
 292   void *get_io_user_info() override {
 293     return req.get_io_user_info();
 294   }
 295
 296   template <class T>
 297   int decode_resource(T *dest);
 298
 299   int read(optional_yield y);
 300
 301   int aio_read();
 302
 303   string to_str() {
 304     return req.to_str();
 305   }
 306
 307   int get_http_status() {
 308     return req.get_http_status();
 309   }
 310
 311   int wait(bufferlist *pbl, optional_yield y) {
 312     int ret = req.wait(y);
 313     if (ret < 0) {
 314       return ret;
 315     }
 316
 317     if (req.get_status() < 0) {
 318       return req.get_status();
 319     }
 320     *pbl = bl;
 321     return 0;
 322   }
 323
 324   template <class T>
 325   int wait(T *dest, optional_yield y);
 326
 327   template <class T>
 328   int fetch(T *dest, optional_yield y);
 329 };
 330
 331
 332 template <class T>
 333 int RGWRESTReadResource::decode_resource(T *dest)
 334 {
 335   int ret = req.get_status();
 336   if (ret < 0) {
 337     return ret;
 338   }
 339   ret = parse_decode_json(*dest, bl);
 340   if (ret < 0) {
 341     return ret;
 342   }
 343   return 0;
 344 }
 345
 346 template <class T>
 347 int RGWRESTReadResource::fetch(T *dest, optional_yield y)
 348 {
 349   int ret = read(y);
 350   if (ret < 0) {
 351     return ret;
 352   }
 353
 354   ret = decode_resource(dest);
 355   if (ret < 0) {
 356     return ret;
 357   }
 358   return 0;
 359 }
 360
 361 template <class T>
 362 int RGWRESTReadResource::wait(T *dest, optional_yield y)
 363 {
 364   int ret = req.wait(y);
 365   if (ret < 0) {
 366     return ret;
 367   }
 368
 369   ret = decode_resource(dest);
 370   if (ret < 0) {
 371     return ret;
 372   }
 373   return 0;
 374 }
 375
 376 class RGWRESTSendResource : public RefCountedObject, public RGWIOProvider {
 377   CephContext *cct;
 378   RGWRESTConn *conn;
 379   string method;
 380   string resource;
 381   param_vec_t params;
 382   map<string, string> headers;
 383   bufferlist bl;
 384   RGWStreamIntoBufferlist cb;
 385
 386   RGWHTTPManager *mgr;
 387   RGWRESTStreamRWRequest req;
 388
 389   void init_common(param_vec_t *extra_headers);
 390
 391 public:
 392   RGWRESTSendResource(RGWRESTConn *_conn,
 393                       const string& _method,
 394                       const string& _resource,
 395                       const rgw_http_param_pair *pp,
 396                       param_vec_t *extra_headers,
 397                       RGWHTTPManager *_mgr);
 398
 399   RGWRESTSendResource(RGWRESTConn *_conn,
 400                       const string& _method,
 401                       const string& _resource,
 402                       param_vec_t& params,
 403                       param_vec_t *extra_headers,
 404                       RGWHTTPManager *_mgr);
 405
 406   ~RGWRESTSendResource() = default;
 407
 408   rgw_io_id get_io_id(int io_type) {
 409     return req.get_io_id(io_type);
 410   }
 411
 412   void set_io_user_info(void *user_info) override {
 413     req.set_io_user_info(user_info);
 414   }
 415
 416   void *get_io_user_info() override {
 417     return req.get_io_user_info();
 418   }
 419
 420   int send(bufferlist& bl, optional_yield y);
 421
 422   int aio_send(bufferlist& bl);
 423
 424   string to_str() {
 425     return req.to_str();
 426   }
 427
 428   int get_http_status() {
 429     return req.get_http_status();
 430   }
 431
 432   template <class E = int>
 433   int wait(bufferlist *pbl, optional_yield y, E *err_result = nullptr) {
 434     int ret = req.wait(y);
 435     *pbl = bl;
 436
 437     if (ret < 0 && err_result ) {
 438       ret = parse_decode_json(*err_result, bl);
 439     }
 440
 441     return req.get_status();
 442   }
 443
 444   template <class T, class E = int>
 445   int wait(T *dest, optional_yield y, E *err_result = nullptr);
 446 };
 447
 448 template <class T, class E>
 449 int RGWRESTSendResource::wait(T *dest, optional_yield y, E *err_result)
 450 {
 451   int ret = req.wait(y);
 452   if (ret >= 0) {
 453     ret = req.get_status();
 454   }
 455
 456   if (ret < 0 && err_result) {
 457     ret = parse_decode_json(*err_result, bl);
 458   }
 459
 460   if (ret < 0) {
 461     return ret;
 462   }
 463
 464   ret = parse_decode_json(*dest, bl);
 465   if (ret < 0) {
 466     return ret;
 467   }
 468   return 0;
 469
 470 }
 471
 472 class RGWRESTPostResource : public RGWRESTSendResource {
 473 public:
 474   RGWRESTPostResource(RGWRESTConn *_conn,
 475                       const string& _resource,
 476                       const rgw_http_param_pair *pp,
 477                       param_vec_t *extra_headers,
 478                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 479                                                                   pp, extra_headers, _mgr) {}
 480
 481   RGWRESTPostResource(RGWRESTConn *_conn,
 482                       const string& _resource,
 483                       param_vec_t& params,
 484                       param_vec_t *extra_headers,
 485                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 486                                                                   params, extra_headers, _mgr) {}
 487
 488 };
 489
 490 class RGWRESTPutResource : public RGWRESTSendResource {
 491 public:
 492   RGWRESTPutResource(RGWRESTConn *_conn,
 493                      const string& _resource,
 494                      const rgw_http_param_pair *pp,
 495                      param_vec_t *extra_headers,
 496                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 497                                                                   pp, extra_headers, _mgr) {}
 498
 499   RGWRESTPutResource(RGWRESTConn *_conn,
 500                      const string& _resource,
 501                      param_vec_t& params,
 502                      param_vec_t *extra_headers,
 503                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 504                                                                   params, extra_headers, _mgr) {}
 505
 506 };
 507
 508 class RGWRESTDeleteResource : public RGWRESTSendResource {
 509 public:
 510   RGWRESTDeleteResource(RGWRESTConn *_conn,
 511                      const string& _resource,
 512                      const rgw_http_param_pair *pp,
 513                      param_vec_t *extra_headers,
 514                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 515                                                                   pp, extra_headers, _mgr) {}
 516
 517   RGWRESTDeleteResource(RGWRESTConn *_conn,
 518                      const string& _resource,
 519                      param_vec_t& params,
 520                      param_vec_t *extra_headers,
 521                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 522                                                                   params, extra_headers, _mgr) {}
 523
 524 };