ceph/src/rgw/rgw_rest_conn.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #pragma once
   5
   6 #include "rgw_rest_client.h"
   7 #include "common/ceph_json.h"
   8 #include "common/RefCountedObj.h"
   9 #include "include/common_fwd.h"
  10
  11 #include <atomic>
  12
  13 class RGWSI_Zone;
  14
  15 template <class T>
  16 static int parse_decode_json(T& t, bufferlist& bl)
  17 {
  18   JSONParser p;
  19   if (!p.parse(bl.c_str(), bl.length())) {
  20     return -EINVAL;
  21   }
  22
  23   try {
  24     decode_json_obj(t, &p);
  25   } catch (JSONDecoder::err& e) {
  26     return -EINVAL;
  27   }
  28   return 0;
  29 }
  30
  31 struct rgw_http_param_pair {
  32   const char *key;
  33   const char *val;
  34 };
  35
  36 // append a null-terminated rgw_http_param_pair list into a list of string pairs
  37 inline void append_param_list(param_vec_t& params, const rgw_http_param_pair* pp)
  38 {
  39   while (pp && pp->key) {
  40     string k = pp->key;
  41     string v = (pp->val ? pp->val : "");
  42     params.emplace_back(make_pair(std::move(k), std::move(v)));
  43     ++pp;
  44   }
  45 }
  46
  47 // copy a null-terminated rgw_http_param_pair list into a list of string pairs
  48 inline param_vec_t make_param_list(const rgw_http_param_pair* pp)
  49 {
  50   param_vec_t params;
  51   append_param_list(params, pp);
  52   return params;
  53 }
  54
  55 inline param_vec_t make_param_list(const map<string, string> *pp)
  56 {
  57   param_vec_t params;
  58   if (!pp) {
  59     return params;
  60   }
  61   for (auto iter : *pp) {
  62     params.emplace_back(make_pair(iter.first, iter.second));
  63   }
  64   return params;
  65 }
  66
  67 class RGWRESTConn
  68 {
  69   CephContext *cct;
  70   vector<string> endpoints;
  71   RGWAccessKey key;
  72   string self_zone_group;
  73   string remote_id;
  74   HostStyle host_style;
  75   std::atomic<int64_t> counter = { 0 };
  76
  77 public:
  78
  79   RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc, const string& _remote_id, const list<string>& endpoints, HostStyle _host_style = PathStyle);
  80   RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc, const string& _remote_id, const list<string>& endpoints, RGWAccessKey _cred, HostStyle _host_style = PathStyle);
  81
  82   // custom move needed for atomic
  83   RGWRESTConn(RGWRESTConn&& other);
  84   RGWRESTConn& operator=(RGWRESTConn&& other);
  85   virtual ~RGWRESTConn() = default;
  86
  87   int get_url(string& endpoint);
  88   string get_url();
  89   const string& get_self_zonegroup() {
  90     return self_zone_group;
  91   }
  92   const string& get_remote_id() {
  93     return remote_id;
  94   }
  95   RGWAccessKey& get_key() {
  96     return key;
  97   }
  98
  99   HostStyle get_host_style() {
 100     return host_style;
 101   }
 102
 103   CephContext *get_ctx() {
 104     return cct;
 105   }
 106   size_t get_endpoint_count() const { return endpoints.size(); }
 107
 108   virtual void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup);
 109
 110   /* sync request */
 111   int forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 112
 113
 114   /* async requests */
 115   int put_obj_send_init(rgw::sal::RGWObject* obj, const rgw_http_param_pair *extra_params, RGWRESTStreamS3PutObj **req);
 116   int put_obj_async(const DoutPrefixProvider *dpp, const rgw_user& uid, rgw::sal::RGWObject* obj, uint64_t obj_size,
 117                     map<string, bufferlist>& attrs, bool send, RGWRESTStreamS3PutObj **req);
 118   int complete_request(RGWRESTStreamS3PutObj *req, string& etag,
 119                        ceph::real_time *mtime, optional_yield y);
 120
 121   struct get_obj_params {
 122     rgw_user uid;
 123     req_info *info{nullptr};
 124     const ceph::real_time *mod_ptr{nullptr};
 125     const ceph::real_time *unmod_ptr{nullptr};
 126     bool high_precision_time{true};
 127
 128     string etag;
 129
 130     uint32_t mod_zone_id{0};
 131     uint64_t mod_pg_ver{0};
 132
 133     bool prepend_metadata{false};
 134     bool get_op{false};
 135     bool rgwx_stat{false};
 136     bool sync_manifest{false};
 137
 138     bool skip_decrypt{true};
 139     RGWHTTPStreamRWRequest::ReceiveCB *cb{nullptr};
 140
 141     bool range_is_set{false};
 142     uint64_t range_start{0};
 143     uint64_t range_end{0};
 144   };
 145
 146   int get_obj(const DoutPrefixProvider *dpp, const rgw::sal::RGWObject* obj, const get_obj_params& params, bool send, RGWRESTStreamRWRequest **req);
 147
 148   int get_obj(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info *info /* optional */, const rgw::sal::RGWObject* obj,
 149               const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr,
 150               uint32_t mod_zone_id, uint64_t mod_pg_ver,
 151               bool prepend_metadata, bool get_op, bool rgwx_stat, bool sync_manifest,
 152               bool skip_decrypt, bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req);
 153   int complete_request(RGWRESTStreamRWRequest *req,
 154                        string *etag,
 155                        ceph::real_time *mtime,
 156                        uint64_t *psize,
 157                        map<string, string> *pattrs,
 158                        map<string, string> *pheaders,
 159                        optional_yield y);
 160
 161   int get_resource(const DoutPrefixProvider *dpp,
 162                    const string& resource,
 163                    param_vec_t *extra_params,
 164                    map<string, string>* extra_headers,
 165                    bufferlist& bl,
 166                    bufferlist *send_data,
 167                    RGWHTTPManager *mgr,
 168                    optional_yield y);
 169
 170   template <class T>
 171   int get_json_resource(const DoutPrefixProvider *dpp, const string& resource, param_vec_t *params,
 172                         bufferlist *in_data, optional_yield y, T& t);
 173   template <class T>
 174   int get_json_resource(const DoutPrefixProvider *dpp, const string& resource, param_vec_t *params,
 175                         optional_yield y, T& t);
 176   template <class T>
 177   int get_json_resource(const DoutPrefixProvider *dpp, const string& resource, const rgw_http_param_pair *pp,
 178                         optional_yield y, T& t);
 179
 180 private:
 181   void populate_zonegroup(param_vec_t& params, const string& zonegroup) {
 182     if (!zonegroup.empty()) {
 183       params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
 184     }
 185   }
 186   void populate_uid(param_vec_t& params, const rgw_user *uid) {
 187     if (uid) {
 188       string uid_str = uid->to_str();
 189       if (!uid->empty()){
 190         params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
 191       }
 192     }
 193   }
 194 };
 195
 196 class S3RESTConn : public RGWRESTConn {
 197
 198 public:
 199
 200   S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const string& _remote_id, const list<string>& endpoints, HostStyle _host_style = PathStyle) :
 201     RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _host_style) {}
 202
 203   S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const string& _remote_id, const list<string>& endpoints, RGWAccessKey _cred, HostStyle _host_style = PathStyle):
 204     RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _cred, _host_style) {}
 205   ~S3RESTConn() override = default;
 206
 207   void populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup) override {
 208     // do not populate any params in S3 REST Connection.
 209     return;
 210   }
 211 };
 212
 213
 214 template<class T>
 215 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const string& resource, param_vec_t *params,
 216                                    bufferlist *in_data, optional_yield y, T& t)
 217 {
 218   bufferlist bl;
 219   int ret = get_resource(dpp, resource, params, nullptr, bl, in_data, nullptr, y);
 220   if (ret < 0) {
 221     return ret;
 222   }
 223
 224   ret = parse_decode_json(t, bl);
 225   if (ret < 0) {
 226     return ret;
 227   }
 228
 229   return 0;
 230 }
 231
 232 template<class T>
 233 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const string& resource, param_vec_t *params,
 234                                    optional_yield y, T& t)
 235 {
 236   return get_json_resource(dpp, resource, params, nullptr, y, t);
 237 }
 238
 239 template<class T>
 240 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const string& resource, const rgw_http_param_pair *pp,
 241                                    optional_yield y, T& t)
 242 {
 243   param_vec_t params = make_param_list(pp);
 244   return get_json_resource(dpp, resource, &params, y, t);
 245 }
 246
 247 class RGWStreamIntoBufferlist : public RGWHTTPStreamRWRequest::ReceiveCB {
 248   bufferlist& bl;
 249 public:
 250   explicit RGWStreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {}
 251   int handle_data(bufferlist& inbl, bool *pause) override {
 252     bl.claim_append(inbl);
 253     return inbl.length();
 254   }
 255 };
 256
 257 class RGWRESTReadResource : public RefCountedObject, public RGWIOProvider {
 258   CephContext *cct;
 259   RGWRESTConn *conn;
 260   string resource;
 261   param_vec_t params;
 262   map<string, string> headers;
 263   bufferlist bl;
 264   RGWStreamIntoBufferlist cb;
 265
 266   RGWHTTPManager *mgr;
 267   RGWRESTStreamReadRequest req;
 268
 269   void init_common(param_vec_t *extra_headers);
 270
 271 public:
 272   RGWRESTReadResource(RGWRESTConn *_conn,
 273                       const string& _resource,
 274                       const rgw_http_param_pair *pp,
 275                       param_vec_t *extra_headers,
 276                       RGWHTTPManager *_mgr);
 277
 278   RGWRESTReadResource(RGWRESTConn *_conn,
 279                       const string& _resource,
 280                       param_vec_t& _params,
 281                       param_vec_t *extra_headers,
 282                       RGWHTTPManager *_mgr);
 283   ~RGWRESTReadResource() = default;
 284
 285   rgw_io_id get_io_id(int io_type) {
 286     return req.get_io_id(io_type);
 287   }
 288
 289   void set_io_user_info(void *user_info) override {
 290     req.set_io_user_info(user_info);
 291   }
 292
 293   void *get_io_user_info() override {
 294     return req.get_io_user_info();
 295   }
 296
 297   template <class T>
 298   int decode_resource(T *dest);
 299
 300   int read(const DoutPrefixProvider *dpp, optional_yield y);
 301
 302   int aio_read(const DoutPrefixProvider *dpp);
 303
 304   string to_str() {
 305     return req.to_str();
 306   }
 307
 308   int get_http_status() {
 309     return req.get_http_status();
 310   }
 311
 312   int wait(bufferlist *pbl, optional_yield y) {
 313     int ret = req.wait(y);
 314     if (ret < 0) {
 315       return ret;
 316     }
 317
 318     if (req.get_status() < 0) {
 319       return req.get_status();
 320     }
 321     *pbl = bl;
 322     return 0;
 323   }
 324
 325   template <class T>
 326   int wait(T *dest, optional_yield y);
 327
 328   template <class T>
 329   int fetch(const DoutPrefixProvider *dpp, T *dest, optional_yield y);
 330 };
 331
 332
 333 template <class T>
 334 int RGWRESTReadResource::decode_resource(T *dest)
 335 {
 336   int ret = req.get_status();
 337   if (ret < 0) {
 338     return ret;
 339   }
 340   ret = parse_decode_json(*dest, bl);
 341   if (ret < 0) {
 342     return ret;
 343   }
 344   return 0;
 345 }
 346
 347 template <class T>
 348 int RGWRESTReadResource::fetch(const DoutPrefixProvider *dpp, T *dest, optional_yield y)
 349 {
 350   int ret = read(dpp, y);
 351   if (ret < 0) {
 352     return ret;
 353   }
 354
 355   ret = decode_resource(dest);
 356   if (ret < 0) {
 357     return ret;
 358   }
 359   return 0;
 360 }
 361
 362 template <class T>
 363 int RGWRESTReadResource::wait(T *dest, optional_yield y)
 364 {
 365   int ret = req.wait(y);
 366   if (ret < 0) {
 367     return ret;
 368   }
 369
 370   ret = decode_resource(dest);
 371   if (ret < 0) {
 372     return ret;
 373   }
 374   return 0;
 375 }
 376
 377 class RGWRESTSendResource : public RefCountedObject, public RGWIOProvider {
 378   CephContext *cct;
 379   RGWRESTConn *conn;
 380   string method;
 381   string resource;
 382   param_vec_t params;
 383   map<string, string> headers;
 384   bufferlist bl;
 385   RGWStreamIntoBufferlist cb;
 386
 387   RGWHTTPManager *mgr;
 388   RGWRESTStreamRWRequest req;
 389
 390   void init_common(param_vec_t *extra_headers);
 391
 392 public:
 393   RGWRESTSendResource(RGWRESTConn *_conn,
 394                       const string& _method,
 395                       const string& _resource,
 396                       const rgw_http_param_pair *pp,
 397                       param_vec_t *extra_headers,
 398                       RGWHTTPManager *_mgr);
 399
 400   RGWRESTSendResource(RGWRESTConn *_conn,
 401                       const string& _method,
 402                       const string& _resource,
 403                       param_vec_t& params,
 404                       param_vec_t *extra_headers,
 405                       RGWHTTPManager *_mgr);
 406
 407   ~RGWRESTSendResource() = default;
 408
 409   rgw_io_id get_io_id(int io_type) {
 410     return req.get_io_id(io_type);
 411   }
 412
 413   void set_io_user_info(void *user_info) override {
 414     req.set_io_user_info(user_info);
 415   }
 416
 417   void *get_io_user_info() override {
 418     return req.get_io_user_info();
 419   }
 420
 421   int send(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y);
 422
 423   int aio_send(const DoutPrefixProvider *dpp, bufferlist& bl);
 424
 425   string to_str() {
 426     return req.to_str();
 427   }
 428
 429   int get_http_status() {
 430     return req.get_http_status();
 431   }
 432
 433   template <class E = int>
 434   int wait(bufferlist *pbl, optional_yield y, E *err_result = nullptr) {
 435     int ret = req.wait(y);
 436     *pbl = bl;
 437
 438     if (ret < 0 && err_result ) {
 439       ret = parse_decode_json(*err_result, bl);
 440     }
 441
 442     return req.get_status();
 443   }
 444
 445   template <class T, class E = int>
 446   int wait(T *dest, optional_yield y, E *err_result = nullptr);
 447 };
 448
 449 template <class T, class E>
 450 int RGWRESTSendResource::wait(T *dest, optional_yield y, E *err_result)
 451 {
 452   int ret = req.wait(y);
 453   if (ret >= 0) {
 454     ret = req.get_status();
 455   }
 456
 457   if (ret < 0 && err_result) {
 458     ret = parse_decode_json(*err_result, bl);
 459   }
 460
 461   if (ret < 0) {
 462     return ret;
 463   }
 464
 465   ret = parse_decode_json(*dest, bl);
 466   if (ret < 0) {
 467     return ret;
 468   }
 469   return 0;
 470
 471 }
 472
 473 class RGWRESTPostResource : public RGWRESTSendResource {
 474 public:
 475   RGWRESTPostResource(RGWRESTConn *_conn,
 476                       const string& _resource,
 477                       const rgw_http_param_pair *pp,
 478                       param_vec_t *extra_headers,
 479                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 480                                                                   pp, extra_headers, _mgr) {}
 481
 482   RGWRESTPostResource(RGWRESTConn *_conn,
 483                       const string& _resource,
 484                       param_vec_t& params,
 485                       param_vec_t *extra_headers,
 486                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 487                                                                   params, extra_headers, _mgr) {}
 488
 489 };
 490
 491 class RGWRESTPutResource : public RGWRESTSendResource {
 492 public:
 493   RGWRESTPutResource(RGWRESTConn *_conn,
 494                      const string& _resource,
 495                      const rgw_http_param_pair *pp,
 496                      param_vec_t *extra_headers,
 497                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 498                                                                   pp, extra_headers, _mgr) {}
 499
 500   RGWRESTPutResource(RGWRESTConn *_conn,
 501                      const string& _resource,
 502                      param_vec_t& params,
 503                      param_vec_t *extra_headers,
 504                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 505                                                                   params, extra_headers, _mgr) {}
 506
 507 };
 508
 509 class RGWRESTDeleteResource : public RGWRESTSendResource {
 510 public:
 511   RGWRESTDeleteResource(RGWRESTConn *_conn,
 512                      const string& _resource,
 513                      const rgw_http_param_pair *pp,
 514                      param_vec_t *extra_headers,
 515                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 516                                                                   pp, extra_headers, _mgr) {}
 517
 518   RGWRESTDeleteResource(RGWRESTConn *_conn,
 519                      const string& _resource,
 520                      param_vec_t& params,
 521                      param_vec_t *extra_headers,
 522                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 523                                                                   params, extra_headers, _mgr) {}
 524
 525 };