ceph/src/rgw/rgw_rest_conn.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #pragma once
   5
   6 #include "rgw_rest_client.h"
   7 #include "common/ceph_json.h"
   8 #include "common/RefCountedObj.h"
   9 #include "include/common_fwd.h"
  10
  11 #include <atomic>
  12
  13 namespace rgw { namespace sal {
  14   class Store;
  15 } }
  16
  17 class RGWSI_Zone;
  18
  19 template <class T>
  20 static int parse_decode_json(T& t, bufferlist& bl)
  21 {
  22   JSONParser p;
  23   if (!p.parse(bl.c_str(), bl.length())) {
  24     return -EINVAL;
  25   }
  26
  27   try {
  28     decode_json_obj(t, &p);
  29   } catch (JSONDecoder::err& e) {
  30     return -EINVAL;
  31   }
  32   return 0;
  33 }
  34
  35 struct rgw_http_param_pair {
  36   const char *key;
  37   const char *val;
  38 };
  39
  40 // append a null-terminated rgw_http_param_pair list into a list of string pairs
  41 inline void append_param_list(param_vec_t& params, const rgw_http_param_pair* pp)
  42 {
  43   while (pp && pp->key) {
  44     std::string k = pp->key;
  45     std::string v = (pp->val ? pp->val : "");
  46     params.emplace_back(make_pair(std::move(k), std::move(v)));
  47     ++pp;
  48   }
  49 }
  50
  51 // copy a null-terminated rgw_http_param_pair list into a list of std::string pairs
  52 inline param_vec_t make_param_list(const rgw_http_param_pair* pp)
  53 {
  54   param_vec_t params;
  55   append_param_list(params, pp);
  56   return params;
  57 }
  58
  59 inline param_vec_t make_param_list(const std::map<std::string, std::string> *pp)
  60 {
  61   param_vec_t params;
  62   if (!pp) {
  63     return params;
  64   }
  65   for (auto iter : *pp) {
  66     params.emplace_back(make_pair(iter.first, iter.second));
  67   }
  68   return params;
  69 }
  70
  71 class RGWRESTConn
  72 {
  73   CephContext *cct;
  74   std::vector<std::string> endpoints;
  75   RGWAccessKey key;
  76   std::string self_zone_group;
  77   std::string remote_id;
  78   std::optional<std::string> api_name;
  79   HostStyle host_style;
  80   std::atomic<int64_t> counter = { 0 };
  81
  82 public:
  83
  84   RGWRESTConn(CephContext *_cct,
  85               RGWSI_Zone *zone_svc,
  86               const std::string& _remote_id,
  87               const std::list<std::string>& endpoints,
  88               std::optional<std::string> _api_name,
  89               HostStyle _host_style = PathStyle);
  90   RGWRESTConn(CephContext *_cct,
  91               rgw::sal::Store* store,
  92               const std::string& _remote_id,
  93               const std::list<std::string>& endpoints,
  94               std::optional<std::string> _api_name,
  95               HostStyle _host_style = PathStyle);
  96   RGWRESTConn(CephContext *_cct,
  97               RGWSI_Zone *zone_svc,
  98               const std::string& _remote_id,
  99               const std::list<std::string>& endpoints,
 100               RGWAccessKey _cred,
 101               std::optional<std::string> _api_name,
 102               HostStyle _host_style = PathStyle);
 103   RGWRESTConn(CephContext *_cct,
 104               rgw::sal::Store* store,
 105               const std::string& _remote_id,
 106               const std::list<std::string>& endpoints,
 107               RGWAccessKey _cred,
 108               std::optional<std::string> _api_name,
 109               HostStyle _host_style = PathStyle);
 110
 111   // custom move needed for atomic
 112   RGWRESTConn(RGWRESTConn&& other);
 113   RGWRESTConn& operator=(RGWRESTConn&& other);
 114   virtual ~RGWRESTConn() = default;
 115
 116   int get_url(std::string& endpoint);
 117   std::string get_url();
 118   const std::string& get_self_zonegroup() {
 119     return self_zone_group;
 120   }
 121   const std::string& get_remote_id() {
 122     return remote_id;
 123   }
 124   RGWAccessKey& get_key() {
 125     return key;
 126   }
 127
 128   std::optional<std::string> get_api_name() const {
 129     return api_name;
 130   }
 131
 132   HostStyle get_host_style() {
 133     return host_style;
 134   }
 135
 136   CephContext *get_ctx() {
 137     return cct;
 138   }
 139   size_t get_endpoint_count() const { return endpoints.size(); }
 140
 141   virtual void populate_params(param_vec_t& params, const rgw_user *uid, const std::string& zonegroup);
 142
 143   /* sync request */
 144   int forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 145
 146
 147   /* async requests */
 148   int put_obj_send_init(rgw::sal::Object* obj, const rgw_http_param_pair *extra_params, RGWRESTStreamS3PutObj **req);
 149   int put_obj_async_init(const DoutPrefixProvider *dpp, const rgw_user& uid, rgw::sal::Object* obj,
 150                          std::map<std::string, bufferlist>& attrs, RGWRESTStreamS3PutObj **req);
 151   int complete_request(RGWRESTStreamS3PutObj *req, std::string& etag,
 152                        ceph::real_time *mtime, optional_yield y);
 153
 154   struct get_obj_params {
 155     rgw_user uid;
 156     req_info *info{nullptr};
 157     const ceph::real_time *mod_ptr{nullptr};
 158     const ceph::real_time *unmod_ptr{nullptr};
 159     bool high_precision_time{true};
 160
 161     std::string etag;
 162
 163     uint32_t mod_zone_id{0};
 164     uint64_t mod_pg_ver{0};
 165
 166     bool prepend_metadata{false};
 167     bool get_op{false};
 168     bool rgwx_stat{false};
 169     bool sync_manifest{false};
 170
 171     bool skip_decrypt{true};
 172     RGWHTTPStreamRWRequest::ReceiveCB *cb{nullptr};
 173
 174     bool range_is_set{false};
 175     uint64_t range_start{0};
 176     uint64_t range_end{0};
 177   };
 178
 179   int get_obj(const DoutPrefixProvider *dpp, const rgw::sal::Object* obj, const get_obj_params& params, bool send, RGWRESTStreamRWRequest **req);
 180
 181   int get_obj(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info *info /* optional */, const rgw::sal::Object* obj,
 182               const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr,
 183               uint32_t mod_zone_id, uint64_t mod_pg_ver,
 184               bool prepend_metadata, bool get_op, bool rgwx_stat, bool sync_manifest,
 185               bool skip_decrypt, bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req);
 186   int complete_request(RGWRESTStreamRWRequest *req,
 187                        std::string *etag,
 188                        ceph::real_time *mtime,
 189                        uint64_t *psize,
 190                        std::map<std::string, std::string> *pattrs,
 191                        std::map<std::string, std::string> *pheaders,
 192                        optional_yield y);
 193
 194   int get_resource(const DoutPrefixProvider *dpp,
 195                    const std::string& resource,
 196                    param_vec_t *extra_params,
 197                    std::map<std::string, std::string>* extra_headers,
 198                    bufferlist& bl,
 199                    bufferlist *send_data,
 200                    RGWHTTPManager *mgr,
 201                    optional_yield y);
 202
 203   int send_resource(const DoutPrefixProvider *dpp,
 204                    const std::string& method,
 205                    const std::string& resource,
 206                            rgw_http_param_pair *extra_params,
 207                    std::map<std::string, std::string>* extra_headers,
 208                    bufferlist& bl,
 209                    bufferlist *send_data,
 210                    RGWHTTPManager *mgr,
 211                    optional_yield y);
 212
 213   template <class T>
 214   int get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 215                         bufferlist *in_data, optional_yield y, T& t);
 216   template <class T>
 217   int get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 218                         optional_yield y, T& t);
 219   template <class T>
 220   int get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, const rgw_http_param_pair *pp,
 221                         optional_yield y, T& t);
 222
 223 private:
 224   void populate_zonegroup(param_vec_t& params, const std::string& zonegroup) {
 225     if (!zonegroup.empty()) {
 226       params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
 227     }
 228   }
 229   void populate_uid(param_vec_t& params, const rgw_user *uid) {
 230     if (uid) {
 231       std::string uid_str = uid->to_str();
 232       if (!uid->empty()){
 233         params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
 234       }
 235     }
 236   }
 237 };
 238
 239 class S3RESTConn : public RGWRESTConn {
 240
 241 public:
 242
 243   S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const std::string& _remote_id, const std::list<std::string>& endpoints, std::optional<std::string> _api_name, HostStyle _host_style = PathStyle) :
 244     RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _api_name, _host_style) {}
 245
 246   S3RESTConn(CephContext *_cct, rgw::sal::Store* store, const std::string& _remote_id, const std::list<std::string>& endpoints, std::optional<std::string> _api_name, HostStyle _host_style = PathStyle) :
 247     RGWRESTConn(_cct, store, _remote_id, endpoints, _api_name, _host_style) {}
 248
 249   S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const std::string& _remote_id, const std::list<std::string>& endpoints, RGWAccessKey _cred, std::optional<std::string> _api_name, HostStyle _host_style = PathStyle):
 250     RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _cred, _api_name, _host_style) {}
 251
 252   S3RESTConn(CephContext *_cct, rgw::sal::Store* store, const std::string& _remote_id, const std::list<std::string>& endpoints, RGWAccessKey _cred, std::optional<std::string> _api_name, HostStyle _host_style = PathStyle):
 253     RGWRESTConn(_cct, store, _remote_id, endpoints, _cred, _api_name, _host_style) {}
 254   ~S3RESTConn() override = default;
 255
 256   void populate_params(param_vec_t& params, const rgw_user *uid, const std::string& zonegroup) override {
 257     // do not populate any params in S3 REST Connection.
 258     return;
 259   }
 260 };
 261
 262
 263 template<class T>
 264 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 265                                    bufferlist *in_data, optional_yield y, T& t)
 266 {
 267   bufferlist bl;
 268   int ret = get_resource(dpp, resource, params, nullptr, bl, in_data, nullptr, y);
 269   if (ret < 0) {
 270     return ret;
 271   }
 272
 273   ret = parse_decode_json(t, bl);
 274   if (ret < 0) {
 275     return ret;
 276   }
 277
 278   return 0;
 279 }
 280
 281 template<class T>
 282 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 283                                    optional_yield y, T& t)
 284 {
 285   return get_json_resource(dpp, resource, params, nullptr, y, t);
 286 }
 287
 288 template<class T>
 289 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, const rgw_http_param_pair *pp,
 290                                    optional_yield y, T& t)
 291 {
 292   param_vec_t params = make_param_list(pp);
 293   return get_json_resource(dpp, resource, &params, y, t);
 294 }
 295
 296 class RGWStreamIntoBufferlist : public RGWHTTPStreamRWRequest::ReceiveCB {
 297   bufferlist& bl;
 298 public:
 299   explicit RGWStreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {}
 300   int handle_data(bufferlist& inbl, bool *pause) override {
 301     bl.claim_append(inbl);
 302     return inbl.length();
 303   }
 304 };
 305
 306 class RGWRESTReadResource : public RefCountedObject, public RGWIOProvider {
 307   CephContext *cct;
 308   RGWRESTConn *conn;
 309   std::string resource;
 310   param_vec_t params;
 311   std::map<std::string, std::string> headers;
 312   bufferlist bl;
 313   RGWStreamIntoBufferlist cb;
 314
 315   RGWHTTPManager *mgr;
 316   RGWRESTStreamReadRequest req;
 317
 318   void init_common(param_vec_t *extra_headers);
 319
 320 public:
 321   RGWRESTReadResource(RGWRESTConn *_conn,
 322                       const std::string& _resource,
 323                       const rgw_http_param_pair *pp,
 324                       param_vec_t *extra_headers,
 325                       RGWHTTPManager *_mgr);
 326
 327   RGWRESTReadResource(RGWRESTConn *_conn,
 328                       const std::string& _resource,
 329                       param_vec_t& _params,
 330                       param_vec_t *extra_headers,
 331                       RGWHTTPManager *_mgr);
 332   ~RGWRESTReadResource() = default;
 333
 334   rgw_io_id get_io_id(int io_type) {
 335     return req.get_io_id(io_type);
 336   }
 337
 338   void set_io_user_info(void *user_info) override {
 339     req.set_io_user_info(user_info);
 340   }
 341
 342   void *get_io_user_info() override {
 343     return req.get_io_user_info();
 344   }
 345
 346   template <class T>
 347   int decode_resource(T *dest);
 348
 349   int read(const DoutPrefixProvider *dpp, optional_yield y);
 350
 351   int aio_read(const DoutPrefixProvider *dpp);
 352
 353   std::string to_str() {
 354     return req.to_str();
 355   }
 356
 357   int get_http_status() {
 358     return req.get_http_status();
 359   }
 360
 361   int wait(bufferlist *pbl, optional_yield y) {
 362     int ret = req.wait(y);
 363     if (ret < 0) {
 364       return ret;
 365     }
 366
 367     if (req.get_status() < 0) {
 368       return req.get_status();
 369     }
 370     *pbl = bl;
 371     return 0;
 372   }
 373
 374   template <class T>
 375   int wait(T *dest, optional_yield y);
 376
 377   template <class T>
 378   int fetch(const DoutPrefixProvider *dpp, T *dest, optional_yield y);
 379 };
 380
 381
 382 template <class T>
 383 int RGWRESTReadResource::decode_resource(T *dest)
 384 {
 385   int ret = req.get_status();
 386   if (ret < 0) {
 387     return ret;
 388   }
 389   ret = parse_decode_json(*dest, bl);
 390   if (ret < 0) {
 391     return ret;
 392   }
 393   return 0;
 394 }
 395
 396 template <class T>
 397 int RGWRESTReadResource::fetch(const DoutPrefixProvider *dpp, T *dest, optional_yield y)
 398 {
 399   int ret = read(dpp, y);
 400   if (ret < 0) {
 401     return ret;
 402   }
 403
 404   ret = decode_resource(dest);
 405   if (ret < 0) {
 406     return ret;
 407   }
 408   return 0;
 409 }
 410
 411 template <class T>
 412 int RGWRESTReadResource::wait(T *dest, optional_yield y)
 413 {
 414   int ret = req.wait(y);
 415   if (ret < 0) {
 416     return ret;
 417   }
 418
 419   ret = decode_resource(dest);
 420   if (ret < 0) {
 421     return ret;
 422   }
 423   return 0;
 424 }
 425
 426 class RGWRESTSendResource : public RefCountedObject, public RGWIOProvider {
 427   CephContext *cct;
 428   RGWRESTConn *conn;
 429   std::string method;
 430   std::string resource;
 431   param_vec_t params;
 432   std::map<std::string, std::string> headers;
 433   bufferlist bl;
 434   RGWStreamIntoBufferlist cb;
 435
 436   RGWHTTPManager *mgr;
 437   RGWRESTStreamRWRequest req;
 438
 439   void init_common(param_vec_t *extra_headers);
 440
 441 public:
 442   RGWRESTSendResource(RGWRESTConn *_conn,
 443                       const std::string& _method,
 444                       const std::string& _resource,
 445                       const rgw_http_param_pair *pp,
 446                       param_vec_t *extra_headers,
 447                       RGWHTTPManager *_mgr);
 448
 449   RGWRESTSendResource(RGWRESTConn *_conn,
 450                       const std::string& _method,
 451                       const std::string& _resource,
 452                       param_vec_t& params,
 453                       param_vec_t *extra_headers,
 454                       RGWHTTPManager *_mgr);
 455
 456   ~RGWRESTSendResource() = default;
 457
 458   rgw_io_id get_io_id(int io_type) {
 459     return req.get_io_id(io_type);
 460   }
 461
 462   void set_io_user_info(void *user_info) override {
 463     req.set_io_user_info(user_info);
 464   }
 465
 466   void *get_io_user_info() override {
 467     return req.get_io_user_info();
 468   }
 469
 470   int send(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y);
 471
 472   int aio_send(const DoutPrefixProvider *dpp, bufferlist& bl);
 473
 474   std::string to_str() {
 475     return req.to_str();
 476   }
 477
 478   int get_http_status() {
 479     return req.get_http_status();
 480   }
 481
 482   template <class E = int>
 483   int wait(bufferlist *pbl, optional_yield y, E *err_result = nullptr) {
 484     int ret = req.wait(y);
 485     *pbl = bl;
 486
 487     if (ret < 0 && err_result ) {
 488       ret = parse_decode_json(*err_result, bl);
 489     }
 490
 491     return req.get_status();
 492   }
 493
 494   template <class T, class E = int>
 495   int wait(T *dest, optional_yield y, E *err_result = nullptr);
 496 };
 497
 498 template <class T, class E>
 499 int RGWRESTSendResource::wait(T *dest, optional_yield y, E *err_result)
 500 {
 501   int ret = req.wait(y);
 502   if (ret >= 0) {
 503     ret = req.get_status();
 504   }
 505
 506   if (ret < 0 && err_result) {
 507     ret = parse_decode_json(*err_result, bl);
 508   }
 509
 510   if (ret < 0) {
 511     return ret;
 512   }
 513
 514   ret = parse_decode_json(*dest, bl);
 515   if (ret < 0) {
 516     return ret;
 517   }
 518   return 0;
 519
 520 }
 521
 522 class RGWRESTPostResource : public RGWRESTSendResource {
 523 public:
 524   RGWRESTPostResource(RGWRESTConn *_conn,
 525                       const std::string& _resource,
 526                       const rgw_http_param_pair *pp,
 527                       param_vec_t *extra_headers,
 528                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 529                                                                   pp, extra_headers, _mgr) {}
 530
 531   RGWRESTPostResource(RGWRESTConn *_conn,
 532                       const std::string& _resource,
 533                       param_vec_t& params,
 534                       param_vec_t *extra_headers,
 535                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 536                                                                   params, extra_headers, _mgr) {}
 537
 538 };
 539
 540 class RGWRESTPutResource : public RGWRESTSendResource {
 541 public:
 542   RGWRESTPutResource(RGWRESTConn *_conn,
 543                      const std::string& _resource,
 544                      const rgw_http_param_pair *pp,
 545                      param_vec_t *extra_headers,
 546                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 547                                                                   pp, extra_headers, _mgr) {}
 548
 549   RGWRESTPutResource(RGWRESTConn *_conn,
 550                      const std::string& _resource,
 551                      param_vec_t& params,
 552                      param_vec_t *extra_headers,
 553                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 554                                                                   params, extra_headers, _mgr) {}
 555
 556 };
 557
 558 class RGWRESTDeleteResource : public RGWRESTSendResource {
 559 public:
 560   RGWRESTDeleteResource(RGWRESTConn *_conn,
 561                      const std::string& _resource,
 562                      const rgw_http_param_pair *pp,
 563                      param_vec_t *extra_headers,
 564                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 565                                                                   pp, extra_headers, _mgr) {}
 566
 567   RGWRESTDeleteResource(RGWRESTConn *_conn,
 568                      const std::string& _resource,
 569                      param_vec_t& params,
 570                      param_vec_t *extra_headers,
 571                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 572                                                                   params, extra_headers, _mgr) {}
 573
 574 };