ceph/src/rgw/rgw_rest_conn.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #pragma once
   5
   6 #include "rgw_rest_client.h"
   7 #include "common/ceph_json.h"
   8 #include "common/RefCountedObj.h"
   9 #include "include/common_fwd.h"
  10 #include "rgw_sal_fwd.h"
  11
  12 #include <atomic>
  13
  14 class RGWSI_Zone;
  15
  16 template<class T>
  17 inline int parse_decode_json(T& t, bufferlist& bl)
  18 {
  19   JSONParser p;
  20   if (!p.parse(bl.c_str(), bl.length())) {
  21     return -EINVAL;
  22   }
  23
  24   try {
  25     decode_json_obj(t, &p);
  26   } catch (JSONDecoder::err& e) {
  27     return -EINVAL;
  28   }
  29   return 0;
  30 }
  31
  32 struct rgw_http_param_pair {
  33   const char *key;
  34   const char *val;
  35 };
  36
  37 // append a null-terminated rgw_http_param_pair list into a list of string pairs
  38 inline void append_param_list(param_vec_t& params, const rgw_http_param_pair* pp)
  39 {
  40   while (pp && pp->key) {
  41     std::string k = pp->key;
  42     std::string v = (pp->val ? pp->val : "");
  43     params.emplace_back(make_pair(std::move(k), std::move(v)));
  44     ++pp;
  45   }
  46 }
  47
  48 // copy a null-terminated rgw_http_param_pair list into a list of std::string pairs
  49 inline param_vec_t make_param_list(const rgw_http_param_pair* pp)
  50 {
  51   param_vec_t params;
  52   append_param_list(params, pp);
  53   return params;
  54 }
  55
  56 inline param_vec_t make_param_list(const std::map<std::string, std::string> *pp)
  57 {
  58   param_vec_t params;
  59   if (!pp) {
  60     return params;
  61   }
  62   for (auto iter : *pp) {
  63     params.emplace_back(make_pair(iter.first, iter.second));
  64   }
  65   return params;
  66 }
  67
  68 class RGWRESTConn
  69 {
  70   CephContext *cct;
  71   std::vector<std::string> endpoints;
  72   RGWAccessKey key;
  73   std::string self_zone_group;
  74   std::string remote_id;
  75   std::optional<std::string> api_name;
  76   HostStyle host_style;
  77   std::atomic<int64_t> counter = { 0 };
  78
  79 public:
  80
  81   RGWRESTConn(CephContext *_cct,
  82               rgw::sal::Driver* driver,
  83               const std::string& _remote_id,
  84               const std::list<std::string>& endpoints,
  85               std::optional<std::string> _api_name,
  86               HostStyle _host_style = PathStyle);
  87   RGWRESTConn(CephContext *_cct,
  88               const std::string& _remote_id,
  89               const std::list<std::string>& endpoints,
  90               RGWAccessKey _cred,
  91               std::string _zone_group,
  92               std::optional<std::string> _api_name,
  93               HostStyle _host_style = PathStyle);
  94
  95   // custom move needed for atomic
  96   RGWRESTConn(RGWRESTConn&& other);
  97   RGWRESTConn& operator=(RGWRESTConn&& other);
  98   virtual ~RGWRESTConn() = default;
  99
 100   int get_url(std::string& endpoint);
 101   std::string get_url();
 102   const std::string& get_self_zonegroup() {
 103     return self_zone_group;
 104   }
 105   const std::string& get_remote_id() {
 106     return remote_id;
 107   }
 108   RGWAccessKey& get_key() {
 109     return key;
 110   }
 111
 112   std::optional<std::string> get_api_name() const {
 113     return api_name;
 114   }
 115
 116   HostStyle get_host_style() {
 117     return host_style;
 118   }
 119
 120   CephContext *get_ctx() {
 121     return cct;
 122   }
 123   size_t get_endpoint_count() const { return endpoints.size(); }
 124
 125   virtual void populate_params(param_vec_t& params, const rgw_user *uid, const std::string& zonegroup);
 126
 127   /* sync request */
 128   int forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 129
 130   /* sync request */
 131   int forward_iam_request(const DoutPrefixProvider *dpp, const RGWAccessKey& key, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y);
 132
 133
 134   /* async requests */
 135   int put_obj_send_init(const rgw_obj& obj, const rgw_http_param_pair *extra_params, RGWRESTStreamS3PutObj **req);
 136   int put_obj_async_init(const DoutPrefixProvider *dpp, const rgw_user& uid, const rgw_obj& obj,
 137                          std::map<std::string, bufferlist>& attrs, RGWRESTStreamS3PutObj **req);
 138   int complete_request(RGWRESTStreamS3PutObj *req, std::string& etag,
 139                        ceph::real_time *mtime, optional_yield y);
 140
 141   struct get_obj_params {
 142     rgw_user uid;
 143     req_info *info{nullptr};
 144     const ceph::real_time *mod_ptr{nullptr};
 145     const ceph::real_time *unmod_ptr{nullptr};
 146     bool high_precision_time{true};
 147
 148     std::string etag;
 149
 150     uint32_t mod_zone_id{0};
 151     uint64_t mod_pg_ver{0};
 152
 153     bool prepend_metadata{false};
 154     bool get_op{false};
 155     bool rgwx_stat{false};
 156     bool sync_manifest{false};
 157     bool sync_cloudtiered{false};
 158
 159     bool skip_decrypt{true};
 160     RGWHTTPStreamRWRequest::ReceiveCB *cb{nullptr};
 161
 162     bool range_is_set{false};
 163     uint64_t range_start{0};
 164     uint64_t range_end{0};
 165     rgw_zone_set_entry *dst_zone_trace{nullptr};
 166   };
 167
 168   int get_obj(const DoutPrefixProvider *dpp, const rgw_obj& obj, const get_obj_params& params, bool send, RGWRESTStreamRWRequest **req);
 169
 170   int get_obj(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info *info /* optional */, const rgw_obj& obj,
 171               const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr,
 172               uint32_t mod_zone_id, uint64_t mod_pg_ver,
 173               bool prepend_metadata, bool get_op, bool rgwx_stat, bool sync_manifest,
 174               bool skip_decrypt, rgw_zone_set_entry *dst_zone_trace, bool sync_cloudtiered,
 175               bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req);
 176   int complete_request(RGWRESTStreamRWRequest *req,
 177                        std::string *etag,
 178                        ceph::real_time *mtime,
 179                        uint64_t *psize,
 180                        std::map<std::string, std::string> *pattrs,
 181                        std::map<std::string, std::string> *pheaders,
 182                        optional_yield y);
 183
 184   int get_resource(const DoutPrefixProvider *dpp,
 185                    const std::string& resource,
 186                    param_vec_t *extra_params,
 187                    std::map<std::string, std::string>* extra_headers,
 188                    bufferlist& bl,
 189                    bufferlist *send_data,
 190                    RGWHTTPManager *mgr,
 191                    optional_yield y);
 192
 193   int send_resource(const DoutPrefixProvider *dpp,
 194                    const std::string& method,
 195                    const std::string& resource,
 196                            rgw_http_param_pair *extra_params,
 197                    std::map<std::string, std::string>* extra_headers,
 198                    bufferlist& bl,
 199                    bufferlist *send_data,
 200                    RGWHTTPManager *mgr,
 201                    optional_yield y);
 202
 203   template <class T>
 204   int get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 205                         bufferlist *in_data, optional_yield y, T& t);
 206   template <class T>
 207   int get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 208                         optional_yield y, T& t);
 209   template <class T>
 210   int get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, const rgw_http_param_pair *pp,
 211                         optional_yield y, T& t);
 212
 213 private:
 214   void populate_zonegroup(param_vec_t& params, const std::string& zonegroup) {
 215     if (!zonegroup.empty()) {
 216       params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "zonegroup", zonegroup));
 217     }
 218   }
 219   void populate_uid(param_vec_t& params, const rgw_user *uid) {
 220     if (uid) {
 221       std::string uid_str = uid->to_str();
 222       if (!uid->empty()){
 223         params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "uid", uid_str));
 224       }
 225     }
 226   }
 227 };
 228
 229 class S3RESTConn : public RGWRESTConn {
 230
 231 public:
 232
 233   S3RESTConn(CephContext *_cct, rgw::sal::Driver* driver, const std::string& _remote_id, const std::list<std::string>& endpoints, std::optional<std::string> _api_name, HostStyle _host_style = PathStyle) :
 234     RGWRESTConn(_cct, driver, _remote_id, endpoints, _api_name, _host_style) {}
 235   S3RESTConn(CephContext *_cct, const std::string& _remote_id, const std::list<std::string>& endpoints, RGWAccessKey _cred, std::string _zone_group, std::optional<std::string> _api_name, HostStyle _host_style = PathStyle):
 236     RGWRESTConn(_cct, _remote_id, endpoints, _cred, _zone_group, _api_name, _host_style) {}
 237   ~S3RESTConn() override = default;
 238
 239   void populate_params(param_vec_t& params, const rgw_user *uid, const std::string& zonegroup) override {
 240     // do not populate any params in S3 REST Connection.
 241     return;
 242   }
 243 };
 244
 245
 246 template<class T>
 247 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 248                                    bufferlist *in_data, optional_yield y, T& t)
 249 {
 250   bufferlist bl;
 251   int ret = get_resource(dpp, resource, params, nullptr, bl, in_data, nullptr, y);
 252   if (ret < 0) {
 253     return ret;
 254   }
 255
 256   ret = parse_decode_json(t, bl);
 257   if (ret < 0) {
 258     return ret;
 259   }
 260
 261   return 0;
 262 }
 263
 264 template<class T>
 265 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, param_vec_t *params,
 266                                    optional_yield y, T& t)
 267 {
 268   return get_json_resource(dpp, resource, params, nullptr, y, t);
 269 }
 270
 271 template<class T>
 272 int RGWRESTConn::get_json_resource(const DoutPrefixProvider *dpp, const std::string& resource, const rgw_http_param_pair *pp,
 273                                    optional_yield y, T& t)
 274 {
 275   param_vec_t params = make_param_list(pp);
 276   return get_json_resource(dpp, resource, &params, y, t);
 277 }
 278
 279 class RGWStreamIntoBufferlist : public RGWHTTPStreamRWRequest::ReceiveCB {
 280   bufferlist& bl;
 281 public:
 282   explicit RGWStreamIntoBufferlist(bufferlist& _bl) : bl(_bl) {}
 283   int handle_data(bufferlist& inbl, bool *pause) override {
 284     bl.claim_append(inbl);
 285     return inbl.length();
 286   }
 287 };
 288
 289 class RGWRESTReadResource : public RefCountedObject, public RGWIOProvider {
 290   CephContext *cct;
 291   RGWRESTConn *conn;
 292   std::string resource;
 293   param_vec_t params;
 294   std::map<std::string, std::string> headers;
 295   bufferlist bl;
 296   RGWStreamIntoBufferlist cb;
 297
 298   RGWHTTPManager *mgr;
 299   RGWRESTStreamReadRequest req;
 300
 301   void init_common(param_vec_t *extra_headers);
 302
 303 public:
 304   RGWRESTReadResource(RGWRESTConn *_conn,
 305                       const std::string& _resource,
 306                       const rgw_http_param_pair *pp,
 307                       param_vec_t *extra_headers,
 308                       RGWHTTPManager *_mgr);
 309
 310   RGWRESTReadResource(RGWRESTConn *_conn,
 311                       const std::string& _resource,
 312                       param_vec_t& _params,
 313                       param_vec_t *extra_headers,
 314                       RGWHTTPManager *_mgr);
 315   ~RGWRESTReadResource() = default;
 316
 317   rgw_io_id get_io_id(int io_type) {
 318     return req.get_io_id(io_type);
 319   }
 320
 321   void set_io_user_info(void *user_info) override {
 322     req.set_io_user_info(user_info);
 323   }
 324
 325   void *get_io_user_info() override {
 326     return req.get_io_user_info();
 327   }
 328
 329   template <class T>
 330   int decode_resource(T *dest);
 331
 332   int read(const DoutPrefixProvider *dpp, optional_yield y);
 333
 334   int aio_read(const DoutPrefixProvider *dpp);
 335
 336   std::string to_str() {
 337     return req.to_str();
 338   }
 339
 340   int get_http_status() {
 341     return req.get_http_status();
 342   }
 343
 344   int wait(bufferlist *pbl, optional_yield y) {
 345     int ret = req.wait(y);
 346     if (ret < 0) {
 347       return ret;
 348     }
 349
 350     if (req.get_status() < 0) {
 351       return req.get_status();
 352     }
 353     *pbl = bl;
 354     return 0;
 355   }
 356
 357   template <class T>
 358   int wait(T *dest, optional_yield y);
 359
 360   template <class T>
 361   int fetch(const DoutPrefixProvider *dpp, T *dest, optional_yield y);
 362 };
 363
 364
 365 template <class T>
 366 int RGWRESTReadResource::decode_resource(T *dest)
 367 {
 368   int ret = req.get_status();
 369   if (ret < 0) {
 370     return ret;
 371   }
 372   ret = parse_decode_json(*dest, bl);
 373   if (ret < 0) {
 374     return ret;
 375   }
 376   return 0;
 377 }
 378
 379 template <class T>
 380 int RGWRESTReadResource::fetch(const DoutPrefixProvider *dpp, T *dest, optional_yield y)
 381 {
 382   int ret = read(dpp, y);
 383   if (ret < 0) {
 384     return ret;
 385   }
 386
 387   ret = decode_resource(dest);
 388   if (ret < 0) {
 389     return ret;
 390   }
 391   return 0;
 392 }
 393
 394 template <class T>
 395 int RGWRESTReadResource::wait(T *dest, optional_yield y)
 396 {
 397   int ret = req.wait(y);
 398   if (ret < 0) {
 399     return ret;
 400   }
 401
 402   ret = decode_resource(dest);
 403   if (ret < 0) {
 404     return ret;
 405   }
 406   return 0;
 407 }
 408
 409 class RGWRESTSendResource : public RefCountedObject, public RGWIOProvider {
 410   CephContext *cct;
 411   RGWRESTConn *conn;
 412   std::string method;
 413   std::string resource;
 414   param_vec_t params;
 415   std::map<std::string, std::string> headers;
 416   bufferlist bl;
 417   RGWStreamIntoBufferlist cb;
 418
 419   RGWHTTPManager *mgr;
 420   RGWRESTStreamRWRequest req;
 421
 422   void init_common(param_vec_t *extra_headers);
 423
 424 public:
 425   RGWRESTSendResource(RGWRESTConn *_conn,
 426                       const std::string& _method,
 427                       const std::string& _resource,
 428                       const rgw_http_param_pair *pp,
 429                       param_vec_t *extra_headers,
 430                       RGWHTTPManager *_mgr);
 431
 432   RGWRESTSendResource(RGWRESTConn *_conn,
 433                       const std::string& _method,
 434                       const std::string& _resource,
 435                       param_vec_t& params,
 436                       param_vec_t *extra_headers,
 437                       RGWHTTPManager *_mgr);
 438
 439   ~RGWRESTSendResource() = default;
 440
 441   rgw_io_id get_io_id(int io_type) {
 442     return req.get_io_id(io_type);
 443   }
 444
 445   void set_io_user_info(void *user_info) override {
 446     req.set_io_user_info(user_info);
 447   }
 448
 449   void *get_io_user_info() override {
 450     return req.get_io_user_info();
 451   }
 452
 453   int send(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y);
 454
 455   int aio_send(const DoutPrefixProvider *dpp, bufferlist& bl);
 456
 457   std::string to_str() {
 458     return req.to_str();
 459   }
 460
 461   int get_http_status() {
 462     return req.get_http_status();
 463   }
 464
 465   template <class E = int>
 466   int wait(bufferlist *pbl, optional_yield y, E *err_result = nullptr) {
 467     int ret = req.wait(y);
 468     *pbl = bl;
 469
 470     if (ret < 0 && err_result ) {
 471       ret = parse_decode_json(*err_result, bl);
 472     }
 473
 474     return req.get_status();
 475   }
 476
 477   template <class T, class E = int>
 478   int wait(T *dest, optional_yield y, E *err_result = nullptr);
 479 };
 480
 481 template <class T, class E>
 482 int RGWRESTSendResource::wait(T *dest, optional_yield y, E *err_result)
 483 {
 484   int ret = req.wait(y);
 485   if (ret >= 0) {
 486     ret = req.get_status();
 487   }
 488
 489   if (ret < 0 && err_result) {
 490     ret = parse_decode_json(*err_result, bl);
 491   }
 492
 493   if (ret < 0) {
 494     return ret;
 495   }
 496
 497   ret = parse_decode_json(*dest, bl);
 498   if (ret < 0) {
 499     return ret;
 500   }
 501   return 0;
 502
 503 }
 504
 505 class RGWRESTPostResource : public RGWRESTSendResource {
 506 public:
 507   RGWRESTPostResource(RGWRESTConn *_conn,
 508                       const std::string& _resource,
 509                       const rgw_http_param_pair *pp,
 510                       param_vec_t *extra_headers,
 511                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 512                                                                   pp, extra_headers, _mgr) {}
 513
 514   RGWRESTPostResource(RGWRESTConn *_conn,
 515                       const std::string& _resource,
 516                       param_vec_t& params,
 517                       param_vec_t *extra_headers,
 518                       RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "POST", _resource,
 519                                                                   params, extra_headers, _mgr) {}
 520
 521 };
 522
 523 class RGWRESTPutResource : public RGWRESTSendResource {
 524 public:
 525   RGWRESTPutResource(RGWRESTConn *_conn,
 526                      const std::string& _resource,
 527                      const rgw_http_param_pair *pp,
 528                      param_vec_t *extra_headers,
 529                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 530                                                                   pp, extra_headers, _mgr) {}
 531
 532   RGWRESTPutResource(RGWRESTConn *_conn,
 533                      const std::string& _resource,
 534                      param_vec_t& params,
 535                      param_vec_t *extra_headers,
 536                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "PUT", _resource,
 537                                                                   params, extra_headers, _mgr) {}
 538
 539 };
 540
 541 class RGWRESTDeleteResource : public RGWRESTSendResource {
 542 public:
 543   RGWRESTDeleteResource(RGWRESTConn *_conn,
 544                      const std::string& _resource,
 545                      const rgw_http_param_pair *pp,
 546                      param_vec_t *extra_headers,
 547                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 548                                                                   pp, extra_headers, _mgr) {}
 549
 550   RGWRESTDeleteResource(RGWRESTConn *_conn,
 551                      const std::string& _resource,
 552                      param_vec_t& params,
 553                      param_vec_t *extra_headers,
 554                      RGWHTTPManager *_mgr) : RGWRESTSendResource(_conn, "DELETE", _resource,
 555                                                                   params, extra_headers, _mgr) {}
 556
 557 };