ceph/src/rgw/rgw_rest_conn.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_zone.h"
   5 #include "rgw_rest_conn.h"
   6 #include "rgw_sal.h"
   7 #include "rgw_rados.h"
   8
   9 #include "services/svc_zone.h"
  10
  11 #define dout_subsys ceph_subsys_rgw
  12
  13 using namespace std;
  14
  15 RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc,
  16                          const string& _remote_id,
  17                          const list<string>& remote_endpoints,
  18                          std::optional<string> _api_name,
  19                          HostStyle _host_style)
  20   : cct(_cct),
  21     endpoints(remote_endpoints.begin(), remote_endpoints.end()),
  22     remote_id(_remote_id),
  23     api_name(_api_name),
  24     host_style(_host_style)
  25 {
  26   if (zone_svc) {
  27     key = zone_svc->get_zone_params().system_key;
  28     self_zone_group = zone_svc->get_zonegroup().get_id();
  29   }
  30 }
  31
  32 RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Store* store,
  33                          const string& _remote_id,
  34                          const list<string>& remote_endpoints,
  35                          std::optional<string> _api_name,
  36                          HostStyle _host_style)
  37   : cct(_cct),
  38     endpoints(remote_endpoints.begin(), remote_endpoints.end()),
  39     remote_id(_remote_id),
  40     api_name(_api_name),
  41     host_style(_host_style)
  42 {
  43   if (store) {
  44     key = store->get_zone()->get_params().system_key;
  45     self_zone_group = store->get_zone()->get_zonegroup().get_id();
  46   }
  47 }
  48
  49 RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc,
  50                          const string& _remote_id,
  51                          const list<string>& remote_endpoints,
  52                          RGWAccessKey _cred,
  53                          std::optional<string> _api_name,
  54                          HostStyle _host_style)
  55   : cct(_cct),
  56     endpoints(remote_endpoints.begin(), remote_endpoints.end()),
  57     key(std::move(_cred)),
  58     remote_id(_remote_id),
  59     api_name(_api_name),
  60     host_style(_host_style)
  61 {
  62   if (zone_svc) {
  63     self_zone_group = zone_svc->get_zonegroup().get_id();
  64   }
  65 }
  66
  67 RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Store* store,
  68                          const string& _remote_id,
  69                          const list<string>& remote_endpoints,
  70                          RGWAccessKey _cred,
  71                          std::optional<string> _api_name,
  72                          HostStyle _host_style)
  73   : cct(_cct),
  74     endpoints(remote_endpoints.begin(), remote_endpoints.end()),
  75     key(std::move(_cred)),
  76     remote_id(_remote_id),
  77     api_name(_api_name),
  78     host_style(_host_style)
  79 {
  80   if (store) {
  81     self_zone_group = store->get_zone()->get_zonegroup().get_id();
  82   }
  83 }
  84
  85 RGWRESTConn::RGWRESTConn(RGWRESTConn&& other)
  86   : cct(other.cct),
  87     endpoints(std::move(other.endpoints)),
  88     key(std::move(other.key)),
  89     self_zone_group(std::move(other.self_zone_group)),
  90     remote_id(std::move(other.remote_id)),
  91     counter(other.counter.load())
  92 {
  93 }
  94
  95 RGWRESTConn& RGWRESTConn::operator=(RGWRESTConn&& other)
  96 {
  97   cct = other.cct;
  98   endpoints = std::move(other.endpoints);
  99   key = std::move(other.key);
 100   self_zone_group = std::move(other.self_zone_group);
 101   remote_id = std::move(other.remote_id);
 102   counter = other.counter.load();
 103   return *this;
 104 }
 105
 106 int RGWRESTConn::get_url(string& endpoint)
 107 {
 108   if (endpoints.empty()) {
 109     ldout(cct, 0) << "ERROR: endpoints not configured for upstream zone" << dendl;
 110     return -EIO;
 111   }
 112
 113   int i = ++counter;
 114   endpoint = endpoints[i % endpoints.size()];
 115
 116   return 0;
 117 }
 118
 119 string RGWRESTConn::get_url()
 120 {
 121   string endpoint;
 122   get_url(endpoint);
 123   return endpoint;
 124 }
 125
 126 void RGWRESTConn::populate_params(param_vec_t& params, const rgw_user *uid, const string& zonegroup)
 127 {
 128   populate_uid(params, uid);
 129   populate_zonegroup(params, zonegroup);
 130 }
 131
 132 int RGWRESTConn::forward(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info& info, obj_version *objv, size_t max_response, bufferlist *inbl, bufferlist *outbl, optional_yield y)
 133 {
 134   string url;
 135   int ret = get_url(url);
 136   if (ret < 0)
 137     return ret;
 138   param_vec_t params;
 139   populate_params(params, &uid, self_zone_group);
 140   if (objv) {
 141     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "tag", objv->tag));
 142     char buf[16];
 143     snprintf(buf, sizeof(buf), "%lld", (long long)objv->ver);
 144     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "ver", buf));
 145   }
 146   RGWRESTSimpleRequest req(cct, info.method, url, NULL, &params, api_name);
 147   return req.forward_request(dpp, key, info, max_response, inbl, outbl, y);
 148 }
 149
 150 int RGWRESTConn::put_obj_send_init(rgw::sal::Object* obj, const rgw_http_param_pair *extra_params, RGWRESTStreamS3PutObj **req)
 151 {
 152   string url;
 153   int ret = get_url(url);
 154   if (ret < 0)
 155     return ret;
 156
 157   rgw_user uid;
 158   param_vec_t params;
 159   populate_params(params, &uid, self_zone_group);
 160
 161   if (extra_params) {
 162     append_param_list(params, extra_params);
 163   }
 164
 165   RGWRESTStreamS3PutObj *wr = new RGWRESTStreamS3PutObj(cct, "PUT", url, NULL, &params, api_name, host_style);
 166   wr->send_init(obj);
 167   *req = wr;
 168   return 0;
 169 }
 170
 171 int RGWRESTConn::put_obj_async_init(const DoutPrefixProvider *dpp, const rgw_user& uid, rgw::sal::Object* obj,
 172                                     map<string, bufferlist>& attrs,
 173                                     RGWRESTStreamS3PutObj **req)
 174 {
 175   string url;
 176   int ret = get_url(url);
 177   if (ret < 0)
 178     return ret;
 179
 180   param_vec_t params;
 181   populate_params(params, &uid, self_zone_group);
 182   RGWRESTStreamS3PutObj *wr = new RGWRESTStreamS3PutObj(cct, "PUT", url, NULL, &params, api_name, host_style);
 183   wr->put_obj_init(dpp, key, obj, attrs);
 184   *req = wr;
 185   return 0;
 186 }
 187
 188 int RGWRESTConn::complete_request(RGWRESTStreamS3PutObj *req, string& etag,
 189                                   real_time *mtime, optional_yield y)
 190 {
 191   int ret = req->complete_request(y, &etag, mtime);
 192   delete req;
 193
 194   return ret;
 195 }
 196
 197 static void set_date_header(const real_time *t, map<string, string>& headers, bool high_precision_time, const string& header_name)
 198 {
 199   if (!t) {
 200     return;
 201   }
 202   stringstream s;
 203   utime_t tm = utime_t(*t);
 204   if (high_precision_time) {
 205     tm.gmtime_nsec(s);
 206   } else {
 207     tm.gmtime(s);
 208   }
 209   headers[header_name] = s.str();
 210 }
 211
 212 template <class T>
 213 static void set_header(T val, map<string, string>& headers, const string& header_name)
 214 {
 215   stringstream s;
 216   s << val;
 217   headers[header_name] = s.str();
 218 }
 219
 220
 221 int RGWRESTConn::get_obj(const DoutPrefixProvider *dpp, const rgw_user& uid, req_info *info /* optional */, const rgw::sal::Object* obj,
 222                          const real_time *mod_ptr, const real_time *unmod_ptr,
 223                          uint32_t mod_zone_id, uint64_t mod_pg_ver,
 224                          bool prepend_metadata, bool get_op, bool rgwx_stat,
 225                          bool sync_manifest, bool skip_decrypt,
 226                          bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req)
 227 {
 228   get_obj_params params;
 229   params.uid = uid;
 230   params.info = info;
 231   params.mod_ptr = mod_ptr;
 232   params.mod_pg_ver = mod_pg_ver;
 233   params.prepend_metadata = prepend_metadata;
 234   params.get_op = get_op;
 235   params.rgwx_stat = rgwx_stat;
 236   params.sync_manifest = sync_manifest;
 237   params.skip_decrypt = skip_decrypt;
 238   params.cb = cb;
 239   return get_obj(dpp, obj, params, send, req);
 240 }
 241
 242 int RGWRESTConn::get_obj(const DoutPrefixProvider *dpp, const rgw::sal::Object* obj, const get_obj_params& in_params, bool send, RGWRESTStreamRWRequest **req)
 243 {
 244   string url;
 245   int ret = get_url(url);
 246   if (ret < 0)
 247     return ret;
 248
 249   param_vec_t params;
 250   populate_params(params, &in_params.uid, self_zone_group);
 251   if (in_params.prepend_metadata) {
 252     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "prepend-metadata", "true"));
 253   }
 254   if (in_params.rgwx_stat) {
 255     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "stat", "true"));
 256   }
 257   if (in_params.sync_manifest) {
 258     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "sync-manifest", ""));
 259   }
 260   if (in_params.skip_decrypt) {
 261     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "skip-decrypt", ""));
 262   }
 263   if (!obj->get_instance().empty()) {
 264     const string& instance = obj->get_instance();
 265     params.push_back(param_pair_t("versionId", instance));
 266   }
 267   if (in_params.get_op) {
 268     *req = new RGWRESTStreamReadRequest(cct, url, in_params.cb, NULL, &params, api_name, host_style);
 269   } else {
 270     *req = new RGWRESTStreamHeadRequest(cct, url, in_params.cb, NULL, &params, api_name);
 271   }
 272   map<string, string> extra_headers;
 273   if (in_params.info) {
 274     const auto& orig_map = in_params.info->env->get_map();
 275
 276     /* add original headers that start with HTTP_X_AMZ_ */
 277     static constexpr char SEARCH_AMZ_PREFIX[] = "HTTP_X_AMZ_";
 278     for (auto iter= orig_map.lower_bound(SEARCH_AMZ_PREFIX); iter != orig_map.end(); ++iter) {
 279       const string& name = iter->first;
 280       if (name == "HTTP_X_AMZ_DATE") /* don't forward date from original request */
 281         continue;
 282       if (name.compare(0, strlen(SEARCH_AMZ_PREFIX), SEARCH_AMZ_PREFIX) != 0)
 283         break;
 284       extra_headers[iter->first] = iter->second;
 285     }
 286   }
 287
 288   set_date_header(in_params.mod_ptr, extra_headers, in_params.high_precision_time, "HTTP_IF_MODIFIED_SINCE");
 289   set_date_header(in_params.unmod_ptr, extra_headers, in_params.high_precision_time, "HTTP_IF_UNMODIFIED_SINCE");
 290   if (!in_params.etag.empty()) {
 291     set_header(in_params.etag, extra_headers, "HTTP_IF_MATCH");
 292   }
 293   if (in_params.mod_zone_id != 0) {
 294     set_header(in_params.mod_zone_id, extra_headers, "HTTP_DEST_ZONE_SHORT_ID");
 295   }
 296   if (in_params.mod_pg_ver != 0) {
 297     set_header(in_params.mod_pg_ver, extra_headers, "HTTP_DEST_PG_VER");
 298   }
 299   if (in_params.range_is_set) {
 300     char buf[64];
 301     snprintf(buf, sizeof(buf), "bytes=%lld-%lld", (long long)in_params.range_start, (long long)in_params.range_end);
 302     set_header(buf, extra_headers, "RANGE");
 303   }
 304
 305   int r = (*req)->send_prepare(dpp, key, extra_headers, obj->get_obj());
 306   if (r < 0) {
 307     goto done_err;
 308   }
 309
 310   if (!send) {
 311     return 0;
 312   }
 313
 314   r = (*req)->send(nullptr);
 315   if (r < 0) {
 316     goto done_err;
 317   }
 318   return 0;
 319 done_err:
 320   delete *req;
 321   *req = nullptr;
 322   return r;
 323 }
 324
 325 int RGWRESTConn::complete_request(RGWRESTStreamRWRequest *req,
 326                                   string *etag,
 327                                   real_time *mtime,
 328                                   uint64_t *psize,
 329                                   map<string, string> *pattrs,
 330                                   map<string, string> *pheaders,
 331                                   optional_yield y)
 332 {
 333   int ret = req->complete_request(y, etag, mtime, psize, pattrs, pheaders);
 334   delete req;
 335
 336   return ret;
 337 }
 338
 339 int RGWRESTConn::get_resource(const DoutPrefixProvider *dpp,
 340                      const string& resource,
 341                      param_vec_t *extra_params,
 342                      map<string, string> *extra_headers,
 343                      bufferlist& bl,
 344                      bufferlist *send_data,
 345                      RGWHTTPManager *mgr,
 346                      optional_yield y)
 347 {
 348   string url;
 349   int ret = get_url(url);
 350   if (ret < 0)
 351     return ret;
 352
 353   param_vec_t params;
 354
 355   if (extra_params) {
 356     params.insert(params.end(), extra_params->begin(), extra_params->end());
 357   }
 358
 359   populate_params(params, nullptr, self_zone_group);
 360
 361   RGWStreamIntoBufferlist cb(bl);
 362
 363   RGWRESTStreamReadRequest req(cct, url, &cb, NULL, &params, api_name, host_style);
 364
 365   map<string, string> headers;
 366   if (extra_headers) {
 367     headers.insert(extra_headers->begin(), extra_headers->end());
 368   }
 369
 370   ret = req.send_request(dpp, &key, headers, resource, mgr, send_data);
 371   if (ret < 0) {
 372     ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
 373     return ret;
 374   }
 375
 376   return req.complete_request(y);
 377 }
 378
 379 int RGWRESTConn::send_resource(const DoutPrefixProvider *dpp, const std::string& method,
 380                         const std::string& resource, rgw_http_param_pair *extra_params,
 381                                 std::map<std::string, std::string> *extra_headers, bufferlist& bl,
 382                         bufferlist *send_data, RGWHTTPManager *mgr, optional_yield y)
 383 {
 384   std::string url;
 385   int ret = get_url(url);
 386   if (ret < 0)
 387     return ret;
 388
 389   param_vec_t params;
 390
 391   if (extra_params) {
 392     params = make_param_list(extra_params);
 393   }
 394
 395   populate_params(params, nullptr, self_zone_group);
 396
 397   RGWStreamIntoBufferlist cb(bl);
 398
 399   RGWRESTStreamSendRequest req(cct, method, url, &cb, NULL, &params, api_name, host_style);
 400
 401   std::map<std::string, std::string> headers;
 402   if (extra_headers) {
 403     headers.insert(extra_headers->begin(), extra_headers->end());
 404   }
 405
 406   ret = req.send_request(dpp, &key, headers, resource, mgr, send_data);
 407   if (ret < 0) {
 408     ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
 409     return ret;
 410   }
 411
 412   return req.complete_request(y);
 413 }
 414
 415 RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn,
 416                                          const string& _resource,
 417                                          const rgw_http_param_pair *pp,
 418                                          param_vec_t *extra_headers,
 419                                          RGWHTTPManager *_mgr)
 420   : cct(_conn->get_ctx()), conn(_conn), resource(_resource),
 421     params(make_param_list(pp)), cb(bl), mgr(_mgr),
 422     req(cct, conn->get_url(), &cb, NULL, NULL, _conn->get_api_name())
 423 {
 424   init_common(extra_headers);
 425 }
 426
 427 RGWRESTReadResource::RGWRESTReadResource(RGWRESTConn *_conn,
 428                                          const string& _resource,
 429                                          param_vec_t& _params,
 430                                          param_vec_t *extra_headers,
 431                                          RGWHTTPManager *_mgr)
 432   : cct(_conn->get_ctx()), conn(_conn), resource(_resource), params(_params),
 433     cb(bl), mgr(_mgr), req(cct, conn->get_url(), &cb, NULL, NULL, _conn->get_api_name())
 434 {
 435   init_common(extra_headers);
 436 }
 437
 438 void RGWRESTReadResource::init_common(param_vec_t *extra_headers)
 439 {
 440   conn->populate_params(params, nullptr, conn->get_self_zonegroup());
 441
 442   if (extra_headers) {
 443     headers.insert(extra_headers->begin(), extra_headers->end());
 444   }
 445
 446   req.set_params(&params);
 447 }
 448
 449 int RGWRESTReadResource::read(const DoutPrefixProvider *dpp, optional_yield y)
 450 {
 451   int ret = req.send_request(dpp, &conn->get_key(), headers, resource, mgr);
 452   if (ret < 0) {
 453     ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
 454     return ret;
 455   }
 456
 457   return req.complete_request(y);
 458 }
 459
 460 int RGWRESTReadResource::aio_read(const DoutPrefixProvider *dpp)
 461 {
 462   int ret = req.send_request(dpp, &conn->get_key(), headers, resource, mgr);
 463   if (ret < 0) {
 464     ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
 465     return ret;
 466   }
 467
 468   return 0;
 469 }
 470
 471 RGWRESTSendResource::RGWRESTSendResource(RGWRESTConn *_conn,
 472                                          const string& _method,
 473                                          const string& _resource,
 474                                          const rgw_http_param_pair *pp,
 475                                          param_vec_t *extra_headers,
 476                                          RGWHTTPManager *_mgr)
 477   : cct(_conn->get_ctx()), conn(_conn), method(_method), resource(_resource),
 478     params(make_param_list(pp)), cb(bl), mgr(_mgr),
 479     req(cct, method.c_str(), conn->get_url(), &cb, NULL, NULL, _conn->get_api_name(), _conn->get_host_style())
 480 {
 481   init_common(extra_headers);
 482 }
 483
 484 RGWRESTSendResource::RGWRESTSendResource(RGWRESTConn *_conn,
 485                                          const string& _method,
 486                                          const string& _resource,
 487                                          param_vec_t& params,
 488                                          param_vec_t *extra_headers,
 489                                          RGWHTTPManager *_mgr)
 490   : cct(_conn->get_ctx()), conn(_conn), method(_method), resource(_resource), params(params),
 491     cb(bl), mgr(_mgr), req(cct, method.c_str(), conn->get_url(), &cb, NULL, NULL, _conn->get_api_name(), _conn->get_host_style())
 492 {
 493   init_common(extra_headers);
 494 }
 495
 496 void RGWRESTSendResource::init_common(param_vec_t *extra_headers)
 497 {
 498   conn->populate_params(params, nullptr, conn->get_self_zonegroup());
 499
 500   if (extra_headers) {
 501     headers.insert(extra_headers->begin(), extra_headers->end());
 502   }
 503
 504   req.set_params(&params);
 505 }
 506
 507 int RGWRESTSendResource::send(const DoutPrefixProvider *dpp, bufferlist& outbl, optional_yield y)
 508 {
 509   req.set_send_length(outbl.length());
 510   req.set_outbl(outbl);
 511
 512   int ret = req.send_request(dpp, &conn->get_key(), headers, resource, mgr);
 513   if (ret < 0) {
 514     ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
 515     return ret;
 516   }
 517
 518   return req.complete_request(y);
 519 }
 520
 521 int RGWRESTSendResource::aio_send(const DoutPrefixProvider *dpp, bufferlist& outbl)
 522 {
 523   req.set_send_length(outbl.length());
 524   req.set_outbl(outbl);
 525
 526   int ret = req.send_request(dpp, &conn->get_key(), headers, resource, mgr);
 527   if (ret < 0) {
 528     ldpp_dout(dpp, 5) << __func__ << ": send_request() resource=" << resource << " returned ret=" << ret << dendl;
 529     return ret;
 530   }
 531
 532   return 0;
 533 }