ceph/src/rgw/rgw_cr_rados.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #ifndef CEPH_RGW_CR_RADOS_H
   5 #define CEPH_RGW_CR_RADOS_H
   6
   7 #include <boost/intrusive_ptr.hpp>
   8 #include "include/ceph_assert.h"
   9 #include "rgw_coroutine.h"
  10 #include "rgw_sal.h"
  11 #include "rgw_sal_rados.h"
  12 #include "common/WorkQueue.h"
  13 #include "common/Throttle.h"
  14
  15 #include <atomic>
  16
  17 #include "services/svc_sys_obj.h"
  18 #include "services/svc_bucket.h"
  19
  20 #define dout_subsys ceph_subsys_rgw
  21
  22 class RGWAsyncRadosRequest : public RefCountedObject {
  23   RGWCoroutine *caller;
  24   RGWAioCompletionNotifier *notifier;
  25
  26   int retcode;
  27
  28   ceph::mutex lock = ceph::make_mutex("RGWAsyncRadosRequest::lock");
  29
  30 protected:
  31   virtual int _send_request(const DoutPrefixProvider *dpp) = 0;
  32 public:
  33   RGWAsyncRadosRequest(RGWCoroutine *_caller, RGWAioCompletionNotifier *_cn)
  34     : caller(_caller), notifier(_cn), retcode(0) {
  35   }
  36   ~RGWAsyncRadosRequest() override {
  37     if (notifier) {
  38       notifier->put();
  39     }
  40   }
  41
  42   void send_request(const DoutPrefixProvider *dpp) {
  43     get();
  44     retcode = _send_request(dpp);
  45     {
  46       std::lock_guard l{lock};
  47       if (notifier) {
  48         notifier->cb(); // drops its own ref
  49         notifier = nullptr;
  50       }
  51     }
  52     put();
  53   }
  54
  55   int get_ret_status() { return retcode; }
  56
  57   void finish() {
  58     {
  59       std::lock_guard l{lock};
  60       if (notifier) {
  61         // we won't call notifier->cb() to drop its ref, so drop it here
  62         notifier->put();
  63         notifier = nullptr;
  64       }
  65     }
  66     put();
  67   }
  68 };
  69
  70
  71 class RGWAsyncRadosProcessor {
  72   std::deque<RGWAsyncRadosRequest *> m_req_queue;
  73   std::atomic<bool> going_down = { false };
  74 protected:
  75   CephContext *cct;
  76   ThreadPool m_tp;
  77   Throttle req_throttle;
  78
  79   struct RGWWQ : public DoutPrefixProvider, public ThreadPool::WorkQueue<RGWAsyncRadosRequest> {
  80     RGWAsyncRadosProcessor *processor;
  81     RGWWQ(RGWAsyncRadosProcessor *p,
  82           ceph::timespan timeout, ceph::timespan suicide_timeout,
  83           ThreadPool *tp)
  84       : ThreadPool::WorkQueue<RGWAsyncRadosRequest>("RGWWQ", timeout, suicide_timeout, tp), processor(p) {}
  85
  86     bool _enqueue(RGWAsyncRadosRequest *req) override;
  87     void _dequeue(RGWAsyncRadosRequest *req) override {
  88       ceph_abort();
  89     }
  90     bool _empty() override;
  91     RGWAsyncRadosRequest *_dequeue() override;
  92     using ThreadPool::WorkQueue<RGWAsyncRadosRequest>::_process;
  93     void _process(RGWAsyncRadosRequest *req, ThreadPool::TPHandle& handle) override;
  94     void _dump_queue();
  95     void _clear() override {
  96       ceph_assert(processor->m_req_queue.empty());
  97     }
  98
  99   CephContext *get_cct() const { return processor->cct; }
 100   unsigned get_subsys() const { return ceph_subsys_rgw; }
 101   std::ostream& gen_prefix(std::ostream& out) const { return out << "rgw async rados processor: ";}
 102
 103   } req_wq;
 104
 105 public:
 106   RGWAsyncRadosProcessor(CephContext *_cct, int num_threads);
 107   ~RGWAsyncRadosProcessor() {}
 108   void start();
 109   void stop();
 110   void handle_request(const DoutPrefixProvider *dpp, RGWAsyncRadosRequest *req);
 111   void queue(RGWAsyncRadosRequest *req);
 112
 113   bool is_going_down() {
 114     return going_down;
 115   }
 116
 117 };
 118
 119 template <class P>
 120 class RGWSimpleWriteOnlyAsyncCR : public RGWSimpleCoroutine {
 121   RGWAsyncRadosProcessor *async_rados;
 122   rgw::sal::RadosStore* store;
 123
 124   P params;
 125   const DoutPrefixProvider *dpp;
 126
 127   class Request : public RGWAsyncRadosRequest {
 128     rgw::sal::RadosStore* store;
 129     P params;
 130     const DoutPrefixProvider *dpp;
 131   protected:
 132     int _send_request(const DoutPrefixProvider *dpp) override;
 133   public:
 134     Request(RGWCoroutine *caller,
 135             RGWAioCompletionNotifier *cn,
 136             rgw::sal::RadosStore* store,
 137             const P& _params,
 138             const DoutPrefixProvider *dpp) : RGWAsyncRadosRequest(caller, cn),
 139                                 store(store),
 140                                 params(_params),
 141                                 dpp(dpp) {}
 142   } *req{nullptr};
 143
 144  public:
 145   RGWSimpleWriteOnlyAsyncCR(RGWAsyncRadosProcessor *_async_rados,
 146                             rgw::sal::RadosStore* _store,
 147                             const P& _params,
 148                             const DoutPrefixProvider *_dpp) : RGWSimpleCoroutine(_store->ctx()),
 149                                                 async_rados(_async_rados),
 150                                                 store(_store),
 151                                                 params(_params),
 152                                                 dpp(_dpp) {}
 153
 154   ~RGWSimpleWriteOnlyAsyncCR() override {
 155     request_cleanup();
 156   }
 157   void request_cleanup() override {
 158     if (req) {
 159       req->finish();
 160       req = NULL;
 161     }
 162   }
 163
 164   int send_request(const DoutPrefixProvider *dpp) override {
 165     req = new Request(this,
 166                       stack->create_completion_notifier(),
 167                       store,
 168                       params,
 169                       dpp);
 170
 171     async_rados->queue(req);
 172     return 0;
 173   }
 174   int request_complete() override {
 175     return req->get_ret_status();
 176   }
 177 };
 178
 179
 180 template <class P, class R>
 181 class RGWSimpleAsyncCR : public RGWSimpleCoroutine {
 182   RGWAsyncRadosProcessor *async_rados;
 183   rgw::sal::RadosStore* store;
 184
 185   P params;
 186   std::shared_ptr<R> result;
 187   const DoutPrefixProvider *dpp;
 188
 189   class Request : public RGWAsyncRadosRequest {
 190     rgw::sal::RadosStore* store;
 191     P params;
 192     std::shared_ptr<R> result;
 193     const DoutPrefixProvider *dpp;
 194   protected:
 195     int _send_request(const DoutPrefixProvider *dpp) override;
 196   public:
 197     Request(const DoutPrefixProvider *dpp,
 198             RGWCoroutine *caller,
 199             RGWAioCompletionNotifier *cn,
 200             rgw::sal::RadosStore* _store,
 201             const P& _params,
 202             std::shared_ptr<R>& _result,
 203             const DoutPrefixProvider *_dpp) : RGWAsyncRadosRequest(caller, cn),
 204                                            store(_store),
 205                                            params(_params),
 206                                            result(_result),
 207                                            dpp(_dpp) {}
 208   } *req{nullptr};
 209
 210  public:
 211   RGWSimpleAsyncCR(RGWAsyncRadosProcessor *_async_rados,
 212                    rgw::sal::RadosStore* _store,
 213                    const P& _params,
 214                    std::shared_ptr<R>& _result,
 215                    const DoutPrefixProvider *_dpp) : RGWSimpleCoroutine(_store->ctx()),
 216                                                   async_rados(_async_rados),
 217                                                   store(_store),
 218                                                   params(_params),
 219                                                   result(_result),
 220                                                   dpp(_dpp) {}
 221
 222   ~RGWSimpleAsyncCR() override {
 223     request_cleanup();
 224   }
 225   void request_cleanup() override {
 226     if (req) {
 227       req->finish();
 228       req = NULL;
 229     }
 230   }
 231
 232   int send_request(const DoutPrefixProvider *dpp) override {
 233     req = new Request(dpp,
 234                       this,
 235                       stack->create_completion_notifier(),
 236                       store,
 237                       params,
 238                       result,
 239                       dpp);
 240
 241     async_rados->queue(req);
 242     return 0;
 243   }
 244   int request_complete() override {
 245     return req->get_ret_status();
 246   }
 247 };
 248
 249 class RGWGenericAsyncCR : public RGWSimpleCoroutine {
 250   RGWAsyncRadosProcessor *async_rados;
 251   rgw::sal::RadosStore* store;
 252
 253
 254 public:
 255   class Action {
 256   public:
 257     virtual ~Action() {}
 258     virtual int operate() = 0;
 259   };
 260
 261 private:
 262   std::shared_ptr<Action> action;
 263
 264   class Request : public RGWAsyncRadosRequest {
 265     std::shared_ptr<Action> action;
 266   protected:
 267     int _send_request(const DoutPrefixProvider *dpp) override {
 268       if (!action) {
 269         return 0;
 270       }
 271       return action->operate();
 272     }
 273   public:
 274     Request(const DoutPrefixProvider *dpp,
 275             RGWCoroutine *caller,
 276             RGWAioCompletionNotifier *cn,
 277             std::shared_ptr<Action>& _action) : RGWAsyncRadosRequest(caller, cn),
 278                                            action(_action) {}
 279   } *req{nullptr};
 280
 281  public:
 282   RGWGenericAsyncCR(CephContext *_cct,
 283                     RGWAsyncRadosProcessor *_async_rados,
 284                     std::shared_ptr<Action>& _action) : RGWSimpleCoroutine(_cct),
 285                                                   async_rados(_async_rados),
 286                                                   action(_action) {}
 287   template<typename T>
 288   RGWGenericAsyncCR(CephContext *_cct,
 289                     RGWAsyncRadosProcessor *_async_rados,
 290                     std::shared_ptr<T>& _action) : RGWSimpleCoroutine(_cct),
 291                                                   async_rados(_async_rados),
 292                                                   action(std::static_pointer_cast<Action>(_action)) {}
 293
 294   ~RGWGenericAsyncCR() override {
 295     request_cleanup();
 296   }
 297   void request_cleanup() override {
 298     if (req) {
 299       req->finish();
 300       req = NULL;
 301     }
 302   }
 303
 304   int send_request(const DoutPrefixProvider *dpp) override {
 305     req = new Request(dpp, this,
 306                       stack->create_completion_notifier(),
 307                       action);
 308
 309     async_rados->queue(req);
 310     return 0;
 311   }
 312   int request_complete() override {
 313     return req->get_ret_status();
 314   }
 315 };
 316
 317
 318 class RGWAsyncGetSystemObj : public RGWAsyncRadosRequest {
 319   const DoutPrefixProvider *dpp;
 320   RGWSysObjectCtx obj_ctx;
 321   rgw_raw_obj obj;
 322   const bool want_attrs;
 323   const bool raw_attrs;
 324 protected:
 325   int _send_request(const DoutPrefixProvider *dpp) override;
 326 public:
 327   RGWAsyncGetSystemObj(const DoutPrefixProvider *dpp,
 328                        RGWCoroutine *caller, RGWAioCompletionNotifier *cn, RGWSI_SysObj *_svc,
 329                        RGWObjVersionTracker *_objv_tracker, const rgw_raw_obj& _obj,
 330                        bool want_attrs, bool raw_attrs);
 331
 332   bufferlist bl;
 333   std::map<std::string, bufferlist> attrs;
 334   RGWObjVersionTracker objv_tracker;
 335 };
 336
 337 class RGWAsyncPutSystemObj : public RGWAsyncRadosRequest {
 338   const DoutPrefixProvider *dpp;
 339   RGWSI_SysObj *svc;
 340   rgw_raw_obj obj;
 341   bool exclusive;
 342   bufferlist bl;
 343
 344 protected:
 345   int _send_request(const DoutPrefixProvider *dpp) override;
 346 public:
 347   RGWAsyncPutSystemObj(const DoutPrefixProvider *dpp, RGWCoroutine *caller,
 348                        RGWAioCompletionNotifier *cn, RGWSI_SysObj *_svc,
 349                        RGWObjVersionTracker *_objv_tracker, const rgw_raw_obj& _obj,
 350                        bool _exclusive, bufferlist _bl);
 351
 352   RGWObjVersionTracker objv_tracker;
 353 };
 354
 355 class RGWAsyncPutSystemObjAttrs : public RGWAsyncRadosRequest {
 356   const DoutPrefixProvider *dpp;
 357   RGWSI_SysObj *svc;
 358   rgw_raw_obj obj;
 359   std::map<std::string, bufferlist> attrs;
 360
 361 protected:
 362   int _send_request(const DoutPrefixProvider *dpp) override;
 363 public:
 364   RGWAsyncPutSystemObjAttrs(const DoutPrefixProvider *dpp, RGWCoroutine *caller, RGWAioCompletionNotifier *cn, RGWSI_SysObj *_svc,
 365                        RGWObjVersionTracker *_objv_tracker, const rgw_raw_obj& _obj,
 366                        std::map<std::string, bufferlist> _attrs);
 367
 368   RGWObjVersionTracker objv_tracker;
 369 };
 370
 371 class RGWAsyncLockSystemObj : public RGWAsyncRadosRequest {
 372   rgw::sal::RadosStore* store;
 373   rgw_raw_obj obj;
 374   std::string lock_name;
 375   std::string cookie;
 376   uint32_t duration_secs;
 377
 378 protected:
 379   int _send_request(const DoutPrefixProvider *dpp) override;
 380 public:
 381   RGWAsyncLockSystemObj(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store,
 382                         RGWObjVersionTracker *_objv_tracker, const rgw_raw_obj& _obj,
 383                         const std::string& _name, const std::string& _cookie, uint32_t _duration_secs);
 384 };
 385
 386 class RGWAsyncUnlockSystemObj : public RGWAsyncRadosRequest {
 387   rgw::sal::RadosStore* store;
 388   rgw_raw_obj obj;
 389   std::string lock_name;
 390   std::string cookie;
 391
 392 protected:
 393   int _send_request(const DoutPrefixProvider *dpp) override;
 394 public:
 395   RGWAsyncUnlockSystemObj(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store,
 396                         RGWObjVersionTracker *_objv_tracker, const rgw_raw_obj& _obj,
 397                         const std::string& _name, const std::string& _cookie);
 398 };
 399
 400 template <class T>
 401 class RGWSimpleRadosReadCR : public RGWSimpleCoroutine {
 402   const DoutPrefixProvider *dpp;
 403   RGWAsyncRadosProcessor *async_rados;
 404   RGWSI_SysObj *svc;
 405
 406   rgw_raw_obj obj;
 407   T *result;
 408   /// on ENOENT, call handle_data() with an empty object instead of failing
 409   const bool empty_on_enoent;
 410   RGWObjVersionTracker *objv_tracker;
 411   RGWAsyncGetSystemObj *req{nullptr};
 412
 413 public:
 414   RGWSimpleRadosReadCR(const DoutPrefixProvider *_dpp,
 415                       RGWAsyncRadosProcessor *_async_rados, RGWSI_SysObj *_svc,
 416                       const rgw_raw_obj& _obj,
 417                       T *_result, bool empty_on_enoent = true,
 418                       RGWObjVersionTracker *objv_tracker = nullptr)
 419     : RGWSimpleCoroutine(_svc->ctx()), dpp(_dpp), async_rados(_async_rados), svc(_svc),
 420       obj(_obj), result(_result),
 421       empty_on_enoent(empty_on_enoent), objv_tracker(objv_tracker) {}
 422   ~RGWSimpleRadosReadCR() override {
 423     request_cleanup();
 424   }
 425
 426   void request_cleanup() override {
 427     if (req) {
 428       req->finish();
 429       req = NULL;
 430     }
 431   }
 432
 433   int send_request(const DoutPrefixProvider *dpp) override;
 434   int request_complete() override;
 435
 436   virtual int handle_data(T& data) {
 437     return 0;
 438   }
 439 };
 440
 441 template <class T>
 442 int RGWSimpleRadosReadCR<T>::send_request(const DoutPrefixProvider *dpp)
 443 {
 444   req = new RGWAsyncGetSystemObj(dpp, this, stack->create_completion_notifier(), svc,
 445                                  objv_tracker, obj, false, false);
 446   async_rados->queue(req);
 447   return 0;
 448 }
 449
 450 template <class T>
 451 int RGWSimpleRadosReadCR<T>::request_complete()
 452 {
 453   int ret = req->get_ret_status();
 454   retcode = ret;
 455   if (ret == -ENOENT && empty_on_enoent) {
 456     *result = T();
 457   } else {
 458     if (ret < 0) {
 459       return ret;
 460     }
 461     try {
 462       auto iter = req->bl.cbegin();
 463       if (iter.end()) {
 464         // allow successful reads with empty buffers. ReadSyncStatus coroutines
 465         // depend on this to be able to read without locking, because the
 466         // cls lock from InitSyncStatus will create an empty object if it didn't
 467         // exist
 468         *result = T();
 469       } else {
 470         decode(*result, iter);
 471       }
 472     } catch (buffer::error& err) {
 473       return -EIO;
 474     }
 475   }
 476
 477   return handle_data(*result);
 478 }
 479
 480 class RGWSimpleRadosReadAttrsCR : public RGWSimpleCoroutine {
 481   const DoutPrefixProvider *dpp;
 482   RGWAsyncRadosProcessor *async_rados;
 483   RGWSI_SysObj *svc;
 484
 485   rgw_raw_obj obj;
 486   std::map<std::string, bufferlist> *pattrs;
 487   bool raw_attrs;
 488   RGWObjVersionTracker* objv_tracker;
 489   RGWAsyncGetSystemObj *req = nullptr;
 490
 491 public:
 492   RGWSimpleRadosReadAttrsCR(const DoutPrefixProvider *_dpp, RGWAsyncRadosProcessor *_async_rados, RGWSI_SysObj *_svc,
 493                             const rgw_raw_obj& _obj, std::map<std::string, bufferlist> *_pattrs,
 494                             bool _raw_attrs, RGWObjVersionTracker* objv_tracker = nullptr)
 495     : RGWSimpleCoroutine(_svc->ctx()),
 496       dpp(_dpp),
 497       async_rados(_async_rados), svc(_svc),
 498       obj(_obj),
 499       pattrs(_pattrs),
 500       raw_attrs(_raw_attrs),
 501       objv_tracker(objv_tracker)
 502   {}
 503   ~RGWSimpleRadosReadAttrsCR() override {
 504     request_cleanup();
 505   }
 506
 507   void request_cleanup() override {
 508     if (req) {
 509       req->finish();
 510       req = NULL;
 511     }
 512   }
 513
 514   int send_request(const DoutPrefixProvider *dpp) override;
 515   int request_complete() override;
 516 };
 517
 518 template <class T>
 519 class RGWSimpleRadosWriteCR : public RGWSimpleCoroutine {
 520   const DoutPrefixProvider *dpp;
 521   RGWAsyncRadosProcessor *async_rados;
 522   RGWSI_SysObj *svc;
 523   bufferlist bl;
 524   rgw_raw_obj obj;
 525   RGWObjVersionTracker *objv_tracker;
 526   RGWAsyncPutSystemObj *req{nullptr};
 527
 528 public:
 529   RGWSimpleRadosWriteCR(const DoutPrefixProvider *_dpp,
 530                       RGWAsyncRadosProcessor *_async_rados, RGWSI_SysObj *_svc,
 531                       const rgw_raw_obj& _obj,
 532                       const T& _data, RGWObjVersionTracker *objv_tracker = nullptr)
 533     : RGWSimpleCoroutine(_svc->ctx()), dpp(_dpp), async_rados(_async_rados),
 534       svc(_svc), obj(_obj), objv_tracker(objv_tracker) {
 535     encode(_data, bl);
 536   }
 537
 538   ~RGWSimpleRadosWriteCR() override {
 539     request_cleanup();
 540   }
 541
 542   void request_cleanup() override {
 543     if (req) {
 544       req->finish();
 545       req = NULL;
 546     }
 547   }
 548
 549   int send_request(const DoutPrefixProvider *dpp) override {
 550     req = new RGWAsyncPutSystemObj(dpp, this, stack->create_completion_notifier(),
 551                                    svc, objv_tracker, obj, false, std::move(bl));
 552     async_rados->queue(req);
 553     return 0;
 554   }
 555
 556   int request_complete() override {
 557     if (objv_tracker) { // copy the updated version
 558       *objv_tracker = req->objv_tracker;
 559     }
 560     return req->get_ret_status();
 561   }
 562 };
 563
 564 class RGWSimpleRadosWriteAttrsCR : public RGWSimpleCoroutine {
 565   const DoutPrefixProvider *dpp;
 566   RGWAsyncRadosProcessor *async_rados;
 567   RGWSI_SysObj *svc;
 568   RGWObjVersionTracker *objv_tracker;
 569
 570   rgw_raw_obj obj;
 571   std::map<std::string, bufferlist> attrs;
 572   RGWAsyncPutSystemObjAttrs *req = nullptr;
 573
 574 public:
 575   RGWSimpleRadosWriteAttrsCR(const DoutPrefixProvider *_dpp,
 576                              RGWAsyncRadosProcessor *_async_rados,
 577                              RGWSI_SysObj *_svc, const rgw_raw_obj& _obj,
 578                              std::map<std::string, bufferlist> _attrs,
 579                              RGWObjVersionTracker *objv_tracker = nullptr)
 580     : RGWSimpleCoroutine(_svc->ctx()), dpp(_dpp), async_rados(_async_rados),
 581       svc(_svc), objv_tracker(objv_tracker), obj(_obj),
 582       attrs(std::move(_attrs)) {
 583   }
 584   ~RGWSimpleRadosWriteAttrsCR() override {
 585     request_cleanup();
 586   }
 587
 588   void request_cleanup() override {
 589     if (req) {
 590       req->finish();
 591       req = NULL;
 592     }
 593   }
 594
 595   int send_request(const DoutPrefixProvider *dpp) override {
 596     req = new RGWAsyncPutSystemObjAttrs(dpp, this, stack->create_completion_notifier(),
 597                                    svc, objv_tracker, obj, std::move(attrs));
 598     async_rados->queue(req);
 599     return 0;
 600   }
 601
 602   int request_complete() override {
 603     if (objv_tracker) { // copy the updated version
 604       *objv_tracker = req->objv_tracker;
 605     }
 606     return req->get_ret_status();
 607   }
 608 };
 609
 610 class RGWRadosSetOmapKeysCR : public RGWSimpleCoroutine {
 611   rgw::sal::RadosStore* store;
 612   std::map<std::string, bufferlist> entries;
 613
 614   rgw_rados_ref ref;
 615
 616   rgw_raw_obj obj;
 617
 618   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 619
 620 public:
 621   RGWRadosSetOmapKeysCR(rgw::sal::RadosStore* _store,
 622                       const rgw_raw_obj& _obj,
 623                       std::map<std::string, bufferlist>& _entries);
 624
 625   int send_request(const DoutPrefixProvider *dpp) override;
 626   int request_complete() override;
 627 };
 628
 629 class RGWRadosGetOmapKeysCR : public RGWSimpleCoroutine {
 630  public:
 631   struct Result {
 632     rgw_rados_ref ref;
 633     std::set<std::string> entries;
 634     bool more = false;
 635   };
 636   using ResultPtr = std::shared_ptr<Result>;
 637
 638   RGWRadosGetOmapKeysCR(rgw::sal::RadosStore* _store, const rgw_raw_obj& _obj,
 639                         const std::string& _marker, int _max_entries,
 640                         ResultPtr result);
 641
 642   int send_request(const DoutPrefixProvider *dpp) override;
 643   int request_complete() override;
 644
 645  private:
 646   rgw::sal::RadosStore* store;
 647   rgw_raw_obj obj;
 648   std::string marker;
 649   int max_entries;
 650   ResultPtr result;
 651   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 652 };
 653
 654 class RGWRadosGetOmapValsCR : public RGWSimpleCoroutine {
 655  public:
 656   struct Result {
 657     rgw_rados_ref ref;
 658     std::map<std::string, bufferlist> entries;
 659     bool more = false;
 660   };
 661   using ResultPtr = std::shared_ptr<Result>;
 662
 663   RGWRadosGetOmapValsCR(rgw::sal::RadosStore* _store, const rgw_raw_obj& _obj,
 664                         const std::string& _marker, int _max_entries,
 665                         ResultPtr result);
 666
 667   int send_request(const DoutPrefixProvider *dpp) override;
 668   int request_complete() override;
 669
 670  private:
 671   rgw::sal::RadosStore* store;
 672   rgw_raw_obj obj;
 673   std::string marker;
 674   int max_entries;
 675   ResultPtr result;
 676   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 677 };
 678
 679 class RGWRadosRemoveOmapKeysCR : public RGWSimpleCoroutine {
 680   rgw::sal::RadosStore* store;
 681
 682   rgw_rados_ref ref;
 683
 684   std::set<std::string> keys;
 685
 686   rgw_raw_obj obj;
 687
 688   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 689
 690 public:
 691   RGWRadosRemoveOmapKeysCR(rgw::sal::RadosStore* _store,
 692                       const rgw_raw_obj& _obj,
 693                       const std::set<std::string>& _keys);
 694
 695   int send_request(const DoutPrefixProvider *dpp) override;
 696
 697   int request_complete() override;
 698 };
 699
 700 class RGWRadosRemoveCR : public RGWSimpleCoroutine {
 701   rgw::sal::RadosStore* store;
 702   librados::IoCtx ioctx;
 703   const rgw_raw_obj obj;
 704   RGWObjVersionTracker* objv_tracker;
 705   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 706
 707 public:
 708   RGWRadosRemoveCR(rgw::sal::RadosStore* store, const rgw_raw_obj& obj,
 709                    RGWObjVersionTracker* objv_tracker = nullptr);
 710
 711   int send_request(const DoutPrefixProvider *dpp) override;
 712   int request_complete() override;
 713 };
 714
 715 class RGWSimpleRadosLockCR : public RGWSimpleCoroutine {
 716   RGWAsyncRadosProcessor *async_rados;
 717   rgw::sal::RadosStore* store;
 718   std::string lock_name;
 719   std::string cookie;
 720   uint32_t duration;
 721
 722   rgw_raw_obj obj;
 723
 724   RGWAsyncLockSystemObj *req;
 725
 726 public:
 727   RGWSimpleRadosLockCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
 728                       const rgw_raw_obj& _obj,
 729                       const std::string& _lock_name,
 730                       const std::string& _cookie,
 731                       uint32_t _duration);
 732   ~RGWSimpleRadosLockCR() override {
 733     request_cleanup();
 734   }
 735   void request_cleanup() override;
 736
 737   int send_request(const DoutPrefixProvider *dpp) override;
 738   int request_complete() override;
 739
 740   static std::string gen_random_cookie(CephContext* cct) {
 741 #define COOKIE_LEN 16
 742     char buf[COOKIE_LEN + 1];
 743     gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1);
 744     return buf;
 745   }
 746 };
 747
 748 class RGWSimpleRadosUnlockCR : public RGWSimpleCoroutine {
 749   RGWAsyncRadosProcessor *async_rados;
 750   rgw::sal::RadosStore* store;
 751   std::string lock_name;
 752   std::string cookie;
 753
 754   rgw_raw_obj obj;
 755
 756   RGWAsyncUnlockSystemObj *req;
 757
 758 public:
 759   RGWSimpleRadosUnlockCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
 760                       const rgw_raw_obj& _obj,
 761                       const std::string& _lock_name,
 762                       const std::string& _cookie);
 763   ~RGWSimpleRadosUnlockCR() override {
 764     request_cleanup();
 765   }
 766   void request_cleanup() override;
 767
 768   int send_request(const DoutPrefixProvider *dpp) override;
 769   int request_complete() override;
 770 };
 771
 772 #define OMAP_APPEND_MAX_ENTRIES_DEFAULT 100
 773
 774 class RGWOmapAppend : public RGWConsumerCR<std::string> {
 775   RGWAsyncRadosProcessor *async_rados;
 776   rgw::sal::RadosStore* store;
 777
 778   rgw_raw_obj obj;
 779
 780   bool going_down;
 781
 782   int num_pending_entries;
 783   std::list<std::string> pending_entries;
 784
 785   std::map<std::string, bufferlist> entries;
 786
 787   uint64_t window_size;
 788   uint64_t total_entries;
 789 public:
 790   RGWOmapAppend(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
 791                 const rgw_raw_obj& _obj,
 792                 uint64_t _window_size = OMAP_APPEND_MAX_ENTRIES_DEFAULT);
 793   int operate(const DoutPrefixProvider *dpp) override;
 794   void flush_pending();
 795   bool append(const std::string& s);
 796   bool finish();
 797
 798   uint64_t get_total_entries() {
 799     return total_entries;
 800   }
 801
 802   const rgw_raw_obj& get_obj() {
 803     return obj;
 804   }
 805 };
 806
 807 class RGWShardedOmapCRManager {
 808   RGWAsyncRadosProcessor *async_rados;
 809   rgw::sal::RadosStore* store;
 810   RGWCoroutine *op;
 811
 812   int num_shards;
 813
 814   std::vector<RGWOmapAppend *> shards;
 815 public:
 816   RGWShardedOmapCRManager(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store, RGWCoroutine *_op, int _num_shards, const rgw_pool& pool, const std::string& oid_prefix)
 817                       : async_rados(_async_rados),
 818                         store(_store), op(_op), num_shards(_num_shards) {
 819     shards.reserve(num_shards);
 820     for (int i = 0; i < num_shards; ++i) {
 821       char buf[oid_prefix.size() + 16];
 822       snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), i);
 823       RGWOmapAppend *shard = new RGWOmapAppend(async_rados, store, rgw_raw_obj(pool, buf));
 824       shard->get();
 825       shards.push_back(shard);
 826       op->spawn(shard, false);
 827     }
 828   }
 829
 830   ~RGWShardedOmapCRManager() {
 831     for (auto shard : shards) {
 832       shard->put();
 833     }
 834   }
 835
 836   bool append(const std::string& entry, int shard_id) {
 837     return shards[shard_id]->append(entry);
 838   }
 839   bool finish() {
 840     bool success = true;
 841     for (auto& append_op : shards) {
 842       success &= (append_op->finish() && (!append_op->is_error()));
 843     }
 844     return success;
 845   }
 846
 847   uint64_t get_total_entries(int shard_id) {
 848     return shards[shard_id]->get_total_entries();
 849   }
 850 };
 851
 852 class RGWAsyncGetBucketInstanceInfo : public RGWAsyncRadosRequest {
 853   rgw::sal::RadosStore* store;
 854   rgw_bucket bucket;
 855   const DoutPrefixProvider *dpp;
 856
 857 protected:
 858   int _send_request(const DoutPrefixProvider *dpp) override;
 859 public:
 860   RGWAsyncGetBucketInstanceInfo(RGWCoroutine *caller, RGWAioCompletionNotifier *cn,
 861                                 rgw::sal::RadosStore* _store, const rgw_bucket& bucket,
 862                                 const DoutPrefixProvider *dpp)
 863     : RGWAsyncRadosRequest(caller, cn), store(_store), bucket(bucket), dpp(dpp) {}
 864
 865   RGWBucketInfo bucket_info;
 866   std::map<std::string, bufferlist> attrs;
 867 };
 868
 869 class RGWGetBucketInstanceInfoCR : public RGWSimpleCoroutine {
 870   RGWAsyncRadosProcessor *async_rados;
 871   rgw::sal::RadosStore* store;
 872   rgw_bucket bucket;
 873   RGWBucketInfo *bucket_info;
 874   std::map<std::string, bufferlist> *pattrs;
 875   const DoutPrefixProvider *dpp;
 876
 877   RGWAsyncGetBucketInstanceInfo *req{nullptr};
 878
 879 public:
 880   // rgw_bucket constructor
 881   RGWGetBucketInstanceInfoCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
 882                              const rgw_bucket& _bucket, RGWBucketInfo *_bucket_info,
 883                              std::map<std::string, bufferlist> *_pattrs, const DoutPrefixProvider *dpp)
 884     : RGWSimpleCoroutine(_store->ctx()), async_rados(_async_rados), store(_store),
 885       bucket(_bucket), bucket_info(_bucket_info), pattrs(_pattrs), dpp(dpp) {}
 886   ~RGWGetBucketInstanceInfoCR() override {
 887     request_cleanup();
 888   }
 889   void request_cleanup() override {
 890     if (req) {
 891       req->finish();
 892       req = NULL;
 893     }
 894   }
 895
 896   int send_request(const DoutPrefixProvider *dpp) override {
 897     req = new RGWAsyncGetBucketInstanceInfo(this, stack->create_completion_notifier(), store, bucket, dpp);
 898     async_rados->queue(req);
 899     return 0;
 900   }
 901   int request_complete() override {
 902     if (bucket_info) {
 903       *bucket_info = std::move(req->bucket_info);
 904     }
 905     if (pattrs) {
 906       *pattrs = std::move(req->attrs);
 907     }
 908     return req->get_ret_status();
 909   }
 910 };
 911
 912 class RGWRadosBILogTrimCR : public RGWSimpleCoroutine {
 913   const RGWBucketInfo& bucket_info;
 914   int shard_id;
 915   RGWRados::BucketShard bs;
 916   std::string start_marker;
 917   std::string end_marker;
 918   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 919  public:
 920   RGWRadosBILogTrimCR(const DoutPrefixProvider *dpp,
 921                       rgw::sal::RadosStore* store, const RGWBucketInfo& bucket_info,
 922                       int shard_id, const std::string& start_marker,
 923                       const std::string& end_marker);
 924
 925   int send_request(const DoutPrefixProvider *dpp) override;
 926   int request_complete() override;
 927 };
 928
 929 class RGWAsyncFetchRemoteObj : public RGWAsyncRadosRequest {
 930   rgw::sal::RadosStore* store;
 931   rgw_zone_id source_zone;
 932
 933   std::optional<rgw_user> user_id;
 934
 935   rgw_bucket src_bucket;
 936   std::optional<rgw_placement_rule> dest_placement_rule;
 937   RGWBucketInfo dest_bucket_info;
 938
 939   rgw_obj_key key;
 940   std::optional<rgw_obj_key> dest_key;
 941   std::optional<uint64_t> versioned_epoch;
 942
 943   real_time src_mtime;
 944
 945   bool copy_if_newer;
 946   std::shared_ptr<RGWFetchObjFilter> filter;
 947   rgw_zone_set zones_trace;
 948   PerfCounters* counters;
 949   const DoutPrefixProvider *dpp;
 950
 951 protected:
 952   int _send_request(const DoutPrefixProvider *dpp) override;
 953 public:
 954   RGWAsyncFetchRemoteObj(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store,
 955                          const rgw_zone_id& _source_zone,
 956                          std::optional<rgw_user>& _user_id,
 957                          const rgw_bucket& _src_bucket,
 958                          std::optional<rgw_placement_rule> _dest_placement_rule,
 959                          const RGWBucketInfo& _dest_bucket_info,
 960                          const rgw_obj_key& _key,
 961                          const std::optional<rgw_obj_key>& _dest_key,
 962                          std::optional<uint64_t> _versioned_epoch,
 963                          bool _if_newer,
 964                          std::shared_ptr<RGWFetchObjFilter> _filter,
 965                          rgw_zone_set *_zones_trace,
 966                          PerfCounters* counters, const DoutPrefixProvider *dpp)
 967     : RGWAsyncRadosRequest(caller, cn), store(_store),
 968       source_zone(_source_zone),
 969       user_id(_user_id),
 970       src_bucket(_src_bucket),
 971       dest_placement_rule(_dest_placement_rule),
 972       dest_bucket_info(_dest_bucket_info),
 973       key(_key),
 974       dest_key(_dest_key),
 975       versioned_epoch(_versioned_epoch),
 976       copy_if_newer(_if_newer),
 977       filter(_filter),
 978       counters(counters),
 979       dpp(dpp)
 980   {
 981     if (_zones_trace) {
 982       zones_trace = *_zones_trace;
 983     }
 984   }
 985 };
 986
 987 class RGWFetchRemoteObjCR : public RGWSimpleCoroutine {
 988   CephContext *cct;
 989   RGWAsyncRadosProcessor *async_rados;
 990   rgw::sal::RadosStore* store;
 991   rgw_zone_id source_zone;
 992
 993   std::optional<rgw_user> user_id;
 994
 995   rgw_bucket src_bucket;
 996   std::optional<rgw_placement_rule> dest_placement_rule;
 997   RGWBucketInfo dest_bucket_info;
 998
 999   rgw_obj_key key;
1000   std::optional<rgw_obj_key> dest_key;
1001   std::optional<uint64_t> versioned_epoch;
1002
1003   real_time src_mtime;
1004
1005   bool copy_if_newer;
1006
1007   std::shared_ptr<RGWFetchObjFilter> filter;
1008
1009   RGWAsyncFetchRemoteObj *req;
1010   rgw_zone_set *zones_trace;
1011   PerfCounters* counters;
1012   const DoutPrefixProvider *dpp;
1013
1014 public:
1015   RGWFetchRemoteObjCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
1016                       const rgw_zone_id& _source_zone,
1017                       std::optional<rgw_user> _user_id,
1018                       const rgw_bucket& _src_bucket,
1019                       std::optional<rgw_placement_rule> _dest_placement_rule,
1020                       const RGWBucketInfo& _dest_bucket_info,
1021                       const rgw_obj_key& _key,
1022                       const std::optional<rgw_obj_key>& _dest_key,
1023                       std::optional<uint64_t> _versioned_epoch,
1024                       bool _if_newer,
1025                       std::shared_ptr<RGWFetchObjFilter> _filter,
1026                       rgw_zone_set *_zones_trace,
1027                       PerfCounters* counters, const DoutPrefixProvider *dpp)
1028     : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()),
1029       async_rados(_async_rados), store(_store),
1030       source_zone(_source_zone),
1031       user_id(_user_id),
1032       src_bucket(_src_bucket),
1033       dest_placement_rule(_dest_placement_rule),
1034       dest_bucket_info(_dest_bucket_info),
1035       key(_key),
1036       dest_key(_dest_key),
1037       versioned_epoch(_versioned_epoch),
1038       copy_if_newer(_if_newer),
1039       filter(_filter),
1040       req(NULL),
1041       zones_trace(_zones_trace), counters(counters), dpp(dpp) {}
1042
1043
1044   ~RGWFetchRemoteObjCR() override {
1045     request_cleanup();
1046   }
1047
1048   void request_cleanup() override {
1049     if (req) {
1050       req->finish();
1051       req = NULL;
1052     }
1053   }
1054
1055   int send_request(const DoutPrefixProvider *dpp) override {
1056     req = new RGWAsyncFetchRemoteObj(this, stack->create_completion_notifier(), store,
1057                                      source_zone, user_id, src_bucket, dest_placement_rule, dest_bucket_info,
1058                                      key, dest_key, versioned_epoch, copy_if_newer, filter,
1059                                      zones_trace, counters, dpp);
1060     async_rados->queue(req);
1061     return 0;
1062   }
1063
1064   int request_complete() override {
1065     return req->get_ret_status();
1066   }
1067 };
1068
1069 class RGWAsyncStatRemoteObj : public RGWAsyncRadosRequest {
1070   rgw::sal::RadosStore* store;
1071   rgw_zone_id source_zone;
1072
1073   rgw_bucket src_bucket;
1074   rgw_obj_key key;
1075
1076   ceph::real_time *pmtime;
1077   uint64_t *psize;
1078   std::string *petag;
1079   std::map<std::string, bufferlist> *pattrs;
1080   std::map<std::string, std::string> *pheaders;
1081
1082 protected:
1083   int _send_request(const DoutPrefixProvider *dpp) override;
1084 public:
1085   RGWAsyncStatRemoteObj(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store,
1086                          const rgw_zone_id& _source_zone,
1087                          rgw_bucket& _src_bucket,
1088                          const rgw_obj_key& _key,
1089                          ceph::real_time *_pmtime,
1090                          uint64_t *_psize,
1091                          std::string *_petag,
1092                          std::map<std::string, bufferlist> *_pattrs,
1093                          std::map<std::string, std::string> *_pheaders) : RGWAsyncRadosRequest(caller, cn), store(_store),
1094                                                       source_zone(_source_zone),
1095                                                       src_bucket(_src_bucket),
1096                                                       key(_key),
1097                                                       pmtime(_pmtime),
1098                                                       psize(_psize),
1099                                                       petag(_petag),
1100                                                       pattrs(_pattrs),
1101                                                       pheaders(_pheaders) {}
1102 };
1103
1104 class RGWStatRemoteObjCR : public RGWSimpleCoroutine {
1105   CephContext *cct;
1106   RGWAsyncRadosProcessor *async_rados;
1107   rgw::sal::RadosStore* store;
1108   rgw_zone_id source_zone;
1109
1110   rgw_bucket src_bucket;
1111   rgw_obj_key key;
1112
1113   ceph::real_time *pmtime;
1114   uint64_t *psize;
1115   std::string *petag;
1116   std::map<std::string, bufferlist> *pattrs;
1117   std::map<std::string, std::string> *pheaders;
1118
1119   RGWAsyncStatRemoteObj *req;
1120
1121 public:
1122   RGWStatRemoteObjCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
1123                       const rgw_zone_id& _source_zone,
1124                       rgw_bucket& _src_bucket,
1125                       const rgw_obj_key& _key,
1126                       ceph::real_time *_pmtime,
1127                       uint64_t *_psize,
1128                       std::string *_petag,
1129                       std::map<std::string, bufferlist> *_pattrs,
1130                       std::map<std::string, std::string> *_pheaders) : RGWSimpleCoroutine(_store->ctx()), cct(_store->ctx()),
1131                                        async_rados(_async_rados), store(_store),
1132                                        source_zone(_source_zone),
1133                                        src_bucket(_src_bucket),
1134                                        key(_key),
1135                                        pmtime(_pmtime),
1136                                        psize(_psize),
1137                                        petag(_petag),
1138                                        pattrs(_pattrs),
1139                                        pheaders(_pheaders),
1140                                        req(NULL) {}
1141
1142
1143   ~RGWStatRemoteObjCR() override {
1144     request_cleanup();
1145   }
1146
1147   void request_cleanup() override {
1148     if (req) {
1149       req->finish();
1150       req = NULL;
1151     }
1152   }
1153
1154   int send_request(const DoutPrefixProvider *dpp) override {
1155     req = new RGWAsyncStatRemoteObj(this, stack->create_completion_notifier(), store, source_zone,
1156                                     src_bucket, key, pmtime, psize, petag, pattrs, pheaders);
1157     async_rados->queue(req);
1158     return 0;
1159   }
1160
1161   int request_complete() override {
1162     return req->get_ret_status();
1163   }
1164 };
1165
1166 class RGWAsyncRemoveObj : public RGWAsyncRadosRequest {
1167   const DoutPrefixProvider *dpp;
1168   rgw::sal::RadosStore* store;
1169   rgw_zone_id source_zone;
1170
1171   RGWBucketInfo bucket_info;
1172
1173   rgw_obj_key key;
1174   std::string owner;
1175   std::string owner_display_name;
1176   bool versioned;
1177   uint64_t versioned_epoch;
1178   std::string marker_version_id;
1179
1180   bool del_if_older;
1181   ceph::real_time timestamp;
1182   rgw_zone_set zones_trace;
1183
1184 protected:
1185   int _send_request(const DoutPrefixProvider *dpp) override;
1186 public:
1187   RGWAsyncRemoveObj(const DoutPrefixProvider *_dpp, RGWCoroutine *caller, RGWAioCompletionNotifier *cn,
1188                          rgw::sal::RadosStore* _store,
1189                          const rgw_zone_id& _source_zone,
1190                          RGWBucketInfo& _bucket_info,
1191                          const rgw_obj_key& _key,
1192                          const std::string& _owner,
1193                          const std::string& _owner_display_name,
1194                          bool _versioned,
1195                          uint64_t _versioned_epoch,
1196                          bool _delete_marker,
1197                          bool _if_older,
1198                          real_time& _timestamp,
1199                          rgw_zone_set* _zones_trace) : RGWAsyncRadosRequest(caller, cn), dpp(_dpp), store(_store),
1200                                                       source_zone(_source_zone),
1201                                                       bucket_info(_bucket_info),
1202                                                       key(_key),
1203                                                       owner(_owner),
1204                                                       owner_display_name(_owner_display_name),
1205                                                       versioned(_versioned),
1206                                                       versioned_epoch(_versioned_epoch),
1207                                                       del_if_older(_if_older),
1208                                                       timestamp(_timestamp) {
1209     if (_delete_marker) {
1210       marker_version_id = key.instance;
1211     }
1212
1213     if (_zones_trace) {
1214       zones_trace = *_zones_trace;
1215     }
1216   }
1217 };
1218
1219 class RGWRemoveObjCR : public RGWSimpleCoroutine {
1220   const DoutPrefixProvider *dpp;
1221   CephContext *cct;
1222   RGWAsyncRadosProcessor *async_rados;
1223   rgw::sal::RadosStore* store;
1224   rgw_zone_id source_zone;
1225
1226   RGWBucketInfo bucket_info;
1227
1228   rgw_obj_key key;
1229   bool versioned;
1230   uint64_t versioned_epoch;
1231   bool delete_marker;
1232   std::string owner;
1233   std::string owner_display_name;
1234
1235   bool del_if_older;
1236   real_time timestamp;
1237
1238   RGWAsyncRemoveObj *req;
1239
1240   rgw_zone_set *zones_trace;
1241
1242 public:
1243   RGWRemoveObjCR(const DoutPrefixProvider *_dpp, RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
1244                       const rgw_zone_id& _source_zone,
1245                       RGWBucketInfo& _bucket_info,
1246                       const rgw_obj_key& _key,
1247                       bool _versioned,
1248                       uint64_t _versioned_epoch,
1249                       std::string *_owner,
1250                       std::string *_owner_display_name,
1251                       bool _delete_marker,
1252                       real_time *_timestamp,
1253                       rgw_zone_set *_zones_trace) : RGWSimpleCoroutine(_store->ctx()), dpp(_dpp), cct(_store->ctx()),
1254                                        async_rados(_async_rados), store(_store),
1255                                        source_zone(_source_zone),
1256                                        bucket_info(_bucket_info),
1257                                        key(_key),
1258                                        versioned(_versioned),
1259                                        versioned_epoch(_versioned_epoch),
1260                                        delete_marker(_delete_marker), req(NULL), zones_trace(_zones_trace) {
1261     del_if_older = (_timestamp != NULL);
1262     if (_timestamp) {
1263       timestamp = *_timestamp;
1264     }
1265
1266     if (_owner) {
1267       owner = *_owner;
1268     }
1269
1270     if (_owner_display_name) {
1271       owner_display_name = *_owner_display_name;
1272     }
1273   }
1274   ~RGWRemoveObjCR() override {
1275     request_cleanup();
1276   }
1277
1278   void request_cleanup() override {
1279     if (req) {
1280       req->finish();
1281       req = NULL;
1282     }
1283   }
1284
1285   int send_request(const DoutPrefixProvider *dpp) override {
1286     req = new RGWAsyncRemoveObj(dpp, this, stack->create_completion_notifier(), store, source_zone, bucket_info,
1287                                 key, owner, owner_display_name, versioned, versioned_epoch,
1288                                 delete_marker, del_if_older, timestamp, zones_trace);
1289     async_rados->queue(req);
1290     return 0;
1291   }
1292
1293   int request_complete() override {
1294     return req->get_ret_status();
1295   }
1296 };
1297
1298 class RGWContinuousLeaseCR : public RGWCoroutine {
1299   RGWAsyncRadosProcessor *async_rados;
1300   rgw::sal::RadosStore* store;
1301
1302   const rgw_raw_obj obj;
1303
1304   const std::string lock_name;
1305   const std::string cookie;
1306
1307   int interval;
1308   bool going_down{ false };
1309   bool locked{false};
1310
1311   RGWCoroutine *caller;
1312
1313   bool aborted{false};
1314
1315 public:
1316   RGWContinuousLeaseCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
1317                        const rgw_raw_obj& _obj,
1318                        const std::string& _lock_name, int _interval, RGWCoroutine *_caller)
1319     : RGWCoroutine(_store->ctx()), async_rados(_async_rados), store(_store),
1320     obj(_obj), lock_name(_lock_name),
1321     cookie(RGWSimpleRadosLockCR::gen_random_cookie(cct)),
1322     interval(_interval), caller(_caller)
1323   {}
1324
1325   virtual ~RGWContinuousLeaseCR() override;
1326
1327   int operate(const DoutPrefixProvider *dpp) override;
1328
1329   bool is_locked() const {
1330     return locked;
1331   }
1332
1333   void set_locked(bool status) {
1334     locked = status;
1335   }
1336
1337   void go_down() {
1338     going_down = true;
1339     wakeup();
1340   }
1341
1342   void abort() {
1343     aborted = true;
1344   }
1345 };
1346
1347 class RGWRadosTimelogAddCR : public RGWSimpleCoroutine {
1348   const DoutPrefixProvider *dpp;
1349   rgw::sal::RadosStore* store;
1350   std::list<cls_log_entry> entries;
1351
1352   std::string oid;
1353
1354   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
1355
1356 public:
1357   RGWRadosTimelogAddCR(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* _store, const std::string& _oid,
1358                         const cls_log_entry& entry);
1359
1360   int send_request(const DoutPrefixProvider *dpp) override;
1361   int request_complete() override;
1362 };
1363
1364 class RGWRadosTimelogTrimCR : public RGWSimpleCoroutine {
1365   const DoutPrefixProvider *dpp;
1366   rgw::sal::RadosStore* store;
1367   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
1368  protected:
1369   std::string oid;
1370   real_time start_time;
1371   real_time end_time;
1372   std::string from_marker;
1373   std::string to_marker;
1374
1375  public:
1376   RGWRadosTimelogTrimCR(const DoutPrefixProvider *dpp,
1377                         rgw::sal::RadosStore* store, const std::string& oid,
1378                         const real_time& start_time, const real_time& end_time,
1379                         const std::string& from_marker,
1380                         const std::string& to_marker);
1381
1382   int send_request(const DoutPrefixProvider *dpp) override;
1383   int request_complete() override;
1384 };
1385
1386 // wrapper to update last_trim_marker on success
1387 class RGWSyncLogTrimCR : public RGWRadosTimelogTrimCR {
1388   CephContext *cct;
1389   std::string *last_trim_marker;
1390  public:
1391   static constexpr const char* max_marker = "99999999";
1392
1393   RGWSyncLogTrimCR(const DoutPrefixProvider *dpp,
1394                    rgw::sal::RadosStore* store, const std::string& oid,
1395                    const std::string& to_marker, std::string *last_trim_marker);
1396   int request_complete() override;
1397 };
1398
1399 class RGWAsyncStatObj : public RGWAsyncRadosRequest {
1400   const DoutPrefixProvider *dpp;
1401   rgw::sal::RadosStore* store;
1402   RGWBucketInfo bucket_info;
1403   rgw_obj obj;
1404   uint64_t *psize;
1405   real_time *pmtime;
1406   uint64_t *pepoch;
1407   RGWObjVersionTracker *objv_tracker;
1408 protected:
1409   int _send_request(const DoutPrefixProvider *dpp) override;
1410 public:
1411   RGWAsyncStatObj(const DoutPrefixProvider *dpp, RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* store,
1412                   const RGWBucketInfo& _bucket_info, const rgw_obj& obj, uint64_t *psize = nullptr,
1413                   real_time *pmtime = nullptr, uint64_t *pepoch = nullptr,
1414                   RGWObjVersionTracker *objv_tracker = nullptr)
1415           : RGWAsyncRadosRequest(caller, cn), dpp(dpp), store(store), obj(obj), psize(psize),
1416           pmtime(pmtime), pepoch(pepoch), objv_tracker(objv_tracker) {}
1417 };
1418
1419 class RGWStatObjCR : public RGWSimpleCoroutine {
1420   const DoutPrefixProvider *dpp;
1421   rgw::sal::RadosStore* store;
1422   RGWAsyncRadosProcessor *async_rados;
1423   RGWBucketInfo bucket_info;
1424   rgw_obj obj;
1425   uint64_t *psize;
1426   real_time *pmtime;
1427   uint64_t *pepoch;
1428   RGWObjVersionTracker *objv_tracker;
1429   RGWAsyncStatObj *req = nullptr;
1430  public:
1431   RGWStatObjCR(const DoutPrefixProvider *dpp, RGWAsyncRadosProcessor *async_rados, rgw::sal::RadosStore* store,
1432           const RGWBucketInfo& _bucket_info, const rgw_obj& obj, uint64_t *psize = nullptr,
1433           real_time* pmtime = nullptr, uint64_t *pepoch = nullptr,
1434           RGWObjVersionTracker *objv_tracker = nullptr);
1435   ~RGWStatObjCR() override {
1436     request_cleanup();
1437   }
1438   void request_cleanup() override;
1439
1440   int send_request(const DoutPrefixProvider *dpp) override;
1441   int request_complete() override;
1442 };
1443
1444 /// coroutine wrapper for IoCtx::aio_notify()
1445 class RGWRadosNotifyCR : public RGWSimpleCoroutine {
1446   rgw::sal::RadosStore* const store;
1447   const rgw_raw_obj obj;
1448   bufferlist request;
1449   const uint64_t timeout_ms;
1450   bufferlist *response;
1451   rgw_rados_ref ref;
1452   boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
1453
1454 public:
1455   RGWRadosNotifyCR(rgw::sal::RadosStore* store, const rgw_raw_obj& obj,
1456                    bufferlist& request, uint64_t timeout_ms,
1457                    bufferlist *response);
1458
1459   int send_request(const DoutPrefixProvider *dpp) override;
1460   int request_complete() override;
1461 };
1462
1463 #endif