ceph/src/librbd/io/ImageRequest.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "librbd/io/ImageRequest.h"
   5 #include "librbd/ImageCtx.h"
   6 #include "librbd/internal.h"
   7 #include "librbd/Journal.h"
   8 #include "librbd/Utils.h"
   9 #include "librbd/cache/ImageCache.h"
  10 #include "librbd/io/AioCompletion.h"
  11 #include "librbd/io/ObjectRequest.h"
  12 #include "librbd/journal/Types.h"
  13 #include "include/rados/librados.hpp"
  14 #include "common/WorkQueue.h"
  15 #include "osdc/Striper.h"
  16
  17 #define dout_subsys ceph_subsys_rbd
  18 #undef dout_prefix
  19 #define dout_prefix *_dout << "librbd::io::ImageRequest: " << this \
  20                            << " " << __func__ << ": "
  21
  22 namespace librbd {
  23 namespace io {
  24
  25 using util::get_image_ctx;
  26
  27 namespace {
  28
  29 template <typename ImageCtxT = ImageCtx>
  30 struct C_DiscardJournalCommit : public Context {
  31   typedef std::vector<ObjectExtent> ObjectExtents;
  32
  33   ImageCtxT &image_ctx;
  34   AioCompletion *aio_comp;
  35   ObjectExtents object_extents;
  36
  37   C_DiscardJournalCommit(ImageCtxT &_image_ctx, AioCompletion *_aio_comp,
  38                          const ObjectExtents &_object_extents, uint64_t tid)
  39     : image_ctx(_image_ctx), aio_comp(_aio_comp),
  40       object_extents(_object_extents) {
  41     CephContext *cct = image_ctx.cct;
  42     ldout(cct, 20) << "delaying cache discard until journal tid " << tid << " "
  43                    << "safe" << dendl;
  44
  45     aio_comp->add_request();
  46   }
  47
  48   void finish(int r) override {
  49     CephContext *cct = image_ctx.cct;
  50     ldout(cct, 20) << "C_DiscardJournalCommit: "
  51                    << "journal committed: discarding from cache" << dendl;
  52
  53     Mutex::Locker cache_locker(image_ctx.cache_lock);
  54     image_ctx.object_cacher->discard_set(image_ctx.object_set, object_extents);
  55     aio_comp->complete_request(r);
  56   }
  57 };
  58
  59 template <typename ImageCtxT = ImageCtx>
  60 struct C_FlushJournalCommit : public Context {
  61   ImageCtxT &image_ctx;
  62   AioCompletion *aio_comp;
  63
  64   C_FlushJournalCommit(ImageCtxT &_image_ctx, AioCompletion *_aio_comp,
  65                        uint64_t tid)
  66     : image_ctx(_image_ctx), aio_comp(_aio_comp) {
  67     CephContext *cct = image_ctx.cct;
  68     ldout(cct, 20) << "delaying flush until journal tid " << tid << " "
  69                    << "safe" << dendl;
  70
  71     aio_comp->add_request();
  72   }
  73
  74   void finish(int r) override {
  75     CephContext *cct = image_ctx.cct;
  76     ldout(cct, 20) << "C_FlushJournalCommit: journal committed" << dendl;
  77     aio_comp->complete_request(r);
  78   }
  79 };
  80
  81 template <typename ImageCtxT>
  82 class C_ObjectCacheRead : public Context {
  83 public:
  84   explicit C_ObjectCacheRead(ImageCtxT &ictx, ObjectReadRequest<ImageCtxT> *req)
  85     : m_image_ctx(ictx), m_req(req), m_enqueued(false) {}
  86
  87   void complete(int r) override {
  88     if (!m_enqueued) {
  89       // cache_lock creates a lock ordering issue -- so re-execute this context
  90       // outside the cache_lock
  91       m_enqueued = true;
  92       m_image_ctx.op_work_queue->queue(this, r);
  93       return;
  94     }
  95     Context::complete(r);
  96   }
  97
  98 protected:
  99   void finish(int r) override {
 100     m_req->complete(r);
 101   }
 102
 103 private:
 104   ImageCtxT &m_image_ctx;
 105   ObjectReadRequest<ImageCtxT> *m_req;
 106   bool m_enqueued;
 107 };
 108
 109 } // anonymous namespace
 110
 111 template <typename I>
 112 void ImageRequest<I>::aio_read(I *ictx, AioCompletion *c,
 113                                Extents &&image_extents,
 114                                ReadResult &&read_result, int op_flags,
 115                                const ZTracer::Trace &parent_trace) {
 116   ImageReadRequest<I> req(*ictx, c, std::move(image_extents),
 117                           std::move(read_result), op_flags, parent_trace);
 118   req.send();
 119 }
 120
 121 template <typename I>
 122 void ImageRequest<I>::aio_write(I *ictx, AioCompletion *c,
 123                                 Extents &&image_extents, bufferlist &&bl,
 124                                 int op_flags,
 125                                 const ZTracer::Trace &parent_trace) {
 126   ImageWriteRequest<I> req(*ictx, c, std::move(image_extents), std::move(bl),
 127                            op_flags, parent_trace);
 128   req.send();
 129 }
 130
 131 template <typename I>
 132 void ImageRequest<I>::aio_discard(I *ictx, AioCompletion *c,
 133                                   uint64_t off, uint64_t len,
 134                                   bool skip_partial_discard,
 135                                   const ZTracer::Trace &parent_trace) {
 136   ImageDiscardRequest<I> req(*ictx, c, off, len, skip_partial_discard,
 137                              parent_trace);
 138   req.send();
 139 }
 140
 141 template <typename I>
 142 void ImageRequest<I>::aio_flush(I *ictx, AioCompletion *c,
 143                                 const ZTracer::Trace &parent_trace) {
 144   ImageFlushRequest<I> req(*ictx, c, parent_trace);
 145   req.send();
 146 }
 147
 148 template <typename I>
 149 void ImageRequest<I>::aio_writesame(I *ictx, AioCompletion *c,
 150                                     uint64_t off, uint64_t len,
 151                                     bufferlist &&bl, int op_flags,
 152                                     const ZTracer::Trace &parent_trace) {
 153   ImageWriteSameRequest<I> req(*ictx, c, off, len, std::move(bl), op_flags,
 154                                parent_trace);
 155   req.send();
 156 }
 157
 158 template <typename I>
 159 void ImageRequest<I>::send() {
 160   I &image_ctx = this->m_image_ctx;
 161   assert(m_aio_comp->is_initialized(get_aio_type()));
 162   assert(m_aio_comp->is_started() ^ (get_aio_type() == AIO_TYPE_FLUSH));
 163
 164   CephContext *cct = image_ctx.cct;
 165   AioCompletion *aio_comp = this->m_aio_comp;
 166   ldout(cct, 20) << get_request_type() << ": ictx=" << &image_ctx << ", "
 167                  << "completion=" << aio_comp <<  dendl;
 168
 169   aio_comp->get();
 170   int r = clip_request();
 171   if (r < 0) {
 172     m_aio_comp->fail(r);
 173     return;
 174   }
 175
 176   if (m_bypass_image_cache || m_image_ctx.image_cache == nullptr) {
 177     send_request();
 178   } else {
 179     send_image_cache_request();
 180   }
 181 }
 182
 183 template <typename I>
 184 int ImageRequest<I>::clip_request() {
 185   RWLock::RLocker snap_locker(m_image_ctx.snap_lock);
 186   for (auto &image_extent : m_image_extents) {
 187     auto clip_len = image_extent.second;
 188     int r = clip_io(get_image_ctx(&m_image_ctx), image_extent.first, &clip_len);
 189     if (r < 0) {
 190       return r;
 191     }
 192
 193     image_extent.second = clip_len;
 194   }
 195   return 0;
 196 }
 197
 198 template <typename I>
 199 void ImageRequest<I>::start_op() {
 200   m_aio_comp->start_op();
 201 }
 202
 203 template <typename I>
 204 void ImageRequest<I>::fail(int r) {
 205   AioCompletion *aio_comp = this->m_aio_comp;
 206   aio_comp->get();
 207   aio_comp->fail(r);
 208 }
 209
 210 template <typename I>
 211 ImageReadRequest<I>::ImageReadRequest(I &image_ctx, AioCompletion *aio_comp,
 212                                       Extents &&image_extents,
 213                                       ReadResult &&read_result, int op_flags,
 214                                       const ZTracer::Trace &parent_trace)
 215   : ImageRequest<I>(image_ctx, aio_comp, std::move(image_extents), "read",
 216                     parent_trace),
 217     m_op_flags(op_flags) {
 218   aio_comp->read_result = std::move(read_result);
 219 }
 220
 221 template <typename I>
 222 int ImageReadRequest<I>::clip_request() {
 223   int r = ImageRequest<I>::clip_request();
 224   if (r < 0) {
 225     return r;
 226   }
 227
 228   uint64_t buffer_length = 0;
 229   auto &image_extents = this->m_image_extents;
 230   for (auto &image_extent : image_extents) {
 231     buffer_length += image_extent.second;
 232   }
 233   this->m_aio_comp->read_result.set_clip_length(buffer_length);
 234   return 0;
 235 }
 236
 237 template <typename I>
 238 void ImageReadRequest<I>::send_request() {
 239   I &image_ctx = this->m_image_ctx;
 240   CephContext *cct = image_ctx.cct;
 241
 242   auto &image_extents = this->m_image_extents;
 243   if (image_ctx.object_cacher && image_ctx.readahead_max_bytes > 0 &&
 244       !(m_op_flags & LIBRADOS_OP_FLAG_FADVISE_RANDOM)) {
 245     readahead(get_image_ctx(&image_ctx), image_extents);
 246   }
 247
 248   AioCompletion *aio_comp = this->m_aio_comp;
 249   librados::snap_t snap_id;
 250   map<object_t,vector<ObjectExtent> > object_extents;
 251   uint64_t buffer_ofs = 0;
 252   {
 253     // prevent image size from changing between computing clip and recording
 254     // pending async operation
 255     RWLock::RLocker snap_locker(image_ctx.snap_lock);
 256     snap_id = image_ctx.snap_id;
 257
 258     // map image extents to object extents
 259     for (auto &extent : image_extents) {
 260       if (extent.second == 0) {
 261         continue;
 262       }
 263
 264       Striper::file_to_extents(cct, image_ctx.format_string, &image_ctx.layout,
 265                                extent.first, extent.second, 0, object_extents,
 266                                buffer_ofs);
 267       buffer_ofs += extent.second;
 268     }
 269   }
 270
 271   // pre-calculate the expected number of read requests
 272   uint32_t request_count = 0;
 273   for (auto &object_extent : object_extents) {
 274     request_count += object_extent.second.size();
 275   }
 276   aio_comp->set_request_count(request_count);
 277
 278   // issue the requests
 279   for (auto &object_extent : object_extents) {
 280     for (auto &extent : object_extent.second) {
 281       ldout(cct, 20) << "oid " << extent.oid << " " << extent.offset << "~"
 282                      << extent.length << " from " << extent.buffer_extents
 283                      << dendl;
 284
 285       auto req_comp = new io::ReadResult::C_SparseReadRequest<I>(
 286         aio_comp);
 287       ObjectReadRequest<I> *req = ObjectReadRequest<I>::create(
 288         &image_ctx, extent.oid.name, extent.objectno, extent.offset,
 289         extent.length, extent.buffer_extents, snap_id, true, m_op_flags,
 290         this->m_trace, req_comp);
 291       req_comp->request = req;
 292
 293       if (image_ctx.object_cacher) {
 294         C_ObjectCacheRead<I> *cache_comp = new C_ObjectCacheRead<I>(image_ctx,
 295                                                                     req);
 296         image_ctx.aio_read_from_cache(
 297           extent.oid, extent.objectno, &req->data(), extent.length,
 298           extent.offset, cache_comp, m_op_flags,
 299           (this->m_trace.valid() ? &this->m_trace : nullptr));
 300       } else {
 301         req->send();
 302       }
 303     }
 304   }
 305
 306   aio_comp->put();
 307
 308   image_ctx.perfcounter->inc(l_librbd_rd);
 309   image_ctx.perfcounter->inc(l_librbd_rd_bytes, buffer_ofs);
 310 }
 311
 312 template <typename I>
 313 void ImageReadRequest<I>::send_image_cache_request() {
 314   I &image_ctx = this->m_image_ctx;
 315   assert(image_ctx.image_cache != nullptr);
 316
 317   AioCompletion *aio_comp = this->m_aio_comp;
 318   aio_comp->set_request_count(1);
 319
 320   auto *req_comp = new io::ReadResult::C_ImageReadRequest(
 321     aio_comp, this->m_image_extents);
 322   image_ctx.image_cache->aio_read(std::move(this->m_image_extents),
 323                                   &req_comp->bl, m_op_flags,
 324                                   req_comp);
 325 }
 326
 327 template <typename I>
 328 void AbstractImageWriteRequest<I>::send_request() {
 329   I &image_ctx = this->m_image_ctx;
 330   CephContext *cct = image_ctx.cct;
 331
 332   RWLock::RLocker md_locker(image_ctx.md_lock);
 333
 334   bool journaling = false;
 335
 336   AioCompletion *aio_comp = this->m_aio_comp;
 337   uint64_t clip_len = 0;
 338   ObjectExtents object_extents;
 339   ::SnapContext snapc;
 340   {
 341     // prevent image size from changing between computing clip and recording
 342     // pending async operation
 343     RWLock::RLocker snap_locker(image_ctx.snap_lock);
 344     if (image_ctx.snap_id != CEPH_NOSNAP || image_ctx.read_only) {
 345       aio_comp->fail(-EROFS);
 346       return;
 347     }
 348
 349     for (auto &extent : this->m_image_extents) {
 350       if (extent.second == 0) {
 351         continue;
 352       }
 353
 354       // map to object extents
 355       Striper::file_to_extents(cct, image_ctx.format_string, &image_ctx.layout,
 356                                extent.first, extent.second, 0, object_extents);
 357       clip_len += extent.second;
 358     }
 359
 360     snapc = image_ctx.snapc;
 361     journaling = (image_ctx.journal != nullptr &&
 362                   image_ctx.journal->is_journal_appending());
 363   }
 364
 365   prune_object_extents(object_extents);
 366
 367   if (!object_extents.empty()) {
 368     uint64_t journal_tid = 0;
 369     aio_comp->set_request_count(
 370       object_extents.size() + get_object_cache_request_count(journaling));
 371
 372     ObjectRequests requests;
 373     send_object_requests(object_extents, snapc,
 374                          (journaling ? &requests : nullptr));
 375
 376     if (journaling) {
 377       // in-flight ops are flushed prior to closing the journal
 378       assert(image_ctx.journal != NULL);
 379       journal_tid = append_journal_event(requests, m_synchronous);
 380     }
 381
 382     if (image_ctx.object_cacher != NULL) {
 383       send_object_cache_requests(object_extents, journal_tid);
 384     }
 385   } else {
 386     // no IO to perform -- fire completion
 387     aio_comp->unblock();
 388   }
 389
 390   update_stats(clip_len);
 391   aio_comp->put();
 392 }
 393
 394 template <typename I>
 395 void AbstractImageWriteRequest<I>::send_object_requests(
 396     const ObjectExtents &object_extents, const ::SnapContext &snapc,
 397     ObjectRequests *object_requests) {
 398   I &image_ctx = this->m_image_ctx;
 399   CephContext *cct = image_ctx.cct;
 400
 401   AioCompletion *aio_comp = this->m_aio_comp;
 402   for (ObjectExtents::const_iterator p = object_extents.begin();
 403        p != object_extents.end(); ++p) {
 404     ldout(cct, 20) << "oid " << p->oid << " " << p->offset << "~" << p->length
 405                    << " from " << p->buffer_extents << dendl;
 406     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 407     ObjectRequestHandle *request = create_object_request(*p, snapc,
 408                                                             req_comp);
 409
 410     // if journaling, stash the request for later; otherwise send
 411     if (request != NULL) {
 412       if (object_requests != NULL) {
 413         object_requests->push_back(request);
 414       } else {
 415         request->send();
 416       }
 417     }
 418   }
 419 }
 420
 421 template <typename I>
 422 void ImageWriteRequest<I>::assemble_extent(const ObjectExtent &object_extent,
 423                                            bufferlist *bl) {
 424   for (auto q = object_extent.buffer_extents.begin();
 425        q != object_extent.buffer_extents.end(); ++q) {
 426     bufferlist sub_bl;
 427     sub_bl.substr_of(m_bl, q->first, q->second);
 428     bl->claim_append(sub_bl);
 429   }
 430 }
 431
 432 template <typename I>
 433 uint64_t ImageWriteRequest<I>::append_journal_event(
 434     const ObjectRequests &requests, bool synchronous) {
 435   I &image_ctx = this->m_image_ctx;
 436
 437   uint64_t tid = 0;
 438   uint64_t buffer_offset = 0;
 439   assert(!this->m_image_extents.empty());
 440   for (auto &extent : this->m_image_extents) {
 441     bufferlist sub_bl;
 442     sub_bl.substr_of(m_bl, buffer_offset, extent.second);
 443     buffer_offset += extent.second;
 444
 445     tid = image_ctx.journal->append_write_event(extent.first, extent.second,
 446                                                 sub_bl, requests, synchronous);
 447   }
 448
 449   if (image_ctx.object_cacher == NULL) {
 450     AioCompletion *aio_comp = this->m_aio_comp;
 451     aio_comp->associate_journal_event(tid);
 452   }
 453   return tid;
 454 }
 455
 456 template <typename I>
 457 void ImageWriteRequest<I>::send_image_cache_request() {
 458   I &image_ctx = this->m_image_ctx;
 459   assert(image_ctx.image_cache != nullptr);
 460
 461   AioCompletion *aio_comp = this->m_aio_comp;
 462   aio_comp->set_request_count(1);
 463   C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 464   image_ctx.image_cache->aio_write(std::move(this->m_image_extents),
 465                                    std::move(m_bl), m_op_flags, req_comp);
 466 }
 467
 468 template <typename I>
 469 void ImageWriteRequest<I>::send_object_cache_requests(
 470     const ObjectExtents &object_extents, uint64_t journal_tid) {
 471   I &image_ctx = this->m_image_ctx;
 472   for (auto p = object_extents.begin(); p != object_extents.end(); ++p) {
 473     const ObjectExtent &object_extent = *p;
 474
 475     bufferlist bl;
 476     assemble_extent(object_extent, &bl);
 477
 478     AioCompletion *aio_comp = this->m_aio_comp;
 479     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 480     image_ctx.write_to_cache(
 481       object_extent.oid, bl, object_extent.length, object_extent.offset,
 482       req_comp, m_op_flags, journal_tid,
 483       (this->m_trace.valid() ? &this->m_trace : nullptr));
 484   }
 485 }
 486
 487 template <typename I>
 488 void ImageWriteRequest<I>::send_object_requests(
 489     const ObjectExtents &object_extents, const ::SnapContext &snapc,
 490     ObjectRequests *object_requests) {
 491   I &image_ctx = this->m_image_ctx;
 492
 493   // cache handles creating object requests during writeback
 494   if (image_ctx.object_cacher == NULL) {
 495     AbstractImageWriteRequest<I>::send_object_requests(object_extents, snapc,
 496                                                        object_requests);
 497   }
 498 }
 499
 500 template <typename I>
 501 ObjectRequestHandle *ImageWriteRequest<I>::create_object_request(
 502     const ObjectExtent &object_extent, const ::SnapContext &snapc,
 503     Context *on_finish) {
 504   I &image_ctx = this->m_image_ctx;
 505   assert(image_ctx.object_cacher == NULL);
 506
 507   bufferlist bl;
 508   assemble_extent(object_extent, &bl);
 509   ObjectRequest<I> *req = ObjectRequest<I>::create_write(
 510     &image_ctx, object_extent.oid.name, object_extent.objectno,
 511     object_extent.offset, bl, snapc, m_op_flags, this->m_trace, on_finish);
 512   return req;
 513 }
 514
 515 template <typename I>
 516 void ImageWriteRequest<I>::update_stats(size_t length) {
 517   I &image_ctx = this->m_image_ctx;
 518   image_ctx.perfcounter->inc(l_librbd_wr);
 519   image_ctx.perfcounter->inc(l_librbd_wr_bytes, length);
 520 }
 521
 522 template <typename I>
 523 uint64_t ImageDiscardRequest<I>::append_journal_event(
 524     const ObjectRequests &requests, bool synchronous) {
 525   I &image_ctx = this->m_image_ctx;
 526
 527   uint64_t tid = 0;
 528   assert(!this->m_image_extents.empty());
 529   for (auto &extent : this->m_image_extents) {
 530     journal::EventEntry event_entry(journal::AioDiscardEvent(extent.first,
 531                                                              extent.second,
 532                                                              this->m_skip_partial_discard));
 533     tid = image_ctx.journal->append_io_event(std::move(event_entry),
 534                                              requests, extent.first,
 535                                              extent.second, synchronous);
 536   }
 537
 538   AioCompletion *aio_comp = this->m_aio_comp;
 539   aio_comp->associate_journal_event(tid);
 540   return tid;
 541 }
 542
 543 template <typename I>
 544 void ImageDiscardRequest<I>::prune_object_extents(ObjectExtents &object_extents) {
 545   I &image_ctx = this->m_image_ctx;
 546   CephContext *cct = image_ctx.cct;
 547   if (!this->m_skip_partial_discard) {
 548     return;
 549   }
 550
 551   for (auto p = object_extents.begin(); p != object_extents.end(); ) {
 552     if (p->offset + p->length < image_ctx.layout.object_size) {
 553       ldout(cct, 20) << "oid " << p->oid << " " << p->offset << "~"
 554                      << p->length << " from " << p->buffer_extents
 555                      << ": skip partial discard" << dendl;
 556       p = object_extents.erase(p);
 557     } else {
 558       ++p;
 559     }
 560   }
 561 }
 562
 563 template <typename I>
 564 uint32_t ImageDiscardRequest<I>::get_object_cache_request_count(bool journaling) const {
 565   // extra completion request is required for tracking journal commit
 566   I &image_ctx = this->m_image_ctx;
 567   return (image_ctx.object_cacher != nullptr && journaling ? 1 : 0);
 568 }
 569
 570 template <typename I>
 571 void ImageDiscardRequest<I>::send_image_cache_request() {
 572   I &image_ctx = this->m_image_ctx;
 573   assert(image_ctx.image_cache != nullptr);
 574
 575   AioCompletion *aio_comp = this->m_aio_comp;
 576   aio_comp->set_request_count(this->m_image_extents.size());
 577   for (auto &extent : this->m_image_extents) {
 578     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 579     image_ctx.image_cache->aio_discard(extent.first, extent.second,
 580                                        this->m_skip_partial_discard, req_comp);
 581   }
 582 }
 583
 584 template <typename I>
 585 void ImageDiscardRequest<I>::send_object_cache_requests(
 586     const ObjectExtents &object_extents, uint64_t journal_tid) {
 587   I &image_ctx = this->m_image_ctx;
 588   if (journal_tid == 0) {
 589     Mutex::Locker cache_locker(image_ctx.cache_lock);
 590     image_ctx.object_cacher->discard_set(image_ctx.object_set,
 591                                          object_extents);
 592   } else {
 593     // cannot discard from cache until journal has committed
 594     assert(image_ctx.journal != NULL);
 595     AioCompletion *aio_comp = this->m_aio_comp;
 596     image_ctx.journal->wait_event(
 597       journal_tid, new C_DiscardJournalCommit<I>(image_ctx, aio_comp,
 598                                                  object_extents, journal_tid));
 599   }
 600 }
 601
 602 template <typename I>
 603 ObjectRequestHandle *ImageDiscardRequest<I>::create_object_request(
 604     const ObjectExtent &object_extent, const ::SnapContext &snapc,
 605     Context *on_finish) {
 606   I &image_ctx = this->m_image_ctx;
 607
 608   ObjectRequest<I> *req;
 609   if (object_extent.length == image_ctx.layout.object_size) {
 610     req = ObjectRequest<I>::create_remove(
 611       &image_ctx, object_extent.oid.name, object_extent.objectno, snapc,
 612       this->m_trace, on_finish);
 613   } else if (object_extent.offset + object_extent.length ==
 614                image_ctx.layout.object_size) {
 615     req = ObjectRequest<I>::create_truncate(
 616       &image_ctx, object_extent.oid.name, object_extent.objectno,
 617       object_extent.offset, snapc, this->m_trace, on_finish);
 618   } else {
 619     req = ObjectRequest<I>::create_zero(
 620       &image_ctx, object_extent.oid.name, object_extent.objectno,
 621       object_extent.offset, object_extent.length, snapc,
 622       this->m_trace, on_finish);
 623   }
 624   return req;
 625 }
 626
 627 template <typename I>
 628 void ImageDiscardRequest<I>::update_stats(size_t length) {
 629   I &image_ctx = this->m_image_ctx;
 630   image_ctx.perfcounter->inc(l_librbd_discard);
 631   image_ctx.perfcounter->inc(l_librbd_discard_bytes, length);
 632 }
 633
 634 template <typename I>
 635 void ImageFlushRequest<I>::send_request() {
 636   I &image_ctx = this->m_image_ctx;
 637   image_ctx.user_flushed();
 638
 639   bool journaling = false;
 640   {
 641     RWLock::RLocker snap_locker(image_ctx.snap_lock);
 642     journaling = (image_ctx.journal != nullptr &&
 643                   image_ctx.journal->is_journal_appending());
 644   }
 645
 646   AioCompletion *aio_comp = this->m_aio_comp;
 647   if (journaling) {
 648     // in-flight ops are flushed prior to closing the journal
 649     uint64_t journal_tid = image_ctx.journal->append_io_event(
 650       journal::EventEntry(journal::AioFlushEvent()),
 651       ObjectRequests(), 0, 0, false);
 652
 653     aio_comp->set_request_count(1);
 654     aio_comp->associate_journal_event(journal_tid);
 655
 656     FunctionContext *flush_ctx = new FunctionContext(
 657       [aio_comp, &image_ctx, journal_tid] (int r) {
 658         auto ctx = new C_FlushJournalCommit<I>(image_ctx, aio_comp,
 659                                                journal_tid);
 660         image_ctx.journal->flush_event(journal_tid, ctx);
 661
 662         // track flush op for block writes
 663         aio_comp->start_op(true);
 664         aio_comp->put();
 665     });
 666
 667     image_ctx.flush_async_operations(flush_ctx);
 668   } else {
 669     // flush rbd cache only when journaling is not enabled
 670     aio_comp->set_request_count(1);
 671     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 672     image_ctx.flush(req_comp);
 673
 674     aio_comp->start_op(true);
 675     aio_comp->put();
 676   }
 677
 678   image_ctx.perfcounter->inc(l_librbd_aio_flush);
 679 }
 680
 681 template <typename I>
 682 void ImageFlushRequest<I>::send_image_cache_request() {
 683   I &image_ctx = this->m_image_ctx;
 684   assert(image_ctx.image_cache != nullptr);
 685
 686   AioCompletion *aio_comp = this->m_aio_comp;
 687   aio_comp->set_request_count(1);
 688   C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 689   image_ctx.image_cache->aio_flush(req_comp);
 690 }
 691
 692 template <typename I>
 693 bool ImageWriteSameRequest<I>::assemble_writesame_extent(const ObjectExtent &object_extent,
 694                                                          bufferlist *bl, bool force_write) {
 695   size_t m_data_len = m_data_bl.length();
 696
 697   if (!force_write) {
 698     bool may_writesame = true;
 699
 700     for (auto q = object_extent.buffer_extents.begin();
 701          q != object_extent.buffer_extents.end(); ++q) {
 702       if (!(q->first % m_data_len == 0 && q->second % m_data_len == 0)) {
 703         may_writesame = false;
 704         break;
 705       }
 706     }
 707
 708     if (may_writesame) {
 709       bl->append(m_data_bl);
 710       return true;
 711     }
 712   }
 713
 714   for (auto q = object_extent.buffer_extents.begin();
 715        q != object_extent.buffer_extents.end(); ++q) {
 716     bufferlist sub_bl;
 717     uint64_t sub_off = q->first % m_data_len;
 718     uint64_t sub_len = m_data_len - sub_off;
 719     uint64_t extent_left = q->second;
 720     while (extent_left >= sub_len) {
 721       sub_bl.substr_of(m_data_bl, sub_off, sub_len);
 722       bl->claim_append(sub_bl);
 723       extent_left -= sub_len;
 724       if (sub_off) {
 725         sub_off = 0;
 726         sub_len = m_data_len;
 727       }
 728     }
 729     if (extent_left) {
 730       sub_bl.substr_of(m_data_bl, sub_off, extent_left);
 731       bl->claim_append(sub_bl);
 732     }
 733   }
 734   return false;
 735 }
 736
 737 template <typename I>
 738 uint64_t ImageWriteSameRequest<I>::append_journal_event(
 739     const ObjectRequests &requests, bool synchronous) {
 740   I &image_ctx = this->m_image_ctx;
 741
 742   uint64_t tid = 0;
 743   assert(!this->m_image_extents.empty());
 744   for (auto &extent : this->m_image_extents) {
 745     journal::EventEntry event_entry(journal::AioWriteSameEvent(extent.first,
 746                                                                extent.second,
 747                                                                m_data_bl));
 748     tid = image_ctx.journal->append_io_event(std::move(event_entry),
 749                                              requests, extent.first,
 750                                              extent.second, synchronous);
 751   }
 752
 753   if (image_ctx.object_cacher == NULL) {
 754     AioCompletion *aio_comp = this->m_aio_comp;
 755     aio_comp->associate_journal_event(tid);
 756   }
 757   return tid;
 758 }
 759
 760 template <typename I>
 761 void ImageWriteSameRequest<I>::send_image_cache_request() {
 762   I &image_ctx = this->m_image_ctx;
 763   assert(image_ctx.image_cache != nullptr);
 764
 765   AioCompletion *aio_comp = this->m_aio_comp;
 766   aio_comp->set_request_count(this->m_image_extents.size());
 767   for (auto &extent : this->m_image_extents) {
 768     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 769     image_ctx.image_cache->aio_writesame(extent.first, extent.second,
 770                                          std::move(m_data_bl), m_op_flags,
 771                                          req_comp);
 772   }
 773 }
 774
 775 template <typename I>
 776 void ImageWriteSameRequest<I>::send_object_cache_requests(
 777     const ObjectExtents &object_extents, uint64_t journal_tid) {
 778   I &image_ctx = this->m_image_ctx;
 779   for (auto p = object_extents.begin(); p != object_extents.end(); ++p) {
 780     const ObjectExtent &object_extent = *p;
 781
 782     bufferlist bl;
 783     assemble_writesame_extent(object_extent, &bl, true);
 784
 785     AioCompletion *aio_comp = this->m_aio_comp;
 786     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 787     image_ctx.write_to_cache(
 788       object_extent.oid, bl, object_extent.length, object_extent.offset,
 789       req_comp, m_op_flags, journal_tid,
 790       (this->m_trace.valid() ? &this->m_trace : nullptr));
 791   }
 792 }
 793
 794 template <typename I>
 795 void ImageWriteSameRequest<I>::send_object_requests(
 796     const ObjectExtents &object_extents, const ::SnapContext &snapc,
 797     ObjectRequests *object_requests) {
 798   I &image_ctx = this->m_image_ctx;
 799
 800   // cache handles creating object requests during writeback
 801   if (image_ctx.object_cacher == NULL) {
 802     AbstractImageWriteRequest<I>::send_object_requests(object_extents, snapc,
 803                                                        object_requests);
 804   }
 805 }
 806
 807 template <typename I>
 808 ObjectRequestHandle *ImageWriteSameRequest<I>::create_object_request(
 809     const ObjectExtent &object_extent, const ::SnapContext &snapc,
 810     Context *on_finish) {
 811   I &image_ctx = this->m_image_ctx;
 812   assert(image_ctx.object_cacher == NULL);
 813
 814   bufferlist bl;
 815   ObjectRequest<I> *req;
 816
 817   if (assemble_writesame_extent(object_extent, &bl, false)) {
 818     req = ObjectRequest<I>::create_writesame(
 819       &image_ctx, object_extent.oid.name, object_extent.objectno,
 820       object_extent.offset, object_extent.length,
 821       bl, snapc, m_op_flags, this->m_trace, on_finish);
 822     return req;
 823   }
 824   req = ObjectRequest<I>::create_write(
 825     &image_ctx, object_extent.oid.name, object_extent.objectno,
 826     object_extent.offset, bl, snapc, m_op_flags, this->m_trace, on_finish);
 827   return req;
 828 }
 829
 830 template <typename I>
 831 void ImageWriteSameRequest<I>::update_stats(size_t length) {
 832   I &image_ctx = this->m_image_ctx;
 833   image_ctx.perfcounter->inc(l_librbd_ws);
 834   image_ctx.perfcounter->inc(l_librbd_ws_bytes, length);
 835 }
 836
 837 } // namespace io
 838 } // namespace librbd
 839
 840 template class librbd::io::ImageRequest<librbd::ImageCtx>;
 841 template class librbd::io::ImageReadRequest<librbd::ImageCtx>;
 842 template class librbd::io::AbstractImageWriteRequest<librbd::ImageCtx>;
 843 template class librbd::io::ImageWriteRequest<librbd::ImageCtx>;
 844 template class librbd::io::ImageDiscardRequest<librbd::ImageCtx>;
 845 template class librbd::io::ImageFlushRequest<librbd::ImageCtx>;
 846 template class librbd::io::ImageWriteSameRequest<librbd::ImageCtx>;