ceph/src/librbd/io/ImageRequest.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "librbd/io/ImageRequest.h"
   5 #include "librbd/ImageCtx.h"
   6 #include "librbd/internal.h"
   7 #include "librbd/Journal.h"
   8 #include "librbd/Types.h"
   9 #include "librbd/Utils.h"
  10 #include "librbd/cache/ImageCache.h"
  11 #include "librbd/io/AioCompletion.h"
  12 #include "librbd/io/ObjectRequest.h"
  13 #include "librbd/journal/Types.h"
  14 #include "include/rados/librados.hpp"
  15 #include "common/WorkQueue.h"
  16 #include "osdc/Striper.h"
  17
  18 #define dout_subsys ceph_subsys_rbd
  19 #undef dout_prefix
  20 #define dout_prefix *_dout << "librbd::io::ImageRequest: " << this \
  21                            << " " << __func__ << ": "
  22
  23 namespace librbd {
  24 namespace io {
  25
  26 using util::get_image_ctx;
  27
  28 namespace {
  29
  30 template <typename ImageCtxT = ImageCtx>
  31 struct C_DiscardRequest {
  32   typedef std::vector<ObjectExtent> ObjectExtents;
  33
  34   ImageCtxT &image_ctx;
  35   AioCompletion *aio_completion;
  36   ObjectExtents object_extents;
  37
  38   C_DiscardRequest(ImageCtxT &image_ctx, AioCompletion *aio_completion,
  39                    const ObjectExtents &object_extents)
  40     : image_ctx(image_ctx), aio_completion(aio_completion),
  41       object_extents(object_extents) {
  42     aio_completion->add_request();
  43   }
  44
  45   void send() {
  46     discard_writeback();
  47   }
  48
  49   void discard_writeback() {
  50     CephContext *cct = image_ctx.cct;
  51     ldout(cct, 20) << "C_DiscardRequest: delaying cache discard until "
  52                    << "writeback flushed" << dendl;
  53
  54     // ensure we aren't holding the cache lock post-write
  55     auto ctx = util::create_async_context_callback(
  56       image_ctx, util::create_context_callback<
  57         C_DiscardRequest<ImageCtxT>,
  58         &C_DiscardRequest<ImageCtxT>::handle_discard_writeback>(this));
  59
  60     // ensure any in-flight writeback is complete before advancing
  61     // the discard request
  62     Mutex::Locker cache_locker(image_ctx.cache_lock);
  63     image_ctx.object_cacher->discard_writeback(image_ctx.object_set,
  64                                                object_extents, ctx);
  65   }
  66
  67   void handle_discard_writeback(int r) {
  68     CephContext *cct = image_ctx.cct;
  69     ldout(cct, 20) << "C_DiscardRequest: writeback flushed: " << r << dendl;
  70
  71     {
  72       Mutex::Locker cache_locker(image_ctx.cache_lock);
  73       image_ctx.object_cacher->discard_set(image_ctx.object_set,
  74                                            object_extents);
  75     }
  76     aio_completion->complete_request(r);
  77     delete this;
  78   }
  79 };
  80
  81 template <typename ImageCtxT = ImageCtx>
  82 struct C_DiscardJournalCommit : public Context {
  83   ImageCtxT &image_ctx;
  84   C_DiscardRequest<ImageCtxT> *discard_request;
  85
  86   C_DiscardJournalCommit(ImageCtxT &_image_ctx,
  87                          C_DiscardRequest<ImageCtxT> *discard_request,
  88                          uint64_t tid)
  89     : image_ctx(_image_ctx), discard_request(discard_request) {
  90     CephContext *cct = image_ctx.cct;
  91     ldout(cct, 20) << "delaying cache discard until journal tid " << tid << " "
  92                    << "safe" << dendl;
  93   }
  94
  95   void finish(int r) override {
  96     CephContext *cct = image_ctx.cct;
  97     ldout(cct, 20) << "C_DiscardJournalCommit: "
  98                    << "journal committed: discarding from cache" << dendl;
  99     discard_request->send();
 100   }
 101 };
 102
 103 template <typename ImageCtxT = ImageCtx>
 104 struct C_FlushJournalCommit : public Context {
 105   ImageCtxT &image_ctx;
 106   AioCompletion *aio_comp;
 107
 108   C_FlushJournalCommit(ImageCtxT &_image_ctx, AioCompletion *_aio_comp,
 109                        uint64_t tid)
 110     : image_ctx(_image_ctx), aio_comp(_aio_comp) {
 111     CephContext *cct = image_ctx.cct;
 112     ldout(cct, 20) << "delaying flush until journal tid " << tid << " "
 113                    << "safe" << dendl;
 114
 115     aio_comp->add_request();
 116   }
 117
 118   void finish(int r) override {
 119     CephContext *cct = image_ctx.cct;
 120     ldout(cct, 20) << "C_FlushJournalCommit: journal committed" << dendl;
 121     aio_comp->complete_request(r);
 122   }
 123 };
 124
 125 } // anonymous namespace
 126
 127 template <typename I>
 128 ImageRequest<I>* ImageRequest<I>::create_read_request(
 129     I &image_ctx, AioCompletion *aio_comp, Extents &&image_extents,
 130     ReadResult &&read_result, int op_flags,
 131     const ZTracer::Trace &parent_trace) {
 132   return new ImageReadRequest<I>(image_ctx, aio_comp,
 133                                  std::move(image_extents),
 134                                  std::move(read_result), op_flags,
 135                                  parent_trace);
 136 }
 137
 138 template <typename I>
 139 ImageRequest<I>* ImageRequest<I>::create_write_request(
 140     I &image_ctx, AioCompletion *aio_comp, Extents &&image_extents,
 141     bufferlist &&bl, int op_flags, const ZTracer::Trace &parent_trace) {
 142   return new ImageWriteRequest<I>(image_ctx, aio_comp, std::move(image_extents),
 143                                   std::move(bl), op_flags, parent_trace);
 144 }
 145
 146 template <typename I>
 147 ImageRequest<I>* ImageRequest<I>::create_discard_request(
 148     I &image_ctx, AioCompletion *aio_comp, uint64_t off, uint64_t len,
 149     bool skip_partial_discard, const ZTracer::Trace &parent_trace) {
 150   return new ImageDiscardRequest<I>(image_ctx, aio_comp, off, len,
 151                                     skip_partial_discard, parent_trace);
 152 }
 153
 154 template <typename I>
 155 ImageRequest<I>* ImageRequest<I>::create_flush_request(
 156     I &image_ctx, AioCompletion *aio_comp,
 157     const ZTracer::Trace &parent_trace) {
 158   return new ImageFlushRequest<I>(image_ctx, aio_comp, parent_trace);
 159 }
 160
 161 template <typename I>
 162 ImageRequest<I>* ImageRequest<I>::create_writesame_request(
 163     I &image_ctx, AioCompletion *aio_comp, uint64_t off, uint64_t len,
 164     bufferlist &&bl, int op_flags, const ZTracer::Trace &parent_trace) {
 165   return new ImageWriteSameRequest<I>(image_ctx, aio_comp, off, len,
 166                                       std::move(bl), op_flags, parent_trace);
 167 }
 168
 169 template <typename I>
 170 ImageRequest<I>* ImageRequest<I>::create_compare_and_write_request(
 171     I &image_ctx, AioCompletion *c, Extents &&image_extents,
 172     bufferlist &&cmp_bl, bufferlist &&bl, uint64_t *mismatch_offset,
 173     int op_flags, const ZTracer::Trace &parent_trace) {
 174   return new ImageCompareAndWriteRequest<I>(image_ctx, c,
 175                                             std::move(image_extents),
 176                                             std::move(cmp_bl),
 177                                             std::move(bl), mismatch_offset,
 178                                             op_flags, parent_trace);
 179 }
 180
 181 template <typename I>
 182 void ImageRequest<I>::aio_read(I *ictx, AioCompletion *c,
 183                                Extents &&image_extents,
 184                                ReadResult &&read_result, int op_flags,
 185                                const ZTracer::Trace &parent_trace) {
 186   ImageReadRequest<I> req(*ictx, c, std::move(image_extents),
 187                           std::move(read_result), op_flags, parent_trace);
 188   req.send();
 189 }
 190
 191 template <typename I>
 192 void ImageRequest<I>::aio_write(I *ictx, AioCompletion *c,
 193                                 Extents &&image_extents, bufferlist &&bl,
 194                                 int op_flags,
 195                                 const ZTracer::Trace &parent_trace) {
 196   ImageWriteRequest<I> req(*ictx, c, std::move(image_extents), std::move(bl),
 197                            op_flags, parent_trace);
 198   req.send();
 199 }
 200
 201 template <typename I>
 202 void ImageRequest<I>::aio_discard(I *ictx, AioCompletion *c,
 203                                   uint64_t off, uint64_t len,
 204                                   bool skip_partial_discard,
 205                                   const ZTracer::Trace &parent_trace) {
 206   ImageDiscardRequest<I> req(*ictx, c, off, len, skip_partial_discard,
 207                              parent_trace);
 208   req.send();
 209 }
 210
 211 template <typename I>
 212 void ImageRequest<I>::aio_flush(I *ictx, AioCompletion *c,
 213                                 const ZTracer::Trace &parent_trace) {
 214   ImageFlushRequest<I> req(*ictx, c, parent_trace);
 215   req.send();
 216 }
 217
 218 template <typename I>
 219 void ImageRequest<I>::aio_writesame(I *ictx, AioCompletion *c,
 220                                     uint64_t off, uint64_t len,
 221                                     bufferlist &&bl, int op_flags,
 222                                     const ZTracer::Trace &parent_trace) {
 223   ImageWriteSameRequest<I> req(*ictx, c, off, len, std::move(bl), op_flags,
 224                                parent_trace);
 225   req.send();
 226 }
 227
 228 template <typename I>
 229 void ImageRequest<I>::aio_compare_and_write(I *ictx, AioCompletion *c,
 230                                             Extents &&image_extents,
 231                                             bufferlist &&cmp_bl,
 232                                             bufferlist &&bl,
 233                                             uint64_t *mismatch_offset,
 234                                             int op_flags,
 235                                             const ZTracer::Trace &parent_trace) {
 236   ImageCompareAndWriteRequest<I> req(*ictx, c, std::move(image_extents),
 237                                      std::move(cmp_bl), std::move(bl),
 238                                      mismatch_offset, op_flags, parent_trace);
 239   req.send();
 240 }
 241
 242
 243 template <typename I>
 244 void ImageRequest<I>::send() {
 245   I &image_ctx = this->m_image_ctx;
 246   assert(m_aio_comp->is_initialized(get_aio_type()));
 247   assert(m_aio_comp->is_started() ^ (get_aio_type() == AIO_TYPE_FLUSH));
 248
 249   CephContext *cct = image_ctx.cct;
 250   AioCompletion *aio_comp = this->m_aio_comp;
 251   ldout(cct, 20) << get_request_type() << ": ictx=" << &image_ctx << ", "
 252                  << "completion=" << aio_comp << dendl;
 253
 254   aio_comp->get();
 255   int r = clip_request();
 256   if (r < 0) {
 257     m_aio_comp->fail(r);
 258     return;
 259   }
 260
 261   if (m_bypass_image_cache || m_image_ctx.image_cache == nullptr) {
 262     send_request();
 263   } else {
 264     send_image_cache_request();
 265   }
 266 }
 267
 268 template <typename I>
 269 int ImageRequest<I>::clip_request() {
 270   RWLock::RLocker snap_locker(m_image_ctx.snap_lock);
 271   for (auto &image_extent : m_image_extents) {
 272     auto clip_len = image_extent.second;
 273     int r = clip_io(get_image_ctx(&m_image_ctx), image_extent.first, &clip_len);
 274     if (r < 0) {
 275       return r;
 276     }
 277
 278     image_extent.second = clip_len;
 279   }
 280   return 0;
 281 }
 282
 283 template <typename I>
 284 void ImageRequest<I>::start_op() {
 285   m_aio_comp->start_op();
 286 }
 287
 288 template <typename I>
 289 void ImageRequest<I>::fail(int r) {
 290   AioCompletion *aio_comp = this->m_aio_comp;
 291   aio_comp->get();
 292   aio_comp->fail(r);
 293 }
 294
 295 template <typename I>
 296 ImageReadRequest<I>::ImageReadRequest(I &image_ctx, AioCompletion *aio_comp,
 297                                       Extents &&image_extents,
 298                                       ReadResult &&read_result, int op_flags,
 299                                       const ZTracer::Trace &parent_trace)
 300   : ImageRequest<I>(image_ctx, aio_comp, std::move(image_extents), "read",
 301                     parent_trace),
 302     m_op_flags(op_flags) {
 303   aio_comp->read_result = std::move(read_result);
 304 }
 305
 306 template <typename I>
 307 int ImageReadRequest<I>::clip_request() {
 308   int r = ImageRequest<I>::clip_request();
 309   if (r < 0) {
 310     return r;
 311   }
 312
 313   uint64_t buffer_length = 0;
 314   auto &image_extents = this->m_image_extents;
 315   for (auto &image_extent : image_extents) {
 316     buffer_length += image_extent.second;
 317   }
 318   this->m_aio_comp->read_result.set_clip_length(buffer_length);
 319   return 0;
 320 }
 321
 322 template <typename I>
 323 void ImageReadRequest<I>::send_request() {
 324   I &image_ctx = this->m_image_ctx;
 325   CephContext *cct = image_ctx.cct;
 326
 327   auto &image_extents = this->m_image_extents;
 328   if (image_ctx.object_cacher && image_ctx.readahead_max_bytes > 0 &&
 329       !(m_op_flags & LIBRADOS_OP_FLAG_FADVISE_RANDOM)) {
 330     readahead(get_image_ctx(&image_ctx), image_extents);
 331   }
 332
 333   AioCompletion *aio_comp = this->m_aio_comp;
 334   librados::snap_t snap_id;
 335   map<object_t,vector<ObjectExtent> > object_extents;
 336   uint64_t buffer_ofs = 0;
 337   {
 338     // prevent image size from changing between computing clip and recording
 339     // pending async operation
 340     RWLock::RLocker snap_locker(image_ctx.snap_lock);
 341     snap_id = image_ctx.snap_id;
 342
 343     // map image extents to object extents
 344     for (auto &extent : image_extents) {
 345       if (extent.second == 0) {
 346         continue;
 347       }
 348
 349       Striper::file_to_extents(cct, image_ctx.format_string, &image_ctx.layout,
 350                                extent.first, extent.second, 0, object_extents,
 351                                buffer_ofs);
 352       buffer_ofs += extent.second;
 353     }
 354   }
 355
 356   // pre-calculate the expected number of read requests
 357   uint32_t request_count = 0;
 358   for (auto &object_extent : object_extents) {
 359     request_count += object_extent.second.size();
 360   }
 361   aio_comp->set_request_count(request_count);
 362
 363   // issue the requests
 364   for (auto &object_extent : object_extents) {
 365     for (auto &extent : object_extent.second) {
 366       ldout(cct, 20) << "oid " << extent.oid << " " << extent.offset << "~"
 367                      << extent.length << " from " << extent.buffer_extents
 368                      << dendl;
 369
 370       auto req_comp = new io::ReadResult::C_SparseReadRequest<I>(
 371         aio_comp, std::move(extent.buffer_extents), true);
 372       ObjectReadRequest<I> *req = ObjectReadRequest<I>::create(
 373         &image_ctx, extent.oid.name, extent.objectno, extent.offset,
 374         extent.length, snap_id, m_op_flags, false, this->m_trace, req_comp);
 375       req_comp->request = req;
 376       req->send();
 377     }
 378   }
 379
 380   aio_comp->put();
 381
 382   image_ctx.perfcounter->inc(l_librbd_rd);
 383   image_ctx.perfcounter->inc(l_librbd_rd_bytes, buffer_ofs);
 384 }
 385
 386 template <typename I>
 387 void ImageReadRequest<I>::send_image_cache_request() {
 388   I &image_ctx = this->m_image_ctx;
 389   assert(image_ctx.image_cache != nullptr);
 390
 391   AioCompletion *aio_comp = this->m_aio_comp;
 392   aio_comp->set_request_count(1);
 393
 394   auto *req_comp = new io::ReadResult::C_ImageReadRequest(
 395     aio_comp, this->m_image_extents);
 396   image_ctx.image_cache->aio_read(std::move(this->m_image_extents),
 397                                   &req_comp->bl, m_op_flags,
 398                                   req_comp);
 399 }
 400
 401 template <typename I>
 402 void AbstractImageWriteRequest<I>::send_request() {
 403   I &image_ctx = this->m_image_ctx;
 404   CephContext *cct = image_ctx.cct;
 405
 406   RWLock::RLocker md_locker(image_ctx.md_lock);
 407
 408   bool journaling = false;
 409
 410   AioCompletion *aio_comp = this->m_aio_comp;
 411   uint64_t clip_len = 0;
 412   ObjectExtents object_extents;
 413   ::SnapContext snapc;
 414   {
 415     // prevent image size from changing between computing clip and recording
 416     // pending async operation
 417     RWLock::RLocker snap_locker(image_ctx.snap_lock);
 418     if (image_ctx.snap_id != CEPH_NOSNAP || image_ctx.read_only) {
 419       aio_comp->fail(-EROFS);
 420       return;
 421     }
 422
 423     for (auto &extent : this->m_image_extents) {
 424       if (extent.second == 0) {
 425         continue;
 426       }
 427
 428       // map to object extents
 429       Striper::file_to_extents(cct, image_ctx.format_string, &image_ctx.layout,
 430                                extent.first, extent.second, 0, object_extents);
 431       clip_len += extent.second;
 432     }
 433
 434     snapc = image_ctx.snapc;
 435     journaling = (image_ctx.journal != nullptr &&
 436                   image_ctx.journal->is_journal_appending());
 437   }
 438
 439   int ret = prune_object_extents(object_extents);
 440   if (ret < 0) {
 441     aio_comp->fail(ret);
 442     return;
 443   }
 444
 445   if (!object_extents.empty()) {
 446     uint64_t journal_tid = 0;
 447     aio_comp->set_request_count(
 448       object_extents.size() + get_object_cache_request_count());
 449
 450     ObjectRequests requests;
 451     send_object_requests(object_extents, snapc,
 452                          (journaling ? &requests : nullptr));
 453
 454     if (journaling) {
 455       // in-flight ops are flushed prior to closing the journal
 456       assert(image_ctx.journal != NULL);
 457       journal_tid = append_journal_event(requests, m_synchronous);
 458     }
 459
 460     if (image_ctx.object_cacher != NULL) {
 461       send_object_cache_requests(object_extents, journal_tid);
 462     }
 463   } else {
 464     // no IO to perform -- fire completion
 465     aio_comp->unblock();
 466   }
 467
 468   update_stats(clip_len);
 469   aio_comp->put();
 470 }
 471
 472 template <typename I>
 473 void AbstractImageWriteRequest<I>::send_object_requests(
 474     const ObjectExtents &object_extents, const ::SnapContext &snapc,
 475     ObjectRequests *object_requests) {
 476   I &image_ctx = this->m_image_ctx;
 477   CephContext *cct = image_ctx.cct;
 478
 479   AioCompletion *aio_comp = this->m_aio_comp;
 480   for (ObjectExtents::const_iterator p = object_extents.begin();
 481        p != object_extents.end(); ++p) {
 482     ldout(cct, 20) << "oid " << p->oid << " " << p->offset << "~" << p->length
 483                    << " from " << p->buffer_extents << dendl;
 484     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 485     ObjectRequestHandle *request = create_object_request(*p, snapc,
 486                                                             req_comp);
 487
 488     // if journaling, stash the request for later; otherwise send
 489     if (request != NULL) {
 490       if (object_requests != NULL) {
 491         object_requests->push_back(request);
 492       } else {
 493         request->send();
 494       }
 495     }
 496   }
 497 }
 498
 499 template <typename I>
 500 void ImageWriteRequest<I>::assemble_extent(const ObjectExtent &object_extent,
 501                                            bufferlist *bl) {
 502   for (auto q = object_extent.buffer_extents.begin();
 503        q != object_extent.buffer_extents.end(); ++q) {
 504     bufferlist sub_bl;
 505     sub_bl.substr_of(m_bl, q->first, q->second);
 506     bl->claim_append(sub_bl);
 507   }
 508 }
 509
 510 template <typename I>
 511 uint64_t ImageWriteRequest<I>::append_journal_event(
 512     const ObjectRequests &requests, bool synchronous) {
 513   I &image_ctx = this->m_image_ctx;
 514
 515   uint64_t tid = 0;
 516   uint64_t buffer_offset = 0;
 517   assert(!this->m_image_extents.empty());
 518   for (auto &extent : this->m_image_extents) {
 519     bufferlist sub_bl;
 520     sub_bl.substr_of(m_bl, buffer_offset, extent.second);
 521     buffer_offset += extent.second;
 522
 523     tid = image_ctx.journal->append_write_event(extent.first, extent.second,
 524                                                 sub_bl, requests, synchronous);
 525   }
 526
 527   if (image_ctx.object_cacher == NULL) {
 528     AioCompletion *aio_comp = this->m_aio_comp;
 529     aio_comp->associate_journal_event(tid);
 530   }
 531   return tid;
 532 }
 533
 534 template <typename I>
 535 void ImageWriteRequest<I>::send_image_cache_request() {
 536   I &image_ctx = this->m_image_ctx;
 537   assert(image_ctx.image_cache != nullptr);
 538
 539   AioCompletion *aio_comp = this->m_aio_comp;
 540   aio_comp->set_request_count(1);
 541   C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 542   image_ctx.image_cache->aio_write(std::move(this->m_image_extents),
 543                                    std::move(m_bl), m_op_flags, req_comp);
 544 }
 545
 546 template <typename I>
 547 void ImageWriteRequest<I>::send_object_cache_requests(
 548     const ObjectExtents &object_extents, uint64_t journal_tid) {
 549   I &image_ctx = this->m_image_ctx;
 550   for (auto p = object_extents.begin(); p != object_extents.end(); ++p) {
 551     const ObjectExtent &object_extent = *p;
 552
 553     bufferlist bl;
 554     assemble_extent(object_extent, &bl);
 555
 556     AioCompletion *aio_comp = this->m_aio_comp;
 557     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 558     image_ctx.write_to_cache(
 559       object_extent.oid, bl, object_extent.length, object_extent.offset,
 560       req_comp, m_op_flags, journal_tid,
 561       (this->m_trace.valid() ? &this->m_trace : nullptr));
 562   }
 563 }
 564
 565 template <typename I>
 566 void ImageWriteRequest<I>::send_object_requests(
 567     const ObjectExtents &object_extents, const ::SnapContext &snapc,
 568     ObjectRequests *object_requests) {
 569   I &image_ctx = this->m_image_ctx;
 570
 571   // cache handles creating object requests during writeback
 572   if (image_ctx.object_cacher == NULL) {
 573     AbstractImageWriteRequest<I>::send_object_requests(object_extents, snapc,
 574                                                        object_requests);
 575   }
 576 }
 577
 578 template <typename I>
 579 ObjectRequestHandle *ImageWriteRequest<I>::create_object_request(
 580     const ObjectExtent &object_extent, const ::SnapContext &snapc,
 581     Context *on_finish) {
 582   I &image_ctx = this->m_image_ctx;
 583   assert(image_ctx.object_cacher == NULL);
 584
 585   bufferlist bl;
 586   assemble_extent(object_extent, &bl);
 587   ObjectRequest<I> *req = ObjectRequest<I>::create_write(
 588     &image_ctx, object_extent.oid.name, object_extent.objectno,
 589     object_extent.offset, bl, snapc, m_op_flags, this->m_trace, on_finish);
 590   return req;
 591 }
 592
 593 template <typename I>
 594 void ImageWriteRequest<I>::update_stats(size_t length) {
 595   I &image_ctx = this->m_image_ctx;
 596   image_ctx.perfcounter->inc(l_librbd_wr);
 597   image_ctx.perfcounter->inc(l_librbd_wr_bytes, length);
 598 }
 599
 600 template <typename I>
 601 uint64_t ImageDiscardRequest<I>::append_journal_event(
 602     const ObjectRequests &requests, bool synchronous) {
 603   I &image_ctx = this->m_image_ctx;
 604
 605   uint64_t tid = 0;
 606   assert(!this->m_image_extents.empty());
 607   for (auto &extent : this->m_image_extents) {
 608     journal::EventEntry event_entry(journal::AioDiscardEvent(extent.first,
 609                                                              extent.second,
 610                                                              this->m_skip_partial_discard));
 611     tid = image_ctx.journal->append_io_event(std::move(event_entry),
 612                                              requests, extent.first,
 613                                              extent.second, synchronous, 0);
 614   }
 615
 616   AioCompletion *aio_comp = this->m_aio_comp;
 617   aio_comp->associate_journal_event(tid);
 618   return tid;
 619 }
 620
 621 template <typename I>
 622 int ImageDiscardRequest<I>::prune_object_extents(ObjectExtents &object_extents) {
 623   I &image_ctx = this->m_image_ctx;
 624   CephContext *cct = image_ctx.cct;
 625   if (!this->m_skip_partial_discard) {
 626     return 0;
 627   }
 628
 629   for (auto p = object_extents.begin(); p != object_extents.end(); ) {
 630     if (p->offset + p->length < image_ctx.layout.object_size) {
 631       ldout(cct, 20) << "oid " << p->oid << " " << p->offset << "~"
 632                      << p->length << " from " << p->buffer_extents
 633                      << ": skip partial discard" << dendl;
 634       p = object_extents.erase(p);
 635     } else {
 636       ++p;
 637     }
 638   }
 639
 640   return 0;
 641 }
 642
 643 template <typename I>
 644 uint32_t ImageDiscardRequest<I>::get_object_cache_request_count() const {
 645   // extra completion request is required for tracking discard
 646   I &image_ctx = this->m_image_ctx;
 647   return (image_ctx.object_cacher != nullptr ? 1 : 0);
 648 }
 649
 650 template <typename I>
 651 void ImageDiscardRequest<I>::send_image_cache_request() {
 652   I &image_ctx = this->m_image_ctx;
 653   assert(image_ctx.image_cache != nullptr);
 654
 655   AioCompletion *aio_comp = this->m_aio_comp;
 656   aio_comp->set_request_count(this->m_image_extents.size());
 657   for (auto &extent : this->m_image_extents) {
 658     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 659     image_ctx.image_cache->aio_discard(extent.first, extent.second,
 660                                        this->m_skip_partial_discard, req_comp);
 661   }
 662 }
 663
 664 template <typename I>
 665 void ImageDiscardRequest<I>::send_object_cache_requests(
 666     const ObjectExtents &object_extents, uint64_t journal_tid) {
 667   I &image_ctx = this->m_image_ctx;
 668   auto aio_comp = this->m_aio_comp;
 669   auto req = new C_DiscardRequest<I>(image_ctx, aio_comp, object_extents);
 670   if (journal_tid == 0) {
 671     req->send();
 672   } else {
 673     // cannot discard from cache until journal has committed
 674     assert(image_ctx.journal != NULL);
 675     image_ctx.journal->wait_event(
 676       journal_tid, new C_DiscardJournalCommit<I>(image_ctx, req, journal_tid));
 677   }
 678 }
 679
 680 template <typename I>
 681 ObjectRequestHandle *ImageDiscardRequest<I>::create_object_request(
 682     const ObjectExtent &object_extent, const ::SnapContext &snapc,
 683     Context *on_finish) {
 684   I &image_ctx = this->m_image_ctx;
 685
 686   auto req = ObjectRequest<I>::create_discard(
 687     &image_ctx, object_extent.oid.name, object_extent.objectno,
 688     object_extent.offset, object_extent.length, snapc, true, true,
 689     this->m_trace, on_finish);
 690   return req;
 691 }
 692
 693 template <typename I>
 694 void ImageDiscardRequest<I>::update_stats(size_t length) {
 695   I &image_ctx = this->m_image_ctx;
 696   image_ctx.perfcounter->inc(l_librbd_discard);
 697   image_ctx.perfcounter->inc(l_librbd_discard_bytes, length);
 698 }
 699
 700 template <typename I>
 701 void ImageFlushRequest<I>::send_request() {
 702   I &image_ctx = this->m_image_ctx;
 703   image_ctx.user_flushed();
 704
 705   bool journaling = false;
 706   {
 707     RWLock::RLocker snap_locker(image_ctx.snap_lock);
 708     journaling = (image_ctx.journal != nullptr &&
 709                   image_ctx.journal->is_journal_appending());
 710   }
 711
 712   AioCompletion *aio_comp = this->m_aio_comp;
 713   if (journaling) {
 714     // in-flight ops are flushed prior to closing the journal
 715     uint64_t journal_tid = image_ctx.journal->append_io_event(
 716       journal::EventEntry(journal::AioFlushEvent()),
 717       ObjectRequests(), 0, 0, false, 0);
 718
 719     aio_comp->set_request_count(1);
 720     aio_comp->associate_journal_event(journal_tid);
 721
 722     FunctionContext *flush_ctx = new FunctionContext(
 723       [aio_comp, &image_ctx, journal_tid] (int r) {
 724         auto ctx = new C_FlushJournalCommit<I>(image_ctx, aio_comp,
 725                                                journal_tid);
 726         image_ctx.journal->flush_event(journal_tid, ctx);
 727
 728         // track flush op for block writes
 729         aio_comp->start_op(true);
 730         aio_comp->put();
 731     });
 732
 733     image_ctx.flush_async_operations(flush_ctx);
 734   } else {
 735     // flush rbd cache only when journaling is not enabled
 736     aio_comp->set_request_count(1);
 737     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 738     image_ctx.flush(req_comp);
 739
 740     aio_comp->start_op(true);
 741     aio_comp->put();
 742   }
 743
 744   image_ctx.perfcounter->inc(l_librbd_aio_flush);
 745 }
 746
 747 template <typename I>
 748 void ImageFlushRequest<I>::send_image_cache_request() {
 749   I &image_ctx = this->m_image_ctx;
 750   assert(image_ctx.image_cache != nullptr);
 751
 752   AioCompletion *aio_comp = this->m_aio_comp;
 753   aio_comp->set_request_count(1);
 754   C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 755   image_ctx.image_cache->aio_flush(req_comp);
 756 }
 757
 758 template <typename I>
 759 bool ImageWriteSameRequest<I>::assemble_writesame_extent(const ObjectExtent &object_extent,
 760                                                          bufferlist *bl, bool force_write) {
 761   size_t m_data_len = m_data_bl.length();
 762
 763   if (!force_write) {
 764     bool may_writesame = true;
 765
 766     for (auto q = object_extent.buffer_extents.begin();
 767          q != object_extent.buffer_extents.end(); ++q) {
 768       if (!(q->first % m_data_len == 0 && q->second % m_data_len == 0)) {
 769         may_writesame = false;
 770         break;
 771       }
 772     }
 773
 774     if (may_writesame) {
 775       bl->append(m_data_bl);
 776       return true;
 777     }
 778   }
 779
 780   for (auto q = object_extent.buffer_extents.begin();
 781        q != object_extent.buffer_extents.end(); ++q) {
 782     bufferlist sub_bl;
 783     uint64_t sub_off = q->first % m_data_len;
 784     uint64_t sub_len = m_data_len - sub_off;
 785     uint64_t extent_left = q->second;
 786     while (extent_left >= sub_len) {
 787       sub_bl.substr_of(m_data_bl, sub_off, sub_len);
 788       bl->claim_append(sub_bl);
 789       extent_left -= sub_len;
 790       if (sub_off) {
 791         sub_off = 0;
 792         sub_len = m_data_len;
 793       }
 794     }
 795     if (extent_left) {
 796       sub_bl.substr_of(m_data_bl, sub_off, extent_left);
 797       bl->claim_append(sub_bl);
 798     }
 799   }
 800   return false;
 801 }
 802
 803 template <typename I>
 804 uint64_t ImageWriteSameRequest<I>::append_journal_event(
 805     const ObjectRequests &requests, bool synchronous) {
 806   I &image_ctx = this->m_image_ctx;
 807
 808   uint64_t tid = 0;
 809   assert(!this->m_image_extents.empty());
 810   for (auto &extent : this->m_image_extents) {
 811     journal::EventEntry event_entry(journal::AioWriteSameEvent(extent.first,
 812                                                                extent.second,
 813                                                                m_data_bl));
 814     tid = image_ctx.journal->append_io_event(std::move(event_entry),
 815                                              requests, extent.first,
 816                                              extent.second, synchronous, 0);
 817   }
 818
 819   if (image_ctx.object_cacher == NULL) {
 820     AioCompletion *aio_comp = this->m_aio_comp;
 821     aio_comp->associate_journal_event(tid);
 822   }
 823   return tid;
 824 }
 825
 826 template <typename I>
 827 void ImageWriteSameRequest<I>::send_image_cache_request() {
 828   I &image_ctx = this->m_image_ctx;
 829   assert(image_ctx.image_cache != nullptr);
 830
 831   AioCompletion *aio_comp = this->m_aio_comp;
 832   aio_comp->set_request_count(this->m_image_extents.size());
 833   for (auto &extent : this->m_image_extents) {
 834     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 835     image_ctx.image_cache->aio_writesame(extent.first, extent.second,
 836                                          std::move(m_data_bl), m_op_flags,
 837                                          req_comp);
 838   }
 839 }
 840
 841 template <typename I>
 842 void ImageWriteSameRequest<I>::send_object_cache_requests(
 843     const ObjectExtents &object_extents, uint64_t journal_tid) {
 844   I &image_ctx = this->m_image_ctx;
 845   for (auto p = object_extents.begin(); p != object_extents.end(); ++p) {
 846     const ObjectExtent &object_extent = *p;
 847
 848     bufferlist bl;
 849     assemble_writesame_extent(object_extent, &bl, true);
 850
 851     AioCompletion *aio_comp = this->m_aio_comp;
 852     C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 853     image_ctx.write_to_cache(
 854       object_extent.oid, bl, object_extent.length, object_extent.offset,
 855       req_comp, m_op_flags, journal_tid,
 856       (this->m_trace.valid() ? &this->m_trace : nullptr));
 857   }
 858 }
 859
 860 template <typename I>
 861 void ImageWriteSameRequest<I>::send_object_requests(
 862     const ObjectExtents &object_extents, const ::SnapContext &snapc,
 863     ObjectRequests *object_requests) {
 864   I &image_ctx = this->m_image_ctx;
 865
 866   // cache handles creating object requests during writeback
 867   if (image_ctx.object_cacher == NULL) {
 868     AbstractImageWriteRequest<I>::send_object_requests(object_extents, snapc,
 869                                                        object_requests);
 870   }
 871 }
 872
 873 template <typename I>
 874 ObjectRequestHandle *ImageWriteSameRequest<I>::create_object_request(
 875     const ObjectExtent &object_extent, const ::SnapContext &snapc,
 876     Context *on_finish) {
 877   I &image_ctx = this->m_image_ctx;
 878   assert(image_ctx.object_cacher == NULL);
 879
 880   bufferlist bl;
 881   ObjectRequest<I> *req;
 882
 883   if (assemble_writesame_extent(object_extent, &bl, false)) {
 884     req = ObjectRequest<I>::create_writesame(
 885       &image_ctx, object_extent.oid.name, object_extent.objectno,
 886       object_extent.offset, object_extent.length,
 887       bl, snapc, m_op_flags, this->m_trace, on_finish);
 888     return req;
 889   }
 890   req = ObjectRequest<I>::create_write(
 891     &image_ctx, object_extent.oid.name, object_extent.objectno,
 892     object_extent.offset, bl, snapc, m_op_flags, this->m_trace, on_finish);
 893   return req;
 894 }
 895
 896 template <typename I>
 897 void ImageWriteSameRequest<I>::update_stats(size_t length) {
 898   I &image_ctx = this->m_image_ctx;
 899   image_ctx.perfcounter->inc(l_librbd_ws);
 900   image_ctx.perfcounter->inc(l_librbd_ws_bytes, length);
 901 }
 902
 903 template <typename I>
 904 uint64_t ImageCompareAndWriteRequest<I>::append_journal_event(
 905     const ObjectRequests &requests, bool synchronous) {
 906
 907   I &image_ctx = this->m_image_ctx;
 908
 909   uint64_t tid = 0;
 910   assert(this->m_image_extents.size() == 1);
 911   auto &extent = this->m_image_extents.front();
 912   journal::EventEntry event_entry(journal::AioCompareAndWriteEvent(extent.first,
 913                                                                    extent.second,
 914                                                                    m_cmp_bl, m_bl));
 915   tid = image_ctx.journal->append_io_event(std::move(event_entry),
 916                                            requests, extent.first,
 917                                            extent.second, synchronous, -EILSEQ);
 918
 919   AioCompletion *aio_comp = this->m_aio_comp;
 920   aio_comp->associate_journal_event(tid);
 921
 922   return tid;
 923 }
 924
 925 template <typename I>
 926 uint32_t ImageCompareAndWriteRequest<I>::get_object_cache_request_count() const {
 927   // extra completion request is required for tracking discard
 928   I &image_ctx = this->m_image_ctx;
 929   return (image_ctx.object_cacher != nullptr ? 1 : 0);
 930 }
 931
 932 template <typename I>
 933 void ImageCompareAndWriteRequest<I>::send_object_cache_requests(
 934   const ObjectExtents &object_extents, uint64_t journal_tid) {
 935   I &image_ctx = this->m_image_ctx;
 936
 937   if (image_ctx.object_cacher != NULL) {
 938     auto aio_comp = this->m_aio_comp;
 939     auto req = new C_DiscardRequest<I>(image_ctx, aio_comp, object_extents);
 940     req->send();
 941   }
 942 }
 943
 944 template <typename I>
 945 void ImageCompareAndWriteRequest<I>::assemble_extent(
 946   const ObjectExtent &object_extent, bufferlist *bl) {
 947   for (auto q = object_extent.buffer_extents.begin();
 948        q != object_extent.buffer_extents.end(); ++q) {
 949     bufferlist sub_bl;
 950     sub_bl.substr_of(m_bl, q->first, q->second);
 951     bl->claim_append(sub_bl);
 952   }
 953 }
 954
 955 template <typename I>
 956 void ImageCompareAndWriteRequest<I>::send_image_cache_request() {
 957   I &image_ctx = this->m_image_ctx;
 958   assert(image_ctx.image_cache != nullptr);
 959
 960   AioCompletion *aio_comp = this->m_aio_comp;
 961   aio_comp->set_request_count(1);
 962   C_AioRequest *req_comp = new C_AioRequest(aio_comp);
 963   image_ctx.image_cache->aio_compare_and_write(
 964     std::move(this->m_image_extents), std::move(m_cmp_bl), std::move(m_bl),
 965     m_mismatch_offset, m_op_flags, req_comp);
 966 }
 967
 968 template <typename I>
 969 ObjectRequestHandle *ImageCompareAndWriteRequest<I>::create_object_request(
 970     const ObjectExtent &object_extent,
 971     const ::SnapContext &snapc,
 972     Context *on_finish) {
 973   I &image_ctx = this->m_image_ctx;
 974
 975   bufferlist bl;
 976   assemble_extent(object_extent, &bl);
 977   ObjectRequest<I> *req = ObjectRequest<I>::create_compare_and_write(
 978                                   &image_ctx, object_extent.oid.name,
 979                                   object_extent.objectno, object_extent.offset,
 980                                   m_cmp_bl, bl, snapc, m_mismatch_offset,
 981                                   m_op_flags, this->m_trace, on_finish);
 982   return req;
 983 }
 984
 985 template <typename I>
 986 void ImageCompareAndWriteRequest<I>::update_stats(size_t length) {
 987   I &image_ctx = this->m_image_ctx;
 988   image_ctx.perfcounter->inc(l_librbd_cmp);
 989   image_ctx.perfcounter->inc(l_librbd_cmp_bytes, length);
 990 }
 991
 992 template <typename I>
 993 int ImageCompareAndWriteRequest<I>::prune_object_extents(ObjectExtents &object_extents) {
 994   if (object_extents.size() > 1)
 995     return -EINVAL;
 996
 997   I &image_ctx = this->m_image_ctx;
 998   uint64_t sector_size = 512ULL;
 999   uint64_t su = image_ctx.layout.stripe_unit;
1000   ObjectExtent object_extent = object_extents.front();
1001   if (object_extent.offset % sector_size + object_extent.length > sector_size ||
1002       (su != 0 && (object_extent.offset % su + object_extent.length > su)))
1003     return -EINVAL;
1004
1005   return 0;
1006 }
1007
1008 } // namespace io
1009 } // namespace librbd
1010
1011 template class librbd::io::ImageRequest<librbd::ImageCtx>;
1012 template class librbd::io::ImageReadRequest<librbd::ImageCtx>;
1013 template class librbd::io::AbstractImageWriteRequest<librbd::ImageCtx>;
1014 template class librbd::io::ImageWriteRequest<librbd::ImageCtx>;
1015 template class librbd::io::ImageDiscardRequest<librbd::ImageCtx>;
1016 template class librbd::io::ImageFlushRequest<librbd::ImageCtx>;
1017 template class librbd::io::ImageWriteSameRequest<librbd::ImageCtx>;
1018 template class librbd::io::ImageCompareAndWriteRequest<librbd::ImageCtx>;