ceph/src/rgw/rgw_d3n_cacherequest.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #pragma once
   5
   6 #include <fcntl.h>
   7 #include <stdlib.h>
   8 #include <aio.h>
   9
  10 #include "include/rados/librados.hpp"
  11 #include "include/Context.h"
  12 #include "common/async/completion.h"
  13
  14 #include <errno.h>
  15 #include "common/error_code.h"
  16 #include "common/errno.h"
  17
  18 #include "rgw_aio.h"
  19 #include "rgw_cache.h"
  20
  21
  22 struct D3nGetObjData {
  23   std::mutex d3n_lock;
  24 };
  25
  26 struct D3nL1CacheRequest {
  27   ~D3nL1CacheRequest() {
  28     lsubdout(g_ceph_context, rgw_datacache, 30) << "D3nDataCache: " << __func__ << "(): Read From Cache, complete" << dendl;
  29   }
  30
  31   // unique_ptr with custom deleter for struct aiocb
  32   struct libaio_aiocb_deleter {
  33     void operator()(struct aiocb* c) {
  34       if(c->aio_fildes > 0) {
  35         if( ::close(c->aio_fildes) != 0) {
  36           lsubdout(g_ceph_context, rgw_datacache, 2) << "D3nDataCache: " << __func__ << "(): Error - can't close file, errno=" << -errno << dendl;
  37         }
  38       }
  39       delete c;
  40     }
  41   };
  42
  43   using unique_aio_cb_ptr = std::unique_ptr<struct aiocb, libaio_aiocb_deleter>;
  44
  45   struct AsyncFileReadOp {
  46     bufferlist result;
  47     unique_aio_cb_ptr aio_cb;
  48     using Signature = void(boost::system::error_code, bufferlist);
  49     using Completion = ceph::async::Completion<Signature, AsyncFileReadOp>;
  50
  51     int init(const DoutPrefixProvider *dpp, const std::string& file_path, off_t read_ofs, off_t read_len, void* arg) {
  52       ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): file_path=" << file_path << dendl;
  53       aio_cb.reset(new struct aiocb);
  54       memset(aio_cb.get(), 0, sizeof(struct aiocb));
  55       aio_cb->aio_fildes = TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_RDONLY|O_CLOEXEC|O_BINARY));
  56       if(aio_cb->aio_fildes < 0) {
  57         int err = errno;
  58         ldpp_dout(dpp, 1) << "ERROR: D3nDataCache: " << __func__ << "(): can't open " << file_path << " : " << cpp_strerror(err) << dendl;
  59         return -err;
  60       }
  61       if (g_conf()->rgw_d3n_l1_fadvise != POSIX_FADV_NORMAL)
  62         posix_fadvise(aio_cb->aio_fildes, 0, 0, g_conf()->rgw_d3n_l1_fadvise);
  63
  64       bufferptr bp(read_len);
  65       aio_cb->aio_buf = bp.c_str();
  66       result.append(std::move(bp));
  67
  68       aio_cb->aio_nbytes = read_len;
  69       aio_cb->aio_offset = read_ofs;
  70       aio_cb->aio_sigevent.sigev_notify = SIGEV_THREAD;
  71       aio_cb->aio_sigevent.sigev_notify_function = libaio_cb_aio_dispatch;
  72       aio_cb->aio_sigevent.sigev_notify_attributes = nullptr;
  73       aio_cb->aio_sigevent.sigev_value.sival_ptr = arg;
  74
  75       return 0;
  76     }
  77
  78     static void libaio_cb_aio_dispatch(sigval sigval) {
  79       lsubdout(g_ceph_context, rgw_datacache, 20) << "D3nDataCache: " << __func__ << "()" << dendl;
  80       auto p = std::unique_ptr<Completion>{static_cast<Completion*>(sigval.sival_ptr)};
  81       auto op = std::move(p->user_data);
  82       const int ret = -aio_error(op.aio_cb.get());
  83       boost::system::error_code ec;
  84       if (ret < 0) {
  85           ec.assign(-ret, boost::system::system_category());
  86       }
  87
  88       ceph::async::dispatch(std::move(p), ec, std::move(op.result));
  89     }
  90
  91     template <typename Executor1, typename CompletionHandler>
  92     static auto create(const Executor1& ex1, CompletionHandler&& handler) {
  93       auto p = Completion::create(ex1, std::move(handler));
  94       return p;
  95     }
  96   };
  97
  98   template <typename ExecutionContext, typename CompletionToken>
  99   auto async_read(const DoutPrefixProvider *dpp, ExecutionContext& ctx, const std::string& file_path,
 100                   off_t read_ofs, off_t read_len, CompletionToken&& token) {
 101     using Op = AsyncFileReadOp;
 102     using Signature = typename Op::Signature;
 103     boost::asio::async_completion<CompletionToken, Signature> init(token);
 104     auto p = Op::create(ctx.get_executor(), init.completion_handler);
 105     auto& op = p->user_data;
 106
 107     ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): file_path=" << file_path << dendl;
 108     int ret = op.init(dpp, file_path, read_ofs, read_len, p.get());
 109     if(0 == ret) {
 110       ret = ::aio_read(op.aio_cb.get());
 111     }
 112     ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): ::aio_read(), ret=" << ret << dendl;
 113     if(ret < 0) {
 114       auto ec = boost::system::error_code{-ret, boost::system::system_category()};
 115       ceph::async::post(std::move(p), ec, bufferlist{});
 116     } else {
 117       (void)p.release();
 118     }
 119     return init.result.get();
 120   }
 121
 122   struct d3n_libaio_handler {
 123     rgw::Aio* throttle = nullptr;
 124     rgw::AioResult& r;
 125     // read callback
 126     void operator()(boost::system::error_code ec, bufferlist bl) const {
 127       r.result = -ec.value();
 128       r.data = std::move(bl);
 129       throttle->put(r);
 130     }
 131   };
 132
 133   void file_aio_read_abstract(const DoutPrefixProvider *dpp, boost::asio::io_context& context, yield_context yield,
 134                               std::string& file_path, off_t read_ofs, off_t read_len,
 135                               rgw::Aio* aio, rgw::AioResult& r) {
 136     using namespace boost::asio;
 137     async_completion<yield_context, void()> init(yield);
 138     auto ex = get_associated_executor(init.completion_handler);
 139
 140     auto& ref = r.obj.get_ref();
 141     ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): oid=" << ref.obj.oid << dendl;
 142     async_read(dpp, context, file_path+"/"+ref.obj.oid, read_ofs, read_len, bind_executor(ex, d3n_libaio_handler{aio, r}));
 143   }
 144
 145 };