ceph/src/os/bluestore/BlockDevice.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5   *
   6  * Copyright (C) 2015 XSky <haomai@xsky.com>
   7  *
   8  * Author: Haomai Wang <haomaiwang@gmail.com>
   9  *
  10  * This is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License version 2.1, as published by the Free Software
  13  * Foundation.  See file COPYING.
  14  *
  15  */
  16
  17 #include <libgen.h>
  18 #include <unistd.h>
  19
  20 #include "BlockDevice.h"
  21
  22 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
  23 #include "KernelDevice.h"
  24 #endif
  25
  26 #if defined(HAVE_SPDK)
  27 #include "NVMEDevice.h"
  28 #endif
  29
  30 #if defined(HAVE_BLUESTORE_PMEM)
  31 #include "PMEMDevice.h"
  32 #include "libpmem.h"
  33 #endif
  34
  35 #include "common/debug.h"
  36 #include "common/EventTrace.h"
  37 #include "common/errno.h"
  38 #include "include/compat.h"
  39
  40 #define dout_context cct
  41 #define dout_subsys ceph_subsys_bdev
  42 #undef dout_prefix
  43 #define dout_prefix *_dout << "bdev "
  44
  45 void IOContext::aio_wait()
  46 {
  47   std::unique_lock l(lock);
  48   // see _aio_thread for waker logic
  49   while (num_running.load() > 0) {
  50     dout(10) << __func__ << " " << this
  51              << " waiting for " << num_running.load() << " aios to complete"
  52              << dendl;
  53     cond.wait(l);
  54   }
  55   dout(20) << __func__ << " " << this << " done" << dendl;
  56 }
  57
  58 uint64_t IOContext::get_num_ios() const
  59 {
  60   // this is about the simplest model for transaction cost you can
  61   // imagine.  there is some fixed overhead cost by saying there is a
  62   // minimum of one "io".  and then we have some cost per "io" that is
  63   // a configurable (with different hdd and ssd defaults), and add
  64   // that to the bytes value.
  65   uint64_t ios = 0;
  66 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
  67   for (auto& p : pending_aios) {
  68     ios += p.iov.size();
  69   }
  70 #endif
  71 #ifdef HAVE_SPDK
  72   ios += total_nseg;
  73 #endif
  74   return ios;
  75 }
  76
  77 void IOContext::release_running_aios()
  78 {
  79   ceph_assert(!num_running);
  80 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
  81   // release aio contexts (including pinned buffers).
  82   running_aios.clear();
  83 #endif
  84 }
  85
  86 BlockDevice *BlockDevice::create(CephContext* cct, const string& path,
  87                                  aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv)
  88 {
  89   string type = "kernel";
  90   char buf[PATH_MAX + 1];
  91   int r = ::readlink(path.c_str(), buf, sizeof(buf) - 1);
  92   if (r >= 0) {
  93     buf[r] = '\0';
  94     char *bname = ::basename(buf);
  95     if (strncmp(bname, SPDK_PREFIX, sizeof(SPDK_PREFIX)-1) == 0)
  96       type = "ust-nvme";
  97   }
  98
  99 #if defined(HAVE_BLUESTORE_PMEM)
 100   if (type == "kernel") {
 101     int is_pmem = 0;
 102     size_t map_len = 0;
 103     void *addr = pmem_map_file(path.c_str(), 0, PMEM_FILE_EXCL, O_RDONLY, &map_len, &is_pmem);
 104     if (addr != NULL) {
 105       if (is_pmem)
 106         type = "pmem";
 107       else
 108         dout(1) << path.c_str() << " isn't pmem file" << dendl;
 109       pmem_unmap(addr, map_len);
 110     } else {
 111       dout(1) << "pmem_map_file:" << path.c_str() << " failed." << pmem_errormsg() << dendl;
 112     }
 113   }
 114 #endif
 115
 116   dout(1) << __func__ << " path " << path << " type " << type << dendl;
 117
 118 #if defined(HAVE_BLUESTORE_PMEM)
 119   if (type == "pmem") {
 120     return new PMEMDevice(cct, cb, cbpriv);
 121   }
 122 #endif
 123 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
 124   if (type == "kernel") {
 125     return new KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv);
 126   }
 127 #endif
 128 #ifndef WITH_SEASTAR
 129 #if defined(HAVE_SPDK)
 130   if (type == "ust-nvme") {
 131     return new NVMEDevice(cct, cb, cbpriv);
 132   }
 133 #endif
 134 #endif
 135
 136   derr << __func__ << " unknown backend " << type << dendl;
 137   ceph_abort();
 138   return NULL;
 139 }
 140
 141 void BlockDevice::queue_reap_ioc(IOContext *ioc)
 142 {
 143   std::lock_guard l(ioc_reap_lock);
 144   if (ioc_reap_count.load() == 0)
 145     ++ioc_reap_count;
 146   ioc_reap_queue.push_back(ioc);
 147 }
 148
 149 void BlockDevice::reap_ioc()
 150 {
 151   if (ioc_reap_count.load()) {
 152     std::lock_guard l(ioc_reap_lock);
 153     for (auto p : ioc_reap_queue) {
 154       dout(20) << __func__ << " reap ioc " << p << dendl;
 155       delete p;
 156     }
 157     ioc_reap_queue.clear();
 158     --ioc_reap_count;
 159   }
 160 }