ceph/src/tools/rbd_wnbd/wnbd_handler.h

   1 /*
   2  * Ceph - scalable distributed file system
   3  *
   4  * Copyright (C) 2020 SUSE LINUX GmbH
   5  *
   6  * This is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License version 2.1, as published by the Free Software
   9  * Foundation.  See file COPYING.
  10  *
  11  */
  12
  13 #ifndef WNBD_HANDLER_H
  14 #define WNBD_HANDLER_H
  15
  16 #include <wnbd.h>
  17
  18 #include "common/admin_socket.h"
  19 #include "common/ceph_context.h"
  20 #include "common/Thread.h"
  21
  22 #include "include/rbd/librbd.hpp"
  23 #include "include/xlist.h"
  24
  25 #include "global/global_context.h"
  26
  27 // TODO: make this configurable.
  28 #define RBD_WNBD_MAX_TRANSFER 2 * 1024 * 1024
  29 #define SOFT_REMOVE_RETRY_INTERVAL 2
  30 #define DEFAULT_SOFT_REMOVE_TIMEOUT 15
  31 #define DEFAULT_IO_WORKER_COUNT 4
  32
  33 // Not defined by mingw.
  34 #ifndef SCSI_ADSENSE_UNRECOVERED_ERROR
  35 #define SCSI_ADSENSE_UNRECOVERED_ERROR 0x11
  36 #endif
  37
  38 // The following will be assigned to the "Owner" field of the WNBD
  39 // parameters, which can be used to determine the application managing
  40 // a disk. We'll ignore other disks.
  41 #define RBD_WNBD_OWNER_NAME "ceph-rbd-wnbd"
  42
  43 class WnbdHandler;
  44
  45 class WnbdAdminHook : public AdminSocketHook {
  46   WnbdHandler *m_handler;
  47
  48 public:
  49   explicit WnbdAdminHook(WnbdHandler *handler) :
  50         m_handler(handler) {
  51     g_ceph_context->get_admin_socket()->register_command(
  52       "wnbd stats", this, "get WNBD stats");
  53   }
  54   ~WnbdAdminHook() override {
  55     g_ceph_context->get_admin_socket()->unregister_commands(this);
  56   }
  57
  58   int call(std::string_view command, const cmdmap_t& cmdmap,
  59      Formatter *f, std::ostream& errss, bufferlist& out) override;
  60 };
  61
  62
  63 class WnbdHandler
  64 {
  65 private:
  66   librbd::Image &image;
  67   std::string instance_name;
  68   uint64_t block_count;
  69   uint32_t block_size;
  70   bool readonly;
  71   bool rbd_cache_enabled;
  72   uint32_t io_req_workers;
  73   uint32_t io_reply_workers;
  74   WnbdAdminHook* admin_hook;
  75   boost::asio::thread_pool* reply_tpool;
  76
  77 public:
  78   WnbdHandler(librbd::Image& _image, std::string _instance_name,
  79               uint64_t _block_count, uint32_t _block_size,
  80               bool _readonly, bool _rbd_cache_enabled,
  81               uint32_t _io_req_workers,
  82               uint32_t _io_reply_workers)
  83     : image(_image)
  84     , instance_name(_instance_name)
  85     , block_count(_block_count)
  86     , block_size(_block_size)
  87     , readonly(_readonly)
  88     , rbd_cache_enabled(_rbd_cache_enabled)
  89     , io_req_workers(_io_req_workers)
  90     , io_reply_workers(_io_reply_workers)
  91   {
  92     admin_hook = new WnbdAdminHook(this);
  93     // Instead of relying on librbd's own thread pool, we're going to use a
  94     // separate one. This allows us to make assumptions on the threads that
  95     // are going to send the IO replies and thus be able to cache Windows
  96     // OVERLAPPED structures.
  97     reply_tpool = new boost::asio::thread_pool(_io_reply_workers);
  98   }
  99
 100   int start();
 101   // Wait for the handler to stop, which normally happens when the driver
 102   // passes the "Disconnect" request.
 103   int wait();
 104   void shutdown();
 105
 106   int dump_stats(Formatter *f);
 107
 108   ~WnbdHandler();
 109
 110   static VOID LogMessage(
 111     WnbdLogLevel LogLevel,
 112     const char* Message,
 113     const char* FileName,
 114     UINT32 Line,
 115     const char* FunctionName);
 116
 117 private:
 118   ceph::mutex shutdown_lock = ceph::make_mutex("WnbdHandler::DisconnectLocker");
 119   bool started = false;
 120   bool terminated = false;
 121   WNBD_DISK* wnbd_disk = nullptr;
 122
 123   struct IOContext
 124   {
 125     xlist<IOContext*>::item item;
 126     WnbdHandler *handler = nullptr;
 127     WNBD_STATUS wnbd_status = {0};
 128     WnbdRequestType req_type = WnbdReqTypeUnknown;
 129     uint64_t req_handle = 0;
 130     uint32_t err_code = 0;
 131     size_t req_size;
 132     uint64_t req_from;
 133     bufferlist data;
 134
 135     IOContext()
 136       : item(this)
 137     {}
 138
 139     void set_sense(uint8_t sense_key, uint8_t asc, uint64_t info);
 140     void set_sense(uint8_t sense_key, uint8_t asc);
 141   };
 142
 143   friend std::ostream &operator<<(std::ostream &os, const IOContext &ctx);
 144
 145   void send_io_response(IOContext *ctx);
 146
 147   static void aio_callback(librbd::completion_t cb, void *arg);
 148
 149   // WNBD IO entry points
 150   static void Read(
 151     PWNBD_DISK Disk,
 152     UINT64 RequestHandle,
 153     PVOID Buffer,
 154     UINT64 BlockAddress,
 155     UINT32 BlockCount,
 156     BOOLEAN ForceUnitAccess);
 157   static void Write(
 158     PWNBD_DISK Disk,
 159     UINT64 RequestHandle,
 160     PVOID Buffer,
 161     UINT64 BlockAddress,
 162     UINT32 BlockCount,
 163     BOOLEAN ForceUnitAccess);
 164   static void Flush(
 165     PWNBD_DISK Disk,
 166     UINT64 RequestHandle,
 167     UINT64 BlockAddress,
 168     UINT32 BlockCount);
 169   static void Unmap(
 170     PWNBD_DISK Disk,
 171     UINT64 RequestHandle,
 172     PWNBD_UNMAP_DESCRIPTOR Descriptors,
 173     UINT32 Count);
 174
 175   static constexpr WNBD_INTERFACE RbdWnbdInterface =
 176   {
 177     Read,
 178     Write,
 179     Flush,
 180     Unmap,
 181   };
 182 };
 183
 184 std::ostream &operator<<(std::ostream &os, const WnbdHandler::IOContext &ctx);
 185
 186 #endif // WNBD_HANDLER_H