ceph/src/tools/rbd_nbd/rbd-nbd.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 /*
   5  * rbd-nbd - RBD in userspace
   6  *
   7  * Copyright (C) 2015 - 2016 Kylin Corporation
   8  *
   9  * Author: Yunchuan Wen <yunchuan.wen@kylin-cloud.com>
  10  *         Li Wang <li.wang@kylin-cloud.com>
  11  *
  12  * This is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU Lesser General Public
  14  * License version 2.1, as published by the Free Software
  15  * Foundation.  See file COPYING.
  16  *
  17 */
  18
  19 #include "include/int_types.h"
  20
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <stddef.h>
  24 #include <errno.h>
  25 #include <fcntl.h>
  26 #include <string.h>
  27 #include <sys/types.h>
  28 #include <unistd.h>
  29
  30 #include <linux/nbd.h>
  31 #include <linux/fs.h>
  32 #include <sys/ioctl.h>
  33 #include <sys/socket.h>
  34
  35 #include <fstream>
  36 #include <iostream>
  37 #include <memory>
  38 #include <regex>
  39 #include <boost/algorithm/string/predicate.hpp>
  40
  41 #include "common/Formatter.h"
  42 #include "common/Preforker.h"
  43 #include "common/TextTable.h"
  44 #include "common/ceph_argparse.h"
  45 #include "common/config.h"
  46 #include "common/dout.h"
  47 #include "common/errno.h"
  48 #include "common/module.h"
  49 #include "common/safe_io.h"
  50 #include "common/version.h"
  51
  52 #include "global/global_init.h"
  53 #include "global/signal_handler.h"
  54
  55 #include "include/rados/librados.hpp"
  56 #include "include/rbd/librbd.hpp"
  57 #include "include/stringify.h"
  58 #include "include/xlist.h"
  59
  60 #include "mon/MonClient.h"
  61
  62 #define dout_context g_ceph_context
  63 #define dout_subsys ceph_subsys_rbd
  64 #undef dout_prefix
  65 #define dout_prefix *_dout << "rbd-nbd: "
  66
  67 struct Config {
  68   int nbds_max = 0;
  69   int max_part = 255;
  70   int timeout = -1;
  71
  72   bool exclusive = false;
  73   bool readonly = false;
  74   bool set_max_part = false;
  75
  76   std::string poolname;
  77   std::string nsname;
  78   std::string imgname;
  79   std::string snapname;
  80   std::string devpath;
  81
  82   std::string format;
  83   bool pretty_format = false;
  84 };
  85
  86 static void usage()
  87 {
  88   std::cout << "Usage: rbd-nbd [options] map <image-or-snap-spec>  Map an image to nbd device\n"
  89             << "               unmap <device|image-or-snap-spec>   Unmap nbd device\n"
  90             << "               [options] list-mapped               List mapped nbd devices\n"
  91             << "Map options:\n"
  92             << "  --device <device path>  Specify nbd device path\n"
  93             << "  --read-only             Map read-only\n"
  94             << "  --nbds_max <limit>      Override for module param nbds_max\n"
  95             << "  --max_part <limit>      Override for module param max_part\n"
  96             << "  --exclusive             Forbid writes by other clients\n"
  97             << "  --timeout <seconds>     Set nbd request timeout\n"
  98             << "\n"
  99             << "List options:\n"
 100             << "  --format plain|json|xml Output format (default: plain)\n"
 101             << "  --pretty-format         Pretty formatting (json and xml)\n"
 102             << std::endl;
 103   generic_server_usage();
 104 }
 105
 106 static int nbd = -1;
 107
 108 enum Command {
 109   None,
 110   Connect,
 111   Disconnect,
 112   List
 113 };
 114
 115 static Command cmd = None;
 116
 117 #define RBD_NBD_BLKSIZE 512UL
 118
 119 #define HELP_INFO 1
 120 #define VERSION_INFO 2
 121
 122 #ifdef CEPH_BIG_ENDIAN
 123 #define ntohll(a) (a)
 124 #elif defined(CEPH_LITTLE_ENDIAN)
 125 #define ntohll(a) swab(a)
 126 #else
 127 #error "Could not determine endianess"
 128 #endif
 129 #define htonll(a) ntohll(a)
 130
 131 static int parse_args(vector<const char*>& args, std::ostream *err_msg,
 132                       Command *command, Config *cfg);
 133
 134 static void handle_signal(int signum)
 135 {
 136   ceph_assert(signum == SIGINT || signum == SIGTERM);
 137   derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
 138   dout(20) << __func__ << ": " << "sending NBD_DISCONNECT" << dendl;
 139   if (ioctl(nbd, NBD_DISCONNECT) < 0) {
 140     derr << "rbd-nbd: disconnect failed: " << cpp_strerror(errno) << dendl;
 141   } else {
 142     dout(20) << __func__ << ": " << "disconnected" << dendl;
 143   }
 144 }
 145
 146 class NBDServer
 147 {
 148 private:
 149   int fd;
 150   librbd::Image &image;
 151
 152 public:
 153   NBDServer(int _fd, librbd::Image& _image)
 154     : fd(_fd)
 155     , image(_image)
 156     , lock("NBDServer::Locker")
 157     , reader_thread(*this, &NBDServer::reader_entry)
 158     , writer_thread(*this, &NBDServer::writer_entry)
 159     , started(false)
 160   {}
 161
 162 private:
 163   std::atomic<bool> terminated = { false };
 164
 165   void shutdown()
 166   {
 167     bool expected = false;
 168     if (terminated.compare_exchange_strong(expected, true)) {
 169       ::shutdown(fd, SHUT_RDWR);
 170
 171       Mutex::Locker l(lock);
 172       cond.Signal();
 173     }
 174   }
 175
 176   struct IOContext
 177   {
 178     xlist<IOContext*>::item item;
 179     NBDServer *server = nullptr;
 180     struct nbd_request request;
 181     struct nbd_reply reply;
 182     bufferlist data;
 183     int command = 0;
 184
 185     IOContext()
 186       : item(this)
 187     {}
 188   };
 189
 190   friend std::ostream &operator<<(std::ostream &os, const IOContext &ctx);
 191
 192   Mutex lock;
 193   Cond cond;
 194   xlist<IOContext*> io_pending;
 195   xlist<IOContext*> io_finished;
 196
 197   void io_start(IOContext *ctx)
 198   {
 199     Mutex::Locker l(lock);
 200     io_pending.push_back(&ctx->item);
 201   }
 202
 203   void io_finish(IOContext *ctx)
 204   {
 205     Mutex::Locker l(lock);
 206     ceph_assert(ctx->item.is_on_list());
 207     ctx->item.remove_myself();
 208     io_finished.push_back(&ctx->item);
 209     cond.Signal();
 210   }
 211
 212   IOContext *wait_io_finish()
 213   {
 214     Mutex::Locker l(lock);
 215     while(io_finished.empty() && !terminated)
 216       cond.Wait(lock);
 217
 218     if (io_finished.empty())
 219       return NULL;
 220
 221     IOContext *ret = io_finished.front();
 222     io_finished.pop_front();
 223
 224     return ret;
 225   }
 226
 227   void wait_clean()
 228   {
 229     ceph_assert(!reader_thread.is_started());
 230     Mutex::Locker l(lock);
 231     while(!io_pending.empty())
 232       cond.Wait(lock);
 233
 234     while(!io_finished.empty()) {
 235       std::unique_ptr<IOContext> free_ctx(io_finished.front());
 236       io_finished.pop_front();
 237     }
 238   }
 239
 240   static void aio_callback(librbd::completion_t cb, void *arg)
 241   {
 242     librbd::RBD::AioCompletion *aio_completion =
 243     reinterpret_cast<librbd::RBD::AioCompletion*>(cb);
 244
 245     IOContext *ctx = reinterpret_cast<IOContext *>(arg);
 246     int ret = aio_completion->get_return_value();
 247
 248     dout(20) << __func__ << ": " << *ctx << dendl;
 249
 250     if (ret == -EINVAL) {
 251       // if shrinking an image, a pagecache writeback might reference
 252       // extents outside of the range of the new image extents
 253       dout(0) << __func__ << ": masking IO out-of-bounds error" << dendl;
 254       ctx->data.clear();
 255       ret = 0;
 256     }
 257
 258     if (ret < 0) {
 259       ctx->reply.error = htonl(-ret);
 260     } else if ((ctx->command == NBD_CMD_READ) &&
 261                 ret < static_cast<int>(ctx->request.len)) {
 262       int pad_byte_count = static_cast<int> (ctx->request.len) - ret;
 263       ctx->data.append_zero(pad_byte_count);
 264       dout(20) << __func__ << ": " << *ctx << ": Pad byte count: "
 265                << pad_byte_count << dendl;
 266       ctx->reply.error = htonl(0);
 267     } else {
 268       ctx->reply.error = htonl(0);
 269     }
 270     ctx->server->io_finish(ctx);
 271
 272     aio_completion->release();
 273   }
 274
 275   void reader_entry()
 276   {
 277     while (!terminated) {
 278       std::unique_ptr<IOContext> ctx(new IOContext());
 279       ctx->server = this;
 280
 281       dout(20) << __func__ << ": waiting for nbd request" << dendl;
 282
 283       int r = safe_read_exact(fd, &ctx->request, sizeof(struct nbd_request));
 284       if (r < 0) {
 285         derr << "failed to read nbd request header: " << cpp_strerror(r)
 286              << dendl;
 287         return;
 288       }
 289
 290       if (ctx->request.magic != htonl(NBD_REQUEST_MAGIC)) {
 291         derr << "invalid nbd request header" << dendl;
 292         return;
 293       }
 294
 295       ctx->request.from = ntohll(ctx->request.from);
 296       ctx->request.type = ntohl(ctx->request.type);
 297       ctx->request.len = ntohl(ctx->request.len);
 298
 299       ctx->reply.magic = htonl(NBD_REPLY_MAGIC);
 300       memcpy(ctx->reply.handle, ctx->request.handle, sizeof(ctx->reply.handle));
 301
 302       ctx->command = ctx->request.type & 0x0000ffff;
 303
 304       dout(20) << *ctx << ": start" << dendl;
 305
 306       switch (ctx->command)
 307       {
 308         case NBD_CMD_DISC:
 309           // NBD_DO_IT will return when pipe is closed
 310           dout(0) << "disconnect request received" << dendl;
 311           return;
 312         case NBD_CMD_WRITE:
 313           bufferptr ptr(ctx->request.len);
 314           r = safe_read_exact(fd, ptr.c_str(), ctx->request.len);
 315           if (r < 0) {
 316             derr << *ctx << ": failed to read nbd request data: "
 317                  << cpp_strerror(r) << dendl;
 318             return;
 319           }
 320           ctx->data.push_back(ptr);
 321           break;
 322       }
 323
 324       IOContext *pctx = ctx.release();
 325       io_start(pctx);
 326       librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion(pctx, aio_callback);
 327       switch (pctx->command)
 328       {
 329         case NBD_CMD_WRITE:
 330           image.aio_write(pctx->request.from, pctx->request.len, pctx->data, c);
 331           break;
 332         case NBD_CMD_READ:
 333           image.aio_read(pctx->request.from, pctx->request.len, pctx->data, c);
 334           break;
 335         case NBD_CMD_FLUSH:
 336           image.aio_flush(c);
 337           break;
 338         case NBD_CMD_TRIM:
 339           image.aio_discard(pctx->request.from, pctx->request.len, c);
 340           break;
 341         default:
 342           derr << *pctx << ": invalid request command" << dendl;
 343           c->release();
 344           return;
 345       }
 346     }
 347     dout(20) << __func__ << ": terminated" << dendl;
 348   }
 349
 350   void writer_entry()
 351   {
 352     while (!terminated) {
 353       dout(20) << __func__ << ": waiting for io request" << dendl;
 354       std::unique_ptr<IOContext> ctx(wait_io_finish());
 355       if (!ctx) {
 356         dout(20) << __func__ << ": no io requests, terminating" << dendl;
 357         return;
 358       }
 359
 360       dout(20) << __func__ << ": got: " << *ctx << dendl;
 361
 362       int r = safe_write(fd, &ctx->reply, sizeof(struct nbd_reply));
 363       if (r < 0) {
 364         derr << *ctx << ": failed to write reply header: " << cpp_strerror(r)
 365              << dendl;
 366         return;
 367       }
 368       if (ctx->command == NBD_CMD_READ && ctx->reply.error == htonl(0)) {
 369         r = ctx->data.write_fd(fd);
 370         if (r < 0) {
 371           derr << *ctx << ": failed to write replay data: " << cpp_strerror(r)
 372                << dendl;
 373           return;
 374         }
 375       }
 376       dout(20) << *ctx << ": finish" << dendl;
 377     }
 378     dout(20) << __func__ << ": terminated" << dendl;
 379   }
 380
 381   class ThreadHelper : public Thread
 382   {
 383   public:
 384     typedef void (NBDServer::*entry_func)();
 385   private:
 386     NBDServer &server;
 387     entry_func func;
 388   public:
 389     ThreadHelper(NBDServer &_server, entry_func _func)
 390       :server(_server)
 391       ,func(_func)
 392     {}
 393   protected:
 394     void* entry() override
 395     {
 396       (server.*func)();
 397       server.shutdown();
 398       return NULL;
 399     }
 400   } reader_thread, writer_thread;
 401
 402   bool started;
 403 public:
 404   void start()
 405   {
 406     if (!started) {
 407       dout(10) << __func__ << ": starting" << dendl;
 408
 409       started = true;
 410
 411       reader_thread.create("rbd_reader");
 412       writer_thread.create("rbd_writer");
 413     }
 414   }
 415
 416   ~NBDServer()
 417   {
 418     if (started) {
 419       dout(10) << __func__ << ": terminating" << dendl;
 420
 421       shutdown();
 422
 423       reader_thread.join();
 424       writer_thread.join();
 425
 426       wait_clean();
 427
 428       started = false;
 429     }
 430   }
 431 };
 432
 433 std::ostream &operator<<(std::ostream &os, const NBDServer::IOContext &ctx) {
 434
 435   os << "[" << std::hex << ntohll(*((uint64_t *)ctx.request.handle));
 436
 437   switch (ctx.command)
 438   {
 439   case NBD_CMD_WRITE:
 440     os << " WRITE ";
 441     break;
 442   case NBD_CMD_READ:
 443     os << " READ ";
 444     break;
 445   case NBD_CMD_FLUSH:
 446     os << " FLUSH ";
 447     break;
 448   case NBD_CMD_TRIM:
 449     os << " TRIM ";
 450     break;
 451   default:
 452     os << " UNKNOWN(" << ctx.command << ") ";
 453     break;
 454   }
 455
 456   os << ctx.request.from << "~" << ctx.request.len << " "
 457      << std::dec << ntohl(ctx.reply.error) << "]";
 458
 459   return os;
 460 }
 461
 462 class NBDWatchCtx : public librbd::UpdateWatchCtx
 463 {
 464 private:
 465   int fd;
 466   librados::IoCtx &io_ctx;
 467   librbd::Image &image;
 468   unsigned long size;
 469 public:
 470   NBDWatchCtx(int _fd,
 471               librados::IoCtx &_io_ctx,
 472               librbd::Image &_image,
 473               unsigned long _size)
 474     : fd(_fd)
 475     , io_ctx(_io_ctx)
 476     , image(_image)
 477     , size(_size)
 478   { }
 479
 480   ~NBDWatchCtx() override {}
 481
 482   void handle_notify() override
 483   {
 484     librbd::image_info_t info;
 485     if (image.stat(info, sizeof(info)) == 0) {
 486       unsigned long new_size = info.size;
 487
 488       if (new_size != size) {
 489         dout(5) << "resize detected" << dendl;
 490         if (ioctl(fd, BLKFLSBUF, NULL) < 0)
 491             derr << "invalidate page cache failed: " << cpp_strerror(errno)
 492                  << dendl;
 493         if (ioctl(fd, NBD_SET_SIZE, new_size) < 0) {
 494             derr << "resize failed: " << cpp_strerror(errno) << dendl;
 495         } else {
 496           size = new_size;
 497         }
 498         if (ioctl(fd, BLKRRPART, NULL) < 0) {
 499           derr << "rescan of partition table failed: " << cpp_strerror(errno)
 500                << dendl;
 501         }
 502         if (image.invalidate_cache() < 0)
 503             derr << "invalidate rbd cache failed" << dendl;
 504       }
 505     }
 506   }
 507 };
 508
 509 class NBDListIterator {
 510 public:
 511   bool get(int *pid, Config *cfg) {
 512     while (true) {
 513       std::string nbd_path = "/sys/block/nbd" + stringify(m_index);
 514       if(access(nbd_path.c_str(), F_OK) != 0) {
 515         return false;
 516       }
 517
 518       *cfg = Config();
 519       cfg->devpath = "/dev/nbd" + stringify(m_index++);
 520
 521       std::ifstream ifs;
 522       ifs.open(nbd_path + "/pid", std::ifstream::in);
 523       if (!ifs.is_open()) {
 524         continue;
 525       }
 526       ifs >> *pid;
 527
 528       int r = get_mapped_info(*pid, cfg);
 529       if (r < 0) {
 530         continue;
 531       }
 532
 533       return true;
 534     }
 535   }
 536
 537 private:
 538   int m_index = 0;
 539
 540   int get_mapped_info(int pid, Config *cfg) {
 541     int r;
 542     std::string path = "/proc/" + stringify(pid) + "/cmdline";
 543     std::ifstream ifs;
 544     std::string cmdline;
 545     std::vector<const char*> args;
 546
 547     ifs.open(path.c_str(), std::ifstream::in);
 548     if (!ifs.is_open())
 549       return -1;
 550     ifs >> cmdline;
 551
 552     for (unsigned i = 0; i < cmdline.size(); i++) {
 553       const char *arg = &cmdline[i];
 554       if (i == 0) {
 555         if (strcmp(basename(arg) , "rbd-nbd") != 0) {
 556           return -EINVAL;
 557         }
 558       } else {
 559         args.push_back(arg);
 560       }
 561
 562       while (cmdline[i] != '\0') {
 563         i++;
 564       }
 565     }
 566
 567     std::ostringstream err_msg;
 568     Command command;
 569     r = parse_args(args, &err_msg, &command, cfg);
 570     if (r < 0) {
 571       return r;
 572     }
 573
 574     if (command != Connect) {
 575       return -ENOENT;
 576     }
 577
 578     return 0;
 579   }
 580 };
 581
 582 static int open_device(const char* path, Config *cfg = nullptr, bool try_load_module = false)
 583 {
 584   int nbd = open(path, O_RDWR);
 585   bool loaded_module = false;
 586
 587   if (nbd < 0 && try_load_module && access("/sys/module/nbd", F_OK) != 0) {
 588     ostringstream param;
 589     int r;
 590     if (cfg->nbds_max) {
 591       param << "nbds_max=" << cfg->nbds_max;
 592     }
 593     if (cfg->max_part) {
 594         param << " max_part=" << cfg->max_part;
 595     }
 596     r = module_load("nbd", param.str().c_str());
 597     if (r < 0) {
 598       cerr << "rbd-nbd: failed to load nbd kernel module: " << cpp_strerror(-r) << std::endl;
 599       return r;
 600     } else {
 601       loaded_module = true;
 602     }
 603     nbd = open(path, O_RDWR);
 604   }
 605
 606   if (try_load_module && !loaded_module &&
 607       (cfg->nbds_max || cfg->set_max_part)) {
 608     cerr << "rbd-nbd: ignoring kernel module parameter options: nbd module already loaded"
 609          << std::endl;
 610   }
 611
 612   return nbd;
 613 }
 614
 615 static int check_device_size(int nbd_index, unsigned long expected_size)
 616 {
 617   // There are bugs with some older kernel versions that result in an
 618   // overflow for large image sizes. This check is to ensure we are
 619   // not affected.
 620
 621   unsigned long size = 0;
 622   std::string path = "/sys/block/nbd" + stringify(nbd_index) + "/size";
 623   std::ifstream ifs;
 624   ifs.open(path.c_str(), std::ifstream::in);
 625   if (!ifs.is_open()) {
 626     cerr << "rbd-nbd: failed to open " << path << std::endl;
 627     return -EINVAL;
 628   }
 629   ifs >> size;
 630   size *= RBD_NBD_BLKSIZE;
 631
 632   if (size == 0) {
 633     // Newer kernel versions will report real size only after nbd
 634     // connect. Assume this is the case and return success.
 635     return 0;
 636   }
 637
 638   if (size != expected_size) {
 639     cerr << "rbd-nbd: kernel reported invalid device size (" << size
 640          << ", expected " << expected_size << ")" << std::endl;
 641     return -EINVAL;
 642   }
 643
 644   return 0;
 645 }
 646
 647 static int do_map(int argc, const char *argv[], Config *cfg)
 648 {
 649   int r;
 650
 651   librados::Rados rados;
 652   librbd::RBD rbd;
 653   librados::IoCtx io_ctx;
 654   librbd::Image image;
 655
 656   int read_only = 0;
 657   unsigned long flags;
 658   unsigned long size;
 659
 660   int index = 0;
 661   int fd[2];
 662
 663   librbd::image_info_t info;
 664
 665   Preforker forker;
 666
 667   vector<const char*> args;
 668   argv_to_vec(argc, argv, args);
 669   if (args.empty()) {
 670     cerr << argv[0] << ": -h or --help for usage" << std::endl;
 671     exit(1);
 672   }
 673   if (ceph_argparse_need_usage(args)) {
 674     usage();
 675     exit(0);
 676   }
 677
 678   auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
 679                          CODE_ENVIRONMENT_DAEMON,
 680                          CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
 681   g_ceph_context->_conf.set_val_or_die("pid_file", "");
 682
 683   if (global_init_prefork(g_ceph_context) >= 0) {
 684     std::string err;
 685     r = forker.prefork(err);
 686     if (r < 0) {
 687       cerr << err << std::endl;
 688       return r;
 689     }
 690     if (forker.is_parent()) {
 691       if (forker.parent_wait(err) != 0) {
 692         return -ENXIO;
 693       }
 694       return 0;
 695     }
 696     global_init_postfork_start(g_ceph_context);
 697   }
 698
 699   common_init_finish(g_ceph_context);
 700   global_init_chdir(g_ceph_context);
 701
 702   if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) == -1) {
 703     r = -errno;
 704     goto close_ret;
 705   }
 706
 707   r = rados.init_with_context(g_ceph_context);
 708   if (r < 0)
 709     goto close_fd;
 710
 711   r = rados.connect();
 712   if (r < 0)
 713     goto close_fd;
 714
 715   r = rados.ioctx_create(cfg->poolname.c_str(), io_ctx);
 716   if (r < 0)
 717     goto close_fd;
 718
 719   io_ctx.set_namespace(cfg->nsname);
 720
 721   r = rbd.open(io_ctx, image, cfg->imgname.c_str());
 722   if (r < 0)
 723     goto close_fd;
 724
 725   if (cfg->exclusive) {
 726     r = image.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE);
 727     if (r < 0) {
 728       cerr << "rbd-nbd: failed to acquire exclusive lock: " << cpp_strerror(r)
 729            << std::endl;
 730       goto close_fd;
 731     }
 732   }
 733
 734   if (!cfg->snapname.empty()) {
 735     r = image.snap_set(cfg->snapname.c_str());
 736     if (r < 0)
 737       goto close_fd;
 738   }
 739
 740   r = image.stat(info, sizeof(info));
 741   if (r < 0)
 742     goto close_fd;
 743
 744   if (cfg->devpath.empty()) {
 745     char dev[64];
 746     bool try_load_module = true;
 747     const char *path = "/sys/module/nbd/parameters/nbds_max";
 748     int nbds_max = -1;
 749     if (access(path, F_OK) == 0) {
 750       std::ifstream ifs;
 751       ifs.open(path, std::ifstream::in);
 752       if (ifs.is_open()) {
 753         ifs >> nbds_max;
 754         ifs.close();
 755       }
 756     }
 757
 758     while (true) {
 759       snprintf(dev, sizeof(dev), "/dev/nbd%d", index);
 760
 761       nbd = open_device(dev, cfg, try_load_module);
 762       try_load_module = false;
 763       if (nbd < 0) {
 764         if (nbd == -EPERM && nbds_max != -1 && index < (nbds_max-1)) {
 765           ++index;
 766           continue;
 767         }
 768         r = nbd;
 769         cerr << "rbd-nbd: failed to find unused device" << std::endl;
 770         goto close_fd;
 771       }
 772
 773       r = ioctl(nbd, NBD_SET_SOCK, fd[0]);
 774       if (r < 0) {
 775         close(nbd);
 776         ++index;
 777         continue;
 778       }
 779
 780       cfg->devpath = dev;
 781       break;
 782     }
 783   } else {
 784     r = sscanf(cfg->devpath.c_str(), "/dev/nbd%d", &index);
 785     if (r < 0) {
 786       cerr << "rbd-nbd: invalid device path: " << cfg->devpath
 787            << " (expected /dev/nbd{num})" << std::endl;
 788       goto close_fd;
 789     }
 790     nbd = open_device(cfg->devpath.c_str(), cfg, true);
 791     if (nbd < 0) {
 792       r = nbd;
 793       cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
 794       goto close_fd;
 795     }
 796
 797     r = ioctl(nbd, NBD_SET_SOCK, fd[0]);
 798     if (r < 0) {
 799       r = -errno;
 800       cerr << "rbd-nbd: the device " << cfg->devpath << " is busy" << std::endl;
 801       close(nbd);
 802       goto close_fd;
 803     }
 804   }
 805
 806   flags = NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_TRIM | NBD_FLAG_HAS_FLAGS;
 807   if (!cfg->snapname.empty() || cfg->readonly) {
 808     flags |= NBD_FLAG_READ_ONLY;
 809     read_only = 1;
 810   }
 811
 812   r = ioctl(nbd, NBD_SET_BLKSIZE, RBD_NBD_BLKSIZE);
 813   if (r < 0) {
 814     r = -errno;
 815     goto close_nbd;
 816   }
 817
 818   if (info.size > ULONG_MAX) {
 819     r = -EFBIG;
 820     cerr << "rbd-nbd: image is too large (" << byte_u_t(info.size)
 821          << ", max is " << byte_u_t(ULONG_MAX) << ")" << std::endl;
 822     goto close_nbd;
 823   }
 824
 825   size = info.size;
 826
 827   r = ioctl(nbd, NBD_SET_SIZE, size);
 828   if (r < 0) {
 829     r = -errno;
 830     goto close_nbd;
 831   }
 832
 833   r = check_device_size(index, size);
 834   if (r < 0) {
 835     goto close_nbd;
 836   }
 837
 838   ioctl(nbd, NBD_SET_FLAGS, flags);
 839
 840   r = ioctl(nbd, BLKROSET, (unsigned long) &read_only);
 841   if (r < 0) {
 842     r = -errno;
 843     goto close_nbd;
 844   }
 845
 846   if (cfg->timeout >= 0) {
 847     r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)cfg->timeout);
 848     if (r < 0) {
 849       r = -errno;
 850       cerr << "rbd-nbd: failed to set timeout: " << cpp_strerror(r)
 851            << std::endl;
 852       goto close_nbd;
 853     }
 854   }
 855
 856   {
 857     uint64_t handle;
 858
 859     NBDWatchCtx watch_ctx(nbd, io_ctx, image, info.size);
 860     r = image.update_watch(&watch_ctx, &handle);
 861     if (r < 0)
 862       goto close_nbd;
 863
 864     cout << cfg->devpath << std::endl;
 865
 866     if (g_conf()->daemonize) {
 867       global_init_postfork_finish(g_ceph_context);
 868       forker.daemonize();
 869     }
 870
 871     {
 872       NBDServer server(fd[1], image);
 873
 874       server.start();
 875
 876       init_async_signal_handler();
 877       register_async_signal_handler(SIGHUP, sighup_handler);
 878       register_async_signal_handler_oneshot(SIGINT, handle_signal);
 879       register_async_signal_handler_oneshot(SIGTERM, handle_signal);
 880
 881       ioctl(nbd, NBD_DO_IT);
 882
 883       unregister_async_signal_handler(SIGHUP, sighup_handler);
 884       unregister_async_signal_handler(SIGINT, handle_signal);
 885       unregister_async_signal_handler(SIGTERM, handle_signal);
 886       shutdown_async_signal_handler();
 887     }
 888
 889     r = image.update_unwatch(handle);
 890     ceph_assert(r == 0);
 891   }
 892
 893 close_nbd:
 894   if (r < 0) {
 895     ioctl(nbd, NBD_CLEAR_SOCK);
 896     cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r) << std::endl;
 897   }
 898   close(nbd);
 899 close_fd:
 900   close(fd[0]);
 901   close(fd[1]);
 902 close_ret:
 903   image.close();
 904   io_ctx.close();
 905   rados.shutdown();
 906
 907   forker.exit(r < 0 ? EXIT_FAILURE : 0);
 908   // Unreachable;
 909   return r;
 910 }
 911
 912 static int do_unmap(const std::string &devpath)
 913 {
 914   int r = 0;
 915
 916   int nbd = open_device(devpath.c_str());
 917   if (nbd < 0) {
 918     cerr << "rbd-nbd: failed to open device: " << devpath << std::endl;
 919     return nbd;
 920   }
 921
 922   r = ioctl(nbd, NBD_DISCONNECT);
 923   if (r < 0) {
 924       cerr << "rbd-nbd: the device is not used" << std::endl;
 925   }
 926
 927   close(nbd);
 928
 929   return r;
 930 }
 931
 932 static int parse_imgpath(const std::string &imgpath, Config *cfg,
 933                          std::ostream *err_msg) {
 934   std::regex pattern("^(?:([^/]+)/(?:([^/@]+)/)?)?([^@]+)(?:@([^/@]+))?$");
 935   std::smatch match;
 936   if (!std::regex_match(imgpath, match, pattern)) {
 937     std::cerr << "rbd-nbd: invalid spec '" << imgpath << "'" << std::endl;
 938     return -EINVAL;
 939   }
 940
 941   if (match[1].matched) {
 942     cfg->poolname = match[1];
 943   }
 944
 945   if (match[2].matched) {
 946     cfg->nsname = match[2];
 947   }
 948
 949   cfg->imgname = match[3];
 950
 951   if (match[4].matched)
 952     cfg->snapname = match[4];
 953
 954   return 0;
 955 }
 956
 957 static int do_list_mapped_devices(const std::string &format, bool pretty_format)
 958 {
 959   bool should_print = false;
 960   std::unique_ptr<ceph::Formatter> f;
 961   TextTable tbl;
 962
 963   if (format == "json") {
 964     f.reset(new JSONFormatter(pretty_format));
 965   } else if (format == "xml") {
 966     f.reset(new XMLFormatter(pretty_format));
 967   } else if (!format.empty() && format != "plain") {
 968     std::cerr << "rbd-nbd: invalid output format: " << format << std::endl;
 969     return -EINVAL;
 970   }
 971
 972   if (f) {
 973     f->open_array_section("devices");
 974   } else {
 975     tbl.define_column("id", TextTable::LEFT, TextTable::LEFT);
 976     tbl.define_column("pool", TextTable::LEFT, TextTable::LEFT);
 977     tbl.define_column("namespace", TextTable::LEFT, TextTable::LEFT);
 978     tbl.define_column("image", TextTable::LEFT, TextTable::LEFT);
 979     tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT);
 980     tbl.define_column("device", TextTable::LEFT, TextTable::LEFT);
 981   }
 982
 983   int pid;
 984   Config cfg;
 985   NBDListIterator it;
 986   while (it.get(&pid, &cfg)) {
 987     if (f) {
 988       f->open_object_section("device");
 989       f->dump_int("id", pid);
 990       f->dump_string("pool", cfg.poolname);
 991       f->dump_string("namespace", cfg.nsname);
 992       f->dump_string("image", cfg.imgname);
 993       f->dump_string("snap", cfg.snapname);
 994       f->dump_string("device", cfg.devpath);
 995       f->close_section();
 996     } else {
 997       should_print = true;
 998       if (cfg.snapname.empty()) {
 999         cfg.snapname = "-";
1000       }
1001       tbl << pid << cfg.poolname << cfg.nsname << cfg.imgname << cfg.snapname
1002           << cfg.devpath << TextTable::endrow;
1003     }
1004   }
1005
1006   if (f) {
1007     f->close_section(); // devices
1008     f->flush(std::cout);
1009   }
1010   if (should_print) {
1011     std::cout << tbl;
1012   }
1013   return 0;
1014 }
1015
1016 static bool find_mapped_dev_by_spec(Config *cfg) {
1017   int pid;
1018   Config c;
1019   NBDListIterator it;
1020   while (it.get(&pid, &c)) {
1021     if (c.poolname == cfg->poolname && c.imgname == cfg->imgname &&
1022         c.snapname == cfg->snapname) {
1023       *cfg = c;
1024       return true;
1025     }
1026   }
1027   return false;
1028 }
1029
1030
1031 static int parse_args(vector<const char*>& args, std::ostream *err_msg,
1032                       Command *command, Config *cfg) {
1033   std::string conf_file_list;
1034   std::string cluster;
1035   CephInitParameters iparams = ceph_argparse_early_args(
1036           args, CEPH_ENTITY_TYPE_CLIENT, &cluster, &conf_file_list);
1037
1038   ConfigProxy config{false};
1039   config->name = iparams.name;
1040   config->cluster = cluster;
1041
1042   if (!conf_file_list.empty()) {
1043     config.parse_config_files(conf_file_list.c_str(), nullptr, 0);
1044   } else {
1045     config.parse_config_files(nullptr, nullptr, 0);
1046   }
1047   config.parse_env(CEPH_ENTITY_TYPE_CLIENT);
1048   config.parse_argv(args);
1049   cfg->poolname = config.get_val<std::string>("rbd_default_pool");
1050
1051   std::vector<const char*>::iterator i;
1052   std::ostringstream err;
1053
1054   for (i = args.begin(); i != args.end(); ) {
1055     if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
1056       return HELP_INFO;
1057     } else if (ceph_argparse_flag(args, i, "-v", "--version", (char*)NULL)) {
1058       return VERSION_INFO;
1059     } else if (ceph_argparse_witharg(args, i, &cfg->devpath, "--device", (char *)NULL)) {
1060     } else if (ceph_argparse_witharg(args, i, &cfg->nbds_max, err, "--nbds_max", (char *)NULL)) {
1061       if (!err.str().empty()) {
1062         *err_msg << "rbd-nbd: " << err.str();
1063         return -EINVAL;
1064       }
1065       if (cfg->nbds_max < 0) {
1066         *err_msg << "rbd-nbd: Invalid argument for nbds_max!";
1067         return -EINVAL;
1068       }
1069     } else if (ceph_argparse_witharg(args, i, &cfg->max_part, err, "--max_part", (char *)NULL)) {
1070       if (!err.str().empty()) {
1071         *err_msg << "rbd-nbd: " << err.str();
1072         return -EINVAL;
1073       }
1074       if ((cfg->max_part < 0) || (cfg->max_part > 255)) {
1075         *err_msg << "rbd-nbd: Invalid argument for max_part(0~255)!";
1076         return -EINVAL;
1077       }
1078       cfg->set_max_part = true;
1079     } else if (ceph_argparse_flag(args, i, "--read-only", (char *)NULL)) {
1080       cfg->readonly = true;
1081     } else if (ceph_argparse_flag(args, i, "--exclusive", (char *)NULL)) {
1082       cfg->exclusive = true;
1083     } else if (ceph_argparse_witharg(args, i, &cfg->timeout, err, "--timeout",
1084                                      (char *)NULL)) {
1085       if (!err.str().empty()) {
1086         *err_msg << "rbd-nbd: " << err.str();
1087         return -EINVAL;
1088       }
1089       if (cfg->timeout < 0) {
1090         *err_msg << "rbd-nbd: Invalid argument for timeout!";
1091         return -EINVAL;
1092       }
1093     } else if (ceph_argparse_witharg(args, i, &cfg->format, err, "--format",
1094                                      (char *)NULL)) {
1095     } else if (ceph_argparse_flag(args, i, "--pretty-format", (char *)NULL)) {
1096       cfg->pretty_format = true;
1097     } else {
1098       ++i;
1099     }
1100   }
1101
1102   Command cmd = None;
1103   if (args.begin() != args.end()) {
1104     if (strcmp(*args.begin(), "map") == 0) {
1105       cmd = Connect;
1106     } else if (strcmp(*args.begin(), "unmap") == 0) {
1107       cmd = Disconnect;
1108     } else if (strcmp(*args.begin(), "list-mapped") == 0) {
1109       cmd = List;
1110     } else {
1111       *err_msg << "rbd-nbd: unknown command: " <<  *args.begin();
1112       return -EINVAL;
1113     }
1114     args.erase(args.begin());
1115   }
1116
1117   if (cmd == None) {
1118     *err_msg << "rbd-nbd: must specify command";
1119     return -EINVAL;
1120   }
1121
1122   switch (cmd) {
1123     case Connect:
1124       if (args.begin() == args.end()) {
1125         *err_msg << "rbd-nbd: must specify image-or-snap-spec";
1126         return -EINVAL;
1127       }
1128       if (parse_imgpath(*args.begin(), cfg, err_msg) < 0) {
1129         return -EINVAL;
1130       }
1131       args.erase(args.begin());
1132       break;
1133     case Disconnect:
1134       if (args.begin() == args.end()) {
1135         *err_msg << "rbd-nbd: must specify nbd device or image-or-snap-spec";
1136         return -EINVAL;
1137       }
1138       if (boost::starts_with(*args.begin(), "/dev/")) {
1139         cfg->devpath = *args.begin();
1140       } else {
1141         if (parse_imgpath(*args.begin(), cfg, err_msg) < 0) {
1142           return -EINVAL;
1143         }
1144         if (!find_mapped_dev_by_spec(cfg)) {
1145           *err_msg << "rbd-nbd: " << *args.begin() << " is not mapped";
1146           return -ENOENT;
1147         }
1148       }
1149       args.erase(args.begin());
1150       break;
1151     default:
1152       //shut up gcc;
1153       break;
1154   }
1155
1156   if (args.begin() != args.end()) {
1157     *err_msg << "rbd-nbd: unknown args: " << *args.begin();
1158     return -EINVAL;
1159   }
1160
1161   *command = cmd;
1162   return 0;
1163 }
1164
1165 static int rbd_nbd(int argc, const char *argv[])
1166 {
1167   int r;
1168   Config cfg;
1169   vector<const char*> args;
1170   argv_to_vec(argc, argv, args);
1171
1172   std::ostringstream err_msg;
1173   r = parse_args(args, &err_msg, &cmd, &cfg);
1174   if (r == HELP_INFO) {
1175     usage();
1176     return 0;
1177   } else if (r == VERSION_INFO) {
1178     std::cout << pretty_version_to_str() << std::endl;
1179     return 0;
1180   } else if (r < 0) {
1181     cerr << err_msg.str() << std::endl;
1182     return r;
1183   }
1184
1185   switch (cmd) {
1186     case Connect:
1187       if (cfg.imgname.empty()) {
1188         cerr << "rbd-nbd: image name was not specified" << std::endl;
1189         return -EINVAL;
1190       }
1191
1192       r = do_map(argc, argv, &cfg);
1193       if (r < 0)
1194         return -EINVAL;
1195       break;
1196     case Disconnect:
1197       r = do_unmap(cfg.devpath);
1198       if (r < 0)
1199         return -EINVAL;
1200       break;
1201     case List:
1202       r = do_list_mapped_devices(cfg.format, cfg.pretty_format);
1203       if (r < 0)
1204         return -EINVAL;
1205       break;
1206     default:
1207       usage();
1208       break;
1209   }
1210
1211   return 0;
1212 }
1213
1214 int main(int argc, const char *argv[])
1215 {
1216   int r = rbd_nbd(argc, argv);
1217   if (r < 0) {
1218     return EXIT_FAILURE;
1219   }
1220   return 0;
1221 }