ceph/src/tools/rbd_nbd/rbd-nbd.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 /*
   5  * rbd-nbd - RBD in userspace
   6  *
   7  * Copyright (C) 2015 - 2016 Kylin Corporation
   8  *
   9  * Author: Yunchuan Wen <yunchuan.wen@kylin-cloud.com>
  10  *         Li Wang <li.wang@kylin-cloud.com>
  11  *
  12  * This is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU Lesser General Public
  14  * License version 2.1, as published by the Free Software
  15  * Foundation.  See file COPYING.
  16  *
  17 */
  18
  19 #include "include/int_types.h"
  20
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <stddef.h>
  24 #include <errno.h>
  25 #include <fcntl.h>
  26 #include <string.h>
  27 #include <sys/types.h>
  28 #include <unistd.h>
  29
  30 #include <linux/nbd.h>
  31 #include <linux/fs.h>
  32 #include <sys/ioctl.h>
  33 #include <sys/socket.h>
  34
  35 #include <iostream>
  36 #include <fstream>
  37 #include <boost/regex.hpp>
  38
  39 #include "mon/MonClient.h"
  40 #include "common/config.h"
  41 #include "common/dout.h"
  42
  43 #include "common/errno.h"
  44 #include "common/module.h"
  45 #include "common/safe_io.h"
  46 #include "common/TextTable.h"
  47 #include "common/ceph_argparse.h"
  48 #include "common/Preforker.h"
  49 #include "global/global_init.h"
  50 #include "global/signal_handler.h"
  51
  52 #include "include/rados/librados.hpp"
  53 #include "include/rbd/librbd.hpp"
  54 #include "include/stringify.h"
  55 #include "include/xlist.h"
  56
  57 #define dout_context g_ceph_context
  58 #define dout_subsys ceph_subsys_rbd
  59 #undef dout_prefix
  60 #define dout_prefix *_dout << "rbd-nbd: "
  61
  62 struct Config {
  63   int nbds_max = 0;
  64   int max_part = 255;
  65
  66   bool exclusive = false;
  67   bool readonly = false;
  68   bool set_max_part = false;
  69
  70   std::string poolname;
  71   std::string imgname;
  72   std::string snapname;
  73   std::string devpath;
  74 };
  75
  76 static void usage()
  77 {
  78   std::cout << "Usage: rbd-nbd [options] map <image-or-snap-spec>  Map an image to nbd device\n"
  79             << "               unmap <device path>                 Unmap nbd device\n"
  80             << "               list-mapped                         List mapped nbd devices\n"
  81             << "Options:\n"
  82             << "  --device <device path>  Specify nbd device path\n"
  83             << "  --read-only             Map read-only\n"
  84             << "  --nbds_max <limit>      Override for module param nbds_max\n"
  85             << "  --max_part <limit>      Override for module param max_part\n"
  86             << "  --exclusive             Forbid writes by other clients\n"
  87             << std::endl;
  88   generic_server_usage();
  89 }
  90
  91 static int nbd = -1;
  92
  93 static enum {
  94   None,
  95   Connect,
  96   Disconnect,
  97   List
  98 } cmd = None;
  99
 100 #define RBD_NBD_BLKSIZE 512UL
 101
 102 #ifdef CEPH_BIG_ENDIAN
 103 #define ntohll(a) (a)
 104 #elif defined(CEPH_LITTLE_ENDIAN)
 105 #define ntohll(a) swab(a)
 106 #else
 107 #error "Could not determine endianess"
 108 #endif
 109 #define htonll(a) ntohll(a)
 110
 111 static int parse_args(vector<const char*>& args, std::ostream *err_msg, Config *cfg);
 112
 113 static void handle_signal(int signum)
 114 {
 115   assert(signum == SIGINT || signum == SIGTERM);
 116   derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
 117   dout(20) << __func__ << ": " << "sending NBD_DISCONNECT" << dendl;
 118   if (ioctl(nbd, NBD_DISCONNECT) < 0) {
 119     derr << "rbd-nbd: disconnect failed: " << cpp_strerror(errno) << dendl;
 120   } else {
 121     dout(20) << __func__ << ": " << "disconnected" << dendl;
 122   }
 123 }
 124
 125 class NBDServer
 126 {
 127 private:
 128   int fd;
 129   librbd::Image &image;
 130
 131 public:
 132   NBDServer(int _fd, librbd::Image& _image)
 133     : fd(_fd)
 134     , image(_image)
 135     , lock("NBDServer::Locker")
 136     , reader_thread(*this, &NBDServer::reader_entry)
 137     , writer_thread(*this, &NBDServer::writer_entry)
 138     , started(false)
 139   {}
 140
 141 private:
 142   std::atomic<bool> terminated = { false };
 143
 144   void shutdown()
 145   {
 146     bool expected = false;
 147     if (terminated.compare_exchange_strong(expected, true)) {
 148       ::shutdown(fd, SHUT_RDWR);
 149
 150       Mutex::Locker l(lock);
 151       cond.Signal();
 152     }
 153   }
 154
 155   struct IOContext
 156   {
 157     xlist<IOContext*>::item item;
 158     NBDServer *server;
 159     struct nbd_request request;
 160     struct nbd_reply reply;
 161     bufferlist data;
 162     int command;
 163
 164     IOContext()
 165       : item(this)
 166     {}
 167   };
 168
 169   friend std::ostream &operator<<(std::ostream &os, const IOContext &ctx);
 170
 171   Mutex lock;
 172   Cond cond;
 173   xlist<IOContext*> io_pending;
 174   xlist<IOContext*> io_finished;
 175
 176   void io_start(IOContext *ctx)
 177   {
 178     Mutex::Locker l(lock);
 179     io_pending.push_back(&ctx->item);
 180   }
 181
 182   void io_finish(IOContext *ctx)
 183   {
 184     Mutex::Locker l(lock);
 185     assert(ctx->item.is_on_list());
 186     ctx->item.remove_myself();
 187     io_finished.push_back(&ctx->item);
 188     cond.Signal();
 189   }
 190
 191   IOContext *wait_io_finish()
 192   {
 193     Mutex::Locker l(lock);
 194     while(io_finished.empty() && !terminated)
 195       cond.Wait(lock);
 196
 197     if (io_finished.empty())
 198       return NULL;
 199
 200     IOContext *ret = io_finished.front();
 201     io_finished.pop_front();
 202
 203     return ret;
 204   }
 205
 206   void wait_clean()
 207   {
 208     assert(!reader_thread.is_started());
 209     Mutex::Locker l(lock);
 210     while(!io_pending.empty())
 211       cond.Wait(lock);
 212
 213     while(!io_finished.empty()) {
 214       ceph::unique_ptr<IOContext> free_ctx(io_finished.front());
 215       io_finished.pop_front();
 216     }
 217   }
 218
 219   static void aio_callback(librbd::completion_t cb, void *arg)
 220   {
 221     librbd::RBD::AioCompletion *aio_completion =
 222     reinterpret_cast<librbd::RBD::AioCompletion*>(cb);
 223
 224     IOContext *ctx = reinterpret_cast<IOContext *>(arg);
 225     int ret = aio_completion->get_return_value();
 226
 227     dout(20) << __func__ << ": " << *ctx << dendl;
 228
 229     if (ret == -EINVAL) {
 230       // if shrinking an image, a pagecache writeback might reference
 231       // extents outside of the range of the new image extents
 232       dout(5) << __func__ << ": masking IO out-of-bounds error" << dendl;
 233       ctx->data.clear();
 234       ret = 0;
 235     }
 236
 237     if (ret < 0) {
 238       ctx->reply.error = htonl(-ret);
 239     } else if ((ctx->command == NBD_CMD_READ) &&
 240                 ret < static_cast<int>(ctx->request.len)) {
 241       int pad_byte_count = static_cast<int> (ctx->request.len) - ret;
 242       ctx->data.append_zero(pad_byte_count);
 243       dout(20) << __func__ << ": " << *ctx << ": Pad byte count: "
 244                << pad_byte_count << dendl;
 245       ctx->reply.error = 0;
 246     } else {
 247       ctx->reply.error = htonl(0);
 248     }
 249     ctx->server->io_finish(ctx);
 250
 251     aio_completion->release();
 252   }
 253
 254   void reader_entry()
 255   {
 256     while (!terminated) {
 257       ceph::unique_ptr<IOContext> ctx(new IOContext());
 258       ctx->server = this;
 259
 260       dout(20) << __func__ << ": waiting for nbd request" << dendl;
 261
 262       int r = safe_read_exact(fd, &ctx->request, sizeof(struct nbd_request));
 263       if (r < 0) {
 264         derr << "failed to read nbd request header: " << cpp_strerror(r)
 265              << dendl;
 266         return;
 267       }
 268
 269       if (ctx->request.magic != htonl(NBD_REQUEST_MAGIC)) {
 270         derr << "invalid nbd request header" << dendl;
 271         return;
 272       }
 273
 274       ctx->request.from = ntohll(ctx->request.from);
 275       ctx->request.type = ntohl(ctx->request.type);
 276       ctx->request.len = ntohl(ctx->request.len);
 277
 278       ctx->reply.magic = htonl(NBD_REPLY_MAGIC);
 279       memcpy(ctx->reply.handle, ctx->request.handle, sizeof(ctx->reply.handle));
 280
 281       ctx->command = ctx->request.type & 0x0000ffff;
 282
 283       dout(20) << *ctx << ": start" << dendl;
 284
 285       switch (ctx->command)
 286       {
 287         case NBD_CMD_DISC:
 288           // NBD_DO_IT will return when pipe is closed
 289           dout(0) << "disconnect request received" << dendl;
 290           return;
 291         case NBD_CMD_WRITE:
 292           bufferptr ptr(ctx->request.len);
 293           r = safe_read_exact(fd, ptr.c_str(), ctx->request.len);
 294           if (r < 0) {
 295             derr << *ctx << ": failed to read nbd request data: "
 296                  << cpp_strerror(r) << dendl;
 297             return;
 298           }
 299           ctx->data.push_back(ptr);
 300           break;
 301       }
 302
 303       IOContext *pctx = ctx.release();
 304       io_start(pctx);
 305       librbd::RBD::AioCompletion *c = new librbd::RBD::AioCompletion(pctx, aio_callback);
 306       switch (pctx->command)
 307       {
 308         case NBD_CMD_WRITE:
 309           image.aio_write(pctx->request.from, pctx->request.len, pctx->data, c);
 310           break;
 311         case NBD_CMD_READ:
 312           image.aio_read(pctx->request.from, pctx->request.len, pctx->data, c);
 313           break;
 314         case NBD_CMD_FLUSH:
 315           image.aio_flush(c);
 316           break;
 317         case NBD_CMD_TRIM:
 318           image.aio_discard(pctx->request.from, pctx->request.len, c);
 319           break;
 320         default:
 321           derr << *pctx << ": invalid request command" << dendl;
 322           c->release();
 323           return;
 324       }
 325     }
 326     dout(20) << __func__ << ": terminated" << dendl;
 327   }
 328
 329   void writer_entry()
 330   {
 331     while (!terminated) {
 332       dout(20) << __func__ << ": waiting for io request" << dendl;
 333       ceph::unique_ptr<IOContext> ctx(wait_io_finish());
 334       if (!ctx) {
 335         dout(20) << __func__ << ": no io requests, terminating" << dendl;
 336         return;
 337       }
 338
 339       dout(20) << __func__ << ": got: " << *ctx << dendl;
 340
 341       int r = safe_write(fd, &ctx->reply, sizeof(struct nbd_reply));
 342       if (r < 0) {
 343         derr << *ctx << ": failed to write reply header: " << cpp_strerror(r)
 344              << dendl;
 345         return;
 346       }
 347       if (ctx->command == NBD_CMD_READ && ctx->reply.error == htonl(0)) {
 348         r = ctx->data.write_fd(fd);
 349         if (r < 0) {
 350           derr << *ctx << ": failed to write replay data: " << cpp_strerror(r)
 351                << dendl;
 352           return;
 353         }
 354       }
 355       dout(20) << *ctx << ": finish" << dendl;
 356     }
 357     dout(20) << __func__ << ": terminated" << dendl;
 358   }
 359
 360   class ThreadHelper : public Thread
 361   {
 362   public:
 363     typedef void (NBDServer::*entry_func)();
 364   private:
 365     NBDServer &server;
 366     entry_func func;
 367   public:
 368     ThreadHelper(NBDServer &_server, entry_func _func)
 369       :server(_server)
 370       ,func(_func)
 371     {}
 372   protected:
 373     void* entry() override
 374     {
 375       (server.*func)();
 376       server.shutdown();
 377       return NULL;
 378     }
 379   } reader_thread, writer_thread;
 380
 381   bool started;
 382 public:
 383   void start()
 384   {
 385     if (!started) {
 386       dout(10) << __func__ << ": starting" << dendl;
 387
 388       started = true;
 389
 390       reader_thread.create("rbd_reader");
 391       writer_thread.create("rbd_writer");
 392     }
 393   }
 394
 395   void stop()
 396   {
 397     if (started) {
 398       dout(10) << __func__ << ": terminating" << dendl;
 399
 400       shutdown();
 401
 402       reader_thread.join();
 403       writer_thread.join();
 404
 405       wait_clean();
 406
 407       started = false;
 408     }
 409   }
 410
 411   ~NBDServer()
 412   {
 413     stop();
 414   }
 415 };
 416
 417 std::ostream &operator<<(std::ostream &os, const NBDServer::IOContext &ctx) {
 418
 419   os << "[" << std::hex << ntohll(*((uint64_t *)ctx.request.handle));
 420
 421   switch (ctx.command)
 422   {
 423   case NBD_CMD_WRITE:
 424     os << " WRITE ";
 425     break;
 426   case NBD_CMD_READ:
 427     os << " READ ";
 428     break;
 429   case NBD_CMD_FLUSH:
 430     os << " FLUSH ";
 431     break;
 432   case NBD_CMD_TRIM:
 433     os << " TRIM ";
 434     break;
 435   default:
 436     os << " UNKNOW(" << ctx.command << ") ";
 437     break;
 438   }
 439
 440   os << ctx.request.from << "~" << ctx.request.len << " "
 441      << ntohl(ctx.reply.error) << "]";
 442
 443   return os;
 444 }
 445
 446 class NBDWatchCtx : public librbd::UpdateWatchCtx
 447 {
 448 private:
 449   int fd;
 450   librados::IoCtx &io_ctx;
 451   librbd::Image &image;
 452   unsigned long size;
 453 public:
 454   NBDWatchCtx(int _fd,
 455               librados::IoCtx &_io_ctx,
 456               librbd::Image &_image,
 457               unsigned long _size)
 458     : fd(_fd)
 459     , io_ctx(_io_ctx)
 460     , image(_image)
 461     , size(_size)
 462   { }
 463
 464   ~NBDWatchCtx() override {}
 465
 466   void handle_notify() override
 467   {
 468     librbd::image_info_t info;
 469     if (image.stat(info, sizeof(info)) == 0) {
 470       unsigned long new_size = info.size;
 471
 472       if (new_size != size) {
 473         if (ioctl(fd, BLKFLSBUF, NULL) < 0)
 474             derr << "invalidate page cache failed: " << cpp_strerror(errno) << dendl;
 475         if (ioctl(fd, NBD_SET_SIZE, new_size) < 0) {
 476             derr << "resize failed: " << cpp_strerror(errno) << dendl;
 477         } else {
 478           size = new_size;
 479         }
 480         if (image.invalidate_cache() < 0)
 481             derr << "invalidate rbd cache failed" << dendl;
 482       }
 483     }
 484   }
 485 };
 486
 487 static int open_device(const char* path, Config *cfg = nullptr, bool try_load_module = false)
 488 {
 489   int nbd = open(path, O_RDWR);
 490   bool loaded_module = false;
 491
 492   if (nbd < 0 && try_load_module && access("/sys/module/nbd", F_OK) != 0) {
 493     ostringstream param;
 494     int r;
 495     if (cfg->nbds_max) {
 496       param << "nbds_max=" << cfg->nbds_max;
 497     }
 498     if (cfg->max_part) {
 499         param << " max_part=" << cfg->max_part;
 500     }
 501     r = module_load("nbd", param.str().c_str());
 502     if (r < 0) {
 503       cerr << "rbd-nbd: failed to load nbd kernel module: " << cpp_strerror(-r) << std::endl;
 504       return r;
 505     } else {
 506       loaded_module = true;
 507     }
 508     nbd = open(path, O_RDWR);
 509   }
 510
 511   if (try_load_module && !loaded_module &&
 512       (cfg->nbds_max || cfg->set_max_part)) {
 513     cerr << "rbd-nbd: ignoring kernel module parameter options: nbd module already loaded"
 514          << std::endl;
 515   }
 516
 517   return nbd;
 518 }
 519
 520 static int check_device_size(int nbd_index, unsigned long expected_size)
 521 {
 522   // There are bugs with some older kernel versions that result in an
 523   // overflow for large image sizes. This check is to ensure we are
 524   // not affected.
 525
 526   unsigned long size = 0;
 527   std::string path = "/sys/block/nbd" + stringify(nbd_index) + "/size";
 528   std::ifstream ifs;
 529   ifs.open(path.c_str(), std::ifstream::in);
 530   if (!ifs.is_open()) {
 531     cerr << "rbd-nbd: failed to open " << path << std::endl;
 532     return -EINVAL;
 533   }
 534   ifs >> size;
 535   size *= RBD_NBD_BLKSIZE;
 536
 537   if (size == 0) {
 538     // Newer kernel versions will report real size only after nbd
 539     // connect. Assume this is the case and return success.
 540     return 0;
 541   }
 542
 543   if (size != expected_size) {
 544     cerr << "rbd-nbd: kernel reported invalid device size (" << size
 545          << ", expected " << expected_size << ")" << std::endl;
 546     return -EINVAL;
 547   }
 548
 549   return 0;
 550 }
 551
 552 static int do_map(int argc, const char *argv[], Config *cfg)
 553 {
 554   int r;
 555
 556   librados::Rados rados;
 557   librbd::RBD rbd;
 558   librados::IoCtx io_ctx;
 559   librbd::Image image;
 560
 561   int read_only = 0;
 562   unsigned long flags;
 563   unsigned long size;
 564
 565   int index = 0;
 566   int fd[2];
 567
 568   librbd::image_info_t info;
 569
 570   Preforker forker;
 571
 572   vector<const char*> args;
 573   argv_to_vec(argc, argv, args);
 574   env_to_vec(args);
 575
 576   auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
 577                          CODE_ENVIRONMENT_DAEMON,
 578                          CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
 579   g_ceph_context->_conf->set_val_or_die("pid_file", "");
 580
 581   if (global_init_prefork(g_ceph_context) >= 0) {
 582     std::string err;
 583     r = forker.prefork(err);
 584     if (r < 0) {
 585       cerr << err << std::endl;
 586       return r;
 587     }
 588
 589     if (forker.is_parent()) {
 590       global_init_postfork_start(g_ceph_context);
 591       if (forker.parent_wait(err) != 0) {
 592         return -ENXIO;
 593       }
 594       return 0;
 595     }
 596   }
 597
 598   common_init_finish(g_ceph_context);
 599   global_init_chdir(g_ceph_context);
 600
 601   if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) == -1) {
 602     r = -errno;
 603     goto close_ret;
 604   }
 605
 606   if (cfg->devpath.empty()) {
 607     char dev[64];
 608     bool try_load_module = true;
 609     while (true) {
 610       snprintf(dev, sizeof(dev), "/dev/nbd%d", index);
 611
 612       nbd = open_device(dev, cfg, try_load_module);
 613       try_load_module = false;
 614       if (nbd < 0) {
 615         r = nbd;
 616         cerr << "rbd-nbd: failed to find unused device" << std::endl;
 617         goto close_fd;
 618       }
 619
 620       r = ioctl(nbd, NBD_SET_SOCK, fd[0]);
 621       if (r < 0) {
 622         close(nbd);
 623         ++index;
 624         continue;
 625       }
 626
 627       cfg->devpath = dev;
 628       break;
 629     }
 630   } else {
 631     r = sscanf(cfg->devpath.c_str(), "/dev/nbd%d", &index);
 632     if (r < 0) {
 633       cerr << "rbd-nbd: invalid device path: " << cfg->devpath
 634            << " (expected /dev/nbd{num})" << std::endl;
 635       goto close_fd;
 636     }
 637     nbd = open_device(cfg->devpath.c_str(), cfg, true);
 638     if (nbd < 0) {
 639       r = nbd;
 640       cerr << "rbd-nbd: failed to open device: " << cfg->devpath << std::endl;
 641       goto close_fd;
 642     }
 643
 644     r = ioctl(nbd, NBD_SET_SOCK, fd[0]);
 645     if (r < 0) {
 646       r = -errno;
 647       cerr << "rbd-nbd: the device " << cfg->devpath << " is busy" << std::endl;
 648       close(nbd);
 649       goto close_fd;
 650     }
 651   }
 652
 653   flags = NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_TRIM | NBD_FLAG_HAS_FLAGS;
 654   if (!cfg->snapname.empty() || cfg->readonly) {
 655     flags |= NBD_FLAG_READ_ONLY;
 656     read_only = 1;
 657   }
 658
 659   r = rados.init_with_context(g_ceph_context);
 660   if (r < 0)
 661     goto close_nbd;
 662
 663   r = rados.connect();
 664   if (r < 0)
 665     goto close_nbd;
 666
 667   r = rados.ioctx_create(cfg->poolname.c_str(), io_ctx);
 668   if (r < 0)
 669     goto close_nbd;
 670
 671   r = rbd.open(io_ctx, image, cfg->imgname.c_str());
 672   if (r < 0)
 673     goto close_nbd;
 674
 675   if (cfg->exclusive) {
 676     r = image.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE);
 677     if (r < 0) {
 678       cerr << "rbd-nbd: failed to acquire exclusive lock: " << cpp_strerror(r)
 679            << std::endl;
 680       goto close_nbd;
 681     }
 682   }
 683
 684   if (!cfg->snapname.empty()) {
 685     r = image.snap_set(cfg->snapname.c_str());
 686     if (r < 0)
 687       goto close_nbd;
 688   }
 689
 690   r = image.stat(info, sizeof(info));
 691   if (r < 0)
 692     goto close_nbd;
 693
 694   r = ioctl(nbd, NBD_SET_BLKSIZE, RBD_NBD_BLKSIZE);
 695   if (r < 0) {
 696     r = -errno;
 697     goto close_nbd;
 698   }
 699
 700   if (info.size > ULONG_MAX) {
 701     r = -EFBIG;
 702     cerr << "rbd-nbd: image is too large (" << prettybyte_t(info.size)
 703          << ", max is " << prettybyte_t(ULONG_MAX) << ")" << std::endl;
 704     goto close_nbd;
 705   }
 706
 707   size = info.size;
 708
 709   r = ioctl(nbd, NBD_SET_SIZE, size);
 710   if (r < 0) {
 711     r = -errno;
 712     goto close_nbd;
 713   }
 714
 715   r = check_device_size(index, size);
 716   if (r < 0) {
 717     goto close_nbd;
 718   }
 719
 720   ioctl(nbd, NBD_SET_FLAGS, flags);
 721
 722   r = ioctl(nbd, BLKROSET, (unsigned long) &read_only);
 723   if (r < 0) {
 724     r = -errno;
 725     goto close_nbd;
 726   }
 727
 728   {
 729     uint64_t handle;
 730
 731     NBDWatchCtx watch_ctx(nbd, io_ctx, image, info.size);
 732     r = image.update_watch(&watch_ctx, &handle);
 733     if (r < 0)
 734       goto close_nbd;
 735
 736     cout << cfg->devpath << std::endl;
 737
 738     if (g_conf->daemonize) {
 739       forker.daemonize();
 740       global_init_postfork_start(g_ceph_context);
 741       global_init_postfork_finish(g_ceph_context);
 742     }
 743
 744     {
 745       NBDServer server(fd[1], image);
 746
 747       server.start();
 748
 749       init_async_signal_handler();
 750       register_async_signal_handler(SIGHUP, sighup_handler);
 751       register_async_signal_handler_oneshot(SIGINT, handle_signal);
 752       register_async_signal_handler_oneshot(SIGTERM, handle_signal);
 753
 754       ioctl(nbd, NBD_DO_IT);
 755
 756       unregister_async_signal_handler(SIGHUP, sighup_handler);
 757       unregister_async_signal_handler(SIGINT, handle_signal);
 758       unregister_async_signal_handler(SIGTERM, handle_signal);
 759       shutdown_async_signal_handler();
 760
 761       server.stop();
 762     }
 763
 764     r = image.update_unwatch(handle);
 765     assert(r == 0);
 766   }
 767
 768 close_nbd:
 769   if (r < 0) {
 770     ioctl(nbd, NBD_CLEAR_SOCK);
 771     cerr << "rbd-nbd: failed to map, status: " << cpp_strerror(-r) << std::endl;
 772   }
 773   close(nbd);
 774 close_fd:
 775   close(fd[0]);
 776   close(fd[1]);
 777 close_ret:
 778   image.close();
 779   io_ctx.close();
 780   rados.shutdown();
 781
 782   forker.exit(r < 0 ? EXIT_FAILURE : 0);
 783   // Unreachable;
 784   return r;
 785 }
 786
 787 static int do_unmap(const std::string &devpath)
 788 {
 789   int r = 0;
 790
 791   int nbd = open_device(devpath.c_str());
 792   if (nbd < 0) {
 793     cerr << "rbd-nbd: failed to open device: " << devpath << std::endl;
 794     return nbd;
 795   }
 796
 797   r = ioctl(nbd, NBD_DISCONNECT);
 798   if (r < 0) {
 799       cerr << "rbd-nbd: the device is not used" << std::endl;
 800   }
 801
 802   close(nbd);
 803
 804   return r;
 805 }
 806
 807 static int parse_imgpath(const std::string &imgpath, Config *cfg)
 808 {
 809   boost::regex pattern("^(?:([^/@]+)/)?([^/@]+)(?:@([^/@]+))?$");
 810   boost::smatch match;
 811   if (!boost::regex_match(imgpath, match, pattern)) {
 812     std::cerr << "rbd-nbd: invalid spec '" << imgpath << "'" << std::endl;
 813     return -EINVAL;
 814   }
 815
 816   if (match[1].matched) {
 817     cfg->poolname = match[1];
 818   }
 819
 820   cfg->imgname = match[2];
 821
 822   if (match[3].matched)
 823     cfg->snapname = match[3];
 824
 825   return 0;
 826 }
 827
 828 static int get_mapped_info(int pid, Config *cfg)
 829 {
 830   int r;
 831   std::string path = "/proc/" + stringify(pid) + "/cmdline";
 832   std::ifstream ifs;
 833   std::string cmdline;
 834   std::vector<const char*> args;
 835
 836   ifs.open(path.c_str(), std::ifstream::in);
 837   assert (ifs.is_open());
 838   ifs >> cmdline;
 839
 840   for (unsigned i = 0; i < cmdline.size(); i++) {
 841     const char *arg = &cmdline[i];
 842     if (i == 0) {
 843       if (strcmp(basename(arg) , "rbd-nbd") != 0) {
 844         return -EINVAL;
 845       }
 846     } else {
 847       args.push_back(arg);
 848     }
 849
 850     while (cmdline[i] != '\0') {
 851       i++;
 852     }
 853   }
 854
 855   std::ostringstream err_msg;
 856   r = parse_args(args, &err_msg, cfg);
 857   return r;
 858 }
 859
 860 static int get_map_pid(const std::string& pid_path)
 861 {
 862   int pid = 0;
 863   std::ifstream ifs;
 864   ifs.open(pid_path.c_str(), std::ifstream::in);
 865   if (!ifs.is_open()) {
 866     return 0;
 867   }
 868   ifs >> pid;
 869   return pid;
 870 }
 871
 872 static int do_list_mapped_devices()
 873 {
 874   int r;
 875   bool should_print = false;
 876   int index = 0;
 877   int pid = 0;
 878
 879   std::string default_pool_name;
 880
 881   TextTable tbl;
 882
 883   tbl.define_column("pid", TextTable::LEFT, TextTable::LEFT);
 884   tbl.define_column("pool", TextTable::LEFT, TextTable::LEFT);
 885   tbl.define_column("image", TextTable::LEFT, TextTable::LEFT);
 886   tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT);
 887   tbl.define_column("device", TextTable::LEFT, TextTable::LEFT);
 888
 889   while (true) {
 890     std::string nbd_path = "/sys/block/nbd" + stringify(index);
 891     if(access(nbd_path.c_str(), F_OK) != 0) {
 892       break;
 893     }
 894     std::string pid_path = nbd_path + "/pid";
 895     pid = get_map_pid(pid_path);
 896
 897     if(pid > 0) {
 898       Config cfg;
 899       r = get_mapped_info(pid, &cfg);
 900       if (r < 0) {
 901         index++;
 902         continue;
 903       }
 904       should_print = true;
 905       if (cfg.snapname.empty()) {
 906         cfg.snapname = "-";
 907       }
 908       tbl << pid << cfg.poolname << cfg.imgname << cfg.snapname
 909           << "/dev/nbd" + stringify(index) << TextTable::endrow;
 910     }
 911
 912     index++;
 913   }
 914
 915   if (should_print) {
 916     cout << tbl;
 917   }
 918   return 0;
 919 }
 920
 921 static int parse_args(vector<const char*>& args, std::ostream *err_msg, Config *cfg)
 922 {
 923   std::vector<const char*>::iterator i;
 924   std::ostringstream err;
 925
 926   md_config_t config;
 927   config.parse_config_files(nullptr, nullptr, 0);
 928   config.parse_env();
 929   config.parse_argv(args);
 930   cfg->poolname = config.rbd_default_pool;
 931
 932   for (i = args.begin(); i != args.end(); ) {
 933     if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
 934       return -ENODATA;
 935     } else if (ceph_argparse_witharg(args, i, &cfg->devpath, "--device", (char *)NULL)) {
 936     } else if (ceph_argparse_witharg(args, i, &cfg->nbds_max, err, "--nbds_max", (char *)NULL)) {
 937       if (!err.str().empty()) {
 938         *err_msg << "rbd-nbd: " << err.str();
 939         return -EINVAL;
 940       }
 941       if (cfg->nbds_max < 0) {
 942         *err_msg << "rbd-nbd: Invalid argument for nbds_max!";
 943         return -EINVAL;
 944       }
 945     } else if (ceph_argparse_witharg(args, i, &cfg->max_part, err, "--max_part", (char *)NULL)) {
 946       if (!err.str().empty()) {
 947         *err_msg << "rbd-nbd: " << err.str();
 948         return -EINVAL;
 949       }
 950       if ((cfg->max_part < 0) || (cfg->max_part > 255)) {
 951         *err_msg << "rbd-nbd: Invalid argument for max_part(0~255)!";
 952         return -EINVAL;
 953       }
 954       cfg->set_max_part = true;
 955     } else if (ceph_argparse_flag(args, i, "--read-only", (char *)NULL)) {
 956       cfg->readonly = true;
 957     } else if (ceph_argparse_flag(args, i, "--exclusive", (char *)NULL)) {
 958       cfg->exclusive = true;
 959     } else {
 960       ++i;
 961     }
 962   }
 963
 964   if (args.begin() != args.end()) {
 965     if (strcmp(*args.begin(), "map") == 0) {
 966       cmd = Connect;
 967     } else if (strcmp(*args.begin(), "unmap") == 0) {
 968       cmd = Disconnect;
 969     } else if (strcmp(*args.begin(), "list-mapped") == 0) {
 970       cmd = List;
 971     } else {
 972       *err_msg << "rbd-nbd: unknown command: " <<  *args.begin();
 973       return -EINVAL;
 974     }
 975     args.erase(args.begin());
 976   }
 977
 978   if (cmd == None) {
 979     *err_msg << "rbd-nbd: must specify command";
 980     return -EINVAL;
 981   }
 982
 983   switch (cmd) {
 984     case Connect:
 985       if (args.begin() == args.end()) {
 986         *err_msg << "rbd-nbd: must specify image-or-snap-spec";
 987         return -EINVAL;
 988       }
 989       if (parse_imgpath(string(*args.begin()), cfg) < 0)
 990         return -EINVAL;
 991       args.erase(args.begin());
 992       break;
 993     case Disconnect:
 994       if (args.begin() == args.end()) {
 995         *err_msg << "rbd-nbd: must specify nbd device path";
 996         return -EINVAL;
 997       }
 998       cfg->devpath = *args.begin();
 999       args.erase(args.begin());
1000       break;
1001     default:
1002       //shut up gcc;
1003       break;
1004   }
1005
1006   if (args.begin() != args.end()) {
1007     *err_msg << "rbd-nbd: unknown args: " << *args.begin();
1008     return -EINVAL;
1009   }
1010
1011   return 0;
1012 }
1013
1014 static int rbd_nbd(int argc, const char *argv[])
1015 {
1016   int r;
1017   Config cfg;
1018   vector<const char*> args;
1019   argv_to_vec(argc, argv, args);
1020
1021   std::ostringstream err_msg;
1022   r = parse_args(args, &err_msg, &cfg);
1023   if (r == -ENODATA) {
1024     usage();
1025     return 0;
1026   } else if (r < 0) {
1027     cerr << err_msg.str() << std::endl;
1028     return r;
1029   }
1030
1031   switch (cmd) {
1032     case Connect:
1033       if (cfg.imgname.empty()) {
1034         cerr << "rbd-nbd: image name was not specified" << std::endl;
1035         return -EINVAL;
1036       }
1037
1038       r = do_map(argc, argv, &cfg);
1039       if (r < 0)
1040         return -EINVAL;
1041       break;
1042     case Disconnect:
1043       r = do_unmap(cfg.devpath);
1044       if (r < 0)
1045         return -EINVAL;
1046       break;
1047     case List:
1048       r = do_list_mapped_devices();
1049       if (r < 0)
1050         return -EINVAL;
1051       break;
1052     default:
1053       usage();
1054       return -EINVAL;
1055   }
1056
1057   return 0;
1058 }
1059
1060 int main(int argc, const char *argv[])
1061 {
1062   int r = rbd_nbd(argc, argv);
1063   if (r < 0) {
1064     return EXIT_FAILURE;
1065   }
1066   return 0;
1067 }