1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
5 * rbd-nbd - RBD in userspace
7 * Copyright (C) 2015 - 2016 Kylin Corporation
9 * Author: Yunchuan Wen <yunchuan.wen@kylin-cloud.com>
10 * Li Wang <li.wang@kylin-cloud.com>
12 * This is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU Lesser General Public
14 * License version 2.1, as published by the Free Software
15 * Foundation. See file COPYING.
19 #include "include/int_types.h"
27 #include <sys/types.h>
30 #include <linux/nbd.h>
32 #include <sys/ioctl.h>
33 #include <sys/socket.h>
35 #include "nbd-netlink.h"
36 #include <libnl3/netlink/genl/genl.h>
37 #include <libnl3/netlink/genl/ctrl.h>
38 #include <libnl3/netlink/genl/mngt.h>
44 #include <boost/algorithm/string/predicate.hpp>
46 #include "common/Formatter.h"
47 #include "common/Preforker.h"
48 #include "common/TextTable.h"
49 #include "common/ceph_argparse.h"
50 #include "common/config.h"
51 #include "common/dout.h"
52 #include "common/errno.h"
53 #include "common/module.h"
54 #include "common/safe_io.h"
55 #include "common/version.h"
57 #include "global/global_init.h"
58 #include "global/signal_handler.h"
60 #include "include/rados/librados.hpp"
61 #include "include/rbd/librbd.hpp"
62 #include "include/stringify.h"
63 #include "include/xlist.h"
65 #include "mon/MonClient.h"
67 #define dout_context g_ceph_context
68 #define dout_subsys ceph_subsys_rbd
70 #define dout_prefix *_dout << "rbd-nbd: "
77 bool exclusive
= false;
78 bool readonly
= false;
79 bool set_max_part
= false;
80 bool try_netlink
= false;
89 bool pretty_format
= false;
94 std::cout
<< "Usage: rbd-nbd [options] map <image-or-snap-spec> Map an image to nbd device\n"
95 << " unmap <device|image-or-snap-spec> Unmap nbd device\n"
96 << " [options] list-mapped List mapped nbd devices\n"
98 << " --device <device path> Specify nbd device path (/dev/nbd{num})\n"
99 << " --read-only Map read-only\n"
100 << " --nbds_max <limit> Override for module param nbds_max\n"
101 << " --max_part <limit> Override for module param max_part\n"
102 << " --exclusive Forbid writes by other clients\n"
103 << " --timeout <seconds> Set nbd request timeout\n"
104 << " --try-netlink Use the nbd netlink interface\n"
107 << " --format plain|json|xml Output format (default: plain)\n"
108 << " --pretty-format Pretty formatting (json and xml)\n"
110 generic_server_usage();
114 static int nbd_index
= -1;
123 static Command cmd
= None
;
125 #define RBD_NBD_BLKSIZE 512UL
128 #define VERSION_INFO 2
130 #ifdef CEPH_BIG_ENDIAN
131 #define ntohll(a) (a)
132 #elif defined(CEPH_LITTLE_ENDIAN)
133 #define ntohll(a) swab(a)
135 #error "Could not determine endianess"
137 #define htonll(a) ntohll(a)
139 static int parse_args(vector
<const char*>& args
, std::ostream
*err_msg
,
140 Command
*command
, Config
*cfg
);
141 static int netlink_resize(int nbd_index
, uint64_t size
);
147 librbd::Image
&image
;
150 NBDServer(int _fd
, librbd::Image
& _image
)
153 , disconnect_lock("NBDServer::DisconnectLocker")
154 , lock("NBDServer::Locker")
155 , reader_thread(*this, &NBDServer::reader_entry
)
156 , writer_thread(*this, &NBDServer::writer_entry
)
161 Mutex disconnect_lock
;
162 Cond disconnect_cond
;
163 std::atomic
<bool> terminated
= { false };
167 bool expected
= false;
168 if (terminated
.compare_exchange_strong(expected
, true)) {
169 ::shutdown(fd
, SHUT_RDWR
);
171 Mutex::Locker
l(lock
);
178 xlist
<IOContext
*>::item item
;
179 NBDServer
*server
= nullptr;
180 struct nbd_request request
;
181 struct nbd_reply reply
;
190 friend std::ostream
&operator<<(std::ostream
&os
, const IOContext
&ctx
);
194 xlist
<IOContext
*> io_pending
;
195 xlist
<IOContext
*> io_finished
;
197 void io_start(IOContext
*ctx
)
199 Mutex::Locker
l(lock
);
200 io_pending
.push_back(&ctx
->item
);
203 void io_finish(IOContext
*ctx
)
205 Mutex::Locker
l(lock
);
206 ceph_assert(ctx
->item
.is_on_list());
207 ctx
->item
.remove_myself();
208 io_finished
.push_back(&ctx
->item
);
212 IOContext
*wait_io_finish()
214 Mutex::Locker
l(lock
);
215 while(io_finished
.empty() && !terminated
)
218 if (io_finished
.empty())
221 IOContext
*ret
= io_finished
.front();
222 io_finished
.pop_front();
229 ceph_assert(!reader_thread
.is_started());
230 Mutex::Locker
l(lock
);
231 while(!io_pending
.empty())
234 while(!io_finished
.empty()) {
235 std::unique_ptr
<IOContext
> free_ctx(io_finished
.front());
236 io_finished
.pop_front();
240 static void aio_callback(librbd::completion_t cb
, void *arg
)
242 librbd::RBD::AioCompletion
*aio_completion
=
243 reinterpret_cast<librbd::RBD::AioCompletion
*>(cb
);
245 IOContext
*ctx
= reinterpret_cast<IOContext
*>(arg
);
246 int ret
= aio_completion
->get_return_value();
248 dout(20) << __func__
<< ": " << *ctx
<< dendl
;
250 if (ret
== -EINVAL
) {
251 // if shrinking an image, a pagecache writeback might reference
252 // extents outside of the range of the new image extents
253 dout(0) << __func__
<< ": masking IO out-of-bounds error" << dendl
;
259 ctx
->reply
.error
= htonl(-ret
);
260 } else if ((ctx
->command
== NBD_CMD_READ
) &&
261 ret
< static_cast<int>(ctx
->request
.len
)) {
262 int pad_byte_count
= static_cast<int> (ctx
->request
.len
) - ret
;
263 ctx
->data
.append_zero(pad_byte_count
);
264 dout(20) << __func__
<< ": " << *ctx
<< ": Pad byte count: "
265 << pad_byte_count
<< dendl
;
266 ctx
->reply
.error
= htonl(0);
268 ctx
->reply
.error
= htonl(0);
270 ctx
->server
->io_finish(ctx
);
272 aio_completion
->release();
277 while (!terminated
) {
278 std::unique_ptr
<IOContext
> ctx(new IOContext());
281 dout(20) << __func__
<< ": waiting for nbd request" << dendl
;
283 int r
= safe_read_exact(fd
, &ctx
->request
, sizeof(struct nbd_request
));
285 derr
<< "failed to read nbd request header: " << cpp_strerror(r
)
290 if (ctx
->request
.magic
!= htonl(NBD_REQUEST_MAGIC
)) {
291 derr
<< "invalid nbd request header" << dendl
;
295 ctx
->request
.from
= ntohll(ctx
->request
.from
);
296 ctx
->request
.type
= ntohl(ctx
->request
.type
);
297 ctx
->request
.len
= ntohl(ctx
->request
.len
);
299 ctx
->reply
.magic
= htonl(NBD_REPLY_MAGIC
);
300 memcpy(ctx
->reply
.handle
, ctx
->request
.handle
, sizeof(ctx
->reply
.handle
));
302 ctx
->command
= ctx
->request
.type
& 0x0000ffff;
304 dout(20) << *ctx
<< ": start" << dendl
;
306 switch (ctx
->command
)
309 // NBD_DO_IT will return when pipe is closed
310 dout(0) << "disconnect request received" << dendl
;
313 bufferptr
ptr(ctx
->request
.len
);
314 r
= safe_read_exact(fd
, ptr
.c_str(), ctx
->request
.len
);
316 derr
<< *ctx
<< ": failed to read nbd request data: "
317 << cpp_strerror(r
) << dendl
;
320 ctx
->data
.push_back(ptr
);
324 IOContext
*pctx
= ctx
.release();
326 librbd::RBD::AioCompletion
*c
= new librbd::RBD::AioCompletion(pctx
, aio_callback
);
327 switch (pctx
->command
)
330 image
.aio_write(pctx
->request
.from
, pctx
->request
.len
, pctx
->data
, c
);
333 image
.aio_read(pctx
->request
.from
, pctx
->request
.len
, pctx
->data
, c
);
339 image
.aio_discard(pctx
->request
.from
, pctx
->request
.len
, c
);
342 derr
<< *pctx
<< ": invalid request command" << dendl
;
347 dout(20) << __func__
<< ": terminated" << dendl
;
350 Mutex::Locker
l(disconnect_lock
);
351 disconnect_cond
.Signal();
356 while (!terminated
) {
357 dout(20) << __func__
<< ": waiting for io request" << dendl
;
358 std::unique_ptr
<IOContext
> ctx(wait_io_finish());
360 dout(20) << __func__
<< ": no io requests, terminating" << dendl
;
364 dout(20) << __func__
<< ": got: " << *ctx
<< dendl
;
366 int r
= safe_write(fd
, &ctx
->reply
, sizeof(struct nbd_reply
));
368 derr
<< *ctx
<< ": failed to write reply header: " << cpp_strerror(r
)
372 if (ctx
->command
== NBD_CMD_READ
&& ctx
->reply
.error
== htonl(0)) {
373 r
= ctx
->data
.write_fd(fd
);
375 derr
<< *ctx
<< ": failed to write replay data: " << cpp_strerror(r
)
380 dout(20) << *ctx
<< ": finish" << dendl
;
382 dout(20) << __func__
<< ": terminated" << dendl
;
385 class ThreadHelper
: public Thread
388 typedef void (NBDServer::*entry_func
)();
393 ThreadHelper(NBDServer
&_server
, entry_func _func
)
398 void* entry() override
404 } reader_thread
, writer_thread
;
411 dout(10) << __func__
<< ": starting" << dendl
;
415 reader_thread
.create("rbd_reader");
416 writer_thread
.create("rbd_writer");
420 void wait_for_disconnect()
425 Mutex::Locker
l(disconnect_lock
);
426 disconnect_cond
.Wait(disconnect_lock
);
432 dout(10) << __func__
<< ": terminating" << dendl
;
436 reader_thread
.join();
437 writer_thread
.join();
446 std::ostream
&operator<<(std::ostream
&os
, const NBDServer::IOContext
&ctx
) {
448 os
<< "[" << std::hex
<< ntohll(*((uint64_t *)ctx
.request
.handle
));
465 os
<< " UNKNOWN(" << ctx
.command
<< ") ";
469 os
<< ctx
.request
.from
<< "~" << ctx
.request
.len
<< " "
470 << std::dec
<< ntohl(ctx
.reply
.error
) << "]";
475 class NBDWatchCtx
: public librbd::UpdateWatchCtx
481 librados::IoCtx
&io_ctx
;
482 librbd::Image
&image
;
488 librados::IoCtx
&_io_ctx
,
489 librbd::Image
&_image
,
492 , nbd_index(_nbd_index
)
493 , use_netlink(_use_netlink
)
499 ~NBDWatchCtx() override
{}
501 void handle_notify() override
503 librbd::image_info_t info
;
504 if (image
.stat(info
, sizeof(info
)) == 0) {
505 unsigned long new_size
= info
.size
;
508 if (new_size
!= size
) {
509 dout(5) << "resize detected" << dendl
;
510 if (ioctl(fd
, BLKFLSBUF
, NULL
) < 0)
511 derr
<< "invalidate page cache failed: " << cpp_strerror(errno
)
514 ret
= netlink_resize(nbd_index
, new_size
);
516 ret
= ioctl(fd
, NBD_SET_SIZE
, new_size
);
518 derr
<< "resize failed: " << cpp_strerror(errno
) << dendl
;
524 if (ioctl(fd
, BLKRRPART
, NULL
) < 0) {
525 derr
<< "rescan of partition table failed: " << cpp_strerror(errno
)
528 if (image
.invalidate_cache() < 0)
529 derr
<< "invalidate rbd cache failed" << dendl
;
535 class NBDListIterator
{
537 bool get(int *pid
, Config
*cfg
) {
539 std::string nbd_path
= "/sys/block/nbd" + stringify(m_index
);
540 if(access(nbd_path
.c_str(), F_OK
) != 0) {
545 cfg
->devpath
= "/dev/nbd" + stringify(m_index
++);
548 ifs
.open(nbd_path
+ "/pid", std::ifstream::in
);
549 if (!ifs
.is_open()) {
554 int r
= get_mapped_info(*pid
, cfg
);
566 int get_mapped_info(int pid
, Config
*cfg
) {
568 std::string path
= "/proc/" + stringify(pid
) + "/cmdline";
571 std::vector
<const char*> args
;
573 ifs
.open(path
.c_str(), std::ifstream::in
);
578 for (unsigned i
= 0; i
< cmdline
.size(); i
++) {
579 const char *arg
= &cmdline
[i
];
581 if (strcmp(basename(arg
) , "rbd-nbd") != 0) {
588 while (cmdline
[i
] != '\0') {
593 std::ostringstream err_msg
;
595 r
= parse_args(args
, &err_msg
, &command
, cfg
);
600 if (command
!= Connect
) {
608 static int load_module(Config
*cfg
)
614 param
<< "nbds_max=" << cfg
->nbds_max
;
617 param
<< " max_part=" << cfg
->max_part
;
619 if (!access("/sys/module/nbd", F_OK
)) {
620 if (cfg
->nbds_max
|| cfg
->set_max_part
)
621 cerr
<< "rbd-nbd: ignoring kernel module parameter options: nbd module already loaded"
626 ret
= module_load("nbd", param
.str().c_str());
628 cerr
<< "rbd-nbd: failed to load nbd kernel module: " << cpp_strerror(-ret
)
634 static int check_device_size(int nbd_index
, unsigned long expected_size
)
636 // There are bugs with some older kernel versions that result in an
637 // overflow for large image sizes. This check is to ensure we are
640 unsigned long size
= 0;
641 std::string path
= "/sys/block/nbd" + stringify(nbd_index
) + "/size";
643 ifs
.open(path
.c_str(), std::ifstream::in
);
644 if (!ifs
.is_open()) {
645 cerr
<< "rbd-nbd: failed to open " << path
<< std::endl
;
649 size
*= RBD_NBD_BLKSIZE
;
652 // Newer kernel versions will report real size only after nbd
653 // connect. Assume this is the case and return success.
657 if (size
!= expected_size
) {
658 cerr
<< "rbd-nbd: kernel reported invalid device size (" << size
659 << ", expected " << expected_size
<< ")" << std::endl
;
666 static int parse_nbd_index(const std::string
& devpath
)
670 ret
= sscanf(devpath
.c_str(), "/dev/nbd%d", &index
);
672 // mean an early matching failure. But some cases need a negative value.
675 cerr
<< "rbd-nbd: invalid device path: " << devpath
676 << " (expected /dev/nbd{num})" << std::endl
;
683 static int try_ioctl_setup(Config
*cfg
, int fd
, uint64_t size
, uint64_t flags
)
687 if (cfg
->devpath
.empty()) {
689 const char *path
= "/sys/module/nbd/parameters/nbds_max";
691 if (access(path
, F_OK
) == 0) {
693 ifs
.open(path
, std::ifstream::in
);
701 snprintf(dev
, sizeof(dev
), "/dev/nbd%d", index
);
703 nbd
= open(dev
, O_RDWR
);
705 if (nbd
== -EPERM
&& nbds_max
!= -1 && index
< (nbds_max
-1)) {
710 cerr
<< "rbd-nbd: failed to find unused device" << std::endl
;
714 r
= ioctl(nbd
, NBD_SET_SOCK
, fd
);
725 r
= parse_nbd_index(cfg
->devpath
);
730 nbd
= open(cfg
->devpath
.c_str(), O_RDWR
);
733 cerr
<< "rbd-nbd: failed to open device: " << cfg
->devpath
<< std::endl
;
737 r
= ioctl(nbd
, NBD_SET_SOCK
, fd
);
740 cerr
<< "rbd-nbd: the device " << cfg
->devpath
<< " is busy" << std::endl
;
746 r
= ioctl(nbd
, NBD_SET_BLKSIZE
, RBD_NBD_BLKSIZE
);
752 r
= ioctl(nbd
, NBD_SET_SIZE
, size
);
758 ioctl(nbd
, NBD_SET_FLAGS
, flags
);
760 if (cfg
->timeout
>= 0) {
761 r
= ioctl(nbd
, NBD_SET_TIMEOUT
, (unsigned long)cfg
->timeout
);
764 cerr
<< "rbd-nbd: failed to set timeout: " << cpp_strerror(r
)
770 dout(10) << "ioctl setup complete for " << cfg
->devpath
<< dendl
;
776 ioctl(nbd
, NBD_CLEAR_SOCK
);
777 cerr
<< "rbd-nbd: failed to map, status: " << cpp_strerror(-r
) << std::endl
;
784 static void netlink_cleanup(struct nl_sock
*sock
)
790 nl_socket_free(sock
);
793 static struct nl_sock
*netlink_init(int *id
)
795 struct nl_sock
*sock
;
798 sock
= nl_socket_alloc();
800 cerr
<< "rbd-nbd: Could not allocate netlink socket." << std::endl
;
804 ret
= genl_connect(sock
);
806 cerr
<< "rbd-nbd: Could not connect netlink socket. Error " << ret
811 *id
= genl_ctrl_resolve(sock
, "nbd");
813 // nbd netlink interface not supported.
821 nl_socket_free(sock
);
825 static int netlink_disconnect(int index
)
827 struct nl_sock
*sock
;
831 sock
= netlink_init(&nl_id
);
836 nl_socket_modify_cb(sock
, NL_CB_VALID
, NL_CB_CUSTOM
, genl_handle_msg
, NULL
);
840 cerr
<< "rbd-nbd: Could not allocate netlink message." << std::endl
;
844 if (!genlmsg_put(msg
, NL_AUTO_PORT
, NL_AUTO_SEQ
, nl_id
, 0, 0,
845 NBD_CMD_DISCONNECT
, 0)) {
846 cerr
<< "rbd-nbd: Could not setup message." << std::endl
;
847 goto nla_put_failure
;
850 NLA_PUT_U32(msg
, NBD_ATTR_INDEX
, index
);
852 ret
= nl_send_sync(sock
, msg
);
853 netlink_cleanup(sock
);
855 cerr
<< "rbd-nbd: netlink disconnect failed: " << nl_geterror(-ret
)
865 netlink_cleanup(sock
);
869 static int netlink_disconnect_by_path(const std::string
& devpath
)
873 index
= parse_nbd_index(devpath
);
877 return netlink_disconnect(index
);
880 static int netlink_resize(int nbd_index
, uint64_t size
)
882 struct nl_sock
*sock
;
886 sock
= netlink_init(&nl_id
);
888 cerr
<< "rbd-nbd: Netlink interface not supported." << std::endl
;
892 nl_socket_modify_cb(sock
, NL_CB_VALID
, NL_CB_CUSTOM
, genl_handle_msg
, NULL
);
896 cerr
<< "rbd-nbd: Could not allocate netlink message." << std::endl
;
900 if (!genlmsg_put(msg
, NL_AUTO_PORT
, NL_AUTO_SEQ
, nl_id
, 0, 0,
901 NBD_CMD_RECONFIGURE
, 0)) {
902 cerr
<< "rbd-nbd: Could not setup message." << std::endl
;
906 NLA_PUT_U32(msg
, NBD_ATTR_INDEX
, nbd_index
);
907 NLA_PUT_U64(msg
, NBD_ATTR_SIZE_BYTES
, size
);
909 ret
= nl_send_sync(sock
, msg
);
911 cerr
<< "rbd-nbd: netlink resize failed: " << nl_geterror(ret
) << std::endl
;
915 netlink_cleanup(sock
);
916 dout(10) << "netlink resize complete for nbd" << nbd_index
<< dendl
;
923 netlink_cleanup(sock
);
927 static int netlink_connect_cb(struct nl_msg
*msg
, void *arg
)
929 struct genlmsghdr
*gnlh
= (struct genlmsghdr
*)nlmsg_data(nlmsg_hdr(msg
));
930 Config
*cfg
= (Config
*)arg
;
931 struct nlattr
*msg_attr
[NBD_ATTR_MAX
+ 1];
935 ret
= nla_parse(msg_attr
, NBD_ATTR_MAX
, genlmsg_attrdata(gnlh
, 0),
936 genlmsg_attrlen(gnlh
, 0), NULL
);
938 cerr
<< "rbd-nbd: Unsupported netlink reply" << std::endl
;
939 return -NLE_MSGTYPE_NOSUPPORT
;
942 if (!msg_attr
[NBD_ATTR_INDEX
]) {
943 cerr
<< "rbd-nbd: netlink connect reply missing device index." << std::endl
;
944 return -NLE_MSGTYPE_NOSUPPORT
;
947 index
= nla_get_u32(msg_attr
[NBD_ATTR_INDEX
]);
948 cfg
->devpath
= "/dev/nbd" + stringify(index
);
954 static int netlink_connect(Config
*cfg
, struct nl_sock
*sock
, int nl_id
, int fd
,
955 uint64_t size
, uint64_t flags
)
957 struct nlattr
*sock_attr
;
958 struct nlattr
*sock_opt
;
962 nl_socket_modify_cb(sock
, NL_CB_VALID
, NL_CB_CUSTOM
, netlink_connect_cb
, cfg
);
966 cerr
<< "rbd-nbd: Could not allocate netlink message." << std::endl
;
970 if (!genlmsg_put(msg
, NL_AUTO_PORT
, NL_AUTO_SEQ
, nl_id
, 0, 0, NBD_CMD_CONNECT
,
972 cerr
<< "rbd-nbd: Could not setup message." << std::endl
;
976 if (!cfg
->devpath
.empty()) {
977 ret
= parse_nbd_index(cfg
->devpath
);
981 NLA_PUT_U32(msg
, NBD_ATTR_INDEX
, ret
);
984 if (cfg
->timeout
>= 0)
985 NLA_PUT_U64(msg
, NBD_ATTR_TIMEOUT
, cfg
->timeout
);
987 NLA_PUT_U64(msg
, NBD_ATTR_SIZE_BYTES
, size
);
988 NLA_PUT_U64(msg
, NBD_ATTR_BLOCK_SIZE_BYTES
, RBD_NBD_BLKSIZE
);
989 NLA_PUT_U64(msg
, NBD_ATTR_SERVER_FLAGS
, flags
);
991 sock_attr
= nla_nest_start(msg
, NBD_ATTR_SOCKETS
);
993 cerr
<< "rbd-nbd: Could not init sockets in netlink message." << std::endl
;
997 sock_opt
= nla_nest_start(msg
, NBD_SOCK_ITEM
);
999 cerr
<< "rbd-nbd: Could not init sock in netlink message." << std::endl
;
1003 NLA_PUT_U32(msg
, NBD_SOCK_FD
, fd
);
1004 nla_nest_end(msg
, sock_opt
);
1005 nla_nest_end(msg
, sock_attr
);
1007 ret
= nl_send_sync(sock
, msg
);
1009 cerr
<< "rbd-nbd: netlink connect failed: " << nl_geterror(ret
)
1014 dout(10) << "netlink connect complete for " << cfg
->devpath
<< dendl
;
1023 static int try_netlink_setup(Config
*cfg
, int fd
, uint64_t size
, uint64_t flags
)
1025 struct nl_sock
*sock
;
1028 sock
= netlink_init(&nl_id
);
1030 cerr
<< "rbd-nbd: Netlink interface not supported. Using ioctl interface."
1035 dout(10) << "netlink interface supported." << dendl
;
1037 ret
= netlink_connect(cfg
, sock
, nl_id
, fd
, size
, flags
);
1038 netlink_cleanup(sock
);
1043 nbd
= open(cfg
->devpath
.c_str(), O_RDWR
);
1045 cerr
<< "rbd-nbd: failed to open device: " << cfg
->devpath
<< std::endl
;
1052 static void handle_signal(int signum
)
1056 ceph_assert(signum
== SIGINT
|| signum
== SIGTERM
);
1057 derr
<< "*** Got signal " << sig_str(signum
) << " ***" << dendl
;
1059 if (nbd
< 0 || nbd_index
< 0) {
1060 dout(20) << __func__
<< ": " << "disconnect not needed." << dendl
;
1064 dout(20) << __func__
<< ": " << "sending NBD_DISCONNECT" << dendl
;
1065 ret
= netlink_disconnect(nbd_index
);
1067 ret
= ioctl(nbd
, NBD_DISCONNECT
);
1070 derr
<< "rbd-nbd: disconnect failed. Error: " << ret
<< dendl
;
1072 dout(20) << __func__
<< ": " << "disconnected" << dendl
;
1076 static NBDServer
*start_server(int fd
, librbd::Image
& image
)
1080 server
= new NBDServer(fd
, image
);
1083 init_async_signal_handler();
1084 register_async_signal_handler(SIGHUP
, sighup_handler
);
1085 register_async_signal_handler_oneshot(SIGINT
, handle_signal
);
1086 register_async_signal_handler_oneshot(SIGTERM
, handle_signal
);
1091 static void run_server(Preforker
& forker
, NBDServer
*server
, bool netlink_used
)
1093 if (g_conf()->daemonize
) {
1094 global_init_postfork_finish(g_ceph_context
);
1099 server
->wait_for_disconnect();
1101 ioctl(nbd
, NBD_DO_IT
);
1103 unregister_async_signal_handler(SIGHUP
, sighup_handler
);
1104 unregister_async_signal_handler(SIGINT
, handle_signal
);
1105 unregister_async_signal_handler(SIGTERM
, handle_signal
);
1106 shutdown_async_signal_handler();
1109 static int do_map(int argc
, const char *argv
[], Config
*cfg
)
1113 librados::Rados rados
;
1115 librados::IoCtx io_ctx
;
1116 librbd::Image image
;
1119 unsigned long flags
;
1125 librbd::image_info_t info
;
1130 vector
<const char*> args
;
1131 argv_to_vec(argc
, argv
, args
);
1133 cerr
<< argv
[0] << ": -h or --help for usage" << std::endl
;
1136 if (ceph_argparse_need_usage(args
)) {
1141 auto cct
= global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
1142 CODE_ENVIRONMENT_DAEMON
,
1143 CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS
);
1144 g_ceph_context
->_conf
.set_val_or_die("pid_file", "");
1146 if (global_init_prefork(g_ceph_context
) >= 0) {
1148 r
= forker
.prefork(err
);
1150 cerr
<< err
<< std::endl
;
1153 if (forker
.is_parent()) {
1154 if (forker
.parent_wait(err
) != 0) {
1159 global_init_postfork_start(g_ceph_context
);
1162 common_init_finish(g_ceph_context
);
1163 global_init_chdir(g_ceph_context
);
1165 if (socketpair(AF_UNIX
, SOCK_STREAM
, 0, fd
) == -1) {
1170 r
= rados
.init_with_context(g_ceph_context
);
1174 r
= rados
.connect();
1178 r
= rados
.ioctx_create(cfg
->poolname
.c_str(), io_ctx
);
1182 io_ctx
.set_namespace(cfg
->nsname
);
1184 r
= rbd
.open(io_ctx
, image
, cfg
->imgname
.c_str());
1188 if (cfg
->exclusive
) {
1189 r
= image
.lock_acquire(RBD_LOCK_MODE_EXCLUSIVE
);
1191 cerr
<< "rbd-nbd: failed to acquire exclusive lock: " << cpp_strerror(r
)
1197 if (!cfg
->snapname
.empty()) {
1198 r
= image
.snap_set(cfg
->snapname
.c_str());
1203 r
= image
.stat(info
, sizeof(info
));
1207 flags
= NBD_FLAG_SEND_FLUSH
| NBD_FLAG_SEND_TRIM
| NBD_FLAG_HAS_FLAGS
;
1208 if (!cfg
->snapname
.empty() || cfg
->readonly
) {
1209 flags
|= NBD_FLAG_READ_ONLY
;
1213 if (info
.size
> ULONG_MAX
) {
1215 cerr
<< "rbd-nbd: image is too large (" << byte_u_t(info
.size
)
1216 << ", max is " << byte_u_t(ULONG_MAX
) << ")" << std::endl
;
1222 r
= load_module(cfg
);
1226 server
= start_server(fd
[1], image
);
1228 use_netlink
= cfg
->try_netlink
;
1230 r
= try_netlink_setup(cfg
, fd
[0], size
, flags
);
1233 } else if (r
== 1) {
1234 use_netlink
= false;
1239 r
= try_ioctl_setup(cfg
, fd
[0], size
, flags
);
1244 r
= check_device_size(nbd_index
, size
);
1248 r
= ioctl(nbd
, BLKROSET
, (unsigned long) &read_only
);
1257 NBDWatchCtx
watch_ctx(nbd
, nbd_index
, use_netlink
, io_ctx
, image
,
1259 r
= image
.update_watch(&watch_ctx
, &handle
);
1263 cout
<< cfg
->devpath
<< std::endl
;
1265 run_server(forker
, server
, use_netlink
);
1267 r
= image
.update_unwatch(handle
);
1268 ceph_assert(r
== 0);
1274 netlink_disconnect(nbd_index
);
1276 ioctl(nbd
, NBD_CLEAR_SOCK
);
1277 cerr
<< "rbd-nbd: failed to map, status: " << cpp_strerror(-r
)
1292 forker
.exit(r
< 0 ? EXIT_FAILURE
: 0);
1297 static int do_unmap(Config
*cfg
)
1302 * The netlink disconnect call supports devices setup with netlink or ioctl,
1303 * so we always try that first.
1305 r
= netlink_disconnect_by_path(cfg
->devpath
);
1309 nbd
= open(cfg
->devpath
.c_str(), O_RDWR
);
1311 cerr
<< "rbd-nbd: failed to open device: " << cfg
->devpath
<< std::endl
;
1315 r
= ioctl(nbd
, NBD_DISCONNECT
);
1317 cerr
<< "rbd-nbd: the device is not used" << std::endl
;
1324 static int parse_imgpath(const std::string
&imgpath
, Config
*cfg
,
1325 std::ostream
*err_msg
) {
1326 std::regex
pattern("^(?:([^/]+)/(?:([^/@]+)/)?)?([^@]+)(?:@([^/@]+))?$");
1328 if (!std::regex_match(imgpath
, match
, pattern
)) {
1329 std::cerr
<< "rbd-nbd: invalid spec '" << imgpath
<< "'" << std::endl
;
1333 if (match
[1].matched
) {
1334 cfg
->poolname
= match
[1];
1337 if (match
[2].matched
) {
1338 cfg
->nsname
= match
[2];
1341 cfg
->imgname
= match
[3];
1343 if (match
[4].matched
)
1344 cfg
->snapname
= match
[4];
1349 static int do_list_mapped_devices(const std::string
&format
, bool pretty_format
)
1351 bool should_print
= false;
1352 std::unique_ptr
<ceph::Formatter
> f
;
1355 if (format
== "json") {
1356 f
.reset(new JSONFormatter(pretty_format
));
1357 } else if (format
== "xml") {
1358 f
.reset(new XMLFormatter(pretty_format
));
1359 } else if (!format
.empty() && format
!= "plain") {
1360 std::cerr
<< "rbd-nbd: invalid output format: " << format
<< std::endl
;
1365 f
->open_array_section("devices");
1367 tbl
.define_column("id", TextTable::LEFT
, TextTable::LEFT
);
1368 tbl
.define_column("pool", TextTable::LEFT
, TextTable::LEFT
);
1369 tbl
.define_column("namespace", TextTable::LEFT
, TextTable::LEFT
);
1370 tbl
.define_column("image", TextTable::LEFT
, TextTable::LEFT
);
1371 tbl
.define_column("snap", TextTable::LEFT
, TextTable::LEFT
);
1372 tbl
.define_column("device", TextTable::LEFT
, TextTable::LEFT
);
1378 while (it
.get(&pid
, &cfg
)) {
1380 f
->open_object_section("device");
1381 f
->dump_int("id", pid
);
1382 f
->dump_string("pool", cfg
.poolname
);
1383 f
->dump_string("namespace", cfg
.nsname
);
1384 f
->dump_string("image", cfg
.imgname
);
1385 f
->dump_string("snap", cfg
.snapname
);
1386 f
->dump_string("device", cfg
.devpath
);
1389 should_print
= true;
1390 if (cfg
.snapname
.empty()) {
1393 tbl
<< pid
<< cfg
.poolname
<< cfg
.nsname
<< cfg
.imgname
<< cfg
.snapname
1394 << cfg
.devpath
<< TextTable::endrow
;
1399 f
->close_section(); // devices
1400 f
->flush(std::cout
);
1408 static bool find_mapped_dev_by_spec(Config
*cfg
) {
1412 while (it
.get(&pid
, &c
)) {
1413 if (c
.poolname
== cfg
->poolname
&& c
.imgname
== cfg
->imgname
&&
1414 c
.snapname
== cfg
->snapname
) {
1423 static int parse_args(vector
<const char*>& args
, std::ostream
*err_msg
,
1424 Command
*command
, Config
*cfg
) {
1425 std::string conf_file_list
;
1426 std::string cluster
;
1427 CephInitParameters iparams
= ceph_argparse_early_args(
1428 args
, CEPH_ENTITY_TYPE_CLIENT
, &cluster
, &conf_file_list
);
1430 ConfigProxy config
{false};
1431 config
->name
= iparams
.name
;
1432 config
->cluster
= cluster
;
1434 if (!conf_file_list
.empty()) {
1435 config
.parse_config_files(conf_file_list
.c_str(), nullptr, 0);
1437 config
.parse_config_files(nullptr, nullptr, 0);
1439 config
.parse_env(CEPH_ENTITY_TYPE_CLIENT
);
1440 config
.parse_argv(args
);
1441 cfg
->poolname
= config
.get_val
<std::string
>("rbd_default_pool");
1443 std::vector
<const char*>::iterator i
;
1444 std::ostringstream err
;
1446 for (i
= args
.begin(); i
!= args
.end(); ) {
1447 if (ceph_argparse_flag(args
, i
, "-h", "--help", (char*)NULL
)) {
1449 } else if (ceph_argparse_flag(args
, i
, "-v", "--version", (char*)NULL
)) {
1450 return VERSION_INFO
;
1451 } else if (ceph_argparse_witharg(args
, i
, &cfg
->devpath
, "--device", (char *)NULL
)) {
1452 } else if (ceph_argparse_witharg(args
, i
, &cfg
->nbds_max
, err
, "--nbds_max", (char *)NULL
)) {
1453 if (!err
.str().empty()) {
1454 *err_msg
<< "rbd-nbd: " << err
.str();
1457 if (cfg
->nbds_max
< 0) {
1458 *err_msg
<< "rbd-nbd: Invalid argument for nbds_max!";
1461 } else if (ceph_argparse_witharg(args
, i
, &cfg
->max_part
, err
, "--max_part", (char *)NULL
)) {
1462 if (!err
.str().empty()) {
1463 *err_msg
<< "rbd-nbd: " << err
.str();
1466 if ((cfg
->max_part
< 0) || (cfg
->max_part
> 255)) {
1467 *err_msg
<< "rbd-nbd: Invalid argument for max_part(0~255)!";
1470 cfg
->set_max_part
= true;
1471 } else if (ceph_argparse_flag(args
, i
, "--read-only", (char *)NULL
)) {
1472 cfg
->readonly
= true;
1473 } else if (ceph_argparse_flag(args
, i
, "--exclusive", (char *)NULL
)) {
1474 cfg
->exclusive
= true;
1475 } else if (ceph_argparse_witharg(args
, i
, &cfg
->timeout
, err
, "--timeout",
1477 if (!err
.str().empty()) {
1478 *err_msg
<< "rbd-nbd: " << err
.str();
1481 if (cfg
->timeout
< 0) {
1482 *err_msg
<< "rbd-nbd: Invalid argument for timeout!";
1485 } else if (ceph_argparse_witharg(args
, i
, &cfg
->format
, err
, "--format",
1487 } else if (ceph_argparse_flag(args
, i
, "--pretty-format", (char *)NULL
)) {
1488 cfg
->pretty_format
= true;
1489 } else if (ceph_argparse_flag(args
, i
, "--try-netlink", (char *)NULL
)) {
1490 cfg
->try_netlink
= true;
1497 if (args
.begin() != args
.end()) {
1498 if (strcmp(*args
.begin(), "map") == 0) {
1500 } else if (strcmp(*args
.begin(), "unmap") == 0) {
1502 } else if (strcmp(*args
.begin(), "list-mapped") == 0) {
1505 *err_msg
<< "rbd-nbd: unknown command: " << *args
.begin();
1508 args
.erase(args
.begin());
1512 *err_msg
<< "rbd-nbd: must specify command";
1518 if (args
.begin() == args
.end()) {
1519 *err_msg
<< "rbd-nbd: must specify image-or-snap-spec";
1522 if (parse_imgpath(*args
.begin(), cfg
, err_msg
) < 0) {
1525 args
.erase(args
.begin());
1528 if (args
.begin() == args
.end()) {
1529 *err_msg
<< "rbd-nbd: must specify nbd device or image-or-snap-spec";
1532 if (boost::starts_with(*args
.begin(), "/dev/")) {
1533 cfg
->devpath
= *args
.begin();
1535 if (parse_imgpath(*args
.begin(), cfg
, err_msg
) < 0) {
1538 if (!find_mapped_dev_by_spec(cfg
)) {
1539 *err_msg
<< "rbd-nbd: " << *args
.begin() << " is not mapped";
1543 args
.erase(args
.begin());
1550 if (args
.begin() != args
.end()) {
1551 *err_msg
<< "rbd-nbd: unknown args: " << *args
.begin();
1559 static int rbd_nbd(int argc
, const char *argv
[])
1563 vector
<const char*> args
;
1564 argv_to_vec(argc
, argv
, args
);
1566 std::ostringstream err_msg
;
1567 r
= parse_args(args
, &err_msg
, &cmd
, &cfg
);
1568 if (r
== HELP_INFO
) {
1571 } else if (r
== VERSION_INFO
) {
1572 std::cout
<< pretty_version_to_str() << std::endl
;
1575 cerr
<< err_msg
.str() << std::endl
;
1581 if (cfg
.imgname
.empty()) {
1582 cerr
<< "rbd-nbd: image name was not specified" << std::endl
;
1586 r
= do_map(argc
, argv
, &cfg
);
1596 r
= do_list_mapped_devices(cfg
.format
, cfg
.pretty_format
);
1608 int main(int argc
, const char *argv
[])
1610 int r
= rbd_nbd(argc
, argv
);
1612 return EXIT_FAILURE
;