2 * GlusterFS backend for QEMU
4 * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
10 #include "qemu/osdep.h"
11 #include <glusterfs/api/glfs.h>
12 #include "block/block_int.h"
13 #include "qapi/error.h"
14 #include "qapi/qmp/qerror.h"
16 #include "qemu/error-report.h"
18 #define GLUSTER_OPT_FILENAME "filename"
19 #define GLUSTER_OPT_VOLUME "volume"
20 #define GLUSTER_OPT_PATH "path"
21 #define GLUSTER_OPT_TYPE "type"
22 #define GLUSTER_OPT_SERVER_PATTERN "server."
23 #define GLUSTER_OPT_HOST "host"
24 #define GLUSTER_OPT_PORT "port"
25 #define GLUSTER_OPT_TO "to"
26 #define GLUSTER_OPT_IPV4 "ipv4"
27 #define GLUSTER_OPT_IPV6 "ipv6"
28 #define GLUSTER_OPT_SOCKET "socket"
29 #define GLUSTER_OPT_DEBUG "debug"
30 #define GLUSTER_DEFAULT_PORT 24007
31 #define GLUSTER_DEBUG_DEFAULT 4
32 #define GLUSTER_DEBUG_MAX 9
33 #define GLUSTER_OPT_LOGFILE "logfile"
34 #define GLUSTER_LOGFILE_DEFAULT "-" /* handled in libgfapi as /dev/stderr */
36 #define GERR_INDEX_HINT "hint: check in 'server' array index '%d'\n"
38 typedef struct GlusterAIOCB
{
42 AioContext
*aio_context
;
45 typedef struct BDRVGlusterState
{
49 bool supports_seek_data
;
53 typedef struct BDRVGlusterReopenState
{
56 } BDRVGlusterReopenState
;
59 static QemuOptsList qemu_gluster_create_opts
= {
60 .name
= "qemu-gluster-create-opts",
61 .head
= QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts
.head
),
64 .name
= BLOCK_OPT_SIZE
,
65 .type
= QEMU_OPT_SIZE
,
66 .help
= "Virtual disk size"
69 .name
= BLOCK_OPT_PREALLOC
,
70 .type
= QEMU_OPT_STRING
,
71 .help
= "Preallocation mode (allowed values: off, full)"
74 .name
= GLUSTER_OPT_DEBUG
,
75 .type
= QEMU_OPT_NUMBER
,
76 .help
= "Gluster log level, valid range is 0-9",
79 .name
= GLUSTER_OPT_LOGFILE
,
80 .type
= QEMU_OPT_STRING
,
81 .help
= "Logfile path of libgfapi",
87 static QemuOptsList runtime_opts
= {
89 .head
= QTAILQ_HEAD_INITIALIZER(runtime_opts
.head
),
92 .name
= GLUSTER_OPT_FILENAME
,
93 .type
= QEMU_OPT_STRING
,
94 .help
= "URL to the gluster image",
97 .name
= GLUSTER_OPT_DEBUG
,
98 .type
= QEMU_OPT_NUMBER
,
99 .help
= "Gluster log level, valid range is 0-9",
102 .name
= GLUSTER_OPT_LOGFILE
,
103 .type
= QEMU_OPT_STRING
,
104 .help
= "Logfile path of libgfapi",
106 { /* end of list */ }
110 static QemuOptsList runtime_json_opts
= {
111 .name
= "gluster_json",
112 .head
= QTAILQ_HEAD_INITIALIZER(runtime_json_opts
.head
),
115 .name
= GLUSTER_OPT_VOLUME
,
116 .type
= QEMU_OPT_STRING
,
117 .help
= "name of gluster volume where VM image resides",
120 .name
= GLUSTER_OPT_PATH
,
121 .type
= QEMU_OPT_STRING
,
122 .help
= "absolute path to image file in gluster volume",
125 .name
= GLUSTER_OPT_DEBUG
,
126 .type
= QEMU_OPT_NUMBER
,
127 .help
= "Gluster log level, valid range is 0-9",
129 { /* end of list */ }
133 static QemuOptsList runtime_type_opts
= {
134 .name
= "gluster_type",
135 .head
= QTAILQ_HEAD_INITIALIZER(runtime_type_opts
.head
),
138 .name
= GLUSTER_OPT_TYPE
,
139 .type
= QEMU_OPT_STRING
,
142 { /* end of list */ }
146 static QemuOptsList runtime_unix_opts
= {
147 .name
= "gluster_unix",
148 .head
= QTAILQ_HEAD_INITIALIZER(runtime_unix_opts
.head
),
151 .name
= GLUSTER_OPT_SOCKET
,
152 .type
= QEMU_OPT_STRING
,
153 .help
= "socket file path)",
155 { /* end of list */ }
159 static QemuOptsList runtime_tcp_opts
= {
160 .name
= "gluster_tcp",
161 .head
= QTAILQ_HEAD_INITIALIZER(runtime_tcp_opts
.head
),
164 .name
= GLUSTER_OPT_TYPE
,
165 .type
= QEMU_OPT_STRING
,
169 .name
= GLUSTER_OPT_HOST
,
170 .type
= QEMU_OPT_STRING
,
171 .help
= "host address (hostname/ipv4/ipv6 addresses)",
174 .name
= GLUSTER_OPT_PORT
,
175 .type
= QEMU_OPT_NUMBER
,
176 .help
= "port number on which glusterd is listening (default 24007)",
180 .type
= QEMU_OPT_NUMBER
,
181 .help
= "max port number, not supported by gluster",
185 .type
= QEMU_OPT_BOOL
,
186 .help
= "ipv4 bool value, not supported by gluster",
190 .type
= QEMU_OPT_BOOL
,
191 .help
= "ipv6 bool value, not supported by gluster",
193 { /* end of list */ }
197 static int parse_volume_options(BlockdevOptionsGluster
*gconf
, char *path
)
206 p
= q
= path
+ strspn(path
, "/");
207 p
+= strcspn(p
, "/");
211 gconf
->volume
= g_strndup(q
, p
- q
);
218 gconf
->path
= g_strdup(p
);
223 * file=gluster[+transport]://[host[:port]]/volume/path[?socket=...]
225 * 'gluster' is the protocol.
227 * 'transport' specifies the transport type used to connect to gluster
228 * management daemon (glusterd). Valid transport types are
229 * tcp or unix. If a transport type isn't specified, then tcp type is assumed.
231 * 'host' specifies the host where the volume file specification for
232 * the given volume resides. This can be either hostname or ipv4 address.
233 * If transport type is 'unix', then 'host' field should not be specified.
234 * The 'socket' field needs to be populated with the path to unix domain
237 * 'port' is the port number on which glusterd is listening. This is optional
238 * and if not specified, QEMU will send 0 which will make gluster to use the
239 * default port. If the transport type is unix, then 'port' should not be
242 * 'volume' is the name of the gluster volume which contains the VM image.
244 * 'path' is the path to the actual VM image that resides on gluster volume.
248 * file=gluster://1.2.3.4/testvol/a.img
249 * file=gluster+tcp://1.2.3.4/testvol/a.img
250 * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
251 * file=gluster+tcp://host.domain.com:24007/testvol/dir/a.img
252 * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
254 static int qemu_gluster_parse_uri(BlockdevOptionsGluster
*gconf
,
255 const char *filename
)
257 GlusterServer
*gsconf
;
259 QueryParams
*qp
= NULL
;
260 bool is_unix
= false;
263 uri
= uri_parse(filename
);
268 gconf
->server
= g_new0(GlusterServerList
, 1);
269 gconf
->server
->value
= gsconf
= g_new0(GlusterServer
, 1);
272 if (!uri
->scheme
|| !strcmp(uri
->scheme
, "gluster")) {
273 gsconf
->type
= GLUSTER_TRANSPORT_TCP
;
274 } else if (!strcmp(uri
->scheme
, "gluster+tcp")) {
275 gsconf
->type
= GLUSTER_TRANSPORT_TCP
;
276 } else if (!strcmp(uri
->scheme
, "gluster+unix")) {
277 gsconf
->type
= GLUSTER_TRANSPORT_UNIX
;
279 } else if (!strcmp(uri
->scheme
, "gluster+rdma")) {
280 gsconf
->type
= GLUSTER_TRANSPORT_TCP
;
281 error_report("Warning: rdma feature is not supported, falling "
288 ret
= parse_volume_options(gconf
, uri
->path
);
293 qp
= query_params_parse(uri
->query
);
294 if (qp
->n
> 1 || (is_unix
&& !qp
->n
) || (!is_unix
&& qp
->n
)) {
300 if (uri
->server
|| uri
->port
) {
304 if (strcmp(qp
->p
[0].name
, "socket")) {
308 gsconf
->u
.q_unix
.path
= g_strdup(qp
->p
[0].value
);
310 gsconf
->u
.tcp
.host
= g_strdup(uri
->server
? uri
->server
: "localhost");
312 gsconf
->u
.tcp
.port
= g_strdup_printf("%d", uri
->port
);
314 gsconf
->u
.tcp
.port
= g_strdup_printf("%d", GLUSTER_DEFAULT_PORT
);
320 query_params_free(qp
);
326 static struct glfs
*qemu_gluster_glfs_init(BlockdevOptionsGluster
*gconf
,
332 GlusterServerList
*server
;
334 glfs
= glfs_new(gconf
->volume
);
339 for (server
= gconf
->server
; server
; server
= server
->next
) {
340 if (server
->value
->type
== GLUSTER_TRANSPORT_UNIX
) {
341 ret
= glfs_set_volfile_server(glfs
,
342 GlusterTransport_lookup
[server
->value
->type
],
343 server
->value
->u
.q_unix
.path
, 0);
345 ret
= glfs_set_volfile_server(glfs
,
346 GlusterTransport_lookup
[server
->value
->type
],
347 server
->value
->u
.tcp
.host
,
348 atoi(server
->value
->u
.tcp
.port
));
356 ret
= glfs_set_logging(glfs
, gconf
->logfile
, gconf
->debug_level
);
361 ret
= glfs_init(glfs
);
363 error_setg(errp
, "Gluster connection for volume %s, path %s failed"
364 " to connect", gconf
->volume
, gconf
->path
);
365 for (server
= gconf
->server
; server
; server
= server
->next
) {
366 if (server
->value
->type
== GLUSTER_TRANSPORT_UNIX
) {
367 error_append_hint(errp
, "hint: failed on socket %s ",
368 server
->value
->u
.q_unix
.path
);
370 error_append_hint(errp
, "hint: failed on host %s and port %s ",
371 server
->value
->u
.tcp
.host
,
372 server
->value
->u
.tcp
.port
);
376 error_append_hint(errp
, "Please refer to gluster logs for more info\n");
378 /* glfs_init sometimes doesn't set errno although docs suggest that */
396 static int qapi_enum_parse(const char *opt
)
401 return GLUSTER_TRANSPORT__MAX
;
404 for (i
= 0; i
< GLUSTER_TRANSPORT__MAX
; i
++) {
405 if (!strcmp(opt
, GlusterTransport_lookup
[i
])) {
414 * Convert the json formatted command line into qapi.
416 static int qemu_gluster_parse_json(BlockdevOptionsGluster
*gconf
,
417 QDict
*options
, Error
**errp
)
420 GlusterServer
*gsconf
;
421 GlusterServerList
*curr
= NULL
;
422 QDict
*backing_options
= NULL
;
423 Error
*local_err
= NULL
;
429 /* create opts info from runtime_json_opts list */
430 opts
= qemu_opts_create(&runtime_json_opts
, NULL
, 0, &error_abort
);
431 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
436 num_servers
= qdict_array_entries(options
, GLUSTER_OPT_SERVER_PATTERN
);
437 if (num_servers
< 1) {
438 error_setg(&local_err
, QERR_MISSING_PARAMETER
, "server");
442 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_VOLUME
);
444 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_VOLUME
);
447 gconf
->volume
= g_strdup(ptr
);
449 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_PATH
);
451 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_PATH
);
454 gconf
->path
= g_strdup(ptr
);
457 for (i
= 0; i
< num_servers
; i
++) {
458 str
= g_strdup_printf(GLUSTER_OPT_SERVER_PATTERN
"%d.", i
);
459 qdict_extract_subqdict(options
, &backing_options
, str
);
461 /* create opts info from runtime_type_opts list */
462 opts
= qemu_opts_create(&runtime_type_opts
, NULL
, 0, &error_abort
);
463 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
468 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_TYPE
);
469 gsconf
= g_new0(GlusterServer
, 1);
470 gsconf
->type
= qapi_enum_parse(ptr
);
472 error_setg(&local_err
, QERR_MISSING_PARAMETER
, GLUSTER_OPT_TYPE
);
473 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
477 if (gsconf
->type
== GLUSTER_TRANSPORT__MAX
) {
478 error_setg(&local_err
, QERR_INVALID_PARAMETER_VALUE
,
479 GLUSTER_OPT_TYPE
, "tcp or unix");
480 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
485 if (gsconf
->type
== GLUSTER_TRANSPORT_TCP
) {
486 /* create opts info from runtime_tcp_opts list */
487 opts
= qemu_opts_create(&runtime_tcp_opts
, NULL
, 0, &error_abort
);
488 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
493 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_HOST
);
495 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
497 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
500 gsconf
->u
.tcp
.host
= g_strdup(ptr
);
501 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_PORT
);
503 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
505 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
508 gsconf
->u
.tcp
.port
= g_strdup(ptr
);
510 /* defend for unsupported fields in InetSocketAddress,
511 * i.e. @ipv4, @ipv6 and @to
513 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_TO
);
515 gsconf
->u
.tcp
.has_to
= true;
517 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_IPV4
);
519 gsconf
->u
.tcp
.has_ipv4
= true;
521 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_IPV6
);
523 gsconf
->u
.tcp
.has_ipv6
= true;
525 if (gsconf
->u
.tcp
.has_to
) {
526 error_setg(&local_err
, "Parameter 'to' not supported");
529 if (gsconf
->u
.tcp
.has_ipv4
|| gsconf
->u
.tcp
.has_ipv6
) {
530 error_setg(&local_err
, "Parameters 'ipv4/ipv6' not supported");
535 /* create opts info from runtime_unix_opts list */
536 opts
= qemu_opts_create(&runtime_unix_opts
, NULL
, 0, &error_abort
);
537 qemu_opts_absorb_qdict(opts
, backing_options
, &local_err
);
542 ptr
= qemu_opt_get(opts
, GLUSTER_OPT_SOCKET
);
544 error_setg(&local_err
, QERR_MISSING_PARAMETER
,
546 error_append_hint(&local_err
, GERR_INDEX_HINT
, i
);
549 gsconf
->u
.q_unix
.path
= g_strdup(ptr
);
553 if (gconf
->server
== NULL
) {
554 gconf
->server
= g_new0(GlusterServerList
, 1);
555 gconf
->server
->value
= gsconf
;
556 curr
= gconf
->server
;
558 curr
->next
= g_new0(GlusterServerList
, 1);
559 curr
->next
->value
= gsconf
;
563 qdict_del(backing_options
, str
);
571 error_propagate(errp
, local_err
);
574 qdict_del(backing_options
, str
);
581 static struct glfs
*qemu_gluster_init(BlockdevOptionsGluster
*gconf
,
582 const char *filename
,
583 QDict
*options
, Error
**errp
)
587 ret
= qemu_gluster_parse_uri(gconf
, filename
);
589 error_setg(errp
, "invalid URI");
590 error_append_hint(errp
, "Usage: file=gluster[+transport]://"
591 "[host[:port]]volume/path[?socket=...]"
593 "[,file.logfile=/path/filename.log]\n");
598 ret
= qemu_gluster_parse_json(gconf
, options
, errp
);
600 error_append_hint(errp
, "Usage: "
601 "-drive driver=qcow2,file.driver=gluster,"
602 "file.volume=testvol,file.path=/path/a.qcow2"
604 "[,file.logfile=/path/filename.log],"
605 "file.server.0.type=tcp,"
606 "file.server.0.host=1.2.3.4,"
607 "file.server.0.port=24007,"
608 "file.server.1.transport=unix,"
609 "file.server.1.socket=/var/run/glusterd.socket ..."
617 return qemu_gluster_glfs_init(gconf
, errp
);
620 static void qemu_gluster_complete_aio(void *opaque
)
622 GlusterAIOCB
*acb
= (GlusterAIOCB
*)opaque
;
624 qemu_coroutine_enter(acb
->coroutine
);
628 * AIO callback routine called from GlusterFS thread.
630 static void gluster_finish_aiocb(struct glfs_fd
*fd
, ssize_t ret
, void *arg
)
632 GlusterAIOCB
*acb
= (GlusterAIOCB
*)arg
;
634 if (!ret
|| ret
== acb
->size
) {
635 acb
->ret
= 0; /* Success */
636 } else if (ret
< 0) {
637 acb
->ret
= -errno
; /* Read/Write failed */
639 acb
->ret
= -EIO
; /* Partial read/write - fail it */
642 aio_bh_schedule_oneshot(acb
->aio_context
, qemu_gluster_complete_aio
, acb
);
645 static void qemu_gluster_parse_flags(int bdrv_flags
, int *open_flags
)
647 assert(open_flags
!= NULL
);
649 *open_flags
|= O_BINARY
;
651 if (bdrv_flags
& BDRV_O_RDWR
) {
652 *open_flags
|= O_RDWR
;
654 *open_flags
|= O_RDONLY
;
657 if ((bdrv_flags
& BDRV_O_NOCACHE
)) {
658 *open_flags
|= O_DIRECT
;
663 * Do SEEK_DATA/HOLE to detect if it is functional. Older broken versions of
664 * gfapi incorrectly return the current offset when SEEK_DATA/HOLE is used.
665 * - Corrected versions return -1 and set errno to EINVAL.
666 * - Versions that support SEEK_DATA/HOLE correctly, will return -1 and set
667 * errno to ENXIO when SEEK_DATA is called with a position of EOF.
669 static bool qemu_gluster_test_seek(struct glfs_fd
*fd
)
673 eof
= glfs_lseek(fd
, 0, SEEK_END
);
675 /* this should never occur */
679 /* this should always fail with ENXIO if SEEK_DATA is supported */
680 ret
= glfs_lseek(fd
, eof
, SEEK_DATA
);
681 return (ret
< 0) && (errno
== ENXIO
);
684 static int qemu_gluster_open(BlockDriverState
*bs
, QDict
*options
,
685 int bdrv_flags
, Error
**errp
)
687 BDRVGlusterState
*s
= bs
->opaque
;
690 BlockdevOptionsGluster
*gconf
= NULL
;
692 Error
*local_err
= NULL
;
693 const char *filename
, *logfile
;
695 opts
= qemu_opts_create(&runtime_opts
, NULL
, 0, &error_abort
);
696 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
698 error_propagate(errp
, local_err
);
703 filename
= qemu_opt_get(opts
, GLUSTER_OPT_FILENAME
);
705 s
->debug_level
= qemu_opt_get_number(opts
, GLUSTER_OPT_DEBUG
,
706 GLUSTER_DEBUG_DEFAULT
);
707 if (s
->debug_level
< 0) {
709 } else if (s
->debug_level
> GLUSTER_DEBUG_MAX
) {
710 s
->debug_level
= GLUSTER_DEBUG_MAX
;
713 gconf
= g_new0(BlockdevOptionsGluster
, 1);
714 gconf
->debug_level
= s
->debug_level
;
715 gconf
->has_debug_level
= true;
717 logfile
= qemu_opt_get(opts
, GLUSTER_OPT_LOGFILE
);
718 s
->logfile
= g_strdup(logfile
? logfile
: GLUSTER_LOGFILE_DEFAULT
);
720 gconf
->logfile
= g_strdup(s
->logfile
);
721 gconf
->has_logfile
= true;
723 s
->glfs
= qemu_gluster_init(gconf
, filename
, options
, errp
);
729 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
730 /* Without this, if fsync fails for a recoverable reason (for instance,
731 * ENOSPC), gluster will dump its cache, preventing retries. This means
732 * almost certain data loss. Not all gluster versions support the
733 * 'resync-failed-syncs-after-fsync' key value, but there is no way to
734 * discover during runtime if it is supported (this api returns success for
735 * unknown key/value pairs) */
736 ret
= glfs_set_xlator_option(s
->glfs
, "*-write-behind",
737 "resync-failed-syncs-after-fsync",
740 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
746 qemu_gluster_parse_flags(bdrv_flags
, &open_flags
);
748 s
->fd
= glfs_open(s
->glfs
, gconf
->path
, open_flags
);
753 s
->supports_seek_data
= qemu_gluster_test_seek(s
->fd
);
757 qapi_free_BlockdevOptionsGluster(gconf
);
771 static int qemu_gluster_reopen_prepare(BDRVReopenState
*state
,
772 BlockReopenQueue
*queue
, Error
**errp
)
776 BDRVGlusterReopenState
*reop_s
;
777 BlockdevOptionsGluster
*gconf
;
780 assert(state
!= NULL
);
781 assert(state
->bs
!= NULL
);
783 s
= state
->bs
->opaque
;
785 state
->opaque
= g_new0(BDRVGlusterReopenState
, 1);
786 reop_s
= state
->opaque
;
788 qemu_gluster_parse_flags(state
->flags
, &open_flags
);
790 gconf
= g_new0(BlockdevOptionsGluster
, 1);
791 gconf
->debug_level
= s
->debug_level
;
792 gconf
->has_debug_level
= true;
793 gconf
->logfile
= g_strdup(s
->logfile
);
794 gconf
->has_logfile
= true;
795 reop_s
->glfs
= qemu_gluster_init(gconf
, state
->bs
->filename
, NULL
, errp
);
796 if (reop_s
->glfs
== NULL
) {
801 #ifdef CONFIG_GLUSTERFS_XLATOR_OPT
802 ret
= glfs_set_xlator_option(reop_s
->glfs
, "*-write-behind",
803 "resync-failed-syncs-after-fsync", "on");
805 error_setg_errno(errp
, errno
, "Unable to set xlator key/value pair");
811 reop_s
->fd
= glfs_open(reop_s
->glfs
, gconf
->path
, open_flags
);
812 if (reop_s
->fd
== NULL
) {
813 /* reops->glfs will be cleaned up in _abort */
819 /* state->opaque will be freed in either the _abort or _commit */
820 qapi_free_BlockdevOptionsGluster(gconf
);
824 static void qemu_gluster_reopen_commit(BDRVReopenState
*state
)
826 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
827 BDRVGlusterState
*s
= state
->bs
->opaque
;
838 /* use the newly opened image / connection */
840 s
->glfs
= reop_s
->glfs
;
842 g_free(state
->opaque
);
843 state
->opaque
= NULL
;
849 static void qemu_gluster_reopen_abort(BDRVReopenState
*state
)
851 BDRVGlusterReopenState
*reop_s
= state
->opaque
;
853 if (reop_s
== NULL
) {
858 glfs_close(reop_s
->fd
);
862 glfs_fini(reop_s
->glfs
);
865 g_free(state
->opaque
);
866 state
->opaque
= NULL
;
871 #ifdef CONFIG_GLUSTERFS_ZEROFILL
872 static coroutine_fn
int qemu_gluster_co_pwrite_zeroes(BlockDriverState
*bs
,
875 BdrvRequestFlags flags
)
879 BDRVGlusterState
*s
= bs
->opaque
;
883 acb
.coroutine
= qemu_coroutine_self();
884 acb
.aio_context
= bdrv_get_aio_context(bs
);
886 ret
= glfs_zerofill_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
891 qemu_coroutine_yield();
895 static inline bool gluster_supports_zerofill(void)
900 static inline int qemu_gluster_zerofill(struct glfs_fd
*fd
, int64_t offset
,
903 return glfs_zerofill(fd
, offset
, size
);
907 static inline bool gluster_supports_zerofill(void)
912 static inline int qemu_gluster_zerofill(struct glfs_fd
*fd
, int64_t offset
,
919 static int qemu_gluster_create(const char *filename
,
920 QemuOpts
*opts
, Error
**errp
)
922 BlockdevOptionsGluster
*gconf
;
927 int64_t total_size
= 0;
930 gconf
= g_new0(BlockdevOptionsGluster
, 1);
931 gconf
->debug_level
= qemu_opt_get_number_del(opts
, GLUSTER_OPT_DEBUG
,
932 GLUSTER_DEBUG_DEFAULT
);
933 if (gconf
->debug_level
< 0) {
934 gconf
->debug_level
= 0;
935 } else if (gconf
->debug_level
> GLUSTER_DEBUG_MAX
) {
936 gconf
->debug_level
= GLUSTER_DEBUG_MAX
;
938 gconf
->has_debug_level
= true;
940 gconf
->logfile
= qemu_opt_get_del(opts
, GLUSTER_OPT_LOGFILE
);
941 if (!gconf
->logfile
) {
942 gconf
->logfile
= g_strdup(GLUSTER_LOGFILE_DEFAULT
);
944 gconf
->has_logfile
= true;
946 glfs
= qemu_gluster_init(gconf
, filename
, NULL
, errp
);
952 total_size
= ROUND_UP(qemu_opt_get_size_del(opts
, BLOCK_OPT_SIZE
, 0),
955 tmp
= qemu_opt_get_del(opts
, BLOCK_OPT_PREALLOC
);
956 if (!tmp
|| !strcmp(tmp
, "off")) {
958 } else if (!strcmp(tmp
, "full") && gluster_supports_zerofill()) {
961 error_setg(errp
, "Invalid preallocation mode: '%s'"
962 " or GlusterFS doesn't support zerofill API", tmp
);
967 fd
= glfs_creat(glfs
, gconf
->path
,
968 O_WRONLY
| O_CREAT
| O_TRUNC
| O_BINARY
, S_IRUSR
| S_IWUSR
);
972 if (!glfs_ftruncate(fd
, total_size
)) {
973 if (prealloc
&& qemu_gluster_zerofill(fd
, 0, total_size
)) {
980 if (glfs_close(fd
) != 0) {
986 qapi_free_BlockdevOptionsGluster(gconf
);
993 static coroutine_fn
int qemu_gluster_co_rw(BlockDriverState
*bs
,
994 int64_t sector_num
, int nb_sectors
,
995 QEMUIOVector
*qiov
, int write
)
999 BDRVGlusterState
*s
= bs
->opaque
;
1000 size_t size
= nb_sectors
* BDRV_SECTOR_SIZE
;
1001 off_t offset
= sector_num
* BDRV_SECTOR_SIZE
;
1005 acb
.coroutine
= qemu_coroutine_self();
1006 acb
.aio_context
= bdrv_get_aio_context(bs
);
1009 ret
= glfs_pwritev_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
1010 gluster_finish_aiocb
, &acb
);
1012 ret
= glfs_preadv_async(s
->fd
, qiov
->iov
, qiov
->niov
, offset
, 0,
1013 gluster_finish_aiocb
, &acb
);
1020 qemu_coroutine_yield();
1024 static int qemu_gluster_truncate(BlockDriverState
*bs
, int64_t offset
)
1027 BDRVGlusterState
*s
= bs
->opaque
;
1029 ret
= glfs_ftruncate(s
->fd
, offset
);
1037 static coroutine_fn
int qemu_gluster_co_readv(BlockDriverState
*bs
,
1042 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 0);
1045 static coroutine_fn
int qemu_gluster_co_writev(BlockDriverState
*bs
,
1050 return qemu_gluster_co_rw(bs
, sector_num
, nb_sectors
, qiov
, 1);
1053 static void qemu_gluster_close(BlockDriverState
*bs
)
1055 BDRVGlusterState
*s
= bs
->opaque
;
1065 static coroutine_fn
int qemu_gluster_co_flush_to_disk(BlockDriverState
*bs
)
1069 BDRVGlusterState
*s
= bs
->opaque
;
1073 acb
.coroutine
= qemu_coroutine_self();
1074 acb
.aio_context
= bdrv_get_aio_context(bs
);
1076 ret
= glfs_fsync_async(s
->fd
, gluster_finish_aiocb
, &acb
);
1082 qemu_coroutine_yield();
1091 /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
1092 * after a fsync failure, so we have no way of allowing the guest to safely
1093 * continue. Gluster versions prior to 3.5.6 don't retain the cache
1094 * either, but will invalidate the fd on error, so this is again our only
1097 * The 'resync-failed-syncs-after-fsync' xlator option for the
1098 * write-behind cache will cause later gluster versions to retain its
1099 * cache after error, so long as the fd remains open. However, we
1100 * currently have no way of knowing if this option is supported.
1102 * TODO: Once gluster provides a way for us to determine if the option
1103 * is supported, bypass the closure and setting drv to NULL. */
1104 qemu_gluster_close(bs
);
1109 #ifdef CONFIG_GLUSTERFS_DISCARD
1110 static coroutine_fn
int qemu_gluster_co_pdiscard(BlockDriverState
*bs
,
1111 int64_t offset
, int size
)
1115 BDRVGlusterState
*s
= bs
->opaque
;
1119 acb
.coroutine
= qemu_coroutine_self();
1120 acb
.aio_context
= bdrv_get_aio_context(bs
);
1122 ret
= glfs_discard_async(s
->fd
, offset
, size
, gluster_finish_aiocb
, &acb
);
1127 qemu_coroutine_yield();
1132 static int64_t qemu_gluster_getlength(BlockDriverState
*bs
)
1134 BDRVGlusterState
*s
= bs
->opaque
;
1137 ret
= glfs_lseek(s
->fd
, 0, SEEK_END
);
1145 static int64_t qemu_gluster_allocated_file_size(BlockDriverState
*bs
)
1147 BDRVGlusterState
*s
= bs
->opaque
;
1151 ret
= glfs_fstat(s
->fd
, &st
);
1155 return st
.st_blocks
* 512;
1159 static int qemu_gluster_has_zero_init(BlockDriverState
*bs
)
1161 /* GlusterFS volume could be backed by a block device */
1166 * Find allocation range in @bs around offset @start.
1167 * May change underlying file descriptor's file offset.
1168 * If @start is not in a hole, store @start in @data, and the
1169 * beginning of the next hole in @hole, and return 0.
1170 * If @start is in a non-trailing hole, store @start in @hole and the
1171 * beginning of the next non-hole in @data, and return 0.
1172 * If @start is in a trailing hole or beyond EOF, return -ENXIO.
1173 * If we can't find out, return a negative errno other than -ENXIO.
1175 * (Shamefully copied from raw-posix.c, only miniscule adaptions.)
1177 static int find_allocation(BlockDriverState
*bs
, off_t start
,
1178 off_t
*data
, off_t
*hole
)
1180 BDRVGlusterState
*s
= bs
->opaque
;
1183 if (!s
->supports_seek_data
) {
1189 * D1. offs == start: start is in data
1190 * D2. offs > start: start is in a hole, next data at offs
1191 * D3. offs < 0, errno = ENXIO: either start is in a trailing hole
1192 * or start is beyond EOF
1193 * If the latter happens, the file has been truncated behind
1194 * our back since we opened it. All bets are off then.
1195 * Treating like a trailing hole is simplest.
1196 * D4. offs < 0, errno != ENXIO: we learned nothing
1198 offs
= glfs_lseek(s
->fd
, start
, SEEK_DATA
);
1200 return -errno
; /* D3 or D4 */
1202 assert(offs
>= start
);
1205 /* D2: in hole, next data at offs */
1211 /* D1: in data, end not yet known */
1215 * H1. offs == start: start is in a hole
1216 * If this happens here, a hole has been dug behind our back
1217 * since the previous lseek().
1218 * H2. offs > start: either start is in data, next hole at offs,
1219 * or start is in trailing hole, EOF at offs
1220 * Linux treats trailing holes like any other hole: offs ==
1221 * start. Solaris seeks to EOF instead: offs > start (blech).
1222 * If that happens here, a hole has been dug behind our back
1223 * since the previous lseek().
1224 * H3. offs < 0, errno = ENXIO: start is beyond EOF
1225 * If this happens, the file has been truncated behind our
1226 * back since we opened it. Treat it like a trailing hole.
1227 * H4. offs < 0, errno != ENXIO: we learned nothing
1228 * Pretend we know nothing at all, i.e. "forget" about D1.
1230 offs
= glfs_lseek(s
->fd
, start
, SEEK_HOLE
);
1232 return -errno
; /* D1 and (H3 or H4) */
1234 assert(offs
>= start
);
1238 * D1 and H2: either in data, next hole at offs, or it was in
1239 * data but is now in a trailing hole. In the latter case,
1240 * all bets are off. Treating it as if it there was data all
1241 * the way to EOF is safe, so simply do that.
1253 * Returns the allocation status of the specified sectors.
1255 * If 'sector_num' is beyond the end of the disk image the return value is 0
1256 * and 'pnum' is set to 0.
1258 * 'pnum' is set to the number of sectors (including and immediately following
1259 * the specified sector) that are known to be in the same
1260 * allocated/unallocated state.
1262 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes
1263 * beyond the end of the disk image it will be clamped.
1265 * (Based on raw_co_get_block_status() from raw-posix.c.)
1267 static int64_t coroutine_fn
qemu_gluster_co_get_block_status(
1268 BlockDriverState
*bs
, int64_t sector_num
, int nb_sectors
, int *pnum
,
1269 BlockDriverState
**file
)
1271 BDRVGlusterState
*s
= bs
->opaque
;
1272 off_t start
, data
= 0, hole
= 0;
1280 start
= sector_num
* BDRV_SECTOR_SIZE
;
1281 total_size
= bdrv_getlength(bs
);
1282 if (total_size
< 0) {
1284 } else if (start
>= total_size
) {
1287 } else if (start
+ nb_sectors
* BDRV_SECTOR_SIZE
> total_size
) {
1288 nb_sectors
= DIV_ROUND_UP(total_size
- start
, BDRV_SECTOR_SIZE
);
1291 ret
= find_allocation(bs
, start
, &data
, &hole
);
1292 if (ret
== -ENXIO
) {
1295 ret
= BDRV_BLOCK_ZERO
;
1296 } else if (ret
< 0) {
1297 /* No info available, so pretend there are no holes */
1299 ret
= BDRV_BLOCK_DATA
;
1300 } else if (data
== start
) {
1301 /* On a data extent, compute sectors to the end of the extent,
1302 * possibly including a partial sector at EOF. */
1303 *pnum
= MIN(nb_sectors
, DIV_ROUND_UP(hole
- start
, BDRV_SECTOR_SIZE
));
1304 ret
= BDRV_BLOCK_DATA
;
1306 /* On a hole, compute sectors to the beginning of the next extent. */
1307 assert(hole
== start
);
1308 *pnum
= MIN(nb_sectors
, (data
- start
) / BDRV_SECTOR_SIZE
);
1309 ret
= BDRV_BLOCK_ZERO
;
1314 return ret
| BDRV_BLOCK_OFFSET_VALID
| start
;
1318 static BlockDriver bdrv_gluster
= {
1319 .format_name
= "gluster",
1320 .protocol_name
= "gluster",
1321 .instance_size
= sizeof(BDRVGlusterState
),
1322 .bdrv_needs_filename
= false,
1323 .bdrv_file_open
= qemu_gluster_open
,
1324 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1325 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1326 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1327 .bdrv_close
= qemu_gluster_close
,
1328 .bdrv_create
= qemu_gluster_create
,
1329 .bdrv_getlength
= qemu_gluster_getlength
,
1330 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1331 .bdrv_truncate
= qemu_gluster_truncate
,
1332 .bdrv_co_readv
= qemu_gluster_co_readv
,
1333 .bdrv_co_writev
= qemu_gluster_co_writev
,
1334 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1335 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1336 #ifdef CONFIG_GLUSTERFS_DISCARD
1337 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1339 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1340 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1342 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1343 .create_opts
= &qemu_gluster_create_opts
,
1346 static BlockDriver bdrv_gluster_tcp
= {
1347 .format_name
= "gluster",
1348 .protocol_name
= "gluster+tcp",
1349 .instance_size
= sizeof(BDRVGlusterState
),
1350 .bdrv_needs_filename
= false,
1351 .bdrv_file_open
= qemu_gluster_open
,
1352 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1353 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1354 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1355 .bdrv_close
= qemu_gluster_close
,
1356 .bdrv_create
= qemu_gluster_create
,
1357 .bdrv_getlength
= qemu_gluster_getlength
,
1358 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1359 .bdrv_truncate
= qemu_gluster_truncate
,
1360 .bdrv_co_readv
= qemu_gluster_co_readv
,
1361 .bdrv_co_writev
= qemu_gluster_co_writev
,
1362 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1363 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1364 #ifdef CONFIG_GLUSTERFS_DISCARD
1365 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1367 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1368 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1370 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1371 .create_opts
= &qemu_gluster_create_opts
,
1374 static BlockDriver bdrv_gluster_unix
= {
1375 .format_name
= "gluster",
1376 .protocol_name
= "gluster+unix",
1377 .instance_size
= sizeof(BDRVGlusterState
),
1378 .bdrv_needs_filename
= true,
1379 .bdrv_file_open
= qemu_gluster_open
,
1380 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1381 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1382 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1383 .bdrv_close
= qemu_gluster_close
,
1384 .bdrv_create
= qemu_gluster_create
,
1385 .bdrv_getlength
= qemu_gluster_getlength
,
1386 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1387 .bdrv_truncate
= qemu_gluster_truncate
,
1388 .bdrv_co_readv
= qemu_gluster_co_readv
,
1389 .bdrv_co_writev
= qemu_gluster_co_writev
,
1390 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1391 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1392 #ifdef CONFIG_GLUSTERFS_DISCARD
1393 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1395 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1396 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1398 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1399 .create_opts
= &qemu_gluster_create_opts
,
1402 /* rdma is deprecated (actually never supported for volfile fetch).
1403 * Let's maintain it for the protocol compatibility, to make sure things
1404 * won't break immediately. For now, gluster+rdma will fall back to gluster+tcp
1405 * protocol with a warning.
1406 * TODO: remove gluster+rdma interface support
1408 static BlockDriver bdrv_gluster_rdma
= {
1409 .format_name
= "gluster",
1410 .protocol_name
= "gluster+rdma",
1411 .instance_size
= sizeof(BDRVGlusterState
),
1412 .bdrv_needs_filename
= true,
1413 .bdrv_file_open
= qemu_gluster_open
,
1414 .bdrv_reopen_prepare
= qemu_gluster_reopen_prepare
,
1415 .bdrv_reopen_commit
= qemu_gluster_reopen_commit
,
1416 .bdrv_reopen_abort
= qemu_gluster_reopen_abort
,
1417 .bdrv_close
= qemu_gluster_close
,
1418 .bdrv_create
= qemu_gluster_create
,
1419 .bdrv_getlength
= qemu_gluster_getlength
,
1420 .bdrv_get_allocated_file_size
= qemu_gluster_allocated_file_size
,
1421 .bdrv_truncate
= qemu_gluster_truncate
,
1422 .bdrv_co_readv
= qemu_gluster_co_readv
,
1423 .bdrv_co_writev
= qemu_gluster_co_writev
,
1424 .bdrv_co_flush_to_disk
= qemu_gluster_co_flush_to_disk
,
1425 .bdrv_has_zero_init
= qemu_gluster_has_zero_init
,
1426 #ifdef CONFIG_GLUSTERFS_DISCARD
1427 .bdrv_co_pdiscard
= qemu_gluster_co_pdiscard
,
1429 #ifdef CONFIG_GLUSTERFS_ZEROFILL
1430 .bdrv_co_pwrite_zeroes
= qemu_gluster_co_pwrite_zeroes
,
1432 .bdrv_co_get_block_status
= qemu_gluster_co_get_block_status
,
1433 .create_opts
= &qemu_gluster_create_opts
,
1436 static void bdrv_gluster_init(void)
1438 bdrv_register(&bdrv_gluster_rdma
);
1439 bdrv_register(&bdrv_gluster_unix
);
1440 bdrv_register(&bdrv_gluster_tcp
);
1441 bdrv_register(&bdrv_gluster
);
1444 block_init(bdrv_gluster_init
);