2 * Copyright (C) 2009-2010 Nippon Telegraph and Telephone Corporation.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
8 * You should have received a copy of the GNU General Public License
9 * along with this program. If not, see <http://www.gnu.org/licenses/>.
11 * Contributions after 2012-01-13 are licensed under the terms of the
12 * GNU GPL, version 2 or (at your option) any later version.
15 #include "qemu/osdep.h"
16 #include "qemu-common.h"
18 #include "qemu/error-report.h"
19 #include "qemu/sockets.h"
20 #include "block/block_int.h"
21 #include "qemu/bitops.h"
23 #define SD_PROTO_VER 0x01
25 #define SD_DEFAULT_ADDR "localhost"
26 #define SD_DEFAULT_PORT 7000
28 #define SD_OP_CREATE_AND_WRITE_OBJ 0x01
29 #define SD_OP_READ_OBJ 0x02
30 #define SD_OP_WRITE_OBJ 0x03
31 /* 0x04 is used internally by Sheepdog */
33 #define SD_OP_NEW_VDI 0x11
34 #define SD_OP_LOCK_VDI 0x12
35 #define SD_OP_RELEASE_VDI 0x13
36 #define SD_OP_GET_VDI_INFO 0x14
37 #define SD_OP_READ_VDIS 0x15
38 #define SD_OP_FLUSH_VDI 0x16
39 #define SD_OP_DEL_VDI 0x17
40 #define SD_OP_GET_CLUSTER_DEFAULT 0x18
42 #define SD_FLAG_CMD_WRITE 0x01
43 #define SD_FLAG_CMD_COW 0x02
44 #define SD_FLAG_CMD_CACHE 0x04 /* Writeback mode for cache */
45 #define SD_FLAG_CMD_DIRECT 0x08 /* Don't use cache */
47 #define SD_RES_SUCCESS 0x00 /* Success */
48 #define SD_RES_UNKNOWN 0x01 /* Unknown error */
49 #define SD_RES_NO_OBJ 0x02 /* No object found */
50 #define SD_RES_EIO 0x03 /* I/O error */
51 #define SD_RES_VDI_EXIST 0x04 /* Vdi exists already */
52 #define SD_RES_INVALID_PARMS 0x05 /* Invalid parameters */
53 #define SD_RES_SYSTEM_ERROR 0x06 /* System error */
54 #define SD_RES_VDI_LOCKED 0x07 /* Vdi is locked */
55 #define SD_RES_NO_VDI 0x08 /* No vdi found */
56 #define SD_RES_NO_BASE_VDI 0x09 /* No base vdi found */
57 #define SD_RES_VDI_READ 0x0A /* Cannot read requested vdi */
58 #define SD_RES_VDI_WRITE 0x0B /* Cannot write requested vdi */
59 #define SD_RES_BASE_VDI_READ 0x0C /* Cannot read base vdi */
60 #define SD_RES_BASE_VDI_WRITE 0x0D /* Cannot write base vdi */
61 #define SD_RES_NO_TAG 0x0E /* Requested tag is not found */
62 #define SD_RES_STARTUP 0x0F /* Sheepdog is on starting up */
63 #define SD_RES_VDI_NOT_LOCKED 0x10 /* Vdi is not locked */
64 #define SD_RES_SHUTDOWN 0x11 /* Sheepdog is shutting down */
65 #define SD_RES_NO_MEM 0x12 /* Cannot allocate memory */
66 #define SD_RES_FULL_VDI 0x13 /* we already have the maximum vdis */
67 #define SD_RES_VER_MISMATCH 0x14 /* Protocol version mismatch */
68 #define SD_RES_NO_SPACE 0x15 /* Server has no room for new objects */
69 #define SD_RES_WAIT_FOR_FORMAT 0x16 /* Waiting for a format operation */
70 #define SD_RES_WAIT_FOR_JOIN 0x17 /* Waiting for other nodes joining */
71 #define SD_RES_JOIN_FAILED 0x18 /* Target node had failed to join sheepdog */
72 #define SD_RES_HALT 0x19 /* Sheepdog is stopped serving IO request */
73 #define SD_RES_READONLY 0x1A /* Object is read-only */
78 * 0 - 19 (20 bits): data object space
79 * 20 - 31 (12 bits): reserved data object space
80 * 32 - 55 (24 bits): vdi object space
81 * 56 - 59 ( 4 bits): reserved vdi object space
82 * 60 - 63 ( 4 bits): object type identifier space
85 #define VDI_SPACE_SHIFT 32
86 #define VDI_BIT (UINT64_C(1) << 63)
87 #define VMSTATE_BIT (UINT64_C(1) << 62)
88 #define MAX_DATA_OBJS (UINT64_C(1) << 20)
89 #define MAX_CHILDREN 1024
90 #define SD_MAX_VDI_LEN 256
91 #define SD_MAX_VDI_TAG_LEN 256
92 #define SD_NR_VDIS (1U << 24)
93 #define SD_DATA_OBJ_SIZE (UINT64_C(1) << 22)
94 #define SD_MAX_VDI_SIZE (SD_DATA_OBJ_SIZE * MAX_DATA_OBJS)
95 #define SD_DEFAULT_BLOCK_SIZE_SHIFT 22
97 * For erasure coding, we use at most SD_EC_MAX_STRIP for data strips and
98 * (SD_EC_MAX_STRIP - 1) for parity strips
100 * SD_MAX_COPIES is sum of number of data strips and parity strips.
102 #define SD_EC_MAX_STRIP 16
103 #define SD_MAX_COPIES (SD_EC_MAX_STRIP * 2 - 1)
105 #define SD_INODE_SIZE (sizeof(SheepdogInode))
106 #define CURRENT_VDI_ID 0
108 #define LOCK_TYPE_NORMAL 0
109 #define LOCK_TYPE_SHARED 1 /* for iSCSI multipath */
111 typedef struct SheepdogReq
{
117 uint32_t data_length
;
118 uint32_t opcode_specific
[8];
121 typedef struct SheepdogRsp
{
127 uint32_t data_length
;
129 uint32_t opcode_specific
[7];
132 typedef struct SheepdogObjReq
{
138 uint32_t data_length
;
147 typedef struct SheepdogObjRsp
{
153 uint32_t data_length
;
161 typedef struct SheepdogVdiReq
{
167 uint32_t data_length
;
169 uint32_t base_vdi_id
;
172 uint8_t store_policy
;
173 uint8_t block_size_shift
;
179 typedef struct SheepdogVdiRsp
{
185 uint32_t data_length
;
192 typedef struct SheepdogClusterRsp
{
198 uint32_t data_length
;
202 uint8_t block_size_shift
;
205 } SheepdogClusterRsp
;
207 typedef struct SheepdogInode
{
208 char name
[SD_MAX_VDI_LEN
];
209 char tag
[SD_MAX_VDI_TAG_LEN
];
212 uint64_t vm_clock_nsec
;
214 uint64_t vm_state_size
;
215 uint16_t copy_policy
;
217 uint8_t block_size_shift
;
220 uint32_t parent_vdi_id
;
221 uint32_t child_vdi_id
[MAX_CHILDREN
];
222 uint32_t data_vdi_id
[MAX_DATA_OBJS
];
225 #define SD_INODE_HEADER_SIZE offsetof(SheepdogInode, data_vdi_id)
228 * 64 bit FNV-1a non-zero initial basis
230 #define FNV1A_64_INIT ((uint64_t)0xcbf29ce484222325ULL)
233 * 64 bit Fowler/Noll/Vo FNV-1a hash code
235 static inline uint64_t fnv_64a_buf(void *buf
, size_t len
, uint64_t hval
)
237 unsigned char *bp
= buf
;
238 unsigned char *be
= bp
+ len
;
240 hval
^= (uint64_t) *bp
++;
241 hval
+= (hval
<< 1) + (hval
<< 4) + (hval
<< 5) +
242 (hval
<< 7) + (hval
<< 8) + (hval
<< 40);
247 static inline bool is_data_obj_writable(SheepdogInode
*inode
, unsigned int idx
)
249 return inode
->vdi_id
== inode
->data_vdi_id
[idx
];
252 static inline bool is_data_obj(uint64_t oid
)
254 return !(VDI_BIT
& oid
);
257 static inline uint64_t data_oid_to_idx(uint64_t oid
)
259 return oid
& (MAX_DATA_OBJS
- 1);
262 static inline uint32_t oid_to_vid(uint64_t oid
)
264 return (oid
& ~VDI_BIT
) >> VDI_SPACE_SHIFT
;
267 static inline uint64_t vid_to_vdi_oid(uint32_t vid
)
269 return VDI_BIT
| ((uint64_t)vid
<< VDI_SPACE_SHIFT
);
272 static inline uint64_t vid_to_vmstate_oid(uint32_t vid
, uint32_t idx
)
274 return VMSTATE_BIT
| ((uint64_t)vid
<< VDI_SPACE_SHIFT
) | idx
;
277 static inline uint64_t vid_to_data_oid(uint32_t vid
, uint32_t idx
)
279 return ((uint64_t)vid
<< VDI_SPACE_SHIFT
) | idx
;
282 static inline bool is_snapshot(struct SheepdogInode
*inode
)
284 return !!inode
->snap_ctime
;
287 static inline size_t count_data_objs(const struct SheepdogInode
*inode
)
289 return DIV_ROUND_UP(inode
->vdi_size
,
290 (1UL << inode
->block_size_shift
));
295 #define DPRINTF(fmt, args...) \
297 fprintf(stdout, "%s %d: " fmt, __func__, __LINE__, ##args); \
300 #define DPRINTF(fmt, args...)
303 typedef struct SheepdogAIOCB SheepdogAIOCB
;
305 typedef struct AIOReq
{
306 SheepdogAIOCB
*aiocb
;
307 unsigned int iov_offset
;
312 unsigned int data_len
;
317 QLIST_ENTRY(AIOReq
) aio_siblings
;
327 #define AIOCBOverlapping(x, y) \
328 (!(x->max_affect_data_idx < y->min_affect_data_idx \
329 || y->max_affect_data_idx < x->min_affect_data_idx))
331 struct SheepdogAIOCB
{
340 enum AIOCBState aiocb_type
;
342 Coroutine
*coroutine
;
343 void (*aio_done_func
)(SheepdogAIOCB
*);
348 uint32_t min_affect_data_idx
;
349 uint32_t max_affect_data_idx
;
352 * The difference between affect_data_idx and dirty_data_idx:
353 * affect_data_idx represents range of index of all request types.
354 * dirty_data_idx represents range of index updated by COW requests.
355 * dirty_data_idx is used for updating an inode object.
357 uint32_t min_dirty_data_idx
;
358 uint32_t max_dirty_data_idx
;
360 QLIST_ENTRY(SheepdogAIOCB
) aiocb_siblings
;
363 typedef struct BDRVSheepdogState
{
364 BlockDriverState
*bs
;
365 AioContext
*aio_context
;
369 char name
[SD_MAX_VDI_LEN
];
371 uint32_t cache_flags
;
372 bool discard_supported
;
382 uint32_t aioreq_seq_num
;
384 /* Every aio request must be linked to either of these queues. */
385 QLIST_HEAD(inflight_aio_head
, AIOReq
) inflight_aio_head
;
386 QLIST_HEAD(failed_aio_head
, AIOReq
) failed_aio_head
;
388 CoQueue overlapping_queue
;
389 QLIST_HEAD(inflight_aiocb_head
, SheepdogAIOCB
) inflight_aiocb_head
;
392 typedef struct BDRVSheepdogReopenState
{
395 } BDRVSheepdogReopenState
;
397 static const char * sd_strerror(int err
)
401 static const struct {
405 {SD_RES_SUCCESS
, "Success"},
406 {SD_RES_UNKNOWN
, "Unknown error"},
407 {SD_RES_NO_OBJ
, "No object found"},
408 {SD_RES_EIO
, "I/O error"},
409 {SD_RES_VDI_EXIST
, "VDI exists already"},
410 {SD_RES_INVALID_PARMS
, "Invalid parameters"},
411 {SD_RES_SYSTEM_ERROR
, "System error"},
412 {SD_RES_VDI_LOCKED
, "VDI is already locked"},
413 {SD_RES_NO_VDI
, "No vdi found"},
414 {SD_RES_NO_BASE_VDI
, "No base VDI found"},
415 {SD_RES_VDI_READ
, "Failed read the requested VDI"},
416 {SD_RES_VDI_WRITE
, "Failed to write the requested VDI"},
417 {SD_RES_BASE_VDI_READ
, "Failed to read the base VDI"},
418 {SD_RES_BASE_VDI_WRITE
, "Failed to write the base VDI"},
419 {SD_RES_NO_TAG
, "Failed to find the requested tag"},
420 {SD_RES_STARTUP
, "The system is still booting"},
421 {SD_RES_VDI_NOT_LOCKED
, "VDI isn't locked"},
422 {SD_RES_SHUTDOWN
, "The system is shutting down"},
423 {SD_RES_NO_MEM
, "Out of memory on the server"},
424 {SD_RES_FULL_VDI
, "We already have the maximum vdis"},
425 {SD_RES_VER_MISMATCH
, "Protocol version mismatch"},
426 {SD_RES_NO_SPACE
, "Server has no space for new objects"},
427 {SD_RES_WAIT_FOR_FORMAT
, "Sheepdog is waiting for a format operation"},
428 {SD_RES_WAIT_FOR_JOIN
, "Sheepdog is waiting for other nodes joining"},
429 {SD_RES_JOIN_FAILED
, "Target node had failed to join sheepdog"},
430 {SD_RES_HALT
, "Sheepdog is stopped serving IO request"},
431 {SD_RES_READONLY
, "Object is read-only"},
434 for (i
= 0; i
< ARRAY_SIZE(errors
); ++i
) {
435 if (errors
[i
].err
== err
) {
436 return errors
[i
].desc
;
440 return "Invalid error code";
444 * Sheepdog I/O handling:
446 * 1. In sd_co_rw_vector, we send the I/O requests to the server and
447 * link the requests to the inflight_list in the
448 * BDRVSheepdogState. The function exits without waiting for
449 * receiving the response.
451 * 2. We receive the response in aio_read_response, the fd handler to
452 * the sheepdog connection. If metadata update is needed, we send
453 * the write request to the vdi object in sd_write_done, the write
454 * completion function. We switch back to sd_co_readv/writev after
455 * all the requests belonging to the AIOCB are finished.
458 static inline AIOReq
*alloc_aio_req(BDRVSheepdogState
*s
, SheepdogAIOCB
*acb
,
459 uint64_t oid
, unsigned int data_len
,
460 uint64_t offset
, uint8_t flags
, bool create
,
461 uint64_t base_oid
, unsigned int iov_offset
)
465 aio_req
= g_malloc(sizeof(*aio_req
));
466 aio_req
->aiocb
= acb
;
467 aio_req
->iov_offset
= iov_offset
;
469 aio_req
->base_oid
= base_oid
;
470 aio_req
->offset
= offset
;
471 aio_req
->data_len
= data_len
;
472 aio_req
->flags
= flags
;
473 aio_req
->id
= s
->aioreq_seq_num
++;
474 aio_req
->create
= create
;
480 static inline void free_aio_req(BDRVSheepdogState
*s
, AIOReq
*aio_req
)
482 SheepdogAIOCB
*acb
= aio_req
->aiocb
;
484 acb
->cancelable
= false;
485 QLIST_REMOVE(aio_req
, aio_siblings
);
491 static void coroutine_fn
sd_finish_aiocb(SheepdogAIOCB
*acb
)
493 qemu_coroutine_enter(acb
->coroutine
, NULL
);
498 * Check whether the specified acb can be canceled
500 * We can cancel aio when any request belonging to the acb is:
501 * - Not processed by the sheepdog server.
502 * - Not linked to the inflight queue.
504 static bool sd_acb_cancelable(const SheepdogAIOCB
*acb
)
506 BDRVSheepdogState
*s
= acb
->common
.bs
->opaque
;
509 if (!acb
->cancelable
) {
513 QLIST_FOREACH(aioreq
, &s
->inflight_aio_head
, aio_siblings
) {
514 if (aioreq
->aiocb
== acb
) {
522 static void sd_aio_cancel(BlockAIOCB
*blockacb
)
524 SheepdogAIOCB
*acb
= (SheepdogAIOCB
*)blockacb
;
525 BDRVSheepdogState
*s
= acb
->common
.bs
->opaque
;
526 AIOReq
*aioreq
, *next
;
528 if (sd_acb_cancelable(acb
)) {
529 /* Remove outstanding requests from failed queue. */
530 QLIST_FOREACH_SAFE(aioreq
, &s
->failed_aio_head
, aio_siblings
,
532 if (aioreq
->aiocb
== acb
) {
533 free_aio_req(s
, aioreq
);
537 assert(acb
->nr_pending
== 0);
538 if (acb
->common
.cb
) {
539 acb
->common
.cb(acb
->common
.opaque
, -ECANCELED
);
541 sd_finish_aiocb(acb
);
545 static const AIOCBInfo sd_aiocb_info
= {
546 .aiocb_size
= sizeof(SheepdogAIOCB
),
547 .cancel_async
= sd_aio_cancel
,
550 static SheepdogAIOCB
*sd_aio_setup(BlockDriverState
*bs
, QEMUIOVector
*qiov
,
551 int64_t sector_num
, int nb_sectors
)
554 uint32_t object_size
;
555 BDRVSheepdogState
*s
= bs
->opaque
;
557 object_size
= (UINT32_C(1) << s
->inode
.block_size_shift
);
559 acb
= qemu_aio_get(&sd_aiocb_info
, bs
, NULL
, NULL
);
563 acb
->sector_num
= sector_num
;
564 acb
->nb_sectors
= nb_sectors
;
566 acb
->aio_done_func
= NULL
;
567 acb
->cancelable
= true;
568 acb
->coroutine
= qemu_coroutine_self();
572 acb
->min_affect_data_idx
= acb
->sector_num
* BDRV_SECTOR_SIZE
/ object_size
;
573 acb
->max_affect_data_idx
= (acb
->sector_num
* BDRV_SECTOR_SIZE
+
574 acb
->nb_sectors
* BDRV_SECTOR_SIZE
) / object_size
;
576 acb
->min_dirty_data_idx
= UINT32_MAX
;
577 acb
->max_dirty_data_idx
= 0;
582 /* Return -EIO in case of error, file descriptor on success */
583 static int connect_to_sdog(BDRVSheepdogState
*s
, Error
**errp
)
588 fd
= unix_connect(s
->host_spec
, errp
);
590 fd
= inet_connect(s
->host_spec
, errp
);
593 int ret
= socket_set_nodelay(fd
);
595 error_report("%s", strerror(errno
));
601 qemu_set_nonblock(fd
);
609 /* Return 0 on success and -errno in case of error */
610 static coroutine_fn
int send_co_req(int sockfd
, SheepdogReq
*hdr
, void *data
,
615 ret
= qemu_co_send(sockfd
, hdr
, sizeof(*hdr
));
616 if (ret
!= sizeof(*hdr
)) {
617 error_report("failed to send a req, %s", strerror(errno
));
621 ret
= qemu_co_send(sockfd
, data
, *wlen
);
623 error_report("failed to send a req, %s", strerror(errno
));
630 static void restart_co_req(void *opaque
)
632 Coroutine
*co
= opaque
;
634 qemu_coroutine_enter(co
, NULL
);
637 typedef struct SheepdogReqCo
{
639 AioContext
*aio_context
;
648 static coroutine_fn
void do_co_req(void *opaque
)
652 SheepdogReqCo
*srco
= opaque
;
653 int sockfd
= srco
->sockfd
;
654 SheepdogReq
*hdr
= srco
->hdr
;
655 void *data
= srco
->data
;
656 unsigned int *wlen
= srco
->wlen
;
657 unsigned int *rlen
= srco
->rlen
;
659 co
= qemu_coroutine_self();
660 aio_set_fd_handler(srco
->aio_context
, sockfd
, false,
661 NULL
, restart_co_req
, co
);
663 ret
= send_co_req(sockfd
, hdr
, data
, wlen
);
668 aio_set_fd_handler(srco
->aio_context
, sockfd
, false,
669 restart_co_req
, NULL
, co
);
671 ret
= qemu_co_recv(sockfd
, hdr
, sizeof(*hdr
));
672 if (ret
!= sizeof(*hdr
)) {
673 error_report("failed to get a rsp, %s", strerror(errno
));
678 if (*rlen
> hdr
->data_length
) {
679 *rlen
= hdr
->data_length
;
683 ret
= qemu_co_recv(sockfd
, data
, *rlen
);
685 error_report("failed to get the data, %s", strerror(errno
));
692 /* there is at most one request for this sockfd, so it is safe to
693 * set each handler to NULL. */
694 aio_set_fd_handler(srco
->aio_context
, sockfd
, false,
698 srco
->finished
= true;
702 * Send the request to the sheep in a synchronous manner.
704 * Return 0 on success, -errno in case of error.
706 static int do_req(int sockfd
, AioContext
*aio_context
, SheepdogReq
*hdr
,
707 void *data
, unsigned int *wlen
, unsigned int *rlen
)
710 SheepdogReqCo srco
= {
712 .aio_context
= aio_context
,
721 if (qemu_in_coroutine()) {
724 co
= qemu_coroutine_create(do_co_req
);
725 qemu_coroutine_enter(co
, &srco
);
726 while (!srco
.finished
) {
727 aio_poll(aio_context
, true);
734 static void coroutine_fn
add_aio_request(BDRVSheepdogState
*s
, AIOReq
*aio_req
,
735 struct iovec
*iov
, int niov
,
736 enum AIOCBState aiocb_type
);
737 static void coroutine_fn
resend_aioreq(BDRVSheepdogState
*s
, AIOReq
*aio_req
);
738 static int reload_inode(BDRVSheepdogState
*s
, uint32_t snapid
, const char *tag
);
739 static int get_sheep_fd(BDRVSheepdogState
*s
, Error
**errp
);
740 static void co_write_request(void *opaque
);
742 static coroutine_fn
void reconnect_to_sdog(void *opaque
)
744 BDRVSheepdogState
*s
= opaque
;
745 AIOReq
*aio_req
, *next
;
747 aio_set_fd_handler(s
->aio_context
, s
->fd
, false, NULL
,
752 /* Wait for outstanding write requests to be completed. */
753 while (s
->co_send
!= NULL
) {
754 co_write_request(opaque
);
757 /* Try to reconnect the sheepdog server every one second. */
759 Error
*local_err
= NULL
;
760 s
->fd
= get_sheep_fd(s
, &local_err
);
762 DPRINTF("Wait for connection to be established\n");
763 error_report_err(local_err
);
764 co_aio_sleep_ns(bdrv_get_aio_context(s
->bs
), QEMU_CLOCK_REALTIME
,
770 * Now we have to resend all the request in the inflight queue. However,
771 * resend_aioreq() can yield and newly created requests can be added to the
772 * inflight queue before the coroutine is resumed. To avoid mixing them, we
773 * have to move all the inflight requests to the failed queue before
774 * resend_aioreq() is called.
776 QLIST_FOREACH_SAFE(aio_req
, &s
->inflight_aio_head
, aio_siblings
, next
) {
777 QLIST_REMOVE(aio_req
, aio_siblings
);
778 QLIST_INSERT_HEAD(&s
->failed_aio_head
, aio_req
, aio_siblings
);
781 /* Resend all the failed aio requests. */
782 while (!QLIST_EMPTY(&s
->failed_aio_head
)) {
783 aio_req
= QLIST_FIRST(&s
->failed_aio_head
);
784 QLIST_REMOVE(aio_req
, aio_siblings
);
785 QLIST_INSERT_HEAD(&s
->inflight_aio_head
, aio_req
, aio_siblings
);
786 resend_aioreq(s
, aio_req
);
791 * Receive responses of the I/O requests.
793 * This function is registered as a fd handler, and called from the
794 * main loop when s->fd is ready for reading responses.
796 static void coroutine_fn
aio_read_response(void *opaque
)
799 BDRVSheepdogState
*s
= opaque
;
802 AIOReq
*aio_req
= NULL
;
807 ret
= qemu_co_recv(fd
, &rsp
, sizeof(rsp
));
808 if (ret
!= sizeof(rsp
)) {
809 error_report("failed to get the header, %s", strerror(errno
));
813 /* find the right aio_req from the inflight aio list */
814 QLIST_FOREACH(aio_req
, &s
->inflight_aio_head
, aio_siblings
) {
815 if (aio_req
->id
== rsp
.id
) {
820 error_report("cannot find aio_req %x", rsp
.id
);
824 acb
= aio_req
->aiocb
;
826 switch (acb
->aiocb_type
) {
827 case AIOCB_WRITE_UDATA
:
828 /* this coroutine context is no longer suitable for co_recv
829 * because we may send data to update vdi objects */
831 if (!is_data_obj(aio_req
->oid
)) {
834 idx
= data_oid_to_idx(aio_req
->oid
);
836 if (aio_req
->create
) {
838 * If the object is newly created one, we need to update
839 * the vdi object (metadata object). min_dirty_data_idx
840 * and max_dirty_data_idx are changed to include updated
841 * index between them.
843 if (rsp
.result
== SD_RES_SUCCESS
) {
844 s
->inode
.data_vdi_id
[idx
] = s
->inode
.vdi_id
;
845 acb
->max_dirty_data_idx
= MAX(idx
, acb
->max_dirty_data_idx
);
846 acb
->min_dirty_data_idx
= MIN(idx
, acb
->min_dirty_data_idx
);
850 case AIOCB_READ_UDATA
:
851 ret
= qemu_co_recvv(fd
, acb
->qiov
->iov
, acb
->qiov
->niov
,
852 aio_req
->iov_offset
, rsp
.data_length
);
853 if (ret
!= rsp
.data_length
) {
854 error_report("failed to get the data, %s", strerror(errno
));
858 case AIOCB_FLUSH_CACHE
:
859 if (rsp
.result
== SD_RES_INVALID_PARMS
) {
860 DPRINTF("disable cache since the server doesn't support it\n");
861 s
->cache_flags
= SD_FLAG_CMD_DIRECT
;
862 rsp
.result
= SD_RES_SUCCESS
;
865 case AIOCB_DISCARD_OBJ
:
866 switch (rsp
.result
) {
867 case SD_RES_INVALID_PARMS
:
868 error_report("sheep(%s) doesn't support discard command",
870 rsp
.result
= SD_RES_SUCCESS
;
871 s
->discard_supported
= false;
878 switch (rsp
.result
) {
881 case SD_RES_READONLY
:
882 if (s
->inode
.vdi_id
== oid_to_vid(aio_req
->oid
)) {
883 ret
= reload_inode(s
, 0, "");
888 if (is_data_obj(aio_req
->oid
)) {
889 aio_req
->oid
= vid_to_data_oid(s
->inode
.vdi_id
,
890 data_oid_to_idx(aio_req
->oid
));
892 aio_req
->oid
= vid_to_vdi_oid(s
->inode
.vdi_id
);
894 resend_aioreq(s
, aio_req
);
898 error_report("%s", sd_strerror(rsp
.result
));
902 free_aio_req(s
, aio_req
);
903 if (!acb
->nr_pending
) {
905 * We've finished all requests which belong to the AIOCB, so
906 * we can switch back to sd_co_readv/writev now.
908 acb
->aio_done_func(acb
);
915 reconnect_to_sdog(opaque
);
918 static void co_read_response(void *opaque
)
920 BDRVSheepdogState
*s
= opaque
;
923 s
->co_recv
= qemu_coroutine_create(aio_read_response
);
926 qemu_coroutine_enter(s
->co_recv
, opaque
);
929 static void co_write_request(void *opaque
)
931 BDRVSheepdogState
*s
= opaque
;
933 qemu_coroutine_enter(s
->co_send
, NULL
);
937 * Return a socket descriptor to read/write objects.
939 * We cannot use this descriptor for other operations because
940 * the block driver may be on waiting response from the server.
942 static int get_sheep_fd(BDRVSheepdogState
*s
, Error
**errp
)
946 fd
= connect_to_sdog(s
, errp
);
951 aio_set_fd_handler(s
->aio_context
, fd
, false,
952 co_read_response
, NULL
, s
);
956 static int sd_parse_uri(BDRVSheepdogState
*s
, const char *filename
,
957 char *vdi
, uint32_t *snapid
, char *tag
)
960 QueryParams
*qp
= NULL
;
963 uri
= uri_parse(filename
);
969 if (!strcmp(uri
->scheme
, "sheepdog")) {
971 } else if (!strcmp(uri
->scheme
, "sheepdog+tcp")) {
973 } else if (!strcmp(uri
->scheme
, "sheepdog+unix")) {
980 if (uri
->path
== NULL
|| !strcmp(uri
->path
, "/")) {
984 pstrcpy(vdi
, SD_MAX_VDI_LEN
, uri
->path
+ 1);
986 qp
= query_params_parse(uri
->query
);
987 if (qp
->n
> 1 || (s
->is_unix
&& !qp
->n
) || (!s
->is_unix
&& qp
->n
)) {
993 /* sheepdog+unix:///vdiname?socket=path */
994 if (uri
->server
|| uri
->port
|| strcmp(qp
->p
[0].name
, "socket")) {
998 s
->host_spec
= g_strdup(qp
->p
[0].value
);
1000 /* sheepdog[+tcp]://[host:port]/vdiname */
1001 s
->host_spec
= g_strdup_printf("%s:%d", uri
->server
?: SD_DEFAULT_ADDR
,
1002 uri
->port
?: SD_DEFAULT_PORT
);
1006 if (uri
->fragment
) {
1007 *snapid
= strtoul(uri
->fragment
, NULL
, 10);
1009 pstrcpy(tag
, SD_MAX_VDI_TAG_LEN
, uri
->fragment
);
1012 *snapid
= CURRENT_VDI_ID
; /* search current vdi */
1017 query_params_free(qp
);
1024 * Parse a filename (old syntax)
1026 * filename must be one of the following formats:
1028 * 2. [vdiname]:[snapid]
1029 * 3. [vdiname]:[tag]
1030 * 4. [hostname]:[port]:[vdiname]
1031 * 5. [hostname]:[port]:[vdiname]:[snapid]
1032 * 6. [hostname]:[port]:[vdiname]:[tag]
1034 * You can boot from the snapshot images by specifying `snapid` or
1037 * You can run VMs outside the Sheepdog cluster by specifying
1038 * `hostname' and `port' (experimental).
1040 static int parse_vdiname(BDRVSheepdogState
*s
, const char *filename
,
1041 char *vdi
, uint32_t *snapid
, char *tag
)
1044 const char *host_spec
, *vdi_spec
;
1047 strstart(filename
, "sheepdog:", (const char **)&filename
);
1048 p
= q
= g_strdup(filename
);
1050 /* count the number of separators */
1060 /* use the first two tokens as host_spec. */
1073 p
= strchr(vdi_spec
, ':');
1078 uri
= g_strdup_printf("sheepdog://%s/%s", host_spec
, vdi_spec
);
1080 ret
= sd_parse_uri(s
, uri
, vdi
, snapid
, tag
);
1088 static int find_vdi_name(BDRVSheepdogState
*s
, const char *filename
,
1089 uint32_t snapid
, const char *tag
, uint32_t *vid
,
1090 bool lock
, Error
**errp
)
1094 SheepdogVdiRsp
*rsp
= (SheepdogVdiRsp
*)&hdr
;
1095 unsigned int wlen
, rlen
= 0;
1096 char buf
[SD_MAX_VDI_LEN
+ SD_MAX_VDI_TAG_LEN
];
1098 fd
= connect_to_sdog(s
, errp
);
1103 /* This pair of strncpy calls ensures that the buffer is zero-filled,
1104 * which is desirable since we'll soon be sending those bytes, and
1105 * don't want the send_req to read uninitialized data.
1107 strncpy(buf
, filename
, SD_MAX_VDI_LEN
);
1108 strncpy(buf
+ SD_MAX_VDI_LEN
, tag
, SD_MAX_VDI_TAG_LEN
);
1110 memset(&hdr
, 0, sizeof(hdr
));
1112 hdr
.opcode
= SD_OP_LOCK_VDI
;
1113 hdr
.type
= LOCK_TYPE_NORMAL
;
1115 hdr
.opcode
= SD_OP_GET_VDI_INFO
;
1117 wlen
= SD_MAX_VDI_LEN
+ SD_MAX_VDI_TAG_LEN
;
1118 hdr
.proto_ver
= SD_PROTO_VER
;
1119 hdr
.data_length
= wlen
;
1120 hdr
.snapid
= snapid
;
1121 hdr
.flags
= SD_FLAG_CMD_WRITE
;
1123 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&hdr
, buf
, &wlen
, &rlen
);
1125 error_setg_errno(errp
, -ret
, "cannot get vdi info");
1129 if (rsp
->result
!= SD_RES_SUCCESS
) {
1130 error_setg(errp
, "cannot get vdi info, %s, %s %" PRIu32
" %s",
1131 sd_strerror(rsp
->result
), filename
, snapid
, tag
);
1132 if (rsp
->result
== SD_RES_NO_VDI
) {
1134 } else if (rsp
->result
== SD_RES_VDI_LOCKED
) {
1149 static void coroutine_fn
add_aio_request(BDRVSheepdogState
*s
, AIOReq
*aio_req
,
1150 struct iovec
*iov
, int niov
,
1151 enum AIOCBState aiocb_type
)
1153 int nr_copies
= s
->inode
.nr_copies
;
1155 unsigned int wlen
= 0;
1157 uint64_t oid
= aio_req
->oid
;
1158 unsigned int datalen
= aio_req
->data_len
;
1159 uint64_t offset
= aio_req
->offset
;
1160 uint8_t flags
= aio_req
->flags
;
1161 uint64_t old_oid
= aio_req
->base_oid
;
1162 bool create
= aio_req
->create
;
1165 error_report("bug");
1168 memset(&hdr
, 0, sizeof(hdr
));
1170 switch (aiocb_type
) {
1171 case AIOCB_FLUSH_CACHE
:
1172 hdr
.opcode
= SD_OP_FLUSH_VDI
;
1174 case AIOCB_READ_UDATA
:
1175 hdr
.opcode
= SD_OP_READ_OBJ
;
1178 case AIOCB_WRITE_UDATA
:
1180 hdr
.opcode
= SD_OP_CREATE_AND_WRITE_OBJ
;
1182 hdr
.opcode
= SD_OP_WRITE_OBJ
;
1185 hdr
.flags
= SD_FLAG_CMD_WRITE
| flags
;
1187 case AIOCB_DISCARD_OBJ
:
1188 hdr
.opcode
= SD_OP_WRITE_OBJ
;
1189 hdr
.flags
= SD_FLAG_CMD_WRITE
| flags
;
1190 s
->inode
.data_vdi_id
[data_oid_to_idx(oid
)] = 0;
1191 offset
= offsetof(SheepdogInode
,
1192 data_vdi_id
[data_oid_to_idx(oid
)]);
1193 oid
= vid_to_vdi_oid(s
->inode
.vdi_id
);
1194 wlen
= datalen
= sizeof(uint32_t);
1198 if (s
->cache_flags
) {
1199 hdr
.flags
|= s
->cache_flags
;
1203 hdr
.cow_oid
= old_oid
;
1204 hdr
.copies
= s
->inode
.nr_copies
;
1206 hdr
.data_length
= datalen
;
1207 hdr
.offset
= offset
;
1209 hdr
.id
= aio_req
->id
;
1211 qemu_co_mutex_lock(&s
->lock
);
1212 s
->co_send
= qemu_coroutine_self();
1213 aio_set_fd_handler(s
->aio_context
, s
->fd
, false,
1214 co_read_response
, co_write_request
, s
);
1215 socket_set_cork(s
->fd
, 1);
1218 ret
= qemu_co_send(s
->fd
, &hdr
, sizeof(hdr
));
1219 if (ret
!= sizeof(hdr
)) {
1220 error_report("failed to send a req, %s", strerror(errno
));
1225 ret
= qemu_co_sendv(s
->fd
, iov
, niov
, aio_req
->iov_offset
, wlen
);
1227 error_report("failed to send a data, %s", strerror(errno
));
1231 socket_set_cork(s
->fd
, 0);
1232 aio_set_fd_handler(s
->aio_context
, s
->fd
, false,
1233 co_read_response
, NULL
, s
);
1235 qemu_co_mutex_unlock(&s
->lock
);
1238 static int read_write_object(int fd
, AioContext
*aio_context
, char *buf
,
1239 uint64_t oid
, uint8_t copies
,
1240 unsigned int datalen
, uint64_t offset
,
1241 bool write
, bool create
, uint32_t cache_flags
)
1244 SheepdogObjRsp
*rsp
= (SheepdogObjRsp
*)&hdr
;
1245 unsigned int wlen
, rlen
;
1248 memset(&hdr
, 0, sizeof(hdr
));
1253 hdr
.flags
= SD_FLAG_CMD_WRITE
;
1255 hdr
.opcode
= SD_OP_CREATE_AND_WRITE_OBJ
;
1257 hdr
.opcode
= SD_OP_WRITE_OBJ
;
1262 hdr
.opcode
= SD_OP_READ_OBJ
;
1265 hdr
.flags
|= cache_flags
;
1268 hdr
.data_length
= datalen
;
1269 hdr
.offset
= offset
;
1270 hdr
.copies
= copies
;
1272 ret
= do_req(fd
, aio_context
, (SheepdogReq
*)&hdr
, buf
, &wlen
, &rlen
);
1274 error_report("failed to send a request to the sheep");
1278 switch (rsp
->result
) {
1279 case SD_RES_SUCCESS
:
1282 error_report("%s", sd_strerror(rsp
->result
));
1287 static int read_object(int fd
, AioContext
*aio_context
, char *buf
,
1288 uint64_t oid
, uint8_t copies
,
1289 unsigned int datalen
, uint64_t offset
,
1290 uint32_t cache_flags
)
1292 return read_write_object(fd
, aio_context
, buf
, oid
, copies
,
1293 datalen
, offset
, false,
1294 false, cache_flags
);
1297 static int write_object(int fd
, AioContext
*aio_context
, char *buf
,
1298 uint64_t oid
, uint8_t copies
,
1299 unsigned int datalen
, uint64_t offset
, bool create
,
1300 uint32_t cache_flags
)
1302 return read_write_object(fd
, aio_context
, buf
, oid
, copies
,
1303 datalen
, offset
, true,
1304 create
, cache_flags
);
1307 /* update inode with the latest state */
1308 static int reload_inode(BDRVSheepdogState
*s
, uint32_t snapid
, const char *tag
)
1310 Error
*local_err
= NULL
;
1311 SheepdogInode
*inode
;
1315 fd
= connect_to_sdog(s
, &local_err
);
1317 error_report_err(local_err
);
1321 inode
= g_malloc(SD_INODE_HEADER_SIZE
);
1323 ret
= find_vdi_name(s
, s
->name
, snapid
, tag
, &vid
, false, &local_err
);
1325 error_report_err(local_err
);
1329 ret
= read_object(fd
, s
->aio_context
, (char *)inode
, vid_to_vdi_oid(vid
),
1330 s
->inode
.nr_copies
, SD_INODE_HEADER_SIZE
, 0,
1336 if (inode
->vdi_id
!= s
->inode
.vdi_id
) {
1337 memcpy(&s
->inode
, inode
, SD_INODE_HEADER_SIZE
);
1347 static void coroutine_fn
resend_aioreq(BDRVSheepdogState
*s
, AIOReq
*aio_req
)
1349 SheepdogAIOCB
*acb
= aio_req
->aiocb
;
1351 aio_req
->create
= false;
1353 /* check whether this request becomes a CoW one */
1354 if (acb
->aiocb_type
== AIOCB_WRITE_UDATA
&& is_data_obj(aio_req
->oid
)) {
1355 int idx
= data_oid_to_idx(aio_req
->oid
);
1357 if (is_data_obj_writable(&s
->inode
, idx
)) {
1361 if (s
->inode
.data_vdi_id
[idx
]) {
1362 aio_req
->base_oid
= vid_to_data_oid(s
->inode
.data_vdi_id
[idx
], idx
);
1363 aio_req
->flags
|= SD_FLAG_CMD_COW
;
1365 aio_req
->create
= true;
1368 if (is_data_obj(aio_req
->oid
)) {
1369 add_aio_request(s
, aio_req
, acb
->qiov
->iov
, acb
->qiov
->niov
,
1373 iov
.iov_base
= &s
->inode
;
1374 iov
.iov_len
= sizeof(s
->inode
);
1375 add_aio_request(s
, aio_req
, &iov
, 1, AIOCB_WRITE_UDATA
);
1379 static void sd_detach_aio_context(BlockDriverState
*bs
)
1381 BDRVSheepdogState
*s
= bs
->opaque
;
1383 aio_set_fd_handler(s
->aio_context
, s
->fd
, false, NULL
,
1387 static void sd_attach_aio_context(BlockDriverState
*bs
,
1388 AioContext
*new_context
)
1390 BDRVSheepdogState
*s
= bs
->opaque
;
1392 s
->aio_context
= new_context
;
1393 aio_set_fd_handler(new_context
, s
->fd
, false,
1394 co_read_response
, NULL
, s
);
1397 /* TODO Convert to fine grained options */
1398 static QemuOptsList runtime_opts
= {
1400 .head
= QTAILQ_HEAD_INITIALIZER(runtime_opts
.head
),
1404 .type
= QEMU_OPT_STRING
,
1405 .help
= "URL to the sheepdog image",
1407 { /* end of list */ }
1411 static int sd_open(BlockDriverState
*bs
, QDict
*options
, int flags
,
1416 BDRVSheepdogState
*s
= bs
->opaque
;
1417 char vdi
[SD_MAX_VDI_LEN
], tag
[SD_MAX_VDI_TAG_LEN
];
1421 Error
*local_err
= NULL
;
1422 const char *filename
;
1425 s
->aio_context
= bdrv_get_aio_context(bs
);
1427 opts
= qemu_opts_create(&runtime_opts
, NULL
, 0, &error_abort
);
1428 qemu_opts_absorb_qdict(opts
, options
, &local_err
);
1430 error_propagate(errp
, local_err
);
1435 filename
= qemu_opt_get(opts
, "filename");
1437 QLIST_INIT(&s
->inflight_aio_head
);
1438 QLIST_INIT(&s
->failed_aio_head
);
1439 QLIST_INIT(&s
->inflight_aiocb_head
);
1442 memset(vdi
, 0, sizeof(vdi
));
1443 memset(tag
, 0, sizeof(tag
));
1445 if (strstr(filename
, "://")) {
1446 ret
= sd_parse_uri(s
, filename
, vdi
, &snapid
, tag
);
1448 ret
= parse_vdiname(s
, filename
, vdi
, &snapid
, tag
);
1451 error_setg(errp
, "Can't parse filename");
1454 s
->fd
= get_sheep_fd(s
, errp
);
1460 ret
= find_vdi_name(s
, vdi
, snapid
, tag
, &vid
, true, errp
);
1466 * QEMU block layer emulates writethrough cache as 'writeback + flush', so
1467 * we always set SD_FLAG_CMD_CACHE (writeback cache) as default.
1469 s
->cache_flags
= SD_FLAG_CMD_CACHE
;
1470 if (flags
& BDRV_O_NOCACHE
) {
1471 s
->cache_flags
= SD_FLAG_CMD_DIRECT
;
1473 s
->discard_supported
= true;
1475 if (snapid
|| tag
[0] != '\0') {
1476 DPRINTF("%" PRIx32
" snapshot inode was open.\n", vid
);
1477 s
->is_snapshot
= true;
1480 fd
= connect_to_sdog(s
, errp
);
1486 buf
= g_malloc(SD_INODE_SIZE
);
1487 ret
= read_object(fd
, s
->aio_context
, buf
, vid_to_vdi_oid(vid
),
1488 0, SD_INODE_SIZE
, 0, s
->cache_flags
);
1493 error_setg(errp
, "Can't read snapshot inode");
1497 memcpy(&s
->inode
, buf
, sizeof(s
->inode
));
1499 bs
->total_sectors
= s
->inode
.vdi_size
/ BDRV_SECTOR_SIZE
;
1500 pstrcpy(s
->name
, sizeof(s
->name
), vdi
);
1501 qemu_co_mutex_init(&s
->lock
);
1502 qemu_co_queue_init(&s
->overlapping_queue
);
1503 qemu_opts_del(opts
);
1507 aio_set_fd_handler(bdrv_get_aio_context(bs
), s
->fd
,
1508 false, NULL
, NULL
, NULL
);
1512 qemu_opts_del(opts
);
1517 static int sd_reopen_prepare(BDRVReopenState
*state
, BlockReopenQueue
*queue
,
1520 BDRVSheepdogState
*s
= state
->bs
->opaque
;
1521 BDRVSheepdogReopenState
*re_s
;
1524 re_s
= state
->opaque
= g_new0(BDRVSheepdogReopenState
, 1);
1526 re_s
->cache_flags
= SD_FLAG_CMD_CACHE
;
1527 if (state
->flags
& BDRV_O_NOCACHE
) {
1528 re_s
->cache_flags
= SD_FLAG_CMD_DIRECT
;
1531 re_s
->fd
= get_sheep_fd(s
, errp
);
1540 static void sd_reopen_commit(BDRVReopenState
*state
)
1542 BDRVSheepdogReopenState
*re_s
= state
->opaque
;
1543 BDRVSheepdogState
*s
= state
->bs
->opaque
;
1546 aio_set_fd_handler(s
->aio_context
, s
->fd
, false,
1552 s
->cache_flags
= re_s
->cache_flags
;
1554 g_free(state
->opaque
);
1555 state
->opaque
= NULL
;
1560 static void sd_reopen_abort(BDRVReopenState
*state
)
1562 BDRVSheepdogReopenState
*re_s
= state
->opaque
;
1563 BDRVSheepdogState
*s
= state
->bs
->opaque
;
1570 aio_set_fd_handler(s
->aio_context
, re_s
->fd
, false,
1572 closesocket(re_s
->fd
);
1575 g_free(state
->opaque
);
1576 state
->opaque
= NULL
;
1581 static int do_sd_create(BDRVSheepdogState
*s
, uint32_t *vdi_id
, int snapshot
,
1585 SheepdogVdiRsp
*rsp
= (SheepdogVdiRsp
*)&hdr
;
1587 unsigned int wlen
, rlen
= 0;
1588 char buf
[SD_MAX_VDI_LEN
];
1590 fd
= connect_to_sdog(s
, errp
);
1595 /* FIXME: would it be better to fail (e.g., return -EIO) when filename
1596 * does not fit in buf? For now, just truncate and avoid buffer overrun.
1598 memset(buf
, 0, sizeof(buf
));
1599 pstrcpy(buf
, sizeof(buf
), s
->name
);
1601 memset(&hdr
, 0, sizeof(hdr
));
1602 hdr
.opcode
= SD_OP_NEW_VDI
;
1603 hdr
.base_vdi_id
= s
->inode
.vdi_id
;
1605 wlen
= SD_MAX_VDI_LEN
;
1607 hdr
.flags
= SD_FLAG_CMD_WRITE
;
1608 hdr
.snapid
= snapshot
;
1610 hdr
.data_length
= wlen
;
1611 hdr
.vdi_size
= s
->inode
.vdi_size
;
1612 hdr
.copy_policy
= s
->inode
.copy_policy
;
1613 hdr
.copies
= s
->inode
.nr_copies
;
1614 hdr
.block_size_shift
= s
->inode
.block_size_shift
;
1616 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&hdr
, buf
, &wlen
, &rlen
);
1621 error_setg_errno(errp
, -ret
, "create failed");
1625 if (rsp
->result
!= SD_RES_SUCCESS
) {
1626 error_setg(errp
, "%s, %s", sd_strerror(rsp
->result
), s
->inode
.name
);
1631 *vdi_id
= rsp
->vdi_id
;
1637 static int sd_prealloc(const char *filename
, Error
**errp
)
1639 BlockDriverState
*bs
= NULL
;
1640 BDRVSheepdogState
*base
= NULL
;
1641 unsigned long buf_size
;
1642 uint32_t idx
, max_idx
;
1643 uint32_t object_size
;
1648 ret
= bdrv_open(&bs
, filename
, NULL
, NULL
, BDRV_O_RDWR
| BDRV_O_PROTOCOL
,
1651 goto out_with_err_set
;
1654 vdi_size
= bdrv_getlength(bs
);
1661 object_size
= (UINT32_C(1) << base
->inode
.block_size_shift
);
1662 buf_size
= MIN(object_size
, SD_DATA_OBJ_SIZE
);
1663 buf
= g_malloc0(buf_size
);
1665 max_idx
= DIV_ROUND_UP(vdi_size
, buf_size
);
1667 for (idx
= 0; idx
< max_idx
; idx
++) {
1669 * The created image can be a cloned image, so we need to read
1670 * a data from the source image.
1672 ret
= bdrv_pread(bs
, idx
* buf_size
, buf
, buf_size
);
1676 ret
= bdrv_pwrite(bs
, idx
* buf_size
, buf
, buf_size
);
1684 error_setg_errno(errp
, -ret
, "Can't pre-allocate");
1696 * Sheepdog support two kinds of redundancy, full replication and erasure
1699 * # create a fully replicated vdi with x copies
1700 * -o redundancy=x (1 <= x <= SD_MAX_COPIES)
1702 * # create a erasure coded vdi with x data strips and y parity strips
1703 * -o redundancy=x:y (x must be one of {2,4,8,16} and 1 <= y < SD_EC_MAX_STRIP)
1705 static int parse_redundancy(BDRVSheepdogState
*s
, const char *opt
)
1707 struct SheepdogInode
*inode
= &s
->inode
;
1708 const char *n1
, *n2
;
1712 pstrcpy(p
, sizeof(p
), opt
);
1713 n1
= strtok(p
, ":");
1714 n2
= strtok(NULL
, ":");
1720 copy
= strtol(n1
, NULL
, 10);
1721 if (copy
> SD_MAX_COPIES
|| copy
< 1) {
1725 inode
->copy_policy
= 0;
1726 inode
->nr_copies
= copy
;
1730 if (copy
!= 2 && copy
!= 4 && copy
!= 8 && copy
!= 16) {
1734 parity
= strtol(n2
, NULL
, 10);
1735 if (parity
>= SD_EC_MAX_STRIP
|| parity
< 1) {
1740 * 4 bits for parity and 4 bits for data.
1741 * We have to compress upper data bits because it can't represent 16
1743 inode
->copy_policy
= ((copy
/ 2) << 4) + parity
;
1744 inode
->nr_copies
= copy
+ parity
;
1749 static int parse_block_size_shift(BDRVSheepdogState
*s
, QemuOpts
*opt
)
1751 struct SheepdogInode
*inode
= &s
->inode
;
1752 uint64_t object_size
;
1755 object_size
= qemu_opt_get_size_del(opt
, BLOCK_OPT_OBJECT_SIZE
, 0);
1757 if ((object_size
- 1) & object_size
) { /* not a power of 2? */
1760 obj_order
= ctz32(object_size
);
1761 if (obj_order
< 20 || obj_order
> 31) {
1764 inode
->block_size_shift
= (uint8_t)obj_order
;
1770 static int sd_create(const char *filename
, QemuOpts
*opts
,
1775 char *backing_file
= NULL
;
1777 BDRVSheepdogState
*s
;
1778 char tag
[SD_MAX_VDI_TAG_LEN
];
1780 uint64_t max_vdi_size
;
1781 bool prealloc
= false;
1783 s
= g_new0(BDRVSheepdogState
, 1);
1785 memset(tag
, 0, sizeof(tag
));
1786 if (strstr(filename
, "://")) {
1787 ret
= sd_parse_uri(s
, filename
, s
->name
, &snapid
, tag
);
1789 ret
= parse_vdiname(s
, filename
, s
->name
, &snapid
, tag
);
1792 error_setg(errp
, "Can't parse filename");
1796 s
->inode
.vdi_size
= ROUND_UP(qemu_opt_get_size_del(opts
, BLOCK_OPT_SIZE
, 0),
1798 backing_file
= qemu_opt_get_del(opts
, BLOCK_OPT_BACKING_FILE
);
1799 buf
= qemu_opt_get_del(opts
, BLOCK_OPT_PREALLOC
);
1800 if (!buf
|| !strcmp(buf
, "off")) {
1802 } else if (!strcmp(buf
, "full")) {
1805 error_setg(errp
, "Invalid preallocation mode: '%s'", buf
);
1811 buf
= qemu_opt_get_del(opts
, BLOCK_OPT_REDUNDANCY
);
1813 ret
= parse_redundancy(s
, buf
);
1815 error_setg(errp
, "Invalid redundancy mode: '%s'", buf
);
1819 ret
= parse_block_size_shift(s
, opts
);
1821 error_setg(errp
, "Invalid object_size."
1822 " obect_size needs to be power of 2"
1823 " and be limited from 2^20 to 2^31");
1828 BlockDriverState
*bs
;
1829 BDRVSheepdogState
*base
;
1832 /* Currently, only Sheepdog backing image is supported. */
1833 drv
= bdrv_find_protocol(backing_file
, true, NULL
);
1834 if (!drv
|| strcmp(drv
->protocol_name
, "sheepdog") != 0) {
1835 error_setg(errp
, "backing_file must be a sheepdog image");
1841 ret
= bdrv_open(&bs
, backing_file
, NULL
, NULL
, BDRV_O_PROTOCOL
, errp
);
1848 if (!is_snapshot(&base
->inode
)) {
1849 error_setg(errp
, "cannot clone from a non snapshot vdi");
1854 s
->inode
.vdi_id
= base
->inode
.vdi_id
;
1858 s
->aio_context
= qemu_get_aio_context();
1860 /* if block_size_shift is not specified, get cluster default value */
1861 if (s
->inode
.block_size_shift
== 0) {
1863 SheepdogClusterRsp
*rsp
= (SheepdogClusterRsp
*)&hdr
;
1864 Error
*local_err
= NULL
;
1866 unsigned int wlen
= 0, rlen
= 0;
1868 fd
= connect_to_sdog(s
, &local_err
);
1870 error_report_err(local_err
);
1875 memset(&hdr
, 0, sizeof(hdr
));
1876 hdr
.opcode
= SD_OP_GET_CLUSTER_DEFAULT
;
1877 hdr
.proto_ver
= SD_PROTO_VER
;
1879 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&hdr
,
1880 NULL
, &wlen
, &rlen
);
1883 error_setg_errno(errp
, -ret
, "failed to get cluster default");
1886 if (rsp
->result
== SD_RES_SUCCESS
) {
1887 s
->inode
.block_size_shift
= rsp
->block_size_shift
;
1889 s
->inode
.block_size_shift
= SD_DEFAULT_BLOCK_SIZE_SHIFT
;
1893 max_vdi_size
= (UINT64_C(1) << s
->inode
.block_size_shift
) * MAX_DATA_OBJS
;
1895 if (s
->inode
.vdi_size
> max_vdi_size
) {
1896 error_setg(errp
, "An image is too large."
1897 " The maximum image size is %"PRIu64
"GB",
1898 max_vdi_size
/ 1024 / 1024 / 1024);
1903 ret
= do_sd_create(s
, &vid
, 0, errp
);
1909 ret
= sd_prealloc(filename
, errp
);
1912 g_free(backing_file
);
1918 static void sd_close(BlockDriverState
*bs
)
1920 Error
*local_err
= NULL
;
1921 BDRVSheepdogState
*s
= bs
->opaque
;
1923 SheepdogVdiRsp
*rsp
= (SheepdogVdiRsp
*)&hdr
;
1924 unsigned int wlen
, rlen
= 0;
1927 DPRINTF("%s\n", s
->name
);
1929 fd
= connect_to_sdog(s
, &local_err
);
1931 error_report_err(local_err
);
1935 memset(&hdr
, 0, sizeof(hdr
));
1937 hdr
.opcode
= SD_OP_RELEASE_VDI
;
1938 hdr
.type
= LOCK_TYPE_NORMAL
;
1939 hdr
.base_vdi_id
= s
->inode
.vdi_id
;
1940 wlen
= strlen(s
->name
) + 1;
1941 hdr
.data_length
= wlen
;
1942 hdr
.flags
= SD_FLAG_CMD_WRITE
;
1944 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&hdr
,
1945 s
->name
, &wlen
, &rlen
);
1949 if (!ret
&& rsp
->result
!= SD_RES_SUCCESS
&&
1950 rsp
->result
!= SD_RES_VDI_NOT_LOCKED
) {
1951 error_report("%s, %s", sd_strerror(rsp
->result
), s
->name
);
1954 aio_set_fd_handler(bdrv_get_aio_context(bs
), s
->fd
,
1955 false, NULL
, NULL
, NULL
);
1957 g_free(s
->host_spec
);
1960 static int64_t sd_getlength(BlockDriverState
*bs
)
1962 BDRVSheepdogState
*s
= bs
->opaque
;
1964 return s
->inode
.vdi_size
;
1967 static int sd_truncate(BlockDriverState
*bs
, int64_t offset
)
1969 Error
*local_err
= NULL
;
1970 BDRVSheepdogState
*s
= bs
->opaque
;
1972 unsigned int datalen
;
1973 uint64_t max_vdi_size
;
1975 max_vdi_size
= (UINT64_C(1) << s
->inode
.block_size_shift
) * MAX_DATA_OBJS
;
1976 if (offset
< s
->inode
.vdi_size
) {
1977 error_report("shrinking is not supported");
1979 } else if (offset
> max_vdi_size
) {
1980 error_report("too big image size");
1984 fd
= connect_to_sdog(s
, &local_err
);
1986 error_report_err(local_err
);
1990 /* we don't need to update entire object */
1991 datalen
= SD_INODE_SIZE
- sizeof(s
->inode
.data_vdi_id
);
1992 s
->inode
.vdi_size
= offset
;
1993 ret
= write_object(fd
, s
->aio_context
, (char *)&s
->inode
,
1994 vid_to_vdi_oid(s
->inode
.vdi_id
), s
->inode
.nr_copies
,
1995 datalen
, 0, false, s
->cache_flags
);
1999 error_report("failed to update an inode.");
2006 * This function is called after writing data objects. If we need to
2007 * update metadata, this sends a write request to the vdi object.
2008 * Otherwise, this switches back to sd_co_readv/writev.
2010 static void coroutine_fn
sd_write_done(SheepdogAIOCB
*acb
)
2012 BDRVSheepdogState
*s
= acb
->common
.bs
->opaque
;
2015 uint32_t offset
, data_len
, mn
, mx
;
2017 mn
= acb
->min_dirty_data_idx
;
2018 mx
= acb
->max_dirty_data_idx
;
2020 /* we need to update the vdi object. */
2021 offset
= sizeof(s
->inode
) - sizeof(s
->inode
.data_vdi_id
) +
2022 mn
* sizeof(s
->inode
.data_vdi_id
[0]);
2023 data_len
= (mx
- mn
+ 1) * sizeof(s
->inode
.data_vdi_id
[0]);
2025 acb
->min_dirty_data_idx
= UINT32_MAX
;
2026 acb
->max_dirty_data_idx
= 0;
2028 iov
.iov_base
= &s
->inode
;
2029 iov
.iov_len
= sizeof(s
->inode
);
2030 aio_req
= alloc_aio_req(s
, acb
, vid_to_vdi_oid(s
->inode
.vdi_id
),
2031 data_len
, offset
, 0, false, 0, offset
);
2032 QLIST_INSERT_HEAD(&s
->inflight_aio_head
, aio_req
, aio_siblings
);
2033 add_aio_request(s
, aio_req
, &iov
, 1, AIOCB_WRITE_UDATA
);
2035 acb
->aio_done_func
= sd_finish_aiocb
;
2036 acb
->aiocb_type
= AIOCB_WRITE_UDATA
;
2040 sd_finish_aiocb(acb
);
2043 /* Delete current working VDI on the snapshot chain */
2044 static bool sd_delete(BDRVSheepdogState
*s
)
2046 Error
*local_err
= NULL
;
2047 unsigned int wlen
= SD_MAX_VDI_LEN
, rlen
= 0;
2048 SheepdogVdiReq hdr
= {
2049 .opcode
= SD_OP_DEL_VDI
,
2050 .base_vdi_id
= s
->inode
.vdi_id
,
2051 .data_length
= wlen
,
2052 .flags
= SD_FLAG_CMD_WRITE
,
2054 SheepdogVdiRsp
*rsp
= (SheepdogVdiRsp
*)&hdr
;
2057 fd
= connect_to_sdog(s
, &local_err
);
2059 error_report_err(local_err
);
2063 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&hdr
,
2064 s
->name
, &wlen
, &rlen
);
2069 switch (rsp
->result
) {
2071 error_report("%s was already deleted", s
->name
);
2073 case SD_RES_SUCCESS
:
2076 error_report("%s, %s", sd_strerror(rsp
->result
), s
->name
);
2084 * Create a writable VDI from a snapshot
2086 static int sd_create_branch(BDRVSheepdogState
*s
)
2088 Error
*local_err
= NULL
;
2094 DPRINTF("%" PRIx32
" is snapshot.\n", s
->inode
.vdi_id
);
2096 buf
= g_malloc(SD_INODE_SIZE
);
2099 * Even If deletion fails, we will just create extra snapshot based on
2100 * the working VDI which was supposed to be deleted. So no need to
2103 deleted
= sd_delete(s
);
2104 ret
= do_sd_create(s
, &vid
, !deleted
, &local_err
);
2106 error_report_err(local_err
);
2110 DPRINTF("%" PRIx32
" is created.\n", vid
);
2112 fd
= connect_to_sdog(s
, &local_err
);
2114 error_report_err(local_err
);
2119 ret
= read_object(fd
, s
->aio_context
, buf
, vid_to_vdi_oid(vid
),
2120 s
->inode
.nr_copies
, SD_INODE_SIZE
, 0, s
->cache_flags
);
2128 memcpy(&s
->inode
, buf
, sizeof(s
->inode
));
2130 s
->is_snapshot
= false;
2132 DPRINTF("%" PRIx32
" was newly created.\n", s
->inode
.vdi_id
);
2141 * Send I/O requests to the server.
2143 * This function sends requests to the server, links the requests to
2144 * the inflight_list in BDRVSheepdogState, and exits without
2145 * waiting the response. The responses are received in the
2146 * `aio_read_response' function which is called from the main loop as
2149 * Returns 1 when we need to wait a response, 0 when there is no sent
2150 * request and -errno in error cases.
2152 static int coroutine_fn
sd_co_rw_vector(void *p
)
2154 SheepdogAIOCB
*acb
= p
;
2156 unsigned long len
, done
= 0, total
= acb
->nb_sectors
* BDRV_SECTOR_SIZE
;
2158 uint32_t object_size
;
2161 BDRVSheepdogState
*s
= acb
->common
.bs
->opaque
;
2162 SheepdogInode
*inode
= &s
->inode
;
2165 if (acb
->aiocb_type
== AIOCB_WRITE_UDATA
&& s
->is_snapshot
) {
2167 * In the case we open the snapshot VDI, Sheepdog creates the
2168 * writable VDI when we do a write operation first.
2170 ret
= sd_create_branch(s
);
2177 object_size
= (UINT32_C(1) << inode
->block_size_shift
);
2178 idx
= acb
->sector_num
* BDRV_SECTOR_SIZE
/ object_size
;
2179 offset
= (acb
->sector_num
* BDRV_SECTOR_SIZE
) % object_size
;
2182 * Make sure we don't free the aiocb before we are done with all requests.
2183 * This additional reference is dropped at the end of this function.
2187 while (done
!= total
) {
2189 uint64_t old_oid
= 0;
2190 bool create
= false;
2192 oid
= vid_to_data_oid(inode
->data_vdi_id
[idx
], idx
);
2194 len
= MIN(total
- done
, object_size
- offset
);
2196 switch (acb
->aiocb_type
) {
2197 case AIOCB_READ_UDATA
:
2198 if (!inode
->data_vdi_id
[idx
]) {
2199 qemu_iovec_memset(acb
->qiov
, done
, 0, len
);
2203 case AIOCB_WRITE_UDATA
:
2204 if (!inode
->data_vdi_id
[idx
]) {
2206 } else if (!is_data_obj_writable(inode
, idx
)) {
2210 flags
= SD_FLAG_CMD_COW
;
2213 case AIOCB_DISCARD_OBJ
:
2215 * We discard the object only when the whole object is
2216 * 1) allocated 2) trimmed. Otherwise, simply skip it.
2218 if (len
!= object_size
|| inode
->data_vdi_id
[idx
] == 0) {
2227 DPRINTF("update ino (%" PRIu32
") %" PRIu64
" %" PRIu64
" %ld\n",
2229 vid_to_data_oid(inode
->data_vdi_id
[idx
], idx
), idx
);
2230 oid
= vid_to_data_oid(inode
->vdi_id
, idx
);
2231 DPRINTF("new oid %" PRIx64
"\n", oid
);
2234 aio_req
= alloc_aio_req(s
, acb
, oid
, len
, offset
, flags
, create
,
2236 acb
->aiocb_type
== AIOCB_DISCARD_OBJ
?
2238 QLIST_INSERT_HEAD(&s
->inflight_aio_head
, aio_req
, aio_siblings
);
2240 add_aio_request(s
, aio_req
, acb
->qiov
->iov
, acb
->qiov
->niov
,
2248 if (!--acb
->nr_pending
) {
2254 static bool check_overlapping_aiocb(BDRVSheepdogState
*s
, SheepdogAIOCB
*aiocb
)
2258 QLIST_FOREACH(cb
, &s
->inflight_aiocb_head
, aiocb_siblings
) {
2259 if (AIOCBOverlapping(aiocb
, cb
)) {
2264 QLIST_INSERT_HEAD(&s
->inflight_aiocb_head
, aiocb
, aiocb_siblings
);
2268 static coroutine_fn
int sd_co_writev(BlockDriverState
*bs
, int64_t sector_num
,
2269 int nb_sectors
, QEMUIOVector
*qiov
)
2273 int64_t offset
= (sector_num
+ nb_sectors
) * BDRV_SECTOR_SIZE
;
2274 BDRVSheepdogState
*s
= bs
->opaque
;
2276 if (offset
> s
->inode
.vdi_size
) {
2277 ret
= sd_truncate(bs
, offset
);
2283 acb
= sd_aio_setup(bs
, qiov
, sector_num
, nb_sectors
);
2284 acb
->aio_done_func
= sd_write_done
;
2285 acb
->aiocb_type
= AIOCB_WRITE_UDATA
;
2288 if (check_overlapping_aiocb(s
, acb
)) {
2289 qemu_co_queue_wait(&s
->overlapping_queue
);
2293 ret
= sd_co_rw_vector(acb
);
2295 QLIST_REMOVE(acb
, aiocb_siblings
);
2296 qemu_co_queue_restart_all(&s
->overlapping_queue
);
2297 qemu_aio_unref(acb
);
2301 qemu_coroutine_yield();
2303 QLIST_REMOVE(acb
, aiocb_siblings
);
2304 qemu_co_queue_restart_all(&s
->overlapping_queue
);
2309 static coroutine_fn
int sd_co_readv(BlockDriverState
*bs
, int64_t sector_num
,
2310 int nb_sectors
, QEMUIOVector
*qiov
)
2314 BDRVSheepdogState
*s
= bs
->opaque
;
2316 acb
= sd_aio_setup(bs
, qiov
, sector_num
, nb_sectors
);
2317 acb
->aiocb_type
= AIOCB_READ_UDATA
;
2318 acb
->aio_done_func
= sd_finish_aiocb
;
2321 if (check_overlapping_aiocb(s
, acb
)) {
2322 qemu_co_queue_wait(&s
->overlapping_queue
);
2326 ret
= sd_co_rw_vector(acb
);
2328 QLIST_REMOVE(acb
, aiocb_siblings
);
2329 qemu_co_queue_restart_all(&s
->overlapping_queue
);
2330 qemu_aio_unref(acb
);
2334 qemu_coroutine_yield();
2336 QLIST_REMOVE(acb
, aiocb_siblings
);
2337 qemu_co_queue_restart_all(&s
->overlapping_queue
);
2341 static int coroutine_fn
sd_co_flush_to_disk(BlockDriverState
*bs
)
2343 BDRVSheepdogState
*s
= bs
->opaque
;
2347 if (s
->cache_flags
!= SD_FLAG_CMD_CACHE
) {
2351 acb
= sd_aio_setup(bs
, NULL
, 0, 0);
2352 acb
->aiocb_type
= AIOCB_FLUSH_CACHE
;
2353 acb
->aio_done_func
= sd_finish_aiocb
;
2355 aio_req
= alloc_aio_req(s
, acb
, vid_to_vdi_oid(s
->inode
.vdi_id
),
2356 0, 0, 0, false, 0, 0);
2357 QLIST_INSERT_HEAD(&s
->inflight_aio_head
, aio_req
, aio_siblings
);
2358 add_aio_request(s
, aio_req
, NULL
, 0, acb
->aiocb_type
);
2360 qemu_coroutine_yield();
2364 static int sd_snapshot_create(BlockDriverState
*bs
, QEMUSnapshotInfo
*sn_info
)
2366 Error
*local_err
= NULL
;
2367 BDRVSheepdogState
*s
= bs
->opaque
;
2370 SheepdogInode
*inode
;
2371 unsigned int datalen
;
2373 DPRINTF("sn_info: name %s id_str %s s: name %s vm_state_size %" PRId64
" "
2374 "is_snapshot %d\n", sn_info
->name
, sn_info
->id_str
,
2375 s
->name
, sn_info
->vm_state_size
, s
->is_snapshot
);
2377 if (s
->is_snapshot
) {
2378 error_report("You can't create a snapshot of a snapshot VDI, "
2379 "%s (%" PRIu32
").", s
->name
, s
->inode
.vdi_id
);
2384 DPRINTF("%s %s\n", sn_info
->name
, sn_info
->id_str
);
2386 s
->inode
.vm_state_size
= sn_info
->vm_state_size
;
2387 s
->inode
.vm_clock_nsec
= sn_info
->vm_clock_nsec
;
2388 /* It appears that inode.tag does not require a NUL terminator,
2389 * which means this use of strncpy is ok.
2391 strncpy(s
->inode
.tag
, sn_info
->name
, sizeof(s
->inode
.tag
));
2392 /* we don't need to update entire object */
2393 datalen
= SD_INODE_SIZE
- sizeof(s
->inode
.data_vdi_id
);
2394 inode
= g_malloc(datalen
);
2396 /* refresh inode. */
2397 fd
= connect_to_sdog(s
, &local_err
);
2399 error_report_err(local_err
);
2404 ret
= write_object(fd
, s
->aio_context
, (char *)&s
->inode
,
2405 vid_to_vdi_oid(s
->inode
.vdi_id
), s
->inode
.nr_copies
,
2406 datalen
, 0, false, s
->cache_flags
);
2408 error_report("failed to write snapshot's inode.");
2412 ret
= do_sd_create(s
, &new_vid
, 1, &local_err
);
2414 error_reportf_err(local_err
,
2415 "failed to create inode for snapshot: ");
2419 ret
= read_object(fd
, s
->aio_context
, (char *)inode
,
2420 vid_to_vdi_oid(new_vid
), s
->inode
.nr_copies
, datalen
, 0,
2424 error_report("failed to read new inode info. %s", strerror(errno
));
2428 memcpy(&s
->inode
, inode
, datalen
);
2429 DPRINTF("s->inode: name %s snap_id %x oid %x\n",
2430 s
->inode
.name
, s
->inode
.snap_id
, s
->inode
.vdi_id
);
2439 * We implement rollback(loadvm) operation to the specified snapshot by
2440 * 1) switch to the snapshot
2441 * 2) rely on sd_create_branch to delete working VDI and
2442 * 3) create a new working VDI based on the specified snapshot
2444 static int sd_snapshot_goto(BlockDriverState
*bs
, const char *snapshot_id
)
2446 BDRVSheepdogState
*s
= bs
->opaque
;
2447 BDRVSheepdogState
*old_s
;
2448 char tag
[SD_MAX_VDI_TAG_LEN
];
2449 uint32_t snapid
= 0;
2452 old_s
= g_new(BDRVSheepdogState
, 1);
2454 memcpy(old_s
, s
, sizeof(BDRVSheepdogState
));
2456 snapid
= strtoul(snapshot_id
, NULL
, 10);
2460 pstrcpy(tag
, sizeof(tag
), snapshot_id
);
2463 ret
= reload_inode(s
, snapid
, tag
);
2468 ret
= sd_create_branch(s
);
2477 /* recover bdrv_sd_state */
2478 memcpy(s
, old_s
, sizeof(BDRVSheepdogState
));
2481 error_report("failed to open. recover old bdrv_sd_state.");
2486 #define NR_BATCHED_DISCARD 128
2488 static bool remove_objects(BDRVSheepdogState
*s
)
2490 int fd
, i
= 0, nr_objs
= 0;
2491 Error
*local_err
= NULL
;
2494 SheepdogInode
*inode
= &s
->inode
;
2496 fd
= connect_to_sdog(s
, &local_err
);
2498 error_report_err(local_err
);
2502 nr_objs
= count_data_objs(inode
);
2503 while (i
< nr_objs
) {
2504 int start_idx
, nr_filled_idx
;
2506 while (i
< nr_objs
&& !inode
->data_vdi_id
[i
]) {
2512 while (i
< nr_objs
&& nr_filled_idx
< NR_BATCHED_DISCARD
) {
2513 if (inode
->data_vdi_id
[i
]) {
2514 inode
->data_vdi_id
[i
] = 0;
2521 ret
= write_object(fd
, s
->aio_context
,
2522 (char *)&inode
->data_vdi_id
[start_idx
],
2523 vid_to_vdi_oid(s
->inode
.vdi_id
), inode
->nr_copies
,
2524 (i
- start_idx
) * sizeof(uint32_t),
2525 offsetof(struct SheepdogInode
,
2526 data_vdi_id
[start_idx
]),
2527 false, s
->cache_flags
);
2529 error_report("failed to discard snapshot inode.");
2540 static int sd_snapshot_delete(BlockDriverState
*bs
,
2541 const char *snapshot_id
,
2545 uint32_t snap_id
= 0;
2546 char snap_tag
[SD_MAX_VDI_TAG_LEN
];
2547 Error
*local_err
= NULL
;
2549 char buf
[SD_MAX_VDI_LEN
+ SD_MAX_VDI_TAG_LEN
];
2550 BDRVSheepdogState
*s
= bs
->opaque
;
2551 unsigned int wlen
= SD_MAX_VDI_LEN
+ SD_MAX_VDI_TAG_LEN
, rlen
= 0;
2553 SheepdogVdiReq hdr
= {
2554 .opcode
= SD_OP_DEL_VDI
,
2555 .data_length
= wlen
,
2556 .flags
= SD_FLAG_CMD_WRITE
,
2558 SheepdogVdiRsp
*rsp
= (SheepdogVdiRsp
*)&hdr
;
2560 if (!remove_objects(s
)) {
2564 memset(buf
, 0, sizeof(buf
));
2565 memset(snap_tag
, 0, sizeof(snap_tag
));
2566 pstrcpy(buf
, SD_MAX_VDI_LEN
, s
->name
);
2567 if (qemu_strtoul(snapshot_id
, NULL
, 10, (unsigned long *)&snap_id
)) {
2572 hdr
.snapid
= snap_id
;
2574 pstrcpy(snap_tag
, sizeof(snap_tag
), snapshot_id
);
2575 pstrcpy(buf
+ SD_MAX_VDI_LEN
, SD_MAX_VDI_TAG_LEN
, snap_tag
);
2578 ret
= find_vdi_name(s
, s
->name
, snap_id
, snap_tag
, &vid
, true,
2584 fd
= connect_to_sdog(s
, &local_err
);
2586 error_report_err(local_err
);
2590 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&hdr
,
2597 switch (rsp
->result
) {
2599 error_report("%s was already deleted", s
->name
);
2600 case SD_RES_SUCCESS
:
2603 error_report("%s, %s", sd_strerror(rsp
->result
), s
->name
);
2610 static int sd_snapshot_list(BlockDriverState
*bs
, QEMUSnapshotInfo
**psn_tab
)
2612 Error
*local_err
= NULL
;
2613 BDRVSheepdogState
*s
= bs
->opaque
;
2615 int fd
, nr
= 1024, ret
, max
= BITS_TO_LONGS(SD_NR_VDIS
) * sizeof(long);
2616 QEMUSnapshotInfo
*sn_tab
= NULL
;
2617 unsigned wlen
, rlen
;
2619 static SheepdogInode inode
;
2620 unsigned long *vdi_inuse
;
2621 unsigned int start_nr
;
2625 vdi_inuse
= g_malloc(max
);
2627 fd
= connect_to_sdog(s
, &local_err
);
2629 error_report_err(local_err
);
2637 memset(&req
, 0, sizeof(req
));
2639 req
.opcode
= SD_OP_READ_VDIS
;
2640 req
.data_length
= max
;
2642 ret
= do_req(fd
, s
->aio_context
, (SheepdogReq
*)&req
,
2643 vdi_inuse
, &wlen
, &rlen
);
2650 sn_tab
= g_new0(QEMUSnapshotInfo
, nr
);
2652 /* calculate a vdi id with hash function */
2653 hval
= fnv_64a_buf(s
->name
, strlen(s
->name
), FNV1A_64_INIT
);
2654 start_nr
= hval
& (SD_NR_VDIS
- 1);
2656 fd
= connect_to_sdog(s
, &local_err
);
2658 error_report_err(local_err
);
2663 for (vid
= start_nr
; found
< nr
; vid
= (vid
+ 1) % SD_NR_VDIS
) {
2664 if (!test_bit(vid
, vdi_inuse
)) {
2668 /* we don't need to read entire object */
2669 ret
= read_object(fd
, s
->aio_context
, (char *)&inode
,
2670 vid_to_vdi_oid(vid
),
2671 0, SD_INODE_SIZE
- sizeof(inode
.data_vdi_id
), 0,
2678 if (!strcmp(inode
.name
, s
->name
) && is_snapshot(&inode
)) {
2679 sn_tab
[found
].date_sec
= inode
.snap_ctime
>> 32;
2680 sn_tab
[found
].date_nsec
= inode
.snap_ctime
& 0xffffffff;
2681 sn_tab
[found
].vm_state_size
= inode
.vm_state_size
;
2682 sn_tab
[found
].vm_clock_nsec
= inode
.vm_clock_nsec
;
2684 snprintf(sn_tab
[found
].id_str
, sizeof(sn_tab
[found
].id_str
),
2685 "%" PRIu32
, inode
.snap_id
);
2686 pstrcpy(sn_tab
[found
].name
,
2687 MIN(sizeof(sn_tab
[found
].name
), sizeof(inode
.tag
)),
2706 static int do_load_save_vmstate(BDRVSheepdogState
*s
, uint8_t *data
,
2707 int64_t pos
, int size
, int load
)
2709 Error
*local_err
= NULL
;
2711 int fd
, ret
= 0, remaining
= size
;
2712 unsigned int data_len
;
2713 uint64_t vmstate_oid
;
2716 uint32_t vdi_id
= load
? s
->inode
.parent_vdi_id
: s
->inode
.vdi_id
;
2717 uint32_t object_size
= (UINT32_C(1) << s
->inode
.block_size_shift
);
2719 fd
= connect_to_sdog(s
, &local_err
);
2721 error_report_err(local_err
);
2726 vdi_index
= pos
/ object_size
;
2727 offset
= pos
% object_size
;
2729 data_len
= MIN(remaining
, object_size
- offset
);
2731 vmstate_oid
= vid_to_vmstate_oid(vdi_id
, vdi_index
);
2733 create
= (offset
== 0);
2735 ret
= read_object(fd
, s
->aio_context
, (char *)data
, vmstate_oid
,
2736 s
->inode
.nr_copies
, data_len
, offset
,
2739 ret
= write_object(fd
, s
->aio_context
, (char *)data
, vmstate_oid
,
2740 s
->inode
.nr_copies
, data_len
, offset
, create
,
2745 error_report("failed to save vmstate %s", strerror(errno
));
2751 remaining
-= data_len
;
2759 static int sd_save_vmstate(BlockDriverState
*bs
, QEMUIOVector
*qiov
,
2762 BDRVSheepdogState
*s
= bs
->opaque
;
2766 buf
= qemu_blockalign(bs
, qiov
->size
);
2767 qemu_iovec_to_buf(qiov
, 0, buf
, qiov
->size
);
2768 ret
= do_load_save_vmstate(s
, (uint8_t *) buf
, pos
, qiov
->size
, 0);
2774 static int sd_load_vmstate(BlockDriverState
*bs
, uint8_t *data
,
2775 int64_t pos
, int size
)
2777 BDRVSheepdogState
*s
= bs
->opaque
;
2779 return do_load_save_vmstate(s
, data
, pos
, size
, 1);
2783 static coroutine_fn
int sd_co_discard(BlockDriverState
*bs
, int64_t sector_num
,
2787 BDRVSheepdogState
*s
= bs
->opaque
;
2789 QEMUIOVector discard_iov
;
2793 if (!s
->discard_supported
) {
2797 memset(&discard_iov
, 0, sizeof(discard_iov
));
2798 memset(&iov
, 0, sizeof(iov
));
2799 iov
.iov_base
= &zero
;
2800 iov
.iov_len
= sizeof(zero
);
2801 discard_iov
.iov
= &iov
;
2802 discard_iov
.niov
= 1;
2803 acb
= sd_aio_setup(bs
, &discard_iov
, sector_num
, nb_sectors
);
2804 acb
->aiocb_type
= AIOCB_DISCARD_OBJ
;
2805 acb
->aio_done_func
= sd_finish_aiocb
;
2808 if (check_overlapping_aiocb(s
, acb
)) {
2809 qemu_co_queue_wait(&s
->overlapping_queue
);
2813 ret
= sd_co_rw_vector(acb
);
2815 QLIST_REMOVE(acb
, aiocb_siblings
);
2816 qemu_co_queue_restart_all(&s
->overlapping_queue
);
2817 qemu_aio_unref(acb
);
2821 qemu_coroutine_yield();
2823 QLIST_REMOVE(acb
, aiocb_siblings
);
2824 qemu_co_queue_restart_all(&s
->overlapping_queue
);
2829 static coroutine_fn
int64_t
2830 sd_co_get_block_status(BlockDriverState
*bs
, int64_t sector_num
, int nb_sectors
,
2831 int *pnum
, BlockDriverState
**file
)
2833 BDRVSheepdogState
*s
= bs
->opaque
;
2834 SheepdogInode
*inode
= &s
->inode
;
2835 uint32_t object_size
= (UINT32_C(1) << inode
->block_size_shift
);
2836 uint64_t offset
= sector_num
* BDRV_SECTOR_SIZE
;
2837 unsigned long start
= offset
/ object_size
,
2838 end
= DIV_ROUND_UP((sector_num
+ nb_sectors
) *
2839 BDRV_SECTOR_SIZE
, object_size
);
2841 int64_t ret
= BDRV_BLOCK_DATA
| BDRV_BLOCK_OFFSET_VALID
| offset
;
2843 for (idx
= start
; idx
< end
; idx
++) {
2844 if (inode
->data_vdi_id
[idx
] == 0) {
2849 /* Get the longest length of unallocated sectors */
2851 for (idx
= start
+ 1; idx
< end
; idx
++) {
2852 if (inode
->data_vdi_id
[idx
] != 0) {
2858 *pnum
= (idx
- start
) * object_size
/ BDRV_SECTOR_SIZE
;
2859 if (*pnum
> nb_sectors
) {
2862 if (ret
> 0 && ret
& BDRV_BLOCK_OFFSET_VALID
) {
2868 static int64_t sd_get_allocated_file_size(BlockDriverState
*bs
)
2870 BDRVSheepdogState
*s
= bs
->opaque
;
2871 SheepdogInode
*inode
= &s
->inode
;
2872 uint32_t object_size
= (UINT32_C(1) << inode
->block_size_shift
);
2873 unsigned long i
, last
= DIV_ROUND_UP(inode
->vdi_size
, object_size
);
2876 for (i
= 0; i
< last
; i
++) {
2877 if (inode
->data_vdi_id
[i
] == 0) {
2880 size
+= object_size
;
2885 static QemuOptsList sd_create_opts
= {
2886 .name
= "sheepdog-create-opts",
2887 .head
= QTAILQ_HEAD_INITIALIZER(sd_create_opts
.head
),
2890 .name
= BLOCK_OPT_SIZE
,
2891 .type
= QEMU_OPT_SIZE
,
2892 .help
= "Virtual disk size"
2895 .name
= BLOCK_OPT_BACKING_FILE
,
2896 .type
= QEMU_OPT_STRING
,
2897 .help
= "File name of a base image"
2900 .name
= BLOCK_OPT_PREALLOC
,
2901 .type
= QEMU_OPT_STRING
,
2902 .help
= "Preallocation mode (allowed values: off, full)"
2905 .name
= BLOCK_OPT_REDUNDANCY
,
2906 .type
= QEMU_OPT_STRING
,
2907 .help
= "Redundancy of the image"
2910 .name
= BLOCK_OPT_OBJECT_SIZE
,
2911 .type
= QEMU_OPT_SIZE
,
2912 .help
= "Object size of the image"
2914 { /* end of list */ }
2918 static BlockDriver bdrv_sheepdog
= {
2919 .format_name
= "sheepdog",
2920 .protocol_name
= "sheepdog",
2921 .instance_size
= sizeof(BDRVSheepdogState
),
2922 .bdrv_needs_filename
= true,
2923 .bdrv_file_open
= sd_open
,
2924 .bdrv_reopen_prepare
= sd_reopen_prepare
,
2925 .bdrv_reopen_commit
= sd_reopen_commit
,
2926 .bdrv_reopen_abort
= sd_reopen_abort
,
2927 .bdrv_close
= sd_close
,
2928 .bdrv_create
= sd_create
,
2929 .bdrv_has_zero_init
= bdrv_has_zero_init_1
,
2930 .bdrv_getlength
= sd_getlength
,
2931 .bdrv_get_allocated_file_size
= sd_get_allocated_file_size
,
2932 .bdrv_truncate
= sd_truncate
,
2934 .bdrv_co_readv
= sd_co_readv
,
2935 .bdrv_co_writev
= sd_co_writev
,
2936 .bdrv_co_flush_to_disk
= sd_co_flush_to_disk
,
2937 .bdrv_co_discard
= sd_co_discard
,
2938 .bdrv_co_get_block_status
= sd_co_get_block_status
,
2940 .bdrv_snapshot_create
= sd_snapshot_create
,
2941 .bdrv_snapshot_goto
= sd_snapshot_goto
,
2942 .bdrv_snapshot_delete
= sd_snapshot_delete
,
2943 .bdrv_snapshot_list
= sd_snapshot_list
,
2945 .bdrv_save_vmstate
= sd_save_vmstate
,
2946 .bdrv_load_vmstate
= sd_load_vmstate
,
2948 .bdrv_detach_aio_context
= sd_detach_aio_context
,
2949 .bdrv_attach_aio_context
= sd_attach_aio_context
,
2951 .create_opts
= &sd_create_opts
,
2954 static BlockDriver bdrv_sheepdog_tcp
= {
2955 .format_name
= "sheepdog",
2956 .protocol_name
= "sheepdog+tcp",
2957 .instance_size
= sizeof(BDRVSheepdogState
),
2958 .bdrv_needs_filename
= true,
2959 .bdrv_file_open
= sd_open
,
2960 .bdrv_reopen_prepare
= sd_reopen_prepare
,
2961 .bdrv_reopen_commit
= sd_reopen_commit
,
2962 .bdrv_reopen_abort
= sd_reopen_abort
,
2963 .bdrv_close
= sd_close
,
2964 .bdrv_create
= sd_create
,
2965 .bdrv_has_zero_init
= bdrv_has_zero_init_1
,
2966 .bdrv_getlength
= sd_getlength
,
2967 .bdrv_get_allocated_file_size
= sd_get_allocated_file_size
,
2968 .bdrv_truncate
= sd_truncate
,
2970 .bdrv_co_readv
= sd_co_readv
,
2971 .bdrv_co_writev
= sd_co_writev
,
2972 .bdrv_co_flush_to_disk
= sd_co_flush_to_disk
,
2973 .bdrv_co_discard
= sd_co_discard
,
2974 .bdrv_co_get_block_status
= sd_co_get_block_status
,
2976 .bdrv_snapshot_create
= sd_snapshot_create
,
2977 .bdrv_snapshot_goto
= sd_snapshot_goto
,
2978 .bdrv_snapshot_delete
= sd_snapshot_delete
,
2979 .bdrv_snapshot_list
= sd_snapshot_list
,
2981 .bdrv_save_vmstate
= sd_save_vmstate
,
2982 .bdrv_load_vmstate
= sd_load_vmstate
,
2984 .bdrv_detach_aio_context
= sd_detach_aio_context
,
2985 .bdrv_attach_aio_context
= sd_attach_aio_context
,
2987 .create_opts
= &sd_create_opts
,
2990 static BlockDriver bdrv_sheepdog_unix
= {
2991 .format_name
= "sheepdog",
2992 .protocol_name
= "sheepdog+unix",
2993 .instance_size
= sizeof(BDRVSheepdogState
),
2994 .bdrv_needs_filename
= true,
2995 .bdrv_file_open
= sd_open
,
2996 .bdrv_reopen_prepare
= sd_reopen_prepare
,
2997 .bdrv_reopen_commit
= sd_reopen_commit
,
2998 .bdrv_reopen_abort
= sd_reopen_abort
,
2999 .bdrv_close
= sd_close
,
3000 .bdrv_create
= sd_create
,
3001 .bdrv_has_zero_init
= bdrv_has_zero_init_1
,
3002 .bdrv_getlength
= sd_getlength
,
3003 .bdrv_get_allocated_file_size
= sd_get_allocated_file_size
,
3004 .bdrv_truncate
= sd_truncate
,
3006 .bdrv_co_readv
= sd_co_readv
,
3007 .bdrv_co_writev
= sd_co_writev
,
3008 .bdrv_co_flush_to_disk
= sd_co_flush_to_disk
,
3009 .bdrv_co_discard
= sd_co_discard
,
3010 .bdrv_co_get_block_status
= sd_co_get_block_status
,
3012 .bdrv_snapshot_create
= sd_snapshot_create
,
3013 .bdrv_snapshot_goto
= sd_snapshot_goto
,
3014 .bdrv_snapshot_delete
= sd_snapshot_delete
,
3015 .bdrv_snapshot_list
= sd_snapshot_list
,
3017 .bdrv_save_vmstate
= sd_save_vmstate
,
3018 .bdrv_load_vmstate
= sd_load_vmstate
,
3020 .bdrv_detach_aio_context
= sd_detach_aio_context
,
3021 .bdrv_attach_aio_context
= sd_attach_aio_context
,
3023 .create_opts
= &sd_create_opts
,
3026 static void bdrv_sheepdog_init(void)
3028 bdrv_register(&bdrv_sheepdog
);
3029 bdrv_register(&bdrv_sheepdog_tcp
);
3030 bdrv_register(&bdrv_sheepdog_unix
);
3032 block_init(bdrv_sheepdog_init
);