4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include "spdk/stdinc.h"
36 #include <sys/eventfd.h>
38 #include <linux/virtio_scsi.h>
40 #include <rte_config.h>
41 #include <rte_malloc.h>
42 #include <rte_alarm.h>
44 #include "virtio_user/vhost.h"
45 #include "spdk/string.h"
46 #include "spdk/config.h"
48 #include "spdk_internal/virtio.h"
50 #define VIRTIO_USER_SUPPORTED_PROTOCOL_FEATURES \
51 ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
52 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))
55 virtio_user_create_queue(struct virtio_dev
*vdev
, uint32_t queue_sel
)
57 struct virtio_user_dev
*dev
= vdev
->ctx
;
59 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come
60 * firstly because vhost depends on this msg to allocate virtqueue
63 struct vhost_vring_file file
;
65 file
.index
= queue_sel
;
66 file
.fd
= dev
->callfds
[queue_sel
];
67 return dev
->ops
->send_request(dev
, VHOST_USER_SET_VRING_CALL
, &file
);
71 virtio_user_set_vring_addr(struct virtio_dev
*vdev
, uint32_t queue_sel
)
73 struct virtio_user_dev
*dev
= vdev
->ctx
;
74 struct vring
*vring
= &dev
->vrings
[queue_sel
];
75 struct vhost_vring_addr addr
= {
77 .desc_user_addr
= (uint64_t)(uintptr_t)vring
->desc
,
78 .avail_user_addr
= (uint64_t)(uintptr_t)vring
->avail
,
79 .used_user_addr
= (uint64_t)(uintptr_t)vring
->used
,
81 .flags
= 0, /* disable log */
84 return dev
->ops
->send_request(dev
, VHOST_USER_SET_VRING_ADDR
, &addr
);
88 virtio_user_kick_queue(struct virtio_dev
*vdev
, uint32_t queue_sel
)
90 struct virtio_user_dev
*dev
= vdev
->ctx
;
91 struct vhost_vring_file file
;
92 struct vhost_vring_state state
;
93 struct vring
*vring
= &dev
->vrings
[queue_sel
];
96 state
.index
= queue_sel
;
97 state
.num
= vring
->num
;
98 rc
= dev
->ops
->send_request(dev
, VHOST_USER_SET_VRING_NUM
, &state
);
103 state
.index
= queue_sel
;
104 state
.num
= 0; /* no reservation */
105 rc
= dev
->ops
->send_request(dev
, VHOST_USER_SET_VRING_BASE
, &state
);
110 virtio_user_set_vring_addr(vdev
, queue_sel
);
112 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes
113 * lastly because vhost depends on this msg to judge if
116 file
.index
= queue_sel
;
117 file
.fd
= dev
->kickfds
[queue_sel
];
118 return dev
->ops
->send_request(dev
, VHOST_USER_SET_VRING_KICK
, &file
);
122 virtio_user_stop_queue(struct virtio_dev
*vdev
, uint32_t queue_sel
)
124 struct virtio_user_dev
*dev
= vdev
->ctx
;
125 struct vhost_vring_state state
;
127 state
.index
= queue_sel
;
130 return dev
->ops
->send_request(dev
, VHOST_USER_GET_VRING_BASE
, &state
);
134 virtio_user_queue_setup(struct virtio_dev
*vdev
,
135 int (*fn
)(struct virtio_dev
*, uint32_t))
140 for (i
= 0; i
< vdev
->max_queues
; ++i
) {
143 SPDK_ERRLOG("setup tx vq fails: %"PRIu32
".\n", i
);
152 virtio_user_map_notify(void *cb_ctx
, struct spdk_mem_map
*map
,
153 enum spdk_mem_map_notify_action action
,
154 void *vaddr
, size_t size
)
156 struct virtio_dev
*vdev
= cb_ctx
;
157 struct virtio_user_dev
*dev
= vdev
->ctx
;
161 /* We have to resend all mappings anyway, so don't bother with any
164 ret
= dev
->ops
->send_request(dev
, VHOST_USER_SET_MEM_TABLE
, NULL
);
169 #ifdef SPDK_CONFIG_VHOST_INTERNAL_LIB
170 /* Our internal rte_vhost lib requires SET_VRING_ADDR to flush a pending
171 * SET_MEM_TABLE. On the other hand, the upstream rte_vhost will invalidate
172 * the entire queue upon receiving SET_VRING_ADDR message, so we mustn't
173 * send it here. Both behaviors are strictly implementation specific, but
174 * this message isn't needed from the point of the spec, so send it only
175 * if vhost is compiled with our internal lib.
177 ret
= virtio_user_queue_setup(vdev
, virtio_user_set_vring_addr
);
183 /* Since we might want to use that mapping straight away, we have to
184 * make sure the guest has already processed our SET_MEM_TABLE message.
185 * F_REPLY_ACK is just a feature and the host is not obliged to
186 * support it, so we send a simple message that always has a response
187 * and we wait for that response. Messages are always processed in order.
189 return dev
->ops
->send_request(dev
, VHOST_USER_GET_FEATURES
, &features
);
193 virtio_user_register_mem(struct virtio_dev
*vdev
)
195 struct virtio_user_dev
*dev
= vdev
->ctx
;
196 const struct spdk_mem_map_ops virtio_user_map_ops
= {
197 .notify_cb
= virtio_user_map_notify
,
198 .are_contiguous
= NULL
201 dev
->mem_map
= spdk_mem_map_alloc(0, &virtio_user_map_ops
, vdev
);
202 if (dev
->mem_map
== NULL
) {
203 SPDK_ERRLOG("spdk_mem_map_alloc() failed\n");
211 virtio_user_unregister_mem(struct virtio_dev
*vdev
)
213 struct virtio_user_dev
*dev
= vdev
->ctx
;
215 spdk_mem_map_free(&dev
->mem_map
);
219 virtio_user_start_device(struct virtio_dev
*vdev
)
221 struct virtio_user_dev
*dev
= vdev
->ctx
;
222 uint64_t host_max_queues
;
225 if ((dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_MQ
)) == 0 &&
226 vdev
->max_queues
> 1 + vdev
->fixed_queues_num
) {
227 SPDK_WARNLOG("%s: requested %"PRIu16
" request queues, but the "
228 "host doesn't support VHOST_USER_PROTOCOL_F_MQ. "
229 "Only one request queue will be used.\n",
230 vdev
->name
, vdev
->max_queues
- vdev
->fixed_queues_num
);
231 vdev
->max_queues
= 1 + vdev
->fixed_queues_num
;
234 /* negotiate the number of I/O queues. */
235 ret
= dev
->ops
->send_request(dev
, VHOST_USER_GET_QUEUE_NUM
, &host_max_queues
);
240 if (vdev
->max_queues
> host_max_queues
+ vdev
->fixed_queues_num
) {
241 SPDK_WARNLOG("%s: requested %"PRIu16
" request queues"
242 "but only %"PRIu64
" available\n",
243 vdev
->name
, vdev
->max_queues
- vdev
->fixed_queues_num
,
245 vdev
->max_queues
= host_max_queues
;
248 /* tell vhost to create queues */
249 ret
= virtio_user_queue_setup(vdev
, virtio_user_create_queue
);
254 ret
= virtio_user_register_mem(vdev
);
259 return virtio_user_queue_setup(vdev
, virtio_user_kick_queue
);
263 virtio_user_stop_device(struct virtio_dev
*vdev
)
267 ret
= virtio_user_queue_setup(vdev
, virtio_user_stop_queue
);
268 /* a queue might fail to stop for various reasons, e.g. socket
269 * connection going down, but this mustn't prevent us from freeing
272 virtio_user_unregister_mem(vdev
);
277 virtio_user_dev_setup(struct virtio_dev
*vdev
)
279 struct virtio_user_dev
*dev
= vdev
->ctx
;
284 for (i
= 0; i
< SPDK_VIRTIO_MAX_VIRTQUEUES
; ++i
) {
285 dev
->callfds
[i
] = -1;
286 dev
->kickfds
[i
] = -1;
289 dev
->ops
= &ops_user
;
291 return dev
->ops
->setup(dev
);
295 virtio_user_read_dev_config(struct virtio_dev
*vdev
, size_t offset
,
296 void *dst
, int length
)
298 struct virtio_user_dev
*dev
= vdev
->ctx
;
299 struct vhost_user_config cfg
= {0};
302 if ((dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_CONFIG
)) == 0) {
307 cfg
.size
= VHOST_USER_MAX_CONFIG_SIZE
;
309 rc
= dev
->ops
->send_request(dev
, VHOST_USER_GET_CONFIG
, &cfg
);
311 SPDK_ERRLOG("get_config failed: %s\n", spdk_strerror(-rc
));
315 memcpy(dst
, cfg
.region
+ offset
, length
);
320 virtio_user_write_dev_config(struct virtio_dev
*vdev
, size_t offset
,
321 const void *src
, int length
)
323 struct virtio_user_dev
*dev
= vdev
->ctx
;
324 struct vhost_user_config cfg
= {0};
327 if ((dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_CONFIG
)) == 0) {
333 memcpy(cfg
.region
, src
, length
);
335 rc
= dev
->ops
->send_request(dev
, VHOST_USER_SET_CONFIG
, &cfg
);
337 SPDK_ERRLOG("set_config failed: %s\n", spdk_strerror(-rc
));
345 virtio_user_set_status(struct virtio_dev
*vdev
, uint8_t status
)
347 struct virtio_user_dev
*dev
= vdev
->ctx
;
350 if ((dev
->status
& VIRTIO_CONFIG_S_NEEDS_RESET
) &&
351 status
!= VIRTIO_CONFIG_S_RESET
) {
353 } else if (status
& VIRTIO_CONFIG_S_DRIVER_OK
) {
354 rc
= virtio_user_start_device(vdev
);
355 } else if (status
== VIRTIO_CONFIG_S_RESET
&&
356 (dev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
)) {
357 rc
= virtio_user_stop_device(vdev
);
361 dev
->status
|= VIRTIO_CONFIG_S_NEEDS_RESET
;
363 dev
->status
= status
;
368 virtio_user_get_status(struct virtio_dev
*vdev
)
370 struct virtio_user_dev
*dev
= vdev
->ctx
;
376 virtio_user_get_features(struct virtio_dev
*vdev
)
378 struct virtio_user_dev
*dev
= vdev
->ctx
;
382 rc
= dev
->ops
->send_request(dev
, VHOST_USER_GET_FEATURES
, &features
);
384 SPDK_ERRLOG("get_features failed: %s\n", spdk_strerror(-rc
));
392 virtio_user_set_features(struct virtio_dev
*vdev
, uint64_t features
)
394 struct virtio_user_dev
*dev
= vdev
->ctx
;
395 uint64_t protocol_features
;
398 ret
= dev
->ops
->send_request(dev
, VHOST_USER_SET_FEATURES
, &features
);
403 vdev
->negotiated_features
= features
;
404 vdev
->modern
= virtio_dev_has_feature(vdev
, VIRTIO_F_VERSION_1
);
406 if (!virtio_dev_has_feature(vdev
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
407 /* nothing else to do */
411 ret
= dev
->ops
->send_request(dev
, VHOST_USER_GET_PROTOCOL_FEATURES
, &protocol_features
);
416 protocol_features
&= VIRTIO_USER_SUPPORTED_PROTOCOL_FEATURES
;
417 ret
= dev
->ops
->send_request(dev
, VHOST_USER_SET_PROTOCOL_FEATURES
, &protocol_features
);
422 dev
->protocol_features
= protocol_features
;
427 virtio_user_get_queue_size(struct virtio_dev
*vdev
, uint16_t queue_id
)
429 struct virtio_user_dev
*dev
= vdev
->ctx
;
431 /* Currently each queue has same queue size */
432 return dev
->queue_size
;
436 virtio_user_setup_queue(struct virtio_dev
*vdev
, struct virtqueue
*vq
)
438 struct virtio_user_dev
*dev
= vdev
->ctx
;
439 struct vhost_vring_state state
;
440 uint16_t queue_idx
= vq
->vq_queue_index
;
442 uint64_t desc_addr
, avail_addr
, used_addr
;
443 int callfd
, kickfd
, rc
;
445 if (dev
->callfds
[queue_idx
] != -1 || dev
->kickfds
[queue_idx
] != -1) {
446 SPDK_ERRLOG("queue %"PRIu16
" already exists\n", queue_idx
);
450 /* May use invalid flag, but some backend uses kickfd and
451 * callfd as criteria to judge if dev is alive. so finally we
454 callfd
= eventfd(0, EFD_CLOEXEC
| EFD_NONBLOCK
);
456 SPDK_ERRLOG("callfd error, %s\n", spdk_strerror(errno
));
460 kickfd
= eventfd(0, EFD_CLOEXEC
| EFD_NONBLOCK
);
462 SPDK_ERRLOG("kickfd error, %s\n", spdk_strerror(errno
));
467 queue_mem
= spdk_zmalloc(vq
->vq_ring_size
, VIRTIO_PCI_VRING_ALIGN
, NULL
,
468 SPDK_ENV_LCORE_ID_ANY
, SPDK_MALLOC_DMA
);
469 if (queue_mem
== NULL
) {
475 vq
->vq_ring_mem
= SPDK_VTOPHYS_ERROR
;
476 vq
->vq_ring_virt_mem
= queue_mem
;
478 state
.index
= vq
->vq_queue_index
;
481 if (virtio_dev_has_feature(vdev
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
482 rc
= dev
->ops
->send_request(dev
, VHOST_USER_SET_VRING_ENABLE
, &state
);
484 SPDK_ERRLOG("failed to send VHOST_USER_SET_VRING_ENABLE: %s\n",
486 spdk_free(queue_mem
);
491 dev
->callfds
[queue_idx
] = callfd
;
492 dev
->kickfds
[queue_idx
] = kickfd
;
494 desc_addr
= (uintptr_t)vq
->vq_ring_virt_mem
;
495 avail_addr
= desc_addr
+ vq
->vq_nentries
* sizeof(struct vring_desc
);
496 used_addr
= RTE_ALIGN_CEIL(avail_addr
+ offsetof(struct vring_avail
,
497 ring
[vq
->vq_nentries
]),
498 VIRTIO_PCI_VRING_ALIGN
);
500 dev
->vrings
[queue_idx
].num
= vq
->vq_nentries
;
501 dev
->vrings
[queue_idx
].desc
= (void *)(uintptr_t)desc_addr
;
502 dev
->vrings
[queue_idx
].avail
= (void *)(uintptr_t)avail_addr
;
503 dev
->vrings
[queue_idx
].used
= (void *)(uintptr_t)used_addr
;
509 virtio_user_del_queue(struct virtio_dev
*vdev
, struct virtqueue
*vq
)
511 /* For legacy devices, write 0 to VIRTIO_PCI_QUEUE_PFN port, QEMU
512 * correspondingly stops the ioeventfds, and reset the status of
514 * For modern devices, set queue desc, avail, used in PCI bar to 0,
515 * not see any more behavior in QEMU.
517 * Here we just care about what information to deliver to vhost-user.
518 * So we just close ioeventfd for now.
520 struct virtio_user_dev
*dev
= vdev
->ctx
;
522 close(dev
->callfds
[vq
->vq_queue_index
]);
523 close(dev
->kickfds
[vq
->vq_queue_index
]);
524 dev
->callfds
[vq
->vq_queue_index
] = -1;
525 dev
->kickfds
[vq
->vq_queue_index
] = -1;
527 spdk_free(vq
->vq_ring_virt_mem
);
531 virtio_user_notify_queue(struct virtio_dev
*vdev
, struct virtqueue
*vq
)
534 struct virtio_user_dev
*dev
= vdev
->ctx
;
536 if (write(dev
->kickfds
[vq
->vq_queue_index
], &buf
, sizeof(buf
)) < 0) {
537 SPDK_ERRLOG("failed to kick backend: %s.\n", spdk_strerror(errno
));
542 virtio_user_destroy(struct virtio_dev
*vdev
)
544 struct virtio_user_dev
*dev
= vdev
->ctx
;
551 virtio_user_dump_json_info(struct virtio_dev
*vdev
, struct spdk_json_write_ctx
*w
)
553 struct virtio_user_dev
*dev
= vdev
->ctx
;
555 spdk_json_write_named_string(w
, "type", "user");
556 spdk_json_write_named_string(w
, "socket", dev
->path
);
560 virtio_user_write_json_config(struct virtio_dev
*vdev
, struct spdk_json_write_ctx
*w
)
562 struct virtio_user_dev
*dev
= vdev
->ctx
;
564 spdk_json_write_named_string(w
, "trtype", "user");
565 spdk_json_write_named_string(w
, "traddr", dev
->path
);
566 spdk_json_write_named_uint32(w
, "vq_count", vdev
->max_queues
- vdev
->fixed_queues_num
);
567 spdk_json_write_named_uint32(w
, "vq_size", virtio_dev_backend_ops(vdev
)->get_queue_size(vdev
, 0));
570 static const struct virtio_dev_ops virtio_user_ops
= {
571 .read_dev_cfg
= virtio_user_read_dev_config
,
572 .write_dev_cfg
= virtio_user_write_dev_config
,
573 .get_status
= virtio_user_get_status
,
574 .set_status
= virtio_user_set_status
,
575 .get_features
= virtio_user_get_features
,
576 .set_features
= virtio_user_set_features
,
577 .destruct_dev
= virtio_user_destroy
,
578 .get_queue_size
= virtio_user_get_queue_size
,
579 .setup_queue
= virtio_user_setup_queue
,
580 .del_queue
= virtio_user_del_queue
,
581 .notify_queue
= virtio_user_notify_queue
,
582 .dump_json_info
= virtio_user_dump_json_info
,
583 .write_json_config
= virtio_user_write_json_config
,
587 virtio_user_dev_init(struct virtio_dev
*vdev
, const char *name
, const char *path
,
590 struct virtio_user_dev
*dev
;
594 SPDK_ERRLOG("No name gived for controller: %s\n", path
);
598 dev
= calloc(1, sizeof(*dev
));
603 rc
= virtio_dev_construct(vdev
, name
, &virtio_user_ops
, dev
);
605 SPDK_ERRLOG("Failed to init device: %s\n", path
);
612 snprintf(dev
->path
, PATH_MAX
, "%s", path
);
613 dev
->queue_size
= queue_size
;
615 rc
= virtio_user_dev_setup(vdev
);
617 SPDK_ERRLOG("backend set up fails\n");
621 rc
= dev
->ops
->send_request(dev
, VHOST_USER_SET_OWNER
, NULL
);
623 SPDK_ERRLOG("set_owner fails: %s\n", spdk_strerror(-rc
));
630 virtio_dev_destruct(vdev
);