4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/migration.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState
{
36 struct vhost_vdpa vhost_vdpa
;
37 Notifier migration_state
;
38 VHostNetState
*vhost_net
;
40 /* Control commands shadow buffers */
41 void *cvq_cmd_out_buffer
;
42 virtio_net_ctrl_ack
*status
;
44 /* The device always have SVQ enabled */
49 const int vdpa_feature_bits
[] = {
50 VIRTIO_F_NOTIFY_ON_EMPTY
,
51 VIRTIO_RING_F_INDIRECT_DESC
,
52 VIRTIO_RING_F_EVENT_IDX
,
56 VIRTIO_NET_F_GUEST_CSUM
,
58 VIRTIO_NET_F_GUEST_TSO4
,
59 VIRTIO_NET_F_GUEST_TSO6
,
60 VIRTIO_NET_F_GUEST_ECN
,
61 VIRTIO_NET_F_GUEST_UFO
,
62 VIRTIO_NET_F_HOST_TSO4
,
63 VIRTIO_NET_F_HOST_TSO6
,
64 VIRTIO_NET_F_HOST_ECN
,
65 VIRTIO_NET_F_HOST_UFO
,
66 VIRTIO_NET_F_MRG_RXBUF
,
69 VIRTIO_NET_F_CTRL_RX_EXTRA
,
70 VIRTIO_NET_F_CTRL_VLAN
,
71 VIRTIO_NET_F_CTRL_MAC_ADDR
,
75 VIRTIO_F_IOMMU_PLATFORM
,
79 VIRTIO_NET_F_HASH_REPORT
,
81 VHOST_INVALID_FEATURE_BIT
84 /** Supported device specific feature bits with SVQ */
85 static const uint64_t vdpa_svq_device_features
=
86 BIT_ULL(VIRTIO_NET_F_CSUM
) |
87 BIT_ULL(VIRTIO_NET_F_GUEST_CSUM
) |
88 BIT_ULL(VIRTIO_NET_F_MTU
) |
89 BIT_ULL(VIRTIO_NET_F_MAC
) |
90 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4
) |
91 BIT_ULL(VIRTIO_NET_F_GUEST_TSO6
) |
92 BIT_ULL(VIRTIO_NET_F_GUEST_ECN
) |
93 BIT_ULL(VIRTIO_NET_F_GUEST_UFO
) |
94 BIT_ULL(VIRTIO_NET_F_HOST_TSO4
) |
95 BIT_ULL(VIRTIO_NET_F_HOST_TSO6
) |
96 BIT_ULL(VIRTIO_NET_F_HOST_ECN
) |
97 BIT_ULL(VIRTIO_NET_F_HOST_UFO
) |
98 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF
) |
99 BIT_ULL(VIRTIO_NET_F_STATUS
) |
100 BIT_ULL(VIRTIO_NET_F_CTRL_VQ
) |
101 BIT_ULL(VIRTIO_NET_F_MQ
) |
102 BIT_ULL(VIRTIO_F_ANY_LAYOUT
) |
103 BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR
) |
104 BIT_ULL(VIRTIO_NET_F_RSC_EXT
) |
105 BIT_ULL(VIRTIO_NET_F_STANDBY
);
107 #define VHOST_VDPA_NET_CVQ_ASID 1
109 VHostNetState
*vhost_vdpa_get_vhost_net(NetClientState
*nc
)
111 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
112 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
116 static bool vhost_vdpa_net_valid_svq_features(uint64_t features
, Error
**errp
)
118 uint64_t invalid_dev_features
=
119 features
& ~vdpa_svq_device_features
&
120 /* Transport are all accepted at this point */
121 ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START
,
122 VIRTIO_TRANSPORT_F_END
- VIRTIO_TRANSPORT_F_START
);
124 if (invalid_dev_features
) {
125 error_setg(errp
, "vdpa svq does not work with features 0x%" PRIx64
,
126 invalid_dev_features
);
130 return vhost_svq_valid_features(features
, errp
);
133 static int vhost_vdpa_net_check_device_id(struct vhost_net
*net
)
137 struct vhost_dev
*hdev
;
139 hdev
= (struct vhost_dev
*)&net
->dev
;
140 ret
= hdev
->vhost_ops
->vhost_get_device_id(hdev
, &device_id
);
141 if (device_id
!= VIRTIO_ID_NET
) {
147 static int vhost_vdpa_add(NetClientState
*ncs
, void *be
,
148 int queue_pair_index
, int nvqs
)
150 VhostNetOptions options
;
151 struct vhost_net
*net
= NULL
;
155 options
.backend_type
= VHOST_BACKEND_TYPE_VDPA
;
156 assert(ncs
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
157 s
= DO_UPCAST(VhostVDPAState
, nc
, ncs
);
158 options
.net_backend
= ncs
;
160 options
.busyloop_timeout
= 0;
163 net
= vhost_net_init(&options
);
165 error_report("failed to init vhost_net for queue");
169 ret
= vhost_vdpa_net_check_device_id(net
);
175 vhost_net_cleanup(net
);
181 static void vhost_vdpa_cleanup(NetClientState
*nc
)
183 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
185 qemu_vfree(s
->cvq_cmd_out_buffer
);
186 qemu_vfree(s
->status
);
188 vhost_net_cleanup(s
->vhost_net
);
189 g_free(s
->vhost_net
);
192 if (s
->vhost_vdpa
.device_fd
>= 0) {
193 qemu_close(s
->vhost_vdpa
.device_fd
);
194 s
->vhost_vdpa
.device_fd
= -1;
198 static bool vhost_vdpa_has_vnet_hdr(NetClientState
*nc
)
200 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
205 static bool vhost_vdpa_has_ufo(NetClientState
*nc
)
207 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
208 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
209 uint64_t features
= 0;
210 features
|= (1ULL << VIRTIO_NET_F_HOST_UFO
);
211 features
= vhost_net_get_features(s
->vhost_net
, features
);
212 return !!(features
& (1ULL << VIRTIO_NET_F_HOST_UFO
));
216 static bool vhost_vdpa_check_peer_type(NetClientState
*nc
, ObjectClass
*oc
,
219 const char *driver
= object_class_get_name(oc
);
221 if (!g_str_has_prefix(driver
, "virtio-net-")) {
222 error_setg(errp
, "vhost-vdpa requires frontend driver virtio-net-*");
229 /** Dummy receive in case qemu falls back to userland tap networking */
230 static ssize_t
vhost_vdpa_receive(NetClientState
*nc
, const uint8_t *buf
,
236 /** From any vdpa net client, get the netclient of the first queue pair */
237 static VhostVDPAState
*vhost_vdpa_net_first_nc_vdpa(VhostVDPAState
*s
)
239 NICState
*nic
= qemu_get_nic(s
->nc
.peer
);
240 NetClientState
*nc0
= qemu_get_peer(nic
->ncs
, 0);
242 return DO_UPCAST(VhostVDPAState
, nc
, nc0
);
245 static void vhost_vdpa_net_log_global_enable(VhostVDPAState
*s
, bool enable
)
247 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
250 int data_queue_pairs
, cvq
, r
;
252 /* We are only called on the first data vqs and only if x-svq is not set */
253 if (s
->vhost_vdpa
.shadow_vqs_enabled
== enable
) {
258 n
= VIRTIO_NET(vdev
);
259 if (!n
->vhost_started
) {
263 data_queue_pairs
= n
->multiqueue
? n
->max_queue_pairs
: 1;
264 cvq
= virtio_vdev_has_feature(vdev
, VIRTIO_NET_F_CTRL_VQ
) ?
265 n
->max_ncs
- n
->max_queue_pairs
: 0;
267 * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
268 * in the future and resume the device if read-only operations between
269 * suspend and reset goes wrong.
271 vhost_net_stop(vdev
, n
->nic
->ncs
, data_queue_pairs
, cvq
);
273 /* Start will check migration setup_or_active to configure or not SVQ */
274 r
= vhost_net_start(vdev
, n
->nic
->ncs
, data_queue_pairs
, cvq
);
275 if (unlikely(r
< 0)) {
276 error_report("unable to start vhost net: %s(%d)", g_strerror(-r
), -r
);
280 static void vdpa_net_migration_state_notifier(Notifier
*notifier
, void *data
)
282 MigrationState
*migration
= data
;
283 VhostVDPAState
*s
= container_of(notifier
, VhostVDPAState
,
286 if (migration_in_setup(migration
)) {
287 vhost_vdpa_net_log_global_enable(s
, true);
288 } else if (migration_has_failed(migration
)) {
289 vhost_vdpa_net_log_global_enable(s
, false);
293 static void vhost_vdpa_net_data_start_first(VhostVDPAState
*s
)
295 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
297 add_migration_state_change_notifier(&s
->migration_state
);
298 if (v
->shadow_vqs_enabled
) {
299 v
->iova_tree
= vhost_iova_tree_new(v
->iova_range
.first
,
304 static int vhost_vdpa_net_data_start(NetClientState
*nc
)
306 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
307 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
309 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
312 migration_is_setup_or_active(migrate_get_current()->state
)) {
313 v
->shadow_vqs_enabled
= true;
314 v
->shadow_data
= true;
316 v
->shadow_vqs_enabled
= false;
317 v
->shadow_data
= false;
321 vhost_vdpa_net_data_start_first(s
);
325 if (v
->shadow_vqs_enabled
) {
326 VhostVDPAState
*s0
= vhost_vdpa_net_first_nc_vdpa(s
);
327 v
->iova_tree
= s0
->vhost_vdpa
.iova_tree
;
333 static void vhost_vdpa_net_client_stop(NetClientState
*nc
)
335 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
336 struct vhost_dev
*dev
;
338 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
340 if (s
->vhost_vdpa
.index
== 0) {
341 remove_migration_state_change_notifier(&s
->migration_state
);
344 dev
= s
->vhost_vdpa
.dev
;
345 if (dev
->vq_index
+ dev
->nvqs
== dev
->vq_index_end
) {
346 g_clear_pointer(&s
->vhost_vdpa
.iova_tree
, vhost_iova_tree_delete
);
350 static NetClientInfo net_vhost_vdpa_info
= {
351 .type
= NET_CLIENT_DRIVER_VHOST_VDPA
,
352 .size
= sizeof(VhostVDPAState
),
353 .receive
= vhost_vdpa_receive
,
354 .start
= vhost_vdpa_net_data_start
,
355 .stop
= vhost_vdpa_net_client_stop
,
356 .cleanup
= vhost_vdpa_cleanup
,
357 .has_vnet_hdr
= vhost_vdpa_has_vnet_hdr
,
358 .has_ufo
= vhost_vdpa_has_ufo
,
359 .check_peer_type
= vhost_vdpa_check_peer_type
,
362 static int64_t vhost_vdpa_get_vring_group(int device_fd
, unsigned vq_index
)
364 struct vhost_vring_state state
= {
367 int r
= ioctl(device_fd
, VHOST_VDPA_GET_VRING_GROUP
, &state
);
369 if (unlikely(r
< 0)) {
370 error_report("Cannot get VQ %u group: %s", vq_index
,
378 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa
*v
,
382 struct vhost_vring_state asid
= {
388 r
= ioctl(v
->device_fd
, VHOST_VDPA_SET_GROUP_ASID
, &asid
);
389 if (unlikely(r
< 0)) {
390 error_report("Can't set vq group %u asid %u, errno=%d (%s)",
391 asid
.index
, asid
.num
, errno
, g_strerror(errno
));
396 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa
*v
, void *addr
)
398 VhostIOVATree
*tree
= v
->iova_tree
;
401 * No need to specify size or to look for more translations since
402 * this contiguous chunk was allocated by us.
404 .translated_addr
= (hwaddr
)(uintptr_t)addr
,
406 const DMAMap
*map
= vhost_iova_tree_find_iova(tree
, &needle
);
409 if (unlikely(!map
)) {
410 error_report("Cannot locate expected map");
414 r
= vhost_vdpa_dma_unmap(v
, v
->address_space_id
, map
->iova
, map
->size
+ 1);
415 if (unlikely(r
!= 0)) {
416 error_report("Device cannot unmap: %s(%d)", g_strerror(r
), r
);
419 vhost_iova_tree_remove(tree
, *map
);
422 static size_t vhost_vdpa_net_cvq_cmd_len(void)
425 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
426 * In buffer is always 1 byte, so it should fit here
428 return sizeof(struct virtio_net_ctrl_hdr
) +
429 2 * sizeof(struct virtio_net_ctrl_mac
) +
430 MAC_TABLE_ENTRIES
* ETH_ALEN
;
433 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
435 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
438 /** Map CVQ buffer. */
439 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa
*v
, void *buf
, size_t size
,
445 map
.translated_addr
= (hwaddr
)(uintptr_t)buf
;
447 map
.perm
= write
? IOMMU_RW
: IOMMU_RO
,
448 r
= vhost_iova_tree_map_alloc(v
->iova_tree
, &map
);
449 if (unlikely(r
!= IOVA_OK
)) {
450 error_report("Cannot map injected element");
454 r
= vhost_vdpa_dma_map(v
, v
->address_space_id
, map
.iova
,
455 vhost_vdpa_net_cvq_cmd_page_len(), buf
, !write
);
456 if (unlikely(r
< 0)) {
463 vhost_iova_tree_remove(v
->iova_tree
, map
);
467 static int vhost_vdpa_net_cvq_start(NetClientState
*nc
)
469 VhostVDPAState
*s
, *s0
;
470 struct vhost_vdpa
*v
;
471 uint64_t backend_features
;
475 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
477 s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
480 s0
= vhost_vdpa_net_first_nc_vdpa(s
);
481 v
->shadow_data
= s0
->vhost_vdpa
.shadow_vqs_enabled
;
482 v
->shadow_vqs_enabled
= s
->always_svq
;
483 s
->vhost_vdpa
.address_space_id
= VHOST_VDPA_GUEST_PA_ASID
;
485 if (s
->vhost_vdpa
.shadow_data
) {
486 /* SVQ is already configured for all virtqueues */
491 * If we early return in these cases SVQ will not be enabled. The migration
492 * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
494 * Calling VHOST_GET_BACKEND_FEATURES as they are not available in v->dev
497 r
= ioctl(v
->device_fd
, VHOST_GET_BACKEND_FEATURES
, &backend_features
);
498 if (unlikely(r
< 0)) {
499 error_report("Cannot get vdpa backend_features: %s(%d)",
500 g_strerror(errno
), errno
);
503 if (!(backend_features
& BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID
)) ||
504 !vhost_vdpa_net_valid_svq_features(v
->dev
->features
, NULL
)) {
509 * Check if all the virtqueues of the virtio device are in a different vq
510 * than the last vq. VQ group of last group passed in cvq_group.
512 cvq_index
= v
->dev
->vq_index_end
- 1;
513 cvq_group
= vhost_vdpa_get_vring_group(v
->device_fd
, cvq_index
);
514 if (unlikely(cvq_group
< 0)) {
517 for (int i
= 0; i
< cvq_index
; ++i
) {
518 int64_t group
= vhost_vdpa_get_vring_group(v
->device_fd
, i
);
520 if (unlikely(group
< 0)) {
524 if (group
== cvq_group
) {
529 r
= vhost_vdpa_set_address_space_id(v
, cvq_group
, VHOST_VDPA_NET_CVQ_ASID
);
530 if (unlikely(r
< 0)) {
534 v
->shadow_vqs_enabled
= true;
535 s
->vhost_vdpa
.address_space_id
= VHOST_VDPA_NET_CVQ_ASID
;
538 if (!s
->vhost_vdpa
.shadow_vqs_enabled
) {
542 if (s0
->vhost_vdpa
.iova_tree
) {
544 * SVQ is already configured for all virtqueues. Reuse IOVA tree for
545 * simplicity, whether CVQ shares ASID with guest or not, because:
546 * - Memory listener need access to guest's memory addresses allocated
548 * - There should be plenty of IOVA address space for both ASID not to
549 * worry about collisions between them. Guest's translations are
550 * still validated with virtio virtqueue_pop so there is no risk for
551 * the guest to access memory that it shouldn't.
553 * To allocate a iova tree per ASID is doable but it complicates the
554 * code and it is not worth it for the moment.
556 v
->iova_tree
= s0
->vhost_vdpa
.iova_tree
;
558 v
->iova_tree
= vhost_iova_tree_new(v
->iova_range
.first
,
562 r
= vhost_vdpa_cvq_map_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
,
563 vhost_vdpa_net_cvq_cmd_page_len(), false);
564 if (unlikely(r
< 0)) {
568 r
= vhost_vdpa_cvq_map_buf(&s
->vhost_vdpa
, s
->status
,
569 vhost_vdpa_net_cvq_cmd_page_len(), true);
570 if (unlikely(r
< 0)) {
571 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
);
577 static void vhost_vdpa_net_cvq_stop(NetClientState
*nc
)
579 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
581 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
583 if (s
->vhost_vdpa
.shadow_vqs_enabled
) {
584 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
);
585 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->status
);
588 vhost_vdpa_net_client_stop(nc
);
591 static ssize_t
vhost_vdpa_net_cvq_add(VhostVDPAState
*s
, size_t out_len
,
594 /* Buffers for the device */
595 const struct iovec out
= {
596 .iov_base
= s
->cvq_cmd_out_buffer
,
599 const struct iovec in
= {
600 .iov_base
= s
->status
,
601 .iov_len
= sizeof(virtio_net_ctrl_ack
),
603 VhostShadowVirtqueue
*svq
= g_ptr_array_index(s
->vhost_vdpa
.shadow_vqs
, 0);
606 r
= vhost_svq_add(svq
, &out
, 1, &in
, 1, NULL
);
607 if (unlikely(r
!= 0)) {
608 if (unlikely(r
== -ENOSPC
)) {
609 qemu_log_mask(LOG_GUEST_ERROR
, "%s: No space on device queue\n",
616 * We can poll here since we've had BQL from the time we sent the
617 * descriptor. Also, we need to take the answer before SVQ pulls by itself,
618 * when BQL is released
620 return vhost_svq_poll(svq
);
623 static ssize_t
vhost_vdpa_net_load_cmd(VhostVDPAState
*s
, uint8_t class,
624 uint8_t cmd
, const void *data
,
627 const struct virtio_net_ctrl_hdr ctrl
= {
632 assert(data_size
< vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl
));
634 memcpy(s
->cvq_cmd_out_buffer
, &ctrl
, sizeof(ctrl
));
635 memcpy(s
->cvq_cmd_out_buffer
+ sizeof(ctrl
), data
, data_size
);
637 return vhost_vdpa_net_cvq_add(s
, sizeof(ctrl
) + data_size
,
638 sizeof(virtio_net_ctrl_ack
));
641 static int vhost_vdpa_net_load_mac(VhostVDPAState
*s
, const VirtIONet
*n
)
643 uint64_t features
= n
->parent_obj
.guest_features
;
644 if (features
& BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR
)) {
645 ssize_t dev_written
= vhost_vdpa_net_load_cmd(s
, VIRTIO_NET_CTRL_MAC
,
646 VIRTIO_NET_CTRL_MAC_ADDR_SET
,
647 n
->mac
, sizeof(n
->mac
));
648 if (unlikely(dev_written
< 0)) {
652 return *s
->status
!= VIRTIO_NET_OK
;
658 static int vhost_vdpa_net_load_mq(VhostVDPAState
*s
,
661 struct virtio_net_ctrl_mq mq
;
662 uint64_t features
= n
->parent_obj
.guest_features
;
665 if (!(features
& BIT_ULL(VIRTIO_NET_F_MQ
))) {
669 mq
.virtqueue_pairs
= cpu_to_le16(n
->curr_queue_pairs
);
670 dev_written
= vhost_vdpa_net_load_cmd(s
, VIRTIO_NET_CTRL_MQ
,
671 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
, &mq
,
673 if (unlikely(dev_written
< 0)) {
677 return *s
->status
!= VIRTIO_NET_OK
;
680 static int vhost_vdpa_net_load(NetClientState
*nc
)
682 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
683 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
687 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
689 if (!v
->shadow_vqs_enabled
) {
693 n
= VIRTIO_NET(v
->dev
->vdev
);
694 r
= vhost_vdpa_net_load_mac(s
, n
);
695 if (unlikely(r
< 0)) {
698 r
= vhost_vdpa_net_load_mq(s
, n
);
706 static NetClientInfo net_vhost_vdpa_cvq_info
= {
707 .type
= NET_CLIENT_DRIVER_VHOST_VDPA
,
708 .size
= sizeof(VhostVDPAState
),
709 .receive
= vhost_vdpa_receive
,
710 .start
= vhost_vdpa_net_cvq_start
,
711 .load
= vhost_vdpa_net_load
,
712 .stop
= vhost_vdpa_net_cvq_stop
,
713 .cleanup
= vhost_vdpa_cleanup
,
714 .has_vnet_hdr
= vhost_vdpa_has_vnet_hdr
,
715 .has_ufo
= vhost_vdpa_has_ufo
,
716 .check_peer_type
= vhost_vdpa_check_peer_type
,
720 * Validate and copy control virtqueue commands.
722 * Following QEMU guidelines, we offer a copy of the buffers to the device to
723 * prevent TOCTOU bugs.
725 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue
*svq
,
726 VirtQueueElement
*elem
,
729 VhostVDPAState
*s
= opaque
;
731 virtio_net_ctrl_ack status
= VIRTIO_NET_ERR
;
732 /* Out buffer sent to both the vdpa device and the device model */
734 .iov_base
= s
->cvq_cmd_out_buffer
,
736 /* in buffer used for device model */
737 const struct iovec in
= {
739 .iov_len
= sizeof(status
),
741 ssize_t dev_written
= -EINVAL
;
743 out
.iov_len
= iov_to_buf(elem
->out_sg
, elem
->out_num
, 0,
744 s
->cvq_cmd_out_buffer
,
745 vhost_vdpa_net_cvq_cmd_len());
746 if (*(uint8_t *)s
->cvq_cmd_out_buffer
== VIRTIO_NET_CTRL_ANNOUNCE
) {
748 * Guest announce capability is emulated by qemu, so don't forward to
751 dev_written
= sizeof(status
);
752 *s
->status
= VIRTIO_NET_OK
;
754 dev_written
= vhost_vdpa_net_cvq_add(s
, out
.iov_len
, sizeof(status
));
755 if (unlikely(dev_written
< 0)) {
760 if (unlikely(dev_written
< sizeof(status
))) {
761 error_report("Insufficient written data (%zu)", dev_written
);
765 if (*s
->status
!= VIRTIO_NET_OK
) {
766 return VIRTIO_NET_ERR
;
769 status
= VIRTIO_NET_ERR
;
770 virtio_net_handle_ctrl_iov(svq
->vdev
, &in
, 1, &out
, 1);
771 if (status
!= VIRTIO_NET_OK
) {
772 error_report("Bad CVQ processing in model");
776 in_len
= iov_from_buf(elem
->in_sg
, elem
->in_num
, 0, &status
,
778 if (unlikely(in_len
< sizeof(status
))) {
779 error_report("Bad device CVQ written length");
781 vhost_svq_push_elem(svq
, elem
, MIN(in_len
, sizeof(status
)));
783 return dev_written
< 0 ? dev_written
: 0;
786 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops
= {
787 .avail_handler
= vhost_vdpa_net_handle_ctrl_avail
,
790 static NetClientState
*net_vhost_vdpa_init(NetClientState
*peer
,
794 int queue_pair_index
,
798 struct vhost_vdpa_iova_range iova_range
,
801 NetClientState
*nc
= NULL
;
806 nc
= qemu_new_net_client(&net_vhost_vdpa_info
, peer
, device
,
809 nc
= qemu_new_net_control_client(&net_vhost_vdpa_cvq_info
, peer
,
812 qemu_set_info_str(nc
, TYPE_VHOST_VDPA
);
813 s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
815 s
->vhost_vdpa
.device_fd
= vdpa_device_fd
;
816 s
->vhost_vdpa
.index
= queue_pair_index
;
818 s
->migration_state
.notify
= vdpa_net_migration_state_notifier
;
819 s
->vhost_vdpa
.shadow_vqs_enabled
= svq
;
820 s
->vhost_vdpa
.iova_range
= iova_range
;
821 s
->vhost_vdpa
.shadow_data
= svq
;
822 if (queue_pair_index
== 0) {
823 vhost_vdpa_net_valid_svq_features(features
,
824 &s
->vhost_vdpa
.migration_blocker
);
825 } else if (!is_datapath
) {
826 s
->cvq_cmd_out_buffer
= qemu_memalign(qemu_real_host_page_size(),
827 vhost_vdpa_net_cvq_cmd_page_len());
828 memset(s
->cvq_cmd_out_buffer
, 0, vhost_vdpa_net_cvq_cmd_page_len());
829 s
->status
= qemu_memalign(qemu_real_host_page_size(),
830 vhost_vdpa_net_cvq_cmd_page_len());
831 memset(s
->status
, 0, vhost_vdpa_net_cvq_cmd_page_len());
833 s
->vhost_vdpa
.shadow_vq_ops
= &vhost_vdpa_net_svq_ops
;
834 s
->vhost_vdpa
.shadow_vq_ops_opaque
= s
;
837 * TODO: We cannot migrate devices with CVQ as there is no way to set
838 * the device state (MAC, MQ, etc) before starting the datapath.
840 * Migration blocker ownership now belongs to s->vhost_vdpa.
842 error_setg(&s
->vhost_vdpa
.migration_blocker
,
843 "net vdpa cannot migrate with CVQ feature");
845 ret
= vhost_vdpa_add(nc
, (void *)&s
->vhost_vdpa
, queue_pair_index
, nvqs
);
847 qemu_del_net_client(nc
);
853 static int vhost_vdpa_get_features(int fd
, uint64_t *features
, Error
**errp
)
855 int ret
= ioctl(fd
, VHOST_GET_FEATURES
, features
);
856 if (unlikely(ret
< 0)) {
857 error_setg_errno(errp
, errno
,
858 "Fail to query features from vhost-vDPA device");
863 static int vhost_vdpa_get_max_queue_pairs(int fd
, uint64_t features
,
864 int *has_cvq
, Error
**errp
)
866 unsigned long config_size
= offsetof(struct vhost_vdpa_config
, buf
);
867 g_autofree
struct vhost_vdpa_config
*config
= NULL
;
868 __virtio16
*max_queue_pairs
;
871 if (features
& (1 << VIRTIO_NET_F_CTRL_VQ
)) {
877 if (features
& (1 << VIRTIO_NET_F_MQ
)) {
878 config
= g_malloc0(config_size
+ sizeof(*max_queue_pairs
));
879 config
->off
= offsetof(struct virtio_net_config
, max_virtqueue_pairs
);
880 config
->len
= sizeof(*max_queue_pairs
);
882 ret
= ioctl(fd
, VHOST_VDPA_GET_CONFIG
, config
);
884 error_setg(errp
, "Fail to get config from vhost-vDPA device");
888 max_queue_pairs
= (__virtio16
*)&config
->buf
;
890 return lduw_le_p(max_queue_pairs
);
896 int net_init_vhost_vdpa(const Netdev
*netdev
, const char *name
,
897 NetClientState
*peer
, Error
**errp
)
899 const NetdevVhostVDPAOptions
*opts
;
902 g_autofree NetClientState
**ncs
= NULL
;
903 struct vhost_vdpa_iova_range iova_range
;
905 int queue_pairs
, r
, i
= 0, has_cvq
= 0;
907 assert(netdev
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
908 opts
= &netdev
->u
.vhost_vdpa
;
909 if (!opts
->vhostdev
&& !opts
->vhostfd
) {
911 "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
915 if (opts
->vhostdev
&& opts
->vhostfd
) {
917 "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
921 if (opts
->vhostdev
) {
922 vdpa_device_fd
= qemu_open(opts
->vhostdev
, O_RDWR
, errp
);
923 if (vdpa_device_fd
== -1) {
928 vdpa_device_fd
= monitor_fd_param(monitor_cur(), opts
->vhostfd
, errp
);
929 if (vdpa_device_fd
== -1) {
930 error_prepend(errp
, "vhost-vdpa: unable to parse vhostfd: ");
935 r
= vhost_vdpa_get_features(vdpa_device_fd
, &features
, errp
);
936 if (unlikely(r
< 0)) {
940 queue_pairs
= vhost_vdpa_get_max_queue_pairs(vdpa_device_fd
, features
,
942 if (queue_pairs
< 0) {
943 qemu_close(vdpa_device_fd
);
947 r
= vhost_vdpa_get_iova_range(vdpa_device_fd
, &iova_range
);
948 if (unlikely(r
< 0)) {
949 error_setg(errp
, "vhost-vdpa: get iova range failed: %s",
954 if (opts
->x_svq
&& !vhost_vdpa_net_valid_svq_features(features
, errp
)) {
958 ncs
= g_malloc0(sizeof(*ncs
) * queue_pairs
);
960 for (i
= 0; i
< queue_pairs
; i
++) {
961 ncs
[i
] = net_vhost_vdpa_init(peer
, TYPE_VHOST_VDPA
, name
,
962 vdpa_device_fd
, i
, 2, true, opts
->x_svq
,
963 iova_range
, features
);
969 nc
= net_vhost_vdpa_init(peer
, TYPE_VHOST_VDPA
, name
,
970 vdpa_device_fd
, i
, 1, false,
971 opts
->x_svq
, iova_range
, features
);
980 for (i
--; i
>= 0; i
--) {
981 qemu_del_net_client(ncs
[i
]);
985 qemu_close(vdpa_device_fd
);