4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/migration.h"
30 #include "migration/misc.h"
31 #include "hw/virtio/vhost.h"
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState
{
36 struct vhost_vdpa vhost_vdpa
;
37 Notifier migration_state
;
38 VHostNetState
*vhost_net
;
40 /* Control commands shadow buffers */
41 void *cvq_cmd_out_buffer
;
42 virtio_net_ctrl_ack
*status
;
44 /* The device always have SVQ enabled */
47 /* The device can isolate CVQ in its own ASID */
54 * The array is sorted alphabetically in ascending order,
55 * with the exception of VHOST_INVALID_FEATURE_BIT,
56 * which should always be the last entry.
58 const int vdpa_feature_bits
[] = {
60 VIRTIO_F_IOMMU_PLATFORM
,
61 VIRTIO_F_NOTIFY_ON_EMPTY
,
66 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
,
67 VIRTIO_NET_F_CTRL_MAC_ADDR
,
69 VIRTIO_NET_F_CTRL_RX_EXTRA
,
70 VIRTIO_NET_F_CTRL_VLAN
,
73 VIRTIO_NET_F_GUEST_CSUM
,
74 VIRTIO_NET_F_GUEST_ECN
,
75 VIRTIO_NET_F_GUEST_TSO4
,
76 VIRTIO_NET_F_GUEST_TSO6
,
77 VIRTIO_NET_F_GUEST_UFO
,
78 VIRTIO_NET_F_HASH_REPORT
,
79 VIRTIO_NET_F_HOST_ECN
,
80 VIRTIO_NET_F_HOST_TSO4
,
81 VIRTIO_NET_F_HOST_TSO6
,
82 VIRTIO_NET_F_HOST_UFO
,
84 VIRTIO_NET_F_MRG_RXBUF
,
88 VIRTIO_RING_F_EVENT_IDX
,
89 VIRTIO_RING_F_INDIRECT_DESC
,
91 /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
92 VHOST_INVALID_FEATURE_BIT
95 /** Supported device specific feature bits with SVQ */
96 static const uint64_t vdpa_svq_device_features
=
97 BIT_ULL(VIRTIO_NET_F_CSUM
) |
98 BIT_ULL(VIRTIO_NET_F_GUEST_CSUM
) |
99 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
) |
100 BIT_ULL(VIRTIO_NET_F_MTU
) |
101 BIT_ULL(VIRTIO_NET_F_MAC
) |
102 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4
) |
103 BIT_ULL(VIRTIO_NET_F_GUEST_TSO6
) |
104 BIT_ULL(VIRTIO_NET_F_GUEST_ECN
) |
105 BIT_ULL(VIRTIO_NET_F_GUEST_UFO
) |
106 BIT_ULL(VIRTIO_NET_F_HOST_TSO4
) |
107 BIT_ULL(VIRTIO_NET_F_HOST_TSO6
) |
108 BIT_ULL(VIRTIO_NET_F_HOST_ECN
) |
109 BIT_ULL(VIRTIO_NET_F_HOST_UFO
) |
110 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF
) |
111 BIT_ULL(VIRTIO_NET_F_STATUS
) |
112 BIT_ULL(VIRTIO_NET_F_CTRL_VQ
) |
113 BIT_ULL(VIRTIO_NET_F_CTRL_RX
) |
114 BIT_ULL(VIRTIO_NET_F_MQ
) |
115 BIT_ULL(VIRTIO_F_ANY_LAYOUT
) |
116 BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR
) |
117 /* VHOST_F_LOG_ALL is exposed by SVQ */
118 BIT_ULL(VHOST_F_LOG_ALL
) |
119 BIT_ULL(VIRTIO_NET_F_RSC_EXT
) |
120 BIT_ULL(VIRTIO_NET_F_STANDBY
) |
121 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX
);
123 #define VHOST_VDPA_NET_CVQ_ASID 1
125 VHostNetState
*vhost_vdpa_get_vhost_net(NetClientState
*nc
)
127 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
128 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
132 static size_t vhost_vdpa_net_cvq_cmd_len(void)
135 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
136 * In buffer is always 1 byte, so it should fit here
138 return sizeof(struct virtio_net_ctrl_hdr
) +
139 2 * sizeof(struct virtio_net_ctrl_mac
) +
140 MAC_TABLE_ENTRIES
* ETH_ALEN
;
143 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
145 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
148 static bool vhost_vdpa_net_valid_svq_features(uint64_t features
, Error
**errp
)
150 uint64_t invalid_dev_features
=
151 features
& ~vdpa_svq_device_features
&
152 /* Transport are all accepted at this point */
153 ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START
,
154 VIRTIO_TRANSPORT_F_END
- VIRTIO_TRANSPORT_F_START
);
156 if (invalid_dev_features
) {
157 error_setg(errp
, "vdpa svq does not work with features 0x%" PRIx64
,
158 invalid_dev_features
);
162 return vhost_svq_valid_features(features
, errp
);
165 static int vhost_vdpa_net_check_device_id(struct vhost_net
*net
)
169 struct vhost_dev
*hdev
;
171 hdev
= (struct vhost_dev
*)&net
->dev
;
172 ret
= hdev
->vhost_ops
->vhost_get_device_id(hdev
, &device_id
);
173 if (device_id
!= VIRTIO_ID_NET
) {
179 static int vhost_vdpa_add(NetClientState
*ncs
, void *be
,
180 int queue_pair_index
, int nvqs
)
182 VhostNetOptions options
;
183 struct vhost_net
*net
= NULL
;
187 options
.backend_type
= VHOST_BACKEND_TYPE_VDPA
;
188 assert(ncs
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
189 s
= DO_UPCAST(VhostVDPAState
, nc
, ncs
);
190 options
.net_backend
= ncs
;
192 options
.busyloop_timeout
= 0;
195 net
= vhost_net_init(&options
);
197 error_report("failed to init vhost_net for queue");
201 ret
= vhost_vdpa_net_check_device_id(net
);
207 vhost_net_cleanup(net
);
213 static void vhost_vdpa_cleanup(NetClientState
*nc
)
215 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
218 * If a peer NIC is attached, do not cleanup anything.
219 * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
220 * when the guest is shutting down.
222 if (nc
->peer
&& nc
->peer
->info
->type
== NET_CLIENT_DRIVER_NIC
) {
225 munmap(s
->cvq_cmd_out_buffer
, vhost_vdpa_net_cvq_cmd_page_len());
226 munmap(s
->status
, vhost_vdpa_net_cvq_cmd_page_len());
228 vhost_net_cleanup(s
->vhost_net
);
229 g_free(s
->vhost_net
);
232 if (s
->vhost_vdpa
.device_fd
>= 0) {
233 qemu_close(s
->vhost_vdpa
.device_fd
);
234 s
->vhost_vdpa
.device_fd
= -1;
238 static bool vhost_vdpa_has_vnet_hdr(NetClientState
*nc
)
240 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
245 static bool vhost_vdpa_has_ufo(NetClientState
*nc
)
247 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
248 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
249 uint64_t features
= 0;
250 features
|= (1ULL << VIRTIO_NET_F_HOST_UFO
);
251 features
= vhost_net_get_features(s
->vhost_net
, features
);
252 return !!(features
& (1ULL << VIRTIO_NET_F_HOST_UFO
));
256 static bool vhost_vdpa_check_peer_type(NetClientState
*nc
, ObjectClass
*oc
,
259 const char *driver
= object_class_get_name(oc
);
261 if (!g_str_has_prefix(driver
, "virtio-net-")) {
262 error_setg(errp
, "vhost-vdpa requires frontend driver virtio-net-*");
269 /** Dummy receive in case qemu falls back to userland tap networking */
270 static ssize_t
vhost_vdpa_receive(NetClientState
*nc
, const uint8_t *buf
,
276 /** From any vdpa net client, get the netclient of the first queue pair */
277 static VhostVDPAState
*vhost_vdpa_net_first_nc_vdpa(VhostVDPAState
*s
)
279 NICState
*nic
= qemu_get_nic(s
->nc
.peer
);
280 NetClientState
*nc0
= qemu_get_peer(nic
->ncs
, 0);
282 return DO_UPCAST(VhostVDPAState
, nc
, nc0
);
285 static void vhost_vdpa_net_log_global_enable(VhostVDPAState
*s
, bool enable
)
287 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
290 int data_queue_pairs
, cvq
, r
;
292 /* We are only called on the first data vqs and only if x-svq is not set */
293 if (s
->vhost_vdpa
.shadow_vqs_enabled
== enable
) {
298 n
= VIRTIO_NET(vdev
);
299 if (!n
->vhost_started
) {
303 data_queue_pairs
= n
->multiqueue
? n
->max_queue_pairs
: 1;
304 cvq
= virtio_vdev_has_feature(vdev
, VIRTIO_NET_F_CTRL_VQ
) ?
305 n
->max_ncs
- n
->max_queue_pairs
: 0;
307 * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
308 * in the future and resume the device if read-only operations between
309 * suspend and reset goes wrong.
311 vhost_net_stop(vdev
, n
->nic
->ncs
, data_queue_pairs
, cvq
);
313 /* Start will check migration setup_or_active to configure or not SVQ */
314 r
= vhost_net_start(vdev
, n
->nic
->ncs
, data_queue_pairs
, cvq
);
315 if (unlikely(r
< 0)) {
316 error_report("unable to start vhost net: %s(%d)", g_strerror(-r
), -r
);
320 static void vdpa_net_migration_state_notifier(Notifier
*notifier
, void *data
)
322 MigrationState
*migration
= data
;
323 VhostVDPAState
*s
= container_of(notifier
, VhostVDPAState
,
326 if (migration_in_setup(migration
)) {
327 vhost_vdpa_net_log_global_enable(s
, true);
328 } else if (migration_has_failed(migration
)) {
329 vhost_vdpa_net_log_global_enable(s
, false);
333 static void vhost_vdpa_net_data_start_first(VhostVDPAState
*s
)
335 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
337 add_migration_state_change_notifier(&s
->migration_state
);
338 if (v
->shadow_vqs_enabled
) {
339 v
->iova_tree
= vhost_iova_tree_new(v
->iova_range
.first
,
344 static int vhost_vdpa_net_data_start(NetClientState
*nc
)
346 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
347 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
349 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
352 migration_is_setup_or_active(migrate_get_current()->state
)) {
353 v
->shadow_vqs_enabled
= true;
354 v
->shadow_data
= true;
356 v
->shadow_vqs_enabled
= false;
357 v
->shadow_data
= false;
361 vhost_vdpa_net_data_start_first(s
);
365 if (v
->shadow_vqs_enabled
) {
366 VhostVDPAState
*s0
= vhost_vdpa_net_first_nc_vdpa(s
);
367 v
->iova_tree
= s0
->vhost_vdpa
.iova_tree
;
373 static void vhost_vdpa_net_client_stop(NetClientState
*nc
)
375 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
376 struct vhost_dev
*dev
;
378 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
380 if (s
->vhost_vdpa
.index
== 0) {
381 remove_migration_state_change_notifier(&s
->migration_state
);
384 dev
= s
->vhost_vdpa
.dev
;
385 if (dev
->vq_index
+ dev
->nvqs
== dev
->vq_index_end
) {
386 g_clear_pointer(&s
->vhost_vdpa
.iova_tree
, vhost_iova_tree_delete
);
390 static NetClientInfo net_vhost_vdpa_info
= {
391 .type
= NET_CLIENT_DRIVER_VHOST_VDPA
,
392 .size
= sizeof(VhostVDPAState
),
393 .receive
= vhost_vdpa_receive
,
394 .start
= vhost_vdpa_net_data_start
,
395 .stop
= vhost_vdpa_net_client_stop
,
396 .cleanup
= vhost_vdpa_cleanup
,
397 .has_vnet_hdr
= vhost_vdpa_has_vnet_hdr
,
398 .has_ufo
= vhost_vdpa_has_ufo
,
399 .check_peer_type
= vhost_vdpa_check_peer_type
,
402 static int64_t vhost_vdpa_get_vring_group(int device_fd
, unsigned vq_index
,
405 struct vhost_vring_state state
= {
408 int r
= ioctl(device_fd
, VHOST_VDPA_GET_VRING_GROUP
, &state
);
410 if (unlikely(r
< 0)) {
412 error_setg_errno(errp
, errno
, "Cannot get VQ %u group", vq_index
);
419 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa
*v
,
423 struct vhost_vring_state asid
= {
429 r
= ioctl(v
->device_fd
, VHOST_VDPA_SET_GROUP_ASID
, &asid
);
430 if (unlikely(r
< 0)) {
431 error_report("Can't set vq group %u asid %u, errno=%d (%s)",
432 asid
.index
, asid
.num
, errno
, g_strerror(errno
));
437 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa
*v
, void *addr
)
439 VhostIOVATree
*tree
= v
->iova_tree
;
442 * No need to specify size or to look for more translations since
443 * this contiguous chunk was allocated by us.
445 .translated_addr
= (hwaddr
)(uintptr_t)addr
,
447 const DMAMap
*map
= vhost_iova_tree_find_iova(tree
, &needle
);
450 if (unlikely(!map
)) {
451 error_report("Cannot locate expected map");
455 r
= vhost_vdpa_dma_unmap(v
, v
->address_space_id
, map
->iova
, map
->size
+ 1);
456 if (unlikely(r
!= 0)) {
457 error_report("Device cannot unmap: %s(%d)", g_strerror(r
), r
);
460 vhost_iova_tree_remove(tree
, *map
);
463 /** Map CVQ buffer. */
464 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa
*v
, void *buf
, size_t size
,
470 map
.translated_addr
= (hwaddr
)(uintptr_t)buf
;
472 map
.perm
= write
? IOMMU_RW
: IOMMU_RO
,
473 r
= vhost_iova_tree_map_alloc(v
->iova_tree
, &map
);
474 if (unlikely(r
!= IOVA_OK
)) {
475 error_report("Cannot map injected element");
479 r
= vhost_vdpa_dma_map(v
, v
->address_space_id
, map
.iova
,
480 vhost_vdpa_net_cvq_cmd_page_len(), buf
, !write
);
481 if (unlikely(r
< 0)) {
488 vhost_iova_tree_remove(v
->iova_tree
, map
);
492 static int vhost_vdpa_net_cvq_start(NetClientState
*nc
)
494 VhostVDPAState
*s
, *s0
;
495 struct vhost_vdpa
*v
;
500 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
502 s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
505 s0
= vhost_vdpa_net_first_nc_vdpa(s
);
506 v
->shadow_data
= s0
->vhost_vdpa
.shadow_vqs_enabled
;
507 v
->shadow_vqs_enabled
= s
->always_svq
;
508 s
->vhost_vdpa
.address_space_id
= VHOST_VDPA_GUEST_PA_ASID
;
510 if (s
->vhost_vdpa
.shadow_data
) {
511 /* SVQ is already configured for all virtqueues */
516 * If we early return in these cases SVQ will not be enabled. The migration
517 * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
519 if (!vhost_vdpa_net_valid_svq_features(v
->dev
->features
, NULL
)) {
523 if (!s
->cvq_isolated
) {
527 cvq_group
= vhost_vdpa_get_vring_group(v
->device_fd
,
528 v
->dev
->vq_index_end
- 1,
530 if (unlikely(cvq_group
< 0)) {
531 error_report_err(err
);
535 r
= vhost_vdpa_set_address_space_id(v
, cvq_group
, VHOST_VDPA_NET_CVQ_ASID
);
536 if (unlikely(r
< 0)) {
540 v
->shadow_vqs_enabled
= true;
541 s
->vhost_vdpa
.address_space_id
= VHOST_VDPA_NET_CVQ_ASID
;
544 if (!s
->vhost_vdpa
.shadow_vqs_enabled
) {
548 if (s0
->vhost_vdpa
.iova_tree
) {
550 * SVQ is already configured for all virtqueues. Reuse IOVA tree for
551 * simplicity, whether CVQ shares ASID with guest or not, because:
552 * - Memory listener need access to guest's memory addresses allocated
554 * - There should be plenty of IOVA address space for both ASID not to
555 * worry about collisions between them. Guest's translations are
556 * still validated with virtio virtqueue_pop so there is no risk for
557 * the guest to access memory that it shouldn't.
559 * To allocate a iova tree per ASID is doable but it complicates the
560 * code and it is not worth it for the moment.
562 v
->iova_tree
= s0
->vhost_vdpa
.iova_tree
;
564 v
->iova_tree
= vhost_iova_tree_new(v
->iova_range
.first
,
568 r
= vhost_vdpa_cvq_map_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
,
569 vhost_vdpa_net_cvq_cmd_page_len(), false);
570 if (unlikely(r
< 0)) {
574 r
= vhost_vdpa_cvq_map_buf(&s
->vhost_vdpa
, s
->status
,
575 vhost_vdpa_net_cvq_cmd_page_len(), true);
576 if (unlikely(r
< 0)) {
577 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
);
583 static void vhost_vdpa_net_cvq_stop(NetClientState
*nc
)
585 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
587 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
589 if (s
->vhost_vdpa
.shadow_vqs_enabled
) {
590 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
);
591 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->status
);
594 vhost_vdpa_net_client_stop(nc
);
597 static ssize_t
vhost_vdpa_net_cvq_add(VhostVDPAState
*s
, size_t out_len
,
600 /* Buffers for the device */
601 const struct iovec out
= {
602 .iov_base
= s
->cvq_cmd_out_buffer
,
605 const struct iovec in
= {
606 .iov_base
= s
->status
,
607 .iov_len
= sizeof(virtio_net_ctrl_ack
),
609 VhostShadowVirtqueue
*svq
= g_ptr_array_index(s
->vhost_vdpa
.shadow_vqs
, 0);
612 r
= vhost_svq_add(svq
, &out
, 1, &in
, 1, NULL
);
613 if (unlikely(r
!= 0)) {
614 if (unlikely(r
== -ENOSPC
)) {
615 qemu_log_mask(LOG_GUEST_ERROR
, "%s: No space on device queue\n",
622 * We can poll here since we've had BQL from the time we sent the
623 * descriptor. Also, we need to take the answer before SVQ pulls by itself,
624 * when BQL is released
626 return vhost_svq_poll(svq
);
629 static ssize_t
vhost_vdpa_net_load_cmd(VhostVDPAState
*s
, uint8_t class,
630 uint8_t cmd
, const struct iovec
*data_sg
,
633 const struct virtio_net_ctrl_hdr ctrl
= {
637 size_t data_size
= iov_size(data_sg
, data_num
);
639 assert(data_size
< vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl
));
641 /* pack the CVQ command header */
642 memcpy(s
->cvq_cmd_out_buffer
, &ctrl
, sizeof(ctrl
));
644 /* pack the CVQ command command-specific-data */
645 iov_to_buf(data_sg
, data_num
, 0,
646 s
->cvq_cmd_out_buffer
+ sizeof(ctrl
), data_size
);
648 return vhost_vdpa_net_cvq_add(s
, data_size
+ sizeof(ctrl
),
649 sizeof(virtio_net_ctrl_ack
));
652 static int vhost_vdpa_net_load_mac(VhostVDPAState
*s
, const VirtIONet
*n
)
654 if (virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_MAC_ADDR
)) {
655 const struct iovec data
= {
656 .iov_base
= (void *)n
->mac
,
657 .iov_len
= sizeof(n
->mac
),
659 ssize_t dev_written
= vhost_vdpa_net_load_cmd(s
, VIRTIO_NET_CTRL_MAC
,
660 VIRTIO_NET_CTRL_MAC_ADDR_SET
,
662 if (unlikely(dev_written
< 0)) {
665 if (*s
->status
!= VIRTIO_NET_OK
) {
671 * According to VirtIO standard, "The device MUST have an
672 * empty MAC filtering table on reset.".
674 * Therefore, there is no need to send this CVQ command if the
675 * driver also sets an empty MAC filter table, which aligns with
676 * the device's defaults.
678 * Note that the device's defaults can mismatch the driver's
679 * configuration only at live migration.
681 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_RX
) ||
682 n
->mac_table
.in_use
== 0) {
686 uint32_t uni_entries
= n
->mac_table
.first_multi
,
687 uni_macs_size
= uni_entries
* ETH_ALEN
,
688 mul_entries
= n
->mac_table
.in_use
- uni_entries
,
689 mul_macs_size
= mul_entries
* ETH_ALEN
;
690 struct virtio_net_ctrl_mac uni
= {
691 .entries
= cpu_to_le32(uni_entries
),
693 struct virtio_net_ctrl_mac mul
= {
694 .entries
= cpu_to_le32(mul_entries
),
696 const struct iovec data
[] = {
699 .iov_len
= sizeof(uni
),
701 .iov_base
= n
->mac_table
.macs
,
702 .iov_len
= uni_macs_size
,
705 .iov_len
= sizeof(mul
),
707 .iov_base
= &n
->mac_table
.macs
[uni_macs_size
],
708 .iov_len
= mul_macs_size
,
711 ssize_t dev_written
= vhost_vdpa_net_load_cmd(s
,
713 VIRTIO_NET_CTRL_MAC_TABLE_SET
,
714 data
, ARRAY_SIZE(data
));
715 if (unlikely(dev_written
< 0)) {
718 if (*s
->status
!= VIRTIO_NET_OK
) {
725 static int vhost_vdpa_net_load_mq(VhostVDPAState
*s
,
728 struct virtio_net_ctrl_mq mq
;
731 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_MQ
)) {
735 mq
.virtqueue_pairs
= cpu_to_le16(n
->curr_queue_pairs
);
736 const struct iovec data
= {
738 .iov_len
= sizeof(mq
),
740 dev_written
= vhost_vdpa_net_load_cmd(s
, VIRTIO_NET_CTRL_MQ
,
741 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
,
743 if (unlikely(dev_written
< 0)) {
746 if (*s
->status
!= VIRTIO_NET_OK
) {
753 static int vhost_vdpa_net_load_offloads(VhostVDPAState
*s
,
759 if (!virtio_vdev_has_feature(&n
->parent_obj
,
760 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
)) {
764 if (n
->curr_guest_offloads
== virtio_net_supported_guest_offloads(n
)) {
766 * According to VirtIO standard, "Upon feature negotiation
767 * corresponding offload gets enabled to preserve
768 * backward compatibility.".
770 * Therefore, there is no need to send this CVQ command if the
771 * driver also enables all supported offloads, which aligns with
772 * the device's defaults.
774 * Note that the device's defaults can mismatch the driver's
775 * configuration only at live migration.
780 offloads
= cpu_to_le64(n
->curr_guest_offloads
);
781 const struct iovec data
= {
782 .iov_base
= &offloads
,
783 .iov_len
= sizeof(offloads
),
785 dev_written
= vhost_vdpa_net_load_cmd(s
, VIRTIO_NET_CTRL_GUEST_OFFLOADS
,
786 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
,
788 if (unlikely(dev_written
< 0)) {
791 if (*s
->status
!= VIRTIO_NET_OK
) {
798 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState
*s
,
802 const struct iovec data
= {
804 .iov_len
= sizeof(on
),
806 return vhost_vdpa_net_load_cmd(s
, VIRTIO_NET_CTRL_RX
,
810 static int vhost_vdpa_net_load_rx(VhostVDPAState
*s
,
815 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_RX
)) {
820 * According to virtio_net_reset(), device turns promiscuous mode
823 * Addtionally, according to VirtIO standard, "Since there are
824 * no guarantees, it can use a hash filter or silently switch to
825 * allmulti or promiscuous mode if it is given too many addresses.".
826 * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
827 * non-multicast MAC addresses, indicating that promiscuous mode
830 * Therefore, QEMU should only send this CVQ command if the
831 * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
832 * which sets promiscuous mode on, different from the device's defaults.
834 * Note that the device's defaults can mismatch the driver's
835 * configuration only at live migration.
837 if (!n
->mac_table
.uni_overflow
&& !n
->promisc
) {
838 dev_written
= vhost_vdpa_net_load_rx_mode(s
,
839 VIRTIO_NET_CTRL_RX_PROMISC
, 0);
840 if (unlikely(dev_written
< 0)) {
843 if (*s
->status
!= VIRTIO_NET_OK
) {
849 * According to virtio_net_reset(), device turns all-multicast mode
852 * According to VirtIO standard, "Since there are no guarantees,
853 * it can use a hash filter or silently switch to allmulti or
854 * promiscuous mode if it is given too many addresses.". QEMU marks
855 * `n->mac_table.multi_overflow` if guest sets too many
856 * non-multicast MAC addresses.
858 * Therefore, QEMU should only send this CVQ command if the
859 * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
860 * which sets all-multicast mode on, different from the device's defaults.
862 * Note that the device's defaults can mismatch the driver's
863 * configuration only at live migration.
865 if (n
->mac_table
.multi_overflow
|| n
->allmulti
) {
866 dev_written
= vhost_vdpa_net_load_rx_mode(s
,
867 VIRTIO_NET_CTRL_RX_ALLMULTI
, 1);
868 if (unlikely(dev_written
< 0)) {
871 if (*s
->status
!= VIRTIO_NET_OK
) {
879 static int vhost_vdpa_net_load(NetClientState
*nc
)
881 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
882 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
886 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
888 if (!v
->shadow_vqs_enabled
) {
892 n
= VIRTIO_NET(v
->dev
->vdev
);
893 r
= vhost_vdpa_net_load_mac(s
, n
);
894 if (unlikely(r
< 0)) {
897 r
= vhost_vdpa_net_load_mq(s
, n
);
901 r
= vhost_vdpa_net_load_offloads(s
, n
);
905 r
= vhost_vdpa_net_load_rx(s
, n
);
913 static NetClientInfo net_vhost_vdpa_cvq_info
= {
914 .type
= NET_CLIENT_DRIVER_VHOST_VDPA
,
915 .size
= sizeof(VhostVDPAState
),
916 .receive
= vhost_vdpa_receive
,
917 .start
= vhost_vdpa_net_cvq_start
,
918 .load
= vhost_vdpa_net_load
,
919 .stop
= vhost_vdpa_net_cvq_stop
,
920 .cleanup
= vhost_vdpa_cleanup
,
921 .has_vnet_hdr
= vhost_vdpa_has_vnet_hdr
,
922 .has_ufo
= vhost_vdpa_has_ufo
,
923 .check_peer_type
= vhost_vdpa_check_peer_type
,
927 * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
930 * Considering that QEMU cannot send the entire filter table to the
931 * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
932 * command to enable promiscuous mode to receive all packets,
933 * according to VirtIO standard, "Since there are no guarantees,
934 * it can use a hash filter or silently switch to allmulti or
935 * promiscuous mode if it is given too many addresses.".
937 * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
938 * marks `n->mac_table.x_overflow` accordingly, it should have
939 * the same effect on the device model to receive
940 * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
941 * The same applies to multicast MAC addresses.
943 * Therefore, QEMU can provide the device model with a fake
944 * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
945 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
946 * MAC addresses. This ensures that the device model marks
947 * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
948 * allowing all packets to be received, which aligns with the
949 * state of the vdpa device.
951 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState
*s
,
952 VirtQueueElement
*elem
,
955 struct virtio_net_ctrl_mac mac_data
, *mac_ptr
;
956 struct virtio_net_ctrl_hdr
*hdr_ptr
;
960 /* parse the non-multicast MAC address entries from CVQ command */
961 cursor
= sizeof(*hdr_ptr
);
962 r
= iov_to_buf(elem
->out_sg
, elem
->out_num
, cursor
,
963 &mac_data
, sizeof(mac_data
));
964 if (unlikely(r
!= sizeof(mac_data
))) {
966 * If the CVQ command is invalid, we should simulate the vdpa device
967 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
969 *s
->status
= VIRTIO_NET_ERR
;
970 return sizeof(*s
->status
);
972 cursor
+= sizeof(mac_data
) + le32_to_cpu(mac_data
.entries
) * ETH_ALEN
;
974 /* parse the multicast MAC address entries from CVQ command */
975 r
= iov_to_buf(elem
->out_sg
, elem
->out_num
, cursor
,
976 &mac_data
, sizeof(mac_data
));
977 if (r
!= sizeof(mac_data
)) {
979 * If the CVQ command is invalid, we should simulate the vdpa device
980 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
982 *s
->status
= VIRTIO_NET_ERR
;
983 return sizeof(*s
->status
);
985 cursor
+= sizeof(mac_data
) + le32_to_cpu(mac_data
.entries
) * ETH_ALEN
;
987 /* validate the CVQ command */
988 if (iov_size(elem
->out_sg
, elem
->out_num
) != cursor
) {
990 * If the CVQ command is invalid, we should simulate the vdpa device
991 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
993 *s
->status
= VIRTIO_NET_ERR
;
994 return sizeof(*s
->status
);
998 * According to VirtIO standard, "Since there are no guarantees,
999 * it can use a hash filter or silently switch to allmulti or
1000 * promiscuous mode if it is given too many addresses.".
1002 * Therefore, considering that QEMU is unable to send the entire
1003 * filter table to the vdpa device, it should send the
1004 * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1006 r
= vhost_vdpa_net_load_rx_mode(s
, VIRTIO_NET_CTRL_RX_PROMISC
, 1);
1007 if (unlikely(r
< 0)) {
1010 if (*s
->status
!= VIRTIO_NET_OK
) {
1011 return sizeof(*s
->status
);
1015 * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1016 * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1017 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1018 * multicast MAC addresses.
1020 * By doing so, the device model can mark `n->mac_table.uni_overflow`
1021 * and `n->mac_table.multi_overflow`, enabling all packets to be
1022 * received, which aligns with the state of the vdpa device.
1025 uint32_t fake_uni_entries
= MAC_TABLE_ENTRIES
+ 1,
1026 fake_mul_entries
= MAC_TABLE_ENTRIES
+ 1,
1027 fake_cvq_size
= sizeof(struct virtio_net_ctrl_hdr
) +
1028 sizeof(mac_data
) + fake_uni_entries
* ETH_ALEN
+
1029 sizeof(mac_data
) + fake_mul_entries
* ETH_ALEN
;
1031 assert(fake_cvq_size
< vhost_vdpa_net_cvq_cmd_page_len());
1032 out
->iov_len
= fake_cvq_size
;
1034 /* pack the header for fake CVQ command */
1035 hdr_ptr
= out
->iov_base
+ cursor
;
1036 hdr_ptr
->class = VIRTIO_NET_CTRL_MAC
;
1037 hdr_ptr
->cmd
= VIRTIO_NET_CTRL_MAC_TABLE_SET
;
1038 cursor
+= sizeof(*hdr_ptr
);
1041 * Pack the non-multicast MAC addresses part for fake CVQ command.
1043 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1044 * addresses provieded in CVQ command. Therefore, only the entries
1045 * field need to be prepared in the CVQ command.
1047 mac_ptr
= out
->iov_base
+ cursor
;
1048 mac_ptr
->entries
= cpu_to_le32(fake_uni_entries
);
1049 cursor
+= sizeof(*mac_ptr
) + fake_uni_entries
* ETH_ALEN
;
1052 * Pack the multicast MAC addresses part for fake CVQ command.
1054 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1055 * addresses provieded in CVQ command. Therefore, only the entries
1056 * field need to be prepared in the CVQ command.
1058 mac_ptr
= out
->iov_base
+ cursor
;
1059 mac_ptr
->entries
= cpu_to_le32(fake_mul_entries
);
1062 * Simulating QEMU poll a vdpa device used buffer
1063 * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1065 return sizeof(*s
->status
);
1069 * Validate and copy control virtqueue commands.
1071 * Following QEMU guidelines, we offer a copy of the buffers to the device to
1072 * prevent TOCTOU bugs.
1074 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue
*svq
,
1075 VirtQueueElement
*elem
,
1078 VhostVDPAState
*s
= opaque
;
1080 const struct virtio_net_ctrl_hdr
*ctrl
;
1081 virtio_net_ctrl_ack status
= VIRTIO_NET_ERR
;
1082 /* Out buffer sent to both the vdpa device and the device model */
1083 struct iovec out
= {
1084 .iov_base
= s
->cvq_cmd_out_buffer
,
1086 /* in buffer used for device model */
1087 const struct iovec in
= {
1088 .iov_base
= &status
,
1089 .iov_len
= sizeof(status
),
1091 ssize_t dev_written
= -EINVAL
;
1093 out
.iov_len
= iov_to_buf(elem
->out_sg
, elem
->out_num
, 0,
1094 s
->cvq_cmd_out_buffer
,
1095 vhost_vdpa_net_cvq_cmd_page_len());
1097 ctrl
= s
->cvq_cmd_out_buffer
;
1098 if (ctrl
->class == VIRTIO_NET_CTRL_ANNOUNCE
) {
1100 * Guest announce capability is emulated by qemu, so don't forward to
1103 dev_written
= sizeof(status
);
1104 *s
->status
= VIRTIO_NET_OK
;
1105 } else if (unlikely(ctrl
->class == VIRTIO_NET_CTRL_MAC
&&
1106 ctrl
->cmd
== VIRTIO_NET_CTRL_MAC_TABLE_SET
&&
1107 iov_size(elem
->out_sg
, elem
->out_num
) > out
.iov_len
)) {
1109 * Due to the size limitation of the out buffer sent to the vdpa device,
1110 * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1111 * MAC addresses set by the driver for the filter table can cause
1112 * truncation of the CVQ command in QEMU. As a result, the vdpa device
1113 * rejects the flawed CVQ command.
1115 * Therefore, QEMU must handle this situation instead of sending
1116 * the CVQ command direclty.
1118 dev_written
= vhost_vdpa_net_excessive_mac_filter_cvq_add(s
, elem
,
1120 if (unlikely(dev_written
< 0)) {
1124 dev_written
= vhost_vdpa_net_cvq_add(s
, out
.iov_len
, sizeof(status
));
1125 if (unlikely(dev_written
< 0)) {
1130 if (unlikely(dev_written
< sizeof(status
))) {
1131 error_report("Insufficient written data (%zu)", dev_written
);
1135 if (*s
->status
!= VIRTIO_NET_OK
) {
1139 status
= VIRTIO_NET_ERR
;
1140 virtio_net_handle_ctrl_iov(svq
->vdev
, &in
, 1, &out
, 1);
1141 if (status
!= VIRTIO_NET_OK
) {
1142 error_report("Bad CVQ processing in model");
1146 in_len
= iov_from_buf(elem
->in_sg
, elem
->in_num
, 0, &status
,
1148 if (unlikely(in_len
< sizeof(status
))) {
1149 error_report("Bad device CVQ written length");
1151 vhost_svq_push_elem(svq
, elem
, MIN(in_len
, sizeof(status
)));
1153 * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1154 * the function successfully forwards the CVQ command, indicated
1155 * by a non-negative value of `dev_written`. Otherwise, it still
1157 * This function should only free the `elem` when it owns.
1159 if (dev_written
>= 0) {
1162 return dev_written
< 0 ? dev_written
: 0;
1165 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops
= {
1166 .avail_handler
= vhost_vdpa_net_handle_ctrl_avail
,
1170 * Probe if CVQ is isolated
1172 * @device_fd The vdpa device fd
1173 * @features Features offered by the device.
1174 * @cvq_index The control vq pair index
1176 * Returns <0 in case of failure, 0 if false and 1 if true.
1178 static int vhost_vdpa_probe_cvq_isolation(int device_fd
, uint64_t features
,
1179 int cvq_index
, Error
**errp
)
1181 uint64_t backend_features
;
1183 uint8_t status
= VIRTIO_CONFIG_S_ACKNOWLEDGE
|
1184 VIRTIO_CONFIG_S_DRIVER
|
1185 VIRTIO_CONFIG_S_FEATURES_OK
;
1190 r
= ioctl(device_fd
, VHOST_GET_BACKEND_FEATURES
, &backend_features
);
1191 if (unlikely(r
< 0)) {
1192 error_setg_errno(errp
, errno
, "Cannot get vdpa backend_features");
1196 if (!(backend_features
& BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID
))) {
1200 r
= ioctl(device_fd
, VHOST_SET_FEATURES
, &features
);
1202 error_setg_errno(errp
, errno
, "Cannot set features");
1205 r
= ioctl(device_fd
, VHOST_VDPA_SET_STATUS
, &status
);
1207 error_setg_errno(errp
, -r
, "Cannot set device features");
1211 cvq_group
= vhost_vdpa_get_vring_group(device_fd
, cvq_index
, errp
);
1212 if (unlikely(cvq_group
< 0)) {
1213 if (cvq_group
!= -ENOTSUP
) {
1219 * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1220 * support ASID even if the parent driver does not. The CVQ cannot be
1221 * isolated in this case.
1229 for (int i
= 0; i
< cvq_index
; ++i
) {
1230 int64_t group
= vhost_vdpa_get_vring_group(device_fd
, i
, errp
);
1231 if (unlikely(group
< 0)) {
1236 if (group
== (int64_t)cvq_group
) {
1246 ioctl(device_fd
, VHOST_VDPA_SET_STATUS
, &status
);
1250 static NetClientState
*net_vhost_vdpa_init(NetClientState
*peer
,
1254 int queue_pair_index
,
1258 struct vhost_vdpa_iova_range iova_range
,
1262 NetClientState
*nc
= NULL
;
1269 nc
= qemu_new_net_client(&net_vhost_vdpa_info
, peer
, device
,
1272 cvq_isolated
= vhost_vdpa_probe_cvq_isolation(vdpa_device_fd
, features
,
1273 queue_pair_index
* 2,
1275 if (unlikely(cvq_isolated
< 0)) {
1279 nc
= qemu_new_net_control_client(&net_vhost_vdpa_cvq_info
, peer
,
1282 qemu_set_info_str(nc
, TYPE_VHOST_VDPA
);
1283 s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
1285 s
->vhost_vdpa
.device_fd
= vdpa_device_fd
;
1286 s
->vhost_vdpa
.index
= queue_pair_index
;
1287 s
->always_svq
= svq
;
1288 s
->migration_state
.notify
= vdpa_net_migration_state_notifier
;
1289 s
->vhost_vdpa
.shadow_vqs_enabled
= svq
;
1290 s
->vhost_vdpa
.iova_range
= iova_range
;
1291 s
->vhost_vdpa
.shadow_data
= svq
;
1292 if (queue_pair_index
== 0) {
1293 vhost_vdpa_net_valid_svq_features(features
,
1294 &s
->vhost_vdpa
.migration_blocker
);
1295 } else if (!is_datapath
) {
1296 s
->cvq_cmd_out_buffer
= mmap(NULL
, vhost_vdpa_net_cvq_cmd_page_len(),
1297 PROT_READ
| PROT_WRITE
,
1298 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
1299 s
->status
= mmap(NULL
, vhost_vdpa_net_cvq_cmd_page_len(),
1300 PROT_READ
| PROT_WRITE
, MAP_SHARED
| MAP_ANONYMOUS
,
1303 s
->vhost_vdpa
.shadow_vq_ops
= &vhost_vdpa_net_svq_ops
;
1304 s
->vhost_vdpa
.shadow_vq_ops_opaque
= s
;
1305 s
->cvq_isolated
= cvq_isolated
;
1308 * TODO: We cannot migrate devices with CVQ and no x-svq enabled as
1309 * there is no way to set the device state (MAC, MQ, etc) before
1310 * starting the datapath.
1312 * Migration blocker ownership now belongs to s->vhost_vdpa.
1315 error_setg(&s
->vhost_vdpa
.migration_blocker
,
1316 "net vdpa cannot migrate with CVQ feature");
1319 ret
= vhost_vdpa_add(nc
, (void *)&s
->vhost_vdpa
, queue_pair_index
, nvqs
);
1321 qemu_del_net_client(nc
);
1327 static int vhost_vdpa_get_features(int fd
, uint64_t *features
, Error
**errp
)
1329 int ret
= ioctl(fd
, VHOST_GET_FEATURES
, features
);
1330 if (unlikely(ret
< 0)) {
1331 error_setg_errno(errp
, errno
,
1332 "Fail to query features from vhost-vDPA device");
1337 static int vhost_vdpa_get_max_queue_pairs(int fd
, uint64_t features
,
1338 int *has_cvq
, Error
**errp
)
1340 unsigned long config_size
= offsetof(struct vhost_vdpa_config
, buf
);
1341 g_autofree
struct vhost_vdpa_config
*config
= NULL
;
1342 __virtio16
*max_queue_pairs
;
1345 if (features
& (1 << VIRTIO_NET_F_CTRL_VQ
)) {
1351 if (features
& (1 << VIRTIO_NET_F_MQ
)) {
1352 config
= g_malloc0(config_size
+ sizeof(*max_queue_pairs
));
1353 config
->off
= offsetof(struct virtio_net_config
, max_virtqueue_pairs
);
1354 config
->len
= sizeof(*max_queue_pairs
);
1356 ret
= ioctl(fd
, VHOST_VDPA_GET_CONFIG
, config
);
1358 error_setg(errp
, "Fail to get config from vhost-vDPA device");
1362 max_queue_pairs
= (__virtio16
*)&config
->buf
;
1364 return lduw_le_p(max_queue_pairs
);
1370 int net_init_vhost_vdpa(const Netdev
*netdev
, const char *name
,
1371 NetClientState
*peer
, Error
**errp
)
1373 const NetdevVhostVDPAOptions
*opts
;
1376 g_autofree NetClientState
**ncs
= NULL
;
1377 struct vhost_vdpa_iova_range iova_range
;
1379 int queue_pairs
, r
, i
= 0, has_cvq
= 0;
1381 assert(netdev
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
1382 opts
= &netdev
->u
.vhost_vdpa
;
1383 if (!opts
->vhostdev
&& !opts
->vhostfd
) {
1385 "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1389 if (opts
->vhostdev
&& opts
->vhostfd
) {
1391 "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1395 if (opts
->vhostdev
) {
1396 vdpa_device_fd
= qemu_open(opts
->vhostdev
, O_RDWR
, errp
);
1397 if (vdpa_device_fd
== -1) {
1402 vdpa_device_fd
= monitor_fd_param(monitor_cur(), opts
->vhostfd
, errp
);
1403 if (vdpa_device_fd
== -1) {
1404 error_prepend(errp
, "vhost-vdpa: unable to parse vhostfd: ");
1409 r
= vhost_vdpa_get_features(vdpa_device_fd
, &features
, errp
);
1410 if (unlikely(r
< 0)) {
1414 queue_pairs
= vhost_vdpa_get_max_queue_pairs(vdpa_device_fd
, features
,
1416 if (queue_pairs
< 0) {
1417 qemu_close(vdpa_device_fd
);
1421 r
= vhost_vdpa_get_iova_range(vdpa_device_fd
, &iova_range
);
1422 if (unlikely(r
< 0)) {
1423 error_setg(errp
, "vhost-vdpa: get iova range failed: %s",
1428 if (opts
->x_svq
&& !vhost_vdpa_net_valid_svq_features(features
, errp
)) {
1432 ncs
= g_malloc0(sizeof(*ncs
) * queue_pairs
);
1434 for (i
= 0; i
< queue_pairs
; i
++) {
1435 ncs
[i
] = net_vhost_vdpa_init(peer
, TYPE_VHOST_VDPA
, name
,
1436 vdpa_device_fd
, i
, 2, true, opts
->x_svq
,
1437 iova_range
, features
, errp
);
1443 nc
= net_vhost_vdpa_init(peer
, TYPE_VHOST_VDPA
, name
,
1444 vdpa_device_fd
, i
, 1, false,
1445 opts
->x_svq
, iova_range
, features
, errp
);
1454 for (i
--; i
>= 0; i
--) {
1455 qemu_del_net_client(ncs
[i
]);
1459 qemu_close(vdpa_device_fd
);