4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <linux/vhost.h>
35 #include <linux/virtio_net.h>
39 #ifdef RTE_LIBRTE_VHOST_NUMA
43 #include <rte_ethdev.h>
45 #include <rte_string_fns.h>
46 #include <rte_memory.h>
47 #include <rte_malloc.h>
48 #include <rte_virtio_net.h>
52 #define VHOST_USER_F_PROTOCOL_FEATURES 30
54 /* Features supported by this lib. */
55 #define VHOST_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \
56 (1ULL << VIRTIO_NET_F_CTRL_VQ) | \
57 (1ULL << VIRTIO_NET_F_CTRL_RX) | \
58 (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
59 (VHOST_SUPPORTS_MQ) | \
60 (1ULL << VIRTIO_F_VERSION_1) | \
61 (1ULL << VHOST_F_LOG_ALL) | \
62 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
63 (1ULL << VIRTIO_NET_F_HOST_TSO4) | \
64 (1ULL << VIRTIO_NET_F_HOST_TSO6) | \
65 (1ULL << VIRTIO_NET_F_CSUM) | \
66 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
67 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
68 (1ULL << VIRTIO_NET_F_GUEST_TSO6))
70 uint64_t VHOST_FEATURES
= VHOST_SUPPORTED_FEATURES
;
72 struct virtio_net
*vhost_devices
[MAX_VHOST_DEVICE
];
74 /* device ops to add/remove device to/from data core. */
75 struct virtio_net_device_ops
const *notify_ops
;
80 struct virtio_net
*dev
= vhost_devices
[vid
];
83 RTE_LOG(ERR
, VHOST_CONFIG
,
84 "(%d) device not found.\n", vid
);
91 cleanup_vq(struct vhost_virtqueue
*vq
, int destroy
)
93 if ((vq
->callfd
>= 0) && (destroy
!= 0))
100 * Unmap any memory, close any file descriptors and
101 * free any memory owned by a device.
104 cleanup_device(struct virtio_net
*dev
, int destroy
)
108 vhost_backend_cleanup(dev
);
110 for (i
= 0; i
< dev
->virt_qp_nb
; i
++) {
111 cleanup_vq(dev
->virtqueue
[i
* VIRTIO_QNUM
+ VIRTIO_RXQ
], destroy
);
112 cleanup_vq(dev
->virtqueue
[i
* VIRTIO_QNUM
+ VIRTIO_TXQ
], destroy
);
117 * Release virtqueues and device memory.
120 free_device(struct virtio_net
*dev
)
123 struct vhost_virtqueue
*rxq
, *txq
;
125 for (i
= 0; i
< dev
->virt_qp_nb
; i
++) {
126 rxq
= dev
->virtqueue
[i
* VIRTIO_QNUM
+ VIRTIO_RXQ
];
127 txq
= dev
->virtqueue
[i
* VIRTIO_QNUM
+ VIRTIO_TXQ
];
129 rte_free(rxq
->shadow_used_ring
);
130 rte_free(txq
->shadow_used_ring
);
132 /* rxq and txq are allocated together as queue-pair */
140 init_vring_queue(struct vhost_virtqueue
*vq
, int qp_idx
)
142 memset(vq
, 0, sizeof(struct vhost_virtqueue
));
144 vq
->kickfd
= VIRTIO_UNINITIALIZED_EVENTFD
;
145 vq
->callfd
= VIRTIO_UNINITIALIZED_EVENTFD
;
147 /* Backends are set to -1 indicating an inactive device. */
150 /* always set the default vq pair to enabled */
154 TAILQ_INIT(&vq
->zmbuf_list
);
158 init_vring_queue_pair(struct virtio_net
*dev
, uint32_t qp_idx
)
160 uint32_t base_idx
= qp_idx
* VIRTIO_QNUM
;
162 init_vring_queue(dev
->virtqueue
[base_idx
+ VIRTIO_RXQ
], qp_idx
);
163 init_vring_queue(dev
->virtqueue
[base_idx
+ VIRTIO_TXQ
], qp_idx
);
167 reset_vring_queue(struct vhost_virtqueue
*vq
, int qp_idx
)
172 init_vring_queue(vq
, qp_idx
);
177 reset_vring_queue_pair(struct virtio_net
*dev
, uint32_t qp_idx
)
179 uint32_t base_idx
= qp_idx
* VIRTIO_QNUM
;
181 reset_vring_queue(dev
->virtqueue
[base_idx
+ VIRTIO_RXQ
], qp_idx
);
182 reset_vring_queue(dev
->virtqueue
[base_idx
+ VIRTIO_TXQ
], qp_idx
);
186 alloc_vring_queue_pair(struct virtio_net
*dev
, uint32_t qp_idx
)
188 struct vhost_virtqueue
*virtqueue
= NULL
;
189 uint32_t virt_rx_q_idx
= qp_idx
* VIRTIO_QNUM
+ VIRTIO_RXQ
;
190 uint32_t virt_tx_q_idx
= qp_idx
* VIRTIO_QNUM
+ VIRTIO_TXQ
;
192 virtqueue
= rte_malloc(NULL
,
193 sizeof(struct vhost_virtqueue
) * VIRTIO_QNUM
, 0);
194 if (virtqueue
== NULL
) {
195 RTE_LOG(ERR
, VHOST_CONFIG
,
196 "Failed to allocate memory for virt qp:%d.\n", qp_idx
);
200 dev
->virtqueue
[virt_rx_q_idx
] = virtqueue
;
201 dev
->virtqueue
[virt_tx_q_idx
] = virtqueue
+ VIRTIO_TXQ
;
203 init_vring_queue_pair(dev
, qp_idx
);
205 dev
->virt_qp_nb
+= 1;
211 * Reset some variables in device structure, while keeping few
212 * others untouched, such as vid, ifname, virt_qp_nb: they
213 * should be same unless the device is removed.
216 reset_device(struct virtio_net
*dev
)
221 dev
->protocol_features
= 0;
224 for (i
= 0; i
< dev
->virt_qp_nb
; i
++)
225 reset_vring_queue_pair(dev
, i
);
229 * Invoked when there is a new vhost-user connection established (when
230 * there is a new virtio device being attached).
233 vhost_new_device(void)
235 struct virtio_net
*dev
;
238 dev
= rte_zmalloc(NULL
, sizeof(struct virtio_net
), 0);
240 RTE_LOG(ERR
, VHOST_CONFIG
,
241 "Failed to allocate memory for new dev.\n");
245 for (i
= 0; i
< MAX_VHOST_DEVICE
; i
++) {
246 if (vhost_devices
[i
] == NULL
)
249 if (i
== MAX_VHOST_DEVICE
) {
250 RTE_LOG(ERR
, VHOST_CONFIG
,
251 "Failed to find a free slot for new device.\n");
255 vhost_devices
[i
] = dev
;
262 * Invoked when there is the vhost-user connection is broken (when
263 * the virtio device is being detached).
266 vhost_destroy_device(int vid
)
268 struct virtio_net
*dev
= get_device(vid
);
273 if (dev
->flags
& VIRTIO_DEV_RUNNING
) {
274 dev
->flags
&= ~VIRTIO_DEV_RUNNING
;
275 notify_ops
->destroy_device(vid
);
278 cleanup_device(dev
, 1);
281 vhost_devices
[vid
] = NULL
;
285 vhost_set_ifname(int vid
, const char *if_name
, unsigned int if_len
)
287 struct virtio_net
*dev
;
290 dev
= get_device(vid
);
294 len
= if_len
> sizeof(dev
->ifname
) ?
295 sizeof(dev
->ifname
) : if_len
;
297 strncpy(dev
->ifname
, if_name
, len
);
298 dev
->ifname
[sizeof(dev
->ifname
) - 1] = '\0';
302 vhost_enable_dequeue_zero_copy(int vid
)
304 struct virtio_net
*dev
= get_device(vid
);
309 dev
->dequeue_zero_copy
= 1;
313 rte_vhost_get_numa_node(int vid
)
315 #ifdef RTE_LIBRTE_VHOST_NUMA
316 struct virtio_net
*dev
= get_device(vid
);
323 ret
= get_mempolicy(&numa_node
, NULL
, 0, dev
,
324 MPOL_F_NODE
| MPOL_F_ADDR
);
326 RTE_LOG(ERR
, VHOST_CONFIG
,
327 "(%d) failed to query numa node: %d\n", vid
, ret
);
339 rte_vhost_get_queue_num(int vid
)
341 struct virtio_net
*dev
= get_device(vid
);
346 return dev
->virt_qp_nb
;
350 rte_vhost_get_ifname(int vid
, char *buf
, size_t len
)
352 struct virtio_net
*dev
= get_device(vid
);
357 len
= RTE_MIN(len
, sizeof(dev
->ifname
));
359 strncpy(buf
, dev
->ifname
, len
);
366 rte_vhost_avail_entries(int vid
, uint16_t queue_id
)
368 struct virtio_net
*dev
;
369 struct vhost_virtqueue
*vq
;
371 dev
= get_device(vid
);
375 vq
= dev
->virtqueue
[queue_id
];
379 return *(volatile uint16_t *)&vq
->avail
->idx
- vq
->last_used_idx
;
383 rte_vhost_enable_guest_notification(int vid
, uint16_t queue_id
, int enable
)
385 struct virtio_net
*dev
= get_device(vid
);
391 RTE_LOG(ERR
, VHOST_CONFIG
,
392 "guest notification isn't supported.\n");
396 dev
->virtqueue
[queue_id
]->used
->flags
= VRING_USED_F_NO_NOTIFY
;
400 uint64_t rte_vhost_feature_get(void)
402 return VHOST_FEATURES
;
405 int rte_vhost_feature_disable(uint64_t feature_mask
)
407 VHOST_FEATURES
= VHOST_FEATURES
& ~feature_mask
;
411 int rte_vhost_feature_enable(uint64_t feature_mask
)
413 if ((feature_mask
& VHOST_SUPPORTED_FEATURES
) == feature_mask
) {
414 VHOST_FEATURES
= VHOST_FEATURES
| feature_mask
;
421 * Register ops so that we can add/remove device to data core.
424 rte_vhost_driver_callback_register(struct virtio_net_device_ops
const * const ops
)