]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2010-2017 Intel Corporation | |
7c673cae FG |
3 | */ |
4 | ||
5 | #include <linux/vhost.h> | |
6 | #include <linux/virtio_net.h> | |
7 | #include <stddef.h> | |
8 | #include <stdint.h> | |
9 | #include <stdlib.h> | |
10 | #ifdef RTE_LIBRTE_VHOST_NUMA | |
9f95a23c | 11 | #include <numa.h> |
7c673cae FG |
12 | #include <numaif.h> |
13 | #endif | |
14 | ||
9f95a23c | 15 | #include <rte_errno.h> |
7c673cae FG |
16 | #include <rte_ethdev.h> |
17 | #include <rte_log.h> | |
18 | #include <rte_string_fns.h> | |
19 | #include <rte_memory.h> | |
20 | #include <rte_malloc.h> | |
11fdf7f2 | 21 | #include <rte_vhost.h> |
9f95a23c | 22 | #include <rte_rwlock.h> |
7c673cae | 23 | |
9f95a23c | 24 | #include "iotlb.h" |
7c673cae | 25 | #include "vhost.h" |
9f95a23c | 26 | #include "vhost_user.h" |
7c673cae | 27 | |
7c673cae FG |
28 | struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; |
29 | ||
9f95a23c TL |
30 | /* Called with iotlb_lock read-locked */ |
31 | uint64_t | |
32 | __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, | |
33 | uint64_t iova, uint64_t *size, uint8_t perm) | |
7c673cae | 34 | { |
9f95a23c | 35 | uint64_t vva, tmp_size; |
7c673cae | 36 | |
9f95a23c TL |
37 | if (unlikely(!*size)) |
38 | return 0; | |
39 | ||
40 | tmp_size = *size; | |
41 | ||
42 | vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm); | |
43 | if (tmp_size == *size) | |
44 | return vva; | |
45 | ||
46 | iova += tmp_size; | |
47 | ||
48 | if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) { | |
49 | /* | |
50 | * iotlb_lock is read-locked for a full burst, | |
51 | * but it only protects the iotlb cache. | |
52 | * In case of IOTLB miss, we might block on the socket, | |
53 | * which could cause a deadlock with QEMU if an IOTLB update | |
54 | * is being handled. We can safely unlock here to avoid it. | |
55 | */ | |
56 | vhost_user_iotlb_rd_unlock(vq); | |
57 | ||
58 | vhost_user_iotlb_pending_insert(vq, iova, perm); | |
59 | if (vhost_user_iotlb_miss(dev, iova, perm)) { | |
60 | RTE_LOG(ERR, VHOST_CONFIG, | |
61 | "IOTLB miss req failed for IOVA 0x%" PRIx64 "\n", | |
62 | iova); | |
63 | vhost_user_iotlb_pending_remove(vq, iova, 1, perm); | |
64 | } | |
65 | ||
66 | vhost_user_iotlb_rd_lock(vq); | |
7c673cae FG |
67 | } |
68 | ||
9f95a23c | 69 | return 0; |
7c673cae FG |
70 | } |
71 | ||
9f95a23c | 72 | void |
7c673cae FG |
73 | cleanup_vq(struct vhost_virtqueue *vq, int destroy) |
74 | { | |
75 | if ((vq->callfd >= 0) && (destroy != 0)) | |
76 | close(vq->callfd); | |
77 | if (vq->kickfd >= 0) | |
78 | close(vq->kickfd); | |
79 | } | |
80 | ||
81 | /* | |
82 | * Unmap any memory, close any file descriptors and | |
83 | * free any memory owned by a device. | |
84 | */ | |
85 | void | |
86 | cleanup_device(struct virtio_net *dev, int destroy) | |
87 | { | |
88 | uint32_t i; | |
89 | ||
90 | vhost_backend_cleanup(dev); | |
91 | ||
11fdf7f2 TL |
92 | for (i = 0; i < dev->nr_vring; i++) |
93 | cleanup_vq(dev->virtqueue[i], destroy); | |
7c673cae FG |
94 | } |
95 | ||
9f95a23c TL |
96 | void |
97 | free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) | |
98 | { | |
99 | if (vq_is_packed(dev)) | |
100 | rte_free(vq->shadow_used_packed); | |
101 | else | |
102 | rte_free(vq->shadow_used_split); | |
103 | rte_free(vq->batch_copy_elems); | |
104 | rte_mempool_free(vq->iotlb_pool); | |
105 | rte_free(vq); | |
106 | } | |
107 | ||
7c673cae FG |
108 | /* |
109 | * Release virtqueues and device memory. | |
110 | */ | |
111 | static void | |
112 | free_device(struct virtio_net *dev) | |
113 | { | |
114 | uint32_t i; | |
7c673cae | 115 | |
9f95a23c TL |
116 | for (i = 0; i < dev->nr_vring; i++) |
117 | free_vq(dev, dev->virtqueue[i]); | |
118 | ||
119 | rte_free(dev); | |
120 | } | |
121 | ||
122 | static int | |
123 | vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq) | |
124 | { | |
125 | uint64_t req_size, size; | |
126 | ||
127 | req_size = sizeof(struct vring_desc) * vq->size; | |
128 | size = req_size; | |
129 | vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, | |
130 | vq->ring_addrs.desc_user_addr, | |
131 | &size, VHOST_ACCESS_RW); | |
132 | if (!vq->desc || size != req_size) | |
133 | return -1; | |
134 | ||
135 | req_size = sizeof(struct vring_avail); | |
136 | req_size += sizeof(uint16_t) * vq->size; | |
137 | if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) | |
138 | req_size += sizeof(uint16_t); | |
139 | size = req_size; | |
140 | vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, | |
141 | vq->ring_addrs.avail_user_addr, | |
142 | &size, VHOST_ACCESS_RW); | |
143 | if (!vq->avail || size != req_size) | |
144 | return -1; | |
145 | ||
146 | req_size = sizeof(struct vring_used); | |
147 | req_size += sizeof(struct vring_used_elem) * vq->size; | |
148 | if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) | |
149 | req_size += sizeof(uint16_t); | |
150 | size = req_size; | |
151 | vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, | |
152 | vq->ring_addrs.used_user_addr, | |
153 | &size, VHOST_ACCESS_RW); | |
154 | if (!vq->used || size != req_size) | |
155 | return -1; | |
7c673cae | 156 | |
9f95a23c TL |
157 | return 0; |
158 | } | |
7c673cae | 159 | |
9f95a23c TL |
160 | static int |
161 | vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) | |
162 | { | |
163 | uint64_t req_size, size; | |
164 | ||
165 | req_size = sizeof(struct vring_packed_desc) * vq->size; | |
166 | size = req_size; | |
167 | vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) | |
168 | vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, | |
169 | &size, VHOST_ACCESS_RW); | |
170 | if (!vq->desc_packed || size != req_size) | |
171 | return -1; | |
172 | ||
173 | req_size = sizeof(struct vring_packed_desc_event); | |
174 | size = req_size; | |
175 | vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) | |
176 | vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, | |
177 | &size, VHOST_ACCESS_RW); | |
178 | if (!vq->driver_event || size != req_size) | |
179 | return -1; | |
180 | ||
181 | req_size = sizeof(struct vring_packed_desc_event); | |
182 | size = req_size; | |
183 | vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) | |
184 | vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, | |
185 | &size, VHOST_ACCESS_RW); | |
186 | if (!vq->device_event || size != req_size) | |
187 | return -1; | |
188 | ||
189 | return 0; | |
190 | } | |
191 | ||
192 | int | |
193 | vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) | |
194 | { | |
195 | ||
196 | if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) | |
197 | goto out; | |
198 | ||
199 | if (vq_is_packed(dev)) { | |
200 | if (vring_translate_packed(dev, vq) < 0) | |
201 | return -1; | |
202 | } else { | |
203 | if (vring_translate_split(dev, vq) < 0) | |
204 | return -1; | |
7c673cae | 205 | } |
9f95a23c TL |
206 | out: |
207 | vq->access_ok = 1; | |
7c673cae | 208 | |
9f95a23c TL |
209 | return 0; |
210 | } | |
211 | ||
212 | void | |
213 | vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq) | |
214 | { | |
215 | if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) | |
216 | vhost_user_iotlb_wr_lock(vq); | |
217 | ||
218 | vq->access_ok = 0; | |
219 | vq->desc = NULL; | |
220 | vq->avail = NULL; | |
221 | vq->used = NULL; | |
222 | ||
223 | if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) | |
224 | vhost_user_iotlb_wr_unlock(vq); | |
7c673cae FG |
225 | } |
226 | ||
227 | static void | |
9f95a23c | 228 | init_vring_queue(struct virtio_net *dev, uint32_t vring_idx) |
7c673cae | 229 | { |
9f95a23c TL |
230 | struct vhost_virtqueue *vq; |
231 | ||
232 | if (vring_idx >= VHOST_MAX_VRING) { | |
233 | RTE_LOG(ERR, VHOST_CONFIG, | |
234 | "Failed not init vring, out of bound (%d)\n", | |
235 | vring_idx); | |
236 | return; | |
237 | } | |
238 | ||
239 | vq = dev->virtqueue[vring_idx]; | |
240 | ||
7c673cae FG |
241 | memset(vq, 0, sizeof(struct vhost_virtqueue)); |
242 | ||
243 | vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; | |
244 | vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; | |
245 | ||
9f95a23c | 246 | vhost_user_iotlb_init(dev, vring_idx); |
7c673cae FG |
247 | /* Backends are set to -1 indicating an inactive device. */ |
248 | vq->backend = -1; | |
249 | ||
7c673cae FG |
250 | TAILQ_INIT(&vq->zmbuf_list); |
251 | } | |
252 | ||
253 | static void | |
9f95a23c | 254 | reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx) |
7c673cae | 255 | { |
9f95a23c | 256 | struct vhost_virtqueue *vq; |
7c673cae FG |
257 | int callfd; |
258 | ||
9f95a23c TL |
259 | if (vring_idx >= VHOST_MAX_VRING) { |
260 | RTE_LOG(ERR, VHOST_CONFIG, | |
261 | "Failed not init vring, out of bound (%d)\n", | |
262 | vring_idx); | |
263 | return; | |
264 | } | |
265 | ||
266 | vq = dev->virtqueue[vring_idx]; | |
7c673cae | 267 | callfd = vq->callfd; |
9f95a23c | 268 | init_vring_queue(dev, vring_idx); |
7c673cae FG |
269 | vq->callfd = callfd; |
270 | } | |
271 | ||
7c673cae | 272 | int |
11fdf7f2 | 273 | alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) |
7c673cae | 274 | { |
11fdf7f2 | 275 | struct vhost_virtqueue *vq; |
7c673cae | 276 | |
11fdf7f2 TL |
277 | vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); |
278 | if (vq == NULL) { | |
7c673cae | 279 | RTE_LOG(ERR, VHOST_CONFIG, |
11fdf7f2 | 280 | "Failed to allocate memory for vring:%u.\n", vring_idx); |
7c673cae FG |
281 | return -1; |
282 | } | |
283 | ||
11fdf7f2 | 284 | dev->virtqueue[vring_idx] = vq; |
9f95a23c TL |
285 | init_vring_queue(dev, vring_idx); |
286 | rte_spinlock_init(&vq->access_lock); | |
287 | vq->avail_wrap_counter = 1; | |
288 | vq->used_wrap_counter = 1; | |
289 | vq->signalled_used_valid = false; | |
7c673cae | 290 | |
11fdf7f2 | 291 | dev->nr_vring += 1; |
7c673cae FG |
292 | |
293 | return 0; | |
294 | } | |
295 | ||
296 | /* | |
297 | * Reset some variables in device structure, while keeping few | |
11fdf7f2 | 298 | * others untouched, such as vid, ifname, nr_vring: they |
7c673cae FG |
299 | * should be same unless the device is removed. |
300 | */ | |
301 | void | |
302 | reset_device(struct virtio_net *dev) | |
303 | { | |
304 | uint32_t i; | |
305 | ||
306 | dev->features = 0; | |
307 | dev->protocol_features = 0; | |
9f95a23c | 308 | dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET; |
7c673cae | 309 | |
11fdf7f2 | 310 | for (i = 0; i < dev->nr_vring; i++) |
9f95a23c | 311 | reset_vring_queue(dev, i); |
7c673cae FG |
312 | } |
313 | ||
314 | /* | |
315 | * Invoked when there is a new vhost-user connection established (when | |
316 | * there is a new virtio device being attached). | |
317 | */ | |
318 | int | |
319 | vhost_new_device(void) | |
320 | { | |
321 | struct virtio_net *dev; | |
322 | int i; | |
323 | ||
7c673cae FG |
324 | for (i = 0; i < MAX_VHOST_DEVICE; i++) { |
325 | if (vhost_devices[i] == NULL) | |
326 | break; | |
327 | } | |
9f95a23c | 328 | |
7c673cae FG |
329 | if (i == MAX_VHOST_DEVICE) { |
330 | RTE_LOG(ERR, VHOST_CONFIG, | |
331 | "Failed to find a free slot for new device.\n"); | |
9f95a23c TL |
332 | return -1; |
333 | } | |
334 | ||
335 | dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); | |
336 | if (dev == NULL) { | |
337 | RTE_LOG(ERR, VHOST_CONFIG, | |
338 | "Failed to allocate memory for new dev.\n"); | |
7c673cae FG |
339 | return -1; |
340 | } | |
341 | ||
342 | vhost_devices[i] = dev; | |
343 | dev->vid = i; | |
9f95a23c TL |
344 | dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; |
345 | dev->slave_req_fd = -1; | |
346 | dev->vdpa_dev_id = -1; | |
347 | dev->postcopy_ufd = -1; | |
348 | rte_spinlock_init(&dev->slave_req_lock); | |
7c673cae FG |
349 | |
350 | return i; | |
351 | } | |
352 | ||
9f95a23c TL |
353 | void |
354 | vhost_destroy_device_notify(struct virtio_net *dev) | |
355 | { | |
356 | struct rte_vdpa_device *vdpa_dev; | |
357 | int did; | |
358 | ||
359 | if (dev->flags & VIRTIO_DEV_RUNNING) { | |
360 | did = dev->vdpa_dev_id; | |
361 | vdpa_dev = rte_vdpa_get_device(did); | |
362 | if (vdpa_dev && vdpa_dev->ops->dev_close) | |
363 | vdpa_dev->ops->dev_close(dev->vid); | |
364 | dev->flags &= ~VIRTIO_DEV_RUNNING; | |
365 | dev->notify_ops->destroy_device(dev->vid); | |
366 | } | |
367 | } | |
368 | ||
7c673cae FG |
369 | /* |
370 | * Invoked when there is the vhost-user connection is broken (when | |
371 | * the virtio device is being detached). | |
372 | */ | |
373 | void | |
374 | vhost_destroy_device(int vid) | |
375 | { | |
376 | struct virtio_net *dev = get_device(vid); | |
377 | ||
378 | if (dev == NULL) | |
379 | return; | |
380 | ||
9f95a23c | 381 | vhost_destroy_device_notify(dev); |
7c673cae FG |
382 | |
383 | cleanup_device(dev, 1); | |
384 | free_device(dev); | |
385 | ||
386 | vhost_devices[vid] = NULL; | |
387 | } | |
388 | ||
9f95a23c TL |
389 | void |
390 | vhost_attach_vdpa_device(int vid, int did) | |
391 | { | |
392 | struct virtio_net *dev = get_device(vid); | |
393 | ||
394 | if (dev == NULL) | |
395 | return; | |
396 | ||
397 | if (rte_vdpa_get_device(did) == NULL) | |
398 | return; | |
399 | ||
400 | dev->vdpa_dev_id = did; | |
401 | } | |
402 | ||
7c673cae FG |
403 | void |
404 | vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) | |
405 | { | |
406 | struct virtio_net *dev; | |
407 | unsigned int len; | |
408 | ||
409 | dev = get_device(vid); | |
410 | if (dev == NULL) | |
411 | return; | |
412 | ||
413 | len = if_len > sizeof(dev->ifname) ? | |
414 | sizeof(dev->ifname) : if_len; | |
415 | ||
416 | strncpy(dev->ifname, if_name, len); | |
417 | dev->ifname[sizeof(dev->ifname) - 1] = '\0'; | |
418 | } | |
419 | ||
420 | void | |
421 | vhost_enable_dequeue_zero_copy(int vid) | |
422 | { | |
423 | struct virtio_net *dev = get_device(vid); | |
424 | ||
425 | if (dev == NULL) | |
426 | return; | |
427 | ||
428 | dev->dequeue_zero_copy = 1; | |
429 | } | |
430 | ||
9f95a23c TL |
431 | void |
432 | vhost_set_builtin_virtio_net(int vid, bool enable) | |
433 | { | |
434 | struct virtio_net *dev = get_device(vid); | |
435 | ||
436 | if (dev == NULL) | |
437 | return; | |
438 | ||
439 | if (enable) | |
440 | dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET; | |
441 | else | |
442 | dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET; | |
443 | } | |
444 | ||
11fdf7f2 TL |
445 | int |
446 | rte_vhost_get_mtu(int vid, uint16_t *mtu) | |
447 | { | |
448 | struct virtio_net *dev = get_device(vid); | |
449 | ||
9f95a23c | 450 | if (dev == NULL || mtu == NULL) |
11fdf7f2 TL |
451 | return -ENODEV; |
452 | ||
453 | if (!(dev->flags & VIRTIO_DEV_READY)) | |
454 | return -EAGAIN; | |
455 | ||
9f95a23c | 456 | if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU))) |
11fdf7f2 TL |
457 | return -ENOTSUP; |
458 | ||
459 | *mtu = dev->mtu; | |
460 | ||
461 | return 0; | |
462 | } | |
463 | ||
7c673cae FG |
464 | int |
465 | rte_vhost_get_numa_node(int vid) | |
466 | { | |
467 | #ifdef RTE_LIBRTE_VHOST_NUMA | |
468 | struct virtio_net *dev = get_device(vid); | |
469 | int numa_node; | |
470 | int ret; | |
471 | ||
9f95a23c | 472 | if (dev == NULL || numa_available() != 0) |
7c673cae FG |
473 | return -1; |
474 | ||
475 | ret = get_mempolicy(&numa_node, NULL, 0, dev, | |
476 | MPOL_F_NODE | MPOL_F_ADDR); | |
477 | if (ret < 0) { | |
478 | RTE_LOG(ERR, VHOST_CONFIG, | |
9f95a23c TL |
479 | "(%d) failed to query numa node: %s\n", |
480 | vid, rte_strerror(errno)); | |
7c673cae FG |
481 | return -1; |
482 | } | |
483 | ||
484 | return numa_node; | |
485 | #else | |
486 | RTE_SET_USED(vid); | |
487 | return -1; | |
488 | #endif | |
489 | } | |
490 | ||
491 | uint32_t | |
492 | rte_vhost_get_queue_num(int vid) | |
493 | { | |
494 | struct virtio_net *dev = get_device(vid); | |
495 | ||
496 | if (dev == NULL) | |
497 | return 0; | |
498 | ||
11fdf7f2 TL |
499 | return dev->nr_vring / 2; |
500 | } | |
501 | ||
502 | uint16_t | |
503 | rte_vhost_get_vring_num(int vid) | |
504 | { | |
505 | struct virtio_net *dev = get_device(vid); | |
506 | ||
507 | if (dev == NULL) | |
508 | return 0; | |
509 | ||
510 | return dev->nr_vring; | |
7c673cae FG |
511 | } |
512 | ||
513 | int | |
514 | rte_vhost_get_ifname(int vid, char *buf, size_t len) | |
515 | { | |
516 | struct virtio_net *dev = get_device(vid); | |
517 | ||
9f95a23c | 518 | if (dev == NULL || buf == NULL) |
7c673cae FG |
519 | return -1; |
520 | ||
521 | len = RTE_MIN(len, sizeof(dev->ifname)); | |
522 | ||
523 | strncpy(buf, dev->ifname, len); | |
524 | buf[len - 1] = '\0'; | |
525 | ||
526 | return 0; | |
527 | } | |
528 | ||
11fdf7f2 TL |
529 | int |
530 | rte_vhost_get_negotiated_features(int vid, uint64_t *features) | |
531 | { | |
532 | struct virtio_net *dev; | |
533 | ||
534 | dev = get_device(vid); | |
9f95a23c | 535 | if (dev == NULL || features == NULL) |
11fdf7f2 TL |
536 | return -1; |
537 | ||
538 | *features = dev->features; | |
539 | return 0; | |
540 | } | |
541 | ||
542 | int | |
543 | rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) | |
544 | { | |
545 | struct virtio_net *dev; | |
546 | struct rte_vhost_memory *m; | |
547 | size_t size; | |
548 | ||
549 | dev = get_device(vid); | |
9f95a23c | 550 | if (dev == NULL || mem == NULL) |
11fdf7f2 TL |
551 | return -1; |
552 | ||
553 | size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); | |
9f95a23c | 554 | m = malloc(sizeof(struct rte_vhost_memory) + size); |
11fdf7f2 TL |
555 | if (!m) |
556 | return -1; | |
557 | ||
558 | m->nregions = dev->mem->nregions; | |
559 | memcpy(m->regions, dev->mem->regions, size); | |
560 | *mem = m; | |
561 | ||
562 | return 0; | |
563 | } | |
564 | ||
565 | int | |
566 | rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, | |
567 | struct rte_vhost_vring *vring) | |
568 | { | |
569 | struct virtio_net *dev; | |
570 | struct vhost_virtqueue *vq; | |
571 | ||
572 | dev = get_device(vid); | |
9f95a23c | 573 | if (dev == NULL || vring == NULL) |
11fdf7f2 TL |
574 | return -1; |
575 | ||
576 | if (vring_idx >= VHOST_MAX_VRING) | |
577 | return -1; | |
578 | ||
579 | vq = dev->virtqueue[vring_idx]; | |
580 | if (!vq) | |
581 | return -1; | |
582 | ||
583 | vring->desc = vq->desc; | |
584 | vring->avail = vq->avail; | |
585 | vring->used = vq->used; | |
586 | vring->log_guest_addr = vq->log_guest_addr; | |
587 | ||
588 | vring->callfd = vq->callfd; | |
589 | vring->kickfd = vq->kickfd; | |
590 | vring->size = vq->size; | |
591 | ||
592 | return 0; | |
593 | } | |
594 | ||
9f95a23c TL |
595 | int |
596 | rte_vhost_vring_call(int vid, uint16_t vring_idx) | |
597 | { | |
598 | struct virtio_net *dev; | |
599 | struct vhost_virtqueue *vq; | |
600 | ||
601 | dev = get_device(vid); | |
602 | if (!dev) | |
603 | return -1; | |
604 | ||
605 | if (vring_idx >= VHOST_MAX_VRING) | |
606 | return -1; | |
607 | ||
608 | vq = dev->virtqueue[vring_idx]; | |
609 | if (!vq) | |
610 | return -1; | |
611 | ||
612 | if (vq_is_packed(dev)) | |
613 | vhost_vring_call_packed(dev, vq); | |
614 | else | |
615 | vhost_vring_call_split(dev, vq); | |
616 | ||
617 | return 0; | |
618 | } | |
619 | ||
7c673cae FG |
620 | uint16_t |
621 | rte_vhost_avail_entries(int vid, uint16_t queue_id) | |
622 | { | |
623 | struct virtio_net *dev; | |
624 | struct vhost_virtqueue *vq; | |
625 | ||
626 | dev = get_device(vid); | |
627 | if (!dev) | |
628 | return 0; | |
629 | ||
630 | vq = dev->virtqueue[queue_id]; | |
631 | if (!vq->enabled) | |
632 | return 0; | |
633 | ||
634 | return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; | |
635 | } | |
636 | ||
9f95a23c TL |
637 | static inline void |
638 | vhost_enable_notify_split(struct virtio_net *dev, | |
639 | struct vhost_virtqueue *vq, int enable) | |
640 | { | |
641 | if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { | |
642 | if (enable) | |
643 | vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; | |
644 | else | |
645 | vq->used->flags |= VRING_USED_F_NO_NOTIFY; | |
646 | } else { | |
647 | if (enable) | |
648 | vhost_avail_event(vq) = vq->last_avail_idx; | |
649 | } | |
650 | } | |
651 | ||
652 | static inline void | |
653 | vhost_enable_notify_packed(struct virtio_net *dev, | |
654 | struct vhost_virtqueue *vq, int enable) | |
655 | { | |
656 | uint16_t flags; | |
657 | ||
658 | if (!enable) { | |
659 | vq->device_event->flags = VRING_EVENT_F_DISABLE; | |
660 | return; | |
661 | } | |
662 | ||
663 | flags = VRING_EVENT_F_ENABLE; | |
664 | if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { | |
665 | flags = VRING_EVENT_F_DESC; | |
666 | vq->device_event->off_wrap = vq->last_avail_idx | | |
667 | vq->avail_wrap_counter << 15; | |
668 | } | |
669 | ||
670 | rte_smp_wmb(); | |
671 | ||
672 | vq->device_event->flags = flags; | |
673 | } | |
674 | ||
7c673cae FG |
675 | int |
676 | rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) | |
677 | { | |
678 | struct virtio_net *dev = get_device(vid); | |
9f95a23c | 679 | struct vhost_virtqueue *vq; |
7c673cae | 680 | |
9f95a23c | 681 | if (!dev) |
7c673cae FG |
682 | return -1; |
683 | ||
9f95a23c TL |
684 | vq = dev->virtqueue[queue_id]; |
685 | ||
686 | if (vq_is_packed(dev)) | |
687 | vhost_enable_notify_packed(dev, vq, enable); | |
688 | else | |
689 | vhost_enable_notify_split(dev, vq, enable); | |
7c673cae | 690 | |
7c673cae FG |
691 | return 0; |
692 | } | |
693 | ||
11fdf7f2 TL |
694 | void |
695 | rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) | |
7c673cae | 696 | { |
11fdf7f2 | 697 | struct virtio_net *dev = get_device(vid); |
7c673cae | 698 | |
11fdf7f2 TL |
699 | if (dev == NULL) |
700 | return; | |
7c673cae | 701 | |
11fdf7f2 | 702 | vhost_log_write(dev, addr, len); |
7c673cae FG |
703 | } |
704 | ||
11fdf7f2 TL |
705 | void |
706 | rte_vhost_log_used_vring(int vid, uint16_t vring_idx, | |
707 | uint64_t offset, uint64_t len) | |
7c673cae | 708 | { |
11fdf7f2 TL |
709 | struct virtio_net *dev; |
710 | struct vhost_virtqueue *vq; | |
7c673cae | 711 | |
11fdf7f2 TL |
712 | dev = get_device(vid); |
713 | if (dev == NULL) | |
714 | return; | |
715 | ||
716 | if (vring_idx >= VHOST_MAX_VRING) | |
717 | return; | |
718 | vq = dev->virtqueue[vring_idx]; | |
719 | if (!vq) | |
720 | return; | |
721 | ||
722 | vhost_log_used_vring(dev, vq, offset, len); | |
7c673cae | 723 | } |
9f95a23c TL |
724 | |
725 | uint32_t | |
726 | rte_vhost_rx_queue_count(int vid, uint16_t qid) | |
727 | { | |
728 | struct virtio_net *dev; | |
729 | struct vhost_virtqueue *vq; | |
730 | ||
731 | dev = get_device(vid); | |
732 | if (dev == NULL) | |
733 | return 0; | |
734 | ||
735 | if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) { | |
736 | RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n", | |
737 | dev->vid, __func__, qid); | |
738 | return 0; | |
739 | } | |
740 | ||
741 | vq = dev->virtqueue[qid]; | |
742 | if (vq == NULL) | |
743 | return 0; | |
744 | ||
745 | if (unlikely(vq->enabled == 0 || vq->avail == NULL)) | |
746 | return 0; | |
747 | ||
748 | return *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; | |
749 | } | |
750 | ||
751 | int rte_vhost_get_vdpa_device_id(int vid) | |
752 | { | |
753 | struct virtio_net *dev = get_device(vid); | |
754 | ||
755 | if (dev == NULL) | |
756 | return -1; | |
757 | ||
758 | return dev->vdpa_dev_id; | |
759 | } | |
760 | ||
761 | int rte_vhost_get_log_base(int vid, uint64_t *log_base, | |
762 | uint64_t *log_size) | |
763 | { | |
764 | struct virtio_net *dev = get_device(vid); | |
765 | ||
766 | if (dev == NULL || log_base == NULL || log_size == NULL) | |
767 | return -1; | |
768 | ||
769 | *log_base = dev->log_base; | |
770 | *log_size = dev->log_size; | |
771 | ||
772 | return 0; | |
773 | } | |
774 | ||
775 | int rte_vhost_get_vring_base(int vid, uint16_t queue_id, | |
776 | uint16_t *last_avail_idx, uint16_t *last_used_idx) | |
777 | { | |
778 | struct virtio_net *dev = get_device(vid); | |
779 | ||
780 | if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) | |
781 | return -1; | |
782 | ||
783 | *last_avail_idx = dev->virtqueue[queue_id]->last_avail_idx; | |
784 | *last_used_idx = dev->virtqueue[queue_id]->last_used_idx; | |
785 | ||
786 | return 0; | |
787 | } | |
788 | ||
789 | int rte_vhost_set_vring_base(int vid, uint16_t queue_id, | |
790 | uint16_t last_avail_idx, uint16_t last_used_idx) | |
791 | { | |
792 | struct virtio_net *dev = get_device(vid); | |
793 | ||
794 | if (!dev) | |
795 | return -1; | |
796 | ||
797 | dev->virtqueue[queue_id]->last_avail_idx = last_avail_idx; | |
798 | dev->virtqueue[queue_id]->last_used_idx = last_used_idx; | |
799 | ||
800 | return 0; | |
801 | } | |
802 | ||
803 | int rte_vhost_extern_callback_register(int vid, | |
804 | struct rte_vhost_user_extern_ops const * const ops, void *ctx) | |
805 | { | |
806 | struct virtio_net *dev = get_device(vid); | |
807 | ||
808 | if (dev == NULL || ops == NULL) | |
809 | return -1; | |
810 | ||
811 | dev->extern_ops = *ops; | |
812 | dev->extern_data = ctx; | |
813 | return 0; | |
814 | } |