]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #include <linux/vhost.h> | |
35 | #include <linux/virtio_net.h> | |
36 | #include <stddef.h> | |
37 | #include <stdint.h> | |
38 | #include <stdlib.h> | |
39 | #ifdef RTE_LIBRTE_VHOST_NUMA | |
40 | #include <numaif.h> | |
41 | #endif | |
42 | ||
43 | #include <rte_ethdev.h> | |
44 | #include <rte_log.h> | |
45 | #include <rte_string_fns.h> | |
46 | #include <rte_memory.h> | |
47 | #include <rte_malloc.h> | |
11fdf7f2 | 48 | #include <rte_vhost.h> |
7c673cae FG |
49 | |
50 | #include "vhost.h" | |
51 | ||
7c673cae FG |
52 | struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; |
53 | ||
7c673cae FG |
54 | struct virtio_net * |
55 | get_device(int vid) | |
56 | { | |
57 | struct virtio_net *dev = vhost_devices[vid]; | |
58 | ||
59 | if (unlikely(!dev)) { | |
60 | RTE_LOG(ERR, VHOST_CONFIG, | |
61 | "(%d) device not found.\n", vid); | |
62 | } | |
63 | ||
64 | return dev; | |
65 | } | |
66 | ||
67 | static void | |
68 | cleanup_vq(struct vhost_virtqueue *vq, int destroy) | |
69 | { | |
70 | if ((vq->callfd >= 0) && (destroy != 0)) | |
71 | close(vq->callfd); | |
72 | if (vq->kickfd >= 0) | |
73 | close(vq->kickfd); | |
74 | } | |
75 | ||
76 | /* | |
77 | * Unmap any memory, close any file descriptors and | |
78 | * free any memory owned by a device. | |
79 | */ | |
80 | void | |
81 | cleanup_device(struct virtio_net *dev, int destroy) | |
82 | { | |
83 | uint32_t i; | |
84 | ||
85 | vhost_backend_cleanup(dev); | |
86 | ||
11fdf7f2 TL |
87 | for (i = 0; i < dev->nr_vring; i++) |
88 | cleanup_vq(dev->virtqueue[i], destroy); | |
7c673cae FG |
89 | } |
90 | ||
91 | /* | |
92 | * Release virtqueues and device memory. | |
93 | */ | |
94 | static void | |
95 | free_device(struct virtio_net *dev) | |
96 | { | |
97 | uint32_t i; | |
11fdf7f2 | 98 | struct vhost_virtqueue *vq; |
7c673cae | 99 | |
11fdf7f2 TL |
100 | for (i = 0; i < dev->nr_vring; i++) { |
101 | vq = dev->virtqueue[i]; | |
7c673cae | 102 | |
11fdf7f2 | 103 | rte_free(vq->shadow_used_ring); |
7c673cae | 104 | |
11fdf7f2 | 105 | rte_free(vq); |
7c673cae FG |
106 | } |
107 | ||
108 | rte_free(dev); | |
109 | } | |
110 | ||
111 | static void | |
11fdf7f2 | 112 | init_vring_queue(struct vhost_virtqueue *vq) |
7c673cae FG |
113 | { |
114 | memset(vq, 0, sizeof(struct vhost_virtqueue)); | |
115 | ||
116 | vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; | |
117 | vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; | |
118 | ||
119 | /* Backends are set to -1 indicating an inactive device. */ | |
120 | vq->backend = -1; | |
121 | ||
11fdf7f2 TL |
122 | /* |
123 | * always set the vq to enabled; this is to keep compatibility | |
124 | * with the old QEMU, whereas there is no SET_VRING_ENABLE message. | |
125 | */ | |
126 | vq->enabled = 1; | |
7c673cae FG |
127 | |
128 | TAILQ_INIT(&vq->zmbuf_list); | |
129 | } | |
130 | ||
131 | static void | |
11fdf7f2 | 132 | reset_vring_queue(struct vhost_virtqueue *vq) |
7c673cae FG |
133 | { |
134 | int callfd; | |
135 | ||
136 | callfd = vq->callfd; | |
11fdf7f2 | 137 | init_vring_queue(vq); |
7c673cae FG |
138 | vq->callfd = callfd; |
139 | } | |
140 | ||
7c673cae | 141 | int |
11fdf7f2 | 142 | alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) |
7c673cae | 143 | { |
11fdf7f2 | 144 | struct vhost_virtqueue *vq; |
7c673cae | 145 | |
11fdf7f2 TL |
146 | vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); |
147 | if (vq == NULL) { | |
7c673cae | 148 | RTE_LOG(ERR, VHOST_CONFIG, |
11fdf7f2 | 149 | "Failed to allocate memory for vring:%u.\n", vring_idx); |
7c673cae FG |
150 | return -1; |
151 | } | |
152 | ||
11fdf7f2 TL |
153 | dev->virtqueue[vring_idx] = vq; |
154 | init_vring_queue(vq); | |
7c673cae | 155 | |
11fdf7f2 | 156 | dev->nr_vring += 1; |
7c673cae FG |
157 | |
158 | return 0; | |
159 | } | |
160 | ||
161 | /* | |
162 | * Reset some variables in device structure, while keeping few | |
11fdf7f2 | 163 | * others untouched, such as vid, ifname, nr_vring: they |
7c673cae FG |
164 | * should be same unless the device is removed. |
165 | */ | |
166 | void | |
167 | reset_device(struct virtio_net *dev) | |
168 | { | |
169 | uint32_t i; | |
170 | ||
11fdf7f2 | 171 | dev->negotiated_features = 0; |
7c673cae FG |
172 | dev->protocol_features = 0; |
173 | dev->flags = 0; | |
174 | ||
11fdf7f2 TL |
175 | for (i = 0; i < dev->nr_vring; i++) |
176 | reset_vring_queue(dev->virtqueue[i]); | |
7c673cae FG |
177 | } |
178 | ||
179 | /* | |
180 | * Invoked when there is a new vhost-user connection established (when | |
181 | * there is a new virtio device being attached). | |
182 | */ | |
183 | int | |
9f95a23c | 184 | vhost_new_device(uint64_t features, struct vhost_device_ops const *ops) |
7c673cae FG |
185 | { |
186 | struct virtio_net *dev; | |
187 | int i; | |
188 | ||
189 | dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); | |
190 | if (dev == NULL) { | |
191 | RTE_LOG(ERR, VHOST_CONFIG, | |
192 | "Failed to allocate memory for new dev.\n"); | |
193 | return -1; | |
194 | } | |
195 | ||
196 | for (i = 0; i < MAX_VHOST_DEVICE; i++) { | |
197 | if (vhost_devices[i] == NULL) | |
198 | break; | |
199 | } | |
200 | if (i == MAX_VHOST_DEVICE) { | |
201 | RTE_LOG(ERR, VHOST_CONFIG, | |
202 | "Failed to find a free slot for new device.\n"); | |
11fdf7f2 | 203 | rte_free(dev); |
7c673cae FG |
204 | return -1; |
205 | } | |
206 | ||
207 | vhost_devices[i] = dev; | |
208 | dev->vid = i; | |
11fdf7f2 | 209 | dev->features = features; |
9f95a23c | 210 | dev->notify_ops = ops; |
7c673cae FG |
211 | |
212 | return i; | |
213 | } | |
214 | ||
215 | /* | |
216 | * Invoked when there is the vhost-user connection is broken (when | |
217 | * the virtio device is being detached). | |
218 | */ | |
219 | void | |
220 | vhost_destroy_device(int vid) | |
221 | { | |
222 | struct virtio_net *dev = get_device(vid); | |
223 | ||
224 | if (dev == NULL) | |
225 | return; | |
226 | ||
227 | if (dev->flags & VIRTIO_DEV_RUNNING) { | |
228 | dev->flags &= ~VIRTIO_DEV_RUNNING; | |
11fdf7f2 | 229 | dev->notify_ops->destroy_device(vid); |
7c673cae FG |
230 | } |
231 | ||
232 | cleanup_device(dev, 1); | |
233 | free_device(dev); | |
234 | ||
235 | vhost_devices[vid] = NULL; | |
236 | } | |
237 | ||
238 | void | |
239 | vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) | |
240 | { | |
241 | struct virtio_net *dev; | |
242 | unsigned int len; | |
243 | ||
244 | dev = get_device(vid); | |
245 | if (dev == NULL) | |
246 | return; | |
247 | ||
248 | len = if_len > sizeof(dev->ifname) ? | |
249 | sizeof(dev->ifname) : if_len; | |
250 | ||
251 | strncpy(dev->ifname, if_name, len); | |
252 | dev->ifname[sizeof(dev->ifname) - 1] = '\0'; | |
253 | } | |
254 | ||
255 | void | |
256 | vhost_enable_dequeue_zero_copy(int vid) | |
257 | { | |
258 | struct virtio_net *dev = get_device(vid); | |
259 | ||
260 | if (dev == NULL) | |
261 | return; | |
262 | ||
263 | dev->dequeue_zero_copy = 1; | |
264 | } | |
265 | ||
11fdf7f2 TL |
266 | int |
267 | rte_vhost_get_mtu(int vid, uint16_t *mtu) | |
268 | { | |
269 | struct virtio_net *dev = get_device(vid); | |
270 | ||
271 | if (!dev) | |
272 | return -ENODEV; | |
273 | ||
274 | if (!(dev->flags & VIRTIO_DEV_READY)) | |
275 | return -EAGAIN; | |
276 | ||
277 | if (!(dev->negotiated_features & VIRTIO_NET_F_MTU)) | |
278 | return -ENOTSUP; | |
279 | ||
280 | *mtu = dev->mtu; | |
281 | ||
282 | return 0; | |
283 | } | |
284 | ||
7c673cae FG |
285 | int |
286 | rte_vhost_get_numa_node(int vid) | |
287 | { | |
288 | #ifdef RTE_LIBRTE_VHOST_NUMA | |
289 | struct virtio_net *dev = get_device(vid); | |
290 | int numa_node; | |
291 | int ret; | |
292 | ||
293 | if (dev == NULL) | |
294 | return -1; | |
295 | ||
296 | ret = get_mempolicy(&numa_node, NULL, 0, dev, | |
297 | MPOL_F_NODE | MPOL_F_ADDR); | |
298 | if (ret < 0) { | |
299 | RTE_LOG(ERR, VHOST_CONFIG, | |
300 | "(%d) failed to query numa node: %d\n", vid, ret); | |
301 | return -1; | |
302 | } | |
303 | ||
304 | return numa_node; | |
305 | #else | |
306 | RTE_SET_USED(vid); | |
307 | return -1; | |
308 | #endif | |
309 | } | |
310 | ||
11fdf7f2 TL |
311 | int |
312 | rte_vhost_get_ifname(int vid, char *buf, size_t len) | |
7c673cae FG |
313 | { |
314 | struct virtio_net *dev = get_device(vid); | |
315 | ||
316 | if (dev == NULL) | |
11fdf7f2 TL |
317 | return -1; |
318 | ||
319 | len = RTE_MIN(len, sizeof(dev->ifname)); | |
320 | ||
321 | strncpy(buf, dev->ifname, len); | |
322 | buf[len - 1] = '\0'; | |
7c673cae | 323 | |
11fdf7f2 | 324 | return 0; |
7c673cae FG |
325 | } |
326 | ||
327 | int | |
11fdf7f2 | 328 | rte_vhost_get_negotiated_features(int vid, uint64_t *features) |
7c673cae | 329 | { |
11fdf7f2 | 330 | struct virtio_net *dev; |
7c673cae | 331 | |
11fdf7f2 TL |
332 | dev = get_device(vid); |
333 | if (!dev) | |
7c673cae FG |
334 | return -1; |
335 | ||
11fdf7f2 TL |
336 | *features = dev->negotiated_features; |
337 | return 0; | |
338 | } | |
7c673cae | 339 | |
11fdf7f2 TL |
340 | int |
341 | rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) | |
342 | { | |
343 | struct virtio_net *dev; | |
344 | struct rte_vhost_memory *m; | |
345 | size_t size; | |
346 | ||
347 | dev = get_device(vid); | |
348 | if (!dev) | |
349 | return -1; | |
350 | ||
351 | size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); | |
352 | m = malloc(sizeof(struct rte_vhost_memory) + size); | |
353 | if (!m) | |
354 | return -1; | |
355 | ||
356 | m->nregions = dev->mem->nregions; | |
357 | memcpy(m->regions, dev->mem->regions, size); | |
358 | *mem = m; | |
359 | ||
360 | return 0; | |
361 | } | |
362 | ||
363 | int | |
364 | rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, | |
365 | struct rte_vhost_vring *vring) | |
366 | { | |
367 | struct virtio_net *dev; | |
368 | struct vhost_virtqueue *vq; | |
369 | ||
370 | dev = get_device(vid); | |
371 | if (!dev) | |
372 | return -1; | |
373 | ||
374 | if (vring_idx >= VHOST_MAX_VRING) | |
375 | return -1; | |
376 | ||
377 | vq = dev->virtqueue[vring_idx]; | |
378 | if (!vq) | |
379 | return -1; | |
380 | ||
381 | vring->desc = vq->desc; | |
382 | vring->avail = vq->avail; | |
383 | vring->used = vq->used; | |
384 | vring->log_guest_addr = vq->log_guest_addr; | |
385 | ||
386 | vring->callfd = vq->callfd; | |
387 | vring->kickfd = vq->kickfd; | |
388 | vring->size = vq->size; | |
389 | ||
7c673cae FG |
390 | return 0; |
391 | } | |
392 | ||
393 | uint16_t | |
394 | rte_vhost_avail_entries(int vid, uint16_t queue_id) | |
395 | { | |
396 | struct virtio_net *dev; | |
397 | struct vhost_virtqueue *vq; | |
398 | ||
399 | dev = get_device(vid); | |
400 | if (!dev) | |
401 | return 0; | |
402 | ||
403 | vq = dev->virtqueue[queue_id]; | |
404 | if (!vq->enabled) | |
405 | return 0; | |
406 | ||
407 | return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; | |
408 | } | |
409 | ||
410 | int | |
411 | rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) | |
412 | { | |
413 | struct virtio_net *dev = get_device(vid); | |
414 | ||
415 | if (dev == NULL) | |
416 | return -1; | |
417 | ||
418 | if (enable) { | |
419 | RTE_LOG(ERR, VHOST_CONFIG, | |
420 | "guest notification isn't supported.\n"); | |
421 | return -1; | |
422 | } | |
423 | ||
424 | dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY; | |
425 | return 0; | |
426 | } | |
427 | ||
11fdf7f2 TL |
428 | void |
429 | rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) | |
7c673cae | 430 | { |
11fdf7f2 | 431 | struct virtio_net *dev = get_device(vid); |
7c673cae | 432 | |
11fdf7f2 TL |
433 | if (dev == NULL) |
434 | return; | |
435 | ||
436 | vhost_log_write(dev, addr, len); | |
7c673cae FG |
437 | } |
438 | ||
11fdf7f2 TL |
439 | void |
440 | rte_vhost_log_used_vring(int vid, uint16_t vring_idx, | |
441 | uint64_t offset, uint64_t len) | |
7c673cae | 442 | { |
11fdf7f2 TL |
443 | struct virtio_net *dev; |
444 | struct vhost_virtqueue *vq; | |
445 | ||
446 | dev = get_device(vid); | |
447 | if (dev == NULL) | |
448 | return; | |
449 | ||
450 | if (vring_idx >= VHOST_MAX_VRING) | |
451 | return; | |
452 | vq = dev->virtqueue[vring_idx]; | |
453 | if (!vq) | |
454 | return; | |
455 | ||
456 | vhost_log_used_vring(dev, vq, offset, len); | |
7c673cae FG |
457 | } |
458 | ||
7c673cae | 459 | int |
9f95a23c TL |
460 | rte_vhost_set_vring_base(int vid, uint16_t vring_idx, |
461 | uint16_t last_avail_idx, uint16_t last_used_idx) | |
462 | { | |
11fdf7f2 TL |
463 | struct virtio_net *dev; |
464 | struct vhost_virtqueue *vq; | |
465 | ||
466 | dev = get_device(vid); | |
467 | if (!dev) | |
468 | return -1; | |
469 | ||
470 | if (vring_idx >= VHOST_MAX_VRING) | |
471 | return -1; | |
472 | ||
473 | vq = dev->virtqueue[vring_idx]; | |
474 | if (!vq) | |
475 | return -1; | |
476 | ||
477 | vq->last_avail_idx = last_avail_idx; | |
478 | vq->last_used_idx = last_used_idx; | |
7c673cae FG |
479 | |
480 | return 0; | |
481 | } | |
9f95a23c TL |
482 | |
483 | int | |
484 | rte_vhost_get_vring_base(int vid, uint16_t vring_idx, | |
485 | uint16_t *last_avail_idx, uint16_t *last_used_idx) | |
486 | { | |
487 | struct virtio_net *dev; | |
488 | struct vhost_virtqueue *vq; | |
489 | ||
490 | dev = get_device(vid); | |
491 | if (!dev) | |
492 | return -1; | |
493 | ||
494 | if (vring_idx >= VHOST_MAX_VRING) | |
495 | return -1; | |
496 | ||
497 | vq = dev->virtqueue[vring_idx]; | |
498 | if (!vq) | |
499 | return -1; | |
500 | ||
501 | *last_avail_idx = vq->last_avail_idx; | |
502 | *last_used_idx = vq->last_used_idx; | |
503 | ||
504 | return 0; | |
505 | } | |
506 | ||
507 | int | |
508 | rte_vhost_vring_call(int vid, uint16_t vring_idx) | |
509 | { | |
510 | struct virtio_net *dev; | |
511 | struct vhost_virtqueue *vq; | |
512 | ||
513 | dev = get_device(vid); | |
514 | if(!dev) | |
515 | return -1; | |
516 | ||
517 | if (vring_idx >= VHOST_MAX_VRING) | |
518 | return -1; | |
519 | ||
520 | vq = dev->virtqueue[vring_idx]; | |
521 | if (!vq) | |
522 | return -1; | |
523 | ||
524 | if (vq->callfd != -1) { | |
525 | eventfd_write(vq->callfd, (eventfd_t)1); | |
526 | return 0; | |
527 | } | |
528 | ||
529 | return -1; | |
530 | } |