]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/lib/vhost/rte_vhost/vhost_user.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / spdk / lib / vhost / rte_vhost / vhost_user.c
1 /*-
2 * BSD LICENSE
3 *
4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <stdint.h>
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #include <stdbool.h>
39 #include <unistd.h>
40 #include <sys/mman.h>
41 #include <asm/mman.h>
42 #include <sys/types.h>
43 #include <sys/stat.h>
44 #include <assert.h>
45 #ifdef RTE_LIBRTE_VHOST_NUMA
46 #include <numaif.h>
47 #endif
48
49 #include <rte_common.h>
50 #include <rte_malloc.h>
51 #include <rte_log.h>
52
53 #include "vhost.h"
54 #include "vhost_user.h"
55
56 #define VIRTIO_MIN_MTU 68
57 #define VIRTIO_MAX_MTU 65535
58
59 static const char *vhost_message_str[VHOST_USER_MAX] = {
60 [VHOST_USER_NONE] = "VHOST_USER_NONE",
61 [VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
62 [VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
63 [VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
64 [VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
65 [VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
66 [VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
67 [VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
68 [VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
69 [VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
70 [VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
71 [VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
72 [VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
73 [VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
74 [VHOST_USER_SET_VRING_ERR] = "VHOST_USER_SET_VRING_ERR",
75 [VHOST_USER_GET_PROTOCOL_FEATURES] = "VHOST_USER_GET_PROTOCOL_FEATURES",
76 [VHOST_USER_SET_PROTOCOL_FEATURES] = "VHOST_USER_SET_PROTOCOL_FEATURES",
77 [VHOST_USER_GET_QUEUE_NUM] = "VHOST_USER_GET_QUEUE_NUM",
78 [VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE",
79 [VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP",
80 [VHOST_USER_NET_SET_MTU] = "VHOST_USER_NET_SET_MTU",
81 [VHOST_USER_GET_CONFIG] = "VHOST_USER_GET_CONFIG",
82 [VHOST_USER_SET_CONFIG] = "VHOST_USER_SET_CONFIG",
83 [VHOST_USER_NVME_ADMIN] = "VHOST_USER_NVME_ADMIN",
84 [VHOST_USER_NVME_SET_CQ_CALL] = "VHOST_USER_NVME_SET_CQ_CALL",
85 [VHOST_USER_NVME_GET_CAP] = "VHOST_USER_NVME_GET_CAP",
86 [VHOST_USER_NVME_START_STOP] = "VHOST_USER_NVME_START_STOP",
87 [VHOST_USER_NVME_IO_CMD] = "VHOST_USER_NVME_IO_CMD"
88 };
89
90 static uint64_t
91 get_blk_size(int fd)
92 {
93 struct stat stat;
94 int ret;
95
96 ret = fstat(fd, &stat);
97 return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
98 }
99
100 static void
101 free_mem_region(struct virtio_net *dev)
102 {
103 uint32_t i;
104 struct rte_vhost_mem_region *reg;
105
106 if (!dev || !dev->mem)
107 return;
108
109 for (i = 0; i < dev->mem->nregions; i++) {
110 reg = &dev->mem->regions[i];
111 if (reg->host_user_addr) {
112 munmap(reg->mmap_addr, reg->mmap_size);
113 close(reg->fd);
114 }
115 }
116 }
117
118 void
119 vhost_backend_cleanup(struct virtio_net *dev)
120 {
121 uint32_t i;
122
123 if (dev->mem) {
124 if (dev->has_new_mem_table) {
125 for (i = 0; i < dev->mem->nregions; i++) {
126 close(dev->mem_table_fds[i]);
127 }
128 dev->has_new_mem_table = 0;
129 }
130 free_mem_region(dev);
131 rte_free(dev->mem);
132 dev->mem = NULL;
133 }
134
135 free(dev->guest_pages);
136 dev->guest_pages = NULL;
137
138 if (dev->log_addr) {
139 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
140 dev->log_addr = 0;
141 }
142 }
143
144 /*
145 * This function just returns success at the moment unless
146 * the device hasn't been initialised.
147 */
148 static int
149 vhost_user_set_owner(void)
150 {
151 return 0;
152 }
153
154 static int
155 vhost_user_reset_owner(struct virtio_net *dev)
156 {
157 if (dev->flags & VIRTIO_DEV_RUNNING) {
158 dev->flags &= ~VIRTIO_DEV_RUNNING;
159 dev->notify_ops->destroy_device(dev->vid);
160 }
161
162 cleanup_device(dev, 0);
163 reset_device(dev);
164 return 0;
165 }
166
167 /*
168 * The features that we support are requested.
169 */
170 static uint64_t
171 vhost_user_get_features(struct virtio_net *dev)
172 {
173 return dev->features;
174 }
175
176 /*
177 * We receive the negotiated features supported by us and the virtio device.
178 */
179 static int
180 vhost_user_set_features(struct virtio_net *dev, uint64_t features)
181 {
182 uint64_t vhost_features = 0;
183
184 vhost_features = vhost_user_get_features(dev);
185 if (features & ~vhost_features) {
186 RTE_LOG(ERR, VHOST_CONFIG,
187 "(%d) received invalid negotiated features.\n",
188 dev->vid);
189 return -1;
190 }
191
192 if ((dev->flags & VIRTIO_DEV_RUNNING) && dev->negotiated_features != features) {
193 if (dev->notify_ops->features_changed) {
194 dev->notify_ops->features_changed(dev->vid, features);
195 } else {
196 dev->flags &= ~VIRTIO_DEV_RUNNING;
197 dev->notify_ops->destroy_device(dev->vid);
198 }
199 }
200
201 dev->negotiated_features = features;
202 if (dev->negotiated_features &
203 ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
204 dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
205 } else {
206 dev->vhost_hlen = sizeof(struct virtio_net_hdr);
207 }
208 VHOST_LOG_DEBUG(VHOST_CONFIG,
209 "(%d) mergeable RX buffers %s, virtio 1 %s\n",
210 dev->vid,
211 (dev->negotiated_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
212 (dev->negotiated_features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
213
214 return 0;
215 }
216
217 /*
218 * The virtio device sends us the size of the descriptor ring.
219 */
220 static int
221 vhost_user_set_vring_num(struct virtio_net *dev,
222 VhostUserMsg *msg)
223 {
224 struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
225
226 vq->size = msg->payload.state.num;
227
228 if (dev->dequeue_zero_copy) {
229 vq->nr_zmbuf = 0;
230 vq->last_zmbuf_idx = 0;
231 vq->zmbuf_size = vq->size;
232 vq->zmbufs = rte_zmalloc(NULL, vq->zmbuf_size *
233 sizeof(struct zcopy_mbuf), 0);
234 if (vq->zmbufs == NULL) {
235 RTE_LOG(WARNING, VHOST_CONFIG,
236 "failed to allocate mem for zero copy; "
237 "zero copy is force disabled\n");
238 dev->dequeue_zero_copy = 0;
239 }
240 }
241
242 vq->shadow_used_ring = rte_malloc(NULL,
243 vq->size * sizeof(struct vring_used_elem),
244 RTE_CACHE_LINE_SIZE);
245 if (!vq->shadow_used_ring) {
246 RTE_LOG(ERR, VHOST_CONFIG,
247 "failed to allocate memory for shadow used ring.\n");
248 return -1;
249 }
250
251 return 0;
252 }
253
254 /*
255 * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
256 * same numa node as the memory of vring descriptor.
257 */
258 #ifdef RTE_LIBRTE_VHOST_NUMA
259 static struct virtio_net*
260 numa_realloc(struct virtio_net *dev, int index)
261 {
262 int oldnode, newnode;
263 struct virtio_net *old_dev;
264 struct vhost_virtqueue *old_vq, *vq;
265 int ret;
266
267 old_dev = dev;
268 vq = old_vq = dev->virtqueue[index];
269
270 ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
271 MPOL_F_NODE | MPOL_F_ADDR);
272
273 /* check if we need to reallocate vq */
274 ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
275 MPOL_F_NODE | MPOL_F_ADDR);
276 if (ret) {
277 RTE_LOG(ERR, VHOST_CONFIG,
278 "Unable to get vq numa information.\n");
279 return dev;
280 }
281 if (oldnode != newnode) {
282 RTE_LOG(INFO, VHOST_CONFIG,
283 "reallocate vq from %d to %d node\n", oldnode, newnode);
284 vq = rte_malloc_socket(NULL, sizeof(*vq), 0, newnode);
285 if (!vq)
286 return dev;
287
288 memcpy(vq, old_vq, sizeof(*vq));
289 rte_free(old_vq);
290 }
291
292 /* check if we need to reallocate dev */
293 ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
294 MPOL_F_NODE | MPOL_F_ADDR);
295 if (ret) {
296 RTE_LOG(ERR, VHOST_CONFIG,
297 "Unable to get dev numa information.\n");
298 goto out;
299 }
300 if (oldnode != newnode) {
301 RTE_LOG(INFO, VHOST_CONFIG,
302 "reallocate dev from %d to %d node\n",
303 oldnode, newnode);
304 dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
305 if (!dev) {
306 dev = old_dev;
307 goto out;
308 }
309
310 memcpy(dev, old_dev, sizeof(*dev));
311 rte_free(old_dev);
312 }
313
314 out:
315 dev->virtqueue[index] = vq;
316 vhost_devices[dev->vid] = dev;
317
318 return dev;
319 }
320 #else
321 static struct virtio_net*
322 numa_realloc(struct virtio_net *dev, int index __rte_unused)
323 {
324 return dev;
325 }
326 #endif
327
328 /*
329 * Converts QEMU virtual address to Vhost virtual address. This function is
330 * used to convert the ring addresses to our address space.
331 */
332 static uint64_t
333 qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
334 {
335 struct rte_vhost_mem_region *reg;
336 uint32_t i;
337
338 /* Find the region where the address lives. */
339 for (i = 0; i < dev->mem->nregions; i++) {
340 reg = &dev->mem->regions[i];
341
342 if (qva >= reg->guest_user_addr &&
343 qva < reg->guest_user_addr + reg->size) {
344
345 if (unlikely(*len > reg->guest_user_addr + reg->size - qva))
346 *len = reg->guest_user_addr + reg->size - qva;
347
348 return qva - reg->guest_user_addr +
349 reg->host_user_addr;
350 }
351 }
352
353 return 0;
354 }
355
356 static int vhost_setup_mem_table(struct virtio_net *dev);
357
358 /*
359 * The virtio device sends us the desc, used and avail ring addresses.
360 * This function then converts these to our address space.
361 */
362 static int
363 vhost_user_set_vring_addr(struct virtio_net *dev, VhostUserMsg *msg)
364 {
365 struct vhost_virtqueue *vq;
366 uint64_t len;
367
368 /* Remove from the data plane. */
369 if (dev->flags & VIRTIO_DEV_RUNNING) {
370 dev->flags &= ~VIRTIO_DEV_RUNNING;
371 dev->notify_ops->destroy_device(dev->vid);
372 }
373
374 if (dev->has_new_mem_table) {
375 vhost_setup_mem_table(dev);
376 dev->has_new_mem_table = 0;
377 }
378
379 if (dev->mem == NULL)
380 return -1;
381
382 /* addr->index refers to the queue index. The txq 1, rxq is 0. */
383 vq = dev->virtqueue[msg->payload.addr.index];
384
385 /* The addresses are converted from QEMU virtual to Vhost virtual. */
386 len = sizeof(struct vring_desc) * vq->size;
387 vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev,
388 msg->payload.addr.desc_user_addr, &len);
389 if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
390 RTE_LOG(ERR, VHOST_CONFIG,
391 "(%d) failed to map desc ring.\n",
392 dev->vid);
393 return -1;
394 }
395
396 dev = numa_realloc(dev, msg->payload.addr.index);
397 vq = dev->virtqueue[msg->payload.addr.index];
398
399 len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
400 vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev,
401 msg->payload.addr.avail_user_addr, &len);
402 if (vq->avail == 0 ||
403 len != sizeof(struct vring_avail)
404 + sizeof(uint16_t) * vq->size) {
405 RTE_LOG(ERR, VHOST_CONFIG,
406 "(%d) failed to find avail ring address.\n",
407 dev->vid);
408 return -1;
409 }
410
411 len = sizeof(struct vring_used) +
412 sizeof(struct vring_used_elem) * vq->size;
413 vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev,
414 msg->payload.addr.used_user_addr, &len);
415 if (vq->used == 0 || len != sizeof(struct vring_used) +
416 sizeof(struct vring_used_elem) * vq->size) {
417
418 RTE_LOG(ERR, VHOST_CONFIG,
419 "(%d) failed to find used ring address.\n",
420 dev->vid);
421 return -1;
422 }
423
424 if (vq->last_used_idx != vq->used->idx) {
425 RTE_LOG(WARNING, VHOST_CONFIG,
426 "last_used_idx (%u) and vq->used->idx (%u) mismatches; "
427 "some packets maybe resent for Tx and dropped for Rx\n",
428 vq->last_used_idx, vq->used->idx);
429 vq->last_used_idx = vq->used->idx;
430 vq->last_avail_idx = vq->used->idx;
431 }
432
433 vq->log_guest_addr = msg->payload.addr.log_guest_addr;
434
435 VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
436 dev->vid, vq->desc);
437 VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n",
438 dev->vid, vq->avail);
439 VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n",
440 dev->vid, vq->used);
441 VHOST_LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n",
442 dev->vid, vq->log_guest_addr);
443
444 return 0;
445 }
446
447 /*
448 * The virtio device sends us the available ring last used index.
449 */
450 static int
451 vhost_user_set_vring_base(struct virtio_net *dev,
452 VhostUserMsg *msg)
453 {
454 /* Remove from the data plane. */
455 if (dev->flags & VIRTIO_DEV_RUNNING) {
456 dev->flags &= ~VIRTIO_DEV_RUNNING;
457 dev->notify_ops->destroy_device(dev->vid);
458 }
459
460 dev->virtqueue[msg->payload.state.index]->last_used_idx = msg->payload.state.num;
461 dev->virtqueue[msg->payload.state.index]->last_avail_idx = msg->payload.state.num;
462
463 return 0;
464 }
465
466 static void
467 add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
468 uint64_t host_phys_addr, uint64_t size)
469 {
470 struct guest_page *page, *last_page;
471
472 if (dev->nr_guest_pages == dev->max_guest_pages) {
473 dev->max_guest_pages = RTE_MAX(8U, dev->max_guest_pages * 2);
474 dev->guest_pages = realloc(dev->guest_pages,
475 dev->max_guest_pages * sizeof(*page));
476 }
477
478 if (dev->nr_guest_pages > 0) {
479 last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
480 /* merge if the two pages are continuous */
481 if (host_phys_addr == last_page->host_phys_addr +
482 last_page->size) {
483 last_page->size += size;
484 return;
485 }
486 }
487
488 page = &dev->guest_pages[dev->nr_guest_pages++];
489 page->guest_phys_addr = guest_phys_addr;
490 page->host_phys_addr = host_phys_addr;
491 page->size = size;
492 }
493
494 static void
495 add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
496 uint64_t page_size)
497 {
498 uint64_t reg_size = reg->size;
499 uint64_t host_user_addr = reg->host_user_addr;
500 uint64_t guest_phys_addr = reg->guest_phys_addr;
501 uint64_t host_phys_addr;
502 uint64_t size;
503
504 host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)host_user_addr);
505 size = page_size - (guest_phys_addr & (page_size - 1));
506 size = RTE_MIN(size, reg_size);
507
508 add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
509 host_user_addr += size;
510 guest_phys_addr += size;
511 reg_size -= size;
512
513 while (reg_size > 0) {
514 size = RTE_MIN(reg_size, page_size);
515 host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)
516 host_user_addr);
517 add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
518
519 host_user_addr += size;
520 guest_phys_addr += size;
521 reg_size -= size;
522 }
523 }
524
525 #ifdef RTE_LIBRTE_VHOST_DEBUG
526 /* TODO: enable it only in debug mode? */
527 static void
528 dump_guest_pages(struct virtio_net *dev)
529 {
530 uint32_t i;
531 struct guest_page *page;
532
533 for (i = 0; i < dev->nr_guest_pages; i++) {
534 page = &dev->guest_pages[i];
535
536 RTE_LOG(INFO, VHOST_CONFIG,
537 "guest physical page region %u\n"
538 "\t guest_phys_addr: %" PRIx64 "\n"
539 "\t host_phys_addr : %" PRIx64 "\n"
540 "\t size : %" PRIx64 "\n",
541 i,
542 page->guest_phys_addr,
543 page->host_phys_addr,
544 page->size);
545 }
546 }
547 #else
548 #define dump_guest_pages(dev)
549 #endif
550
551 static int
552 vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg)
553 {
554 uint32_t i;
555
556 if (dev->has_new_mem_table) {
557 /*
558 * The previous mem table was not consumed, so close the
559 * file descriptors from that mem table before copying
560 * the new one.
561 */
562 for (i = 0; i < dev->mem_table.nregions; i++) {
563 close(dev->mem_table_fds[i]);
564 }
565 }
566
567 memcpy(&dev->mem_table, &pmsg->payload.memory, sizeof(dev->mem_table));
568 memcpy(dev->mem_table_fds, pmsg->fds, sizeof(dev->mem_table_fds));
569 dev->has_new_mem_table = 1;
570 /* vhost-user-nvme will not send
571 * set vring addr message, enable
572 * memory address table now.
573 */
574 if (dev->has_new_mem_table && dev->is_nvme) {
575 vhost_setup_mem_table(dev);
576 dev->has_new_mem_table = 0;
577 }
578
579 return 0;
580 }
581
582 static int
583 vhost_setup_mem_table(struct virtio_net *dev)
584 {
585 struct VhostUserMemory memory = dev->mem_table;
586 struct rte_vhost_mem_region *reg;
587 struct vhost_virtqueue *vq;
588 void *mmap_addr;
589 uint64_t mmap_size;
590 uint64_t mmap_offset;
591 uint64_t alignment;
592 uint32_t i;
593 int fd;
594
595 if (dev->mem) {
596 free_mem_region(dev);
597 rte_free(dev->mem);
598 dev->mem = NULL;
599 }
600
601 for (i = 0; i < dev->nr_vring; i++) {
602 vq = dev->virtqueue[i];
603 /* Those addresses won't be valid anymore in host address space
604 * after setting new mem table. Initiator need to resend these
605 * addresses.
606 */
607 vq->desc = NULL;
608 vq->avail = NULL;
609 vq->used = NULL;
610 }
611
612 dev->nr_guest_pages = 0;
613 if (!dev->guest_pages) {
614 dev->max_guest_pages = 8;
615 dev->guest_pages = malloc(dev->max_guest_pages *
616 sizeof(struct guest_page));
617 }
618
619 dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct rte_vhost_memory) +
620 sizeof(struct rte_vhost_mem_region) * memory.nregions, 0);
621 if (dev->mem == NULL) {
622 RTE_LOG(ERR, VHOST_CONFIG,
623 "(%d) failed to allocate memory for dev->mem\n",
624 dev->vid);
625 return -1;
626 }
627 dev->mem->nregions = memory.nregions;
628
629 for (i = 0; i < memory.nregions; i++) {
630 fd = dev->mem_table_fds[i];
631 reg = &dev->mem->regions[i];
632
633 reg->guest_phys_addr = memory.regions[i].guest_phys_addr;
634 reg->guest_user_addr = memory.regions[i].userspace_addr;
635 reg->size = memory.regions[i].memory_size;
636 reg->fd = fd;
637
638 mmap_offset = memory.regions[i].mmap_offset;
639 mmap_size = reg->size + mmap_offset;
640
641 /* mmap() without flag of MAP_ANONYMOUS, should be called
642 * with length argument aligned with hugepagesz at older
643 * longterm version Linux, like 2.6.32 and 3.2.72, or
644 * mmap() will fail with EINVAL.
645 *
646 * to avoid failure, make sure in caller to keep length
647 * aligned.
648 */
649 alignment = get_blk_size(fd);
650 if (alignment == (uint64_t)-1) {
651 RTE_LOG(ERR, VHOST_CONFIG,
652 "couldn't get hugepage size through fstat\n");
653 goto err_mmap;
654 }
655 mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
656
657 mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
658 MAP_SHARED | MAP_POPULATE, fd, 0);
659
660 if (mmap_addr == MAP_FAILED) {
661 RTE_LOG(ERR, VHOST_CONFIG,
662 "mmap region %u failed.\n", i);
663 goto err_mmap;
664 }
665
666 if (madvise(mmap_addr, mmap_size, MADV_DONTDUMP) != 0) {
667 RTE_LOG(INFO, VHOST_CONFIG,
668 "MADV_DONTDUMP advice setting failed.\n");
669 }
670
671 reg->mmap_addr = mmap_addr;
672 reg->mmap_size = mmap_size;
673 reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr +
674 mmap_offset;
675
676 if (dev->dequeue_zero_copy)
677 add_guest_pages(dev, reg, alignment);
678
679 RTE_LOG(INFO, VHOST_CONFIG,
680 "guest memory region %u, size: 0x%" PRIx64 "\n"
681 "\t guest physical addr: 0x%" PRIx64 "\n"
682 "\t guest virtual addr: 0x%" PRIx64 "\n"
683 "\t host virtual addr: 0x%" PRIx64 "\n"
684 "\t mmap addr : 0x%" PRIx64 "\n"
685 "\t mmap size : 0x%" PRIx64 "\n"
686 "\t mmap align: 0x%" PRIx64 "\n"
687 "\t mmap off : 0x%" PRIx64 "\n",
688 i, reg->size,
689 reg->guest_phys_addr,
690 reg->guest_user_addr,
691 reg->host_user_addr,
692 (uint64_t)(uintptr_t)mmap_addr,
693 mmap_size,
694 alignment,
695 mmap_offset);
696 }
697
698 dump_guest_pages(dev);
699
700 return 0;
701
702 err_mmap:
703 free_mem_region(dev);
704 rte_free(dev->mem);
705 dev->mem = NULL;
706 return -1;
707 }
708
709 static int
710 vq_is_ready(struct vhost_virtqueue *vq)
711 {
712 return vq && vq->desc &&
713 vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
714 vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD &&
715 vq->kickfd != VIRTIO_INVALID_EVENTFD &&
716 vq->callfd != VIRTIO_INVALID_EVENTFD;
717 }
718
719 static int
720 virtio_is_ready(struct virtio_net *dev)
721 {
722 struct vhost_virtqueue *vq;
723 uint32_t i;
724
725 if (dev->nr_vring == 0)
726 return 0;
727
728 for (i = 0; i < dev->nr_vring; i++) {
729 vq = dev->virtqueue[i];
730
731 if (vq_is_ready(vq)) {
732 RTE_LOG(INFO, VHOST_CONFIG,
733 "virtio is now ready for processing.\n");
734 return 1;
735 }
736 }
737
738 return 0;
739 }
740
741 static void
742 vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg)
743 {
744 struct vhost_vring_file file;
745 struct vhost_virtqueue *vq;
746
747 /* Remove from the data plane. */
748 if (dev->flags & VIRTIO_DEV_RUNNING) {
749 dev->flags &= ~VIRTIO_DEV_RUNNING;
750 dev->notify_ops->destroy_device(dev->vid);
751 }
752
753 file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
754 if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
755 file.fd = VIRTIO_INVALID_EVENTFD;
756 else
757 file.fd = pmsg->fds[0];
758 RTE_LOG(INFO, VHOST_CONFIG,
759 "vring call idx:%d file:%d\n", file.index, file.fd);
760
761 vq = dev->virtqueue[file.index];
762 if (vq->callfd >= 0)
763 close(vq->callfd);
764
765 vq->callfd = file.fd;
766 }
767
768 static void
769 vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg)
770 {
771 struct vhost_vring_file file;
772 struct vhost_virtqueue *vq;
773
774 /* Remove from the data plane. */
775 if (dev->flags & VIRTIO_DEV_RUNNING) {
776 dev->flags &= ~VIRTIO_DEV_RUNNING;
777 dev->notify_ops->destroy_device(dev->vid);
778 }
779
780 file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
781 if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
782 file.fd = VIRTIO_INVALID_EVENTFD;
783 else
784 file.fd = pmsg->fds[0];
785 RTE_LOG(INFO, VHOST_CONFIG,
786 "vring kick idx:%d file:%d\n", file.index, file.fd);
787
788 vq = dev->virtqueue[file.index];
789 if (vq->kickfd >= 0)
790 close(vq->kickfd);
791 vq->kickfd = file.fd;
792 }
793
794 static void
795 free_zmbufs(struct vhost_virtqueue *vq)
796 {
797 struct zcopy_mbuf *zmbuf, *next;
798
799 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
800 zmbuf != NULL; zmbuf = next) {
801 next = TAILQ_NEXT(zmbuf, next);
802
803 rte_pktmbuf_free(zmbuf->mbuf);
804 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
805 }
806
807 rte_free(vq->zmbufs);
808 }
809
810 /*
811 * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
812 */
813 static int
814 vhost_user_get_vring_base(struct virtio_net *dev,
815 VhostUserMsg *msg)
816 {
817 struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
818
819 /* We have to stop the queue (virtio) if it is running. */
820 if (dev->flags & VIRTIO_DEV_RUNNING) {
821 dev->flags &= ~VIRTIO_DEV_RUNNING;
822 dev->notify_ops->destroy_device(dev->vid);
823 }
824
825 dev->flags &= ~VIRTIO_DEV_READY;
826
827 /* Here we are safe to get the last used index */
828 msg->payload.state.num = vq->last_used_idx;
829
830 RTE_LOG(INFO, VHOST_CONFIG,
831 "vring base idx:%d file:%d\n", msg->payload.state.index, msg->payload.state.num);
832 /*
833 * Based on current qemu vhost-user implementation, this message is
834 * sent and only sent in vhost_vring_stop.
835 * TODO: cleanup the vring, it isn't usable since here.
836 */
837 if (vq->kickfd >= 0)
838 close(vq->kickfd);
839
840 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
841
842 if (vq->callfd >= 0)
843 close(vq->callfd);
844
845 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
846
847 if (dev->dequeue_zero_copy)
848 free_zmbufs(vq);
849 rte_free(vq->shadow_used_ring);
850 vq->shadow_used_ring = NULL;
851
852 return 0;
853 }
854
855 /*
856 * when virtio queues are ready to work, qemu will send us to
857 * enable the virtio queue pair.
858 */
859 static int
860 vhost_user_set_vring_enable(struct virtio_net *dev,
861 VhostUserMsg *msg)
862 {
863 int enable = (int)msg->payload.state.num;
864
865 RTE_LOG(INFO, VHOST_CONFIG,
866 "set queue enable: %d to qp idx: %d\n",
867 enable, msg->payload.state.index);
868
869 if (dev->notify_ops->vring_state_changed)
870 dev->notify_ops->vring_state_changed(dev->vid, msg->payload.state.index, enable);
871
872 dev->virtqueue[msg->payload.state.index]->enabled = enable;
873
874 return 0;
875 }
876
877 static void
878 vhost_user_set_protocol_features(struct virtio_net *dev,
879 uint64_t protocol_features)
880 {
881 if (protocol_features & ~VHOST_USER_PROTOCOL_FEATURES)
882 return;
883
884 /* Remove from the data plane. */
885 if (dev->flags & VIRTIO_DEV_RUNNING) {
886 dev->flags &= ~VIRTIO_DEV_RUNNING;
887 dev->notify_ops->destroy_device(dev->vid);
888 }
889
890 dev->protocol_features = protocol_features;
891 }
892
893 static int
894 vhost_user_set_log_base(struct virtio_net *dev, struct VhostUserMsg *msg)
895 {
896 int fd = msg->fds[0];
897 uint64_t size, off;
898 void *addr;
899
900 if (fd < 0) {
901 RTE_LOG(ERR, VHOST_CONFIG, "invalid log fd: %d\n", fd);
902 return -1;
903 }
904
905 if (msg->size != sizeof(VhostUserLog)) {
906 RTE_LOG(ERR, VHOST_CONFIG,
907 "invalid log base msg size: %"PRId32" != %d\n",
908 msg->size, (int)sizeof(VhostUserLog));
909 return -1;
910 }
911
912 /* Remove from the data plane. */
913 if (dev->flags & VIRTIO_DEV_RUNNING) {
914 dev->flags &= ~VIRTIO_DEV_RUNNING;
915 dev->notify_ops->destroy_device(dev->vid);
916 }
917
918 size = msg->payload.log.mmap_size;
919 off = msg->payload.log.mmap_offset;
920 RTE_LOG(INFO, VHOST_CONFIG,
921 "log mmap size: %"PRId64", offset: %"PRId64"\n",
922 size, off);
923
924 /*
925 * mmap from 0 to workaround a hugepage mmap bug: mmap will
926 * fail when offset is not page size aligned.
927 */
928 addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
929 close(fd);
930 if (addr == MAP_FAILED) {
931 RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n");
932 return -1;
933 }
934
935 /*
936 * Free previously mapped log memory on occasionally
937 * multiple VHOST_USER_SET_LOG_BASE.
938 */
939 if (dev->log_addr) {
940 munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
941 }
942 dev->log_addr = (uint64_t)(uintptr_t)addr;
943 dev->log_base = dev->log_addr + off;
944 dev->log_size = size;
945
946 return 0;
947 }
948
949 /*
950 * An rarp packet is constructed and broadcasted to notify switches about
951 * the new location of the migrated VM, so that packets from outside will
952 * not be lost after migration.
953 *
954 * However, we don't actually "send" a rarp packet here, instead, we set
955 * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
956 */
957 static int
958 vhost_user_send_rarp(struct virtio_net *dev, struct VhostUserMsg *msg)
959 {
960 uint8_t *mac = (uint8_t *)&msg->payload.u64;
961
962 RTE_LOG(DEBUG, VHOST_CONFIG,
963 ":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
964 mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
965 memcpy(dev->mac.addr_bytes, mac, 6);
966
967 /*
968 * Set the flag to inject a RARP broadcast packet at
969 * rte_vhost_dequeue_burst().
970 *
971 * rte_smp_wmb() is for making sure the mac is copied
972 * before the flag is set.
973 */
974 rte_smp_wmb();
975 rte_atomic16_set(&dev->broadcast_rarp, 1);
976
977 return 0;
978 }
979
980 static int
981 vhost_user_net_set_mtu(struct virtio_net *dev, struct VhostUserMsg *msg)
982 {
983 if (msg->payload.u64 < VIRTIO_MIN_MTU ||
984 msg->payload.u64 > VIRTIO_MAX_MTU) {
985 RTE_LOG(ERR, VHOST_CONFIG, "Invalid MTU size (%"PRIu64")\n",
986 msg->payload.u64);
987
988 return -1;
989 }
990
991 dev->mtu = msg->payload.u64;
992
993 return 0;
994 }
995
996 /* return bytes# of read on success or negative val on failure. */
997 static int
998 read_vhost_message(int sockfd, struct VhostUserMsg *msg)
999 {
1000 int ret;
1001
1002 ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
1003 msg->fds, VHOST_MEMORY_MAX_NREGIONS);
1004 if (ret <= 0)
1005 return ret;
1006
1007 if (msg && msg->size) {
1008 if (msg->size > sizeof(msg->payload)) {
1009 RTE_LOG(ERR, VHOST_CONFIG,
1010 "invalid msg size: %d\n", msg->size);
1011 return -1;
1012 }
1013 ret = read(sockfd, &msg->payload, msg->size);
1014 if (ret <= 0)
1015 return ret;
1016 if (ret != (int)msg->size) {
1017 RTE_LOG(ERR, VHOST_CONFIG,
1018 "read control message failed\n");
1019 return -1;
1020 }
1021 }
1022
1023 return ret;
1024 }
1025
1026 static int
1027 send_vhost_message(int sockfd, struct VhostUserMsg *msg)
1028 {
1029 int ret;
1030
1031 if (!msg)
1032 return 0;
1033
1034 msg->flags &= ~VHOST_USER_VERSION_MASK;
1035 msg->flags &= ~VHOST_USER_NEED_REPLY;
1036 msg->flags |= VHOST_USER_VERSION;
1037 msg->flags |= VHOST_USER_REPLY_MASK;
1038
1039 ret = send_fd_message(sockfd, (char *)msg,
1040 VHOST_USER_HDR_SIZE + msg->size, NULL, 0);
1041
1042 return ret;
1043 }
1044
1045 /*
1046 * Allocate a queue pair if it hasn't been allocated yet
1047 */
1048 static int
1049 vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev, VhostUserMsg *msg)
1050 {
1051 uint16_t vring_idx;
1052
1053 switch (msg->request) {
1054 case VHOST_USER_SET_VRING_KICK:
1055 case VHOST_USER_SET_VRING_CALL:
1056 case VHOST_USER_SET_VRING_ERR:
1057 vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1058 break;
1059 case VHOST_USER_SET_VRING_NUM:
1060 case VHOST_USER_SET_VRING_BASE:
1061 case VHOST_USER_SET_VRING_ENABLE:
1062 vring_idx = msg->payload.state.index;
1063 break;
1064 case VHOST_USER_SET_VRING_ADDR:
1065 vring_idx = msg->payload.addr.index;
1066 break;
1067 default:
1068 return 0;
1069 }
1070
1071 if (vring_idx >= VHOST_MAX_VRING) {
1072 RTE_LOG(ERR, VHOST_CONFIG,
1073 "invalid vring index: %u\n", vring_idx);
1074 return -1;
1075 }
1076
1077 if (dev->virtqueue[vring_idx])
1078 return 0;
1079
1080 return alloc_vring_queue(dev, vring_idx);
1081 }
1082
1083 static int
1084 vhost_user_nvme_io_request_passthrough(struct virtio_net *dev,
1085 uint16_t qid, uint16_t tail_head,
1086 bool is_submission_queue)
1087 {
1088 return -1;
1089 }
1090
1091 static int
1092 vhost_user_nvme_admin_passthrough(struct virtio_net *dev,
1093 void *cmd, void *cqe, void *buf)
1094 {
1095 if (dev->notify_ops->vhost_nvme_admin_passthrough) {
1096 return dev->notify_ops->vhost_nvme_admin_passthrough(dev->vid, cmd, cqe, buf);
1097 }
1098
1099 return -1;
1100 }
1101
1102 static int
1103 vhost_user_nvme_set_cq_call(struct virtio_net *dev, uint16_t qid, int fd)
1104 {
1105 if (dev->notify_ops->vhost_nvme_set_cq_call) {
1106 return dev->notify_ops->vhost_nvme_set_cq_call(dev->vid, qid, fd);
1107 }
1108
1109 return -1;
1110 }
1111
1112 static int
1113 vhost_user_nvme_get_cap(struct virtio_net *dev, uint64_t *cap)
1114 {
1115 if (dev->notify_ops->vhost_nvme_get_cap) {
1116 return dev->notify_ops->vhost_nvme_get_cap(dev->vid, cap);
1117 }
1118
1119 return -1;
1120 }
1121
1122 int
1123 vhost_user_msg_handler(int vid, int fd)
1124 {
1125 struct virtio_net *dev;
1126 struct VhostUserMsg msg;
1127 struct vhost_vring_file file;
1128 int ret;
1129 uint64_t cap;
1130 uint64_t enable;
1131 uint8_t cqe[16];
1132 uint8_t cmd[64];
1133 uint8_t buf[4096];
1134 uint16_t qid, tail_head;
1135 bool is_submission_queue;
1136
1137 dev = get_device(vid);
1138 if (dev == NULL)
1139 return -1;
1140
1141 if (!dev->notify_ops) {
1142 dev->notify_ops = vhost_driver_callback_get(dev->ifname);
1143 if (!dev->notify_ops) {
1144 RTE_LOG(ERR, VHOST_CONFIG,
1145 "failed to get callback ops for driver %s\n",
1146 dev->ifname);
1147 return -1;
1148 }
1149 }
1150
1151 ret = read_vhost_message(fd, &msg);
1152 if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
1153 if (ret < 0)
1154 RTE_LOG(ERR, VHOST_CONFIG,
1155 "vhost read message failed\n");
1156 else if (ret == 0)
1157 RTE_LOG(INFO, VHOST_CONFIG,
1158 "vhost peer closed\n");
1159 else
1160 RTE_LOG(ERR, VHOST_CONFIG,
1161 "vhost read incorrect message\n");
1162
1163 return -1;
1164 }
1165
1166 RTE_LOG(INFO, VHOST_CONFIG, "%s: read message %s\n",
1167 dev->ifname, vhost_message_str[msg.request]);
1168
1169 ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);
1170 if (ret < 0) {
1171 RTE_LOG(ERR, VHOST_CONFIG,
1172 "failed to alloc queue\n");
1173 return -1;
1174 }
1175
1176 switch (msg.request) {
1177 case VHOST_USER_GET_CONFIG:
1178 if (dev->notify_ops->get_config(dev->vid,
1179 msg.payload.config.region,
1180 msg.payload.config.size) != 0) {
1181 msg.size = sizeof(uint64_t);
1182 }
1183 send_vhost_message(fd, &msg);
1184 break;
1185 case VHOST_USER_SET_CONFIG:
1186 if ((dev->notify_ops->set_config(dev->vid,
1187 msg.payload.config.region,
1188 msg.payload.config.offset,
1189 msg.payload.config.size,
1190 msg.payload.config.flags)) != 0) {
1191 ret = 1;
1192 } else {
1193 ret = 0;
1194 }
1195 break;
1196 case VHOST_USER_NVME_ADMIN:
1197 if (!dev->is_nvme) {
1198 dev->is_nvme = 1;
1199 }
1200 memcpy(cmd, msg.payload.nvme.cmd.req, sizeof(cmd));
1201 ret = vhost_user_nvme_admin_passthrough(dev, cmd, cqe, buf);
1202 memcpy(msg.payload.nvme.cmd.cqe, cqe, sizeof(cqe));
1203 msg.size = sizeof(cqe);
1204 /* NVMe Identify Command */
1205 if (cmd[0] == 0x06) {
1206 memcpy(msg.payload.nvme.buf, &buf, 4096);
1207 msg.size += 4096;
1208 }
1209 send_vhost_message(fd, &msg);
1210 break;
1211 case VHOST_USER_NVME_SET_CQ_CALL:
1212 file.index = msg.payload.u64 & VHOST_USER_VRING_IDX_MASK;
1213 file.fd = msg.fds[0];
1214 ret = vhost_user_nvme_set_cq_call(dev, file.index, file.fd);
1215 break;
1216 case VHOST_USER_NVME_GET_CAP:
1217 ret = vhost_user_nvme_get_cap(dev, &cap);
1218 if (!ret)
1219 msg.payload.u64 = cap;
1220 else
1221 msg.payload.u64 = 0;
1222 msg.size = sizeof(msg.payload.u64);
1223 send_vhost_message(fd, &msg);
1224 break;
1225 case VHOST_USER_NVME_START_STOP:
1226 enable = msg.payload.u64;
1227 /* device must be started before set cq call */
1228 if (enable) {
1229 if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
1230 if (dev->notify_ops->new_device(dev->vid) == 0)
1231 dev->flags |= VIRTIO_DEV_RUNNING;
1232 }
1233 } else {
1234 if (dev->flags & VIRTIO_DEV_RUNNING) {
1235 dev->flags &= ~VIRTIO_DEV_RUNNING;
1236 dev->notify_ops->destroy_device(dev->vid);
1237 }
1238 }
1239 break;
1240 case VHOST_USER_NVME_IO_CMD:
1241 qid = msg.payload.nvme_io.qid;
1242 tail_head = msg.payload.nvme_io.tail_head;
1243 is_submission_queue = (msg.payload.nvme_io.queue_type == VHOST_USER_NVME_SUBMISSION_QUEUE) ? true : false;
1244 vhost_user_nvme_io_request_passthrough(dev, qid, tail_head, is_submission_queue);
1245 break;
1246 case VHOST_USER_GET_FEATURES:
1247 msg.payload.u64 = vhost_user_get_features(dev);
1248 msg.size = sizeof(msg.payload.u64);
1249 send_vhost_message(fd, &msg);
1250 break;
1251 case VHOST_USER_SET_FEATURES:
1252 vhost_user_set_features(dev, msg.payload.u64);
1253 break;
1254
1255 case VHOST_USER_GET_PROTOCOL_FEATURES:
1256 msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES;
1257 msg.size = sizeof(msg.payload.u64);
1258 send_vhost_message(fd, &msg);
1259 break;
1260 case VHOST_USER_SET_PROTOCOL_FEATURES:
1261 vhost_user_set_protocol_features(dev, msg.payload.u64);
1262 break;
1263
1264 case VHOST_USER_SET_OWNER:
1265 vhost_user_set_owner();
1266 break;
1267 case VHOST_USER_RESET_OWNER:
1268 vhost_user_reset_owner(dev);
1269 break;
1270
1271 case VHOST_USER_SET_MEM_TABLE:
1272 ret = vhost_user_set_mem_table(dev, &msg);
1273 break;
1274
1275 case VHOST_USER_SET_LOG_BASE:
1276 vhost_user_set_log_base(dev, &msg);
1277
1278 /* it needs a reply */
1279 msg.size = sizeof(msg.payload.u64);
1280 send_vhost_message(fd, &msg);
1281 break;
1282 case VHOST_USER_SET_LOG_FD:
1283 close(msg.fds[0]);
1284 RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n");
1285 break;
1286
1287 case VHOST_USER_SET_VRING_NUM:
1288 vhost_user_set_vring_num(dev, &msg);
1289 break;
1290 case VHOST_USER_SET_VRING_ADDR:
1291 vhost_user_set_vring_addr(dev, &msg);
1292 break;
1293 case VHOST_USER_SET_VRING_BASE:
1294 vhost_user_set_vring_base(dev, &msg);
1295 break;
1296
1297 case VHOST_USER_GET_VRING_BASE:
1298 vhost_user_get_vring_base(dev, &msg);
1299 msg.size = sizeof(msg.payload.state);
1300 send_vhost_message(fd, &msg);
1301 break;
1302
1303 case VHOST_USER_SET_VRING_KICK:
1304 vhost_user_set_vring_kick(dev, &msg);
1305 break;
1306 case VHOST_USER_SET_VRING_CALL:
1307 vhost_user_set_vring_call(dev, &msg);
1308 break;
1309
1310 case VHOST_USER_SET_VRING_ERR:
1311 if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1312 close(msg.fds[0]);
1313 RTE_LOG(INFO, VHOST_CONFIG, "not implemented\n");
1314 break;
1315
1316 case VHOST_USER_GET_QUEUE_NUM:
1317 msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS;
1318 msg.size = sizeof(msg.payload.u64);
1319 send_vhost_message(fd, &msg);
1320 break;
1321
1322 case VHOST_USER_SET_VRING_ENABLE:
1323 vhost_user_set_vring_enable(dev, &msg);
1324 break;
1325 case VHOST_USER_SEND_RARP:
1326 vhost_user_send_rarp(dev, &msg);
1327 break;
1328
1329 case VHOST_USER_NET_SET_MTU:
1330 ret = vhost_user_net_set_mtu(dev, &msg);
1331 break;
1332
1333 default:
1334 ret = -1;
1335 break;
1336
1337 }
1338
1339 if (msg.flags & VHOST_USER_NEED_REPLY) {
1340 msg.payload.u64 = !!ret;
1341 msg.size = sizeof(msg.payload.u64);
1342 send_vhost_message(fd, &msg);
1343 }
1344
1345 if (!(dev->flags & VIRTIO_DEV_RUNNING) && virtio_is_ready(dev)) {
1346 dev->flags |= VIRTIO_DEV_READY;
1347
1348 if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
1349 if (dev->dequeue_zero_copy) {
1350 RTE_LOG(INFO, VHOST_CONFIG,
1351 "dequeue zero copy is enabled\n");
1352 }
1353
1354 if (dev->notify_ops->new_device(dev->vid) == 0)
1355 dev->flags |= VIRTIO_DEV_RUNNING;
1356 }
1357 }
1358
1359 return 0;
1360 }
Ceph