4 * Copyright (c) 2013 Virtual Open Systems Sarl.
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/vhost.h"
14 #include "hw/virtio/vhost-user.h"
15 #include "hw/virtio/vhost-backend.h"
16 #include "hw/virtio/virtio.h"
17 #include "hw/virtio/virtio-net.h"
18 #include "chardev/char-fe.h"
19 #include "io/channel-socket.h"
20 #include "sysemu/kvm.h"
21 #include "qemu/error-report.h"
22 #include "qemu/main-loop.h"
23 #include "qemu/sockets.h"
24 #include "sysemu/runstate.h"
25 #include "sysemu/cryptodev.h"
26 #include "migration/migration.h"
27 #include "migration/postcopy-ram.h"
29 #include "exec/ramblock.h"
31 #include <sys/ioctl.h>
32 #include <sys/socket.h>
35 #include "standard-headers/linux/vhost_types.h"
38 #include <linux/userfaultfd.h>
41 #define VHOST_MEMORY_BASELINE_NREGIONS 8
42 #define VHOST_USER_F_PROTOCOL_FEATURES 30
43 #define VHOST_USER_BACKEND_MAX_FDS 8
45 #if defined(TARGET_PPC) || defined(TARGET_PPC64)
46 #include "hw/ppc/spapr.h"
47 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
50 #define VHOST_USER_MAX_RAM_SLOTS 512
54 * Maximum size of virtio device config space
56 #define VHOST_USER_MAX_CONFIG_SIZE 256
58 enum VhostUserProtocolFeature
{
59 VHOST_USER_PROTOCOL_F_MQ
= 0,
60 VHOST_USER_PROTOCOL_F_LOG_SHMFD
= 1,
61 VHOST_USER_PROTOCOL_F_RARP
= 2,
62 VHOST_USER_PROTOCOL_F_REPLY_ACK
= 3,
63 VHOST_USER_PROTOCOL_F_NET_MTU
= 4,
64 VHOST_USER_PROTOCOL_F_BACKEND_REQ
= 5,
65 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
= 6,
66 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
= 7,
67 VHOST_USER_PROTOCOL_F_PAGEFAULT
= 8,
68 VHOST_USER_PROTOCOL_F_CONFIG
= 9,
69 VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD
= 10,
70 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
= 11,
71 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
= 12,
72 VHOST_USER_PROTOCOL_F_RESET_DEVICE
= 13,
73 /* Feature 14 reserved for VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. */
74 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
= 15,
75 VHOST_USER_PROTOCOL_F_STATUS
= 16,
76 VHOST_USER_PROTOCOL_F_MAX
79 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
81 typedef enum VhostUserRequest
{
83 VHOST_USER_GET_FEATURES
= 1,
84 VHOST_USER_SET_FEATURES
= 2,
85 VHOST_USER_SET_OWNER
= 3,
86 VHOST_USER_RESET_OWNER
= 4,
87 VHOST_USER_SET_MEM_TABLE
= 5,
88 VHOST_USER_SET_LOG_BASE
= 6,
89 VHOST_USER_SET_LOG_FD
= 7,
90 VHOST_USER_SET_VRING_NUM
= 8,
91 VHOST_USER_SET_VRING_ADDR
= 9,
92 VHOST_USER_SET_VRING_BASE
= 10,
93 VHOST_USER_GET_VRING_BASE
= 11,
94 VHOST_USER_SET_VRING_KICK
= 12,
95 VHOST_USER_SET_VRING_CALL
= 13,
96 VHOST_USER_SET_VRING_ERR
= 14,
97 VHOST_USER_GET_PROTOCOL_FEATURES
= 15,
98 VHOST_USER_SET_PROTOCOL_FEATURES
= 16,
99 VHOST_USER_GET_QUEUE_NUM
= 17,
100 VHOST_USER_SET_VRING_ENABLE
= 18,
101 VHOST_USER_SEND_RARP
= 19,
102 VHOST_USER_NET_SET_MTU
= 20,
103 VHOST_USER_SET_BACKEND_REQ_FD
= 21,
104 VHOST_USER_IOTLB_MSG
= 22,
105 VHOST_USER_SET_VRING_ENDIAN
= 23,
106 VHOST_USER_GET_CONFIG
= 24,
107 VHOST_USER_SET_CONFIG
= 25,
108 VHOST_USER_CREATE_CRYPTO_SESSION
= 26,
109 VHOST_USER_CLOSE_CRYPTO_SESSION
= 27,
110 VHOST_USER_POSTCOPY_ADVISE
= 28,
111 VHOST_USER_POSTCOPY_LISTEN
= 29,
112 VHOST_USER_POSTCOPY_END
= 30,
113 VHOST_USER_GET_INFLIGHT_FD
= 31,
114 VHOST_USER_SET_INFLIGHT_FD
= 32,
115 VHOST_USER_GPU_SET_SOCKET
= 33,
116 VHOST_USER_RESET_DEVICE
= 34,
117 /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
118 VHOST_USER_GET_MAX_MEM_SLOTS
= 36,
119 VHOST_USER_ADD_MEM_REG
= 37,
120 VHOST_USER_REM_MEM_REG
= 38,
121 VHOST_USER_SET_STATUS
= 39,
122 VHOST_USER_GET_STATUS
= 40,
126 typedef enum VhostUserSlaveRequest
{
127 VHOST_USER_BACKEND_NONE
= 0,
128 VHOST_USER_BACKEND_IOTLB_MSG
= 1,
129 VHOST_USER_BACKEND_CONFIG_CHANGE_MSG
= 2,
130 VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG
= 3,
131 VHOST_USER_BACKEND_MAX
132 } VhostUserSlaveRequest
;
134 typedef struct VhostUserMemoryRegion
{
135 uint64_t guest_phys_addr
;
136 uint64_t memory_size
;
137 uint64_t userspace_addr
;
138 uint64_t mmap_offset
;
139 } VhostUserMemoryRegion
;
141 typedef struct VhostUserMemory
{
144 VhostUserMemoryRegion regions
[VHOST_MEMORY_BASELINE_NREGIONS
];
147 typedef struct VhostUserMemRegMsg
{
149 VhostUserMemoryRegion region
;
150 } VhostUserMemRegMsg
;
152 typedef struct VhostUserLog
{
154 uint64_t mmap_offset
;
157 typedef struct VhostUserConfig
{
161 uint8_t region
[VHOST_USER_MAX_CONFIG_SIZE
];
164 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
165 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
167 typedef struct VhostUserCryptoSession
{
168 /* session id for success, -1 on errors */
170 CryptoDevBackendSymSessionInfo session_setup_data
;
171 uint8_t key
[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN
];
172 uint8_t auth_key
[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN
];
173 } VhostUserCryptoSession
;
175 static VhostUserConfig c
__attribute__ ((unused
));
176 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
180 typedef struct VhostUserVringArea
{
184 } VhostUserVringArea
;
186 typedef struct VhostUserInflight
{
188 uint64_t mmap_offset
;
194 VhostUserRequest request
;
196 #define VHOST_USER_VERSION_MASK (0x3)
197 #define VHOST_USER_REPLY_MASK (0x1 << 2)
198 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
200 uint32_t size
; /* the following payload size */
201 } QEMU_PACKED VhostUserHeader
;
204 #define VHOST_USER_VRING_IDX_MASK (0xff)
205 #define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
207 struct vhost_vring_state state
;
208 struct vhost_vring_addr addr
;
209 VhostUserMemory memory
;
210 VhostUserMemRegMsg mem_reg
;
212 struct vhost_iotlb_msg iotlb
;
213 VhostUserConfig config
;
214 VhostUserCryptoSession session
;
215 VhostUserVringArea area
;
216 VhostUserInflight inflight
;
219 typedef struct VhostUserMsg
{
221 VhostUserPayload payload
;
222 } QEMU_PACKED VhostUserMsg
;
224 static VhostUserMsg m
__attribute__ ((unused
));
225 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
227 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
229 /* The version of the protocol we support */
230 #define VHOST_USER_VERSION (0x1)
233 struct vhost_dev
*dev
;
234 /* Shared between vhost devs of the same virtio device */
235 VhostUserState
*user
;
236 QIOChannel
*slave_ioc
;
238 NotifierWithReturn postcopy_notifier
;
239 struct PostCopyFD postcopy_fd
;
240 uint64_t postcopy_client_bases
[VHOST_USER_MAX_RAM_SLOTS
];
241 /* Length of the region_rb and region_rb_offset arrays */
242 size_t region_rb_len
;
243 /* RAMBlock associated with a given region */
244 RAMBlock
**region_rb
;
246 * The offset from the start of the RAMBlock to the start of the
249 ram_addr_t
*region_rb_offset
;
251 /* True once we've entered postcopy_listen */
252 bool postcopy_listen
;
254 /* Our current regions */
255 int num_shadow_regions
;
256 struct vhost_memory_region shadow_regions
[VHOST_USER_MAX_RAM_SLOTS
];
259 struct scrub_regions
{
260 struct vhost_memory_region
*region
;
265 static bool ioeventfd_enabled(void)
267 return !kvm_enabled() || kvm_eventfds_enabled();
270 static int vhost_user_read_header(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
272 struct vhost_user
*u
= dev
->opaque
;
273 CharBackend
*chr
= u
->user
->chr
;
274 uint8_t *p
= (uint8_t *) msg
;
275 int r
, size
= VHOST_USER_HDR_SIZE
;
277 r
= qemu_chr_fe_read_all(chr
, p
, size
);
279 int saved_errno
= errno
;
280 error_report("Failed to read msg header. Read %d instead of %d."
281 " Original request %d.", r
, size
, msg
->hdr
.request
);
282 return r
< 0 ? -saved_errno
: -EIO
;
285 /* validate received flags */
286 if (msg
->hdr
.flags
!= (VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
)) {
287 error_report("Failed to read msg header."
288 " Flags 0x%x instead of 0x%x.", msg
->hdr
.flags
,
289 VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
);
293 trace_vhost_user_read(msg
->hdr
.request
, msg
->hdr
.flags
);
298 static int vhost_user_read(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
300 struct vhost_user
*u
= dev
->opaque
;
301 CharBackend
*chr
= u
->user
->chr
;
302 uint8_t *p
= (uint8_t *) msg
;
305 r
= vhost_user_read_header(dev
, msg
);
310 /* validate message size is sane */
311 if (msg
->hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
312 error_report("Failed to read msg header."
313 " Size %d exceeds the maximum %zu.", msg
->hdr
.size
,
314 VHOST_USER_PAYLOAD_SIZE
);
319 p
+= VHOST_USER_HDR_SIZE
;
320 size
= msg
->hdr
.size
;
321 r
= qemu_chr_fe_read_all(chr
, p
, size
);
323 int saved_errno
= errno
;
324 error_report("Failed to read msg payload."
325 " Read %d instead of %d.", r
, msg
->hdr
.size
);
326 return r
< 0 ? -saved_errno
: -EIO
;
333 static int process_message_reply(struct vhost_dev
*dev
,
334 const VhostUserMsg
*msg
)
337 VhostUserMsg msg_reply
;
339 if ((msg
->hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) == 0) {
343 ret
= vhost_user_read(dev
, &msg_reply
);
348 if (msg_reply
.hdr
.request
!= msg
->hdr
.request
) {
349 error_report("Received unexpected msg type. "
350 "Expected %d received %d",
351 msg
->hdr
.request
, msg_reply
.hdr
.request
);
355 return msg_reply
.payload
.u64
? -EIO
: 0;
358 static bool vhost_user_one_time_request(VhostUserRequest request
)
361 case VHOST_USER_SET_OWNER
:
362 case VHOST_USER_RESET_OWNER
:
363 case VHOST_USER_SET_MEM_TABLE
:
364 case VHOST_USER_GET_QUEUE_NUM
:
365 case VHOST_USER_NET_SET_MTU
:
366 case VHOST_USER_ADD_MEM_REG
:
367 case VHOST_USER_REM_MEM_REG
:
374 /* most non-init callers ignore the error */
375 static int vhost_user_write(struct vhost_dev
*dev
, VhostUserMsg
*msg
,
376 int *fds
, int fd_num
)
378 struct vhost_user
*u
= dev
->opaque
;
379 CharBackend
*chr
= u
->user
->chr
;
380 int ret
, size
= VHOST_USER_HDR_SIZE
+ msg
->hdr
.size
;
383 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
384 * we just need send it once in the first time. For later such
385 * request, we just ignore it.
387 if (vhost_user_one_time_request(msg
->hdr
.request
) && dev
->vq_index
!= 0) {
388 msg
->hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
392 if (qemu_chr_fe_set_msgfds(chr
, fds
, fd_num
) < 0) {
393 error_report("Failed to set msg fds.");
397 ret
= qemu_chr_fe_write_all(chr
, (const uint8_t *) msg
, size
);
399 int saved_errno
= errno
;
400 error_report("Failed to write msg."
401 " Wrote %d instead of %d.", ret
, size
);
402 return ret
< 0 ? -saved_errno
: -EIO
;
405 trace_vhost_user_write(msg
->hdr
.request
, msg
->hdr
.flags
);
410 int vhost_user_gpu_set_socket(struct vhost_dev
*dev
, int fd
)
413 .hdr
.request
= VHOST_USER_GPU_SET_SOCKET
,
414 .hdr
.flags
= VHOST_USER_VERSION
,
417 return vhost_user_write(dev
, &msg
, &fd
, 1);
420 static int vhost_user_set_log_base(struct vhost_dev
*dev
, uint64_t base
,
421 struct vhost_log
*log
)
423 int fds
[VHOST_USER_MAX_RAM_SLOTS
];
425 bool shmfd
= virtio_has_feature(dev
->protocol_features
,
426 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
429 .hdr
.request
= VHOST_USER_SET_LOG_BASE
,
430 .hdr
.flags
= VHOST_USER_VERSION
,
431 .payload
.log
.mmap_size
= log
->size
* sizeof(*(log
->log
)),
432 .payload
.log
.mmap_offset
= 0,
433 .hdr
.size
= sizeof(msg
.payload
.log
),
436 /* Send only once with first queue pair */
437 if (dev
->vq_index
!= 0) {
441 if (shmfd
&& log
->fd
!= -1) {
442 fds
[fd_num
++] = log
->fd
;
445 ret
= vhost_user_write(dev
, &msg
, fds
, fd_num
);
452 ret
= vhost_user_read(dev
, &msg
);
457 if (msg
.hdr
.request
!= VHOST_USER_SET_LOG_BASE
) {
458 error_report("Received unexpected msg type. "
459 "Expected %d received %d",
460 VHOST_USER_SET_LOG_BASE
, msg
.hdr
.request
);
468 static MemoryRegion
*vhost_user_get_mr_data(uint64_t addr
, ram_addr_t
*offset
,
473 assert((uintptr_t)addr
== addr
);
474 mr
= memory_region_from_host((void *)(uintptr_t)addr
, offset
);
475 *fd
= memory_region_get_fd(mr
);
476 *offset
+= mr
->ram_block
->fd_offset
;
481 static void vhost_user_fill_msg_region(VhostUserMemoryRegion
*dst
,
482 struct vhost_memory_region
*src
,
483 uint64_t mmap_offset
)
485 assert(src
!= NULL
&& dst
!= NULL
);
486 dst
->userspace_addr
= src
->userspace_addr
;
487 dst
->memory_size
= src
->memory_size
;
488 dst
->guest_phys_addr
= src
->guest_phys_addr
;
489 dst
->mmap_offset
= mmap_offset
;
492 static int vhost_user_fill_set_mem_table_msg(struct vhost_user
*u
,
493 struct vhost_dev
*dev
,
495 int *fds
, size_t *fd_num
,
496 bool track_ramblocks
)
501 struct vhost_memory_region
*reg
;
502 VhostUserMemoryRegion region_buffer
;
504 msg
->hdr
.request
= VHOST_USER_SET_MEM_TABLE
;
506 for (i
= 0; i
< dev
->mem
->nregions
; ++i
) {
507 reg
= dev
->mem
->regions
+ i
;
509 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
511 if (track_ramblocks
) {
512 assert(*fd_num
< VHOST_MEMORY_BASELINE_NREGIONS
);
513 trace_vhost_user_set_mem_table_withfd(*fd_num
, mr
->name
,
515 reg
->guest_phys_addr
,
518 u
->region_rb_offset
[i
] = offset
;
519 u
->region_rb
[i
] = mr
->ram_block
;
520 } else if (*fd_num
== VHOST_MEMORY_BASELINE_NREGIONS
) {
521 error_report("Failed preparing vhost-user memory table msg");
524 vhost_user_fill_msg_region(®ion_buffer
, reg
, offset
);
525 msg
->payload
.memory
.regions
[*fd_num
] = region_buffer
;
526 fds
[(*fd_num
)++] = fd
;
527 } else if (track_ramblocks
) {
528 u
->region_rb_offset
[i
] = 0;
529 u
->region_rb
[i
] = NULL
;
533 msg
->payload
.memory
.nregions
= *fd_num
;
536 error_report("Failed initializing vhost-user memory map, "
537 "consider using -object memory-backend-file share=on");
541 msg
->hdr
.size
= sizeof(msg
->payload
.memory
.nregions
);
542 msg
->hdr
.size
+= sizeof(msg
->payload
.memory
.padding
);
543 msg
->hdr
.size
+= *fd_num
* sizeof(VhostUserMemoryRegion
);
548 static inline bool reg_equal(struct vhost_memory_region
*shadow_reg
,
549 struct vhost_memory_region
*vdev_reg
)
551 return shadow_reg
->guest_phys_addr
== vdev_reg
->guest_phys_addr
&&
552 shadow_reg
->userspace_addr
== vdev_reg
->userspace_addr
&&
553 shadow_reg
->memory_size
== vdev_reg
->memory_size
;
556 static void scrub_shadow_regions(struct vhost_dev
*dev
,
557 struct scrub_regions
*add_reg
,
559 struct scrub_regions
*rem_reg
,
560 int *nr_rem_reg
, uint64_t *shadow_pcb
,
561 bool track_ramblocks
)
563 struct vhost_user
*u
= dev
->opaque
;
564 bool found
[VHOST_USER_MAX_RAM_SLOTS
] = {};
565 struct vhost_memory_region
*reg
, *shadow_reg
;
566 int i
, j
, fd
, add_idx
= 0, rm_idx
= 0, fd_num
= 0;
572 * Find memory regions present in our shadow state which are not in
573 * the device's current memory state.
575 * Mark regions in both the shadow and device state as "found".
577 for (i
= 0; i
< u
->num_shadow_regions
; i
++) {
578 shadow_reg
= &u
->shadow_regions
[i
];
581 for (j
= 0; j
< dev
->mem
->nregions
; j
++) {
582 reg
= &dev
->mem
->regions
[j
];
584 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
586 if (reg_equal(shadow_reg
, reg
)) {
589 if (track_ramblocks
) {
591 * Reset postcopy client bases, region_rb, and
592 * region_rb_offset in case regions are removed.
595 u
->region_rb_offset
[j
] = offset
;
596 u
->region_rb
[j
] = mr
->ram_block
;
597 shadow_pcb
[j
] = u
->postcopy_client_bases
[i
];
599 u
->region_rb_offset
[j
] = 0;
600 u
->region_rb
[j
] = NULL
;
608 * If the region was not found in the current device memory state
609 * create an entry for it in the removed list.
612 rem_reg
[rm_idx
].region
= shadow_reg
;
613 rem_reg
[rm_idx
++].reg_idx
= i
;
618 * For regions not marked "found", create entries in the added list.
620 * Note their indexes in the device memory state and the indexes of their
623 for (i
= 0; i
< dev
->mem
->nregions
; i
++) {
624 reg
= &dev
->mem
->regions
[i
];
625 vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
631 * If the region was in both the shadow and device state we don't
632 * need to send a VHOST_USER_ADD_MEM_REG message for it.
638 add_reg
[add_idx
].region
= reg
;
639 add_reg
[add_idx
].reg_idx
= i
;
640 add_reg
[add_idx
++].fd_idx
= fd_num
;
642 *nr_rem_reg
= rm_idx
;
643 *nr_add_reg
= add_idx
;
648 static int send_remove_regions(struct vhost_dev
*dev
,
649 struct scrub_regions
*remove_reg
,
650 int nr_rem_reg
, VhostUserMsg
*msg
,
651 bool reply_supported
)
653 struct vhost_user
*u
= dev
->opaque
;
654 struct vhost_memory_region
*shadow_reg
;
655 int i
, fd
, shadow_reg_idx
, ret
;
657 VhostUserMemoryRegion region_buffer
;
660 * The regions in remove_reg appear in the same order they do in the
661 * shadow table. Therefore we can minimize memory copies by iterating
662 * through remove_reg backwards.
664 for (i
= nr_rem_reg
- 1; i
>= 0; i
--) {
665 shadow_reg
= remove_reg
[i
].region
;
666 shadow_reg_idx
= remove_reg
[i
].reg_idx
;
668 vhost_user_get_mr_data(shadow_reg
->userspace_addr
, &offset
, &fd
);
671 msg
->hdr
.request
= VHOST_USER_REM_MEM_REG
;
672 vhost_user_fill_msg_region(®ion_buffer
, shadow_reg
, 0);
673 msg
->payload
.mem_reg
.region
= region_buffer
;
675 ret
= vhost_user_write(dev
, msg
, NULL
, 0);
680 if (reply_supported
) {
681 ret
= process_message_reply(dev
, msg
);
689 * At this point we know the backend has unmapped the region. It is now
690 * safe to remove it from the shadow table.
692 memmove(&u
->shadow_regions
[shadow_reg_idx
],
693 &u
->shadow_regions
[shadow_reg_idx
+ 1],
694 sizeof(struct vhost_memory_region
) *
695 (u
->num_shadow_regions
- shadow_reg_idx
- 1));
696 u
->num_shadow_regions
--;
702 static int send_add_regions(struct vhost_dev
*dev
,
703 struct scrub_regions
*add_reg
, int nr_add_reg
,
704 VhostUserMsg
*msg
, uint64_t *shadow_pcb
,
705 bool reply_supported
, bool track_ramblocks
)
707 struct vhost_user
*u
= dev
->opaque
;
708 int i
, fd
, ret
, reg_idx
, reg_fd_idx
;
709 struct vhost_memory_region
*reg
;
712 VhostUserMsg msg_reply
;
713 VhostUserMemoryRegion region_buffer
;
715 for (i
= 0; i
< nr_add_reg
; i
++) {
716 reg
= add_reg
[i
].region
;
717 reg_idx
= add_reg
[i
].reg_idx
;
718 reg_fd_idx
= add_reg
[i
].fd_idx
;
720 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
723 if (track_ramblocks
) {
724 trace_vhost_user_set_mem_table_withfd(reg_fd_idx
, mr
->name
,
726 reg
->guest_phys_addr
,
729 u
->region_rb_offset
[reg_idx
] = offset
;
730 u
->region_rb
[reg_idx
] = mr
->ram_block
;
732 msg
->hdr
.request
= VHOST_USER_ADD_MEM_REG
;
733 vhost_user_fill_msg_region(®ion_buffer
, reg
, offset
);
734 msg
->payload
.mem_reg
.region
= region_buffer
;
736 ret
= vhost_user_write(dev
, msg
, &fd
, 1);
741 if (track_ramblocks
) {
744 ret
= vhost_user_read(dev
, &msg_reply
);
749 reply_gpa
= msg_reply
.payload
.mem_reg
.region
.guest_phys_addr
;
751 if (msg_reply
.hdr
.request
!= VHOST_USER_ADD_MEM_REG
) {
752 error_report("%s: Received unexpected msg type."
753 "Expected %d received %d", __func__
,
754 VHOST_USER_ADD_MEM_REG
,
755 msg_reply
.hdr
.request
);
760 * We're using the same structure, just reusing one of the
761 * fields, so it should be the same size.
763 if (msg_reply
.hdr
.size
!= msg
->hdr
.size
) {
764 error_report("%s: Unexpected size for postcopy reply "
765 "%d vs %d", __func__
, msg_reply
.hdr
.size
,
770 /* Get the postcopy client base from the backend's reply. */
771 if (reply_gpa
== dev
->mem
->regions
[reg_idx
].guest_phys_addr
) {
772 shadow_pcb
[reg_idx
] =
773 msg_reply
.payload
.mem_reg
.region
.userspace_addr
;
774 trace_vhost_user_set_mem_table_postcopy(
775 msg_reply
.payload
.mem_reg
.region
.userspace_addr
,
776 msg
->payload
.mem_reg
.region
.userspace_addr
,
777 reg_fd_idx
, reg_idx
);
779 error_report("%s: invalid postcopy reply for region. "
780 "Got guest physical address %" PRIX64
", expected "
781 "%" PRIX64
, __func__
, reply_gpa
,
782 dev
->mem
->regions
[reg_idx
].guest_phys_addr
);
785 } else if (reply_supported
) {
786 ret
= process_message_reply(dev
, msg
);
791 } else if (track_ramblocks
) {
792 u
->region_rb_offset
[reg_idx
] = 0;
793 u
->region_rb
[reg_idx
] = NULL
;
797 * At this point, we know the backend has mapped in the new
798 * region, if the region has a valid file descriptor.
800 * The region should now be added to the shadow table.
802 u
->shadow_regions
[u
->num_shadow_regions
].guest_phys_addr
=
803 reg
->guest_phys_addr
;
804 u
->shadow_regions
[u
->num_shadow_regions
].userspace_addr
=
806 u
->shadow_regions
[u
->num_shadow_regions
].memory_size
=
808 u
->num_shadow_regions
++;
814 static int vhost_user_add_remove_regions(struct vhost_dev
*dev
,
816 bool reply_supported
,
817 bool track_ramblocks
)
819 struct vhost_user
*u
= dev
->opaque
;
820 struct scrub_regions add_reg
[VHOST_USER_MAX_RAM_SLOTS
];
821 struct scrub_regions rem_reg
[VHOST_USER_MAX_RAM_SLOTS
];
822 uint64_t shadow_pcb
[VHOST_USER_MAX_RAM_SLOTS
] = {};
823 int nr_add_reg
, nr_rem_reg
;
826 msg
->hdr
.size
= sizeof(msg
->payload
.mem_reg
);
828 /* Find the regions which need to be removed or added. */
829 scrub_shadow_regions(dev
, add_reg
, &nr_add_reg
, rem_reg
, &nr_rem_reg
,
830 shadow_pcb
, track_ramblocks
);
833 ret
= send_remove_regions(dev
, rem_reg
, nr_rem_reg
, msg
,
841 ret
= send_add_regions(dev
, add_reg
, nr_add_reg
, msg
, shadow_pcb
,
842 reply_supported
, track_ramblocks
);
848 if (track_ramblocks
) {
849 memcpy(u
->postcopy_client_bases
, shadow_pcb
,
850 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
852 * Now we've registered this with the postcopy code, we ack to the
853 * client, because now we're in the position to be able to deal with
854 * any faults it generates.
856 /* TODO: Use this for failure cases as well with a bad value. */
857 msg
->hdr
.size
= sizeof(msg
->payload
.u64
);
858 msg
->payload
.u64
= 0; /* OK */
860 ret
= vhost_user_write(dev
, msg
, NULL
, 0);
869 if (track_ramblocks
) {
870 memcpy(u
->postcopy_client_bases
, shadow_pcb
,
871 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
877 static int vhost_user_set_mem_table_postcopy(struct vhost_dev
*dev
,
878 struct vhost_memory
*mem
,
879 bool reply_supported
,
880 bool config_mem_slots
)
882 struct vhost_user
*u
= dev
->opaque
;
883 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
885 VhostUserMsg msg_reply
;
890 .hdr
.flags
= VHOST_USER_VERSION
,
893 if (u
->region_rb_len
< dev
->mem
->nregions
) {
894 u
->region_rb
= g_renew(RAMBlock
*, u
->region_rb
, dev
->mem
->nregions
);
895 u
->region_rb_offset
= g_renew(ram_addr_t
, u
->region_rb_offset
,
897 memset(&(u
->region_rb
[u
->region_rb_len
]), '\0',
898 sizeof(RAMBlock
*) * (dev
->mem
->nregions
- u
->region_rb_len
));
899 memset(&(u
->region_rb_offset
[u
->region_rb_len
]), '\0',
900 sizeof(ram_addr_t
) * (dev
->mem
->nregions
- u
->region_rb_len
));
901 u
->region_rb_len
= dev
->mem
->nregions
;
904 if (config_mem_slots
) {
905 ret
= vhost_user_add_remove_regions(dev
, &msg
, reply_supported
, true);
910 ret
= vhost_user_fill_set_mem_table_msg(u
, dev
, &msg
, fds
, &fd_num
,
916 ret
= vhost_user_write(dev
, &msg
, fds
, fd_num
);
921 ret
= vhost_user_read(dev
, &msg_reply
);
926 if (msg_reply
.hdr
.request
!= VHOST_USER_SET_MEM_TABLE
) {
927 error_report("%s: Received unexpected msg type."
928 "Expected %d received %d", __func__
,
929 VHOST_USER_SET_MEM_TABLE
, msg_reply
.hdr
.request
);
934 * We're using the same structure, just reusing one of the
935 * fields, so it should be the same size.
937 if (msg_reply
.hdr
.size
!= msg
.hdr
.size
) {
938 error_report("%s: Unexpected size for postcopy reply "
939 "%d vs %d", __func__
, msg_reply
.hdr
.size
,
944 memset(u
->postcopy_client_bases
, 0,
945 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
948 * They're in the same order as the regions that were sent
949 * but some of the regions were skipped (above) if they
952 for (msg_i
= 0, region_i
= 0;
953 region_i
< dev
->mem
->nregions
;
955 if (msg_i
< fd_num
&&
956 msg_reply
.payload
.memory
.regions
[msg_i
].guest_phys_addr
==
957 dev
->mem
->regions
[region_i
].guest_phys_addr
) {
958 u
->postcopy_client_bases
[region_i
] =
959 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
;
960 trace_vhost_user_set_mem_table_postcopy(
961 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
,
962 msg
.payload
.memory
.regions
[msg_i
].userspace_addr
,
967 if (msg_i
!= fd_num
) {
968 error_report("%s: postcopy reply not fully consumed "
970 __func__
, msg_i
, fd_num
);
975 * Now we've registered this with the postcopy code, we ack to the
976 * client, because now we're in the position to be able to deal
977 * with any faults it generates.
979 /* TODO: Use this for failure cases as well with a bad value. */
980 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
981 msg
.payload
.u64
= 0; /* OK */
982 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
991 static int vhost_user_set_mem_table(struct vhost_dev
*dev
,
992 struct vhost_memory
*mem
)
994 struct vhost_user
*u
= dev
->opaque
;
995 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
997 bool do_postcopy
= u
->postcopy_listen
&& u
->postcopy_fd
.handler
;
998 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
999 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1000 bool config_mem_slots
=
1001 virtio_has_feature(dev
->protocol_features
,
1002 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
);
1007 * Postcopy has enough differences that it's best done in it's own
1010 return vhost_user_set_mem_table_postcopy(dev
, mem
, reply_supported
,
1014 VhostUserMsg msg
= {
1015 .hdr
.flags
= VHOST_USER_VERSION
,
1018 if (reply_supported
) {
1019 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1022 if (config_mem_slots
) {
1023 ret
= vhost_user_add_remove_regions(dev
, &msg
, reply_supported
, false);
1028 ret
= vhost_user_fill_set_mem_table_msg(u
, dev
, &msg
, fds
, &fd_num
,
1034 ret
= vhost_user_write(dev
, &msg
, fds
, fd_num
);
1039 if (reply_supported
) {
1040 return process_message_reply(dev
, &msg
);
1047 static int vhost_user_set_vring_endian(struct vhost_dev
*dev
,
1048 struct vhost_vring_state
*ring
)
1050 bool cross_endian
= virtio_has_feature(dev
->protocol_features
,
1051 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
);
1052 VhostUserMsg msg
= {
1053 .hdr
.request
= VHOST_USER_SET_VRING_ENDIAN
,
1054 .hdr
.flags
= VHOST_USER_VERSION
,
1055 .payload
.state
= *ring
,
1056 .hdr
.size
= sizeof(msg
.payload
.state
),
1059 if (!cross_endian
) {
1060 error_report("vhost-user trying to send unhandled ioctl");
1064 return vhost_user_write(dev
, &msg
, NULL
, 0);
1067 static int vhost_set_vring(struct vhost_dev
*dev
,
1068 unsigned long int request
,
1069 struct vhost_vring_state
*ring
)
1071 VhostUserMsg msg
= {
1072 .hdr
.request
= request
,
1073 .hdr
.flags
= VHOST_USER_VERSION
,
1074 .payload
.state
= *ring
,
1075 .hdr
.size
= sizeof(msg
.payload
.state
),
1078 return vhost_user_write(dev
, &msg
, NULL
, 0);
1081 static int vhost_user_set_vring_num(struct vhost_dev
*dev
,
1082 struct vhost_vring_state
*ring
)
1084 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_NUM
, ring
);
1087 static void vhost_user_host_notifier_free(VhostUserHostNotifier
*n
)
1089 assert(n
&& n
->unmap_addr
);
1090 munmap(n
->unmap_addr
, qemu_real_host_page_size());
1091 n
->unmap_addr
= NULL
;
1095 * clean-up function for notifier, will finally free the structure
1098 static void vhost_user_host_notifier_remove(VhostUserHostNotifier
*n
,
1103 virtio_queue_set_host_notifier_mr(vdev
, n
->idx
, &n
->mr
, false);
1105 assert(!n
->unmap_addr
);
1106 n
->unmap_addr
= n
->addr
;
1108 call_rcu(n
, vhost_user_host_notifier_free
, rcu
);
1112 static int vhost_user_set_vring_base(struct vhost_dev
*dev
,
1113 struct vhost_vring_state
*ring
)
1115 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_BASE
, ring
);
1118 static int vhost_user_set_vring_enable(struct vhost_dev
*dev
, int enable
)
1122 if (!virtio_has_feature(dev
->features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
1126 for (i
= 0; i
< dev
->nvqs
; ++i
) {
1128 struct vhost_vring_state state
= {
1129 .index
= dev
->vq_index
+ i
,
1133 ret
= vhost_set_vring(dev
, VHOST_USER_SET_VRING_ENABLE
, &state
);
1136 * Restoring the previous state is likely infeasible, as well as
1137 * proceeding regardless the error, so just bail out and hope for
1138 * the device-level recovery.
1147 static VhostUserHostNotifier
*fetch_notifier(VhostUserState
*u
,
1150 if (idx
>= u
->notifiers
->len
) {
1153 return g_ptr_array_index(u
->notifiers
, idx
);
1156 static int vhost_user_get_vring_base(struct vhost_dev
*dev
,
1157 struct vhost_vring_state
*ring
)
1160 VhostUserMsg msg
= {
1161 .hdr
.request
= VHOST_USER_GET_VRING_BASE
,
1162 .hdr
.flags
= VHOST_USER_VERSION
,
1163 .payload
.state
= *ring
,
1164 .hdr
.size
= sizeof(msg
.payload
.state
),
1166 struct vhost_user
*u
= dev
->opaque
;
1168 VhostUserHostNotifier
*n
= fetch_notifier(u
->user
, ring
->index
);
1170 vhost_user_host_notifier_remove(n
, dev
->vdev
);
1173 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1178 ret
= vhost_user_read(dev
, &msg
);
1183 if (msg
.hdr
.request
!= VHOST_USER_GET_VRING_BASE
) {
1184 error_report("Received unexpected msg type. Expected %d received %d",
1185 VHOST_USER_GET_VRING_BASE
, msg
.hdr
.request
);
1189 if (msg
.hdr
.size
!= sizeof(msg
.payload
.state
)) {
1190 error_report("Received bad msg size.");
1194 *ring
= msg
.payload
.state
;
1199 static int vhost_set_vring_file(struct vhost_dev
*dev
,
1200 VhostUserRequest request
,
1201 struct vhost_vring_file
*file
)
1203 int fds
[VHOST_USER_MAX_RAM_SLOTS
];
1205 VhostUserMsg msg
= {
1206 .hdr
.request
= request
,
1207 .hdr
.flags
= VHOST_USER_VERSION
,
1208 .payload
.u64
= file
->index
& VHOST_USER_VRING_IDX_MASK
,
1209 .hdr
.size
= sizeof(msg
.payload
.u64
),
1212 if (ioeventfd_enabled() && file
->fd
> 0) {
1213 fds
[fd_num
++] = file
->fd
;
1215 msg
.payload
.u64
|= VHOST_USER_VRING_NOFD_MASK
;
1218 return vhost_user_write(dev
, &msg
, fds
, fd_num
);
1221 static int vhost_user_set_vring_kick(struct vhost_dev
*dev
,
1222 struct vhost_vring_file
*file
)
1224 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_KICK
, file
);
1227 static int vhost_user_set_vring_call(struct vhost_dev
*dev
,
1228 struct vhost_vring_file
*file
)
1230 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_CALL
, file
);
1233 static int vhost_user_set_vring_err(struct vhost_dev
*dev
,
1234 struct vhost_vring_file
*file
)
1236 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_ERR
, file
);
1239 static int vhost_user_get_u64(struct vhost_dev
*dev
, int request
, uint64_t *u64
)
1242 VhostUserMsg msg
= {
1243 .hdr
.request
= request
,
1244 .hdr
.flags
= VHOST_USER_VERSION
,
1247 if (vhost_user_one_time_request(request
) && dev
->vq_index
!= 0) {
1251 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1256 ret
= vhost_user_read(dev
, &msg
);
1261 if (msg
.hdr
.request
!= request
) {
1262 error_report("Received unexpected msg type. Expected %d received %d",
1263 request
, msg
.hdr
.request
);
1267 if (msg
.hdr
.size
!= sizeof(msg
.payload
.u64
)) {
1268 error_report("Received bad msg size.");
1272 *u64
= msg
.payload
.u64
;
1277 static int vhost_user_get_features(struct vhost_dev
*dev
, uint64_t *features
)
1279 if (vhost_user_get_u64(dev
, VHOST_USER_GET_FEATURES
, features
) < 0) {
1286 static int enforce_reply(struct vhost_dev
*dev
,
1287 const VhostUserMsg
*msg
)
1291 if (msg
->hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) {
1292 return process_message_reply(dev
, msg
);
1296 * We need to wait for a reply but the backend does not
1297 * support replies for the command we just sent.
1298 * Send VHOST_USER_GET_FEATURES which makes all backends
1301 return vhost_user_get_features(dev
, &dummy
);
1304 static int vhost_user_set_vring_addr(struct vhost_dev
*dev
,
1305 struct vhost_vring_addr
*addr
)
1308 VhostUserMsg msg
= {
1309 .hdr
.request
= VHOST_USER_SET_VRING_ADDR
,
1310 .hdr
.flags
= VHOST_USER_VERSION
,
1311 .payload
.addr
= *addr
,
1312 .hdr
.size
= sizeof(msg
.payload
.addr
),
1315 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1316 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1319 * wait for a reply if logging is enabled to make sure
1320 * backend is actually logging changes
1322 bool wait_for_reply
= addr
->flags
& (1 << VHOST_VRING_F_LOG
);
1324 if (reply_supported
&& wait_for_reply
) {
1325 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1328 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1333 if (wait_for_reply
) {
1334 return enforce_reply(dev
, &msg
);
1340 static int vhost_user_set_u64(struct vhost_dev
*dev
, int request
, uint64_t u64
,
1341 bool wait_for_reply
)
1343 VhostUserMsg msg
= {
1344 .hdr
.request
= request
,
1345 .hdr
.flags
= VHOST_USER_VERSION
,
1347 .hdr
.size
= sizeof(msg
.payload
.u64
),
1351 if (wait_for_reply
) {
1352 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1353 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1354 if (reply_supported
) {
1355 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1359 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1364 if (wait_for_reply
) {
1365 return enforce_reply(dev
, &msg
);
1371 static int vhost_user_set_status(struct vhost_dev
*dev
, uint8_t status
)
1373 return vhost_user_set_u64(dev
, VHOST_USER_SET_STATUS
, status
, false);
1376 static int vhost_user_get_status(struct vhost_dev
*dev
, uint8_t *status
)
1381 ret
= vhost_user_get_u64(dev
, VHOST_USER_GET_STATUS
, &value
);
1390 static int vhost_user_add_status(struct vhost_dev
*dev
, uint8_t status
)
1395 ret
= vhost_user_get_status(dev
, &s
);
1400 if ((s
& status
) == status
) {
1405 return vhost_user_set_status(dev
, s
);
1408 static int vhost_user_set_features(struct vhost_dev
*dev
,
1412 * wait for a reply if logging is enabled to make sure
1413 * backend is actually logging changes
1415 bool log_enabled
= features
& (0x1ULL
<< VHOST_F_LOG_ALL
);
1419 * We need to include any extra backend only feature bits that
1420 * might be needed by our device. Currently this includes the
1421 * VHOST_USER_F_PROTOCOL_FEATURES bit for enabling protocol
1424 ret
= vhost_user_set_u64(dev
, VHOST_USER_SET_FEATURES
,
1425 features
| dev
->backend_features
,
1428 if (virtio_has_feature(dev
->protocol_features
,
1429 VHOST_USER_PROTOCOL_F_STATUS
)) {
1431 return vhost_user_add_status(dev
, VIRTIO_CONFIG_S_FEATURES_OK
);
1438 static int vhost_user_set_protocol_features(struct vhost_dev
*dev
,
1441 return vhost_user_set_u64(dev
, VHOST_USER_SET_PROTOCOL_FEATURES
, features
,
1445 static int vhost_user_set_owner(struct vhost_dev
*dev
)
1447 VhostUserMsg msg
= {
1448 .hdr
.request
= VHOST_USER_SET_OWNER
,
1449 .hdr
.flags
= VHOST_USER_VERSION
,
1452 return vhost_user_write(dev
, &msg
, NULL
, 0);
1455 static int vhost_user_get_max_memslots(struct vhost_dev
*dev
,
1456 uint64_t *max_memslots
)
1458 uint64_t backend_max_memslots
;
1461 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_MAX_MEM_SLOTS
,
1462 &backend_max_memslots
);
1467 *max_memslots
= backend_max_memslots
;
1472 static int vhost_user_reset_device(struct vhost_dev
*dev
)
1474 VhostUserMsg msg
= {
1475 .hdr
.flags
= VHOST_USER_VERSION
,
1478 msg
.hdr
.request
= virtio_has_feature(dev
->protocol_features
,
1479 VHOST_USER_PROTOCOL_F_RESET_DEVICE
)
1480 ? VHOST_USER_RESET_DEVICE
1481 : VHOST_USER_RESET_OWNER
;
1483 return vhost_user_write(dev
, &msg
, NULL
, 0);
1486 static int vhost_user_slave_handle_config_change(struct vhost_dev
*dev
)
1488 if (!dev
->config_ops
|| !dev
->config_ops
->vhost_dev_config_notifier
) {
1492 return dev
->config_ops
->vhost_dev_config_notifier(dev
);
1496 * Fetch or create the notifier for a given idx. Newly created
1497 * notifiers are added to the pointer array that tracks them.
1499 static VhostUserHostNotifier
*fetch_or_create_notifier(VhostUserState
*u
,
1502 VhostUserHostNotifier
*n
= NULL
;
1503 if (idx
>= u
->notifiers
->len
) {
1504 g_ptr_array_set_size(u
->notifiers
, idx
+ 1);
1507 n
= g_ptr_array_index(u
->notifiers
, idx
);
1510 * In case notification arrive out-of-order,
1511 * make room for current index.
1513 g_ptr_array_remove_index(u
->notifiers
, idx
);
1514 n
= g_new0(VhostUserHostNotifier
, 1);
1516 g_ptr_array_insert(u
->notifiers
, idx
, n
);
1517 trace_vhost_user_create_notifier(idx
, n
);
1523 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev
*dev
,
1524 VhostUserVringArea
*area
,
1527 int queue_idx
= area
->u64
& VHOST_USER_VRING_IDX_MASK
;
1528 size_t page_size
= qemu_real_host_page_size();
1529 struct vhost_user
*u
= dev
->opaque
;
1530 VhostUserState
*user
= u
->user
;
1531 VirtIODevice
*vdev
= dev
->vdev
;
1532 VhostUserHostNotifier
*n
;
1536 if (!virtio_has_feature(dev
->protocol_features
,
1537 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
) ||
1538 vdev
== NULL
|| queue_idx
>= virtio_get_num_queues(vdev
)) {
1543 * Fetch notifier and invalidate any old data before setting up
1544 * new mapped address.
1546 n
= fetch_or_create_notifier(user
, queue_idx
);
1547 vhost_user_host_notifier_remove(n
, vdev
);
1549 if (area
->u64
& VHOST_USER_VRING_NOFD_MASK
) {
1554 if (area
->size
!= page_size
) {
1558 addr
= mmap(NULL
, page_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
1560 if (addr
== MAP_FAILED
) {
1564 name
= g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1566 if (!n
->mr
.ram
) { /* Don't init again after suspend. */
1567 memory_region_init_ram_device_ptr(&n
->mr
, OBJECT(vdev
), name
,
1570 n
->mr
.ram_block
->host
= addr
;
1574 if (virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, true)) {
1575 object_unparent(OBJECT(&n
->mr
));
1576 munmap(addr
, page_size
);
1585 static void close_slave_channel(struct vhost_user
*u
)
1587 g_source_destroy(u
->slave_src
);
1588 g_source_unref(u
->slave_src
);
1589 u
->slave_src
= NULL
;
1590 object_unref(OBJECT(u
->slave_ioc
));
1591 u
->slave_ioc
= NULL
;
1594 static gboolean
slave_read(QIOChannel
*ioc
, GIOCondition condition
,
1597 struct vhost_dev
*dev
= opaque
;
1598 struct vhost_user
*u
= dev
->opaque
;
1599 VhostUserHeader hdr
= { 0, };
1600 VhostUserPayload payload
= { 0, };
1601 Error
*local_err
= NULL
;
1602 gboolean rc
= G_SOURCE_CONTINUE
;
1605 g_autofree
int *fd
= NULL
;
1610 iov
.iov_base
= &hdr
;
1611 iov
.iov_len
= VHOST_USER_HDR_SIZE
;
1613 if (qio_channel_readv_full_all(ioc
, &iov
, 1, &fd
, &fdsize
, &local_err
)) {
1614 error_report_err(local_err
);
1618 if (hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
1619 error_report("Failed to read msg header."
1620 " Size %d exceeds the maximum %zu.", hdr
.size
,
1621 VHOST_USER_PAYLOAD_SIZE
);
1626 if (qio_channel_read_all(ioc
, (char *) &payload
, hdr
.size
, &local_err
)) {
1627 error_report_err(local_err
);
1631 switch (hdr
.request
) {
1632 case VHOST_USER_BACKEND_IOTLB_MSG
:
1633 ret
= vhost_backend_handle_iotlb_msg(dev
, &payload
.iotlb
);
1635 case VHOST_USER_BACKEND_CONFIG_CHANGE_MSG
:
1636 ret
= vhost_user_slave_handle_config_change(dev
);
1638 case VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG
:
1639 ret
= vhost_user_slave_handle_vring_host_notifier(dev
, &payload
.area
,
1643 error_report("Received unexpected msg type: %d.", hdr
.request
);
1648 * REPLY_ACK feature handling. Other reply types has to be managed
1649 * directly in their request handlers.
1651 if (hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) {
1652 struct iovec iovec
[2];
1655 hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
1656 hdr
.flags
|= VHOST_USER_REPLY_MASK
;
1658 payload
.u64
= !!ret
;
1659 hdr
.size
= sizeof(payload
.u64
);
1661 iovec
[0].iov_base
= &hdr
;
1662 iovec
[0].iov_len
= VHOST_USER_HDR_SIZE
;
1663 iovec
[1].iov_base
= &payload
;
1664 iovec
[1].iov_len
= hdr
.size
;
1666 if (qio_channel_writev_all(ioc
, iovec
, ARRAY_SIZE(iovec
), &local_err
)) {
1667 error_report_err(local_err
);
1675 close_slave_channel(u
);
1676 rc
= G_SOURCE_REMOVE
;
1680 for (i
= 0; i
< fdsize
; i
++) {
1687 static int vhost_setup_slave_channel(struct vhost_dev
*dev
)
1689 VhostUserMsg msg
= {
1690 .hdr
.request
= VHOST_USER_SET_BACKEND_REQ_FD
,
1691 .hdr
.flags
= VHOST_USER_VERSION
,
1693 struct vhost_user
*u
= dev
->opaque
;
1695 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1696 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1697 Error
*local_err
= NULL
;
1700 if (!virtio_has_feature(dev
->protocol_features
,
1701 VHOST_USER_PROTOCOL_F_BACKEND_REQ
)) {
1705 if (qemu_socketpair(PF_UNIX
, SOCK_STREAM
, 0, sv
) == -1) {
1706 int saved_errno
= errno
;
1707 error_report("socketpair() failed");
1708 return -saved_errno
;
1711 ioc
= QIO_CHANNEL(qio_channel_socket_new_fd(sv
[0], &local_err
));
1713 error_report_err(local_err
);
1714 return -ECONNREFUSED
;
1717 u
->slave_src
= qio_channel_add_watch_source(u
->slave_ioc
,
1719 slave_read
, dev
, NULL
, NULL
);
1721 if (reply_supported
) {
1722 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1725 ret
= vhost_user_write(dev
, &msg
, &sv
[1], 1);
1730 if (reply_supported
) {
1731 ret
= process_message_reply(dev
, &msg
);
1737 close_slave_channel(u
);
1745 * Called back from the postcopy fault thread when a fault is received on our
1747 * TODO: This is Linux specific
1749 static int vhost_user_postcopy_fault_handler(struct PostCopyFD
*pcfd
,
1752 struct vhost_dev
*dev
= pcfd
->data
;
1753 struct vhost_user
*u
= dev
->opaque
;
1754 struct uffd_msg
*msg
= ufd
;
1755 uint64_t faultaddr
= msg
->arg
.pagefault
.address
;
1756 RAMBlock
*rb
= NULL
;
1760 trace_vhost_user_postcopy_fault_handler(pcfd
->idstr
, faultaddr
,
1761 dev
->mem
->nregions
);
1762 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1763 trace_vhost_user_postcopy_fault_handler_loop(i
,
1764 u
->postcopy_client_bases
[i
], dev
->mem
->regions
[i
].memory_size
);
1765 if (faultaddr
>= u
->postcopy_client_bases
[i
]) {
1766 /* Ofset of the fault address in the vhost region */
1767 uint64_t region_offset
= faultaddr
- u
->postcopy_client_bases
[i
];
1768 if (region_offset
< dev
->mem
->regions
[i
].memory_size
) {
1769 rb_offset
= region_offset
+ u
->region_rb_offset
[i
];
1770 trace_vhost_user_postcopy_fault_handler_found(i
,
1771 region_offset
, rb_offset
);
1772 rb
= u
->region_rb
[i
];
1773 return postcopy_request_shared_page(pcfd
, rb
, faultaddr
,
1778 error_report("%s: Failed to find region for fault %" PRIx64
,
1779 __func__
, faultaddr
);
1783 static int vhost_user_postcopy_waker(struct PostCopyFD
*pcfd
, RAMBlock
*rb
,
1786 struct vhost_dev
*dev
= pcfd
->data
;
1787 struct vhost_user
*u
= dev
->opaque
;
1790 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb
), offset
);
1795 /* Translate the offset into an address in the clients address space */
1796 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1797 if (u
->region_rb
[i
] == rb
&&
1798 offset
>= u
->region_rb_offset
[i
] &&
1799 offset
< (u
->region_rb_offset
[i
] +
1800 dev
->mem
->regions
[i
].memory_size
)) {
1801 uint64_t client_addr
= (offset
- u
->region_rb_offset
[i
]) +
1802 u
->postcopy_client_bases
[i
];
1803 trace_vhost_user_postcopy_waker_found(client_addr
);
1804 return postcopy_wake_shared(pcfd
, client_addr
, rb
);
1808 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb
), offset
);
1814 * Called at the start of an inbound postcopy on reception of the
1817 static int vhost_user_postcopy_advise(struct vhost_dev
*dev
, Error
**errp
)
1820 struct vhost_user
*u
= dev
->opaque
;
1821 CharBackend
*chr
= u
->user
->chr
;
1824 VhostUserMsg msg
= {
1825 .hdr
.request
= VHOST_USER_POSTCOPY_ADVISE
,
1826 .hdr
.flags
= VHOST_USER_VERSION
,
1829 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1831 error_setg(errp
, "Failed to send postcopy_advise to vhost");
1835 ret
= vhost_user_read(dev
, &msg
);
1837 error_setg(errp
, "Failed to get postcopy_advise reply from vhost");
1841 if (msg
.hdr
.request
!= VHOST_USER_POSTCOPY_ADVISE
) {
1842 error_setg(errp
, "Unexpected msg type. Expected %d received %d",
1843 VHOST_USER_POSTCOPY_ADVISE
, msg
.hdr
.request
);
1848 error_setg(errp
, "Received bad msg size.");
1851 ufd
= qemu_chr_fe_get_msgfd(chr
);
1853 error_setg(errp
, "%s: Failed to get ufd", __func__
);
1856 qemu_socket_set_nonblock(ufd
);
1858 /* register ufd with userfault thread */
1859 u
->postcopy_fd
.fd
= ufd
;
1860 u
->postcopy_fd
.data
= dev
;
1861 u
->postcopy_fd
.handler
= vhost_user_postcopy_fault_handler
;
1862 u
->postcopy_fd
.waker
= vhost_user_postcopy_waker
;
1863 u
->postcopy_fd
.idstr
= "vhost-user"; /* Need to find unique name */
1864 postcopy_register_shared_ufd(&u
->postcopy_fd
);
1867 error_setg(errp
, "Postcopy not supported on non-Linux systems");
1873 * Called at the switch to postcopy on reception of the 'listen' command.
1875 static int vhost_user_postcopy_listen(struct vhost_dev
*dev
, Error
**errp
)
1877 struct vhost_user
*u
= dev
->opaque
;
1879 VhostUserMsg msg
= {
1880 .hdr
.request
= VHOST_USER_POSTCOPY_LISTEN
,
1881 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1883 u
->postcopy_listen
= true;
1885 trace_vhost_user_postcopy_listen();
1887 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1889 error_setg(errp
, "Failed to send postcopy_listen to vhost");
1893 ret
= process_message_reply(dev
, &msg
);
1895 error_setg(errp
, "Failed to receive reply to postcopy_listen");
1903 * Called at the end of postcopy
1905 static int vhost_user_postcopy_end(struct vhost_dev
*dev
, Error
**errp
)
1907 VhostUserMsg msg
= {
1908 .hdr
.request
= VHOST_USER_POSTCOPY_END
,
1909 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1912 struct vhost_user
*u
= dev
->opaque
;
1914 trace_vhost_user_postcopy_end_entry();
1916 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1918 error_setg(errp
, "Failed to send postcopy_end to vhost");
1922 ret
= process_message_reply(dev
, &msg
);
1924 error_setg(errp
, "Failed to receive reply to postcopy_end");
1927 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
1928 close(u
->postcopy_fd
.fd
);
1929 u
->postcopy_fd
.handler
= NULL
;
1931 trace_vhost_user_postcopy_end_exit();
1936 static int vhost_user_postcopy_notifier(NotifierWithReturn
*notifier
,
1939 struct PostcopyNotifyData
*pnd
= opaque
;
1940 struct vhost_user
*u
= container_of(notifier
, struct vhost_user
,
1942 struct vhost_dev
*dev
= u
->dev
;
1944 switch (pnd
->reason
) {
1945 case POSTCOPY_NOTIFY_PROBE
:
1946 if (!virtio_has_feature(dev
->protocol_features
,
1947 VHOST_USER_PROTOCOL_F_PAGEFAULT
)) {
1948 /* TODO: Get the device name into this error somehow */
1949 error_setg(pnd
->errp
,
1950 "vhost-user backend not capable of postcopy");
1955 case POSTCOPY_NOTIFY_INBOUND_ADVISE
:
1956 return vhost_user_postcopy_advise(dev
, pnd
->errp
);
1958 case POSTCOPY_NOTIFY_INBOUND_LISTEN
:
1959 return vhost_user_postcopy_listen(dev
, pnd
->errp
);
1961 case POSTCOPY_NOTIFY_INBOUND_END
:
1962 return vhost_user_postcopy_end(dev
, pnd
->errp
);
1965 /* We ignore notifications we don't know */
1972 static int vhost_user_backend_init(struct vhost_dev
*dev
, void *opaque
,
1975 uint64_t features
, ram_slots
;
1976 struct vhost_user
*u
;
1977 VhostUserState
*vus
= (VhostUserState
*) opaque
;
1980 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1982 u
= g_new0(struct vhost_user
, 1);
1987 err
= vhost_user_get_features(dev
, &features
);
1989 error_setg_errno(errp
, -err
, "vhost_backend_init failed");
1993 if (virtio_has_feature(features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
1994 bool supports_f_config
= vus
->supports_config
||
1995 (dev
->config_ops
&& dev
->config_ops
->vhost_dev_config_notifier
);
1996 uint64_t protocol_features
;
1998 dev
->backend_features
|= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES
;
2000 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_PROTOCOL_FEATURES
,
2001 &protocol_features
);
2003 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2008 * We will use all the protocol features we support - although
2009 * we suppress F_CONFIG if we know QEMUs internal code can not support
2012 protocol_features
&= VHOST_USER_PROTOCOL_FEATURE_MASK
;
2014 if (supports_f_config
) {
2015 if (!virtio_has_feature(protocol_features
,
2016 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2017 error_setg(errp
, "vhost-user device expecting "
2018 "VHOST_USER_PROTOCOL_F_CONFIG but the vhost-user backend does "
2023 if (virtio_has_feature(protocol_features
,
2024 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2025 warn_report("vhost-user backend supports "
2026 "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not.");
2027 protocol_features
&= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG
);
2031 /* final set of protocol features */
2032 dev
->protocol_features
= protocol_features
;
2033 err
= vhost_user_set_protocol_features(dev
, dev
->protocol_features
);
2035 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2039 /* query the max queues we support if backend supports Multiple Queue */
2040 if (dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_MQ
)) {
2041 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_QUEUE_NUM
,
2044 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2048 dev
->max_queues
= 1;
2051 if (dev
->num_queues
&& dev
->max_queues
< dev
->num_queues
) {
2052 error_setg(errp
, "The maximum number of queues supported by the "
2053 "backend is %" PRIu64
, dev
->max_queues
);
2057 if (virtio_has_feature(features
, VIRTIO_F_IOMMU_PLATFORM
) &&
2058 !(virtio_has_feature(dev
->protocol_features
,
2059 VHOST_USER_PROTOCOL_F_BACKEND_REQ
) &&
2060 virtio_has_feature(dev
->protocol_features
,
2061 VHOST_USER_PROTOCOL_F_REPLY_ACK
))) {
2062 error_setg(errp
, "IOMMU support requires reply-ack and "
2063 "slave-req protocol features.");
2067 /* get max memory regions if backend supports configurable RAM slots */
2068 if (!virtio_has_feature(dev
->protocol_features
,
2069 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
)) {
2070 u
->user
->memory_slots
= VHOST_MEMORY_BASELINE_NREGIONS
;
2072 err
= vhost_user_get_max_memslots(dev
, &ram_slots
);
2074 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2078 if (ram_slots
< u
->user
->memory_slots
) {
2079 error_setg(errp
, "The backend specified a max ram slots limit "
2080 "of %" PRIu64
", when the prior validated limit was "
2081 "%d. This limit should never decrease.", ram_slots
,
2082 u
->user
->memory_slots
);
2086 u
->user
->memory_slots
= MIN(ram_slots
, VHOST_USER_MAX_RAM_SLOTS
);
2090 if (dev
->migration_blocker
== NULL
&&
2091 !virtio_has_feature(dev
->protocol_features
,
2092 VHOST_USER_PROTOCOL_F_LOG_SHMFD
)) {
2093 error_setg(&dev
->migration_blocker
,
2094 "Migration disabled: vhost-user backend lacks "
2095 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
2098 if (dev
->vq_index
== 0) {
2099 err
= vhost_setup_slave_channel(dev
);
2101 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2106 u
->postcopy_notifier
.notify
= vhost_user_postcopy_notifier
;
2107 postcopy_add_notifier(&u
->postcopy_notifier
);
2112 static int vhost_user_backend_cleanup(struct vhost_dev
*dev
)
2114 struct vhost_user
*u
;
2116 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2119 if (u
->postcopy_notifier
.notify
) {
2120 postcopy_remove_notifier(&u
->postcopy_notifier
);
2121 u
->postcopy_notifier
.notify
= NULL
;
2123 u
->postcopy_listen
= false;
2124 if (u
->postcopy_fd
.handler
) {
2125 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
2126 close(u
->postcopy_fd
.fd
);
2127 u
->postcopy_fd
.handler
= NULL
;
2130 close_slave_channel(u
);
2132 g_free(u
->region_rb
);
2133 u
->region_rb
= NULL
;
2134 g_free(u
->region_rb_offset
);
2135 u
->region_rb_offset
= NULL
;
2136 u
->region_rb_len
= 0;
2143 static int vhost_user_get_vq_index(struct vhost_dev
*dev
, int idx
)
2145 assert(idx
>= dev
->vq_index
&& idx
< dev
->vq_index
+ dev
->nvqs
);
2150 static int vhost_user_memslots_limit(struct vhost_dev
*dev
)
2152 struct vhost_user
*u
= dev
->opaque
;
2154 return u
->user
->memory_slots
;
2157 static bool vhost_user_requires_shm_log(struct vhost_dev
*dev
)
2159 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2161 return virtio_has_feature(dev
->protocol_features
,
2162 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
2165 static int vhost_user_migration_done(struct vhost_dev
*dev
, char* mac_addr
)
2167 VhostUserMsg msg
= { };
2169 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2171 /* If guest supports GUEST_ANNOUNCE do nothing */
2172 if (virtio_has_feature(dev
->acked_features
, VIRTIO_NET_F_GUEST_ANNOUNCE
)) {
2176 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
2177 if (virtio_has_feature(dev
->protocol_features
,
2178 VHOST_USER_PROTOCOL_F_RARP
)) {
2179 msg
.hdr
.request
= VHOST_USER_SEND_RARP
;
2180 msg
.hdr
.flags
= VHOST_USER_VERSION
;
2181 memcpy((char *)&msg
.payload
.u64
, mac_addr
, 6);
2182 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
2184 return vhost_user_write(dev
, &msg
, NULL
, 0);
2189 static bool vhost_user_can_merge(struct vhost_dev
*dev
,
2190 uint64_t start1
, uint64_t size1
,
2191 uint64_t start2
, uint64_t size2
)
2196 (void)vhost_user_get_mr_data(start1
, &offset
, &mfd
);
2197 (void)vhost_user_get_mr_data(start2
, &offset
, &rfd
);
2202 static int vhost_user_net_set_mtu(struct vhost_dev
*dev
, uint16_t mtu
)
2205 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
2206 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2209 if (!(dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU
))) {
2213 msg
.hdr
.request
= VHOST_USER_NET_SET_MTU
;
2214 msg
.payload
.u64
= mtu
;
2215 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
2216 msg
.hdr
.flags
= VHOST_USER_VERSION
;
2217 if (reply_supported
) {
2218 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2221 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2226 /* If reply_ack supported, slave has to ack specified MTU is valid */
2227 if (reply_supported
) {
2228 return process_message_reply(dev
, &msg
);
2234 static int vhost_user_send_device_iotlb_msg(struct vhost_dev
*dev
,
2235 struct vhost_iotlb_msg
*imsg
)
2238 VhostUserMsg msg
= {
2239 .hdr
.request
= VHOST_USER_IOTLB_MSG
,
2240 .hdr
.size
= sizeof(msg
.payload
.iotlb
),
2241 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
2242 .payload
.iotlb
= *imsg
,
2245 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2250 return process_message_reply(dev
, &msg
);
2254 static void vhost_user_set_iotlb_callback(struct vhost_dev
*dev
, int enabled
)
2256 /* No-op as the receive channel is not dedicated to IOTLB messages. */
2259 static int vhost_user_get_config(struct vhost_dev
*dev
, uint8_t *config
,
2260 uint32_t config_len
, Error
**errp
)
2263 VhostUserMsg msg
= {
2264 .hdr
.request
= VHOST_USER_GET_CONFIG
,
2265 .hdr
.flags
= VHOST_USER_VERSION
,
2266 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
,
2269 if (!virtio_has_feature(dev
->protocol_features
,
2270 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2271 error_setg(errp
, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
2275 assert(config_len
<= VHOST_USER_MAX_CONFIG_SIZE
);
2277 msg
.payload
.config
.offset
= 0;
2278 msg
.payload
.config
.size
= config_len
;
2279 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2281 error_setg_errno(errp
, -ret
, "vhost_get_config failed");
2285 ret
= vhost_user_read(dev
, &msg
);
2287 error_setg_errno(errp
, -ret
, "vhost_get_config failed");
2291 if (msg
.hdr
.request
!= VHOST_USER_GET_CONFIG
) {
2293 "Received unexpected msg type. Expected %d received %d",
2294 VHOST_USER_GET_CONFIG
, msg
.hdr
.request
);
2298 if (msg
.hdr
.size
!= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
) {
2299 error_setg(errp
, "Received bad msg size.");
2303 memcpy(config
, msg
.payload
.config
.region
, config_len
);
2308 static int vhost_user_set_config(struct vhost_dev
*dev
, const uint8_t *data
,
2309 uint32_t offset
, uint32_t size
, uint32_t flags
)
2313 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
2314 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2316 VhostUserMsg msg
= {
2317 .hdr
.request
= VHOST_USER_SET_CONFIG
,
2318 .hdr
.flags
= VHOST_USER_VERSION
,
2319 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ size
,
2322 if (!virtio_has_feature(dev
->protocol_features
,
2323 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2327 if (reply_supported
) {
2328 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2331 if (size
> VHOST_USER_MAX_CONFIG_SIZE
) {
2335 msg
.payload
.config
.offset
= offset
,
2336 msg
.payload
.config
.size
= size
,
2337 msg
.payload
.config
.flags
= flags
,
2338 p
= msg
.payload
.config
.region
;
2339 memcpy(p
, data
, size
);
2341 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2346 if (reply_supported
) {
2347 return process_message_reply(dev
, &msg
);
2353 static int vhost_user_crypto_create_session(struct vhost_dev
*dev
,
2355 uint64_t *session_id
)
2358 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
2359 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
2360 CryptoDevBackendSymSessionInfo
*sess_info
= session_info
;
2361 VhostUserMsg msg
= {
2362 .hdr
.request
= VHOST_USER_CREATE_CRYPTO_SESSION
,
2363 .hdr
.flags
= VHOST_USER_VERSION
,
2364 .hdr
.size
= sizeof(msg
.payload
.session
),
2367 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2369 if (!crypto_session
) {
2370 error_report("vhost-user trying to send unhandled ioctl");
2374 memcpy(&msg
.payload
.session
.session_setup_data
, sess_info
,
2375 sizeof(CryptoDevBackendSymSessionInfo
));
2376 if (sess_info
->key_len
) {
2377 memcpy(&msg
.payload
.session
.key
, sess_info
->cipher_key
,
2378 sess_info
->key_len
);
2380 if (sess_info
->auth_key_len
> 0) {
2381 memcpy(&msg
.payload
.session
.auth_key
, sess_info
->auth_key
,
2382 sess_info
->auth_key_len
);
2384 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2386 error_report("vhost_user_write() return %d, create session failed",
2391 ret
= vhost_user_read(dev
, &msg
);
2393 error_report("vhost_user_read() return %d, create session failed",
2398 if (msg
.hdr
.request
!= VHOST_USER_CREATE_CRYPTO_SESSION
) {
2399 error_report("Received unexpected msg type. Expected %d received %d",
2400 VHOST_USER_CREATE_CRYPTO_SESSION
, msg
.hdr
.request
);
2404 if (msg
.hdr
.size
!= sizeof(msg
.payload
.session
)) {
2405 error_report("Received bad msg size.");
2409 if (msg
.payload
.session
.session_id
< 0) {
2410 error_report("Bad session id: %" PRId64
"",
2411 msg
.payload
.session
.session_id
);
2414 *session_id
= msg
.payload
.session
.session_id
;
2420 vhost_user_crypto_close_session(struct vhost_dev
*dev
, uint64_t session_id
)
2423 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
2424 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
2425 VhostUserMsg msg
= {
2426 .hdr
.request
= VHOST_USER_CLOSE_CRYPTO_SESSION
,
2427 .hdr
.flags
= VHOST_USER_VERSION
,
2428 .hdr
.size
= sizeof(msg
.payload
.u64
),
2430 msg
.payload
.u64
= session_id
;
2432 if (!crypto_session
) {
2433 error_report("vhost-user trying to send unhandled ioctl");
2437 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2439 error_report("vhost_user_write() return %d, close session failed",
2447 static bool vhost_user_mem_section_filter(struct vhost_dev
*dev
,
2448 MemoryRegionSection
*section
)
2450 return memory_region_get_fd(section
->mr
) >= 0;
2453 static int vhost_user_get_inflight_fd(struct vhost_dev
*dev
,
2454 uint16_t queue_size
,
2455 struct vhost_inflight
*inflight
)
2460 struct vhost_user
*u
= dev
->opaque
;
2461 CharBackend
*chr
= u
->user
->chr
;
2462 VhostUserMsg msg
= {
2463 .hdr
.request
= VHOST_USER_GET_INFLIGHT_FD
,
2464 .hdr
.flags
= VHOST_USER_VERSION
,
2465 .payload
.inflight
.num_queues
= dev
->nvqs
,
2466 .payload
.inflight
.queue_size
= queue_size
,
2467 .hdr
.size
= sizeof(msg
.payload
.inflight
),
2470 if (!virtio_has_feature(dev
->protocol_features
,
2471 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2475 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2480 ret
= vhost_user_read(dev
, &msg
);
2485 if (msg
.hdr
.request
!= VHOST_USER_GET_INFLIGHT_FD
) {
2486 error_report("Received unexpected msg type. "
2487 "Expected %d received %d",
2488 VHOST_USER_GET_INFLIGHT_FD
, msg
.hdr
.request
);
2492 if (msg
.hdr
.size
!= sizeof(msg
.payload
.inflight
)) {
2493 error_report("Received bad msg size.");
2497 if (!msg
.payload
.inflight
.mmap_size
) {
2501 fd
= qemu_chr_fe_get_msgfd(chr
);
2503 error_report("Failed to get mem fd");
2507 addr
= mmap(0, msg
.payload
.inflight
.mmap_size
, PROT_READ
| PROT_WRITE
,
2508 MAP_SHARED
, fd
, msg
.payload
.inflight
.mmap_offset
);
2510 if (addr
== MAP_FAILED
) {
2511 error_report("Failed to mmap mem fd");
2516 inflight
->addr
= addr
;
2518 inflight
->size
= msg
.payload
.inflight
.mmap_size
;
2519 inflight
->offset
= msg
.payload
.inflight
.mmap_offset
;
2520 inflight
->queue_size
= queue_size
;
2525 static int vhost_user_set_inflight_fd(struct vhost_dev
*dev
,
2526 struct vhost_inflight
*inflight
)
2528 VhostUserMsg msg
= {
2529 .hdr
.request
= VHOST_USER_SET_INFLIGHT_FD
,
2530 .hdr
.flags
= VHOST_USER_VERSION
,
2531 .payload
.inflight
.mmap_size
= inflight
->size
,
2532 .payload
.inflight
.mmap_offset
= inflight
->offset
,
2533 .payload
.inflight
.num_queues
= dev
->nvqs
,
2534 .payload
.inflight
.queue_size
= inflight
->queue_size
,
2535 .hdr
.size
= sizeof(msg
.payload
.inflight
),
2538 if (!virtio_has_feature(dev
->protocol_features
,
2539 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2543 return vhost_user_write(dev
, &msg
, &inflight
->fd
, 1);
2546 static void vhost_user_state_destroy(gpointer data
)
2548 VhostUserHostNotifier
*n
= (VhostUserHostNotifier
*) data
;
2550 vhost_user_host_notifier_remove(n
, NULL
);
2551 object_unparent(OBJECT(&n
->mr
));
2553 * We can't free until vhost_user_host_notifier_remove has
2554 * done it's thing so schedule the free with RCU.
2560 bool vhost_user_init(VhostUserState
*user
, CharBackend
*chr
, Error
**errp
)
2563 error_setg(errp
, "Cannot initialize vhost-user state");
2567 user
->memory_slots
= 0;
2568 user
->notifiers
= g_ptr_array_new_full(VIRTIO_QUEUE_MAX
/ 4,
2569 &vhost_user_state_destroy
);
2573 void vhost_user_cleanup(VhostUserState
*user
)
2578 memory_region_transaction_begin();
2579 user
->notifiers
= (GPtrArray
*) g_ptr_array_free(user
->notifiers
, true);
2580 memory_region_transaction_commit();
2586 vu_async_close_fn cb
;
2589 struct vhost_dev
*vhost
;
2590 } VhostAsyncCallback
;
2592 static void vhost_user_async_close_bh(void *opaque
)
2594 VhostAsyncCallback
*data
= opaque
;
2595 struct vhost_dev
*vhost
= data
->vhost
;
2598 * If the vhost_dev has been cleared in the meantime there is
2599 * nothing left to do as some other path has completed the
2603 data
->cb(data
->dev
);
2610 * We only schedule the work if the machine is running. If suspended
2611 * we want to keep all the in-flight data as is for migration
2614 void vhost_user_async_close(DeviceState
*d
,
2615 CharBackend
*chardev
, struct vhost_dev
*vhost
,
2616 vu_async_close_fn cb
)
2618 if (!runstate_check(RUN_STATE_SHUTDOWN
)) {
2620 * A close event may happen during a read/write, but vhost
2621 * code assumes the vhost_dev remains setup, so delay the
2624 AioContext
*ctx
= qemu_get_current_aio_context();
2625 VhostAsyncCallback
*data
= g_new0(VhostAsyncCallback
, 1);
2627 /* Save data for the callback */
2631 data
->vhost
= vhost
;
2633 /* Disable any further notifications on the chardev */
2634 qemu_chr_fe_set_handlers(chardev
,
2635 NULL
, NULL
, NULL
, NULL
, NULL
, NULL
,
2638 aio_bh_schedule_oneshot(ctx
, vhost_user_async_close_bh
, data
);
2641 * Move vhost device to the stopped state. The vhost-user device
2642 * will be clean up and disconnected in BH. This can be useful in
2643 * the vhost migration code. If disconnect was caught there is an
2644 * option for the general vhost code to get the dev state without
2645 * knowing its type (in this case vhost-user).
2647 * Note if the vhost device is fully cleared by the time we
2648 * execute the bottom half we won't continue with the cleanup.
2650 vhost
->started
= false;
2654 static int vhost_user_dev_start(struct vhost_dev
*dev
, bool started
)
2656 if (!virtio_has_feature(dev
->protocol_features
,
2657 VHOST_USER_PROTOCOL_F_STATUS
)) {
2661 /* Set device status only for last queue pair */
2662 if (dev
->vq_index
+ dev
->nvqs
!= dev
->vq_index_end
) {
2667 return vhost_user_add_status(dev
, VIRTIO_CONFIG_S_ACKNOWLEDGE
|
2668 VIRTIO_CONFIG_S_DRIVER
|
2669 VIRTIO_CONFIG_S_DRIVER_OK
);
2675 static void vhost_user_reset_status(struct vhost_dev
*dev
)
2677 /* Set device status only for last queue pair */
2678 if (dev
->vq_index
+ dev
->nvqs
!= dev
->vq_index_end
) {
2682 if (virtio_has_feature(dev
->protocol_features
,
2683 VHOST_USER_PROTOCOL_F_STATUS
)) {
2684 vhost_user_set_status(dev
, 0);
2688 const VhostOps user_ops
= {
2689 .backend_type
= VHOST_BACKEND_TYPE_USER
,
2690 .vhost_backend_init
= vhost_user_backend_init
,
2691 .vhost_backend_cleanup
= vhost_user_backend_cleanup
,
2692 .vhost_backend_memslots_limit
= vhost_user_memslots_limit
,
2693 .vhost_set_log_base
= vhost_user_set_log_base
,
2694 .vhost_set_mem_table
= vhost_user_set_mem_table
,
2695 .vhost_set_vring_addr
= vhost_user_set_vring_addr
,
2696 .vhost_set_vring_endian
= vhost_user_set_vring_endian
,
2697 .vhost_set_vring_num
= vhost_user_set_vring_num
,
2698 .vhost_set_vring_base
= vhost_user_set_vring_base
,
2699 .vhost_get_vring_base
= vhost_user_get_vring_base
,
2700 .vhost_set_vring_kick
= vhost_user_set_vring_kick
,
2701 .vhost_set_vring_call
= vhost_user_set_vring_call
,
2702 .vhost_set_vring_err
= vhost_user_set_vring_err
,
2703 .vhost_set_features
= vhost_user_set_features
,
2704 .vhost_get_features
= vhost_user_get_features
,
2705 .vhost_set_owner
= vhost_user_set_owner
,
2706 .vhost_reset_device
= vhost_user_reset_device
,
2707 .vhost_get_vq_index
= vhost_user_get_vq_index
,
2708 .vhost_set_vring_enable
= vhost_user_set_vring_enable
,
2709 .vhost_requires_shm_log
= vhost_user_requires_shm_log
,
2710 .vhost_migration_done
= vhost_user_migration_done
,
2711 .vhost_backend_can_merge
= vhost_user_can_merge
,
2712 .vhost_net_set_mtu
= vhost_user_net_set_mtu
,
2713 .vhost_set_iotlb_callback
= vhost_user_set_iotlb_callback
,
2714 .vhost_send_device_iotlb_msg
= vhost_user_send_device_iotlb_msg
,
2715 .vhost_get_config
= vhost_user_get_config
,
2716 .vhost_set_config
= vhost_user_set_config
,
2717 .vhost_crypto_create_session
= vhost_user_crypto_create_session
,
2718 .vhost_crypto_close_session
= vhost_user_crypto_close_session
,
2719 .vhost_backend_mem_section_filter
= vhost_user_mem_section_filter
,
2720 .vhost_get_inflight_fd
= vhost_user_get_inflight_fd
,
2721 .vhost_set_inflight_fd
= vhost_user_set_inflight_fd
,
2722 .vhost_dev_start
= vhost_user_dev_start
,
2723 .vhost_reset_status
= vhost_user_reset_status
,