]>
Commit | Line | Data |
---|---|---|
d5970055 MT |
1 | /* |
2 | * vhost support | |
3 | * | |
4 | * Copyright Red Hat, Inc. 2010 | |
5 | * | |
6 | * Authors: | |
7 | * Michael S. Tsirkin <mst@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
d5970055 | 13 | #include <sys/ioctl.h> |
d5970055 MT |
14 | #include "vhost.h" |
15 | #include "hw/hw.h" | |
bf1b0071 | 16 | #include "range.h" |
11078ae3 | 17 | #include <linux/vhost.h> |
d5970055 MT |
18 | |
19 | static void vhost_dev_sync_region(struct vhost_dev *dev, | |
20 | uint64_t mfirst, uint64_t mlast, | |
21 | uint64_t rfirst, uint64_t rlast) | |
22 | { | |
23 | uint64_t start = MAX(mfirst, rfirst); | |
24 | uint64_t end = MIN(mlast, rlast); | |
25 | vhost_log_chunk_t *from = dev->log + start / VHOST_LOG_CHUNK; | |
26 | vhost_log_chunk_t *to = dev->log + end / VHOST_LOG_CHUNK + 1; | |
27 | uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK; | |
28 | ||
29 | assert(end / VHOST_LOG_CHUNK < dev->log_size); | |
30 | assert(start / VHOST_LOG_CHUNK < dev->log_size); | |
31 | if (end < start) { | |
32 | return; | |
33 | } | |
34 | for (;from < to; ++from) { | |
35 | vhost_log_chunk_t log; | |
36 | int bit; | |
37 | /* We first check with non-atomic: much cheaper, | |
38 | * and we expect non-dirty to be the common case. */ | |
39 | if (!*from) { | |
0c600ce2 | 40 | addr += VHOST_LOG_CHUNK; |
d5970055 MT |
41 | continue; |
42 | } | |
43 | /* Data must be read atomically. We don't really | |
44 | * need the barrier semantics of __sync | |
45 | * builtins, but it's easier to use them than | |
46 | * roll our own. */ | |
47 | log = __sync_fetch_and_and(from, 0); | |
48 | while ((bit = sizeof(log) > sizeof(int) ? | |
49 | ffsll(log) : ffs(log))) { | |
7b67b18a | 50 | ram_addr_t ram_addr; |
d5970055 | 51 | bit -= 1; |
7b67b18a MT |
52 | ram_addr = cpu_get_physical_page_desc(addr + bit * VHOST_LOG_PAGE); |
53 | cpu_physical_memory_set_dirty(ram_addr); | |
d5970055 MT |
54 | log &= ~(0x1ull << bit); |
55 | } | |
56 | addr += VHOST_LOG_CHUNK; | |
57 | } | |
58 | } | |
59 | ||
60 | static int vhost_client_sync_dirty_bitmap(CPUPhysMemoryClient *client, | |
61 | target_phys_addr_t start_addr, | |
62 | target_phys_addr_t end_addr) | |
63 | { | |
64 | struct vhost_dev *dev = container_of(client, struct vhost_dev, client); | |
65 | int i; | |
66 | if (!dev->log_enabled || !dev->started) { | |
67 | return 0; | |
68 | } | |
69 | for (i = 0; i < dev->mem->nregions; ++i) { | |
70 | struct vhost_memory_region *reg = dev->mem->regions + i; | |
71 | vhost_dev_sync_region(dev, start_addr, end_addr, | |
72 | reg->guest_phys_addr, | |
73 | range_get_last(reg->guest_phys_addr, | |
74 | reg->memory_size)); | |
75 | } | |
76 | for (i = 0; i < dev->nvqs; ++i) { | |
77 | struct vhost_virtqueue *vq = dev->vqs + i; | |
78 | vhost_dev_sync_region(dev, start_addr, end_addr, vq->used_phys, | |
79 | range_get_last(vq->used_phys, vq->used_size)); | |
80 | } | |
81 | return 0; | |
82 | } | |
83 | ||
84 | /* Assign/unassign. Keep an unsorted array of non-overlapping | |
85 | * memory regions in dev->mem. */ | |
86 | static void vhost_dev_unassign_memory(struct vhost_dev *dev, | |
87 | uint64_t start_addr, | |
88 | uint64_t size) | |
89 | { | |
90 | int from, to, n = dev->mem->nregions; | |
91 | /* Track overlapping/split regions for sanity checking. */ | |
92 | int overlap_start = 0, overlap_end = 0, overlap_middle = 0, split = 0; | |
93 | ||
94 | for (from = 0, to = 0; from < n; ++from, ++to) { | |
95 | struct vhost_memory_region *reg = dev->mem->regions + to; | |
96 | uint64_t reglast; | |
97 | uint64_t memlast; | |
98 | uint64_t change; | |
99 | ||
100 | /* clone old region */ | |
101 | if (to != from) { | |
102 | memcpy(reg, dev->mem->regions + from, sizeof *reg); | |
103 | } | |
104 | ||
105 | /* No overlap is simple */ | |
106 | if (!ranges_overlap(reg->guest_phys_addr, reg->memory_size, | |
107 | start_addr, size)) { | |
108 | continue; | |
109 | } | |
110 | ||
111 | /* Split only happens if supplied region | |
112 | * is in the middle of an existing one. Thus it can not | |
113 | * overlap with any other existing region. */ | |
114 | assert(!split); | |
115 | ||
116 | reglast = range_get_last(reg->guest_phys_addr, reg->memory_size); | |
117 | memlast = range_get_last(start_addr, size); | |
118 | ||
119 | /* Remove whole region */ | |
120 | if (start_addr <= reg->guest_phys_addr && memlast >= reglast) { | |
121 | --dev->mem->nregions; | |
122 | --to; | |
123 | assert(to >= 0); | |
124 | ++overlap_middle; | |
125 | continue; | |
126 | } | |
127 | ||
128 | /* Shrink region */ | |
129 | if (memlast >= reglast) { | |
130 | reg->memory_size = start_addr - reg->guest_phys_addr; | |
131 | assert(reg->memory_size); | |
132 | assert(!overlap_end); | |
133 | ++overlap_end; | |
134 | continue; | |
135 | } | |
136 | ||
137 | /* Shift region */ | |
138 | if (start_addr <= reg->guest_phys_addr) { | |
139 | change = memlast + 1 - reg->guest_phys_addr; | |
140 | reg->memory_size -= change; | |
141 | reg->guest_phys_addr += change; | |
142 | reg->userspace_addr += change; | |
143 | assert(reg->memory_size); | |
144 | assert(!overlap_start); | |
145 | ++overlap_start; | |
146 | continue; | |
147 | } | |
148 | ||
149 | /* This only happens if supplied region | |
150 | * is in the middle of an existing one. Thus it can not | |
151 | * overlap with any other existing region. */ | |
152 | assert(!overlap_start); | |
153 | assert(!overlap_end); | |
154 | assert(!overlap_middle); | |
155 | /* Split region: shrink first part, shift second part. */ | |
156 | memcpy(dev->mem->regions + n, reg, sizeof *reg); | |
157 | reg->memory_size = start_addr - reg->guest_phys_addr; | |
158 | assert(reg->memory_size); | |
159 | change = memlast + 1 - reg->guest_phys_addr; | |
160 | reg = dev->mem->regions + n; | |
161 | reg->memory_size -= change; | |
162 | assert(reg->memory_size); | |
163 | reg->guest_phys_addr += change; | |
164 | reg->userspace_addr += change; | |
165 | /* Never add more than 1 region */ | |
166 | assert(dev->mem->nregions == n); | |
167 | ++dev->mem->nregions; | |
168 | ++split; | |
169 | } | |
170 | } | |
171 | ||
172 | /* Called after unassign, so no regions overlap the given range. */ | |
173 | static void vhost_dev_assign_memory(struct vhost_dev *dev, | |
174 | uint64_t start_addr, | |
175 | uint64_t size, | |
176 | uint64_t uaddr) | |
177 | { | |
178 | int from, to; | |
179 | struct vhost_memory_region *merged = NULL; | |
180 | for (from = 0, to = 0; from < dev->mem->nregions; ++from, ++to) { | |
181 | struct vhost_memory_region *reg = dev->mem->regions + to; | |
182 | uint64_t prlast, urlast; | |
183 | uint64_t pmlast, umlast; | |
184 | uint64_t s, e, u; | |
185 | ||
186 | /* clone old region */ | |
187 | if (to != from) { | |
188 | memcpy(reg, dev->mem->regions + from, sizeof *reg); | |
189 | } | |
190 | prlast = range_get_last(reg->guest_phys_addr, reg->memory_size); | |
191 | pmlast = range_get_last(start_addr, size); | |
192 | urlast = range_get_last(reg->userspace_addr, reg->memory_size); | |
193 | umlast = range_get_last(uaddr, size); | |
194 | ||
195 | /* check for overlapping regions: should never happen. */ | |
196 | assert(prlast < start_addr || pmlast < reg->guest_phys_addr); | |
197 | /* Not an adjacent or overlapping region - do not merge. */ | |
198 | if ((prlast + 1 != start_addr || urlast + 1 != uaddr) && | |
199 | (pmlast + 1 != reg->guest_phys_addr || | |
200 | umlast + 1 != reg->userspace_addr)) { | |
201 | continue; | |
202 | } | |
203 | ||
204 | if (merged) { | |
205 | --to; | |
206 | assert(to >= 0); | |
207 | } else { | |
208 | merged = reg; | |
209 | } | |
210 | u = MIN(uaddr, reg->userspace_addr); | |
211 | s = MIN(start_addr, reg->guest_phys_addr); | |
212 | e = MAX(pmlast, prlast); | |
213 | uaddr = merged->userspace_addr = u; | |
214 | start_addr = merged->guest_phys_addr = s; | |
215 | size = merged->memory_size = e - s + 1; | |
216 | assert(merged->memory_size); | |
217 | } | |
218 | ||
219 | if (!merged) { | |
220 | struct vhost_memory_region *reg = dev->mem->regions + to; | |
221 | memset(reg, 0, sizeof *reg); | |
222 | reg->memory_size = size; | |
223 | assert(reg->memory_size); | |
224 | reg->guest_phys_addr = start_addr; | |
225 | reg->userspace_addr = uaddr; | |
226 | ++to; | |
227 | } | |
228 | assert(to <= dev->mem->nregions + 1); | |
229 | dev->mem->nregions = to; | |
230 | } | |
231 | ||
232 | static uint64_t vhost_get_log_size(struct vhost_dev *dev) | |
233 | { | |
234 | uint64_t log_size = 0; | |
235 | int i; | |
236 | for (i = 0; i < dev->mem->nregions; ++i) { | |
237 | struct vhost_memory_region *reg = dev->mem->regions + i; | |
238 | uint64_t last = range_get_last(reg->guest_phys_addr, | |
239 | reg->memory_size); | |
240 | log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1); | |
241 | } | |
242 | for (i = 0; i < dev->nvqs; ++i) { | |
243 | struct vhost_virtqueue *vq = dev->vqs + i; | |
244 | uint64_t last = vq->used_phys + vq->used_size - 1; | |
245 | log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1); | |
246 | } | |
247 | return log_size; | |
248 | } | |
249 | ||
250 | static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size) | |
251 | { | |
252 | vhost_log_chunk_t *log; | |
253 | uint64_t log_base; | |
254 | int r; | |
255 | if (size) { | |
256 | log = qemu_mallocz(size * sizeof *log); | |
257 | } else { | |
258 | log = NULL; | |
259 | } | |
260 | log_base = (uint64_t)(unsigned long)log; | |
261 | r = ioctl(dev->control, VHOST_SET_LOG_BASE, &log_base); | |
262 | assert(r >= 0); | |
263 | vhost_client_sync_dirty_bitmap(&dev->client, 0, | |
264 | (target_phys_addr_t)~0x0ull); | |
265 | if (dev->log) { | |
266 | qemu_free(dev->log); | |
267 | } | |
268 | dev->log = log; | |
269 | dev->log_size = size; | |
270 | } | |
271 | ||
272 | static int vhost_verify_ring_mappings(struct vhost_dev *dev, | |
273 | uint64_t start_addr, | |
274 | uint64_t size) | |
275 | { | |
276 | int i; | |
277 | for (i = 0; i < dev->nvqs; ++i) { | |
278 | struct vhost_virtqueue *vq = dev->vqs + i; | |
279 | target_phys_addr_t l; | |
280 | void *p; | |
281 | ||
282 | if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) { | |
283 | continue; | |
284 | } | |
285 | l = vq->ring_size; | |
286 | p = cpu_physical_memory_map(vq->ring_phys, &l, 1); | |
287 | if (!p || l != vq->ring_size) { | |
288 | fprintf(stderr, "Unable to map ring buffer for ring %d\n", i); | |
289 | return -ENOMEM; | |
290 | } | |
291 | if (p != vq->ring) { | |
292 | fprintf(stderr, "Ring buffer relocated for ring %d\n", i); | |
293 | return -EBUSY; | |
294 | } | |
295 | cpu_physical_memory_unmap(p, l, 0, 0); | |
296 | } | |
297 | return 0; | |
298 | } | |
299 | ||
300 | static void vhost_client_set_memory(CPUPhysMemoryClient *client, | |
301 | target_phys_addr_t start_addr, | |
302 | ram_addr_t size, | |
303 | ram_addr_t phys_offset) | |
304 | { | |
305 | struct vhost_dev *dev = container_of(client, struct vhost_dev, client); | |
306 | ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK; | |
307 | int s = offsetof(struct vhost_memory, regions) + | |
308 | (dev->mem->nregions + 1) * sizeof dev->mem->regions[0]; | |
309 | uint64_t log_size; | |
310 | int r; | |
311 | dev->mem = qemu_realloc(dev->mem, s); | |
312 | ||
313 | assert(size); | |
314 | ||
315 | vhost_dev_unassign_memory(dev, start_addr, size); | |
316 | if (flags == IO_MEM_RAM) { | |
317 | /* Add given mapping, merging adjacent regions if any */ | |
318 | vhost_dev_assign_memory(dev, start_addr, size, | |
319 | (uintptr_t)qemu_get_ram_ptr(phys_offset)); | |
320 | } else { | |
321 | /* Remove old mapping for this memory, if any. */ | |
322 | vhost_dev_unassign_memory(dev, start_addr, size); | |
323 | } | |
324 | ||
325 | if (!dev->started) { | |
326 | return; | |
327 | } | |
328 | ||
329 | if (dev->started) { | |
330 | r = vhost_verify_ring_mappings(dev, start_addr, size); | |
331 | assert(r >= 0); | |
332 | } | |
333 | ||
334 | if (!dev->log_enabled) { | |
335 | r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); | |
336 | assert(r >= 0); | |
337 | return; | |
338 | } | |
339 | log_size = vhost_get_log_size(dev); | |
340 | /* We allocate an extra 4K bytes to log, | |
341 | * to reduce the * number of reallocations. */ | |
342 | #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log) | |
343 | /* To log more, must increase log size before table update. */ | |
344 | if (dev->log_size < log_size) { | |
345 | vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER); | |
346 | } | |
347 | r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); | |
348 | assert(r >= 0); | |
349 | /* To log less, can only decrease log size after table update. */ | |
350 | if (dev->log_size > log_size + VHOST_LOG_BUFFER) { | |
351 | vhost_dev_log_resize(dev, log_size); | |
352 | } | |
353 | } | |
354 | ||
355 | static int vhost_virtqueue_set_addr(struct vhost_dev *dev, | |
356 | struct vhost_virtqueue *vq, | |
357 | unsigned idx, bool enable_log) | |
358 | { | |
359 | struct vhost_vring_addr addr = { | |
360 | .index = idx, | |
2b3af999 SW |
361 | .desc_user_addr = (uint64_t)(unsigned long)vq->desc, |
362 | .avail_user_addr = (uint64_t)(unsigned long)vq->avail, | |
363 | .used_user_addr = (uint64_t)(unsigned long)vq->used, | |
d5970055 MT |
364 | .log_guest_addr = vq->used_phys, |
365 | .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0, | |
366 | }; | |
367 | int r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr); | |
368 | if (r < 0) { | |
369 | return -errno; | |
370 | } | |
371 | return 0; | |
372 | } | |
373 | ||
374 | static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log) | |
375 | { | |
376 | uint64_t features = dev->acked_features; | |
377 | int r; | |
378 | if (enable_log) { | |
379 | features |= 0x1 << VHOST_F_LOG_ALL; | |
380 | } | |
381 | r = ioctl(dev->control, VHOST_SET_FEATURES, &features); | |
382 | return r < 0 ? -errno : 0; | |
383 | } | |
384 | ||
385 | static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log) | |
386 | { | |
387 | int r, t, i; | |
388 | r = vhost_dev_set_features(dev, enable_log); | |
389 | if (r < 0) { | |
390 | goto err_features; | |
391 | } | |
392 | for (i = 0; i < dev->nvqs; ++i) { | |
393 | r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i, | |
394 | enable_log); | |
395 | if (r < 0) { | |
396 | goto err_vq; | |
397 | } | |
398 | } | |
399 | return 0; | |
400 | err_vq: | |
401 | for (; i >= 0; --i) { | |
402 | t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i, | |
403 | dev->log_enabled); | |
404 | assert(t >= 0); | |
405 | } | |
406 | t = vhost_dev_set_features(dev, dev->log_enabled); | |
407 | assert(t >= 0); | |
408 | err_features: | |
409 | return r; | |
410 | } | |
411 | ||
412 | static int vhost_client_migration_log(CPUPhysMemoryClient *client, | |
413 | int enable) | |
414 | { | |
415 | struct vhost_dev *dev = container_of(client, struct vhost_dev, client); | |
416 | int r; | |
417 | if (!!enable == dev->log_enabled) { | |
418 | return 0; | |
419 | } | |
420 | if (!dev->started) { | |
421 | dev->log_enabled = enable; | |
422 | return 0; | |
423 | } | |
424 | if (!enable) { | |
425 | r = vhost_dev_set_log(dev, false); | |
426 | if (r < 0) { | |
427 | return r; | |
428 | } | |
429 | if (dev->log) { | |
430 | qemu_free(dev->log); | |
431 | } | |
432 | dev->log = NULL; | |
433 | dev->log_size = 0; | |
434 | } else { | |
435 | vhost_dev_log_resize(dev, vhost_get_log_size(dev)); | |
436 | r = vhost_dev_set_log(dev, true); | |
437 | if (r < 0) { | |
438 | return r; | |
439 | } | |
440 | } | |
441 | dev->log_enabled = enable; | |
442 | return 0; | |
443 | } | |
444 | ||
445 | static int vhost_virtqueue_init(struct vhost_dev *dev, | |
446 | struct VirtIODevice *vdev, | |
447 | struct vhost_virtqueue *vq, | |
448 | unsigned idx) | |
449 | { | |
450 | target_phys_addr_t s, l, a; | |
451 | int r; | |
452 | struct vhost_vring_file file = { | |
453 | .index = idx, | |
454 | }; | |
455 | struct vhost_vring_state state = { | |
456 | .index = idx, | |
457 | }; | |
458 | struct VirtQueue *vvq = virtio_get_queue(vdev, idx); | |
459 | ||
d5970055 MT |
460 | if (!vdev->binding->set_host_notifier) { |
461 | fprintf(stderr, "binding does not support host notifiers\n"); | |
462 | return -ENOSYS; | |
463 | } | |
464 | ||
465 | vq->num = state.num = virtio_queue_get_num(vdev, idx); | |
466 | r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state); | |
467 | if (r) { | |
468 | return -errno; | |
469 | } | |
470 | ||
471 | state.num = virtio_queue_get_last_avail_idx(vdev, idx); | |
472 | r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state); | |
473 | if (r) { | |
474 | return -errno; | |
475 | } | |
476 | ||
477 | s = l = virtio_queue_get_desc_size(vdev, idx); | |
478 | a = virtio_queue_get_desc_addr(vdev, idx); | |
479 | vq->desc = cpu_physical_memory_map(a, &l, 0); | |
480 | if (!vq->desc || l != s) { | |
481 | r = -ENOMEM; | |
482 | goto fail_alloc_desc; | |
483 | } | |
484 | s = l = virtio_queue_get_avail_size(vdev, idx); | |
485 | a = virtio_queue_get_avail_addr(vdev, idx); | |
486 | vq->avail = cpu_physical_memory_map(a, &l, 0); | |
487 | if (!vq->avail || l != s) { | |
488 | r = -ENOMEM; | |
489 | goto fail_alloc_avail; | |
490 | } | |
491 | vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx); | |
492 | vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx); | |
493 | vq->used = cpu_physical_memory_map(a, &l, 1); | |
494 | if (!vq->used || l != s) { | |
495 | r = -ENOMEM; | |
496 | goto fail_alloc_used; | |
497 | } | |
498 | ||
499 | vq->ring_size = s = l = virtio_queue_get_ring_size(vdev, idx); | |
500 | vq->ring_phys = a = virtio_queue_get_ring_addr(vdev, idx); | |
501 | vq->ring = cpu_physical_memory_map(a, &l, 1); | |
502 | if (!vq->ring || l != s) { | |
503 | r = -ENOMEM; | |
504 | goto fail_alloc_ring; | |
505 | } | |
506 | ||
507 | r = vhost_virtqueue_set_addr(dev, vq, idx, dev->log_enabled); | |
508 | if (r < 0) { | |
509 | r = -errno; | |
510 | goto fail_alloc; | |
511 | } | |
d5970055 MT |
512 | r = vdev->binding->set_host_notifier(vdev->binding_opaque, idx, true); |
513 | if (r < 0) { | |
514 | fprintf(stderr, "Error binding host notifier: %d\n", -r); | |
515 | goto fail_host_notifier; | |
516 | } | |
517 | ||
518 | file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq)); | |
519 | r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file); | |
520 | if (r) { | |
c8852121 | 521 | r = -errno; |
d5970055 MT |
522 | goto fail_kick; |
523 | } | |
524 | ||
525 | file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq)); | |
526 | r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file); | |
527 | if (r) { | |
c8852121 | 528 | r = -errno; |
d5970055 MT |
529 | goto fail_call; |
530 | } | |
531 | ||
532 | return 0; | |
533 | ||
534 | fail_call: | |
535 | fail_kick: | |
536 | vdev->binding->set_host_notifier(vdev->binding_opaque, idx, false); | |
537 | fail_host_notifier: | |
d5970055 MT |
538 | fail_alloc: |
539 | cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx), | |
540 | 0, 0); | |
541 | fail_alloc_ring: | |
542 | cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx), | |
543 | 0, 0); | |
544 | fail_alloc_used: | |
545 | cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx), | |
546 | 0, 0); | |
547 | fail_alloc_avail: | |
548 | cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx), | |
549 | 0, 0); | |
550 | fail_alloc_desc: | |
551 | return r; | |
552 | } | |
553 | ||
554 | static void vhost_virtqueue_cleanup(struct vhost_dev *dev, | |
555 | struct VirtIODevice *vdev, | |
556 | struct vhost_virtqueue *vq, | |
557 | unsigned idx) | |
558 | { | |
559 | struct vhost_vring_state state = { | |
560 | .index = idx, | |
561 | }; | |
562 | int r; | |
d5970055 MT |
563 | r = vdev->binding->set_host_notifier(vdev->binding_opaque, idx, false); |
564 | if (r < 0) { | |
565 | fprintf(stderr, "vhost VQ %d host cleanup failed: %d\n", idx, r); | |
566 | fflush(stderr); | |
567 | } | |
568 | assert (r >= 0); | |
569 | r = ioctl(dev->control, VHOST_GET_VRING_BASE, &state); | |
570 | if (r < 0) { | |
571 | fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r); | |
572 | fflush(stderr); | |
573 | } | |
574 | virtio_queue_set_last_avail_idx(vdev, idx, state.num); | |
575 | assert (r >= 0); | |
576 | cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx), | |
577 | 0, virtio_queue_get_ring_size(vdev, idx)); | |
578 | cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx), | |
579 | 1, virtio_queue_get_used_size(vdev, idx)); | |
580 | cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx), | |
581 | 0, virtio_queue_get_avail_size(vdev, idx)); | |
582 | cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx), | |
583 | 0, virtio_queue_get_desc_size(vdev, idx)); | |
584 | } | |
585 | ||
5430a28f | 586 | int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) |
d5970055 MT |
587 | { |
588 | uint64_t features; | |
589 | int r; | |
590 | if (devfd >= 0) { | |
591 | hdev->control = devfd; | |
592 | } else { | |
593 | hdev->control = open("/dev/vhost-net", O_RDWR); | |
594 | if (hdev->control < 0) { | |
595 | return -errno; | |
596 | } | |
597 | } | |
598 | r = ioctl(hdev->control, VHOST_SET_OWNER, NULL); | |
599 | if (r < 0) { | |
600 | goto fail; | |
601 | } | |
602 | ||
603 | r = ioctl(hdev->control, VHOST_GET_FEATURES, &features); | |
604 | if (r < 0) { | |
605 | goto fail; | |
606 | } | |
607 | hdev->features = features; | |
608 | ||
609 | hdev->client.set_memory = vhost_client_set_memory; | |
610 | hdev->client.sync_dirty_bitmap = vhost_client_sync_dirty_bitmap; | |
611 | hdev->client.migration_log = vhost_client_migration_log; | |
e5896b12 AP |
612 | hdev->client.log_start = NULL; |
613 | hdev->client.log_stop = NULL; | |
d5970055 MT |
614 | hdev->mem = qemu_mallocz(offsetof(struct vhost_memory, regions)); |
615 | hdev->log = NULL; | |
616 | hdev->log_size = 0; | |
617 | hdev->log_enabled = false; | |
618 | hdev->started = false; | |
619 | cpu_register_phys_memory_client(&hdev->client); | |
5430a28f | 620 | hdev->force = force; |
d5970055 MT |
621 | return 0; |
622 | fail: | |
623 | r = -errno; | |
624 | close(hdev->control); | |
625 | return r; | |
626 | } | |
627 | ||
628 | void vhost_dev_cleanup(struct vhost_dev *hdev) | |
629 | { | |
630 | cpu_unregister_phys_memory_client(&hdev->client); | |
631 | qemu_free(hdev->mem); | |
632 | close(hdev->control); | |
633 | } | |
634 | ||
5430a28f | 635 | bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev) |
636 | { | |
637 | return !vdev->binding->query_guest_notifiers || | |
638 | vdev->binding->query_guest_notifiers(vdev->binding_opaque) || | |
639 | hdev->force; | |
640 | } | |
641 | ||
d5970055 MT |
642 | int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) |
643 | { | |
644 | int i, r; | |
54dd9321 MT |
645 | if (!vdev->binding->set_guest_notifiers) { |
646 | fprintf(stderr, "binding does not support guest notifiers\n"); | |
647 | r = -ENOSYS; | |
648 | goto fail; | |
649 | } | |
650 | ||
651 | r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, true); | |
652 | if (r < 0) { | |
653 | fprintf(stderr, "Error binding guest notifier: %d\n", -r); | |
654 | goto fail_notifiers; | |
655 | } | |
d5970055 MT |
656 | |
657 | r = vhost_dev_set_features(hdev, hdev->log_enabled); | |
658 | if (r < 0) { | |
54dd9321 | 659 | goto fail_features; |
d5970055 MT |
660 | } |
661 | r = ioctl(hdev->control, VHOST_SET_MEM_TABLE, hdev->mem); | |
662 | if (r < 0) { | |
663 | r = -errno; | |
54dd9321 | 664 | goto fail_mem; |
d5970055 | 665 | } |
d154e0ba MT |
666 | for (i = 0; i < hdev->nvqs; ++i) { |
667 | r = vhost_virtqueue_init(hdev, | |
668 | vdev, | |
669 | hdev->vqs + i, | |
670 | i); | |
671 | if (r < 0) { | |
672 | goto fail_vq; | |
673 | } | |
674 | } | |
675 | ||
d5970055 MT |
676 | if (hdev->log_enabled) { |
677 | hdev->log_size = vhost_get_log_size(hdev); | |
678 | hdev->log = hdev->log_size ? | |
679 | qemu_mallocz(hdev->log_size * sizeof *hdev->log) : NULL; | |
680 | r = ioctl(hdev->control, VHOST_SET_LOG_BASE, | |
681 | (uint64_t)(unsigned long)hdev->log); | |
682 | if (r < 0) { | |
683 | r = -errno; | |
54dd9321 | 684 | goto fail_log; |
d5970055 MT |
685 | } |
686 | } | |
d154e0ba | 687 | |
d5970055 MT |
688 | hdev->started = true; |
689 | ||
690 | return 0; | |
54dd9321 | 691 | fail_log: |
d5970055 MT |
692 | fail_vq: |
693 | while (--i >= 0) { | |
694 | vhost_virtqueue_cleanup(hdev, | |
695 | vdev, | |
696 | hdev->vqs + i, | |
697 | i); | |
698 | } | |
54dd9321 MT |
699 | fail_mem: |
700 | fail_features: | |
701 | vdev->binding->set_guest_notifiers(vdev->binding_opaque, false); | |
702 | fail_notifiers: | |
d5970055 MT |
703 | fail: |
704 | return r; | |
705 | } | |
706 | ||
707 | void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev) | |
708 | { | |
54dd9321 MT |
709 | int i, r; |
710 | ||
d5970055 MT |
711 | for (i = 0; i < hdev->nvqs; ++i) { |
712 | vhost_virtqueue_cleanup(hdev, | |
713 | vdev, | |
714 | hdev->vqs + i, | |
715 | i); | |
716 | } | |
717 | vhost_client_sync_dirty_bitmap(&hdev->client, 0, | |
718 | (target_phys_addr_t)~0x0ull); | |
54dd9321 MT |
719 | r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, false); |
720 | if (r < 0) { | |
721 | fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", r); | |
722 | fflush(stderr); | |
723 | } | |
724 | assert (r >= 0); | |
725 | ||
d5970055 MT |
726 | hdev->started = false; |
727 | qemu_free(hdev->log); | |
728 | hdev->log_size = 0; | |
729 | } |