]>
Commit | Line | Data |
---|---|---|
e2c7d025 EA |
1 | /* |
2 | * generic functions used by VFIO devices | |
3 | * | |
4 | * Copyright Red Hat, Inc. 2012 | |
5 | * | |
6 | * Authors: | |
7 | * Alex Williamson <alex.williamson@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | * | |
12 | * Based on qemu-kvm device-assignment: | |
13 | * Adapted for KVM by Qumranet. | |
14 | * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) | |
15 | * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) | |
16 | * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) | |
17 | * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) | |
18 | * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) | |
19 | */ | |
20 | ||
c6eacb1a | 21 | #include "qemu/osdep.h" |
e2c7d025 EA |
22 | #include <sys/ioctl.h> |
23 | #include <sys/mman.h> | |
24 | #include <linux/vfio.h> | |
25 | ||
26 | #include "hw/vfio/vfio-common.h" | |
27 | #include "hw/vfio/vfio.h" | |
28 | #include "exec/address-spaces.h" | |
29 | #include "exec/memory.h" | |
30 | #include "hw/hw.h" | |
31 | #include "qemu/error-report.h" | |
32 | #include "sysemu/kvm.h" | |
e81096b1 PB |
33 | #ifdef CONFIG_KVM |
34 | #include "linux/kvm.h" | |
35 | #endif | |
e2c7d025 EA |
36 | #include "trace.h" |
37 | ||
38 | struct vfio_group_head vfio_group_list = | |
39cb514f | 39 | QLIST_HEAD_INITIALIZER(vfio_group_list); |
e2c7d025 EA |
40 | struct vfio_as_head vfio_address_spaces = |
41 | QLIST_HEAD_INITIALIZER(vfio_address_spaces); | |
42 | ||
43 | #ifdef CONFIG_KVM | |
44 | /* | |
45 | * We have a single VFIO pseudo device per KVM VM. Once created it lives | |
46 | * for the life of the VM. Closing the file descriptor only drops our | |
47 | * reference to it and the device's reference to kvm. Therefore once | |
48 | * initialized, this file descriptor is only released on QEMU exit and | |
49 | * we'll re-use it should another vfio device be attached before then. | |
50 | */ | |
51 | static int vfio_kvm_device_fd = -1; | |
52 | #endif | |
53 | ||
54 | /* | |
55 | * Common VFIO interrupt disable | |
56 | */ | |
57 | void vfio_disable_irqindex(VFIODevice *vbasedev, int index) | |
58 | { | |
59 | struct vfio_irq_set irq_set = { | |
60 | .argsz = sizeof(irq_set), | |
61 | .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, | |
62 | .index = index, | |
63 | .start = 0, | |
64 | .count = 0, | |
65 | }; | |
66 | ||
67 | ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); | |
68 | } | |
69 | ||
70 | void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index) | |
71 | { | |
72 | struct vfio_irq_set irq_set = { | |
73 | .argsz = sizeof(irq_set), | |
74 | .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, | |
75 | .index = index, | |
76 | .start = 0, | |
77 | .count = 1, | |
78 | }; | |
79 | ||
80 | ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); | |
81 | } | |
82 | ||
83 | void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index) | |
84 | { | |
85 | struct vfio_irq_set irq_set = { | |
86 | .argsz = sizeof(irq_set), | |
87 | .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK, | |
88 | .index = index, | |
89 | .start = 0, | |
90 | .count = 1, | |
91 | }; | |
92 | ||
93 | ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); | |
94 | } | |
95 | ||
96 | /* | |
97 | * IO Port/MMIO - Beware of the endians, VFIO is always little endian | |
98 | */ | |
99 | void vfio_region_write(void *opaque, hwaddr addr, | |
100 | uint64_t data, unsigned size) | |
101 | { | |
102 | VFIORegion *region = opaque; | |
103 | VFIODevice *vbasedev = region->vbasedev; | |
104 | union { | |
105 | uint8_t byte; | |
106 | uint16_t word; | |
107 | uint32_t dword; | |
108 | uint64_t qword; | |
109 | } buf; | |
110 | ||
111 | switch (size) { | |
112 | case 1: | |
113 | buf.byte = data; | |
114 | break; | |
115 | case 2: | |
116 | buf.word = cpu_to_le16(data); | |
117 | break; | |
118 | case 4: | |
119 | buf.dword = cpu_to_le32(data); | |
120 | break; | |
121 | default: | |
122 | hw_error("vfio: unsupported write size, %d bytes", size); | |
123 | break; | |
124 | } | |
125 | ||
126 | if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { | |
127 | error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64 | |
128 | ",%d) failed: %m", | |
129 | __func__, vbasedev->name, region->nr, | |
130 | addr, data, size); | |
131 | } | |
132 | ||
133 | trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size); | |
134 | ||
135 | /* | |
136 | * A read or write to a BAR always signals an INTx EOI. This will | |
137 | * do nothing if not pending (including not in INTx mode). We assume | |
138 | * that a BAR access is in response to an interrupt and that BAR | |
139 | * accesses will service the interrupt. Unfortunately, we don't know | |
140 | * which access will service the interrupt, so we're potentially | |
141 | * getting quite a few host interrupts per guest interrupt. | |
142 | */ | |
143 | vbasedev->ops->vfio_eoi(vbasedev); | |
144 | } | |
145 | ||
146 | uint64_t vfio_region_read(void *opaque, | |
147 | hwaddr addr, unsigned size) | |
148 | { | |
149 | VFIORegion *region = opaque; | |
150 | VFIODevice *vbasedev = region->vbasedev; | |
151 | union { | |
152 | uint8_t byte; | |
153 | uint16_t word; | |
154 | uint32_t dword; | |
155 | uint64_t qword; | |
156 | } buf; | |
157 | uint64_t data = 0; | |
158 | ||
159 | if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { | |
160 | error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m", | |
161 | __func__, vbasedev->name, region->nr, | |
162 | addr, size); | |
163 | return (uint64_t)-1; | |
164 | } | |
165 | switch (size) { | |
166 | case 1: | |
167 | data = buf.byte; | |
168 | break; | |
169 | case 2: | |
170 | data = le16_to_cpu(buf.word); | |
171 | break; | |
172 | case 4: | |
173 | data = le32_to_cpu(buf.dword); | |
174 | break; | |
175 | default: | |
176 | hw_error("vfio: unsupported read size, %d bytes", size); | |
177 | break; | |
178 | } | |
179 | ||
180 | trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data); | |
181 | ||
182 | /* Same as write above */ | |
183 | vbasedev->ops->vfio_eoi(vbasedev); | |
184 | ||
185 | return data; | |
186 | } | |
187 | ||
188 | const MemoryRegionOps vfio_region_ops = { | |
189 | .read = vfio_region_read, | |
190 | .write = vfio_region_write, | |
191 | .endianness = DEVICE_LITTLE_ENDIAN, | |
192 | }; | |
193 | ||
194 | /* | |
195 | * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 | |
196 | */ | |
197 | static int vfio_dma_unmap(VFIOContainer *container, | |
198 | hwaddr iova, ram_addr_t size) | |
199 | { | |
200 | struct vfio_iommu_type1_dma_unmap unmap = { | |
201 | .argsz = sizeof(unmap), | |
202 | .flags = 0, | |
203 | .iova = iova, | |
204 | .size = size, | |
205 | }; | |
206 | ||
207 | if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { | |
78e5b17f | 208 | error_report("VFIO_UNMAP_DMA: %d", -errno); |
e2c7d025 EA |
209 | return -errno; |
210 | } | |
211 | ||
212 | return 0; | |
213 | } | |
214 | ||
215 | static int vfio_dma_map(VFIOContainer *container, hwaddr iova, | |
216 | ram_addr_t size, void *vaddr, bool readonly) | |
217 | { | |
218 | struct vfio_iommu_type1_dma_map map = { | |
219 | .argsz = sizeof(map), | |
220 | .flags = VFIO_DMA_MAP_FLAG_READ, | |
221 | .vaddr = (__u64)(uintptr_t)vaddr, | |
222 | .iova = iova, | |
223 | .size = size, | |
224 | }; | |
225 | ||
226 | if (!readonly) { | |
227 | map.flags |= VFIO_DMA_MAP_FLAG_WRITE; | |
228 | } | |
229 | ||
230 | /* | |
231 | * Try the mapping, if it fails with EBUSY, unmap the region and try | |
232 | * again. This shouldn't be necessary, but we sometimes see it in | |
b6af0975 | 233 | * the VGA ROM space. |
e2c7d025 EA |
234 | */ |
235 | if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 || | |
236 | (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 && | |
237 | ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) { | |
238 | return 0; | |
239 | } | |
240 | ||
78e5b17f | 241 | error_report("VFIO_MAP_DMA: %d", -errno); |
e2c7d025 EA |
242 | return -errno; |
243 | } | |
244 | ||
245 | static bool vfio_listener_skipped_section(MemoryRegionSection *section) | |
246 | { | |
247 | return (!memory_region_is_ram(section->mr) && | |
248 | !memory_region_is_iommu(section->mr)) || | |
249 | /* | |
250 | * Sizing an enabled 64-bit BAR can cause spurious mappings to | |
251 | * addresses in the upper part of the 64-bit address space. These | |
252 | * are never accessed by the CPU and beyond the address width of | |
253 | * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. | |
254 | */ | |
255 | section->offset_within_address_space & (1ULL << 63); | |
256 | } | |
257 | ||
258 | static void vfio_iommu_map_notify(Notifier *n, void *data) | |
259 | { | |
260 | VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); | |
261 | VFIOContainer *container = giommu->container; | |
262 | IOMMUTLBEntry *iotlb = data; | |
263 | MemoryRegion *mr; | |
264 | hwaddr xlat; | |
265 | hwaddr len = iotlb->addr_mask + 1; | |
266 | void *vaddr; | |
267 | int ret; | |
268 | ||
269 | trace_vfio_iommu_map_notify(iotlb->iova, | |
270 | iotlb->iova + iotlb->addr_mask); | |
271 | ||
272 | /* | |
273 | * The IOMMU TLB entry we have just covers translation through | |
274 | * this IOMMU to its immediate target. We need to translate | |
275 | * it the rest of the way through to memory. | |
276 | */ | |
41063e1e | 277 | rcu_read_lock(); |
e2c7d025 EA |
278 | mr = address_space_translate(&address_space_memory, |
279 | iotlb->translated_addr, | |
280 | &xlat, &len, iotlb->perm & IOMMU_WO); | |
281 | if (!memory_region_is_ram(mr)) { | |
78e5b17f | 282 | error_report("iommu map to non memory area %"HWADDR_PRIx"", |
e2c7d025 | 283 | xlat); |
41063e1e | 284 | goto out; |
e2c7d025 EA |
285 | } |
286 | /* | |
287 | * Translation truncates length to the IOMMU page size, | |
288 | * check that it did not truncate too much. | |
289 | */ | |
290 | if (len & iotlb->addr_mask) { | |
78e5b17f | 291 | error_report("iommu has granularity incompatible with target AS"); |
41063e1e | 292 | goto out; |
e2c7d025 EA |
293 | } |
294 | ||
295 | if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { | |
296 | vaddr = memory_region_get_ram_ptr(mr) + xlat; | |
297 | ret = vfio_dma_map(container, iotlb->iova, | |
298 | iotlb->addr_mask + 1, vaddr, | |
299 | !(iotlb->perm & IOMMU_WO) || mr->readonly); | |
300 | if (ret) { | |
301 | error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " | |
302 | "0x%"HWADDR_PRIx", %p) = %d (%m)", | |
303 | container, iotlb->iova, | |
304 | iotlb->addr_mask + 1, vaddr, ret); | |
305 | } | |
306 | } else { | |
307 | ret = vfio_dma_unmap(container, iotlb->iova, iotlb->addr_mask + 1); | |
308 | if (ret) { | |
309 | error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " | |
310 | "0x%"HWADDR_PRIx") = %d (%m)", | |
311 | container, iotlb->iova, | |
312 | iotlb->addr_mask + 1, ret); | |
313 | } | |
314 | } | |
41063e1e PB |
315 | out: |
316 | rcu_read_unlock(); | |
e2c7d025 EA |
317 | } |
318 | ||
508ce5eb DG |
319 | static hwaddr vfio_container_granularity(VFIOContainer *container) |
320 | { | |
321 | return (hwaddr)1 << ctz64(container->iova_pgsizes); | |
322 | } | |
323 | ||
e2c7d025 EA |
324 | static void vfio_listener_region_add(MemoryListener *listener, |
325 | MemoryRegionSection *section) | |
326 | { | |
ee0bf0e5 | 327 | VFIOContainer *container = container_of(listener, VFIOContainer, listener); |
e2c7d025 | 328 | hwaddr iova, end; |
55efcc53 | 329 | Int128 llend, llsize; |
e2c7d025 EA |
330 | void *vaddr; |
331 | int ret; | |
332 | ||
333 | if (vfio_listener_skipped_section(section)) { | |
334 | trace_vfio_listener_region_add_skip( | |
335 | section->offset_within_address_space, | |
336 | section->offset_within_address_space + | |
337 | int128_get64(int128_sub(section->size, int128_one()))); | |
338 | return; | |
339 | } | |
340 | ||
341 | if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != | |
342 | (section->offset_within_region & ~TARGET_PAGE_MASK))) { | |
343 | error_report("%s received unaligned region", __func__); | |
344 | return; | |
345 | } | |
346 | ||
347 | iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); | |
348 | llend = int128_make64(section->offset_within_address_space); | |
349 | llend = int128_add(llend, section->size); | |
350 | llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); | |
351 | ||
352 | if (int128_ge(int128_make64(iova), llend)) { | |
353 | return; | |
354 | } | |
55efcc53 | 355 | end = int128_get64(int128_sub(llend, int128_one())); |
3898aad3 | 356 | |
55efcc53 | 357 | if ((iova < container->min_iova) || (end > container->max_iova)) { |
3898aad3 DG |
358 | error_report("vfio: IOMMU container %p can't map guest IOVA region" |
359 | " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, | |
55efcc53 | 360 | container, iova, end); |
3898aad3 DG |
361 | ret = -EFAULT; |
362 | goto fail; | |
363 | } | |
e2c7d025 EA |
364 | |
365 | memory_region_ref(section->mr); | |
366 | ||
367 | if (memory_region_is_iommu(section->mr)) { | |
368 | VFIOGuestIOMMU *giommu; | |
369 | ||
55efcc53 | 370 | trace_vfio_listener_region_add_iommu(iova, end); |
e2c7d025 EA |
371 | /* |
372 | * FIXME: We should do some checking to see if the | |
373 | * capabilities of the host VFIO IOMMU are adequate to model | |
374 | * the guest IOMMU | |
375 | * | |
376 | * FIXME: For VFIO iommu types which have KVM acceleration to | |
377 | * avoid bouncing all map/unmaps through qemu this way, this | |
378 | * would be the right place to wire that up (tell the KVM | |
379 | * device emulation the VFIO iommu handles to use). | |
380 | */ | |
e2c7d025 EA |
381 | giommu = g_malloc0(sizeof(*giommu)); |
382 | giommu->iommu = section->mr; | |
383 | giommu->container = container; | |
384 | giommu->n.notify = vfio_iommu_map_notify; | |
385 | QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); | |
508ce5eb | 386 | |
e2c7d025 | 387 | memory_region_register_iommu_notifier(giommu->iommu, &giommu->n); |
508ce5eb DG |
388 | memory_region_iommu_replay(giommu->iommu, &giommu->n, |
389 | vfio_container_granularity(container), | |
390 | false); | |
e2c7d025 EA |
391 | |
392 | return; | |
393 | } | |
394 | ||
395 | /* Here we assume that memory_region_is_ram(section->mr)==true */ | |
396 | ||
e2c7d025 EA |
397 | vaddr = memory_region_get_ram_ptr(section->mr) + |
398 | section->offset_within_region + | |
399 | (iova - section->offset_within_address_space); | |
400 | ||
55efcc53 | 401 | trace_vfio_listener_region_add_ram(iova, end, vaddr); |
e2c7d025 | 402 | |
55efcc53 BD |
403 | llsize = int128_sub(llend, int128_make64(iova)); |
404 | ||
405 | ret = vfio_dma_map(container, iova, int128_get64(llsize), | |
406 | vaddr, section->readonly); | |
e2c7d025 EA |
407 | if (ret) { |
408 | error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " | |
409 | "0x%"HWADDR_PRIx", %p) = %d (%m)", | |
55efcc53 | 410 | container, iova, int128_get64(llsize), vaddr, ret); |
ac6dc389 DG |
411 | goto fail; |
412 | } | |
e2c7d025 | 413 | |
ac6dc389 DG |
414 | return; |
415 | ||
416 | fail: | |
417 | /* | |
418 | * On the initfn path, store the first error in the container so we | |
419 | * can gracefully fail. Runtime, there's not much we can do other | |
420 | * than throw a hardware error. | |
421 | */ | |
422 | if (!container->initialized) { | |
423 | if (!container->error) { | |
424 | container->error = ret; | |
e2c7d025 | 425 | } |
ac6dc389 DG |
426 | } else { |
427 | hw_error("vfio: DMA mapping failed, unable to continue"); | |
e2c7d025 EA |
428 | } |
429 | } | |
430 | ||
431 | static void vfio_listener_region_del(MemoryListener *listener, | |
432 | MemoryRegionSection *section) | |
433 | { | |
ee0bf0e5 | 434 | VFIOContainer *container = container_of(listener, VFIOContainer, listener); |
e2c7d025 | 435 | hwaddr iova, end; |
7a057b4f | 436 | Int128 llend, llsize; |
e2c7d025 EA |
437 | int ret; |
438 | ||
439 | if (vfio_listener_skipped_section(section)) { | |
440 | trace_vfio_listener_region_del_skip( | |
441 | section->offset_within_address_space, | |
442 | section->offset_within_address_space + | |
443 | int128_get64(int128_sub(section->size, int128_one()))); | |
444 | return; | |
445 | } | |
446 | ||
447 | if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != | |
448 | (section->offset_within_region & ~TARGET_PAGE_MASK))) { | |
449 | error_report("%s received unaligned region", __func__); | |
450 | return; | |
451 | } | |
452 | ||
453 | if (memory_region_is_iommu(section->mr)) { | |
454 | VFIOGuestIOMMU *giommu; | |
455 | ||
456 | QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { | |
457 | if (giommu->iommu == section->mr) { | |
458 | memory_region_unregister_iommu_notifier(&giommu->n); | |
459 | QLIST_REMOVE(giommu, giommu_next); | |
460 | g_free(giommu); | |
461 | break; | |
462 | } | |
463 | } | |
464 | ||
465 | /* | |
466 | * FIXME: We assume the one big unmap below is adequate to | |
467 | * remove any individual page mappings in the IOMMU which | |
468 | * might have been copied into VFIO. This works for a page table | |
469 | * based IOMMU where a big unmap flattens a large range of IO-PTEs. | |
470 | * That may not be true for all IOMMU types. | |
471 | */ | |
472 | } | |
473 | ||
474 | iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); | |
7a057b4f AK |
475 | llend = int128_make64(section->offset_within_address_space); |
476 | llend = int128_add(llend, section->size); | |
477 | llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); | |
e2c7d025 | 478 | |
7a057b4f | 479 | if (int128_ge(int128_make64(iova), llend)) { |
e2c7d025 EA |
480 | return; |
481 | } | |
7a057b4f AK |
482 | end = int128_get64(int128_sub(llend, int128_one())); |
483 | ||
484 | llsize = int128_sub(llend, int128_make64(iova)); | |
e2c7d025 | 485 | |
7a057b4f | 486 | trace_vfio_listener_region_del(iova, end); |
e2c7d025 | 487 | |
7a057b4f | 488 | ret = vfio_dma_unmap(container, iova, int128_get64(llsize)); |
e2c7d025 EA |
489 | memory_region_unref(section->mr); |
490 | if (ret) { | |
491 | error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " | |
492 | "0x%"HWADDR_PRIx") = %d (%m)", | |
7a057b4f | 493 | container, iova, int128_get64(llsize), ret); |
e2c7d025 EA |
494 | } |
495 | } | |
496 | ||
51b833f4 | 497 | static const MemoryListener vfio_memory_listener = { |
e2c7d025 EA |
498 | .region_add = vfio_listener_region_add, |
499 | .region_del = vfio_listener_region_del, | |
500 | }; | |
501 | ||
51b833f4 | 502 | static void vfio_listener_release(VFIOContainer *container) |
e2c7d025 | 503 | { |
ee0bf0e5 | 504 | memory_listener_unregister(&container->listener); |
e2c7d025 EA |
505 | } |
506 | ||
b53b0f69 AW |
507 | static struct vfio_info_cap_header * |
508 | vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id) | |
509 | { | |
510 | struct vfio_info_cap_header *hdr; | |
511 | void *ptr = info; | |
512 | ||
513 | if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) { | |
514 | return NULL; | |
515 | } | |
516 | ||
517 | for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) { | |
518 | if (hdr->id == id) { | |
519 | return hdr; | |
520 | } | |
521 | } | |
522 | ||
523 | return NULL; | |
524 | } | |
525 | ||
526 | static void vfio_setup_region_sparse_mmaps(VFIORegion *region, | |
527 | struct vfio_region_info *info) | |
528 | { | |
529 | struct vfio_info_cap_header *hdr; | |
530 | struct vfio_region_info_cap_sparse_mmap *sparse; | |
531 | int i; | |
532 | ||
533 | hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); | |
534 | if (!hdr) { | |
535 | return; | |
536 | } | |
537 | ||
538 | sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header); | |
539 | ||
540 | trace_vfio_region_sparse_mmap_header(region->vbasedev->name, | |
541 | region->nr, sparse->nr_areas); | |
542 | ||
543 | region->nr_mmaps = sparse->nr_areas; | |
544 | region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); | |
545 | ||
546 | for (i = 0; i < region->nr_mmaps; i++) { | |
547 | region->mmaps[i].offset = sparse->areas[i].offset; | |
548 | region->mmaps[i].size = sparse->areas[i].size; | |
549 | trace_vfio_region_sparse_mmap_entry(i, region->mmaps[i].offset, | |
550 | region->mmaps[i].offset + | |
551 | region->mmaps[i].size); | |
552 | } | |
553 | } | |
554 | ||
db0da029 AW |
555 | int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, |
556 | int index, const char *name) | |
e2c7d025 | 557 | { |
db0da029 AW |
558 | struct vfio_region_info *info; |
559 | int ret; | |
560 | ||
561 | ret = vfio_get_region_info(vbasedev, index, &info); | |
562 | if (ret) { | |
563 | return ret; | |
564 | } | |
565 | ||
566 | region->vbasedev = vbasedev; | |
567 | region->flags = info->flags; | |
568 | region->size = info->size; | |
569 | region->fd_offset = info->offset; | |
570 | region->nr = index; | |
571 | ||
572 | if (region->size) { | |
573 | region->mem = g_new0(MemoryRegion, 1); | |
574 | memory_region_init_io(region->mem, obj, &vfio_region_ops, | |
575 | region, name, region->size); | |
e2c7d025 | 576 | |
db0da029 AW |
577 | if (!vbasedev->no_mmap && |
578 | region->flags & VFIO_REGION_INFO_FLAG_MMAP && | |
579 | !(region->size & ~qemu_real_host_page_mask)) { | |
e2c7d025 | 580 | |
b53b0f69 | 581 | vfio_setup_region_sparse_mmaps(region, info); |
db0da029 | 582 | |
b53b0f69 AW |
583 | if (!region->nr_mmaps) { |
584 | region->nr_mmaps = 1; | |
585 | region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); | |
586 | region->mmaps[0].offset = 0; | |
587 | region->mmaps[0].size = region->size; | |
588 | } | |
e2c7d025 | 589 | } |
db0da029 AW |
590 | } |
591 | ||
592 | g_free(info); | |
593 | ||
594 | trace_vfio_region_setup(vbasedev->name, index, name, | |
595 | region->flags, region->fd_offset, region->size); | |
596 | return 0; | |
597 | } | |
e2c7d025 | 598 | |
db0da029 AW |
599 | int vfio_region_mmap(VFIORegion *region) |
600 | { | |
601 | int i, prot = 0; | |
602 | char *name; | |
603 | ||
604 | if (!region->mem) { | |
605 | return 0; | |
606 | } | |
607 | ||
608 | prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; | |
609 | prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; | |
610 | ||
611 | for (i = 0; i < region->nr_mmaps; i++) { | |
612 | region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot, | |
613 | MAP_SHARED, region->vbasedev->fd, | |
614 | region->fd_offset + | |
615 | region->mmaps[i].offset); | |
616 | if (region->mmaps[i].mmap == MAP_FAILED) { | |
617 | int ret = -errno; | |
618 | ||
619 | trace_vfio_region_mmap_fault(memory_region_name(region->mem), i, | |
620 | region->fd_offset + | |
621 | region->mmaps[i].offset, | |
622 | region->fd_offset + | |
623 | region->mmaps[i].offset + | |
624 | region->mmaps[i].size - 1, ret); | |
625 | ||
626 | region->mmaps[i].mmap = NULL; | |
627 | ||
628 | for (i--; i >= 0; i--) { | |
629 | memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); | |
630 | munmap(region->mmaps[i].mmap, region->mmaps[i].size); | |
631 | object_unparent(OBJECT(®ion->mmaps[i].mem)); | |
632 | region->mmaps[i].mmap = NULL; | |
633 | } | |
634 | ||
635 | return ret; | |
e2c7d025 EA |
636 | } |
637 | ||
db0da029 AW |
638 | name = g_strdup_printf("%s mmaps[%d]", |
639 | memory_region_name(region->mem), i); | |
640 | memory_region_init_ram_ptr(®ion->mmaps[i].mem, | |
641 | memory_region_owner(region->mem), | |
642 | name, region->mmaps[i].size, | |
643 | region->mmaps[i].mmap); | |
644 | g_free(name); | |
645 | memory_region_set_skip_dump(®ion->mmaps[i].mem); | |
646 | memory_region_add_subregion(region->mem, region->mmaps[i].offset, | |
647 | ®ion->mmaps[i].mem); | |
648 | ||
649 | trace_vfio_region_mmap(memory_region_name(®ion->mmaps[i].mem), | |
650 | region->mmaps[i].offset, | |
651 | region->mmaps[i].offset + | |
652 | region->mmaps[i].size - 1); | |
653 | } | |
654 | ||
655 | return 0; | |
656 | } | |
657 | ||
658 | void vfio_region_exit(VFIORegion *region) | |
659 | { | |
660 | int i; | |
661 | ||
662 | if (!region->mem) { | |
663 | return; | |
664 | } | |
665 | ||
666 | for (i = 0; i < region->nr_mmaps; i++) { | |
667 | if (region->mmaps[i].mmap) { | |
668 | memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); | |
e2c7d025 | 669 | } |
db0da029 | 670 | } |
e2c7d025 | 671 | |
db0da029 AW |
672 | trace_vfio_region_exit(region->vbasedev->name, region->nr); |
673 | } | |
674 | ||
675 | void vfio_region_finalize(VFIORegion *region) | |
676 | { | |
677 | int i; | |
678 | ||
679 | if (!region->mem) { | |
680 | return; | |
e2c7d025 EA |
681 | } |
682 | ||
db0da029 AW |
683 | for (i = 0; i < region->nr_mmaps; i++) { |
684 | if (region->mmaps[i].mmap) { | |
685 | munmap(region->mmaps[i].mmap, region->mmaps[i].size); | |
686 | object_unparent(OBJECT(®ion->mmaps[i].mem)); | |
687 | } | |
688 | } | |
689 | ||
690 | object_unparent(OBJECT(region->mem)); | |
691 | ||
692 | g_free(region->mem); | |
693 | g_free(region->mmaps); | |
694 | ||
695 | trace_vfio_region_finalize(region->vbasedev->name, region->nr); | |
696 | } | |
697 | ||
698 | void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled) | |
699 | { | |
700 | int i; | |
701 | ||
702 | if (!region->mem) { | |
703 | return; | |
704 | } | |
705 | ||
706 | for (i = 0; i < region->nr_mmaps; i++) { | |
707 | if (region->mmaps[i].mmap) { | |
708 | memory_region_set_enabled(®ion->mmaps[i].mem, enabled); | |
709 | } | |
710 | } | |
e2c7d025 | 711 | |
db0da029 AW |
712 | trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem), |
713 | enabled); | |
e2c7d025 EA |
714 | } |
715 | ||
716 | void vfio_reset_handler(void *opaque) | |
717 | { | |
718 | VFIOGroup *group; | |
719 | VFIODevice *vbasedev; | |
720 | ||
721 | QLIST_FOREACH(group, &vfio_group_list, next) { | |
722 | QLIST_FOREACH(vbasedev, &group->device_list, next) { | |
723 | vbasedev->ops->vfio_compute_needs_reset(vbasedev); | |
724 | } | |
725 | } | |
726 | ||
727 | QLIST_FOREACH(group, &vfio_group_list, next) { | |
728 | QLIST_FOREACH(vbasedev, &group->device_list, next) { | |
729 | if (vbasedev->needs_reset) { | |
730 | vbasedev->ops->vfio_hot_reset_multi(vbasedev); | |
731 | } | |
732 | } | |
733 | } | |
734 | } | |
735 | ||
736 | static void vfio_kvm_device_add_group(VFIOGroup *group) | |
737 | { | |
738 | #ifdef CONFIG_KVM | |
739 | struct kvm_device_attr attr = { | |
740 | .group = KVM_DEV_VFIO_GROUP, | |
741 | .attr = KVM_DEV_VFIO_GROUP_ADD, | |
742 | .addr = (uint64_t)(unsigned long)&group->fd, | |
743 | }; | |
744 | ||
745 | if (!kvm_enabled()) { | |
746 | return; | |
747 | } | |
748 | ||
749 | if (vfio_kvm_device_fd < 0) { | |
750 | struct kvm_create_device cd = { | |
751 | .type = KVM_DEV_TYPE_VFIO, | |
752 | }; | |
753 | ||
754 | if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { | |
78e5b17f | 755 | error_report("Failed to create KVM VFIO device: %m"); |
e2c7d025 EA |
756 | return; |
757 | } | |
758 | ||
759 | vfio_kvm_device_fd = cd.fd; | |
760 | } | |
761 | ||
762 | if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { | |
763 | error_report("Failed to add group %d to KVM VFIO device: %m", | |
764 | group->groupid); | |
765 | } | |
766 | #endif | |
767 | } | |
768 | ||
769 | static void vfio_kvm_device_del_group(VFIOGroup *group) | |
770 | { | |
771 | #ifdef CONFIG_KVM | |
772 | struct kvm_device_attr attr = { | |
773 | .group = KVM_DEV_VFIO_GROUP, | |
774 | .attr = KVM_DEV_VFIO_GROUP_DEL, | |
775 | .addr = (uint64_t)(unsigned long)&group->fd, | |
776 | }; | |
777 | ||
778 | if (vfio_kvm_device_fd < 0) { | |
779 | return; | |
780 | } | |
781 | ||
782 | if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { | |
783 | error_report("Failed to remove group %d from KVM VFIO device: %m", | |
784 | group->groupid); | |
785 | } | |
786 | #endif | |
787 | } | |
788 | ||
789 | static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) | |
790 | { | |
791 | VFIOAddressSpace *space; | |
792 | ||
793 | QLIST_FOREACH(space, &vfio_address_spaces, list) { | |
794 | if (space->as == as) { | |
795 | return space; | |
796 | } | |
797 | } | |
798 | ||
799 | /* No suitable VFIOAddressSpace, create a new one */ | |
800 | space = g_malloc0(sizeof(*space)); | |
801 | space->as = as; | |
802 | QLIST_INIT(&space->containers); | |
803 | ||
804 | QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); | |
805 | ||
806 | return space; | |
807 | } | |
808 | ||
809 | static void vfio_put_address_space(VFIOAddressSpace *space) | |
810 | { | |
811 | if (QLIST_EMPTY(&space->containers)) { | |
812 | QLIST_REMOVE(space, list); | |
813 | g_free(space); | |
814 | } | |
815 | } | |
816 | ||
817 | static int vfio_connect_container(VFIOGroup *group, AddressSpace *as) | |
818 | { | |
819 | VFIOContainer *container; | |
820 | int ret, fd; | |
821 | VFIOAddressSpace *space; | |
822 | ||
823 | space = vfio_get_address_space(as); | |
824 | ||
825 | QLIST_FOREACH(container, &space->containers, next) { | |
826 | if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { | |
827 | group->container = container; | |
828 | QLIST_INSERT_HEAD(&container->group_list, group, container_next); | |
829 | return 0; | |
830 | } | |
831 | } | |
832 | ||
833 | fd = qemu_open("/dev/vfio/vfio", O_RDWR); | |
834 | if (fd < 0) { | |
835 | error_report("vfio: failed to open /dev/vfio/vfio: %m"); | |
836 | ret = -errno; | |
837 | goto put_space_exit; | |
838 | } | |
839 | ||
840 | ret = ioctl(fd, VFIO_GET_API_VERSION); | |
841 | if (ret != VFIO_API_VERSION) { | |
842 | error_report("vfio: supported vfio version: %d, " | |
843 | "reported version: %d", VFIO_API_VERSION, ret); | |
844 | ret = -EINVAL; | |
845 | goto close_fd_exit; | |
846 | } | |
847 | ||
848 | container = g_malloc0(sizeof(*container)); | |
849 | container->space = space; | |
850 | container->fd = fd; | |
2e6e697e AW |
851 | if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) || |
852 | ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) { | |
853 | bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU); | |
7a140a57 | 854 | struct vfio_iommu_type1_info info; |
2e6e697e | 855 | |
e2c7d025 EA |
856 | ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); |
857 | if (ret) { | |
858 | error_report("vfio: failed to set group container: %m"); | |
859 | ret = -errno; | |
860 | goto free_container_exit; | |
861 | } | |
862 | ||
2e6e697e AW |
863 | ret = ioctl(fd, VFIO_SET_IOMMU, |
864 | v2 ? VFIO_TYPE1v2_IOMMU : VFIO_TYPE1_IOMMU); | |
e2c7d025 EA |
865 | if (ret) { |
866 | error_report("vfio: failed to set iommu for container: %m"); | |
867 | ret = -errno; | |
868 | goto free_container_exit; | |
869 | } | |
3898aad3 DG |
870 | |
871 | /* | |
872 | * FIXME: This assumes that a Type1 IOMMU can map any 64-bit | |
873 | * IOVA whatsoever. That's not actually true, but the current | |
874 | * kernel interface doesn't tell us what it can map, and the | |
875 | * existing Type1 IOMMUs generally support any IOVA we're | |
876 | * going to actually try in practice. | |
877 | */ | |
878 | container->min_iova = 0; | |
879 | container->max_iova = (hwaddr)-1; | |
7a140a57 DG |
880 | |
881 | /* Assume just 4K IOVA page size */ | |
882 | container->iova_pgsizes = 0x1000; | |
883 | info.argsz = sizeof(info); | |
884 | ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info); | |
885 | /* Ignore errors */ | |
886 | if ((ret == 0) && (info.flags & VFIO_IOMMU_INFO_PGSIZES)) { | |
887 | container->iova_pgsizes = info.iova_pgsizes; | |
888 | } | |
e2c7d025 | 889 | } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU)) { |
3898aad3 DG |
890 | struct vfio_iommu_spapr_tce_info info; |
891 | ||
e2c7d025 EA |
892 | ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); |
893 | if (ret) { | |
894 | error_report("vfio: failed to set group container: %m"); | |
895 | ret = -errno; | |
896 | goto free_container_exit; | |
897 | } | |
898 | ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_IOMMU); | |
899 | if (ret) { | |
900 | error_report("vfio: failed to set iommu for container: %m"); | |
901 | ret = -errno; | |
902 | goto free_container_exit; | |
903 | } | |
904 | ||
905 | /* | |
906 | * The host kernel code implementing VFIO_IOMMU_DISABLE is called | |
907 | * when container fd is closed so we do not call it explicitly | |
908 | * in this file. | |
909 | */ | |
910 | ret = ioctl(fd, VFIO_IOMMU_ENABLE); | |
911 | if (ret) { | |
912 | error_report("vfio: failed to enable container: %m"); | |
913 | ret = -errno; | |
914 | goto free_container_exit; | |
915 | } | |
3898aad3 DG |
916 | |
917 | /* | |
918 | * This only considers the host IOMMU's 32-bit window. At | |
919 | * some point we need to add support for the optional 64-bit | |
920 | * window and dynamic windows | |
921 | */ | |
922 | info.argsz = sizeof(info); | |
923 | ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); | |
924 | if (ret) { | |
925 | error_report("vfio: VFIO_IOMMU_SPAPR_TCE_GET_INFO failed: %m"); | |
926 | ret = -errno; | |
927 | goto free_container_exit; | |
928 | } | |
929 | container->min_iova = info.dma32_window_start; | |
930 | container->max_iova = container->min_iova + info.dma32_window_size - 1; | |
7a140a57 DG |
931 | |
932 | /* Assume just 4K IOVA pages for now */ | |
933 | container->iova_pgsizes = 0x1000; | |
e2c7d025 EA |
934 | } else { |
935 | error_report("vfio: No available IOMMU models"); | |
936 | ret = -EINVAL; | |
937 | goto free_container_exit; | |
938 | } | |
939 | ||
ee0bf0e5 DG |
940 | container->listener = vfio_memory_listener; |
941 | ||
942 | memory_listener_register(&container->listener, container->space->as); | |
943 | ||
944 | if (container->error) { | |
945 | ret = container->error; | |
946 | error_report("vfio: memory listener initialization failed for container"); | |
947 | goto listener_release_exit; | |
948 | } | |
949 | ||
950 | container->initialized = true; | |
951 | ||
e2c7d025 EA |
952 | QLIST_INIT(&container->group_list); |
953 | QLIST_INSERT_HEAD(&space->containers, container, next); | |
954 | ||
955 | group->container = container; | |
956 | QLIST_INSERT_HEAD(&container->group_list, group, container_next); | |
957 | ||
958 | return 0; | |
959 | listener_release_exit: | |
960 | vfio_listener_release(container); | |
961 | ||
962 | free_container_exit: | |
963 | g_free(container); | |
964 | ||
965 | close_fd_exit: | |
966 | close(fd); | |
967 | ||
968 | put_space_exit: | |
969 | vfio_put_address_space(space); | |
970 | ||
971 | return ret; | |
972 | } | |
973 | ||
974 | static void vfio_disconnect_container(VFIOGroup *group) | |
975 | { | |
976 | VFIOContainer *container = group->container; | |
977 | ||
978 | if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) { | |
979 | error_report("vfio: error disconnecting group %d from container", | |
980 | group->groupid); | |
981 | } | |
982 | ||
983 | QLIST_REMOVE(group, container_next); | |
984 | group->container = NULL; | |
985 | ||
986 | if (QLIST_EMPTY(&container->group_list)) { | |
987 | VFIOAddressSpace *space = container->space; | |
f8d8a944 | 988 | VFIOGuestIOMMU *giommu, *tmp; |
e2c7d025 | 989 | |
ee0bf0e5 | 990 | vfio_listener_release(container); |
e2c7d025 | 991 | QLIST_REMOVE(container, next); |
f8d8a944 AK |
992 | |
993 | QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) { | |
994 | memory_region_unregister_iommu_notifier(&giommu->n); | |
995 | QLIST_REMOVE(giommu, giommu_next); | |
996 | g_free(giommu); | |
997 | } | |
998 | ||
e2c7d025 EA |
999 | trace_vfio_disconnect_container(container->fd); |
1000 | close(container->fd); | |
1001 | g_free(container); | |
1002 | ||
1003 | vfio_put_address_space(space); | |
1004 | } | |
1005 | } | |
1006 | ||
1007 | VFIOGroup *vfio_get_group(int groupid, AddressSpace *as) | |
1008 | { | |
1009 | VFIOGroup *group; | |
1010 | char path[32]; | |
1011 | struct vfio_group_status status = { .argsz = sizeof(status) }; | |
1012 | ||
1013 | QLIST_FOREACH(group, &vfio_group_list, next) { | |
1014 | if (group->groupid == groupid) { | |
1015 | /* Found it. Now is it already in the right context? */ | |
1016 | if (group->container->space->as == as) { | |
1017 | return group; | |
1018 | } else { | |
1019 | error_report("vfio: group %d used in multiple address spaces", | |
1020 | group->groupid); | |
1021 | return NULL; | |
1022 | } | |
1023 | } | |
1024 | } | |
1025 | ||
1026 | group = g_malloc0(sizeof(*group)); | |
1027 | ||
1028 | snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); | |
1029 | group->fd = qemu_open(path, O_RDWR); | |
1030 | if (group->fd < 0) { | |
1031 | error_report("vfio: error opening %s: %m", path); | |
1032 | goto free_group_exit; | |
1033 | } | |
1034 | ||
1035 | if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) { | |
1036 | error_report("vfio: error getting group status: %m"); | |
1037 | goto close_fd_exit; | |
1038 | } | |
1039 | ||
1040 | if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { | |
1041 | error_report("vfio: error, group %d is not viable, please ensure " | |
1042 | "all devices within the iommu_group are bound to their " | |
1043 | "vfio bus driver.", groupid); | |
1044 | goto close_fd_exit; | |
1045 | } | |
1046 | ||
1047 | group->groupid = groupid; | |
1048 | QLIST_INIT(&group->device_list); | |
1049 | ||
1050 | if (vfio_connect_container(group, as)) { | |
1051 | error_report("vfio: failed to setup container for group %d", groupid); | |
1052 | goto close_fd_exit; | |
1053 | } | |
1054 | ||
1055 | if (QLIST_EMPTY(&vfio_group_list)) { | |
1056 | qemu_register_reset(vfio_reset_handler, NULL); | |
1057 | } | |
1058 | ||
1059 | QLIST_INSERT_HEAD(&vfio_group_list, group, next); | |
1060 | ||
1061 | vfio_kvm_device_add_group(group); | |
1062 | ||
1063 | return group; | |
1064 | ||
1065 | close_fd_exit: | |
1066 | close(group->fd); | |
1067 | ||
1068 | free_group_exit: | |
1069 | g_free(group); | |
1070 | ||
1071 | return NULL; | |
1072 | } | |
1073 | ||
1074 | void vfio_put_group(VFIOGroup *group) | |
1075 | { | |
77a10d04 | 1076 | if (!group || !QLIST_EMPTY(&group->device_list)) { |
e2c7d025 EA |
1077 | return; |
1078 | } | |
1079 | ||
1080 | vfio_kvm_device_del_group(group); | |
1081 | vfio_disconnect_container(group); | |
1082 | QLIST_REMOVE(group, next); | |
1083 | trace_vfio_put_group(group->fd); | |
1084 | close(group->fd); | |
1085 | g_free(group); | |
1086 | ||
1087 | if (QLIST_EMPTY(&vfio_group_list)) { | |
1088 | qemu_unregister_reset(vfio_reset_handler, NULL); | |
1089 | } | |
1090 | } | |
1091 | ||
1092 | int vfio_get_device(VFIOGroup *group, const char *name, | |
1093 | VFIODevice *vbasedev) | |
1094 | { | |
1095 | struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) }; | |
217e9fdc | 1096 | int ret, fd; |
e2c7d025 | 1097 | |
217e9fdc PB |
1098 | fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name); |
1099 | if (fd < 0) { | |
e2c7d025 EA |
1100 | error_report("vfio: error getting device %s from group %d: %m", |
1101 | name, group->groupid); | |
1102 | error_printf("Verify all devices in group %d are bound to vfio-<bus> " | |
1103 | "or pci-stub and not already in use\n", group->groupid); | |
217e9fdc | 1104 | return fd; |
e2c7d025 EA |
1105 | } |
1106 | ||
217e9fdc | 1107 | ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &dev_info); |
e2c7d025 EA |
1108 | if (ret) { |
1109 | error_report("vfio: error getting device info: %m"); | |
217e9fdc PB |
1110 | close(fd); |
1111 | return ret; | |
e2c7d025 EA |
1112 | } |
1113 | ||
217e9fdc PB |
1114 | vbasedev->fd = fd; |
1115 | vbasedev->group = group; | |
1116 | QLIST_INSERT_HEAD(&group->device_list, vbasedev, next); | |
1117 | ||
e2c7d025 EA |
1118 | vbasedev->num_irqs = dev_info.num_irqs; |
1119 | vbasedev->num_regions = dev_info.num_regions; | |
1120 | vbasedev->flags = dev_info.flags; | |
1121 | ||
1122 | trace_vfio_get_device(name, dev_info.flags, dev_info.num_regions, | |
1123 | dev_info.num_irqs); | |
1124 | ||
1125 | vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET); | |
217e9fdc | 1126 | return 0; |
e2c7d025 EA |
1127 | } |
1128 | ||
1129 | void vfio_put_base_device(VFIODevice *vbasedev) | |
1130 | { | |
77a10d04 PB |
1131 | if (!vbasedev->group) { |
1132 | return; | |
1133 | } | |
e2c7d025 EA |
1134 | QLIST_REMOVE(vbasedev, next); |
1135 | vbasedev->group = NULL; | |
1136 | trace_vfio_put_base_device(vbasedev->fd); | |
1137 | close(vbasedev->fd); | |
1138 | } | |
1139 | ||
46900226 AW |
1140 | int vfio_get_region_info(VFIODevice *vbasedev, int index, |
1141 | struct vfio_region_info **info) | |
1142 | { | |
1143 | size_t argsz = sizeof(struct vfio_region_info); | |
1144 | ||
1145 | *info = g_malloc0(argsz); | |
1146 | ||
1147 | (*info)->index = index; | |
b53b0f69 | 1148 | retry: |
46900226 AW |
1149 | (*info)->argsz = argsz; |
1150 | ||
1151 | if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) { | |
1152 | g_free(*info); | |
e61a424f | 1153 | *info = NULL; |
46900226 AW |
1154 | return -errno; |
1155 | } | |
1156 | ||
b53b0f69 AW |
1157 | if ((*info)->argsz > argsz) { |
1158 | argsz = (*info)->argsz; | |
1159 | *info = g_realloc(*info, argsz); | |
1160 | ||
1161 | goto retry; | |
1162 | } | |
1163 | ||
46900226 AW |
1164 | return 0; |
1165 | } | |
1166 | ||
e61a424f AW |
1167 | int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type, |
1168 | uint32_t subtype, struct vfio_region_info **info) | |
1169 | { | |
1170 | int i; | |
1171 | ||
1172 | for (i = 0; i < vbasedev->num_regions; i++) { | |
1173 | struct vfio_info_cap_header *hdr; | |
1174 | struct vfio_region_info_cap_type *cap_type; | |
1175 | ||
1176 | if (vfio_get_region_info(vbasedev, i, info)) { | |
1177 | continue; | |
1178 | } | |
1179 | ||
1180 | hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE); | |
1181 | if (!hdr) { | |
1182 | g_free(*info); | |
1183 | continue; | |
1184 | } | |
1185 | ||
1186 | cap_type = container_of(hdr, struct vfio_region_info_cap_type, header); | |
1187 | ||
1188 | trace_vfio_get_dev_region(vbasedev->name, i, | |
1189 | cap_type->type, cap_type->subtype); | |
1190 | ||
1191 | if (cap_type->type == type && cap_type->subtype == subtype) { | |
1192 | return 0; | |
1193 | } | |
1194 | ||
1195 | g_free(*info); | |
1196 | } | |
1197 | ||
1198 | *info = NULL; | |
1199 | return -ENODEV; | |
1200 | } | |
1201 | ||
3153119e DG |
1202 | /* |
1203 | * Interfaces for IBM EEH (Enhanced Error Handling) | |
1204 | */ | |
1205 | static bool vfio_eeh_container_ok(VFIOContainer *container) | |
1206 | { | |
1207 | /* | |
1208 | * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO | |
1209 | * implementation is broken if there are multiple groups in a | |
1210 | * container. The hardware works in units of Partitionable | |
1211 | * Endpoints (== IOMMU groups) and the EEH operations naively | |
1212 | * iterate across all groups in the container, without any logic | |
1213 | * to make sure the groups have their state synchronized. For | |
1214 | * certain operations (ENABLE) that might be ok, until an error | |
1215 | * occurs, but for others (GET_STATE) it's clearly broken. | |
1216 | */ | |
1217 | ||
1218 | /* | |
1219 | * XXX Once fixed kernels exist, test for them here | |
1220 | */ | |
1221 | ||
1222 | if (QLIST_EMPTY(&container->group_list)) { | |
1223 | return false; | |
1224 | } | |
1225 | ||
1226 | if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) { | |
1227 | return false; | |
1228 | } | |
1229 | ||
1230 | return true; | |
1231 | } | |
1232 | ||
1233 | static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op) | |
1234 | { | |
1235 | struct vfio_eeh_pe_op pe_op = { | |
1236 | .argsz = sizeof(pe_op), | |
1237 | .op = op, | |
1238 | }; | |
1239 | int ret; | |
1240 | ||
1241 | if (!vfio_eeh_container_ok(container)) { | |
1242 | error_report("vfio/eeh: EEH_PE_OP 0x%x: " | |
1243 | "kernel requires a container with exactly one group", op); | |
1244 | return -EPERM; | |
1245 | } | |
1246 | ||
1247 | ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op); | |
1248 | if (ret < 0) { | |
1249 | error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op); | |
1250 | return -errno; | |
1251 | } | |
1252 | ||
1253 | return 0; | |
1254 | } | |
1255 | ||
1256 | static VFIOContainer *vfio_eeh_as_container(AddressSpace *as) | |
1257 | { | |
1258 | VFIOAddressSpace *space = vfio_get_address_space(as); | |
1259 | VFIOContainer *container = NULL; | |
1260 | ||
1261 | if (QLIST_EMPTY(&space->containers)) { | |
1262 | /* No containers to act on */ | |
1263 | goto out; | |
1264 | } | |
1265 | ||
1266 | container = QLIST_FIRST(&space->containers); | |
1267 | ||
1268 | if (QLIST_NEXT(container, next)) { | |
1269 | /* We don't yet have logic to synchronize EEH state across | |
1270 | * multiple containers */ | |
1271 | container = NULL; | |
1272 | goto out; | |
1273 | } | |
1274 | ||
1275 | out: | |
1276 | vfio_put_address_space(space); | |
1277 | return container; | |
1278 | } | |
1279 | ||
1280 | bool vfio_eeh_as_ok(AddressSpace *as) | |
1281 | { | |
1282 | VFIOContainer *container = vfio_eeh_as_container(as); | |
1283 | ||
1284 | return (container != NULL) && vfio_eeh_container_ok(container); | |
1285 | } | |
1286 | ||
1287 | int vfio_eeh_as_op(AddressSpace *as, uint32_t op) | |
1288 | { | |
1289 | VFIOContainer *container = vfio_eeh_as_container(as); | |
1290 | ||
1291 | if (!container) { | |
1292 | return -ENODEV; | |
1293 | } | |
1294 | return vfio_eeh_container_op(container, op); | |
1295 | } |