]>
Commit | Line | Data |
---|---|---|
e2c7d025 EA |
1 | /* |
2 | * generic functions used by VFIO devices | |
3 | * | |
4 | * Copyright Red Hat, Inc. 2012 | |
5 | * | |
6 | * Authors: | |
7 | * Alex Williamson <alex.williamson@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | * | |
12 | * Based on qemu-kvm device-assignment: | |
13 | * Adapted for KVM by Qumranet. | |
14 | * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) | |
15 | * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) | |
16 | * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) | |
17 | * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) | |
18 | * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) | |
19 | */ | |
20 | ||
c6eacb1a | 21 | #include "qemu/osdep.h" |
e2c7d025 | 22 | #include <sys/ioctl.h> |
e2c7d025 EA |
23 | #include <linux/vfio.h> |
24 | ||
25 | #include "hw/vfio/vfio-common.h" | |
26 | #include "hw/vfio/vfio.h" | |
27 | #include "exec/address-spaces.h" | |
28 | #include "exec/memory.h" | |
29 | #include "hw/hw.h" | |
30 | #include "qemu/error-report.h" | |
31 | #include "sysemu/kvm.h" | |
e81096b1 PB |
32 | #ifdef CONFIG_KVM |
33 | #include "linux/kvm.h" | |
34 | #endif | |
e2c7d025 EA |
35 | #include "trace.h" |
36 | ||
37 | struct vfio_group_head vfio_group_list = | |
39cb514f | 38 | QLIST_HEAD_INITIALIZER(vfio_group_list); |
e2c7d025 EA |
39 | struct vfio_as_head vfio_address_spaces = |
40 | QLIST_HEAD_INITIALIZER(vfio_address_spaces); | |
41 | ||
42 | #ifdef CONFIG_KVM | |
43 | /* | |
44 | * We have a single VFIO pseudo device per KVM VM. Once created it lives | |
45 | * for the life of the VM. Closing the file descriptor only drops our | |
46 | * reference to it and the device's reference to kvm. Therefore once | |
47 | * initialized, this file descriptor is only released on QEMU exit and | |
48 | * we'll re-use it should another vfio device be attached before then. | |
49 | */ | |
50 | static int vfio_kvm_device_fd = -1; | |
51 | #endif | |
52 | ||
53 | /* | |
54 | * Common VFIO interrupt disable | |
55 | */ | |
56 | void vfio_disable_irqindex(VFIODevice *vbasedev, int index) | |
57 | { | |
58 | struct vfio_irq_set irq_set = { | |
59 | .argsz = sizeof(irq_set), | |
60 | .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, | |
61 | .index = index, | |
62 | .start = 0, | |
63 | .count = 0, | |
64 | }; | |
65 | ||
66 | ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); | |
67 | } | |
68 | ||
69 | void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index) | |
70 | { | |
71 | struct vfio_irq_set irq_set = { | |
72 | .argsz = sizeof(irq_set), | |
73 | .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, | |
74 | .index = index, | |
75 | .start = 0, | |
76 | .count = 1, | |
77 | }; | |
78 | ||
79 | ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); | |
80 | } | |
81 | ||
82 | void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index) | |
83 | { | |
84 | struct vfio_irq_set irq_set = { | |
85 | .argsz = sizeof(irq_set), | |
86 | .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK, | |
87 | .index = index, | |
88 | .start = 0, | |
89 | .count = 1, | |
90 | }; | |
91 | ||
92 | ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); | |
93 | } | |
94 | ||
95 | /* | |
96 | * IO Port/MMIO - Beware of the endians, VFIO is always little endian | |
97 | */ | |
98 | void vfio_region_write(void *opaque, hwaddr addr, | |
99 | uint64_t data, unsigned size) | |
100 | { | |
101 | VFIORegion *region = opaque; | |
102 | VFIODevice *vbasedev = region->vbasedev; | |
103 | union { | |
104 | uint8_t byte; | |
105 | uint16_t word; | |
106 | uint32_t dword; | |
107 | uint64_t qword; | |
108 | } buf; | |
109 | ||
110 | switch (size) { | |
111 | case 1: | |
112 | buf.byte = data; | |
113 | break; | |
114 | case 2: | |
115 | buf.word = cpu_to_le16(data); | |
116 | break; | |
117 | case 4: | |
118 | buf.dword = cpu_to_le32(data); | |
119 | break; | |
120 | default: | |
121 | hw_error("vfio: unsupported write size, %d bytes", size); | |
122 | break; | |
123 | } | |
124 | ||
125 | if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { | |
126 | error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64 | |
127 | ",%d) failed: %m", | |
128 | __func__, vbasedev->name, region->nr, | |
129 | addr, data, size); | |
130 | } | |
131 | ||
132 | trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size); | |
133 | ||
134 | /* | |
135 | * A read or write to a BAR always signals an INTx EOI. This will | |
136 | * do nothing if not pending (including not in INTx mode). We assume | |
137 | * that a BAR access is in response to an interrupt and that BAR | |
138 | * accesses will service the interrupt. Unfortunately, we don't know | |
139 | * which access will service the interrupt, so we're potentially | |
140 | * getting quite a few host interrupts per guest interrupt. | |
141 | */ | |
142 | vbasedev->ops->vfio_eoi(vbasedev); | |
143 | } | |
144 | ||
145 | uint64_t vfio_region_read(void *opaque, | |
146 | hwaddr addr, unsigned size) | |
147 | { | |
148 | VFIORegion *region = opaque; | |
149 | VFIODevice *vbasedev = region->vbasedev; | |
150 | union { | |
151 | uint8_t byte; | |
152 | uint16_t word; | |
153 | uint32_t dword; | |
154 | uint64_t qword; | |
155 | } buf; | |
156 | uint64_t data = 0; | |
157 | ||
158 | if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { | |
159 | error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m", | |
160 | __func__, vbasedev->name, region->nr, | |
161 | addr, size); | |
162 | return (uint64_t)-1; | |
163 | } | |
164 | switch (size) { | |
165 | case 1: | |
166 | data = buf.byte; | |
167 | break; | |
168 | case 2: | |
169 | data = le16_to_cpu(buf.word); | |
170 | break; | |
171 | case 4: | |
172 | data = le32_to_cpu(buf.dword); | |
173 | break; | |
174 | default: | |
175 | hw_error("vfio: unsupported read size, %d bytes", size); | |
176 | break; | |
177 | } | |
178 | ||
179 | trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data); | |
180 | ||
181 | /* Same as write above */ | |
182 | vbasedev->ops->vfio_eoi(vbasedev); | |
183 | ||
184 | return data; | |
185 | } | |
186 | ||
187 | const MemoryRegionOps vfio_region_ops = { | |
188 | .read = vfio_region_read, | |
189 | .write = vfio_region_write, | |
190 | .endianness = DEVICE_LITTLE_ENDIAN, | |
191 | }; | |
192 | ||
193 | /* | |
194 | * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 | |
195 | */ | |
196 | static int vfio_dma_unmap(VFIOContainer *container, | |
197 | hwaddr iova, ram_addr_t size) | |
198 | { | |
199 | struct vfio_iommu_type1_dma_unmap unmap = { | |
200 | .argsz = sizeof(unmap), | |
201 | .flags = 0, | |
202 | .iova = iova, | |
203 | .size = size, | |
204 | }; | |
205 | ||
206 | if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { | |
78e5b17f | 207 | error_report("VFIO_UNMAP_DMA: %d", -errno); |
e2c7d025 EA |
208 | return -errno; |
209 | } | |
210 | ||
211 | return 0; | |
212 | } | |
213 | ||
214 | static int vfio_dma_map(VFIOContainer *container, hwaddr iova, | |
215 | ram_addr_t size, void *vaddr, bool readonly) | |
216 | { | |
217 | struct vfio_iommu_type1_dma_map map = { | |
218 | .argsz = sizeof(map), | |
219 | .flags = VFIO_DMA_MAP_FLAG_READ, | |
220 | .vaddr = (__u64)(uintptr_t)vaddr, | |
221 | .iova = iova, | |
222 | .size = size, | |
223 | }; | |
224 | ||
225 | if (!readonly) { | |
226 | map.flags |= VFIO_DMA_MAP_FLAG_WRITE; | |
227 | } | |
228 | ||
229 | /* | |
230 | * Try the mapping, if it fails with EBUSY, unmap the region and try | |
231 | * again. This shouldn't be necessary, but we sometimes see it in | |
b6af0975 | 232 | * the VGA ROM space. |
e2c7d025 EA |
233 | */ |
234 | if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 || | |
235 | (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 && | |
236 | ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) { | |
237 | return 0; | |
238 | } | |
239 | ||
78e5b17f | 240 | error_report("VFIO_MAP_DMA: %d", -errno); |
e2c7d025 EA |
241 | return -errno; |
242 | } | |
243 | ||
244 | static bool vfio_listener_skipped_section(MemoryRegionSection *section) | |
245 | { | |
246 | return (!memory_region_is_ram(section->mr) && | |
247 | !memory_region_is_iommu(section->mr)) || | |
248 | /* | |
249 | * Sizing an enabled 64-bit BAR can cause spurious mappings to | |
250 | * addresses in the upper part of the 64-bit address space. These | |
251 | * are never accessed by the CPU and beyond the address width of | |
252 | * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. | |
253 | */ | |
254 | section->offset_within_address_space & (1ULL << 63); | |
255 | } | |
256 | ||
257 | static void vfio_iommu_map_notify(Notifier *n, void *data) | |
258 | { | |
259 | VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); | |
260 | VFIOContainer *container = giommu->container; | |
261 | IOMMUTLBEntry *iotlb = data; | |
d78c19b5 | 262 | hwaddr iova = iotlb->iova + giommu->iommu_offset; |
e2c7d025 EA |
263 | MemoryRegion *mr; |
264 | hwaddr xlat; | |
265 | hwaddr len = iotlb->addr_mask + 1; | |
266 | void *vaddr; | |
267 | int ret; | |
268 | ||
d78c19b5 | 269 | trace_vfio_iommu_map_notify(iova, iova + iotlb->addr_mask); |
e2c7d025 | 270 | |
f1f93650 AK |
271 | if (iotlb->target_as != &address_space_memory) { |
272 | error_report("Wrong target AS \"%s\", only system memory is allowed", | |
273 | iotlb->target_as->name ? iotlb->target_as->name : "none"); | |
274 | return; | |
275 | } | |
276 | ||
e2c7d025 EA |
277 | /* |
278 | * The IOMMU TLB entry we have just covers translation through | |
279 | * this IOMMU to its immediate target. We need to translate | |
280 | * it the rest of the way through to memory. | |
281 | */ | |
41063e1e | 282 | rcu_read_lock(); |
e2c7d025 EA |
283 | mr = address_space_translate(&address_space_memory, |
284 | iotlb->translated_addr, | |
285 | &xlat, &len, iotlb->perm & IOMMU_WO); | |
286 | if (!memory_region_is_ram(mr)) { | |
78e5b17f | 287 | error_report("iommu map to non memory area %"HWADDR_PRIx"", |
e2c7d025 | 288 | xlat); |
41063e1e | 289 | goto out; |
e2c7d025 EA |
290 | } |
291 | /* | |
292 | * Translation truncates length to the IOMMU page size, | |
293 | * check that it did not truncate too much. | |
294 | */ | |
295 | if (len & iotlb->addr_mask) { | |
78e5b17f | 296 | error_report("iommu has granularity incompatible with target AS"); |
41063e1e | 297 | goto out; |
e2c7d025 EA |
298 | } |
299 | ||
300 | if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { | |
301 | vaddr = memory_region_get_ram_ptr(mr) + xlat; | |
d78c19b5 | 302 | ret = vfio_dma_map(container, iova, |
e2c7d025 EA |
303 | iotlb->addr_mask + 1, vaddr, |
304 | !(iotlb->perm & IOMMU_WO) || mr->readonly); | |
305 | if (ret) { | |
306 | error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " | |
307 | "0x%"HWADDR_PRIx", %p) = %d (%m)", | |
d78c19b5 | 308 | container, iova, |
e2c7d025 EA |
309 | iotlb->addr_mask + 1, vaddr, ret); |
310 | } | |
311 | } else { | |
d78c19b5 | 312 | ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1); |
e2c7d025 EA |
313 | if (ret) { |
314 | error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " | |
315 | "0x%"HWADDR_PRIx") = %d (%m)", | |
d78c19b5 | 316 | container, iova, |
e2c7d025 EA |
317 | iotlb->addr_mask + 1, ret); |
318 | } | |
319 | } | |
41063e1e PB |
320 | out: |
321 | rcu_read_unlock(); | |
e2c7d025 EA |
322 | } |
323 | ||
324 | static void vfio_listener_region_add(MemoryListener *listener, | |
325 | MemoryRegionSection *section) | |
326 | { | |
ee0bf0e5 | 327 | VFIOContainer *container = container_of(listener, VFIOContainer, listener); |
e2c7d025 | 328 | hwaddr iova, end; |
55efcc53 | 329 | Int128 llend, llsize; |
e2c7d025 EA |
330 | void *vaddr; |
331 | int ret; | |
332 | ||
333 | if (vfio_listener_skipped_section(section)) { | |
334 | trace_vfio_listener_region_add_skip( | |
335 | section->offset_within_address_space, | |
336 | section->offset_within_address_space + | |
337 | int128_get64(int128_sub(section->size, int128_one()))); | |
338 | return; | |
339 | } | |
340 | ||
341 | if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != | |
342 | (section->offset_within_region & ~TARGET_PAGE_MASK))) { | |
343 | error_report("%s received unaligned region", __func__); | |
344 | return; | |
345 | } | |
346 | ||
347 | iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); | |
348 | llend = int128_make64(section->offset_within_address_space); | |
349 | llend = int128_add(llend, section->size); | |
350 | llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); | |
351 | ||
352 | if (int128_ge(int128_make64(iova), llend)) { | |
353 | return; | |
354 | } | |
55efcc53 | 355 | end = int128_get64(int128_sub(llend, int128_one())); |
3898aad3 | 356 | |
55efcc53 | 357 | if ((iova < container->min_iova) || (end > container->max_iova)) { |
3898aad3 DG |
358 | error_report("vfio: IOMMU container %p can't map guest IOVA region" |
359 | " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, | |
55efcc53 | 360 | container, iova, end); |
3898aad3 DG |
361 | ret = -EFAULT; |
362 | goto fail; | |
363 | } | |
e2c7d025 EA |
364 | |
365 | memory_region_ref(section->mr); | |
366 | ||
367 | if (memory_region_is_iommu(section->mr)) { | |
368 | VFIOGuestIOMMU *giommu; | |
369 | ||
55efcc53 | 370 | trace_vfio_listener_region_add_iommu(iova, end); |
e2c7d025 EA |
371 | /* |
372 | * FIXME: We should do some checking to see if the | |
373 | * capabilities of the host VFIO IOMMU are adequate to model | |
374 | * the guest IOMMU | |
375 | * | |
376 | * FIXME: For VFIO iommu types which have KVM acceleration to | |
377 | * avoid bouncing all map/unmaps through qemu this way, this | |
378 | * would be the right place to wire that up (tell the KVM | |
379 | * device emulation the VFIO iommu handles to use). | |
380 | */ | |
e2c7d025 EA |
381 | giommu = g_malloc0(sizeof(*giommu)); |
382 | giommu->iommu = section->mr; | |
d78c19b5 AK |
383 | giommu->iommu_offset = section->offset_within_address_space - |
384 | section->offset_within_region; | |
e2c7d025 EA |
385 | giommu->container = container; |
386 | giommu->n.notify = vfio_iommu_map_notify; | |
387 | QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); | |
508ce5eb | 388 | |
e2c7d025 | 389 | memory_region_register_iommu_notifier(giommu->iommu, &giommu->n); |
f682e9c2 | 390 | memory_region_iommu_replay(giommu->iommu, &giommu->n, false); |
e2c7d025 EA |
391 | |
392 | return; | |
393 | } | |
394 | ||
395 | /* Here we assume that memory_region_is_ram(section->mr)==true */ | |
396 | ||
e2c7d025 EA |
397 | vaddr = memory_region_get_ram_ptr(section->mr) + |
398 | section->offset_within_region + | |
399 | (iova - section->offset_within_address_space); | |
400 | ||
55efcc53 | 401 | trace_vfio_listener_region_add_ram(iova, end, vaddr); |
e2c7d025 | 402 | |
55efcc53 BD |
403 | llsize = int128_sub(llend, int128_make64(iova)); |
404 | ||
405 | ret = vfio_dma_map(container, iova, int128_get64(llsize), | |
406 | vaddr, section->readonly); | |
e2c7d025 EA |
407 | if (ret) { |
408 | error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " | |
409 | "0x%"HWADDR_PRIx", %p) = %d (%m)", | |
55efcc53 | 410 | container, iova, int128_get64(llsize), vaddr, ret); |
ac6dc389 DG |
411 | goto fail; |
412 | } | |
e2c7d025 | 413 | |
ac6dc389 DG |
414 | return; |
415 | ||
416 | fail: | |
417 | /* | |
418 | * On the initfn path, store the first error in the container so we | |
419 | * can gracefully fail. Runtime, there's not much we can do other | |
420 | * than throw a hardware error. | |
421 | */ | |
422 | if (!container->initialized) { | |
423 | if (!container->error) { | |
424 | container->error = ret; | |
e2c7d025 | 425 | } |
ac6dc389 DG |
426 | } else { |
427 | hw_error("vfio: DMA mapping failed, unable to continue"); | |
e2c7d025 EA |
428 | } |
429 | } | |
430 | ||
431 | static void vfio_listener_region_del(MemoryListener *listener, | |
432 | MemoryRegionSection *section) | |
433 | { | |
ee0bf0e5 | 434 | VFIOContainer *container = container_of(listener, VFIOContainer, listener); |
e2c7d025 | 435 | hwaddr iova, end; |
7a057b4f | 436 | Int128 llend, llsize; |
e2c7d025 EA |
437 | int ret; |
438 | ||
439 | if (vfio_listener_skipped_section(section)) { | |
440 | trace_vfio_listener_region_del_skip( | |
441 | section->offset_within_address_space, | |
442 | section->offset_within_address_space + | |
443 | int128_get64(int128_sub(section->size, int128_one()))); | |
444 | return; | |
445 | } | |
446 | ||
447 | if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != | |
448 | (section->offset_within_region & ~TARGET_PAGE_MASK))) { | |
449 | error_report("%s received unaligned region", __func__); | |
450 | return; | |
451 | } | |
452 | ||
453 | if (memory_region_is_iommu(section->mr)) { | |
454 | VFIOGuestIOMMU *giommu; | |
455 | ||
456 | QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { | |
457 | if (giommu->iommu == section->mr) { | |
d22d8956 AK |
458 | memory_region_unregister_iommu_notifier(giommu->iommu, |
459 | &giommu->n); | |
e2c7d025 EA |
460 | QLIST_REMOVE(giommu, giommu_next); |
461 | g_free(giommu); | |
462 | break; | |
463 | } | |
464 | } | |
465 | ||
466 | /* | |
467 | * FIXME: We assume the one big unmap below is adequate to | |
468 | * remove any individual page mappings in the IOMMU which | |
469 | * might have been copied into VFIO. This works for a page table | |
470 | * based IOMMU where a big unmap flattens a large range of IO-PTEs. | |
471 | * That may not be true for all IOMMU types. | |
472 | */ | |
473 | } | |
474 | ||
475 | iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); | |
7a057b4f AK |
476 | llend = int128_make64(section->offset_within_address_space); |
477 | llend = int128_add(llend, section->size); | |
478 | llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); | |
e2c7d025 | 479 | |
7a057b4f | 480 | if (int128_ge(int128_make64(iova), llend)) { |
e2c7d025 EA |
481 | return; |
482 | } | |
7a057b4f AK |
483 | end = int128_get64(int128_sub(llend, int128_one())); |
484 | ||
485 | llsize = int128_sub(llend, int128_make64(iova)); | |
e2c7d025 | 486 | |
7a057b4f | 487 | trace_vfio_listener_region_del(iova, end); |
e2c7d025 | 488 | |
7a057b4f | 489 | ret = vfio_dma_unmap(container, iova, int128_get64(llsize)); |
e2c7d025 EA |
490 | memory_region_unref(section->mr); |
491 | if (ret) { | |
492 | error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " | |
493 | "0x%"HWADDR_PRIx") = %d (%m)", | |
7a057b4f | 494 | container, iova, int128_get64(llsize), ret); |
e2c7d025 EA |
495 | } |
496 | } | |
497 | ||
51b833f4 | 498 | static const MemoryListener vfio_memory_listener = { |
e2c7d025 EA |
499 | .region_add = vfio_listener_region_add, |
500 | .region_del = vfio_listener_region_del, | |
501 | }; | |
502 | ||
51b833f4 | 503 | static void vfio_listener_release(VFIOContainer *container) |
e2c7d025 | 504 | { |
ee0bf0e5 | 505 | memory_listener_unregister(&container->listener); |
318f67ce AK |
506 | if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { |
507 | memory_listener_unregister(&container->prereg_listener); | |
508 | } | |
e2c7d025 EA |
509 | } |
510 | ||
b53b0f69 AW |
511 | static struct vfio_info_cap_header * |
512 | vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id) | |
513 | { | |
514 | struct vfio_info_cap_header *hdr; | |
515 | void *ptr = info; | |
516 | ||
517 | if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) { | |
518 | return NULL; | |
519 | } | |
520 | ||
521 | for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) { | |
522 | if (hdr->id == id) { | |
523 | return hdr; | |
524 | } | |
525 | } | |
526 | ||
527 | return NULL; | |
528 | } | |
529 | ||
530 | static void vfio_setup_region_sparse_mmaps(VFIORegion *region, | |
531 | struct vfio_region_info *info) | |
532 | { | |
533 | struct vfio_info_cap_header *hdr; | |
534 | struct vfio_region_info_cap_sparse_mmap *sparse; | |
535 | int i; | |
536 | ||
537 | hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); | |
538 | if (!hdr) { | |
539 | return; | |
540 | } | |
541 | ||
542 | sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header); | |
543 | ||
544 | trace_vfio_region_sparse_mmap_header(region->vbasedev->name, | |
545 | region->nr, sparse->nr_areas); | |
546 | ||
547 | region->nr_mmaps = sparse->nr_areas; | |
548 | region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); | |
549 | ||
550 | for (i = 0; i < region->nr_mmaps; i++) { | |
551 | region->mmaps[i].offset = sparse->areas[i].offset; | |
552 | region->mmaps[i].size = sparse->areas[i].size; | |
553 | trace_vfio_region_sparse_mmap_entry(i, region->mmaps[i].offset, | |
554 | region->mmaps[i].offset + | |
555 | region->mmaps[i].size); | |
556 | } | |
557 | } | |
558 | ||
db0da029 AW |
559 | int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, |
560 | int index, const char *name) | |
e2c7d025 | 561 | { |
db0da029 AW |
562 | struct vfio_region_info *info; |
563 | int ret; | |
564 | ||
565 | ret = vfio_get_region_info(vbasedev, index, &info); | |
566 | if (ret) { | |
567 | return ret; | |
568 | } | |
569 | ||
570 | region->vbasedev = vbasedev; | |
571 | region->flags = info->flags; | |
572 | region->size = info->size; | |
573 | region->fd_offset = info->offset; | |
574 | region->nr = index; | |
575 | ||
576 | if (region->size) { | |
577 | region->mem = g_new0(MemoryRegion, 1); | |
578 | memory_region_init_io(region->mem, obj, &vfio_region_ops, | |
579 | region, name, region->size); | |
e2c7d025 | 580 | |
db0da029 AW |
581 | if (!vbasedev->no_mmap && |
582 | region->flags & VFIO_REGION_INFO_FLAG_MMAP && | |
583 | !(region->size & ~qemu_real_host_page_mask)) { | |
e2c7d025 | 584 | |
b53b0f69 | 585 | vfio_setup_region_sparse_mmaps(region, info); |
db0da029 | 586 | |
b53b0f69 AW |
587 | if (!region->nr_mmaps) { |
588 | region->nr_mmaps = 1; | |
589 | region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); | |
590 | region->mmaps[0].offset = 0; | |
591 | region->mmaps[0].size = region->size; | |
592 | } | |
e2c7d025 | 593 | } |
db0da029 AW |
594 | } |
595 | ||
596 | g_free(info); | |
597 | ||
598 | trace_vfio_region_setup(vbasedev->name, index, name, | |
599 | region->flags, region->fd_offset, region->size); | |
600 | return 0; | |
601 | } | |
e2c7d025 | 602 | |
db0da029 AW |
603 | int vfio_region_mmap(VFIORegion *region) |
604 | { | |
605 | int i, prot = 0; | |
606 | char *name; | |
607 | ||
608 | if (!region->mem) { | |
609 | return 0; | |
610 | } | |
611 | ||
612 | prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; | |
613 | prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; | |
614 | ||
615 | for (i = 0; i < region->nr_mmaps; i++) { | |
616 | region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot, | |
617 | MAP_SHARED, region->vbasedev->fd, | |
618 | region->fd_offset + | |
619 | region->mmaps[i].offset); | |
620 | if (region->mmaps[i].mmap == MAP_FAILED) { | |
621 | int ret = -errno; | |
622 | ||
623 | trace_vfio_region_mmap_fault(memory_region_name(region->mem), i, | |
624 | region->fd_offset + | |
625 | region->mmaps[i].offset, | |
626 | region->fd_offset + | |
627 | region->mmaps[i].offset + | |
628 | region->mmaps[i].size - 1, ret); | |
629 | ||
630 | region->mmaps[i].mmap = NULL; | |
631 | ||
632 | for (i--; i >= 0; i--) { | |
633 | memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); | |
634 | munmap(region->mmaps[i].mmap, region->mmaps[i].size); | |
635 | object_unparent(OBJECT(®ion->mmaps[i].mem)); | |
636 | region->mmaps[i].mmap = NULL; | |
637 | } | |
638 | ||
639 | return ret; | |
e2c7d025 EA |
640 | } |
641 | ||
db0da029 AW |
642 | name = g_strdup_printf("%s mmaps[%d]", |
643 | memory_region_name(region->mem), i); | |
644 | memory_region_init_ram_ptr(®ion->mmaps[i].mem, | |
645 | memory_region_owner(region->mem), | |
646 | name, region->mmaps[i].size, | |
647 | region->mmaps[i].mmap); | |
648 | g_free(name); | |
649 | memory_region_set_skip_dump(®ion->mmaps[i].mem); | |
650 | memory_region_add_subregion(region->mem, region->mmaps[i].offset, | |
651 | ®ion->mmaps[i].mem); | |
652 | ||
653 | trace_vfio_region_mmap(memory_region_name(®ion->mmaps[i].mem), | |
654 | region->mmaps[i].offset, | |
655 | region->mmaps[i].offset + | |
656 | region->mmaps[i].size - 1); | |
657 | } | |
658 | ||
659 | return 0; | |
660 | } | |
661 | ||
662 | void vfio_region_exit(VFIORegion *region) | |
663 | { | |
664 | int i; | |
665 | ||
666 | if (!region->mem) { | |
667 | return; | |
668 | } | |
669 | ||
670 | for (i = 0; i < region->nr_mmaps; i++) { | |
671 | if (region->mmaps[i].mmap) { | |
672 | memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); | |
e2c7d025 | 673 | } |
db0da029 | 674 | } |
e2c7d025 | 675 | |
db0da029 AW |
676 | trace_vfio_region_exit(region->vbasedev->name, region->nr); |
677 | } | |
678 | ||
679 | void vfio_region_finalize(VFIORegion *region) | |
680 | { | |
681 | int i; | |
682 | ||
683 | if (!region->mem) { | |
684 | return; | |
e2c7d025 EA |
685 | } |
686 | ||
db0da029 AW |
687 | for (i = 0; i < region->nr_mmaps; i++) { |
688 | if (region->mmaps[i].mmap) { | |
689 | munmap(region->mmaps[i].mmap, region->mmaps[i].size); | |
690 | object_unparent(OBJECT(®ion->mmaps[i].mem)); | |
691 | } | |
692 | } | |
693 | ||
694 | object_unparent(OBJECT(region->mem)); | |
695 | ||
696 | g_free(region->mem); | |
697 | g_free(region->mmaps); | |
698 | ||
699 | trace_vfio_region_finalize(region->vbasedev->name, region->nr); | |
700 | } | |
701 | ||
702 | void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled) | |
703 | { | |
704 | int i; | |
705 | ||
706 | if (!region->mem) { | |
707 | return; | |
708 | } | |
709 | ||
710 | for (i = 0; i < region->nr_mmaps; i++) { | |
711 | if (region->mmaps[i].mmap) { | |
712 | memory_region_set_enabled(®ion->mmaps[i].mem, enabled); | |
713 | } | |
714 | } | |
e2c7d025 | 715 | |
db0da029 AW |
716 | trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem), |
717 | enabled); | |
e2c7d025 EA |
718 | } |
719 | ||
720 | void vfio_reset_handler(void *opaque) | |
721 | { | |
722 | VFIOGroup *group; | |
723 | VFIODevice *vbasedev; | |
724 | ||
725 | QLIST_FOREACH(group, &vfio_group_list, next) { | |
726 | QLIST_FOREACH(vbasedev, &group->device_list, next) { | |
727 | vbasedev->ops->vfio_compute_needs_reset(vbasedev); | |
728 | } | |
729 | } | |
730 | ||
731 | QLIST_FOREACH(group, &vfio_group_list, next) { | |
732 | QLIST_FOREACH(vbasedev, &group->device_list, next) { | |
733 | if (vbasedev->needs_reset) { | |
734 | vbasedev->ops->vfio_hot_reset_multi(vbasedev); | |
735 | } | |
736 | } | |
737 | } | |
738 | } | |
739 | ||
740 | static void vfio_kvm_device_add_group(VFIOGroup *group) | |
741 | { | |
742 | #ifdef CONFIG_KVM | |
743 | struct kvm_device_attr attr = { | |
744 | .group = KVM_DEV_VFIO_GROUP, | |
745 | .attr = KVM_DEV_VFIO_GROUP_ADD, | |
746 | .addr = (uint64_t)(unsigned long)&group->fd, | |
747 | }; | |
748 | ||
749 | if (!kvm_enabled()) { | |
750 | return; | |
751 | } | |
752 | ||
753 | if (vfio_kvm_device_fd < 0) { | |
754 | struct kvm_create_device cd = { | |
755 | .type = KVM_DEV_TYPE_VFIO, | |
756 | }; | |
757 | ||
758 | if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { | |
78e5b17f | 759 | error_report("Failed to create KVM VFIO device: %m"); |
e2c7d025 EA |
760 | return; |
761 | } | |
762 | ||
763 | vfio_kvm_device_fd = cd.fd; | |
764 | } | |
765 | ||
766 | if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { | |
767 | error_report("Failed to add group %d to KVM VFIO device: %m", | |
768 | group->groupid); | |
769 | } | |
770 | #endif | |
771 | } | |
772 | ||
773 | static void vfio_kvm_device_del_group(VFIOGroup *group) | |
774 | { | |
775 | #ifdef CONFIG_KVM | |
776 | struct kvm_device_attr attr = { | |
777 | .group = KVM_DEV_VFIO_GROUP, | |
778 | .attr = KVM_DEV_VFIO_GROUP_DEL, | |
779 | .addr = (uint64_t)(unsigned long)&group->fd, | |
780 | }; | |
781 | ||
782 | if (vfio_kvm_device_fd < 0) { | |
783 | return; | |
784 | } | |
785 | ||
786 | if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { | |
787 | error_report("Failed to remove group %d from KVM VFIO device: %m", | |
788 | group->groupid); | |
789 | } | |
790 | #endif | |
791 | } | |
792 | ||
793 | static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) | |
794 | { | |
795 | VFIOAddressSpace *space; | |
796 | ||
797 | QLIST_FOREACH(space, &vfio_address_spaces, list) { | |
798 | if (space->as == as) { | |
799 | return space; | |
800 | } | |
801 | } | |
802 | ||
803 | /* No suitable VFIOAddressSpace, create a new one */ | |
804 | space = g_malloc0(sizeof(*space)); | |
805 | space->as = as; | |
806 | QLIST_INIT(&space->containers); | |
807 | ||
808 | QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); | |
809 | ||
810 | return space; | |
811 | } | |
812 | ||
813 | static void vfio_put_address_space(VFIOAddressSpace *space) | |
814 | { | |
815 | if (QLIST_EMPTY(&space->containers)) { | |
816 | QLIST_REMOVE(space, list); | |
817 | g_free(space); | |
818 | } | |
819 | } | |
820 | ||
821 | static int vfio_connect_container(VFIOGroup *group, AddressSpace *as) | |
822 | { | |
823 | VFIOContainer *container; | |
824 | int ret, fd; | |
825 | VFIOAddressSpace *space; | |
826 | ||
827 | space = vfio_get_address_space(as); | |
828 | ||
829 | QLIST_FOREACH(container, &space->containers, next) { | |
830 | if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { | |
831 | group->container = container; | |
832 | QLIST_INSERT_HEAD(&container->group_list, group, container_next); | |
833 | return 0; | |
834 | } | |
835 | } | |
836 | ||
837 | fd = qemu_open("/dev/vfio/vfio", O_RDWR); | |
838 | if (fd < 0) { | |
839 | error_report("vfio: failed to open /dev/vfio/vfio: %m"); | |
840 | ret = -errno; | |
841 | goto put_space_exit; | |
842 | } | |
843 | ||
844 | ret = ioctl(fd, VFIO_GET_API_VERSION); | |
845 | if (ret != VFIO_API_VERSION) { | |
846 | error_report("vfio: supported vfio version: %d, " | |
847 | "reported version: %d", VFIO_API_VERSION, ret); | |
848 | ret = -EINVAL; | |
849 | goto close_fd_exit; | |
850 | } | |
851 | ||
852 | container = g_malloc0(sizeof(*container)); | |
853 | container->space = space; | |
854 | container->fd = fd; | |
2e6e697e AW |
855 | if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) || |
856 | ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) { | |
857 | bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU); | |
7a140a57 | 858 | struct vfio_iommu_type1_info info; |
2e6e697e | 859 | |
e2c7d025 EA |
860 | ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); |
861 | if (ret) { | |
862 | error_report("vfio: failed to set group container: %m"); | |
863 | ret = -errno; | |
864 | goto free_container_exit; | |
865 | } | |
866 | ||
318f67ce AK |
867 | container->iommu_type = v2 ? VFIO_TYPE1v2_IOMMU : VFIO_TYPE1_IOMMU; |
868 | ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type); | |
e2c7d025 EA |
869 | if (ret) { |
870 | error_report("vfio: failed to set iommu for container: %m"); | |
871 | ret = -errno; | |
872 | goto free_container_exit; | |
873 | } | |
3898aad3 DG |
874 | |
875 | /* | |
876 | * FIXME: This assumes that a Type1 IOMMU can map any 64-bit | |
877 | * IOVA whatsoever. That's not actually true, but the current | |
878 | * kernel interface doesn't tell us what it can map, and the | |
879 | * existing Type1 IOMMUs generally support any IOVA we're | |
880 | * going to actually try in practice. | |
881 | */ | |
882 | container->min_iova = 0; | |
883 | container->max_iova = (hwaddr)-1; | |
7a140a57 DG |
884 | |
885 | /* Assume just 4K IOVA page size */ | |
886 | container->iova_pgsizes = 0x1000; | |
887 | info.argsz = sizeof(info); | |
888 | ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info); | |
889 | /* Ignore errors */ | |
890 | if ((ret == 0) && (info.flags & VFIO_IOMMU_INFO_PGSIZES)) { | |
891 | container->iova_pgsizes = info.iova_pgsizes; | |
892 | } | |
318f67ce AK |
893 | } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU) || |
894 | ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU)) { | |
3898aad3 | 895 | struct vfio_iommu_spapr_tce_info info; |
318f67ce | 896 | bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU); |
3898aad3 | 897 | |
e2c7d025 EA |
898 | ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); |
899 | if (ret) { | |
900 | error_report("vfio: failed to set group container: %m"); | |
901 | ret = -errno; | |
902 | goto free_container_exit; | |
903 | } | |
318f67ce AK |
904 | container->iommu_type = |
905 | v2 ? VFIO_SPAPR_TCE_v2_IOMMU : VFIO_SPAPR_TCE_IOMMU; | |
906 | ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type); | |
e2c7d025 EA |
907 | if (ret) { |
908 | error_report("vfio: failed to set iommu for container: %m"); | |
909 | ret = -errno; | |
910 | goto free_container_exit; | |
911 | } | |
912 | ||
913 | /* | |
914 | * The host kernel code implementing VFIO_IOMMU_DISABLE is called | |
915 | * when container fd is closed so we do not call it explicitly | |
916 | * in this file. | |
917 | */ | |
318f67ce AK |
918 | if (!v2) { |
919 | ret = ioctl(fd, VFIO_IOMMU_ENABLE); | |
920 | if (ret) { | |
921 | error_report("vfio: failed to enable container: %m"); | |
922 | ret = -errno; | |
923 | goto free_container_exit; | |
924 | } | |
925 | } else { | |
926 | container->prereg_listener = vfio_prereg_listener; | |
927 | ||
928 | memory_listener_register(&container->prereg_listener, | |
929 | &address_space_memory); | |
930 | if (container->error) { | |
931 | memory_listener_unregister(&container->prereg_listener); | |
932 | error_report("vfio: RAM memory listener initialization failed for container"); | |
933 | goto free_container_exit; | |
934 | } | |
e2c7d025 | 935 | } |
3898aad3 DG |
936 | |
937 | /* | |
938 | * This only considers the host IOMMU's 32-bit window. At | |
939 | * some point we need to add support for the optional 64-bit | |
940 | * window and dynamic windows | |
941 | */ | |
942 | info.argsz = sizeof(info); | |
943 | ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); | |
944 | if (ret) { | |
945 | error_report("vfio: VFIO_IOMMU_SPAPR_TCE_GET_INFO failed: %m"); | |
946 | ret = -errno; | |
318f67ce AK |
947 | if (v2) { |
948 | memory_listener_unregister(&container->prereg_listener); | |
949 | } | |
3898aad3 DG |
950 | goto free_container_exit; |
951 | } | |
952 | container->min_iova = info.dma32_window_start; | |
953 | container->max_iova = container->min_iova + info.dma32_window_size - 1; | |
7a140a57 DG |
954 | |
955 | /* Assume just 4K IOVA pages for now */ | |
956 | container->iova_pgsizes = 0x1000; | |
e2c7d025 EA |
957 | } else { |
958 | error_report("vfio: No available IOMMU models"); | |
959 | ret = -EINVAL; | |
960 | goto free_container_exit; | |
961 | } | |
962 | ||
ee0bf0e5 DG |
963 | container->listener = vfio_memory_listener; |
964 | ||
965 | memory_listener_register(&container->listener, container->space->as); | |
966 | ||
967 | if (container->error) { | |
968 | ret = container->error; | |
969 | error_report("vfio: memory listener initialization failed for container"); | |
970 | goto listener_release_exit; | |
971 | } | |
972 | ||
973 | container->initialized = true; | |
974 | ||
e2c7d025 EA |
975 | QLIST_INIT(&container->group_list); |
976 | QLIST_INSERT_HEAD(&space->containers, container, next); | |
977 | ||
978 | group->container = container; | |
979 | QLIST_INSERT_HEAD(&container->group_list, group, container_next); | |
980 | ||
981 | return 0; | |
982 | listener_release_exit: | |
983 | vfio_listener_release(container); | |
984 | ||
985 | free_container_exit: | |
986 | g_free(container); | |
987 | ||
988 | close_fd_exit: | |
989 | close(fd); | |
990 | ||
991 | put_space_exit: | |
992 | vfio_put_address_space(space); | |
993 | ||
994 | return ret; | |
995 | } | |
996 | ||
997 | static void vfio_disconnect_container(VFIOGroup *group) | |
998 | { | |
999 | VFIOContainer *container = group->container; | |
1000 | ||
1001 | if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) { | |
1002 | error_report("vfio: error disconnecting group %d from container", | |
1003 | group->groupid); | |
1004 | } | |
1005 | ||
1006 | QLIST_REMOVE(group, container_next); | |
1007 | group->container = NULL; | |
1008 | ||
1009 | if (QLIST_EMPTY(&container->group_list)) { | |
1010 | VFIOAddressSpace *space = container->space; | |
f8d8a944 | 1011 | VFIOGuestIOMMU *giommu, *tmp; |
e2c7d025 | 1012 | |
ee0bf0e5 | 1013 | vfio_listener_release(container); |
e2c7d025 | 1014 | QLIST_REMOVE(container, next); |
f8d8a944 AK |
1015 | |
1016 | QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) { | |
d22d8956 | 1017 | memory_region_unregister_iommu_notifier(giommu->iommu, &giommu->n); |
f8d8a944 AK |
1018 | QLIST_REMOVE(giommu, giommu_next); |
1019 | g_free(giommu); | |
1020 | } | |
1021 | ||
e2c7d025 EA |
1022 | trace_vfio_disconnect_container(container->fd); |
1023 | close(container->fd); | |
1024 | g_free(container); | |
1025 | ||
1026 | vfio_put_address_space(space); | |
1027 | } | |
1028 | } | |
1029 | ||
1030 | VFIOGroup *vfio_get_group(int groupid, AddressSpace *as) | |
1031 | { | |
1032 | VFIOGroup *group; | |
1033 | char path[32]; | |
1034 | struct vfio_group_status status = { .argsz = sizeof(status) }; | |
1035 | ||
1036 | QLIST_FOREACH(group, &vfio_group_list, next) { | |
1037 | if (group->groupid == groupid) { | |
1038 | /* Found it. Now is it already in the right context? */ | |
1039 | if (group->container->space->as == as) { | |
1040 | return group; | |
1041 | } else { | |
1042 | error_report("vfio: group %d used in multiple address spaces", | |
1043 | group->groupid); | |
1044 | return NULL; | |
1045 | } | |
1046 | } | |
1047 | } | |
1048 | ||
1049 | group = g_malloc0(sizeof(*group)); | |
1050 | ||
1051 | snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); | |
1052 | group->fd = qemu_open(path, O_RDWR); | |
1053 | if (group->fd < 0) { | |
1054 | error_report("vfio: error opening %s: %m", path); | |
1055 | goto free_group_exit; | |
1056 | } | |
1057 | ||
1058 | if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) { | |
1059 | error_report("vfio: error getting group status: %m"); | |
1060 | goto close_fd_exit; | |
1061 | } | |
1062 | ||
1063 | if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { | |
1064 | error_report("vfio: error, group %d is not viable, please ensure " | |
1065 | "all devices within the iommu_group are bound to their " | |
1066 | "vfio bus driver.", groupid); | |
1067 | goto close_fd_exit; | |
1068 | } | |
1069 | ||
1070 | group->groupid = groupid; | |
1071 | QLIST_INIT(&group->device_list); | |
1072 | ||
1073 | if (vfio_connect_container(group, as)) { | |
1074 | error_report("vfio: failed to setup container for group %d", groupid); | |
1075 | goto close_fd_exit; | |
1076 | } | |
1077 | ||
1078 | if (QLIST_EMPTY(&vfio_group_list)) { | |
1079 | qemu_register_reset(vfio_reset_handler, NULL); | |
1080 | } | |
1081 | ||
1082 | QLIST_INSERT_HEAD(&vfio_group_list, group, next); | |
1083 | ||
1084 | vfio_kvm_device_add_group(group); | |
1085 | ||
1086 | return group; | |
1087 | ||
1088 | close_fd_exit: | |
1089 | close(group->fd); | |
1090 | ||
1091 | free_group_exit: | |
1092 | g_free(group); | |
1093 | ||
1094 | return NULL; | |
1095 | } | |
1096 | ||
1097 | void vfio_put_group(VFIOGroup *group) | |
1098 | { | |
77a10d04 | 1099 | if (!group || !QLIST_EMPTY(&group->device_list)) { |
e2c7d025 EA |
1100 | return; |
1101 | } | |
1102 | ||
1103 | vfio_kvm_device_del_group(group); | |
1104 | vfio_disconnect_container(group); | |
1105 | QLIST_REMOVE(group, next); | |
1106 | trace_vfio_put_group(group->fd); | |
1107 | close(group->fd); | |
1108 | g_free(group); | |
1109 | ||
1110 | if (QLIST_EMPTY(&vfio_group_list)) { | |
1111 | qemu_unregister_reset(vfio_reset_handler, NULL); | |
1112 | } | |
1113 | } | |
1114 | ||
1115 | int vfio_get_device(VFIOGroup *group, const char *name, | |
1116 | VFIODevice *vbasedev) | |
1117 | { | |
1118 | struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) }; | |
217e9fdc | 1119 | int ret, fd; |
e2c7d025 | 1120 | |
217e9fdc PB |
1121 | fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name); |
1122 | if (fd < 0) { | |
e2c7d025 EA |
1123 | error_report("vfio: error getting device %s from group %d: %m", |
1124 | name, group->groupid); | |
1125 | error_printf("Verify all devices in group %d are bound to vfio-<bus> " | |
1126 | "or pci-stub and not already in use\n", group->groupid); | |
217e9fdc | 1127 | return fd; |
e2c7d025 EA |
1128 | } |
1129 | ||
217e9fdc | 1130 | ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &dev_info); |
e2c7d025 EA |
1131 | if (ret) { |
1132 | error_report("vfio: error getting device info: %m"); | |
217e9fdc PB |
1133 | close(fd); |
1134 | return ret; | |
e2c7d025 EA |
1135 | } |
1136 | ||
217e9fdc PB |
1137 | vbasedev->fd = fd; |
1138 | vbasedev->group = group; | |
1139 | QLIST_INSERT_HEAD(&group->device_list, vbasedev, next); | |
1140 | ||
e2c7d025 EA |
1141 | vbasedev->num_irqs = dev_info.num_irqs; |
1142 | vbasedev->num_regions = dev_info.num_regions; | |
1143 | vbasedev->flags = dev_info.flags; | |
1144 | ||
1145 | trace_vfio_get_device(name, dev_info.flags, dev_info.num_regions, | |
1146 | dev_info.num_irqs); | |
1147 | ||
1148 | vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET); | |
217e9fdc | 1149 | return 0; |
e2c7d025 EA |
1150 | } |
1151 | ||
1152 | void vfio_put_base_device(VFIODevice *vbasedev) | |
1153 | { | |
77a10d04 PB |
1154 | if (!vbasedev->group) { |
1155 | return; | |
1156 | } | |
e2c7d025 EA |
1157 | QLIST_REMOVE(vbasedev, next); |
1158 | vbasedev->group = NULL; | |
1159 | trace_vfio_put_base_device(vbasedev->fd); | |
1160 | close(vbasedev->fd); | |
1161 | } | |
1162 | ||
46900226 AW |
1163 | int vfio_get_region_info(VFIODevice *vbasedev, int index, |
1164 | struct vfio_region_info **info) | |
1165 | { | |
1166 | size_t argsz = sizeof(struct vfio_region_info); | |
1167 | ||
1168 | *info = g_malloc0(argsz); | |
1169 | ||
1170 | (*info)->index = index; | |
b53b0f69 | 1171 | retry: |
46900226 AW |
1172 | (*info)->argsz = argsz; |
1173 | ||
1174 | if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) { | |
1175 | g_free(*info); | |
e61a424f | 1176 | *info = NULL; |
46900226 AW |
1177 | return -errno; |
1178 | } | |
1179 | ||
b53b0f69 AW |
1180 | if ((*info)->argsz > argsz) { |
1181 | argsz = (*info)->argsz; | |
1182 | *info = g_realloc(*info, argsz); | |
1183 | ||
1184 | goto retry; | |
1185 | } | |
1186 | ||
46900226 AW |
1187 | return 0; |
1188 | } | |
1189 | ||
e61a424f AW |
1190 | int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type, |
1191 | uint32_t subtype, struct vfio_region_info **info) | |
1192 | { | |
1193 | int i; | |
1194 | ||
1195 | for (i = 0; i < vbasedev->num_regions; i++) { | |
1196 | struct vfio_info_cap_header *hdr; | |
1197 | struct vfio_region_info_cap_type *cap_type; | |
1198 | ||
1199 | if (vfio_get_region_info(vbasedev, i, info)) { | |
1200 | continue; | |
1201 | } | |
1202 | ||
1203 | hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE); | |
1204 | if (!hdr) { | |
1205 | g_free(*info); | |
1206 | continue; | |
1207 | } | |
1208 | ||
1209 | cap_type = container_of(hdr, struct vfio_region_info_cap_type, header); | |
1210 | ||
1211 | trace_vfio_get_dev_region(vbasedev->name, i, | |
1212 | cap_type->type, cap_type->subtype); | |
1213 | ||
1214 | if (cap_type->type == type && cap_type->subtype == subtype) { | |
1215 | return 0; | |
1216 | } | |
1217 | ||
1218 | g_free(*info); | |
1219 | } | |
1220 | ||
1221 | *info = NULL; | |
1222 | return -ENODEV; | |
1223 | } | |
1224 | ||
3153119e DG |
1225 | /* |
1226 | * Interfaces for IBM EEH (Enhanced Error Handling) | |
1227 | */ | |
1228 | static bool vfio_eeh_container_ok(VFIOContainer *container) | |
1229 | { | |
1230 | /* | |
1231 | * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO | |
1232 | * implementation is broken if there are multiple groups in a | |
1233 | * container. The hardware works in units of Partitionable | |
1234 | * Endpoints (== IOMMU groups) and the EEH operations naively | |
1235 | * iterate across all groups in the container, without any logic | |
1236 | * to make sure the groups have their state synchronized. For | |
1237 | * certain operations (ENABLE) that might be ok, until an error | |
1238 | * occurs, but for others (GET_STATE) it's clearly broken. | |
1239 | */ | |
1240 | ||
1241 | /* | |
1242 | * XXX Once fixed kernels exist, test for them here | |
1243 | */ | |
1244 | ||
1245 | if (QLIST_EMPTY(&container->group_list)) { | |
1246 | return false; | |
1247 | } | |
1248 | ||
1249 | if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) { | |
1250 | return false; | |
1251 | } | |
1252 | ||
1253 | return true; | |
1254 | } | |
1255 | ||
1256 | static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op) | |
1257 | { | |
1258 | struct vfio_eeh_pe_op pe_op = { | |
1259 | .argsz = sizeof(pe_op), | |
1260 | .op = op, | |
1261 | }; | |
1262 | int ret; | |
1263 | ||
1264 | if (!vfio_eeh_container_ok(container)) { | |
1265 | error_report("vfio/eeh: EEH_PE_OP 0x%x: " | |
1266 | "kernel requires a container with exactly one group", op); | |
1267 | return -EPERM; | |
1268 | } | |
1269 | ||
1270 | ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op); | |
1271 | if (ret < 0) { | |
1272 | error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op); | |
1273 | return -errno; | |
1274 | } | |
1275 | ||
d917e88d | 1276 | return ret; |
3153119e DG |
1277 | } |
1278 | ||
1279 | static VFIOContainer *vfio_eeh_as_container(AddressSpace *as) | |
1280 | { | |
1281 | VFIOAddressSpace *space = vfio_get_address_space(as); | |
1282 | VFIOContainer *container = NULL; | |
1283 | ||
1284 | if (QLIST_EMPTY(&space->containers)) { | |
1285 | /* No containers to act on */ | |
1286 | goto out; | |
1287 | } | |
1288 | ||
1289 | container = QLIST_FIRST(&space->containers); | |
1290 | ||
1291 | if (QLIST_NEXT(container, next)) { | |
1292 | /* We don't yet have logic to synchronize EEH state across | |
1293 | * multiple containers */ | |
1294 | container = NULL; | |
1295 | goto out; | |
1296 | } | |
1297 | ||
1298 | out: | |
1299 | vfio_put_address_space(space); | |
1300 | return container; | |
1301 | } | |
1302 | ||
1303 | bool vfio_eeh_as_ok(AddressSpace *as) | |
1304 | { | |
1305 | VFIOContainer *container = vfio_eeh_as_container(as); | |
1306 | ||
1307 | return (container != NULL) && vfio_eeh_container_ok(container); | |
1308 | } | |
1309 | ||
1310 | int vfio_eeh_as_op(AddressSpace *as, uint32_t op) | |
1311 | { | |
1312 | VFIOContainer *container = vfio_eeh_as_container(as); | |
1313 | ||
1314 | if (!container) { | |
1315 | return -ENODEV; | |
1316 | } | |
1317 | return vfio_eeh_container_op(container, op); | |
1318 | } |