]>
Commit | Line | Data |
---|---|---|
c00d61d8 AW |
1 | /* |
2 | * device quirks for PCI devices | |
3 | * | |
4 | * Copyright Red Hat, Inc. 2012-2015 | |
5 | * | |
6 | * Authors: | |
7 | * Alex Williamson <alex.williamson@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
c6eacb1a | 13 | #include "qemu/osdep.h" |
e0255bb1 | 14 | #include "qemu/units.h" |
c4c45e94 | 15 | #include "qemu/error-report.h" |
c958c51d | 16 | #include "qemu/main-loop.h" |
c4c45e94 AW |
17 | #include "qemu/range.h" |
18 | #include "qapi/error.h" | |
dfbee78d | 19 | #include "qapi/visitor.h" |
2b1dbd0d | 20 | #include <sys/ioctl.h> |
c4c45e94 | 21 | #include "hw/nvram/fw_cfg.h" |
c00d61d8 AW |
22 | #include "pci.h" |
23 | #include "trace.h" | |
c00d61d8 | 24 | |
056dfcb6 AW |
25 | /* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */ |
26 | static bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device) | |
27 | { | |
ff635e37 AW |
28 | return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) && |
29 | (device == PCI_ANY_ID || device == vdev->device_id); | |
056dfcb6 AW |
30 | } |
31 | ||
0d38fb1c AW |
32 | static bool vfio_is_vga(VFIOPCIDevice *vdev) |
33 | { | |
34 | PCIDevice *pdev = &vdev->pdev; | |
35 | uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE); | |
36 | ||
37 | return class == PCI_CLASS_DISPLAY_VGA; | |
38 | } | |
39 | ||
c00d61d8 AW |
40 | /* |
41 | * List of device ids/vendor ids for which to disable | |
42 | * option rom loading. This avoids the guest hangs during rom | |
43 | * execution as noticed with the BCM 57810 card for lack of a | |
44 | * more better way to handle such issues. | |
45 | * The user can still override by specifying a romfile or | |
46 | * rombar=1. | |
47 | * Please see https://bugs.launchpad.net/qemu/+bug/1284874 | |
48 | * for an analysis of the 57810 card hang. When adding | |
49 | * a new vendor id/device id combination below, please also add | |
50 | * your card/environment details and information that could | |
51 | * help in debugging to the bug tracking this issue | |
52 | */ | |
056dfcb6 AW |
53 | static const struct { |
54 | uint32_t vendor; | |
55 | uint32_t device; | |
56 | } romblacklist[] = { | |
57 | { 0x14e4, 0x168e }, /* Broadcom BCM 57810 */ | |
c00d61d8 AW |
58 | }; |
59 | ||
60 | bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev) | |
61 | { | |
056dfcb6 | 62 | int i; |
c00d61d8 | 63 | |
056dfcb6 AW |
64 | for (i = 0 ; i < ARRAY_SIZE(romblacklist); i++) { |
65 | if (vfio_pci_is(vdev, romblacklist[i].vendor, romblacklist[i].device)) { | |
66 | trace_vfio_quirk_rom_blacklisted(vdev->vbasedev.name, | |
67 | romblacklist[i].vendor, | |
68 | romblacklist[i].device); | |
69 | return true; | |
c00d61d8 | 70 | } |
c00d61d8 | 71 | } |
c00d61d8 AW |
72 | return false; |
73 | } | |
74 | ||
75 | /* | |
0e54f24a | 76 | * Device specific region quirks (mostly backdoors to PCI config space) |
c00d61d8 AW |
77 | */ |
78 | ||
0e54f24a AW |
79 | /* |
80 | * The generic window quirks operate on an address and data register, | |
81 | * vfio_generic_window_address_quirk handles the address register and | |
82 | * vfio_generic_window_data_quirk handles the data register. These ops | |
83 | * pass reads and writes through to hardware until a value matching the | |
84 | * stored address match/mask is written. When this occurs, the data | |
85 | * register access emulated PCI config space for the device rather than | |
86 | * passing through accesses. This enables devices where PCI config space | |
87 | * is accessible behind a window register to maintain the virtualization | |
88 | * provided through vfio. | |
89 | */ | |
90 | typedef struct VFIOConfigWindowMatch { | |
91 | uint32_t match; | |
92 | uint32_t mask; | |
93 | } VFIOConfigWindowMatch; | |
94 | ||
95 | typedef struct VFIOConfigWindowQuirk { | |
96 | struct VFIOPCIDevice *vdev; | |
97 | ||
98 | uint32_t address_val; | |
99 | ||
100 | uint32_t address_offset; | |
101 | uint32_t data_offset; | |
102 | ||
103 | bool window_enabled; | |
104 | uint8_t bar; | |
105 | ||
106 | MemoryRegion *addr_mem; | |
107 | MemoryRegion *data_mem; | |
108 | ||
109 | uint32_t nr_matches; | |
110 | VFIOConfigWindowMatch matches[]; | |
111 | } VFIOConfigWindowQuirk; | |
112 | ||
113 | static uint64_t vfio_generic_window_quirk_address_read(void *opaque, | |
114 | hwaddr addr, | |
115 | unsigned size) | |
116 | { | |
117 | VFIOConfigWindowQuirk *window = opaque; | |
118 | VFIOPCIDevice *vdev = window->vdev; | |
119 | ||
120 | return vfio_region_read(&vdev->bars[window->bar].region, | |
121 | addr + window->address_offset, size); | |
122 | } | |
123 | ||
124 | static void vfio_generic_window_quirk_address_write(void *opaque, hwaddr addr, | |
125 | uint64_t data, | |
126 | unsigned size) | |
127 | { | |
128 | VFIOConfigWindowQuirk *window = opaque; | |
129 | VFIOPCIDevice *vdev = window->vdev; | |
130 | int i; | |
131 | ||
132 | window->window_enabled = false; | |
133 | ||
134 | vfio_region_write(&vdev->bars[window->bar].region, | |
135 | addr + window->address_offset, data, size); | |
136 | ||
137 | for (i = 0; i < window->nr_matches; i++) { | |
138 | if ((data & ~window->matches[i].mask) == window->matches[i].match) { | |
139 | window->window_enabled = true; | |
140 | window->address_val = data & window->matches[i].mask; | |
141 | trace_vfio_quirk_generic_window_address_write(vdev->vbasedev.name, | |
142 | memory_region_name(window->addr_mem), data); | |
143 | break; | |
144 | } | |
145 | } | |
146 | } | |
147 | ||
148 | static const MemoryRegionOps vfio_generic_window_address_quirk = { | |
149 | .read = vfio_generic_window_quirk_address_read, | |
150 | .write = vfio_generic_window_quirk_address_write, | |
151 | .endianness = DEVICE_LITTLE_ENDIAN, | |
152 | }; | |
153 | ||
154 | static uint64_t vfio_generic_window_quirk_data_read(void *opaque, | |
155 | hwaddr addr, unsigned size) | |
156 | { | |
157 | VFIOConfigWindowQuirk *window = opaque; | |
158 | VFIOPCIDevice *vdev = window->vdev; | |
159 | uint64_t data; | |
160 | ||
161 | /* Always read data reg, discard if window enabled */ | |
162 | data = vfio_region_read(&vdev->bars[window->bar].region, | |
163 | addr + window->data_offset, size); | |
164 | ||
165 | if (window->window_enabled) { | |
166 | data = vfio_pci_read_config(&vdev->pdev, window->address_val, size); | |
167 | trace_vfio_quirk_generic_window_data_read(vdev->vbasedev.name, | |
168 | memory_region_name(window->data_mem), data); | |
169 | } | |
170 | ||
171 | return data; | |
172 | } | |
173 | ||
174 | static void vfio_generic_window_quirk_data_write(void *opaque, hwaddr addr, | |
175 | uint64_t data, unsigned size) | |
176 | { | |
177 | VFIOConfigWindowQuirk *window = opaque; | |
178 | VFIOPCIDevice *vdev = window->vdev; | |
179 | ||
180 | if (window->window_enabled) { | |
181 | vfio_pci_write_config(&vdev->pdev, window->address_val, data, size); | |
182 | trace_vfio_quirk_generic_window_data_write(vdev->vbasedev.name, | |
183 | memory_region_name(window->data_mem), data); | |
184 | return; | |
185 | } | |
186 | ||
187 | vfio_region_write(&vdev->bars[window->bar].region, | |
188 | addr + window->data_offset, data, size); | |
189 | } | |
190 | ||
191 | static const MemoryRegionOps vfio_generic_window_data_quirk = { | |
192 | .read = vfio_generic_window_quirk_data_read, | |
193 | .write = vfio_generic_window_quirk_data_write, | |
194 | .endianness = DEVICE_LITTLE_ENDIAN, | |
195 | }; | |
196 | ||
0d38fb1c AW |
197 | /* |
198 | * The generic mirror quirk handles devices which expose PCI config space | |
199 | * through a region within a BAR. When enabled, reads and writes are | |
200 | * redirected through to emulated PCI config space. XXX if PCI config space | |
201 | * used memory regions, this could just be an alias. | |
202 | */ | |
203 | typedef struct VFIOConfigMirrorQuirk { | |
204 | struct VFIOPCIDevice *vdev; | |
205 | uint32_t offset; | |
206 | uint8_t bar; | |
207 | MemoryRegion *mem; | |
c958c51d | 208 | uint8_t data[]; |
0d38fb1c AW |
209 | } VFIOConfigMirrorQuirk; |
210 | ||
211 | static uint64_t vfio_generic_quirk_mirror_read(void *opaque, | |
212 | hwaddr addr, unsigned size) | |
213 | { | |
214 | VFIOConfigMirrorQuirk *mirror = opaque; | |
215 | VFIOPCIDevice *vdev = mirror->vdev; | |
216 | uint64_t data; | |
217 | ||
218 | /* Read and discard in case the hardware cares */ | |
219 | (void)vfio_region_read(&vdev->bars[mirror->bar].region, | |
220 | addr + mirror->offset, size); | |
221 | ||
222 | data = vfio_pci_read_config(&vdev->pdev, addr, size); | |
223 | trace_vfio_quirk_generic_mirror_read(vdev->vbasedev.name, | |
224 | memory_region_name(mirror->mem), | |
225 | addr, data); | |
226 | return data; | |
227 | } | |
228 | ||
229 | static void vfio_generic_quirk_mirror_write(void *opaque, hwaddr addr, | |
230 | uint64_t data, unsigned size) | |
231 | { | |
232 | VFIOConfigMirrorQuirk *mirror = opaque; | |
233 | VFIOPCIDevice *vdev = mirror->vdev; | |
234 | ||
235 | vfio_pci_write_config(&vdev->pdev, addr, data, size); | |
236 | trace_vfio_quirk_generic_mirror_write(vdev->vbasedev.name, | |
237 | memory_region_name(mirror->mem), | |
238 | addr, data); | |
239 | } | |
240 | ||
241 | static const MemoryRegionOps vfio_generic_mirror_quirk = { | |
242 | .read = vfio_generic_quirk_mirror_read, | |
243 | .write = vfio_generic_quirk_mirror_write, | |
244 | .endianness = DEVICE_LITTLE_ENDIAN, | |
245 | }; | |
246 | ||
c00d61d8 AW |
247 | /* Is range1 fully contained within range2? */ |
248 | static bool vfio_range_contained(uint64_t first1, uint64_t len1, | |
249 | uint64_t first2, uint64_t len2) { | |
250 | return (first1 >= first2 && first1 + len1 <= first2 + len2); | |
251 | } | |
252 | ||
c00d61d8 AW |
253 | #define PCI_VENDOR_ID_ATI 0x1002 |
254 | ||
255 | /* | |
256 | * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR | |
257 | * through VGA register 0x3c3. On newer cards, the I/O port BAR is always | |
258 | * BAR4 (older cards like the X550 used BAR1, but we don't care to support | |
259 | * those). Note that on bare metal, a read of 0x3c3 doesn't always return the | |
260 | * I/O port BAR address. Originally this was coded to return the virtual BAR | |
261 | * address only if the physical register read returns the actual BAR address, | |
262 | * but users have reported greater success if we return the virtual address | |
263 | * unconditionally. | |
264 | */ | |
265 | static uint64_t vfio_ati_3c3_quirk_read(void *opaque, | |
266 | hwaddr addr, unsigned size) | |
267 | { | |
b946d286 | 268 | VFIOPCIDevice *vdev = opaque; |
c00d61d8 | 269 | uint64_t data = vfio_pci_read_config(&vdev->pdev, |
b946d286 AW |
270 | PCI_BASE_ADDRESS_4 + 1, size); |
271 | ||
272 | trace_vfio_quirk_ati_3c3_read(vdev->vbasedev.name, data); | |
c00d61d8 AW |
273 | |
274 | return data; | |
275 | } | |
276 | ||
277 | static const MemoryRegionOps vfio_ati_3c3_quirk = { | |
278 | .read = vfio_ati_3c3_quirk_read, | |
279 | .endianness = DEVICE_LITTLE_ENDIAN, | |
280 | }; | |
281 | ||
bcf3c3d0 AW |
282 | static VFIOQuirk *vfio_quirk_alloc(int nr_mem) |
283 | { | |
284 | VFIOQuirk *quirk = g_new0(VFIOQuirk, 1); | |
c958c51d | 285 | QLIST_INIT(&quirk->ioeventfds); |
bcf3c3d0 AW |
286 | quirk->mem = g_new0(MemoryRegion, nr_mem); |
287 | quirk->nr_mem = nr_mem; | |
288 | ||
289 | return quirk; | |
290 | } | |
291 | ||
2b1dbd0d | 292 | static void vfio_ioeventfd_exit(VFIOPCIDevice *vdev, VFIOIOEventFD *ioeventfd) |
c958c51d AW |
293 | { |
294 | QLIST_REMOVE(ioeventfd, next); | |
295 | memory_region_del_eventfd(ioeventfd->mr, ioeventfd->addr, ioeventfd->size, | |
296 | true, ioeventfd->data, &ioeventfd->e); | |
2b1dbd0d AW |
297 | |
298 | if (ioeventfd->vfio) { | |
299 | struct vfio_device_ioeventfd vfio_ioeventfd; | |
300 | ||
301 | vfio_ioeventfd.argsz = sizeof(vfio_ioeventfd); | |
302 | vfio_ioeventfd.flags = ioeventfd->size; | |
303 | vfio_ioeventfd.data = ioeventfd->data; | |
304 | vfio_ioeventfd.offset = ioeventfd->region->fd_offset + | |
305 | ioeventfd->region_addr; | |
306 | vfio_ioeventfd.fd = -1; | |
307 | ||
308 | if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_IOEVENTFD, &vfio_ioeventfd)) { | |
309 | error_report("Failed to remove vfio ioeventfd for %s+0x%" | |
310 | HWADDR_PRIx"[%d]:0x%"PRIx64" (%m)", | |
311 | memory_region_name(ioeventfd->mr), ioeventfd->addr, | |
312 | ioeventfd->size, ioeventfd->data); | |
313 | } | |
314 | } else { | |
315 | qemu_set_fd_handler(event_notifier_get_fd(&ioeventfd->e), | |
316 | NULL, NULL, NULL); | |
317 | } | |
318 | ||
c958c51d AW |
319 | event_notifier_cleanup(&ioeventfd->e); |
320 | trace_vfio_ioeventfd_exit(memory_region_name(ioeventfd->mr), | |
321 | (uint64_t)ioeventfd->addr, ioeventfd->size, | |
322 | ioeventfd->data); | |
323 | g_free(ioeventfd); | |
324 | } | |
325 | ||
326 | static void vfio_drop_dynamic_eventfds(VFIOPCIDevice *vdev, VFIOQuirk *quirk) | |
327 | { | |
328 | VFIOIOEventFD *ioeventfd, *tmp; | |
329 | ||
330 | QLIST_FOREACH_SAFE(ioeventfd, &quirk->ioeventfds, next, tmp) { | |
331 | if (ioeventfd->dynamic) { | |
2b1dbd0d | 332 | vfio_ioeventfd_exit(vdev, ioeventfd); |
c958c51d AW |
333 | } |
334 | } | |
335 | } | |
336 | ||
337 | static void vfio_ioeventfd_handler(void *opaque) | |
338 | { | |
339 | VFIOIOEventFD *ioeventfd = opaque; | |
340 | ||
341 | if (event_notifier_test_and_clear(&ioeventfd->e)) { | |
342 | vfio_region_write(ioeventfd->region, ioeventfd->region_addr, | |
343 | ioeventfd->data, ioeventfd->size); | |
344 | trace_vfio_ioeventfd_handler(memory_region_name(ioeventfd->mr), | |
345 | (uint64_t)ioeventfd->addr, ioeventfd->size, | |
346 | ioeventfd->data); | |
347 | } | |
348 | } | |
349 | ||
350 | static VFIOIOEventFD *vfio_ioeventfd_init(VFIOPCIDevice *vdev, | |
351 | MemoryRegion *mr, hwaddr addr, | |
352 | unsigned size, uint64_t data, | |
353 | VFIORegion *region, | |
354 | hwaddr region_addr, bool dynamic) | |
355 | { | |
356 | VFIOIOEventFD *ioeventfd; | |
357 | ||
358 | if (vdev->no_kvm_ioeventfd) { | |
359 | return NULL; | |
360 | } | |
361 | ||
362 | ioeventfd = g_malloc0(sizeof(*ioeventfd)); | |
363 | ||
364 | if (event_notifier_init(&ioeventfd->e, 0)) { | |
365 | g_free(ioeventfd); | |
366 | return NULL; | |
367 | } | |
368 | ||
369 | /* | |
370 | * MemoryRegion and relative offset, plus additional ioeventfd setup | |
371 | * parameters for configuring and later tearing down KVM ioeventfd. | |
372 | */ | |
373 | ioeventfd->mr = mr; | |
374 | ioeventfd->addr = addr; | |
375 | ioeventfd->size = size; | |
376 | ioeventfd->data = data; | |
377 | ioeventfd->dynamic = dynamic; | |
378 | /* | |
379 | * VFIORegion and relative offset for implementing the userspace | |
380 | * handler. data & size fields shared for both uses. | |
381 | */ | |
382 | ioeventfd->region = region; | |
383 | ioeventfd->region_addr = region_addr; | |
384 | ||
2b1dbd0d AW |
385 | if (!vdev->no_vfio_ioeventfd) { |
386 | struct vfio_device_ioeventfd vfio_ioeventfd; | |
387 | ||
388 | vfio_ioeventfd.argsz = sizeof(vfio_ioeventfd); | |
389 | vfio_ioeventfd.flags = ioeventfd->size; | |
390 | vfio_ioeventfd.data = ioeventfd->data; | |
391 | vfio_ioeventfd.offset = ioeventfd->region->fd_offset + | |
392 | ioeventfd->region_addr; | |
393 | vfio_ioeventfd.fd = event_notifier_get_fd(&ioeventfd->e); | |
394 | ||
395 | ioeventfd->vfio = !ioctl(vdev->vbasedev.fd, | |
396 | VFIO_DEVICE_IOEVENTFD, &vfio_ioeventfd); | |
397 | } | |
398 | ||
399 | if (!ioeventfd->vfio) { | |
400 | qemu_set_fd_handler(event_notifier_get_fd(&ioeventfd->e), | |
401 | vfio_ioeventfd_handler, NULL, ioeventfd); | |
402 | } | |
403 | ||
c958c51d AW |
404 | memory_region_add_eventfd(ioeventfd->mr, ioeventfd->addr, ioeventfd->size, |
405 | true, ioeventfd->data, &ioeventfd->e); | |
406 | trace_vfio_ioeventfd_init(memory_region_name(mr), (uint64_t)addr, | |
2b1dbd0d | 407 | size, data, ioeventfd->vfio); |
c958c51d AW |
408 | |
409 | return ioeventfd; | |
410 | } | |
411 | ||
c00d61d8 AW |
412 | static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev) |
413 | { | |
c00d61d8 | 414 | VFIOQuirk *quirk; |
c00d61d8 AW |
415 | |
416 | /* | |
417 | * As long as the BAR is >= 256 bytes it will be aligned such that the | |
418 | * lower byte is always zero. Filter out anything else, if it exists. | |
419 | */ | |
b946d286 AW |
420 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
421 | !vdev->bars[4].ioport || vdev->bars[4].region.size < 256) { | |
c00d61d8 AW |
422 | return; |
423 | } | |
424 | ||
bcf3c3d0 | 425 | quirk = vfio_quirk_alloc(1); |
c00d61d8 | 426 | |
b946d286 | 427 | memory_region_init_io(quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, vdev, |
c00d61d8 | 428 | "vfio-ati-3c3-quirk", 1); |
2d82f8a3 | 429 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
8c4f2348 | 430 | 3 /* offset 3 bytes from 0x3c0 */, quirk->mem); |
c00d61d8 | 431 | |
2d82f8a3 | 432 | QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks, |
c00d61d8 AW |
433 | quirk, next); |
434 | ||
b946d286 | 435 | trace_vfio_quirk_ati_3c3_probe(vdev->vbasedev.name); |
c00d61d8 AW |
436 | } |
437 | ||
438 | /* | |
0e54f24a | 439 | * Newer ATI/AMD devices, including HD5450 and HD7850, have a mirror to PCI |
c00d61d8 AW |
440 | * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access |
441 | * the MMIO space directly, but a window to this space is provided through | |
442 | * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the | |
443 | * data register. When the address is programmed to a range of 0x4000-0x4fff | |
444 | * PCI configuration space is available. Experimentation seems to indicate | |
0e54f24a | 445 | * that read-only may be provided by hardware. |
c00d61d8 | 446 | */ |
0e54f24a | 447 | static void vfio_probe_ati_bar4_quirk(VFIOPCIDevice *vdev, int nr) |
c00d61d8 | 448 | { |
c00d61d8 | 449 | VFIOQuirk *quirk; |
0e54f24a | 450 | VFIOConfigWindowQuirk *window; |
c00d61d8 | 451 | |
0e54f24a AW |
452 | /* This windows doesn't seem to be used except by legacy VGA code */ |
453 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || | |
4d3fc4fd | 454 | !vdev->vga || nr != 4) { |
c00d61d8 AW |
455 | return; |
456 | } | |
457 | ||
bcf3c3d0 | 458 | quirk = vfio_quirk_alloc(2); |
0e54f24a AW |
459 | window = quirk->data = g_malloc0(sizeof(*window) + |
460 | sizeof(VFIOConfigWindowMatch)); | |
461 | window->vdev = vdev; | |
462 | window->address_offset = 0; | |
463 | window->data_offset = 4; | |
464 | window->nr_matches = 1; | |
465 | window->matches[0].match = 0x4000; | |
f5793fd9 | 466 | window->matches[0].mask = vdev->config_size - 1; |
0e54f24a AW |
467 | window->bar = nr; |
468 | window->addr_mem = &quirk->mem[0]; | |
469 | window->data_mem = &quirk->mem[1]; | |
470 | ||
471 | memory_region_init_io(window->addr_mem, OBJECT(vdev), | |
472 | &vfio_generic_window_address_quirk, window, | |
473 | "vfio-ati-bar4-window-address-quirk", 4); | |
db0da029 | 474 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0e54f24a AW |
475 | window->address_offset, |
476 | window->addr_mem, 1); | |
8c4f2348 | 477 | |
0e54f24a AW |
478 | memory_region_init_io(window->data_mem, OBJECT(vdev), |
479 | &vfio_generic_window_data_quirk, window, | |
480 | "vfio-ati-bar4-window-data-quirk", 4); | |
db0da029 | 481 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0e54f24a AW |
482 | window->data_offset, |
483 | window->data_mem, 1); | |
c00d61d8 AW |
484 | |
485 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
486 | ||
0e54f24a | 487 | trace_vfio_quirk_ati_bar4_probe(vdev->vbasedev.name); |
c00d61d8 AW |
488 | } |
489 | ||
490 | /* | |
0d38fb1c | 491 | * Trap the BAR2 MMIO mirror to config space as well. |
c00d61d8 | 492 | */ |
0d38fb1c | 493 | static void vfio_probe_ati_bar2_quirk(VFIOPCIDevice *vdev, int nr) |
c00d61d8 | 494 | { |
c00d61d8 | 495 | VFIOQuirk *quirk; |
0d38fb1c | 496 | VFIOConfigMirrorQuirk *mirror; |
c00d61d8 AW |
497 | |
498 | /* Only enable on newer devices where BAR2 is 64bit */ | |
0d38fb1c | 499 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
4d3fc4fd | 500 | !vdev->vga || nr != 2 || !vdev->bars[2].mem64) { |
c00d61d8 AW |
501 | return; |
502 | } | |
503 | ||
bcf3c3d0 | 504 | quirk = vfio_quirk_alloc(1); |
0d38fb1c | 505 | mirror = quirk->data = g_malloc0(sizeof(*mirror)); |
bcf3c3d0 | 506 | mirror->mem = quirk->mem; |
0d38fb1c AW |
507 | mirror->vdev = vdev; |
508 | mirror->offset = 0x4000; | |
509 | mirror->bar = nr; | |
510 | ||
511 | memory_region_init_io(mirror->mem, OBJECT(vdev), | |
512 | &vfio_generic_mirror_quirk, mirror, | |
513 | "vfio-ati-bar2-4000-quirk", PCI_CONFIG_SPACE_SIZE); | |
db0da029 | 514 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0d38fb1c | 515 | mirror->offset, mirror->mem, 1); |
c00d61d8 AW |
516 | |
517 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
518 | ||
0d38fb1c | 519 | trace_vfio_quirk_ati_bar2_probe(vdev->vbasedev.name); |
c00d61d8 AW |
520 | } |
521 | ||
522 | /* | |
523 | * Older ATI/AMD cards like the X550 have a similar window to that above. | |
524 | * I/O port BAR1 provides a window to a mirror of PCI config space located | |
525 | * in BAR2 at offset 0xf00. We don't care to support such older cards, but | |
526 | * note it for future reference. | |
527 | */ | |
528 | ||
529 | #define PCI_VENDOR_ID_NVIDIA 0x10de | |
530 | ||
531 | /* | |
532 | * Nvidia has several different methods to get to config space, the | |
533 | * nouveu project has several of these documented here: | |
534 | * https://github.com/pathscale/envytools/tree/master/hwdocs | |
535 | * | |
536 | * The first quirk is actually not documented in envytools and is found | |
537 | * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an | |
538 | * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access | |
539 | * the mirror of PCI config space found at BAR0 offset 0x1800. The access | |
540 | * sequence first writes 0x338 to I/O port 0x3d4. The target offset is | |
541 | * then written to 0x3d0. Finally 0x538 is written for a read and 0x738 | |
542 | * is written for a write to 0x3d4. The BAR0 offset is then accessible | |
543 | * through 0x3d0. This quirk doesn't seem to be necessary on newer cards | |
544 | * that use the I/O port BAR5 window but it doesn't hurt to leave it. | |
545 | */ | |
6029a424 AW |
546 | typedef enum {NONE = 0, SELECT, WINDOW, READ, WRITE} VFIONvidia3d0State; |
547 | static const char *nv3d0_states[] = { "NONE", "SELECT", | |
548 | "WINDOW", "READ", "WRITE" }; | |
c00d61d8 | 549 | |
6029a424 AW |
550 | typedef struct VFIONvidia3d0Quirk { |
551 | VFIOPCIDevice *vdev; | |
552 | VFIONvidia3d0State state; | |
553 | uint32_t offset; | |
554 | } VFIONvidia3d0Quirk; | |
555 | ||
556 | static uint64_t vfio_nvidia_3d4_quirk_read(void *opaque, | |
c00d61d8 AW |
557 | hwaddr addr, unsigned size) |
558 | { | |
6029a424 | 559 | VFIONvidia3d0Quirk *quirk = opaque; |
c00d61d8 | 560 | VFIOPCIDevice *vdev = quirk->vdev; |
c00d61d8 | 561 | |
6029a424 | 562 | quirk->state = NONE; |
c00d61d8 | 563 | |
2d82f8a3 | 564 | return vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
6029a424 | 565 | addr + 0x14, size); |
c00d61d8 AW |
566 | } |
567 | ||
6029a424 | 568 | static void vfio_nvidia_3d4_quirk_write(void *opaque, hwaddr addr, |
c00d61d8 AW |
569 | uint64_t data, unsigned size) |
570 | { | |
6029a424 | 571 | VFIONvidia3d0Quirk *quirk = opaque; |
c00d61d8 | 572 | VFIOPCIDevice *vdev = quirk->vdev; |
6029a424 | 573 | VFIONvidia3d0State old_state = quirk->state; |
c00d61d8 | 574 | |
6029a424 AW |
575 | quirk->state = NONE; |
576 | ||
577 | switch (data) { | |
578 | case 0x338: | |
579 | if (old_state == NONE) { | |
580 | quirk->state = SELECT; | |
581 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, | |
582 | nv3d0_states[quirk->state]); | |
c00d61d8 AW |
583 | } |
584 | break; | |
6029a424 AW |
585 | case 0x538: |
586 | if (old_state == WINDOW) { | |
587 | quirk->state = READ; | |
588 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, | |
589 | nv3d0_states[quirk->state]); | |
c00d61d8 AW |
590 | } |
591 | break; | |
6029a424 AW |
592 | case 0x738: |
593 | if (old_state == WINDOW) { | |
594 | quirk->state = WRITE; | |
595 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, | |
596 | nv3d0_states[quirk->state]); | |
c00d61d8 AW |
597 | } |
598 | break; | |
6029a424 AW |
599 | } |
600 | ||
2d82f8a3 | 601 | vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
6029a424 AW |
602 | addr + 0x14, data, size); |
603 | } | |
604 | ||
605 | static const MemoryRegionOps vfio_nvidia_3d4_quirk = { | |
606 | .read = vfio_nvidia_3d4_quirk_read, | |
607 | .write = vfio_nvidia_3d4_quirk_write, | |
608 | .endianness = DEVICE_LITTLE_ENDIAN, | |
609 | }; | |
610 | ||
611 | static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque, | |
612 | hwaddr addr, unsigned size) | |
613 | { | |
614 | VFIONvidia3d0Quirk *quirk = opaque; | |
615 | VFIOPCIDevice *vdev = quirk->vdev; | |
616 | VFIONvidia3d0State old_state = quirk->state; | |
2d82f8a3 | 617 | uint64_t data = vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
6029a424 AW |
618 | addr + 0x10, size); |
619 | ||
620 | quirk->state = NONE; | |
621 | ||
622 | if (old_state == READ && | |
623 | (quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) { | |
624 | uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1); | |
625 | ||
626 | data = vfio_pci_read_config(&vdev->pdev, offset, size); | |
627 | trace_vfio_quirk_nvidia_3d0_read(vdev->vbasedev.name, | |
628 | offset, size, data); | |
629 | } | |
630 | ||
631 | return data; | |
632 | } | |
633 | ||
634 | static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr, | |
635 | uint64_t data, unsigned size) | |
636 | { | |
637 | VFIONvidia3d0Quirk *quirk = opaque; | |
638 | VFIOPCIDevice *vdev = quirk->vdev; | |
639 | VFIONvidia3d0State old_state = quirk->state; | |
640 | ||
641 | quirk->state = NONE; | |
642 | ||
643 | if (old_state == SELECT) { | |
644 | quirk->offset = (uint32_t)data; | |
645 | quirk->state = WINDOW; | |
646 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, | |
647 | nv3d0_states[quirk->state]); | |
648 | } else if (old_state == WRITE) { | |
649 | if ((quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) { | |
650 | uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1); | |
651 | ||
652 | vfio_pci_write_config(&vdev->pdev, offset, data, size); | |
653 | trace_vfio_quirk_nvidia_3d0_write(vdev->vbasedev.name, | |
654 | offset, data, size); | |
c00d61d8 AW |
655 | return; |
656 | } | |
c00d61d8 AW |
657 | } |
658 | ||
2d82f8a3 | 659 | vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
6029a424 | 660 | addr + 0x10, data, size); |
c00d61d8 AW |
661 | } |
662 | ||
663 | static const MemoryRegionOps vfio_nvidia_3d0_quirk = { | |
664 | .read = vfio_nvidia_3d0_quirk_read, | |
665 | .write = vfio_nvidia_3d0_quirk_write, | |
666 | .endianness = DEVICE_LITTLE_ENDIAN, | |
667 | }; | |
668 | ||
669 | static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev) | |
670 | { | |
c00d61d8 | 671 | VFIOQuirk *quirk; |
6029a424 | 672 | VFIONvidia3d0Quirk *data; |
c00d61d8 | 673 | |
db32d0f4 AW |
674 | if (vdev->no_geforce_quirks || |
675 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || | |
c00d61d8 AW |
676 | !vdev->bars[1].region.size) { |
677 | return; | |
678 | } | |
679 | ||
bcf3c3d0 | 680 | quirk = vfio_quirk_alloc(2); |
6029a424 | 681 | quirk->data = data = g_malloc0(sizeof(*data)); |
6029a424 AW |
682 | data->vdev = vdev; |
683 | ||
684 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_nvidia_3d4_quirk, | |
685 | data, "vfio-nvidia-3d4-quirk", 2); | |
2d82f8a3 | 686 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
6029a424 | 687 | 0x14 /* 0x3c0 + 0x14 */, &quirk->mem[0]); |
8c4f2348 | 688 | |
6029a424 AW |
689 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_nvidia_3d0_quirk, |
690 | data, "vfio-nvidia-3d0-quirk", 2); | |
2d82f8a3 | 691 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
6029a424 | 692 | 0x10 /* 0x3c0 + 0x10 */, &quirk->mem[1]); |
c00d61d8 | 693 | |
2d82f8a3 | 694 | QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks, |
c00d61d8 AW |
695 | quirk, next); |
696 | ||
6029a424 | 697 | trace_vfio_quirk_nvidia_3d0_probe(vdev->vbasedev.name); |
c00d61d8 AW |
698 | } |
699 | ||
700 | /* | |
701 | * The second quirk is documented in envytools. The I/O port BAR5 is just | |
702 | * a set of address/data ports to the MMIO BARs. The BAR we care about is | |
703 | * again BAR0. This backdoor is apparently a bit newer than the one above | |
704 | * so we need to not only trap 256 bytes @0x1800, but all of PCI config | |
705 | * space, including extended space is available at the 4k @0x88000. | |
706 | */ | |
0e54f24a AW |
707 | typedef struct VFIONvidiaBAR5Quirk { |
708 | uint32_t master; | |
709 | uint32_t enable; | |
710 | MemoryRegion *addr_mem; | |
711 | MemoryRegion *data_mem; | |
712 | bool enabled; | |
713 | VFIOConfigWindowQuirk window; /* last for match data */ | |
714 | } VFIONvidiaBAR5Quirk; | |
715 | ||
716 | static void vfio_nvidia_bar5_enable(VFIONvidiaBAR5Quirk *bar5) | |
717 | { | |
718 | VFIOPCIDevice *vdev = bar5->window.vdev; | |
719 | ||
720 | if (((bar5->master & bar5->enable) & 0x1) == bar5->enabled) { | |
721 | return; | |
722 | } | |
723 | ||
724 | bar5->enabled = !bar5->enabled; | |
725 | trace_vfio_quirk_nvidia_bar5_state(vdev->vbasedev.name, | |
726 | bar5->enabled ? "Enable" : "Disable"); | |
727 | memory_region_set_enabled(bar5->addr_mem, bar5->enabled); | |
728 | memory_region_set_enabled(bar5->data_mem, bar5->enabled); | |
729 | } | |
730 | ||
731 | static uint64_t vfio_nvidia_bar5_quirk_master_read(void *opaque, | |
732 | hwaddr addr, unsigned size) | |
733 | { | |
734 | VFIONvidiaBAR5Quirk *bar5 = opaque; | |
735 | VFIOPCIDevice *vdev = bar5->window.vdev; | |
736 | ||
737 | return vfio_region_read(&vdev->bars[5].region, addr, size); | |
738 | } | |
739 | ||
740 | static void vfio_nvidia_bar5_quirk_master_write(void *opaque, hwaddr addr, | |
741 | uint64_t data, unsigned size) | |
742 | { | |
743 | VFIONvidiaBAR5Quirk *bar5 = opaque; | |
744 | VFIOPCIDevice *vdev = bar5->window.vdev; | |
745 | ||
746 | vfio_region_write(&vdev->bars[5].region, addr, data, size); | |
747 | ||
748 | bar5->master = data; | |
749 | vfio_nvidia_bar5_enable(bar5); | |
750 | } | |
751 | ||
752 | static const MemoryRegionOps vfio_nvidia_bar5_quirk_master = { | |
753 | .read = vfio_nvidia_bar5_quirk_master_read, | |
754 | .write = vfio_nvidia_bar5_quirk_master_write, | |
755 | .endianness = DEVICE_LITTLE_ENDIAN, | |
c00d61d8 AW |
756 | }; |
757 | ||
0e54f24a AW |
758 | static uint64_t vfio_nvidia_bar5_quirk_enable_read(void *opaque, |
759 | hwaddr addr, unsigned size) | |
760 | { | |
761 | VFIONvidiaBAR5Quirk *bar5 = opaque; | |
762 | VFIOPCIDevice *vdev = bar5->window.vdev; | |
763 | ||
764 | return vfio_region_read(&vdev->bars[5].region, addr + 4, size); | |
765 | } | |
766 | ||
767 | static void vfio_nvidia_bar5_quirk_enable_write(void *opaque, hwaddr addr, | |
c00d61d8 AW |
768 | uint64_t data, unsigned size) |
769 | { | |
0e54f24a AW |
770 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
771 | VFIOPCIDevice *vdev = bar5->window.vdev; | |
c00d61d8 | 772 | |
0e54f24a | 773 | vfio_region_write(&vdev->bars[5].region, addr + 4, data, size); |
c00d61d8 | 774 | |
0e54f24a AW |
775 | bar5->enable = data; |
776 | vfio_nvidia_bar5_enable(bar5); | |
c00d61d8 AW |
777 | } |
778 | ||
0e54f24a AW |
779 | static const MemoryRegionOps vfio_nvidia_bar5_quirk_enable = { |
780 | .read = vfio_nvidia_bar5_quirk_enable_read, | |
781 | .write = vfio_nvidia_bar5_quirk_enable_write, | |
c00d61d8 AW |
782 | .endianness = DEVICE_LITTLE_ENDIAN, |
783 | }; | |
784 | ||
0e54f24a | 785 | static void vfio_probe_nvidia_bar5_quirk(VFIOPCIDevice *vdev, int nr) |
c00d61d8 | 786 | { |
c00d61d8 | 787 | VFIOQuirk *quirk; |
0e54f24a AW |
788 | VFIONvidiaBAR5Quirk *bar5; |
789 | VFIOConfigWindowQuirk *window; | |
c00d61d8 | 790 | |
db32d0f4 AW |
791 | if (vdev->no_geforce_quirks || |
792 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || | |
8f419c5b | 793 | !vdev->vga || nr != 5 || !vdev->bars[5].ioport) { |
c00d61d8 AW |
794 | return; |
795 | } | |
796 | ||
bcf3c3d0 | 797 | quirk = vfio_quirk_alloc(4); |
0e54f24a AW |
798 | bar5 = quirk->data = g_malloc0(sizeof(*bar5) + |
799 | (sizeof(VFIOConfigWindowMatch) * 2)); | |
800 | window = &bar5->window; | |
801 | ||
802 | window->vdev = vdev; | |
803 | window->address_offset = 0x8; | |
804 | window->data_offset = 0xc; | |
805 | window->nr_matches = 2; | |
806 | window->matches[0].match = 0x1800; | |
807 | window->matches[0].mask = PCI_CONFIG_SPACE_SIZE - 1; | |
808 | window->matches[1].match = 0x88000; | |
f5793fd9 | 809 | window->matches[1].mask = vdev->config_size - 1; |
0e54f24a AW |
810 | window->bar = nr; |
811 | window->addr_mem = bar5->addr_mem = &quirk->mem[0]; | |
812 | window->data_mem = bar5->data_mem = &quirk->mem[1]; | |
813 | ||
814 | memory_region_init_io(window->addr_mem, OBJECT(vdev), | |
815 | &vfio_generic_window_address_quirk, window, | |
816 | "vfio-nvidia-bar5-window-address-quirk", 4); | |
db0da029 | 817 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0e54f24a AW |
818 | window->address_offset, |
819 | window->addr_mem, 1); | |
820 | memory_region_set_enabled(window->addr_mem, false); | |
821 | ||
822 | memory_region_init_io(window->data_mem, OBJECT(vdev), | |
823 | &vfio_generic_window_data_quirk, window, | |
824 | "vfio-nvidia-bar5-window-data-quirk", 4); | |
db0da029 | 825 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0e54f24a AW |
826 | window->data_offset, |
827 | window->data_mem, 1); | |
828 | memory_region_set_enabled(window->data_mem, false); | |
829 | ||
830 | memory_region_init_io(&quirk->mem[2], OBJECT(vdev), | |
831 | &vfio_nvidia_bar5_quirk_master, bar5, | |
832 | "vfio-nvidia-bar5-master-quirk", 4); | |
db0da029 | 833 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0e54f24a | 834 | 0, &quirk->mem[2], 1); |
8c4f2348 | 835 | |
0e54f24a AW |
836 | memory_region_init_io(&quirk->mem[3], OBJECT(vdev), |
837 | &vfio_nvidia_bar5_quirk_enable, bar5, | |
838 | "vfio-nvidia-bar5-enable-quirk", 4); | |
db0da029 | 839 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0e54f24a | 840 | 4, &quirk->mem[3], 1); |
c00d61d8 AW |
841 | |
842 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
843 | ||
0e54f24a | 844 | trace_vfio_quirk_nvidia_bar5_probe(vdev->vbasedev.name); |
c00d61d8 AW |
845 | } |
846 | ||
c958c51d AW |
847 | typedef struct LastDataSet { |
848 | VFIOQuirk *quirk; | |
849 | hwaddr addr; | |
850 | uint64_t data; | |
851 | unsigned size; | |
852 | int hits; | |
853 | int added; | |
854 | } LastDataSet; | |
855 | ||
856 | #define MAX_DYN_IOEVENTFD 10 | |
857 | #define HITS_FOR_IOEVENTFD 10 | |
858 | ||
0d38fb1c AW |
859 | /* |
860 | * Finally, BAR0 itself. We want to redirect any accesses to either | |
861 | * 0x1800 or 0x88000 through the PCI config space access functions. | |
862 | */ | |
863 | static void vfio_nvidia_quirk_mirror_write(void *opaque, hwaddr addr, | |
864 | uint64_t data, unsigned size) | |
c00d61d8 | 865 | { |
0d38fb1c AW |
866 | VFIOConfigMirrorQuirk *mirror = opaque; |
867 | VFIOPCIDevice *vdev = mirror->vdev; | |
c00d61d8 | 868 | PCIDevice *pdev = &vdev->pdev; |
c958c51d | 869 | LastDataSet *last = (LastDataSet *)&mirror->data; |
c00d61d8 | 870 | |
0d38fb1c | 871 | vfio_generic_quirk_mirror_write(opaque, addr, data, size); |
c00d61d8 AW |
872 | |
873 | /* | |
874 | * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the | |
875 | * MSI capability ID register. Both the ID and next register are | |
876 | * read-only, so we allow writes covering either of those to real hw. | |
c00d61d8 AW |
877 | */ |
878 | if ((pdev->cap_present & QEMU_PCI_CAP_MSI) && | |
879 | vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) { | |
0d38fb1c AW |
880 | vfio_region_write(&vdev->bars[mirror->bar].region, |
881 | addr + mirror->offset, data, size); | |
882 | trace_vfio_quirk_nvidia_bar0_msi_ack(vdev->vbasedev.name); | |
c00d61d8 | 883 | } |
c958c51d AW |
884 | |
885 | /* | |
886 | * Automatically add an ioeventfd to handle any repeated write with the | |
887 | * same data and size above the standard PCI config space header. This is | |
888 | * primarily expected to accelerate the MSI-ACK behavior, such as noted | |
889 | * above. Current hardware/drivers should trigger an ioeventfd at config | |
890 | * offset 0x704 (region offset 0x88704), with data 0x0, size 4. | |
891 | * | |
892 | * The criteria of 10 successive hits is arbitrary but reliably adds the | |
893 | * MSI-ACK region. Note that as some writes are bypassed via the ioeventfd, | |
894 | * the remaining ones have a greater chance of being seen successively. | |
895 | * To avoid the pathological case of burning up all of QEMU's open file | |
896 | * handles, arbitrarily limit this algorithm from adding no more than 10 | |
897 | * ioeventfds, print an error if we would have added an 11th, and then | |
898 | * stop counting. | |
899 | */ | |
900 | if (!vdev->no_kvm_ioeventfd && | |
901 | addr >= PCI_STD_HEADER_SIZEOF && last->added <= MAX_DYN_IOEVENTFD) { | |
902 | if (addr != last->addr || data != last->data || size != last->size) { | |
903 | last->addr = addr; | |
904 | last->data = data; | |
905 | last->size = size; | |
906 | last->hits = 1; | |
907 | } else if (++last->hits >= HITS_FOR_IOEVENTFD) { | |
908 | if (last->added < MAX_DYN_IOEVENTFD) { | |
909 | VFIOIOEventFD *ioeventfd; | |
910 | ioeventfd = vfio_ioeventfd_init(vdev, mirror->mem, addr, size, | |
911 | data, &vdev->bars[mirror->bar].region, | |
912 | mirror->offset + addr, true); | |
913 | if (ioeventfd) { | |
914 | VFIOQuirk *quirk = last->quirk; | |
915 | ||
916 | QLIST_INSERT_HEAD(&quirk->ioeventfds, ioeventfd, next); | |
917 | last->added++; | |
918 | } | |
919 | } else { | |
920 | last->added++; | |
921 | warn_report("NVIDIA ioeventfd queue full for %s, unable to " | |
922 | "accelerate 0x%"HWADDR_PRIx", data 0x%"PRIx64", " | |
923 | "size %u", vdev->vbasedev.name, addr, data, size); | |
924 | } | |
925 | } | |
926 | } | |
c00d61d8 AW |
927 | } |
928 | ||
0d38fb1c AW |
929 | static const MemoryRegionOps vfio_nvidia_mirror_quirk = { |
930 | .read = vfio_generic_quirk_mirror_read, | |
931 | .write = vfio_nvidia_quirk_mirror_write, | |
c00d61d8 AW |
932 | .endianness = DEVICE_LITTLE_ENDIAN, |
933 | }; | |
934 | ||
c958c51d AW |
935 | static void vfio_nvidia_bar0_quirk_reset(VFIOPCIDevice *vdev, VFIOQuirk *quirk) |
936 | { | |
937 | VFIOConfigMirrorQuirk *mirror = quirk->data; | |
938 | LastDataSet *last = (LastDataSet *)&mirror->data; | |
939 | ||
940 | last->addr = last->data = last->size = last->hits = last->added = 0; | |
941 | ||
942 | vfio_drop_dynamic_eventfds(vdev, quirk); | |
943 | } | |
944 | ||
0d38fb1c | 945 | static void vfio_probe_nvidia_bar0_quirk(VFIOPCIDevice *vdev, int nr) |
c00d61d8 | 946 | { |
c00d61d8 | 947 | VFIOQuirk *quirk; |
0d38fb1c | 948 | VFIOConfigMirrorQuirk *mirror; |
c958c51d | 949 | LastDataSet *last; |
c00d61d8 | 950 | |
db32d0f4 AW |
951 | if (vdev->no_geforce_quirks || |
952 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || | |
0d38fb1c | 953 | !vfio_is_vga(vdev) || nr != 0) { |
c00d61d8 AW |
954 | return; |
955 | } | |
956 | ||
bcf3c3d0 | 957 | quirk = vfio_quirk_alloc(1); |
c958c51d AW |
958 | quirk->reset = vfio_nvidia_bar0_quirk_reset; |
959 | mirror = quirk->data = g_malloc0(sizeof(*mirror) + sizeof(LastDataSet)); | |
bcf3c3d0 | 960 | mirror->mem = quirk->mem; |
0d38fb1c AW |
961 | mirror->vdev = vdev; |
962 | mirror->offset = 0x88000; | |
963 | mirror->bar = nr; | |
c958c51d AW |
964 | last = (LastDataSet *)&mirror->data; |
965 | last->quirk = quirk; | |
0d38fb1c AW |
966 | |
967 | memory_region_init_io(mirror->mem, OBJECT(vdev), | |
968 | &vfio_nvidia_mirror_quirk, mirror, | |
969 | "vfio-nvidia-bar0-88000-mirror-quirk", | |
f5793fd9 | 970 | vdev->config_size); |
db0da029 | 971 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0d38fb1c | 972 | mirror->offset, mirror->mem, 1); |
c00d61d8 AW |
973 | |
974 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
975 | ||
0d38fb1c | 976 | /* The 0x1800 offset mirror only seems to get used by legacy VGA */ |
4d3fc4fd | 977 | if (vdev->vga) { |
bcf3c3d0 | 978 | quirk = vfio_quirk_alloc(1); |
c958c51d AW |
979 | quirk->reset = vfio_nvidia_bar0_quirk_reset; |
980 | mirror = quirk->data = g_malloc0(sizeof(*mirror) + sizeof(LastDataSet)); | |
bcf3c3d0 | 981 | mirror->mem = quirk->mem; |
0d38fb1c AW |
982 | mirror->vdev = vdev; |
983 | mirror->offset = 0x1800; | |
984 | mirror->bar = nr; | |
c958c51d AW |
985 | last = (LastDataSet *)&mirror->data; |
986 | last->quirk = quirk; | |
0d38fb1c AW |
987 | |
988 | memory_region_init_io(mirror->mem, OBJECT(vdev), | |
989 | &vfio_nvidia_mirror_quirk, mirror, | |
990 | "vfio-nvidia-bar0-1800-mirror-quirk", | |
991 | PCI_CONFIG_SPACE_SIZE); | |
db0da029 | 992 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
0d38fb1c AW |
993 | mirror->offset, mirror->mem, 1); |
994 | ||
995 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
c00d61d8 AW |
996 | } |
997 | ||
0d38fb1c | 998 | trace_vfio_quirk_nvidia_bar0_probe(vdev->vbasedev.name); |
c00d61d8 AW |
999 | } |
1000 | ||
1001 | /* | |
1002 | * TODO - Some Nvidia devices provide config access to their companion HDA | |
1003 | * device and even to their parent bridge via these config space mirrors. | |
1004 | * Add quirks for those regions. | |
1005 | */ | |
1006 | ||
1007 | #define PCI_VENDOR_ID_REALTEK 0x10ec | |
1008 | ||
1009 | /* | |
1010 | * RTL8168 devices have a backdoor that can access the MSI-X table. At BAR2 | |
1011 | * offset 0x70 there is a dword data register, offset 0x74 is a dword address | |
1012 | * register. According to the Linux r8169 driver, the MSI-X table is addressed | |
1013 | * when the "type" portion of the address register is set to 0x1. This appears | |
1014 | * to be bits 16:30. Bit 31 is both a write indicator and some sort of | |
1015 | * "address latched" indicator. Bits 12:15 are a mask field, which we can | |
1016 | * ignore because the MSI-X table should always be accessed as a dword (full | |
1017 | * mask). Bits 0:11 is offset within the type. | |
1018 | * | |
1019 | * Example trace: | |
1020 | * | |
1021 | * Read from MSI-X table offset 0 | |
1022 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr | |
1023 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch | |
1024 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data | |
1025 | * | |
1026 | * Write 0xfee00000 to MSI-X table offset 0 | |
1027 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data | |
1028 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write | |
1029 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete | |
1030 | */ | |
954258a5 AW |
1031 | typedef struct VFIOrtl8168Quirk { |
1032 | VFIOPCIDevice *vdev; | |
1033 | uint32_t addr; | |
1034 | uint32_t data; | |
1035 | bool enabled; | |
1036 | } VFIOrtl8168Quirk; | |
c00d61d8 | 1037 | |
954258a5 AW |
1038 | static uint64_t vfio_rtl8168_quirk_address_read(void *opaque, |
1039 | hwaddr addr, unsigned size) | |
1040 | { | |
1041 | VFIOrtl8168Quirk *rtl = opaque; | |
1042 | VFIOPCIDevice *vdev = rtl->vdev; | |
1043 | uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size); | |
c00d61d8 | 1044 | |
954258a5 AW |
1045 | if (rtl->enabled) { |
1046 | data = rtl->addr ^ 0x80000000U; /* latch/complete */ | |
1047 | trace_vfio_quirk_rtl8168_fake_latch(vdev->vbasedev.name, data); | |
c00d61d8 AW |
1048 | } |
1049 | ||
954258a5 | 1050 | return data; |
c00d61d8 AW |
1051 | } |
1052 | ||
954258a5 AW |
1053 | static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr, |
1054 | uint64_t data, unsigned size) | |
c00d61d8 | 1055 | { |
954258a5 AW |
1056 | VFIOrtl8168Quirk *rtl = opaque; |
1057 | VFIOPCIDevice *vdev = rtl->vdev; | |
c00d61d8 | 1058 | |
954258a5 AW |
1059 | rtl->enabled = false; |
1060 | ||
1061 | if ((data & 0x7fff0000) == 0x10000) { /* MSI-X table */ | |
1062 | rtl->enabled = true; | |
1063 | rtl->addr = (uint32_t)data; | |
1064 | ||
1065 | if (data & 0x80000000U) { /* Do write */ | |
1066 | if (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) { | |
1067 | hwaddr offset = data & 0xfff; | |
1068 | uint64_t val = rtl->data; | |
1069 | ||
1070 | trace_vfio_quirk_rtl8168_msix_write(vdev->vbasedev.name, | |
1071 | (uint16_t)offset, val); | |
1072 | ||
1073 | /* Write to the proper guest MSI-X table instead */ | |
1074 | memory_region_dispatch_write(&vdev->pdev.msix_table_mmio, | |
1075 | offset, val, size, | |
1076 | MEMTXATTRS_UNSPECIFIED); | |
c00d61d8 | 1077 | } |
954258a5 | 1078 | return; /* Do not write guest MSI-X data to hardware */ |
c00d61d8 | 1079 | } |
c00d61d8 AW |
1080 | } |
1081 | ||
954258a5 | 1082 | vfio_region_write(&vdev->bars[2].region, addr + 0x74, data, size); |
c00d61d8 AW |
1083 | } |
1084 | ||
954258a5 AW |
1085 | static const MemoryRegionOps vfio_rtl_address_quirk = { |
1086 | .read = vfio_rtl8168_quirk_address_read, | |
1087 | .write = vfio_rtl8168_quirk_address_write, | |
c00d61d8 AW |
1088 | .valid = { |
1089 | .min_access_size = 4, | |
1090 | .max_access_size = 4, | |
1091 | .unaligned = false, | |
1092 | }, | |
1093 | .endianness = DEVICE_LITTLE_ENDIAN, | |
1094 | }; | |
1095 | ||
954258a5 AW |
1096 | static uint64_t vfio_rtl8168_quirk_data_read(void *opaque, |
1097 | hwaddr addr, unsigned size) | |
1098 | { | |
1099 | VFIOrtl8168Quirk *rtl = opaque; | |
1100 | VFIOPCIDevice *vdev = rtl->vdev; | |
31e6a7b1 | 1101 | uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x70, size); |
954258a5 AW |
1102 | |
1103 | if (rtl->enabled && (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) { | |
1104 | hwaddr offset = rtl->addr & 0xfff; | |
1105 | memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, offset, | |
1106 | &data, size, MEMTXATTRS_UNSPECIFIED); | |
1107 | trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data); | |
1108 | } | |
1109 | ||
1110 | return data; | |
1111 | } | |
1112 | ||
1113 | static void vfio_rtl8168_quirk_data_write(void *opaque, hwaddr addr, | |
1114 | uint64_t data, unsigned size) | |
1115 | { | |
1116 | VFIOrtl8168Quirk *rtl = opaque; | |
1117 | VFIOPCIDevice *vdev = rtl->vdev; | |
1118 | ||
1119 | rtl->data = (uint32_t)data; | |
1120 | ||
1121 | vfio_region_write(&vdev->bars[2].region, addr + 0x70, data, size); | |
1122 | } | |
1123 | ||
1124 | static const MemoryRegionOps vfio_rtl_data_quirk = { | |
1125 | .read = vfio_rtl8168_quirk_data_read, | |
1126 | .write = vfio_rtl8168_quirk_data_write, | |
1127 | .valid = { | |
1128 | .min_access_size = 4, | |
1129 | .max_access_size = 4, | |
1130 | .unaligned = false, | |
1131 | }, | |
1132 | .endianness = DEVICE_LITTLE_ENDIAN, | |
1133 | }; | |
1134 | ||
1135 | static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr) | |
c00d61d8 | 1136 | { |
c00d61d8 | 1137 | VFIOQuirk *quirk; |
954258a5 | 1138 | VFIOrtl8168Quirk *rtl; |
c00d61d8 | 1139 | |
954258a5 | 1140 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_REALTEK, 0x8168) || nr != 2) { |
c00d61d8 AW |
1141 | return; |
1142 | } | |
1143 | ||
bcf3c3d0 | 1144 | quirk = vfio_quirk_alloc(2); |
954258a5 AW |
1145 | quirk->data = rtl = g_malloc0(sizeof(*rtl)); |
1146 | rtl->vdev = vdev; | |
1147 | ||
1148 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), | |
1149 | &vfio_rtl_address_quirk, rtl, | |
1150 | "vfio-rtl8168-window-address-quirk", 4); | |
db0da029 | 1151 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
954258a5 | 1152 | 0x74, &quirk->mem[0], 1); |
8c4f2348 | 1153 | |
954258a5 AW |
1154 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), |
1155 | &vfio_rtl_data_quirk, rtl, | |
1156 | "vfio-rtl8168-window-data-quirk", 4); | |
db0da029 | 1157 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
954258a5 | 1158 | 0x70, &quirk->mem[1], 1); |
c00d61d8 AW |
1159 | |
1160 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
1161 | ||
954258a5 | 1162 | trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name); |
c00d61d8 AW |
1163 | } |
1164 | ||
c4c45e94 AW |
1165 | /* |
1166 | * Intel IGD support | |
1167 | * | |
1168 | * Obviously IGD is not a discrete device, this is evidenced not only by it | |
1169 | * being integrated into the CPU, but by the various chipset and BIOS | |
1170 | * dependencies that it brings along with it. Intel is trying to move away | |
1171 | * from this and Broadwell and newer devices can run in what Intel calls | |
1172 | * "Universal Pass-Through" mode, or UPT. Theoretically in UPT mode, nothing | |
1173 | * more is required beyond assigning the IGD device to a VM. There are | |
1174 | * however support limitations to this mode. It only supports IGD as a | |
1175 | * secondary graphics device in the VM and it doesn't officially support any | |
1176 | * physical outputs. | |
1177 | * | |
1178 | * The code here attempts to enable what we'll call legacy mode assignment, | |
1179 | * IGD retains most of the capabilities we expect for it to have on bare | |
1180 | * metal. To enable this mode, the IGD device must be assigned to the VM | |
1181 | * at PCI address 00:02.0, it must have a ROM, it very likely needs VGA | |
1182 | * support, we must have VM BIOS support for reserving and populating some | |
1183 | * of the required tables, and we need to tweak the chipset with revisions | |
1184 | * and IDs and an LPC/ISA bridge device. The intention is to make all of | |
1185 | * this happen automatically by installing the device at the correct VM PCI | |
1186 | * bus address. If any of the conditions are not met, we cross our fingers | |
1187 | * and hope the user knows better. | |
1188 | * | |
1189 | * NB - It is possible to enable physical outputs in UPT mode by supplying | |
1190 | * an OpRegion table. We don't do this by default because the guest driver | |
1191 | * behaves differently if an OpRegion is provided and no monitor is attached | |
1192 | * vs no OpRegion and a monitor being attached or not. Effectively, if a | |
1193 | * headless setup is desired, the OpRegion gets in the way of that. | |
1194 | */ | |
1195 | ||
1196 | /* | |
1197 | * This presumes the device is already known to be an Intel VGA device, so we | |
1198 | * take liberties in which device ID bits match which generation. This should | |
1199 | * not be taken as an indication that all the devices are supported, or even | |
1200 | * supportable, some of them don't even support VT-d. | |
1201 | * See linux:include/drm/i915_pciids.h for IDs. | |
1202 | */ | |
1203 | static int igd_gen(VFIOPCIDevice *vdev) | |
1204 | { | |
1205 | if ((vdev->device_id & 0xfff) == 0xa84) { | |
1206 | return 8; /* Broxton */ | |
1207 | } | |
1208 | ||
1209 | switch (vdev->device_id & 0xff00) { | |
1210 | /* Old, untested, unavailable, unknown */ | |
1211 | case 0x0000: | |
1212 | case 0x2500: | |
1213 | case 0x2700: | |
1214 | case 0x2900: | |
1215 | case 0x2a00: | |
1216 | case 0x2e00: | |
1217 | case 0x3500: | |
1218 | case 0xa000: | |
1219 | return -1; | |
1220 | /* SandyBridge, IvyBridge, ValleyView, Haswell */ | |
1221 | case 0x0100: | |
1222 | case 0x0400: | |
1223 | case 0x0a00: | |
1224 | case 0x0c00: | |
1225 | case 0x0d00: | |
1226 | case 0x0f00: | |
1227 | return 6; | |
1228 | /* BroadWell, CherryView, SkyLake, KabyLake */ | |
1229 | case 0x1600: | |
1230 | case 0x1900: | |
1231 | case 0x2200: | |
1232 | case 0x5900: | |
1233 | return 8; | |
1234 | } | |
1235 | ||
1236 | return 8; /* Assume newer is compatible */ | |
1237 | } | |
1238 | ||
1239 | typedef struct VFIOIGDQuirk { | |
1240 | struct VFIOPCIDevice *vdev; | |
1241 | uint32_t index; | |
ac2a9862 | 1242 | uint32_t bdsm; |
c4c45e94 AW |
1243 | } VFIOIGDQuirk; |
1244 | ||
1245 | #define IGD_GMCH 0x50 /* Graphics Control Register */ | |
1246 | #define IGD_BDSM 0x5c /* Base Data of Stolen Memory */ | |
1247 | #define IGD_ASLS 0xfc /* ASL Storage Register */ | |
1248 | ||
1249 | /* | |
1250 | * The OpRegion includes the Video BIOS Table, which seems important for | |
1251 | * telling the driver what sort of outputs it has. Without this, the device | |
1252 | * may work in the guest, but we may not get output. This also requires BIOS | |
1253 | * support to reserve and populate a section of guest memory sufficient for | |
1254 | * the table and to write the base address of that memory to the ASLS register | |
1255 | * of the IGD device. | |
1256 | */ | |
6ced0bba | 1257 | int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev, |
7237011d | 1258 | struct vfio_region_info *info, Error **errp) |
c4c45e94 AW |
1259 | { |
1260 | int ret; | |
1261 | ||
1262 | vdev->igd_opregion = g_malloc0(info->size); | |
1263 | ret = pread(vdev->vbasedev.fd, vdev->igd_opregion, | |
1264 | info->size, info->offset); | |
1265 | if (ret != info->size) { | |
7237011d | 1266 | error_setg(errp, "failed to read IGD OpRegion"); |
c4c45e94 AW |
1267 | g_free(vdev->igd_opregion); |
1268 | vdev->igd_opregion = NULL; | |
1269 | return -EINVAL; | |
1270 | } | |
1271 | ||
1272 | /* | |
1273 | * Provide fw_cfg with a copy of the OpRegion which the VM firmware is to | |
1274 | * allocate 32bit reserved memory for, copy these contents into, and write | |
1275 | * the reserved memory base address to the device ASLS register at 0xFC. | |
1276 | * Alignment of this reserved region seems flexible, but using a 4k page | |
1277 | * alignment seems to work well. This interface assumes a single IGD | |
1278 | * device, which may be at VM address 00:02.0 in legacy mode or another | |
1279 | * address in UPT mode. | |
1280 | * | |
1281 | * NB, there may be future use cases discovered where the VM should have | |
1282 | * direct interaction with the host OpRegion, in which case the write to | |
1283 | * the ASLS register would trigger MemoryRegion setup to enable that. | |
1284 | */ | |
1285 | fw_cfg_add_file(fw_cfg_find(), "etc/igd-opregion", | |
1286 | vdev->igd_opregion, info->size); | |
1287 | ||
1288 | trace_vfio_pci_igd_opregion_enabled(vdev->vbasedev.name); | |
1289 | ||
1290 | pci_set_long(vdev->pdev.config + IGD_ASLS, 0); | |
1291 | pci_set_long(vdev->pdev.wmask + IGD_ASLS, ~0); | |
1292 | pci_set_long(vdev->emulated_config_bits + IGD_ASLS, ~0); | |
1293 | ||
1294 | return 0; | |
1295 | } | |
1296 | ||
1297 | /* | |
1298 | * The rather short list of registers that we copy from the host devices. | |
1299 | * The LPC/ISA bridge values are definitely needed to support the vBIOS, the | |
1300 | * host bridge values may or may not be needed depending on the guest OS. | |
1301 | * Since we're only munging revision and subsystem values on the host bridge, | |
1302 | * we don't require our own device. The LPC/ISA bridge needs to be our very | |
1303 | * own though. | |
1304 | */ | |
1305 | typedef struct { | |
1306 | uint8_t offset; | |
1307 | uint8_t len; | |
1308 | } IGDHostInfo; | |
1309 | ||
1310 | static const IGDHostInfo igd_host_bridge_infos[] = { | |
1311 | {PCI_REVISION_ID, 2}, | |
1312 | {PCI_SUBSYSTEM_VENDOR_ID, 2}, | |
1313 | {PCI_SUBSYSTEM_ID, 2}, | |
1314 | }; | |
1315 | ||
1316 | static const IGDHostInfo igd_lpc_bridge_infos[] = { | |
1317 | {PCI_VENDOR_ID, 2}, | |
1318 | {PCI_DEVICE_ID, 2}, | |
1319 | {PCI_REVISION_ID, 2}, | |
1320 | {PCI_SUBSYSTEM_VENDOR_ID, 2}, | |
1321 | {PCI_SUBSYSTEM_ID, 2}, | |
1322 | }; | |
1323 | ||
1324 | static int vfio_pci_igd_copy(VFIOPCIDevice *vdev, PCIDevice *pdev, | |
1325 | struct vfio_region_info *info, | |
1326 | const IGDHostInfo *list, int len) | |
1327 | { | |
1328 | int i, ret; | |
1329 | ||
1330 | for (i = 0; i < len; i++) { | |
1331 | ret = pread(vdev->vbasedev.fd, pdev->config + list[i].offset, | |
1332 | list[i].len, info->offset + list[i].offset); | |
1333 | if (ret != list[i].len) { | |
1334 | error_report("IGD copy failed: %m"); | |
1335 | return -errno; | |
1336 | } | |
1337 | } | |
1338 | ||
1339 | return 0; | |
1340 | } | |
1341 | ||
1342 | /* | |
1343 | * Stuff a few values into the host bridge. | |
1344 | */ | |
1345 | static int vfio_pci_igd_host_init(VFIOPCIDevice *vdev, | |
1346 | struct vfio_region_info *info) | |
1347 | { | |
1348 | PCIBus *bus; | |
1349 | PCIDevice *host_bridge; | |
1350 | int ret; | |
1351 | ||
1352 | bus = pci_device_root_bus(&vdev->pdev); | |
1353 | host_bridge = pci_find_device(bus, 0, PCI_DEVFN(0, 0)); | |
1354 | ||
1355 | if (!host_bridge) { | |
1356 | error_report("Can't find host bridge"); | |
1357 | return -ENODEV; | |
1358 | } | |
1359 | ||
1360 | ret = vfio_pci_igd_copy(vdev, host_bridge, info, igd_host_bridge_infos, | |
1361 | ARRAY_SIZE(igd_host_bridge_infos)); | |
1362 | if (!ret) { | |
1363 | trace_vfio_pci_igd_host_bridge_enabled(vdev->vbasedev.name); | |
1364 | } | |
1365 | ||
1366 | return ret; | |
1367 | } | |
1368 | ||
1369 | /* | |
1370 | * IGD LPC/ISA bridge support code. The vBIOS needs this, but we can't write | |
1371 | * arbitrary values into just any bridge, so we must create our own. We try | |
1372 | * to handle if the user has created it for us, which they might want to do | |
b12227af | 1373 | * to enable multifunction so we don't occupy the whole PCI slot. |
c4c45e94 AW |
1374 | */ |
1375 | static void vfio_pci_igd_lpc_bridge_realize(PCIDevice *pdev, Error **errp) | |
1376 | { | |
1377 | if (pdev->devfn != PCI_DEVFN(0x1f, 0)) { | |
1378 | error_setg(errp, "VFIO dummy ISA/LPC bridge must have address 1f.0"); | |
1379 | } | |
1380 | } | |
1381 | ||
1382 | static void vfio_pci_igd_lpc_bridge_class_init(ObjectClass *klass, void *data) | |
1383 | { | |
1384 | DeviceClass *dc = DEVICE_CLASS(klass); | |
1385 | PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); | |
1386 | ||
f23363ea | 1387 | set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); |
c4c45e94 AW |
1388 | dc->desc = "VFIO dummy ISA/LPC bridge for IGD assignment"; |
1389 | dc->hotpluggable = false; | |
1390 | k->realize = vfio_pci_igd_lpc_bridge_realize; | |
1391 | k->class_id = PCI_CLASS_BRIDGE_ISA; | |
1392 | } | |
1393 | ||
1394 | static TypeInfo vfio_pci_igd_lpc_bridge_info = { | |
1395 | .name = "vfio-pci-igd-lpc-bridge", | |
1396 | .parent = TYPE_PCI_DEVICE, | |
1397 | .class_init = vfio_pci_igd_lpc_bridge_class_init, | |
fd3b02c8 EH |
1398 | .interfaces = (InterfaceInfo[]) { |
1399 | { INTERFACE_CONVENTIONAL_PCI_DEVICE }, | |
1400 | { }, | |
1401 | }, | |
c4c45e94 AW |
1402 | }; |
1403 | ||
1404 | static void vfio_pci_igd_register_types(void) | |
1405 | { | |
1406 | type_register_static(&vfio_pci_igd_lpc_bridge_info); | |
1407 | } | |
1408 | ||
1409 | type_init(vfio_pci_igd_register_types) | |
1410 | ||
1411 | static int vfio_pci_igd_lpc_init(VFIOPCIDevice *vdev, | |
1412 | struct vfio_region_info *info) | |
1413 | { | |
1414 | PCIDevice *lpc_bridge; | |
1415 | int ret; | |
1416 | ||
1417 | lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), | |
1418 | 0, PCI_DEVFN(0x1f, 0)); | |
1419 | if (!lpc_bridge) { | |
1420 | lpc_bridge = pci_create_simple(pci_device_root_bus(&vdev->pdev), | |
1421 | PCI_DEVFN(0x1f, 0), "vfio-pci-igd-lpc-bridge"); | |
1422 | } | |
1423 | ||
1424 | ret = vfio_pci_igd_copy(vdev, lpc_bridge, info, igd_lpc_bridge_infos, | |
1425 | ARRAY_SIZE(igd_lpc_bridge_infos)); | |
1426 | if (!ret) { | |
1427 | trace_vfio_pci_igd_lpc_bridge_enabled(vdev->vbasedev.name); | |
1428 | } | |
1429 | ||
1430 | return ret; | |
1431 | } | |
1432 | ||
1433 | /* | |
1434 | * IGD Gen8 and newer support up to 8MB for the GTT and use a 64bit PTE | |
1435 | * entry, older IGDs use 2MB and 32bit. Each PTE maps a 4k page. Therefore | |
1436 | * we either have 2M/4k * 4 = 2k or 8M/4k * 8 = 16k as the maximum iobar index | |
1437 | * for programming the GTT. | |
1438 | * | |
1439 | * See linux:include/drm/i915_drm.h for shift and mask values. | |
1440 | */ | |
1441 | static int vfio_igd_gtt_max(VFIOPCIDevice *vdev) | |
1442 | { | |
1443 | uint32_t gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); | |
1444 | int ggms, gen = igd_gen(vdev); | |
1445 | ||
1446 | gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); | |
1447 | ggms = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; | |
1448 | if (gen > 6) { | |
1449 | ggms = 1 << ggms; | |
1450 | } | |
1451 | ||
e0255bb1 | 1452 | ggms *= MiB; |
c4c45e94 | 1453 | |
e0255bb1 | 1454 | return (ggms / (4 * KiB)) * (gen < 8 ? 4 : 8); |
c4c45e94 AW |
1455 | } |
1456 | ||
1457 | /* | |
1458 | * The IGD ROM will make use of stolen memory (GGMS) for support of VESA modes. | |
1459 | * Somehow the host stolen memory range is used for this, but how the ROM gets | |
1460 | * it is a mystery, perhaps it's hardcoded into the ROM. Thankfully though, it | |
1461 | * reprograms the GTT through the IOBAR where we can trap it and transpose the | |
1462 | * programming to the VM allocated buffer. That buffer gets reserved by the VM | |
1463 | * firmware via the fw_cfg entry added below. Here we're just monitoring the | |
1464 | * IOBAR address and data registers to detect a write sequence targeting the | |
1465 | * GTTADR. This code is developed by observed behavior and doesn't have a | |
1466 | * direct spec reference, unfortunately. | |
1467 | */ | |
1468 | static uint64_t vfio_igd_quirk_data_read(void *opaque, | |
1469 | hwaddr addr, unsigned size) | |
1470 | { | |
1471 | VFIOIGDQuirk *igd = opaque; | |
1472 | VFIOPCIDevice *vdev = igd->vdev; | |
1473 | ||
1474 | igd->index = ~0; | |
1475 | ||
1476 | return vfio_region_read(&vdev->bars[4].region, addr + 4, size); | |
1477 | } | |
1478 | ||
1479 | static void vfio_igd_quirk_data_write(void *opaque, hwaddr addr, | |
1480 | uint64_t data, unsigned size) | |
1481 | { | |
1482 | VFIOIGDQuirk *igd = opaque; | |
1483 | VFIOPCIDevice *vdev = igd->vdev; | |
1484 | uint64_t val = data; | |
1485 | int gen = igd_gen(vdev); | |
1486 | ||
1487 | /* | |
1488 | * Programming the GGMS starts at index 0x1 and uses every 4th index (ie. | |
1489 | * 0x1, 0x5, 0x9, 0xd,...). For pre-Gen8 each 4-byte write is a whole PTE | |
1490 | * entry, with 0th bit enable set. For Gen8 and up, PTEs are 64bit, so | |
1491 | * entries 0x5 & 0xd are the high dword, in our case zero. Each PTE points | |
1492 | * to a 4k page, which we translate to a page from the VM allocated region, | |
1493 | * pointed to by the BDSM register. If this is not set, we fail. | |
1494 | * | |
1495 | * We trap writes to the full configured GTT size, but we typically only | |
1496 | * see the vBIOS writing up to (nearly) the 1MB barrier. In fact it often | |
1497 | * seems to miss the last entry for an even 1MB GTT. Doing a gratuitous | |
1498 | * write of that last entry does work, but is hopefully unnecessary since | |
1499 | * we clear the previous GTT on initialization. | |
1500 | */ | |
1501 | if ((igd->index % 4 == 1) && igd->index < vfio_igd_gtt_max(vdev)) { | |
1502 | if (gen < 8 || (igd->index % 8 == 1)) { | |
1503 | uint32_t base; | |
1504 | ||
1505 | base = pci_get_long(vdev->pdev.config + IGD_BDSM); | |
1506 | if (!base) { | |
1507 | hw_error("vfio-igd: Guest attempted to program IGD GTT before " | |
1508 | "BIOS reserved stolen memory. Unsupported BIOS?"); | |
1509 | } | |
1510 | ||
ac2a9862 | 1511 | val = data - igd->bdsm + base; |
c4c45e94 AW |
1512 | } else { |
1513 | val = 0; /* upper 32bits of pte, we only enable below 4G PTEs */ | |
1514 | } | |
1515 | ||
1516 | trace_vfio_pci_igd_bar4_write(vdev->vbasedev.name, | |
1517 | igd->index, data, val); | |
1518 | } | |
1519 | ||
1520 | vfio_region_write(&vdev->bars[4].region, addr + 4, val, size); | |
1521 | ||
1522 | igd->index = ~0; | |
1523 | } | |
1524 | ||
1525 | static const MemoryRegionOps vfio_igd_data_quirk = { | |
1526 | .read = vfio_igd_quirk_data_read, | |
1527 | .write = vfio_igd_quirk_data_write, | |
1528 | .endianness = DEVICE_LITTLE_ENDIAN, | |
1529 | }; | |
1530 | ||
1531 | static uint64_t vfio_igd_quirk_index_read(void *opaque, | |
1532 | hwaddr addr, unsigned size) | |
1533 | { | |
1534 | VFIOIGDQuirk *igd = opaque; | |
1535 | VFIOPCIDevice *vdev = igd->vdev; | |
1536 | ||
1537 | igd->index = ~0; | |
1538 | ||
1539 | return vfio_region_read(&vdev->bars[4].region, addr, size); | |
1540 | } | |
1541 | ||
1542 | static void vfio_igd_quirk_index_write(void *opaque, hwaddr addr, | |
1543 | uint64_t data, unsigned size) | |
1544 | { | |
1545 | VFIOIGDQuirk *igd = opaque; | |
1546 | VFIOPCIDevice *vdev = igd->vdev; | |
1547 | ||
1548 | igd->index = data; | |
1549 | ||
1550 | vfio_region_write(&vdev->bars[4].region, addr, data, size); | |
1551 | } | |
1552 | ||
1553 | static const MemoryRegionOps vfio_igd_index_quirk = { | |
1554 | .read = vfio_igd_quirk_index_read, | |
1555 | .write = vfio_igd_quirk_index_write, | |
1556 | .endianness = DEVICE_LITTLE_ENDIAN, | |
1557 | }; | |
1558 | ||
1559 | static void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr) | |
1560 | { | |
1561 | struct vfio_region_info *rom = NULL, *opregion = NULL, | |
1562 | *host = NULL, *lpc = NULL; | |
1563 | VFIOQuirk *quirk; | |
1564 | VFIOIGDQuirk *igd; | |
1565 | PCIDevice *lpc_bridge; | |
1566 | int i, ret, ggms_mb, gms_mb = 0, gen; | |
1567 | uint64_t *bdsm_size; | |
1568 | uint32_t gmch; | |
1569 | uint16_t cmd_orig, cmd; | |
cde4279b | 1570 | Error *err = NULL; |
c4c45e94 | 1571 | |
c2b2e158 | 1572 | /* |
93587e3a XZ |
1573 | * This must be an Intel VGA device at address 00:02.0 for us to even |
1574 | * consider enabling legacy mode. The vBIOS has dependencies on the | |
1575 | * PCI bus address. | |
c2b2e158 | 1576 | */ |
93587e3a XZ |
1577 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || |
1578 | !vfio_is_vga(vdev) || nr != 4 || | |
1579 | &vdev->pdev != pci_find_device(pci_device_root_bus(&vdev->pdev), | |
c4c45e94 AW |
1580 | 0, PCI_DEVFN(0x2, 0))) { |
1581 | return; | |
1582 | } | |
1583 | ||
1584 | /* | |
1585 | * We need to create an LPC/ISA bridge at PCI bus address 00:1f.0 that we | |
1586 | * can stuff host values into, so if there's already one there and it's not | |
1587 | * one we can hack on, legacy mode is no-go. Sorry Q35. | |
1588 | */ | |
1589 | lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), | |
1590 | 0, PCI_DEVFN(0x1f, 0)); | |
1591 | if (lpc_bridge && !object_dynamic_cast(OBJECT(lpc_bridge), | |
1592 | "vfio-pci-igd-lpc-bridge")) { | |
1593 | error_report("IGD device %s cannot support legacy mode due to existing " | |
1594 | "devices at address 1f.0", vdev->vbasedev.name); | |
1595 | return; | |
1596 | } | |
1597 | ||
93587e3a XZ |
1598 | /* |
1599 | * IGD is not a standard, they like to change their specs often. We | |
1600 | * only attempt to support back to SandBridge and we hope that newer | |
1601 | * devices maintain compatibility with generation 8. | |
1602 | */ | |
1603 | gen = igd_gen(vdev); | |
1604 | if (gen != 6 && gen != 8) { | |
1605 | error_report("IGD device %s is unsupported in legacy mode, " | |
1606 | "try SandyBridge or newer", vdev->vbasedev.name); | |
1607 | return; | |
1608 | } | |
1609 | ||
c4c45e94 AW |
1610 | /* |
1611 | * Most of what we're doing here is to enable the ROM to run, so if | |
1612 | * there's no ROM, there's no point in setting up this quirk. | |
1613 | * NB. We only seem to get BIOS ROMs, so a UEFI VM would need CSM support. | |
1614 | */ | |
1615 | ret = vfio_get_region_info(&vdev->vbasedev, | |
1616 | VFIO_PCI_ROM_REGION_INDEX, &rom); | |
1617 | if ((ret || !rom->size) && !vdev->pdev.romfile) { | |
1618 | error_report("IGD device %s has no ROM, legacy mode disabled", | |
1619 | vdev->vbasedev.name); | |
1620 | goto out; | |
1621 | } | |
1622 | ||
1623 | /* | |
1624 | * Ignore the hotplug corner case, mark the ROM failed, we can't | |
1625 | * create the devices we need for legacy mode in the hotplug scenario. | |
1626 | */ | |
1627 | if (vdev->pdev.qdev.hotplugged) { | |
1628 | error_report("IGD device %s hotplugged, ROM disabled, " | |
1629 | "legacy mode disabled", vdev->vbasedev.name); | |
1630 | vdev->rom_read_failed = true; | |
1631 | goto out; | |
1632 | } | |
1633 | ||
1634 | /* | |
1635 | * Check whether we have all the vfio device specific regions to | |
1636 | * support legacy mode (added in Linux v4.6). If not, bail. | |
1637 | */ | |
1638 | ret = vfio_get_dev_region_info(&vdev->vbasedev, | |
1639 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, | |
1640 | VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion); | |
1641 | if (ret) { | |
1642 | error_report("IGD device %s does not support OpRegion access," | |
1643 | "legacy mode disabled", vdev->vbasedev.name); | |
1644 | goto out; | |
1645 | } | |
1646 | ||
1647 | ret = vfio_get_dev_region_info(&vdev->vbasedev, | |
1648 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, | |
1649 | VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG, &host); | |
1650 | if (ret) { | |
1651 | error_report("IGD device %s does not support host bridge access," | |
1652 | "legacy mode disabled", vdev->vbasedev.name); | |
1653 | goto out; | |
1654 | } | |
1655 | ||
1656 | ret = vfio_get_dev_region_info(&vdev->vbasedev, | |
1657 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, | |
1658 | VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG, &lpc); | |
1659 | if (ret) { | |
1660 | error_report("IGD device %s does not support LPC bridge access," | |
1661 | "legacy mode disabled", vdev->vbasedev.name); | |
1662 | goto out; | |
1663 | } | |
1664 | ||
93587e3a XZ |
1665 | gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, 4); |
1666 | ||
c4c45e94 AW |
1667 | /* |
1668 | * If IGD VGA Disable is clear (expected) and VGA is not already enabled, | |
1669 | * try to enable it. Probably shouldn't be using legacy mode without VGA, | |
1670 | * but also no point in us enabling VGA if disabled in hardware. | |
1671 | */ | |
cde4279b | 1672 | if (!(gmch & 0x2) && !vdev->vga && vfio_populate_vga(vdev, &err)) { |
c3b8e3e0 | 1673 | error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); |
c4c45e94 AW |
1674 | error_report("IGD device %s failed to enable VGA access, " |
1675 | "legacy mode disabled", vdev->vbasedev.name); | |
1676 | goto out; | |
1677 | } | |
1678 | ||
1679 | /* Create our LPC/ISA bridge */ | |
1680 | ret = vfio_pci_igd_lpc_init(vdev, lpc); | |
1681 | if (ret) { | |
1682 | error_report("IGD device %s failed to create LPC bridge, " | |
1683 | "legacy mode disabled", vdev->vbasedev.name); | |
1684 | goto out; | |
1685 | } | |
1686 | ||
1687 | /* Stuff some host values into the VM PCI host bridge */ | |
1688 | ret = vfio_pci_igd_host_init(vdev, host); | |
1689 | if (ret) { | |
1690 | error_report("IGD device %s failed to modify host bridge, " | |
1691 | "legacy mode disabled", vdev->vbasedev.name); | |
1692 | goto out; | |
1693 | } | |
1694 | ||
1695 | /* Setup OpRegion access */ | |
7237011d | 1696 | ret = vfio_pci_igd_opregion_init(vdev, opregion, &err); |
c4c45e94 | 1697 | if (ret) { |
7237011d | 1698 | error_append_hint(&err, "IGD legacy mode disabled\n"); |
c3b8e3e0 | 1699 | error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); |
c4c45e94 AW |
1700 | goto out; |
1701 | } | |
1702 | ||
1703 | /* Setup our quirk to munge GTT addresses to the VM allocated buffer */ | |
bcf3c3d0 | 1704 | quirk = vfio_quirk_alloc(2); |
c4c45e94 AW |
1705 | igd = quirk->data = g_malloc0(sizeof(*igd)); |
1706 | igd->vdev = vdev; | |
1707 | igd->index = ~0; | |
ac2a9862 | 1708 | igd->bdsm = vfio_pci_read_config(&vdev->pdev, IGD_BDSM, 4); |
e0255bb1 | 1709 | igd->bdsm &= ~((1 * MiB) - 1); /* 1MB aligned */ |
c4c45e94 AW |
1710 | |
1711 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_igd_index_quirk, | |
1712 | igd, "vfio-igd-index-quirk", 4); | |
1713 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, | |
1714 | 0, &quirk->mem[0], 1); | |
1715 | ||
1716 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_igd_data_quirk, | |
1717 | igd, "vfio-igd-data-quirk", 4); | |
1718 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, | |
1719 | 4, &quirk->mem[1], 1); | |
1720 | ||
1721 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); | |
1722 | ||
1723 | /* Determine the size of stolen memory needed for GTT */ | |
1724 | ggms_mb = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; | |
1725 | if (gen > 6) { | |
1726 | ggms_mb = 1 << ggms_mb; | |
1727 | } | |
1728 | ||
1729 | /* | |
1730 | * Assume we have no GMS memory, but allow it to be overrided by device | |
1731 | * option (experimental). The spec doesn't actually allow zero GMS when | |
1732 | * when IVD (IGD VGA Disable) is clear, but the claim is that it's unused, | |
1733 | * so let's not waste VM memory for it. | |
1734 | */ | |
93587e3a XZ |
1735 | gmch &= ~((gen < 8 ? 0x1f : 0xff) << (gen < 8 ? 3 : 8)); |
1736 | ||
c4c45e94 AW |
1737 | if (vdev->igd_gms) { |
1738 | if (vdev->igd_gms <= 0x10) { | |
1739 | gms_mb = vdev->igd_gms * 32; | |
1740 | gmch |= vdev->igd_gms << (gen < 8 ? 3 : 8); | |
1741 | } else { | |
1742 | error_report("Unsupported IGD GMS value 0x%x", vdev->igd_gms); | |
1743 | vdev->igd_gms = 0; | |
1744 | } | |
1745 | } | |
1746 | ||
1747 | /* | |
1748 | * Request reserved memory for stolen memory via fw_cfg. VM firmware | |
1749 | * must allocate a 1MB aligned reserved memory region below 4GB with | |
1750 | * the requested size (in bytes) for use by the Intel PCI class VGA | |
1751 | * device at VM address 00:02.0. The base address of this reserved | |
1752 | * memory region must be written to the device BDSM regsiter at PCI | |
1753 | * config offset 0x5C. | |
1754 | */ | |
1755 | bdsm_size = g_malloc(sizeof(*bdsm_size)); | |
e0255bb1 | 1756 | *bdsm_size = cpu_to_le64((ggms_mb + gms_mb) * MiB); |
c4c45e94 AW |
1757 | fw_cfg_add_file(fw_cfg_find(), "etc/igd-bdsm-size", |
1758 | bdsm_size, sizeof(*bdsm_size)); | |
1759 | ||
93587e3a XZ |
1760 | /* GMCH is read-only, emulated */ |
1761 | pci_set_long(vdev->pdev.config + IGD_GMCH, gmch); | |
1762 | pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0); | |
1763 | pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0); | |
1764 | ||
c4c45e94 AW |
1765 | /* BDSM is read-write, emulated. The BIOS needs to be able to write it */ |
1766 | pci_set_long(vdev->pdev.config + IGD_BDSM, 0); | |
1767 | pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0); | |
1768 | pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0); | |
1769 | ||
1770 | /* | |
1771 | * This IOBAR gives us access to GTTADR, which allows us to write to | |
1772 | * the GTT itself. So let's go ahead and write zero to all the GTT | |
1773 | * entries to avoid spurious DMA faults. Be sure I/O access is enabled | |
1774 | * before talking to the device. | |
1775 | */ | |
1776 | if (pread(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), | |
1777 | vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { | |
1778 | error_report("IGD device %s - failed to read PCI command register", | |
1779 | vdev->vbasedev.name); | |
1780 | } | |
1781 | ||
1782 | cmd = cmd_orig | PCI_COMMAND_IO; | |
1783 | ||
1784 | if (pwrite(vdev->vbasedev.fd, &cmd, sizeof(cmd), | |
1785 | vdev->config_offset + PCI_COMMAND) != sizeof(cmd)) { | |
1786 | error_report("IGD device %s - failed to write PCI command register", | |
1787 | vdev->vbasedev.name); | |
1788 | } | |
1789 | ||
1790 | for (i = 1; i < vfio_igd_gtt_max(vdev); i += 4) { | |
1791 | vfio_region_write(&vdev->bars[4].region, 0, i, 4); | |
1792 | vfio_region_write(&vdev->bars[4].region, 4, 0, 4); | |
1793 | } | |
1794 | ||
1795 | if (pwrite(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), | |
1796 | vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { | |
1797 | error_report("IGD device %s - failed to restore PCI command register", | |
1798 | vdev->vbasedev.name); | |
1799 | } | |
1800 | ||
1801 | trace_vfio_pci_igd_bdsm_enabled(vdev->vbasedev.name, ggms_mb + gms_mb); | |
1802 | ||
1803 | out: | |
1804 | g_free(rom); | |
1805 | g_free(opregion); | |
1806 | g_free(host); | |
1807 | g_free(lpc); | |
1808 | } | |
1809 | ||
c00d61d8 AW |
1810 | /* |
1811 | * Common quirk probe entry points. | |
1812 | */ | |
1813 | void vfio_vga_quirk_setup(VFIOPCIDevice *vdev) | |
1814 | { | |
1815 | vfio_vga_probe_ati_3c3_quirk(vdev); | |
1816 | vfio_vga_probe_nvidia_3d0_quirk(vdev); | |
1817 | } | |
1818 | ||
2d82f8a3 | 1819 | void vfio_vga_quirk_exit(VFIOPCIDevice *vdev) |
c00d61d8 AW |
1820 | { |
1821 | VFIOQuirk *quirk; | |
8c4f2348 | 1822 | int i, j; |
c00d61d8 | 1823 | |
2d82f8a3 AW |
1824 | for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { |
1825 | QLIST_FOREACH(quirk, &vdev->vga->region[i].quirks, next) { | |
8c4f2348 | 1826 | for (j = 0; j < quirk->nr_mem; j++) { |
2d82f8a3 | 1827 | memory_region_del_subregion(&vdev->vga->region[i].mem, |
8c4f2348 AW |
1828 | &quirk->mem[j]); |
1829 | } | |
c00d61d8 AW |
1830 | } |
1831 | } | |
1832 | } | |
1833 | ||
2d82f8a3 | 1834 | void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev) |
c00d61d8 | 1835 | { |
8c4f2348 | 1836 | int i, j; |
c00d61d8 | 1837 | |
2d82f8a3 AW |
1838 | for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { |
1839 | while (!QLIST_EMPTY(&vdev->vga->region[i].quirks)) { | |
1840 | VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga->region[i].quirks); | |
c00d61d8 | 1841 | QLIST_REMOVE(quirk, next); |
8c4f2348 AW |
1842 | for (j = 0; j < quirk->nr_mem; j++) { |
1843 | object_unparent(OBJECT(&quirk->mem[j])); | |
1844 | } | |
1845 | g_free(quirk->mem); | |
1846 | g_free(quirk->data); | |
c00d61d8 AW |
1847 | g_free(quirk); |
1848 | } | |
1849 | } | |
1850 | } | |
1851 | ||
1852 | void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr) | |
1853 | { | |
0e54f24a | 1854 | vfio_probe_ati_bar4_quirk(vdev, nr); |
0d38fb1c | 1855 | vfio_probe_ati_bar2_quirk(vdev, nr); |
0e54f24a | 1856 | vfio_probe_nvidia_bar5_quirk(vdev, nr); |
0d38fb1c | 1857 | vfio_probe_nvidia_bar0_quirk(vdev, nr); |
954258a5 | 1858 | vfio_probe_rtl8168_bar2_quirk(vdev, nr); |
c4c45e94 | 1859 | vfio_probe_igd_bar4_quirk(vdev, nr); |
c00d61d8 AW |
1860 | } |
1861 | ||
2d82f8a3 | 1862 | void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr) |
c00d61d8 AW |
1863 | { |
1864 | VFIOBAR *bar = &vdev->bars[nr]; | |
1865 | VFIOQuirk *quirk; | |
8c4f2348 | 1866 | int i; |
c00d61d8 AW |
1867 | |
1868 | QLIST_FOREACH(quirk, &bar->quirks, next) { | |
c958c51d | 1869 | while (!QLIST_EMPTY(&quirk->ioeventfds)) { |
2b1dbd0d | 1870 | vfio_ioeventfd_exit(vdev, QLIST_FIRST(&quirk->ioeventfds)); |
c958c51d AW |
1871 | } |
1872 | ||
8c4f2348 | 1873 | for (i = 0; i < quirk->nr_mem; i++) { |
db0da029 | 1874 | memory_region_del_subregion(bar->region.mem, &quirk->mem[i]); |
8c4f2348 | 1875 | } |
c00d61d8 AW |
1876 | } |
1877 | } | |
1878 | ||
2d82f8a3 | 1879 | void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr) |
c00d61d8 AW |
1880 | { |
1881 | VFIOBAR *bar = &vdev->bars[nr]; | |
8c4f2348 | 1882 | int i; |
c00d61d8 AW |
1883 | |
1884 | while (!QLIST_EMPTY(&bar->quirks)) { | |
1885 | VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks); | |
c00d61d8 | 1886 | QLIST_REMOVE(quirk, next); |
8c4f2348 AW |
1887 | for (i = 0; i < quirk->nr_mem; i++) { |
1888 | object_unparent(OBJECT(&quirk->mem[i])); | |
1889 | } | |
1890 | g_free(quirk->mem); | |
1891 | g_free(quirk->data); | |
c00d61d8 AW |
1892 | g_free(quirk); |
1893 | } | |
1894 | } | |
c9c50009 AW |
1895 | |
1896 | /* | |
1897 | * Reset quirks | |
1898 | */ | |
469d02de AW |
1899 | void vfio_quirk_reset(VFIOPCIDevice *vdev) |
1900 | { | |
1901 | int i; | |
1902 | ||
1903 | for (i = 0; i < PCI_ROM_SLOT; i++) { | |
1904 | VFIOQuirk *quirk; | |
1905 | VFIOBAR *bar = &vdev->bars[i]; | |
1906 | ||
1907 | QLIST_FOREACH(quirk, &bar->quirks, next) { | |
1908 | if (quirk->reset) { | |
1909 | quirk->reset(vdev, quirk); | |
1910 | } | |
1911 | } | |
1912 | } | |
1913 | } | |
c9c50009 AW |
1914 | |
1915 | /* | |
1916 | * AMD Radeon PCI config reset, based on Linux: | |
1917 | * drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running() | |
1918 | * drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset | |
1919 | * drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc() | |
1920 | * drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock() | |
1921 | * IDs: include/drm/drm_pciids.h | |
1922 | * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0 | |
1923 | * | |
1924 | * Bonaire and Hawaii GPUs do not respond to a bus reset. This is a bug in the | |
1925 | * hardware that should be fixed on future ASICs. The symptom of this is that | |
1926 | * once the accerlated driver loads, Windows guests will bsod on subsequent | |
1927 | * attmpts to load the driver, such as after VM reset or shutdown/restart. To | |
1928 | * work around this, we do an AMD specific PCI config reset, followed by an SMC | |
1929 | * reset. The PCI config reset only works if SMC firmware is running, so we | |
1930 | * have a dependency on the state of the device as to whether this reset will | |
1931 | * be effective. There are still cases where we won't be able to kick the | |
1932 | * device into working, but this greatly improves the usability overall. The | |
1933 | * config reset magic is relatively common on AMD GPUs, but the setup and SMC | |
1934 | * poking is largely ASIC specific. | |
1935 | */ | |
1936 | static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev) | |
1937 | { | |
1938 | uint32_t clk, pc_c; | |
1939 | ||
1940 | /* | |
1941 | * Registers 200h and 204h are index and data registers for accessing | |
1942 | * indirect configuration registers within the device. | |
1943 | */ | |
1944 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); | |
1945 | clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4); | |
1946 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4); | |
1947 | pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4); | |
1948 | ||
1949 | return (!(clk & 1) && (0x20100 <= pc_c)); | |
1950 | } | |
1951 | ||
1952 | /* | |
1953 | * The scope of a config reset is controlled by a mode bit in the misc register | |
1954 | * and a fuse, exposed as a bit in another register. The fuse is the default | |
1955 | * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula | |
1956 | * scope = !(misc ^ fuse), where the resulting scope is defined the same as | |
1957 | * the fuse. A truth table therefore tells us that if misc == fuse, we need | |
1958 | * to flip the value of the bit in the misc register. | |
1959 | */ | |
1960 | static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev) | |
1961 | { | |
1962 | uint32_t misc, fuse; | |
1963 | bool a, b; | |
1964 | ||
1965 | vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4); | |
1966 | fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4); | |
1967 | b = fuse & 64; | |
1968 | ||
1969 | vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4); | |
1970 | misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4); | |
1971 | a = misc & 2; | |
1972 | ||
1973 | if (a == b) { | |
1974 | vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4); | |
1975 | vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */ | |
1976 | } | |
1977 | } | |
1978 | ||
1979 | static int vfio_radeon_reset(VFIOPCIDevice *vdev) | |
1980 | { | |
1981 | PCIDevice *pdev = &vdev->pdev; | |
1982 | int i, ret = 0; | |
1983 | uint32_t data; | |
1984 | ||
1985 | /* Defer to a kernel implemented reset */ | |
1986 | if (vdev->vbasedev.reset_works) { | |
1987 | trace_vfio_quirk_ati_bonaire_reset_skipped(vdev->vbasedev.name); | |
1988 | return -ENODEV; | |
1989 | } | |
1990 | ||
1991 | /* Enable only memory BAR access */ | |
1992 | vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2); | |
1993 | ||
1994 | /* Reset only works if SMC firmware is loaded and running */ | |
1995 | if (!vfio_radeon_smc_is_running(vdev)) { | |
1996 | ret = -EINVAL; | |
1997 | trace_vfio_quirk_ati_bonaire_reset_no_smc(vdev->vbasedev.name); | |
1998 | goto out; | |
1999 | } | |
2000 | ||
2001 | /* Make sure only the GFX function is reset */ | |
2002 | vfio_radeon_set_gfx_only_reset(vdev); | |
2003 | ||
2004 | /* AMD PCI config reset */ | |
2005 | vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4); | |
2006 | usleep(100); | |
2007 | ||
2008 | /* Read back the memory size to make sure we're out of reset */ | |
2009 | for (i = 0; i < 100000; i++) { | |
2010 | if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) { | |
2011 | goto reset_smc; | |
2012 | } | |
2013 | usleep(1); | |
2014 | } | |
2015 | ||
2016 | trace_vfio_quirk_ati_bonaire_reset_timeout(vdev->vbasedev.name); | |
2017 | ||
2018 | reset_smc: | |
2019 | /* Reset SMC */ | |
2020 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4); | |
2021 | data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); | |
2022 | data |= 1; | |
2023 | vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); | |
2024 | ||
2025 | /* Disable SMC clock */ | |
2026 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); | |
2027 | data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); | |
2028 | data |= 1; | |
2029 | vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); | |
2030 | ||
2031 | trace_vfio_quirk_ati_bonaire_reset_done(vdev->vbasedev.name); | |
2032 | ||
2033 | out: | |
2034 | /* Restore PCI command register */ | |
2035 | vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2); | |
2036 | ||
2037 | return ret; | |
2038 | } | |
2039 | ||
2040 | void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev) | |
2041 | { | |
ff635e37 | 2042 | switch (vdev->vendor_id) { |
c9c50009 | 2043 | case 0x1002: |
ff635e37 | 2044 | switch (vdev->device_id) { |
c9c50009 AW |
2045 | /* Bonaire */ |
2046 | case 0x6649: /* Bonaire [FirePro W5100] */ | |
2047 | case 0x6650: | |
2048 | case 0x6651: | |
2049 | case 0x6658: /* Bonaire XTX [Radeon R7 260X] */ | |
2050 | case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */ | |
2051 | case 0x665d: /* Bonaire [Radeon R7 200 Series] */ | |
2052 | /* Hawaii */ | |
2053 | case 0x67A0: /* Hawaii XT GL [FirePro W9100] */ | |
2054 | case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */ | |
2055 | case 0x67A2: | |
2056 | case 0x67A8: | |
2057 | case 0x67A9: | |
2058 | case 0x67AA: | |
2059 | case 0x67B0: /* Hawaii XT [Radeon R9 290X] */ | |
2060 | case 0x67B1: /* Hawaii PRO [Radeon R9 290] */ | |
2061 | case 0x67B8: | |
2062 | case 0x67B9: | |
2063 | case 0x67BA: | |
2064 | case 0x67BE: | |
2065 | vdev->resetfn = vfio_radeon_reset; | |
2066 | trace_vfio_quirk_ati_bonaire_reset(vdev->vbasedev.name); | |
2067 | break; | |
2068 | } | |
2069 | break; | |
2070 | } | |
2071 | } | |
dfbee78d AW |
2072 | |
2073 | /* | |
2074 | * The NVIDIA GPUDirect P2P Vendor capability allows the user to specify | |
2075 | * devices as a member of a clique. Devices within the same clique ID | |
2076 | * are capable of direct P2P. It's the user's responsibility that this | |
2077 | * is correct. The spec says that this may reside at any unused config | |
2078 | * offset, but reserves and recommends hypervisors place this at C8h. | |
2079 | * The spec also states that the hypervisor should place this capability | |
2080 | * at the end of the capability list, thus next is defined as 0h. | |
2081 | * | |
2082 | * +----------------+----------------+----------------+----------------+ | |
2083 | * | sig 7:0 ('P') | vndr len (8h) | next (0h) | cap id (9h) | | |
2084 | * +----------------+----------------+----------------+----------------+ | |
2085 | * | rsvd 15:7(0h),id 6:3,ver 2:0(0h)| sig 23:8 ('P2') | | |
2086 | * +---------------------------------+---------------------------------+ | |
2087 | * | |
2088 | * https://lists.gnu.org/archive/html/qemu-devel/2017-08/pdfUda5iEpgOS.pdf | |
2089 | */ | |
2090 | static void get_nv_gpudirect_clique_id(Object *obj, Visitor *v, | |
2091 | const char *name, void *opaque, | |
2092 | Error **errp) | |
2093 | { | |
2094 | DeviceState *dev = DEVICE(obj); | |
2095 | Property *prop = opaque; | |
2096 | uint8_t *ptr = qdev_get_prop_ptr(dev, prop); | |
2097 | ||
2098 | visit_type_uint8(v, name, ptr, errp); | |
2099 | } | |
2100 | ||
2101 | static void set_nv_gpudirect_clique_id(Object *obj, Visitor *v, | |
2102 | const char *name, void *opaque, | |
2103 | Error **errp) | |
2104 | { | |
2105 | DeviceState *dev = DEVICE(obj); | |
2106 | Property *prop = opaque; | |
2107 | uint8_t value, *ptr = qdev_get_prop_ptr(dev, prop); | |
2108 | Error *local_err = NULL; | |
2109 | ||
2110 | if (dev->realized) { | |
2111 | qdev_prop_set_after_realize(dev, name, errp); | |
2112 | return; | |
2113 | } | |
2114 | ||
2115 | visit_type_uint8(v, name, &value, &local_err); | |
2116 | if (local_err) { | |
2117 | error_propagate(errp, local_err); | |
2118 | return; | |
2119 | } | |
2120 | ||
2121 | if (value & ~0xF) { | |
2122 | error_setg(errp, "Property %s: valid range 0-15", name); | |
2123 | return; | |
2124 | } | |
2125 | ||
2126 | *ptr = value; | |
2127 | } | |
2128 | ||
2129 | const PropertyInfo qdev_prop_nv_gpudirect_clique = { | |
2130 | .name = "uint4", | |
2131 | .description = "NVIDIA GPUDirect Clique ID (0 - 15)", | |
2132 | .get = get_nv_gpudirect_clique_id, | |
2133 | .set = set_nv_gpudirect_clique_id, | |
2134 | }; | |
2135 | ||
2136 | static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp) | |
2137 | { | |
2138 | PCIDevice *pdev = &vdev->pdev; | |
2139 | int ret, pos = 0xC8; | |
2140 | ||
2141 | if (vdev->nv_gpudirect_clique == 0xFF) { | |
2142 | return 0; | |
2143 | } | |
2144 | ||
2145 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID)) { | |
2146 | error_setg(errp, "NVIDIA GPUDirect Clique ID: invalid device vendor"); | |
2147 | return -EINVAL; | |
2148 | } | |
2149 | ||
2150 | if (pci_get_byte(pdev->config + PCI_CLASS_DEVICE + 1) != | |
2151 | PCI_BASE_CLASS_DISPLAY) { | |
2152 | error_setg(errp, "NVIDIA GPUDirect Clique ID: unsupported PCI class"); | |
2153 | return -EINVAL; | |
2154 | } | |
2155 | ||
2156 | ret = pci_add_capability(pdev, PCI_CAP_ID_VNDR, pos, 8, errp); | |
2157 | if (ret < 0) { | |
2158 | error_prepend(errp, "Failed to add NVIDIA GPUDirect cap: "); | |
2159 | return ret; | |
2160 | } | |
2161 | ||
2162 | memset(vdev->emulated_config_bits + pos, 0xFF, 8); | |
2163 | pos += PCI_CAP_FLAGS; | |
2164 | pci_set_byte(pdev->config + pos++, 8); | |
2165 | pci_set_byte(pdev->config + pos++, 'P'); | |
2166 | pci_set_byte(pdev->config + pos++, '2'); | |
2167 | pci_set_byte(pdev->config + pos++, 'P'); | |
2168 | pci_set_byte(pdev->config + pos++, vdev->nv_gpudirect_clique << 3); | |
2169 | pci_set_byte(pdev->config + pos, 0); | |
2170 | ||
2171 | return 0; | |
2172 | } | |
2173 | ||
e3f79f3b AW |
2174 | int vfio_add_virt_caps(VFIOPCIDevice *vdev, Error **errp) |
2175 | { | |
dfbee78d AW |
2176 | int ret; |
2177 | ||
2178 | ret = vfio_add_nv_gpudirect_cap(vdev, errp); | |
2179 | if (ret) { | |
2180 | return ret; | |
2181 | } | |
2182 | ||
e3f79f3b AW |
2183 | return 0; |
2184 | } |