]>
Commit | Line | Data |
---|---|---|
c3ebd3ba JK |
1 | /* |
2 | * Copyright (c) 2007, Neocleus Corporation. | |
3 | * | |
4 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
5 | * the COPYING file in the top-level directory. | |
6 | * | |
7 | * | |
8 | * Assign a PCI device from the host to a guest VM. | |
9 | * | |
10 | * This implementation uses the classic device assignment interface of KVM | |
11 | * and is only available on x86 hosts. It is expected to be obsoleted by VFIO | |
12 | * based device assignment. | |
13 | * | |
14 | * Adapted for KVM (qemu-kvm) by Qumranet. QEMU version was based on qemu-kvm | |
15 | * revision 4144fe9d48. See its repository for the history. | |
16 | * | |
17 | * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) | |
18 | * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) | |
19 | * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) | |
20 | * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) | |
21 | * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) | |
22 | */ | |
23 | #include <stdio.h> | |
24 | #include <unistd.h> | |
25 | #include <sys/io.h> | |
26 | #include <sys/mman.h> | |
27 | #include <sys/types.h> | |
28 | #include <sys/stat.h> | |
29 | #include "hw/hw.h" | |
0d09e41a | 30 | #include "hw/i386/pc.h" |
1de7afc9 | 31 | #include "qemu/error-report.h" |
28ecbaee | 32 | #include "ui/console.h" |
c3ebd3ba | 33 | #include "hw/loader.h" |
83c9089e | 34 | #include "monitor/monitor.h" |
1de7afc9 | 35 | #include "qemu/range.h" |
9c17d615 | 36 | #include "sysemu/sysemu.h" |
a2cb15b0 MT |
37 | #include "hw/pci/pci.h" |
38 | #include "hw/pci/msi.h" | |
c3ebd3ba JK |
39 | #include "kvm_i386.h" |
40 | ||
41 | #define MSIX_PAGE_SIZE 0x1000 | |
42 | ||
43 | /* From linux/ioport.h */ | |
44 | #define IORESOURCE_IO 0x00000100 /* Resource type */ | |
45 | #define IORESOURCE_MEM 0x00000200 | |
46 | #define IORESOURCE_IRQ 0x00000400 | |
47 | #define IORESOURCE_DMA 0x00000800 | |
48 | #define IORESOURCE_PREFETCH 0x00002000 /* No side effects */ | |
0a2a59d3 | 49 | #define IORESOURCE_MEM_64 0x00100000 |
c3ebd3ba JK |
50 | |
51 | //#define DEVICE_ASSIGNMENT_DEBUG | |
52 | ||
53 | #ifdef DEVICE_ASSIGNMENT_DEBUG | |
54 | #define DEBUG(fmt, ...) \ | |
55 | do { \ | |
56 | fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__); \ | |
57 | } while (0) | |
58 | #else | |
59 | #define DEBUG(fmt, ...) | |
60 | #endif | |
61 | ||
62 | typedef struct PCIRegion { | |
63 | int type; /* Memory or port I/O */ | |
64 | int valid; | |
65 | uint64_t base_addr; | |
66 | uint64_t size; /* size of the region */ | |
67 | int resource_fd; | |
68 | } PCIRegion; | |
69 | ||
70 | typedef struct PCIDevRegions { | |
71 | uint8_t bus, dev, func; /* Bus inside domain, device and function */ | |
72 | int irq; /* IRQ number */ | |
73 | uint16_t region_number; /* number of active regions */ | |
74 | ||
75 | /* Port I/O or MMIO Regions */ | |
76 | PCIRegion regions[PCI_NUM_REGIONS - 1]; | |
77 | int config_fd; | |
78 | } PCIDevRegions; | |
79 | ||
80 | typedef struct AssignedDevRegion { | |
81 | MemoryRegion container; | |
82 | MemoryRegion real_iomem; | |
83 | union { | |
84 | uint8_t *r_virtbase; /* mmapped access address for memory regions */ | |
85 | uint32_t r_baseport; /* the base guest port for I/O regions */ | |
86 | } u; | |
87 | pcibus_t e_size; /* emulated size of region in bytes */ | |
88 | pcibus_t r_size; /* real size of region in bytes */ | |
89 | PCIRegion *region; | |
90 | } AssignedDevRegion; | |
91 | ||
92 | #define ASSIGNED_DEVICE_PREFER_MSI_BIT 0 | |
93 | #define ASSIGNED_DEVICE_SHARE_INTX_BIT 1 | |
94 | ||
95 | #define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) | |
96 | #define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT) | |
97 | ||
98 | typedef struct MSIXTableEntry { | |
99 | uint32_t addr_lo; | |
100 | uint32_t addr_hi; | |
101 | uint32_t data; | |
102 | uint32_t ctrl; | |
103 | } MSIXTableEntry; | |
104 | ||
105 | typedef enum AssignedIRQType { | |
106 | ASSIGNED_IRQ_NONE = 0, | |
107 | ASSIGNED_IRQ_INTX_HOST_INTX, | |
108 | ASSIGNED_IRQ_INTX_HOST_MSI, | |
109 | ASSIGNED_IRQ_MSI, | |
110 | ASSIGNED_IRQ_MSIX | |
111 | } AssignedIRQType; | |
112 | ||
113 | typedef struct AssignedDevice { | |
114 | PCIDevice dev; | |
115 | PCIHostDeviceAddress host; | |
116 | uint32_t dev_id; | |
117 | uint32_t features; | |
118 | int intpin; | |
119 | AssignedDevRegion v_addrs[PCI_NUM_REGIONS - 1]; | |
120 | PCIDevRegions real_device; | |
121 | PCIINTxRoute intx_route; | |
122 | AssignedIRQType assigned_irq_type; | |
123 | struct { | |
124 | #define ASSIGNED_DEVICE_CAP_MSI (1 << 0) | |
125 | #define ASSIGNED_DEVICE_CAP_MSIX (1 << 1) | |
126 | uint32_t available; | |
127 | #define ASSIGNED_DEVICE_MSI_ENABLED (1 << 0) | |
128 | #define ASSIGNED_DEVICE_MSIX_ENABLED (1 << 1) | |
129 | #define ASSIGNED_DEVICE_MSIX_MASKED (1 << 2) | |
130 | uint32_t state; | |
131 | } cap; | |
132 | uint8_t emulate_config_read[PCI_CONFIG_SPACE_SIZE]; | |
133 | uint8_t emulate_config_write[PCI_CONFIG_SPACE_SIZE]; | |
134 | int msi_virq_nr; | |
135 | int *msi_virq; | |
136 | MSIXTableEntry *msix_table; | |
a8170e5e | 137 | hwaddr msix_table_addr; |
c3ebd3ba JK |
138 | uint16_t msix_max; |
139 | MemoryRegion mmio; | |
140 | char *configfd_name; | |
141 | int32_t bootindex; | |
142 | } AssignedDevice; | |
143 | ||
144 | static void assigned_dev_update_irq_routing(PCIDevice *dev); | |
145 | ||
146 | static void assigned_dev_load_option_rom(AssignedDevice *dev); | |
147 | ||
148 | static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev); | |
149 | ||
150 | static uint64_t assigned_dev_ioport_rw(AssignedDevRegion *dev_region, | |
a8170e5e | 151 | hwaddr addr, int size, |
c3ebd3ba JK |
152 | uint64_t *data) |
153 | { | |
154 | uint64_t val = 0; | |
155 | int fd = dev_region->region->resource_fd; | |
156 | ||
157 | if (fd >= 0) { | |
158 | if (data) { | |
159 | DEBUG("pwrite data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
160 | ", addr="TARGET_FMT_plx"\n", *data, size, addr, addr); | |
161 | if (pwrite(fd, data, size, addr) != size) { | |
162 | error_report("%s - pwrite failed %s", | |
163 | __func__, strerror(errno)); | |
164 | } | |
165 | } else { | |
166 | if (pread(fd, &val, size, addr) != size) { | |
167 | error_report("%s - pread failed %s", | |
168 | __func__, strerror(errno)); | |
169 | val = (1UL << (size * 8)) - 1; | |
170 | } | |
171 | DEBUG("pread val=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
172 | ", addr=" TARGET_FMT_plx "\n", val, size, addr, addr); | |
173 | } | |
174 | } else { | |
175 | uint32_t port = addr + dev_region->u.r_baseport; | |
176 | ||
177 | if (data) { | |
178 | DEBUG("out data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
179 | ", host=%x\n", *data, size, addr, port); | |
180 | switch (size) { | |
181 | case 1: | |
182 | outb(*data, port); | |
183 | break; | |
184 | case 2: | |
185 | outw(*data, port); | |
186 | break; | |
187 | case 4: | |
188 | outl(*data, port); | |
189 | break; | |
190 | } | |
191 | } else { | |
192 | switch (size) { | |
193 | case 1: | |
194 | val = inb(port); | |
195 | break; | |
196 | case 2: | |
197 | val = inw(port); | |
198 | break; | |
199 | case 4: | |
200 | val = inl(port); | |
201 | break; | |
202 | } | |
203 | DEBUG("in data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
204 | ", host=%x\n", val, size, addr, port); | |
205 | } | |
206 | } | |
207 | return val; | |
208 | } | |
209 | ||
a8170e5e | 210 | static void assigned_dev_ioport_write(void *opaque, hwaddr addr, |
c3ebd3ba JK |
211 | uint64_t data, unsigned size) |
212 | { | |
213 | assigned_dev_ioport_rw(opaque, addr, size, &data); | |
214 | } | |
215 | ||
216 | static uint64_t assigned_dev_ioport_read(void *opaque, | |
a8170e5e | 217 | hwaddr addr, unsigned size) |
c3ebd3ba JK |
218 | { |
219 | return assigned_dev_ioport_rw(opaque, addr, size, NULL); | |
220 | } | |
221 | ||
a8170e5e | 222 | static uint32_t slow_bar_readb(void *opaque, hwaddr addr) |
c3ebd3ba JK |
223 | { |
224 | AssignedDevRegion *d = opaque; | |
225 | uint8_t *in = d->u.r_virtbase + addr; | |
226 | uint32_t r; | |
227 | ||
228 | r = *in; | |
229 | DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); | |
230 | ||
231 | return r; | |
232 | } | |
233 | ||
a8170e5e | 234 | static uint32_t slow_bar_readw(void *opaque, hwaddr addr) |
c3ebd3ba JK |
235 | { |
236 | AssignedDevRegion *d = opaque; | |
237 | uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr); | |
238 | uint32_t r; | |
239 | ||
240 | r = *in; | |
241 | DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); | |
242 | ||
243 | return r; | |
244 | } | |
245 | ||
a8170e5e | 246 | static uint32_t slow_bar_readl(void *opaque, hwaddr addr) |
c3ebd3ba JK |
247 | { |
248 | AssignedDevRegion *d = opaque; | |
249 | uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr); | |
250 | uint32_t r; | |
251 | ||
252 | r = *in; | |
253 | DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); | |
254 | ||
255 | return r; | |
256 | } | |
257 | ||
a8170e5e | 258 | static void slow_bar_writeb(void *opaque, hwaddr addr, uint32_t val) |
c3ebd3ba JK |
259 | { |
260 | AssignedDevRegion *d = opaque; | |
261 | uint8_t *out = d->u.r_virtbase + addr; | |
262 | ||
263 | DEBUG("slow_bar_writeb addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr, val); | |
264 | *out = val; | |
265 | } | |
266 | ||
a8170e5e | 267 | static void slow_bar_writew(void *opaque, hwaddr addr, uint32_t val) |
c3ebd3ba JK |
268 | { |
269 | AssignedDevRegion *d = opaque; | |
270 | uint16_t *out = (uint16_t *)(d->u.r_virtbase + addr); | |
271 | ||
272 | DEBUG("slow_bar_writew addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr, val); | |
273 | *out = val; | |
274 | } | |
275 | ||
a8170e5e | 276 | static void slow_bar_writel(void *opaque, hwaddr addr, uint32_t val) |
c3ebd3ba JK |
277 | { |
278 | AssignedDevRegion *d = opaque; | |
279 | uint32_t *out = (uint32_t *)(d->u.r_virtbase + addr); | |
280 | ||
281 | DEBUG("slow_bar_writel addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, val); | |
282 | *out = val; | |
283 | } | |
284 | ||
285 | static const MemoryRegionOps slow_bar_ops = { | |
286 | .old_mmio = { | |
287 | .read = { slow_bar_readb, slow_bar_readw, slow_bar_readl, }, | |
288 | .write = { slow_bar_writeb, slow_bar_writew, slow_bar_writel, }, | |
289 | }, | |
290 | .endianness = DEVICE_NATIVE_ENDIAN, | |
291 | }; | |
292 | ||
293 | static void assigned_dev_iomem_setup(PCIDevice *pci_dev, int region_num, | |
294 | pcibus_t e_size) | |
295 | { | |
296 | AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
297 | AssignedDevRegion *region = &r_dev->v_addrs[region_num]; | |
298 | PCIRegion *real_region = &r_dev->real_device.regions[region_num]; | |
299 | ||
300 | if (e_size > 0) { | |
2c9b15ca | 301 | memory_region_init(®ion->container, NULL, "assigned-dev-container", |
c3ebd3ba JK |
302 | e_size); |
303 | memory_region_add_subregion(®ion->container, 0, ®ion->real_iomem); | |
304 | ||
305 | /* deal with MSI-X MMIO page */ | |
306 | if (real_region->base_addr <= r_dev->msix_table_addr && | |
307 | real_region->base_addr + real_region->size > | |
308 | r_dev->msix_table_addr) { | |
309 | uint64_t offset = r_dev->msix_table_addr - real_region->base_addr; | |
310 | ||
311 | memory_region_add_subregion_overlap(®ion->container, | |
312 | offset, | |
313 | &r_dev->mmio, | |
314 | 1); | |
315 | } | |
316 | } | |
317 | } | |
318 | ||
319 | static const MemoryRegionOps assigned_dev_ioport_ops = { | |
320 | .read = assigned_dev_ioport_read, | |
321 | .write = assigned_dev_ioport_write, | |
322 | .endianness = DEVICE_NATIVE_ENDIAN, | |
323 | }; | |
324 | ||
325 | static void assigned_dev_ioport_setup(PCIDevice *pci_dev, int region_num, | |
326 | pcibus_t size) | |
327 | { | |
328 | AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
329 | AssignedDevRegion *region = &r_dev->v_addrs[region_num]; | |
330 | ||
331 | region->e_size = size; | |
2c9b15ca PB |
332 | memory_region_init(®ion->container, NULL, "assigned-dev-container", size); |
333 | memory_region_init_io(®ion->real_iomem, NULL, &assigned_dev_ioport_ops, | |
c3ebd3ba JK |
334 | r_dev->v_addrs + region_num, |
335 | "assigned-dev-iomem", size); | |
336 | memory_region_add_subregion(®ion->container, 0, ®ion->real_iomem); | |
337 | } | |
338 | ||
339 | static uint32_t assigned_dev_pci_read(PCIDevice *d, int pos, int len) | |
340 | { | |
341 | AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d); | |
342 | uint32_t val; | |
343 | ssize_t ret; | |
344 | int fd = pci_dev->real_device.config_fd; | |
345 | ||
346 | again: | |
347 | ret = pread(fd, &val, len, pos); | |
348 | if (ret != len) { | |
349 | if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) { | |
350 | goto again; | |
351 | } | |
352 | ||
353 | hw_error("pci read failed, ret = %zd errno = %d\n", ret, errno); | |
354 | } | |
355 | ||
356 | return val; | |
357 | } | |
358 | ||
359 | static uint8_t assigned_dev_pci_read_byte(PCIDevice *d, int pos) | |
360 | { | |
361 | return (uint8_t)assigned_dev_pci_read(d, pos, 1); | |
362 | } | |
363 | ||
364 | static void assigned_dev_pci_write(PCIDevice *d, int pos, uint32_t val, int len) | |
365 | { | |
366 | AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d); | |
367 | ssize_t ret; | |
368 | int fd = pci_dev->real_device.config_fd; | |
369 | ||
370 | again: | |
371 | ret = pwrite(fd, &val, len, pos); | |
372 | if (ret != len) { | |
373 | if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) { | |
374 | goto again; | |
375 | } | |
376 | ||
377 | hw_error("pci write failed, ret = %zd errno = %d\n", ret, errno); | |
378 | } | |
379 | } | |
380 | ||
381 | static void assigned_dev_emulate_config_read(AssignedDevice *dev, | |
382 | uint32_t offset, uint32_t len) | |
383 | { | |
384 | memset(dev->emulate_config_read + offset, 0xff, len); | |
385 | } | |
386 | ||
387 | static void assigned_dev_direct_config_read(AssignedDevice *dev, | |
388 | uint32_t offset, uint32_t len) | |
389 | { | |
390 | memset(dev->emulate_config_read + offset, 0, len); | |
391 | } | |
392 | ||
393 | static void assigned_dev_direct_config_write(AssignedDevice *dev, | |
394 | uint32_t offset, uint32_t len) | |
395 | { | |
396 | memset(dev->emulate_config_write + offset, 0, len); | |
397 | } | |
398 | ||
399 | static uint8_t pci_find_cap_offset(PCIDevice *d, uint8_t cap, uint8_t start) | |
400 | { | |
401 | int id; | |
402 | int max_cap = 48; | |
403 | int pos = start ? start : PCI_CAPABILITY_LIST; | |
404 | int status; | |
405 | ||
406 | status = assigned_dev_pci_read_byte(d, PCI_STATUS); | |
407 | if ((status & PCI_STATUS_CAP_LIST) == 0) { | |
408 | return 0; | |
409 | } | |
410 | ||
411 | while (max_cap--) { | |
412 | pos = assigned_dev_pci_read_byte(d, pos); | |
413 | if (pos < 0x40) { | |
414 | break; | |
415 | } | |
416 | ||
417 | pos &= ~3; | |
418 | id = assigned_dev_pci_read_byte(d, pos + PCI_CAP_LIST_ID); | |
419 | ||
420 | if (id == 0xff) { | |
421 | break; | |
422 | } | |
423 | if (id == cap) { | |
424 | return pos; | |
425 | } | |
426 | ||
427 | pos += PCI_CAP_LIST_NEXT; | |
428 | } | |
429 | return 0; | |
430 | } | |
431 | ||
432 | static int assigned_dev_register_regions(PCIRegion *io_regions, | |
433 | unsigned long regions_num, | |
434 | AssignedDevice *pci_dev) | |
435 | { | |
436 | uint32_t i; | |
437 | PCIRegion *cur_region = io_regions; | |
438 | ||
439 | for (i = 0; i < regions_num; i++, cur_region++) { | |
440 | if (!cur_region->valid) { | |
441 | continue; | |
442 | } | |
443 | ||
444 | /* handle memory io regions */ | |
445 | if (cur_region->type & IORESOURCE_MEM) { | |
0a2a59d3 XH |
446 | int t = PCI_BASE_ADDRESS_SPACE_MEMORY; |
447 | if (cur_region->type & IORESOURCE_PREFETCH) { | |
448 | t |= PCI_BASE_ADDRESS_MEM_PREFETCH; | |
449 | } | |
450 | if (cur_region->type & IORESOURCE_MEM_64) { | |
451 | t |= PCI_BASE_ADDRESS_MEM_TYPE_64; | |
452 | } | |
c3ebd3ba JK |
453 | |
454 | /* map physical memory */ | |
455 | pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL, cur_region->size, | |
456 | PROT_WRITE | PROT_READ, | |
457 | MAP_SHARED, | |
458 | cur_region->resource_fd, | |
459 | (off_t)0); | |
460 | ||
461 | if (pci_dev->v_addrs[i].u.r_virtbase == MAP_FAILED) { | |
462 | pci_dev->v_addrs[i].u.r_virtbase = NULL; | |
463 | error_report("%s: Error: Couldn't mmap 0x%" PRIx64 "!", | |
464 | __func__, cur_region->base_addr); | |
465 | return -1; | |
466 | } | |
467 | ||
468 | pci_dev->v_addrs[i].r_size = cur_region->size; | |
469 | pci_dev->v_addrs[i].e_size = 0; | |
470 | ||
471 | /* add offset */ | |
472 | pci_dev->v_addrs[i].u.r_virtbase += | |
473 | (cur_region->base_addr & 0xFFF); | |
474 | ||
475 | if (cur_region->size & 0xFFF) { | |
476 | error_report("PCI region %d at address 0x%" PRIx64 " has " | |
477 | "size 0x%" PRIx64 ", which is not a multiple of " | |
478 | "4K. You might experience some performance hit " | |
479 | "due to that.", | |
480 | i, cur_region->base_addr, cur_region->size); | |
2c9b15ca | 481 | memory_region_init_io(&pci_dev->v_addrs[i].real_iomem, NULL, |
c3ebd3ba JK |
482 | &slow_bar_ops, &pci_dev->v_addrs[i], |
483 | "assigned-dev-slow-bar", | |
484 | cur_region->size); | |
485 | } else { | |
486 | void *virtbase = pci_dev->v_addrs[i].u.r_virtbase; | |
487 | char name[32]; | |
488 | snprintf(name, sizeof(name), "%s.bar%d", | |
489 | object_get_typename(OBJECT(pci_dev)), i); | |
2c9b15ca | 490 | memory_region_init_ram_ptr(&pci_dev->v_addrs[i].real_iomem, NULL, |
c3ebd3ba JK |
491 | name, cur_region->size, |
492 | virtbase); | |
493 | vmstate_register_ram(&pci_dev->v_addrs[i].real_iomem, | |
494 | &pci_dev->dev.qdev); | |
495 | } | |
496 | ||
497 | assigned_dev_iomem_setup(&pci_dev->dev, i, cur_region->size); | |
498 | pci_register_bar((PCIDevice *) pci_dev, i, t, | |
499 | &pci_dev->v_addrs[i].container); | |
500 | continue; | |
501 | } else { | |
502 | /* handle port io regions */ | |
503 | uint32_t val; | |
504 | int ret; | |
505 | ||
506 | /* Test kernel support for ioport resource read/write. Old | |
507 | * kernels return EIO. New kernels only allow 1/2/4 byte reads | |
508 | * so should return EINVAL for a 3 byte read */ | |
509 | ret = pread(pci_dev->v_addrs[i].region->resource_fd, &val, 3, 0); | |
510 | if (ret >= 0) { | |
511 | error_report("Unexpected return from I/O port read: %d", ret); | |
512 | abort(); | |
513 | } else if (errno != EINVAL) { | |
514 | error_report("Kernel doesn't support ioport resource " | |
515 | "access, hiding this region."); | |
516 | close(pci_dev->v_addrs[i].region->resource_fd); | |
517 | cur_region->valid = 0; | |
518 | continue; | |
519 | } | |
520 | ||
521 | pci_dev->v_addrs[i].u.r_baseport = cur_region->base_addr; | |
522 | pci_dev->v_addrs[i].r_size = cur_region->size; | |
523 | pci_dev->v_addrs[i].e_size = 0; | |
524 | ||
525 | assigned_dev_ioport_setup(&pci_dev->dev, i, cur_region->size); | |
526 | pci_register_bar((PCIDevice *) pci_dev, i, | |
527 | PCI_BASE_ADDRESS_SPACE_IO, | |
528 | &pci_dev->v_addrs[i].container); | |
529 | } | |
530 | } | |
531 | ||
532 | /* success */ | |
533 | return 0; | |
534 | } | |
535 | ||
536 | static int get_real_id(const char *devpath, const char *idname, uint16_t *val) | |
537 | { | |
538 | FILE *f; | |
539 | char name[128]; | |
540 | long id; | |
541 | ||
542 | snprintf(name, sizeof(name), "%s%s", devpath, idname); | |
543 | f = fopen(name, "r"); | |
544 | if (f == NULL) { | |
545 | error_report("%s: %s: %m", __func__, name); | |
546 | return -1; | |
547 | } | |
548 | if (fscanf(f, "%li\n", &id) == 1) { | |
549 | *val = id; | |
550 | } else { | |
90527d2a | 551 | fclose(f); |
c3ebd3ba JK |
552 | return -1; |
553 | } | |
554 | fclose(f); | |
555 | ||
556 | return 0; | |
557 | } | |
558 | ||
559 | static int get_real_vendor_id(const char *devpath, uint16_t *val) | |
560 | { | |
561 | return get_real_id(devpath, "vendor", val); | |
562 | } | |
563 | ||
564 | static int get_real_device_id(const char *devpath, uint16_t *val) | |
565 | { | |
566 | return get_real_id(devpath, "device", val); | |
567 | } | |
568 | ||
569 | static int get_real_device(AssignedDevice *pci_dev, uint16_t r_seg, | |
570 | uint8_t r_bus, uint8_t r_dev, uint8_t r_func) | |
571 | { | |
572 | char dir[128], name[128]; | |
573 | int fd, r = 0, v; | |
574 | FILE *f; | |
575 | uint64_t start, end, size, flags; | |
576 | uint16_t id; | |
577 | PCIRegion *rp; | |
578 | PCIDevRegions *dev = &pci_dev->real_device; | |
579 | ||
580 | dev->region_number = 0; | |
581 | ||
582 | snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%x/", | |
583 | r_seg, r_bus, r_dev, r_func); | |
584 | ||
585 | snprintf(name, sizeof(name), "%sconfig", dir); | |
586 | ||
587 | if (pci_dev->configfd_name && *pci_dev->configfd_name) { | |
9a3a8895 PB |
588 | dev->config_fd = monitor_handle_fd_param(cur_mon, pci_dev->configfd_name); |
589 | if (dev->config_fd < 0) { | |
590 | return 1; | |
c3ebd3ba JK |
591 | } |
592 | } else { | |
593 | dev->config_fd = open(name, O_RDWR); | |
594 | ||
595 | if (dev->config_fd == -1) { | |
596 | error_report("%s: %s: %m", __func__, name); | |
597 | return 1; | |
598 | } | |
599 | } | |
600 | again: | |
601 | r = read(dev->config_fd, pci_dev->dev.config, | |
602 | pci_config_size(&pci_dev->dev)); | |
603 | if (r < 0) { | |
604 | if (errno == EINTR || errno == EAGAIN) { | |
605 | goto again; | |
606 | } | |
607 | error_report("%s: read failed, errno = %d", __func__, errno); | |
608 | } | |
609 | ||
610 | /* Restore or clear multifunction, this is always controlled by qemu */ | |
611 | if (pci_dev->dev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) { | |
612 | pci_dev->dev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION; | |
613 | } else { | |
614 | pci_dev->dev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION; | |
615 | } | |
616 | ||
617 | /* Clear host resource mapping info. If we choose not to register a | |
618 | * BAR, such as might be the case with the option ROM, we can get | |
619 | * confusing, unwritable, residual addresses from the host here. */ | |
620 | memset(&pci_dev->dev.config[PCI_BASE_ADDRESS_0], 0, 24); | |
621 | memset(&pci_dev->dev.config[PCI_ROM_ADDRESS], 0, 4); | |
622 | ||
623 | snprintf(name, sizeof(name), "%sresource", dir); | |
624 | ||
625 | f = fopen(name, "r"); | |
626 | if (f == NULL) { | |
627 | error_report("%s: %s: %m", __func__, name); | |
628 | return 1; | |
629 | } | |
630 | ||
631 | for (r = 0; r < PCI_ROM_SLOT; r++) { | |
632 | if (fscanf(f, "%" SCNi64 " %" SCNi64 " %" SCNi64 "\n", | |
633 | &start, &end, &flags) != 3) { | |
634 | break; | |
635 | } | |
636 | ||
637 | rp = dev->regions + r; | |
638 | rp->valid = 0; | |
639 | rp->resource_fd = -1; | |
640 | size = end - start + 1; | |
0a2a59d3 XH |
641 | flags &= IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH |
642 | | IORESOURCE_MEM_64; | |
c3ebd3ba JK |
643 | if (size == 0 || (flags & ~IORESOURCE_PREFETCH) == 0) { |
644 | continue; | |
645 | } | |
646 | if (flags & IORESOURCE_MEM) { | |
647 | flags &= ~IORESOURCE_IO; | |
648 | } else { | |
649 | flags &= ~IORESOURCE_PREFETCH; | |
650 | } | |
651 | snprintf(name, sizeof(name), "%sresource%d", dir, r); | |
652 | fd = open(name, O_RDWR); | |
653 | if (fd == -1) { | |
654 | continue; | |
655 | } | |
656 | rp->resource_fd = fd; | |
657 | ||
658 | rp->type = flags; | |
659 | rp->valid = 1; | |
660 | rp->base_addr = start; | |
661 | rp->size = size; | |
662 | pci_dev->v_addrs[r].region = rp; | |
663 | DEBUG("region %d size %" PRIu64 " start 0x%" PRIx64 | |
664 | " type %d resource_fd %d\n", | |
665 | r, rp->size, start, rp->type, rp->resource_fd); | |
666 | } | |
667 | ||
668 | fclose(f); | |
669 | ||
670 | /* read and fill vendor ID */ | |
671 | v = get_real_vendor_id(dir, &id); | |
672 | if (v) { | |
673 | return 1; | |
674 | } | |
675 | pci_dev->dev.config[0] = id & 0xff; | |
676 | pci_dev->dev.config[1] = (id & 0xff00) >> 8; | |
677 | ||
678 | /* read and fill device ID */ | |
679 | v = get_real_device_id(dir, &id); | |
680 | if (v) { | |
681 | return 1; | |
682 | } | |
683 | pci_dev->dev.config[2] = id & 0xff; | |
684 | pci_dev->dev.config[3] = (id & 0xff00) >> 8; | |
685 | ||
686 | pci_word_test_and_clear_mask(pci_dev->emulate_config_write + PCI_COMMAND, | |
687 | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE); | |
688 | ||
689 | dev->region_number = r; | |
690 | return 0; | |
691 | } | |
692 | ||
693 | static void free_msi_virqs(AssignedDevice *dev) | |
694 | { | |
695 | int i; | |
696 | ||
697 | for (i = 0; i < dev->msi_virq_nr; i++) { | |
698 | if (dev->msi_virq[i] >= 0) { | |
699 | kvm_irqchip_release_virq(kvm_state, dev->msi_virq[i]); | |
700 | dev->msi_virq[i] = -1; | |
701 | } | |
702 | } | |
703 | g_free(dev->msi_virq); | |
704 | dev->msi_virq = NULL; | |
705 | dev->msi_virq_nr = 0; | |
706 | } | |
707 | ||
708 | static void free_assigned_device(AssignedDevice *dev) | |
709 | { | |
710 | int i; | |
711 | ||
712 | if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) { | |
713 | assigned_dev_unregister_msix_mmio(dev); | |
714 | } | |
715 | for (i = 0; i < dev->real_device.region_number; i++) { | |
716 | PCIRegion *pci_region = &dev->real_device.regions[i]; | |
717 | AssignedDevRegion *region = &dev->v_addrs[i]; | |
718 | ||
719 | if (!pci_region->valid) { | |
720 | continue; | |
721 | } | |
722 | if (pci_region->type & IORESOURCE_IO) { | |
723 | if (region->u.r_baseport) { | |
724 | memory_region_del_subregion(®ion->container, | |
725 | ®ion->real_iomem); | |
726 | memory_region_destroy(®ion->real_iomem); | |
727 | memory_region_destroy(®ion->container); | |
728 | } | |
729 | } else if (pci_region->type & IORESOURCE_MEM) { | |
730 | if (region->u.r_virtbase) { | |
731 | memory_region_del_subregion(®ion->container, | |
732 | ®ion->real_iomem); | |
733 | ||
734 | /* Remove MSI-X table subregion */ | |
735 | if (pci_region->base_addr <= dev->msix_table_addr && | |
736 | pci_region->base_addr + pci_region->size > | |
737 | dev->msix_table_addr) { | |
738 | memory_region_del_subregion(®ion->container, | |
739 | &dev->mmio); | |
740 | } | |
741 | ||
742 | memory_region_destroy(®ion->real_iomem); | |
743 | memory_region_destroy(®ion->container); | |
744 | if (munmap(region->u.r_virtbase, | |
745 | (pci_region->size + 0xFFF) & 0xFFFFF000)) { | |
746 | error_report("Failed to unmap assigned device region: %s", | |
747 | strerror(errno)); | |
748 | } | |
749 | } | |
750 | } | |
751 | if (pci_region->resource_fd >= 0) { | |
752 | close(pci_region->resource_fd); | |
753 | } | |
754 | } | |
755 | ||
756 | if (dev->real_device.config_fd >= 0) { | |
757 | close(dev->real_device.config_fd); | |
758 | } | |
759 | ||
760 | free_msi_virqs(dev); | |
761 | } | |
762 | ||
763 | static void assign_failed_examine(AssignedDevice *dev) | |
764 | { | |
765 | char name[PATH_MAX], dir[PATH_MAX], driver[PATH_MAX] = {}, *ns; | |
766 | uint16_t vendor_id, device_id; | |
767 | int r; | |
768 | ||
769 | snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/", | |
770 | dev->host.domain, dev->host.bus, dev->host.slot, | |
771 | dev->host.function); | |
772 | ||
773 | snprintf(name, sizeof(name), "%sdriver", dir); | |
774 | ||
775 | r = readlink(name, driver, sizeof(driver)); | |
776 | if ((r <= 0) || r >= sizeof(driver)) { | |
777 | goto fail; | |
778 | } | |
779 | ||
780 | ns = strrchr(driver, '/'); | |
781 | if (!ns) { | |
782 | goto fail; | |
783 | } | |
784 | ||
785 | ns++; | |
786 | ||
787 | if (get_real_vendor_id(dir, &vendor_id) || | |
788 | get_real_device_id(dir, &device_id)) { | |
789 | goto fail; | |
790 | } | |
791 | ||
792 | error_report("*** The driver '%s' is occupying your device " | |
793 | "%04x:%02x:%02x.%x.", | |
794 | ns, dev->host.domain, dev->host.bus, dev->host.slot, | |
795 | dev->host.function); | |
796 | error_report("***"); | |
797 | error_report("*** You can try the following commands to free it:"); | |
798 | error_report("***"); | |
799 | error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub/" | |
800 | "new_id", vendor_id, device_id); | |
801 | error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/" | |
802 | "%s/unbind", | |
803 | dev->host.domain, dev->host.bus, dev->host.slot, | |
804 | dev->host.function, ns); | |
805 | error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/" | |
806 | "pci-stub/bind", | |
807 | dev->host.domain, dev->host.bus, dev->host.slot, | |
808 | dev->host.function); | |
809 | error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub" | |
810 | "/remove_id", vendor_id, device_id); | |
811 | error_report("***"); | |
812 | ||
813 | return; | |
814 | ||
815 | fail: | |
816 | error_report("Couldn't find out why."); | |
817 | } | |
818 | ||
819 | static int assign_device(AssignedDevice *dev) | |
820 | { | |
821 | uint32_t flags = KVM_DEV_ASSIGN_ENABLE_IOMMU; | |
822 | int r; | |
823 | ||
824 | /* Only pass non-zero PCI segment to capable module */ | |
825 | if (!kvm_check_extension(kvm_state, KVM_CAP_PCI_SEGMENT) && | |
826 | dev->host.domain) { | |
827 | error_report("Can't assign device inside non-zero PCI segment " | |
828 | "as this KVM module doesn't support it."); | |
829 | return -ENODEV; | |
830 | } | |
831 | ||
832 | if (!kvm_check_extension(kvm_state, KVM_CAP_IOMMU)) { | |
833 | error_report("No IOMMU found. Unable to assign device \"%s\"", | |
834 | dev->dev.qdev.id); | |
835 | return -ENODEV; | |
836 | } | |
837 | ||
838 | if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK && | |
839 | kvm_has_intx_set_mask()) { | |
840 | flags |= KVM_DEV_ASSIGN_PCI_2_3; | |
841 | } | |
842 | ||
843 | r = kvm_device_pci_assign(kvm_state, &dev->host, flags, &dev->dev_id); | |
844 | if (r < 0) { | |
845 | error_report("Failed to assign device \"%s\" : %s", | |
846 | dev->dev.qdev.id, strerror(-r)); | |
847 | ||
848 | switch (r) { | |
849 | case -EBUSY: | |
850 | assign_failed_examine(dev); | |
851 | break; | |
852 | default: | |
853 | break; | |
854 | } | |
855 | } | |
856 | return r; | |
857 | } | |
858 | ||
859 | static bool check_irqchip_in_kernel(void) | |
860 | { | |
861 | if (kvm_irqchip_in_kernel()) { | |
862 | return true; | |
863 | } | |
864 | error_report("pci-assign: error: requires KVM with in-kernel irqchip " | |
865 | "enabled"); | |
866 | return false; | |
867 | } | |
868 | ||
869 | static int assign_intx(AssignedDevice *dev) | |
870 | { | |
871 | AssignedIRQType new_type; | |
872 | PCIINTxRoute intx_route; | |
873 | bool intx_host_msi; | |
874 | int r; | |
875 | ||
876 | /* Interrupt PIN 0 means don't use INTx */ | |
877 | if (assigned_dev_pci_read_byte(&dev->dev, PCI_INTERRUPT_PIN) == 0) { | |
878 | pci_device_set_intx_routing_notifier(&dev->dev, NULL); | |
879 | return 0; | |
880 | } | |
881 | ||
882 | if (!check_irqchip_in_kernel()) { | |
883 | return -ENOTSUP; | |
884 | } | |
885 | ||
886 | pci_device_set_intx_routing_notifier(&dev->dev, | |
887 | assigned_dev_update_irq_routing); | |
888 | ||
889 | intx_route = pci_device_route_intx_to_irq(&dev->dev, dev->intpin); | |
890 | assert(intx_route.mode != PCI_INTX_INVERTED); | |
891 | ||
4774d7b2 | 892 | if (!pci_intx_route_changed(&dev->intx_route, &intx_route)) { |
c3ebd3ba JK |
893 | return 0; |
894 | } | |
895 | ||
896 | switch (dev->assigned_irq_type) { | |
897 | case ASSIGNED_IRQ_INTX_HOST_INTX: | |
898 | case ASSIGNED_IRQ_INTX_HOST_MSI: | |
899 | intx_host_msi = dev->assigned_irq_type == ASSIGNED_IRQ_INTX_HOST_MSI; | |
900 | r = kvm_device_intx_deassign(kvm_state, dev->dev_id, intx_host_msi); | |
901 | break; | |
902 | case ASSIGNED_IRQ_MSI: | |
903 | r = kvm_device_msi_deassign(kvm_state, dev->dev_id); | |
904 | break; | |
905 | case ASSIGNED_IRQ_MSIX: | |
906 | r = kvm_device_msix_deassign(kvm_state, dev->dev_id); | |
907 | break; | |
908 | default: | |
909 | r = 0; | |
910 | break; | |
911 | } | |
912 | if (r) { | |
913 | perror("assign_intx: deassignment of previous interrupt failed"); | |
914 | } | |
915 | dev->assigned_irq_type = ASSIGNED_IRQ_NONE; | |
916 | ||
917 | if (intx_route.mode == PCI_INTX_DISABLED) { | |
918 | dev->intx_route = intx_route; | |
919 | return 0; | |
920 | } | |
921 | ||
922 | retry: | |
923 | if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK && | |
924 | dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) { | |
925 | intx_host_msi = true; | |
926 | new_type = ASSIGNED_IRQ_INTX_HOST_MSI; | |
927 | } else { | |
928 | intx_host_msi = false; | |
929 | new_type = ASSIGNED_IRQ_INTX_HOST_INTX; | |
930 | } | |
931 | ||
932 | r = kvm_device_intx_assign(kvm_state, dev->dev_id, intx_host_msi, | |
933 | intx_route.irq); | |
934 | if (r < 0) { | |
935 | if (r == -EIO && !(dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK) && | |
936 | dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) { | |
937 | /* Retry with host-side MSI. There might be an IRQ conflict and | |
938 | * either the kernel or the device doesn't support sharing. */ | |
939 | error_report("Host-side INTx sharing not supported, " | |
474c2134 MA |
940 | "using MSI instead"); |
941 | error_printf("Some devices do not work properly in this mode.\n"); | |
c3ebd3ba JK |
942 | dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK; |
943 | goto retry; | |
944 | } | |
945 | error_report("Failed to assign irq for \"%s\": %s", | |
946 | dev->dev.qdev.id, strerror(-r)); | |
947 | error_report("Perhaps you are assigning a device " | |
948 | "that shares an IRQ with another device?"); | |
949 | return r; | |
950 | } | |
951 | ||
952 | dev->intx_route = intx_route; | |
953 | dev->assigned_irq_type = new_type; | |
954 | return r; | |
955 | } | |
956 | ||
957 | static void deassign_device(AssignedDevice *dev) | |
958 | { | |
959 | int r; | |
960 | ||
961 | r = kvm_device_pci_deassign(kvm_state, dev->dev_id); | |
962 | assert(r == 0); | |
963 | } | |
964 | ||
965 | /* The pci config space got updated. Check if irq numbers have changed | |
966 | * for our devices | |
967 | */ | |
968 | static void assigned_dev_update_irq_routing(PCIDevice *dev) | |
969 | { | |
970 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, dev); | |
971 | Error *err = NULL; | |
972 | int r; | |
973 | ||
974 | r = assign_intx(assigned_dev); | |
975 | if (r < 0) { | |
976 | qdev_unplug(&dev->qdev, &err); | |
977 | assert(!err); | |
978 | } | |
979 | } | |
980 | ||
981 | static void assigned_dev_update_msi(PCIDevice *pci_dev) | |
982 | { | |
983 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
984 | uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap + | |
985 | PCI_MSI_FLAGS); | |
986 | int r; | |
987 | ||
988 | /* Some guests gratuitously disable MSI even if they're not using it, | |
989 | * try to catch this by only deassigning irqs if the guest is using | |
990 | * MSI or intends to start. */ | |
991 | if (assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSI || | |
992 | (ctrl_byte & PCI_MSI_FLAGS_ENABLE)) { | |
993 | r = kvm_device_msi_deassign(kvm_state, assigned_dev->dev_id); | |
994 | /* -ENXIO means no assigned irq */ | |
995 | if (r && r != -ENXIO) { | |
996 | perror("assigned_dev_update_msi: deassign irq"); | |
997 | } | |
998 | ||
999 | free_msi_virqs(assigned_dev); | |
1000 | ||
1001 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE; | |
1002 | pci_device_set_intx_routing_notifier(pci_dev, NULL); | |
1003 | } | |
1004 | ||
1005 | if (ctrl_byte & PCI_MSI_FLAGS_ENABLE) { | |
2b199f93 | 1006 | MSIMessage msg = msi_get_message(pci_dev, 0); |
c3ebd3ba JK |
1007 | int virq; |
1008 | ||
c3ebd3ba JK |
1009 | virq = kvm_irqchip_add_msi_route(kvm_state, msg); |
1010 | if (virq < 0) { | |
1011 | perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route"); | |
1012 | return; | |
1013 | } | |
1014 | ||
1015 | assigned_dev->msi_virq = g_malloc(sizeof(*assigned_dev->msi_virq)); | |
1016 | assigned_dev->msi_virq_nr = 1; | |
1017 | assigned_dev->msi_virq[0] = virq; | |
1018 | if (kvm_device_msi_assign(kvm_state, assigned_dev->dev_id, virq) < 0) { | |
1019 | perror("assigned_dev_update_msi: kvm_device_msi_assign"); | |
1020 | } | |
1021 | ||
1022 | assigned_dev->intx_route.mode = PCI_INTX_DISABLED; | |
1023 | assigned_dev->intx_route.irq = -1; | |
1024 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSI; | |
1025 | } else { | |
1026 | assign_intx(assigned_dev); | |
1027 | } | |
1028 | } | |
1029 | ||
3459f01b AW |
1030 | static void assigned_dev_update_msi_msg(PCIDevice *pci_dev) |
1031 | { | |
1032 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1033 | uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap + | |
1034 | PCI_MSI_FLAGS); | |
1035 | ||
1036 | if (assigned_dev->assigned_irq_type != ASSIGNED_IRQ_MSI || | |
1037 | !(ctrl_byte & PCI_MSI_FLAGS_ENABLE)) { | |
1038 | return; | |
1039 | } | |
1040 | ||
1041 | kvm_irqchip_update_msi_route(kvm_state, assigned_dev->msi_virq[0], | |
1042 | msi_get_message(pci_dev, 0)); | |
1043 | } | |
1044 | ||
c3ebd3ba JK |
1045 | static bool assigned_dev_msix_masked(MSIXTableEntry *entry) |
1046 | { | |
1047 | return (entry->ctrl & cpu_to_le32(0x1)) != 0; | |
1048 | } | |
1049 | ||
feb9a2ab AW |
1050 | /* |
1051 | * When MSI-X is first enabled the vector table typically has all the | |
1052 | * vectors masked, so we can't use that as the obvious test to figure out | |
1053 | * how many vectors to initially enable. Instead we look at the data field | |
1054 | * because this is what worked for pci-assign for a long time. This makes | |
1055 | * sure the physical MSI-X state tracks the guest's view, which is important | |
1056 | * for some VF/PF and PF/fw communication channels. | |
1057 | */ | |
1058 | static bool assigned_dev_msix_skipped(MSIXTableEntry *entry) | |
1059 | { | |
1060 | return !entry->data; | |
1061 | } | |
1062 | ||
c3ebd3ba JK |
1063 | static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev) |
1064 | { | |
1065 | AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1066 | uint16_t entries_nr = 0; | |
1067 | int i, r = 0; | |
1068 | MSIXTableEntry *entry = adev->msix_table; | |
1069 | MSIMessage msg; | |
1070 | ||
1071 | /* Get the usable entry number for allocating */ | |
1072 | for (i = 0; i < adev->msix_max; i++, entry++) { | |
feb9a2ab | 1073 | if (assigned_dev_msix_skipped(entry)) { |
c3ebd3ba JK |
1074 | continue; |
1075 | } | |
1076 | entries_nr++; | |
1077 | } | |
1078 | ||
1079 | DEBUG("MSI-X entries: %d\n", entries_nr); | |
1080 | ||
1081 | /* It's valid to enable MSI-X with all entries masked */ | |
1082 | if (!entries_nr) { | |
1083 | return 0; | |
1084 | } | |
1085 | ||
1086 | r = kvm_device_msix_init_vectors(kvm_state, adev->dev_id, entries_nr); | |
1087 | if (r != 0) { | |
1088 | error_report("fail to set MSI-X entry number for MSIX! %s", | |
1089 | strerror(-r)); | |
1090 | return r; | |
1091 | } | |
1092 | ||
1093 | free_msi_virqs(adev); | |
1094 | ||
1095 | adev->msi_virq_nr = adev->msix_max; | |
1096 | adev->msi_virq = g_malloc(adev->msix_max * sizeof(*adev->msi_virq)); | |
1097 | ||
1098 | entry = adev->msix_table; | |
1099 | for (i = 0; i < adev->msix_max; i++, entry++) { | |
1100 | adev->msi_virq[i] = -1; | |
1101 | ||
feb9a2ab | 1102 | if (assigned_dev_msix_skipped(entry)) { |
c3ebd3ba JK |
1103 | continue; |
1104 | } | |
1105 | ||
1106 | msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32); | |
1107 | msg.data = entry->data; | |
1108 | r = kvm_irqchip_add_msi_route(kvm_state, msg); | |
1109 | if (r < 0) { | |
1110 | return r; | |
1111 | } | |
1112 | adev->msi_virq[i] = r; | |
1113 | ||
1114 | DEBUG("MSI-X vector %d, gsi %d, addr %08x_%08x, data %08x\n", i, | |
1115 | r, entry->addr_hi, entry->addr_lo, entry->data); | |
1116 | ||
1117 | r = kvm_device_msix_set_vector(kvm_state, adev->dev_id, i, | |
1118 | adev->msi_virq[i]); | |
1119 | if (r) { | |
1120 | error_report("fail to set MSI-X entry! %s", strerror(-r)); | |
1121 | break; | |
1122 | } | |
1123 | } | |
1124 | ||
1125 | return r; | |
1126 | } | |
1127 | ||
1128 | static void assigned_dev_update_msix(PCIDevice *pci_dev) | |
1129 | { | |
1130 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1131 | uint16_t ctrl_word = pci_get_word(pci_dev->config + pci_dev->msix_cap + | |
1132 | PCI_MSIX_FLAGS); | |
1133 | int r; | |
1134 | ||
1135 | /* Some guests gratuitously disable MSIX even if they're not using it, | |
1136 | * try to catch this by only deassigning irqs if the guest is using | |
1137 | * MSIX or intends to start. */ | |
1138 | if ((assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSIX) || | |
1139 | (ctrl_word & PCI_MSIX_FLAGS_ENABLE)) { | |
1140 | r = kvm_device_msix_deassign(kvm_state, assigned_dev->dev_id); | |
1141 | /* -ENXIO means no assigned irq */ | |
1142 | if (r && r != -ENXIO) { | |
1143 | perror("assigned_dev_update_msix: deassign irq"); | |
1144 | } | |
1145 | ||
1146 | free_msi_virqs(assigned_dev); | |
1147 | ||
1148 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE; | |
1149 | pci_device_set_intx_routing_notifier(pci_dev, NULL); | |
1150 | } | |
1151 | ||
1152 | if (ctrl_word & PCI_MSIX_FLAGS_ENABLE) { | |
1153 | if (assigned_dev_update_msix_mmio(pci_dev) < 0) { | |
1154 | perror("assigned_dev_update_msix_mmio"); | |
1155 | return; | |
1156 | } | |
1157 | ||
1158 | if (assigned_dev->msi_virq_nr > 0) { | |
1159 | if (kvm_device_msix_assign(kvm_state, assigned_dev->dev_id) < 0) { | |
1160 | perror("assigned_dev_enable_msix: assign irq"); | |
1161 | return; | |
1162 | } | |
1163 | } | |
1164 | assigned_dev->intx_route.mode = PCI_INTX_DISABLED; | |
1165 | assigned_dev->intx_route.irq = -1; | |
1166 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSIX; | |
1167 | } else { | |
1168 | assign_intx(assigned_dev); | |
1169 | } | |
1170 | } | |
1171 | ||
1172 | static uint32_t assigned_dev_pci_read_config(PCIDevice *pci_dev, | |
1173 | uint32_t address, int len) | |
1174 | { | |
1175 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1176 | uint32_t virt_val = pci_default_read_config(pci_dev, address, len); | |
1177 | uint32_t real_val, emulate_mask, full_emulation_mask; | |
1178 | ||
1179 | emulate_mask = 0; | |
1180 | memcpy(&emulate_mask, assigned_dev->emulate_config_read + address, len); | |
1181 | emulate_mask = le32_to_cpu(emulate_mask); | |
1182 | ||
1183 | full_emulation_mask = 0xffffffff >> (32 - len * 8); | |
1184 | ||
1185 | if (emulate_mask != full_emulation_mask) { | |
1186 | real_val = assigned_dev_pci_read(pci_dev, address, len); | |
1187 | return (virt_val & emulate_mask) | (real_val & ~emulate_mask); | |
1188 | } else { | |
1189 | return virt_val; | |
1190 | } | |
1191 | } | |
1192 | ||
1193 | static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, | |
1194 | uint32_t val, int len) | |
1195 | { | |
1196 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1197 | uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND); | |
1198 | uint32_t emulate_mask, full_emulation_mask; | |
1199 | int ret; | |
1200 | ||
1201 | pci_default_write_config(pci_dev, address, val, len); | |
1202 | ||
1203 | if (kvm_has_intx_set_mask() && | |
1204 | range_covers_byte(address, len, PCI_COMMAND + 1)) { | |
1205 | bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) & | |
1206 | PCI_COMMAND_INTX_DISABLE); | |
1207 | ||
1208 | if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) { | |
1209 | ret = kvm_device_intx_set_mask(kvm_state, assigned_dev->dev_id, | |
1210 | intx_masked); | |
1211 | if (ret) { | |
1212 | perror("assigned_dev_pci_write_config: set intx mask"); | |
1213 | } | |
1214 | } | |
1215 | } | |
1216 | if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) { | |
1217 | if (range_covers_byte(address, len, | |
1218 | pci_dev->msi_cap + PCI_MSI_FLAGS)) { | |
1219 | assigned_dev_update_msi(pci_dev); | |
3459f01b AW |
1220 | } else if (ranges_overlap(address, len, /* 32bit MSI only */ |
1221 | pci_dev->msi_cap + PCI_MSI_ADDRESS_LO, 6)) { | |
1222 | assigned_dev_update_msi_msg(pci_dev); | |
c3ebd3ba JK |
1223 | } |
1224 | } | |
1225 | if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) { | |
1226 | if (range_covers_byte(address, len, | |
1227 | pci_dev->msix_cap + PCI_MSIX_FLAGS + 1)) { | |
1228 | assigned_dev_update_msix(pci_dev); | |
1229 | } | |
1230 | } | |
1231 | ||
1232 | emulate_mask = 0; | |
1233 | memcpy(&emulate_mask, assigned_dev->emulate_config_write + address, len); | |
1234 | emulate_mask = le32_to_cpu(emulate_mask); | |
1235 | ||
1236 | full_emulation_mask = 0xffffffff >> (32 - len * 8); | |
1237 | ||
1238 | if (emulate_mask != full_emulation_mask) { | |
1239 | if (emulate_mask) { | |
1240 | val &= ~emulate_mask; | |
1241 | val |= assigned_dev_pci_read(pci_dev, address, len) & emulate_mask; | |
1242 | } | |
1243 | assigned_dev_pci_write(pci_dev, address, val, len); | |
1244 | } | |
1245 | } | |
1246 | ||
1247 | static void assigned_dev_setup_cap_read(AssignedDevice *dev, uint32_t offset, | |
1248 | uint32_t len) | |
1249 | { | |
1250 | assigned_dev_direct_config_read(dev, offset, len); | |
1251 | assigned_dev_emulate_config_read(dev, offset + PCI_CAP_LIST_NEXT, 1); | |
1252 | } | |
1253 | ||
1254 | static int assigned_device_pci_cap_init(PCIDevice *pci_dev) | |
1255 | { | |
1256 | AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1257 | PCIRegion *pci_region = dev->real_device.regions; | |
1258 | int ret, pos; | |
1259 | ||
1260 | /* Clear initial capabilities pointer and status copied from hw */ | |
1261 | pci_set_byte(pci_dev->config + PCI_CAPABILITY_LIST, 0); | |
1262 | pci_set_word(pci_dev->config + PCI_STATUS, | |
1263 | pci_get_word(pci_dev->config + PCI_STATUS) & | |
1264 | ~PCI_STATUS_CAP_LIST); | |
1265 | ||
1266 | /* Expose MSI capability | |
1267 | * MSI capability is the 1st capability in capability config */ | |
1268 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSI, 0); | |
1269 | if (pos != 0 && kvm_check_extension(kvm_state, KVM_CAP_ASSIGN_DEV_IRQ)) { | |
1270 | if (!check_irqchip_in_kernel()) { | |
1271 | return -ENOTSUP; | |
1272 | } | |
1273 | dev->cap.available |= ASSIGNED_DEVICE_CAP_MSI; | |
1274 | /* Only 32-bit/no-mask currently supported */ | |
1275 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSI, pos, 10); | |
1276 | if (ret < 0) { | |
1277 | return ret; | |
1278 | } | |
1279 | pci_dev->msi_cap = pos; | |
1280 | ||
1281 | pci_set_word(pci_dev->config + pos + PCI_MSI_FLAGS, | |
1282 | pci_get_word(pci_dev->config + pos + PCI_MSI_FLAGS) & | |
1283 | PCI_MSI_FLAGS_QMASK); | |
1284 | pci_set_long(pci_dev->config + pos + PCI_MSI_ADDRESS_LO, 0); | |
1285 | pci_set_word(pci_dev->config + pos + PCI_MSI_DATA_32, 0); | |
1286 | ||
1287 | /* Set writable fields */ | |
1288 | pci_set_word(pci_dev->wmask + pos + PCI_MSI_FLAGS, | |
1289 | PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); | |
1290 | pci_set_long(pci_dev->wmask + pos + PCI_MSI_ADDRESS_LO, 0xfffffffc); | |
1291 | pci_set_word(pci_dev->wmask + pos + PCI_MSI_DATA_32, 0xffff); | |
1292 | } | |
1293 | /* Expose MSI-X capability */ | |
1294 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSIX, 0); | |
1295 | if (pos != 0 && kvm_device_msix_supported(kvm_state)) { | |
1296 | int bar_nr; | |
1297 | uint32_t msix_table_entry; | |
1298 | ||
1299 | if (!check_irqchip_in_kernel()) { | |
1300 | return -ENOTSUP; | |
1301 | } | |
1302 | dev->cap.available |= ASSIGNED_DEVICE_CAP_MSIX; | |
1303 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSIX, pos, 12); | |
1304 | if (ret < 0) { | |
1305 | return ret; | |
1306 | } | |
1307 | pci_dev->msix_cap = pos; | |
1308 | ||
1309 | pci_set_word(pci_dev->config + pos + PCI_MSIX_FLAGS, | |
1310 | pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS) & | |
1311 | PCI_MSIX_FLAGS_QSIZE); | |
1312 | ||
1313 | /* Only enable and function mask bits are writable */ | |
1314 | pci_set_word(pci_dev->wmask + pos + PCI_MSIX_FLAGS, | |
1315 | PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); | |
1316 | ||
1317 | msix_table_entry = pci_get_long(pci_dev->config + pos + PCI_MSIX_TABLE); | |
1318 | bar_nr = msix_table_entry & PCI_MSIX_FLAGS_BIRMASK; | |
1319 | msix_table_entry &= ~PCI_MSIX_FLAGS_BIRMASK; | |
1320 | dev->msix_table_addr = pci_region[bar_nr].base_addr + msix_table_entry; | |
1321 | dev->msix_max = pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS); | |
1322 | dev->msix_max &= PCI_MSIX_FLAGS_QSIZE; | |
1323 | dev->msix_max += 1; | |
1324 | } | |
1325 | ||
1326 | /* Minimal PM support, nothing writable, device appears to NAK changes */ | |
1327 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PM, 0); | |
1328 | if (pos) { | |
1329 | uint16_t pmc; | |
1330 | ||
1331 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, pos, PCI_PM_SIZEOF); | |
1332 | if (ret < 0) { | |
1333 | return ret; | |
1334 | } | |
1335 | ||
1336 | assigned_dev_setup_cap_read(dev, pos, PCI_PM_SIZEOF); | |
1337 | ||
1338 | pmc = pci_get_word(pci_dev->config + pos + PCI_CAP_FLAGS); | |
1339 | pmc &= (PCI_PM_CAP_VER_MASK | PCI_PM_CAP_DSI); | |
1340 | pci_set_word(pci_dev->config + pos + PCI_CAP_FLAGS, pmc); | |
1341 | ||
1342 | /* assign_device will bring the device up to D0, so we don't need | |
1343 | * to worry about doing that ourselves here. */ | |
1344 | pci_set_word(pci_dev->config + pos + PCI_PM_CTRL, | |
1345 | PCI_PM_CTRL_NO_SOFT_RESET); | |
1346 | ||
1347 | pci_set_byte(pci_dev->config + pos + PCI_PM_PPB_EXTENSIONS, 0); | |
1348 | pci_set_byte(pci_dev->config + pos + PCI_PM_DATA_REGISTER, 0); | |
1349 | } | |
1350 | ||
1351 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_EXP, 0); | |
1352 | if (pos) { | |
1353 | uint8_t version, size = 0; | |
1354 | uint16_t type, devctl, lnksta; | |
1355 | uint32_t devcap, lnkcap; | |
1356 | ||
1357 | version = pci_get_byte(pci_dev->config + pos + PCI_EXP_FLAGS); | |
1358 | version &= PCI_EXP_FLAGS_VERS; | |
1359 | if (version == 1) { | |
1360 | size = 0x14; | |
1361 | } else if (version == 2) { | |
1362 | /* | |
1363 | * Check for non-std size, accept reduced size to 0x34, | |
1364 | * which is what bcm5761 implemented, violating the | |
1365 | * PCIe v3.0 spec that regs should exist and be read as 0, | |
1366 | * not optionally provided and shorten the struct size. | |
1367 | */ | |
1368 | size = MIN(0x3c, PCI_CONFIG_SPACE_SIZE - pos); | |
1369 | if (size < 0x34) { | |
1370 | error_report("%s: Invalid size PCIe cap-id 0x%x", | |
1371 | __func__, PCI_CAP_ID_EXP); | |
1372 | return -EINVAL; | |
1373 | } else if (size != 0x3c) { | |
1374 | error_report("WARNING, %s: PCIe cap-id 0x%x has " | |
1375 | "non-standard size 0x%x; std size should be 0x3c", | |
1376 | __func__, PCI_CAP_ID_EXP, size); | |
1377 | } | |
1378 | } else if (version == 0) { | |
1379 | uint16_t vid, did; | |
1380 | vid = pci_get_word(pci_dev->config + PCI_VENDOR_ID); | |
1381 | did = pci_get_word(pci_dev->config + PCI_DEVICE_ID); | |
1382 | if (vid == PCI_VENDOR_ID_INTEL && did == 0x10ed) { | |
1383 | /* | |
1384 | * quirk for Intel 82599 VF with invalid PCIe capability | |
1385 | * version, should really be version 2 (same as PF) | |
1386 | */ | |
1387 | size = 0x3c; | |
1388 | } | |
1389 | } | |
1390 | ||
1391 | if (size == 0) { | |
1392 | error_report("%s: Unsupported PCI express capability version %d", | |
1393 | __func__, version); | |
1394 | return -EINVAL; | |
1395 | } | |
1396 | ||
1397 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_EXP, pos, size); | |
1398 | if (ret < 0) { | |
1399 | return ret; | |
1400 | } | |
1401 | ||
1402 | assigned_dev_setup_cap_read(dev, pos, size); | |
1403 | ||
1404 | type = pci_get_word(pci_dev->config + pos + PCI_EXP_FLAGS); | |
1405 | type = (type & PCI_EXP_FLAGS_TYPE) >> 4; | |
1406 | if (type != PCI_EXP_TYPE_ENDPOINT && | |
1407 | type != PCI_EXP_TYPE_LEG_END && type != PCI_EXP_TYPE_RC_END) { | |
1408 | error_report("Device assignment only supports endpoint assignment," | |
1409 | " device type %d", type); | |
1410 | return -EINVAL; | |
1411 | } | |
1412 | ||
1413 | /* capabilities, pass existing read-only copy | |
1414 | * PCI_EXP_FLAGS_IRQ: updated by hardware, should be direct read */ | |
1415 | ||
1416 | /* device capabilities: hide FLR */ | |
1417 | devcap = pci_get_long(pci_dev->config + pos + PCI_EXP_DEVCAP); | |
1418 | devcap &= ~PCI_EXP_DEVCAP_FLR; | |
1419 | pci_set_long(pci_dev->config + pos + PCI_EXP_DEVCAP, devcap); | |
1420 | ||
1421 | /* device control: clear all error reporting enable bits, leaving | |
1422 | * only a few host values. Note, these are | |
1423 | * all writable, but not passed to hw. | |
1424 | */ | |
1425 | devctl = pci_get_word(pci_dev->config + pos + PCI_EXP_DEVCTL); | |
1426 | devctl = (devctl & (PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_PAYLOAD)) | | |
1427 | PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN; | |
1428 | pci_set_word(pci_dev->config + pos + PCI_EXP_DEVCTL, devctl); | |
1429 | devctl = PCI_EXP_DEVCTL_BCR_FLR | PCI_EXP_DEVCTL_AUX_PME; | |
1430 | pci_set_word(pci_dev->wmask + pos + PCI_EXP_DEVCTL, ~devctl); | |
1431 | ||
1432 | /* Clear device status */ | |
1433 | pci_set_word(pci_dev->config + pos + PCI_EXP_DEVSTA, 0); | |
1434 | ||
1435 | /* Link capabilities, expose links and latencues, clear reporting */ | |
1436 | lnkcap = pci_get_long(pci_dev->config + pos + PCI_EXP_LNKCAP); | |
1437 | lnkcap &= (PCI_EXP_LNKCAP_SLS | PCI_EXP_LNKCAP_MLW | | |
1438 | PCI_EXP_LNKCAP_ASPMS | PCI_EXP_LNKCAP_L0SEL | | |
1439 | PCI_EXP_LNKCAP_L1EL); | |
1440 | pci_set_long(pci_dev->config + pos + PCI_EXP_LNKCAP, lnkcap); | |
1441 | ||
1442 | /* Link control, pass existing read-only copy. Should be writable? */ | |
1443 | ||
1444 | /* Link status, only expose current speed and width */ | |
1445 | lnksta = pci_get_word(pci_dev->config + pos + PCI_EXP_LNKSTA); | |
1446 | lnksta &= (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW); | |
1447 | pci_set_word(pci_dev->config + pos + PCI_EXP_LNKSTA, lnksta); | |
1448 | ||
1449 | if (version >= 2) { | |
1450 | /* Slot capabilities, control, status - not needed for endpoints */ | |
1451 | pci_set_long(pci_dev->config + pos + PCI_EXP_SLTCAP, 0); | |
1452 | pci_set_word(pci_dev->config + pos + PCI_EXP_SLTCTL, 0); | |
1453 | pci_set_word(pci_dev->config + pos + PCI_EXP_SLTSTA, 0); | |
1454 | ||
1455 | /* Root control, capabilities, status - not needed for endpoints */ | |
1456 | pci_set_word(pci_dev->config + pos + PCI_EXP_RTCTL, 0); | |
1457 | pci_set_word(pci_dev->config + pos + PCI_EXP_RTCAP, 0); | |
1458 | pci_set_long(pci_dev->config + pos + PCI_EXP_RTSTA, 0); | |
1459 | ||
1460 | /* Device capabilities/control 2, pass existing read-only copy */ | |
1461 | /* Link control 2, pass existing read-only copy */ | |
1462 | } | |
1463 | } | |
1464 | ||
1465 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PCIX, 0); | |
1466 | if (pos) { | |
1467 | uint16_t cmd; | |
1468 | uint32_t status; | |
1469 | ||
1470 | /* Only expose the minimum, 8 byte capability */ | |
1471 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_PCIX, pos, 8); | |
1472 | if (ret < 0) { | |
1473 | return ret; | |
1474 | } | |
1475 | ||
1476 | assigned_dev_setup_cap_read(dev, pos, 8); | |
1477 | ||
1478 | /* Command register, clear upper bits, including extended modes */ | |
1479 | cmd = pci_get_word(pci_dev->config + pos + PCI_X_CMD); | |
1480 | cmd &= (PCI_X_CMD_DPERR_E | PCI_X_CMD_ERO | PCI_X_CMD_MAX_READ | | |
1481 | PCI_X_CMD_MAX_SPLIT); | |
1482 | pci_set_word(pci_dev->config + pos + PCI_X_CMD, cmd); | |
1483 | ||
1484 | /* Status register, update with emulated PCI bus location, clear | |
1485 | * error bits, leave the rest. */ | |
1486 | status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS); | |
1487 | status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN); | |
1488 | status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn; | |
1489 | status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL | | |
1490 | PCI_X_STATUS_SPL_ERR); | |
1491 | pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status); | |
1492 | } | |
1493 | ||
1494 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VPD, 0); | |
1495 | if (pos) { | |
1496 | /* Direct R/W passthrough */ | |
1497 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_VPD, pos, 8); | |
1498 | if (ret < 0) { | |
1499 | return ret; | |
1500 | } | |
1501 | ||
1502 | assigned_dev_setup_cap_read(dev, pos, 8); | |
1503 | ||
1504 | /* direct write for cap content */ | |
1505 | assigned_dev_direct_config_write(dev, pos + 2, 6); | |
1506 | } | |
1507 | ||
1508 | /* Devices can have multiple vendor capabilities, get them all */ | |
1509 | for (pos = 0; (pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VNDR, pos)); | |
1510 | pos += PCI_CAP_LIST_NEXT) { | |
1511 | uint8_t len = pci_get_byte(pci_dev->config + pos + PCI_CAP_FLAGS); | |
1512 | /* Direct R/W passthrough */ | |
1513 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_VNDR, pos, len); | |
1514 | if (ret < 0) { | |
1515 | return ret; | |
1516 | } | |
1517 | ||
1518 | assigned_dev_setup_cap_read(dev, pos, len); | |
1519 | ||
1520 | /* direct write for cap content */ | |
1521 | assigned_dev_direct_config_write(dev, pos + 2, len - 2); | |
1522 | } | |
1523 | ||
1524 | /* If real and virtual capability list status bits differ, virtualize the | |
1525 | * access. */ | |
1526 | if ((pci_get_word(pci_dev->config + PCI_STATUS) & PCI_STATUS_CAP_LIST) != | |
1527 | (assigned_dev_pci_read_byte(pci_dev, PCI_STATUS) & | |
1528 | PCI_STATUS_CAP_LIST)) { | |
1529 | dev->emulate_config_read[PCI_STATUS] |= PCI_STATUS_CAP_LIST; | |
1530 | } | |
1531 | ||
1532 | return 0; | |
1533 | } | |
1534 | ||
1535 | static uint64_t | |
a8170e5e | 1536 | assigned_dev_msix_mmio_read(void *opaque, hwaddr addr, |
c3ebd3ba JK |
1537 | unsigned size) |
1538 | { | |
1539 | AssignedDevice *adev = opaque; | |
1540 | uint64_t val; | |
1541 | ||
1542 | memcpy(&val, (void *)((uint8_t *)adev->msix_table + addr), size); | |
1543 | ||
1544 | return val; | |
1545 | } | |
1546 | ||
a8170e5e | 1547 | static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr, |
c3ebd3ba JK |
1548 | uint64_t val, unsigned size) |
1549 | { | |
1550 | AssignedDevice *adev = opaque; | |
1551 | PCIDevice *pdev = &adev->dev; | |
1552 | uint16_t ctrl; | |
1553 | MSIXTableEntry orig; | |
1554 | int i = addr >> 4; | |
1555 | ||
1556 | if (i >= adev->msix_max) { | |
1557 | return; /* Drop write */ | |
1558 | } | |
1559 | ||
1560 | ctrl = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS); | |
1561 | ||
1562 | DEBUG("write to MSI-X table offset 0x%lx, val 0x%lx\n", addr, val); | |
1563 | ||
1564 | if (ctrl & PCI_MSIX_FLAGS_ENABLE) { | |
1565 | orig = adev->msix_table[i]; | |
1566 | } | |
1567 | ||
1568 | memcpy((uint8_t *)adev->msix_table + addr, &val, size); | |
1569 | ||
1570 | if (ctrl & PCI_MSIX_FLAGS_ENABLE) { | |
1571 | MSIXTableEntry *entry = &adev->msix_table[i]; | |
1572 | ||
1573 | if (!assigned_dev_msix_masked(&orig) && | |
1574 | assigned_dev_msix_masked(entry)) { | |
1575 | /* | |
1576 | * Vector masked, disable it | |
1577 | * | |
1578 | * XXX It's not clear if we can or should actually attempt | |
1579 | * to mask or disable the interrupt. KVM doesn't have | |
1580 | * support for pending bits and kvm_assign_set_msix_entry | |
1581 | * doesn't modify the device hardware mask. Interrupts | |
1582 | * while masked are simply not injected to the guest, so | |
1583 | * are lost. Can we get away with always injecting an | |
1584 | * interrupt on unmask? | |
1585 | */ | |
1586 | } else if (assigned_dev_msix_masked(&orig) && | |
1587 | !assigned_dev_msix_masked(entry)) { | |
1588 | /* Vector unmasked */ | |
1589 | if (i >= adev->msi_virq_nr || adev->msi_virq[i] < 0) { | |
1590 | /* Previously unassigned vector, start from scratch */ | |
1591 | assigned_dev_update_msix(pdev); | |
1592 | return; | |
1593 | } else { | |
1594 | /* Update an existing, previously masked vector */ | |
1595 | MSIMessage msg; | |
1596 | int ret; | |
1597 | ||
1598 | msg.address = entry->addr_lo | | |
1599 | ((uint64_t)entry->addr_hi << 32); | |
1600 | msg.data = entry->data; | |
1601 | ||
1602 | ret = kvm_irqchip_update_msi_route(kvm_state, | |
1603 | adev->msi_virq[i], msg); | |
1604 | if (ret) { | |
1605 | error_report("Error updating irq routing entry (%d)", ret); | |
1606 | } | |
1607 | } | |
1608 | } | |
1609 | } | |
1610 | } | |
1611 | ||
1612 | static const MemoryRegionOps assigned_dev_msix_mmio_ops = { | |
1613 | .read = assigned_dev_msix_mmio_read, | |
1614 | .write = assigned_dev_msix_mmio_write, | |
1615 | .endianness = DEVICE_NATIVE_ENDIAN, | |
1616 | .valid = { | |
1617 | .min_access_size = 4, | |
1618 | .max_access_size = 8, | |
1619 | }, | |
1620 | .impl = { | |
1621 | .min_access_size = 4, | |
1622 | .max_access_size = 8, | |
1623 | }, | |
1624 | }; | |
1625 | ||
1626 | static void assigned_dev_msix_reset(AssignedDevice *dev) | |
1627 | { | |
1628 | MSIXTableEntry *entry; | |
1629 | int i; | |
1630 | ||
1631 | if (!dev->msix_table) { | |
1632 | return; | |
1633 | } | |
1634 | ||
1635 | memset(dev->msix_table, 0, MSIX_PAGE_SIZE); | |
1636 | ||
1637 | for (i = 0, entry = dev->msix_table; i < dev->msix_max; i++, entry++) { | |
1638 | entry->ctrl = cpu_to_le32(0x1); /* Masked */ | |
1639 | } | |
1640 | } | |
1641 | ||
1642 | static int assigned_dev_register_msix_mmio(AssignedDevice *dev) | |
1643 | { | |
1644 | dev->msix_table = mmap(NULL, MSIX_PAGE_SIZE, PROT_READ|PROT_WRITE, | |
1645 | MAP_ANONYMOUS|MAP_PRIVATE, 0, 0); | |
1646 | if (dev->msix_table == MAP_FAILED) { | |
1647 | error_report("fail allocate msix_table! %s", strerror(errno)); | |
1648 | return -EFAULT; | |
1649 | } | |
1650 | ||
1651 | assigned_dev_msix_reset(dev); | |
1652 | ||
2c9b15ca | 1653 | memory_region_init_io(&dev->mmio, NULL, &assigned_dev_msix_mmio_ops, dev, |
c3ebd3ba JK |
1654 | "assigned-dev-msix", MSIX_PAGE_SIZE); |
1655 | return 0; | |
1656 | } | |
1657 | ||
1658 | static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev) | |
1659 | { | |
1660 | if (!dev->msix_table) { | |
1661 | return; | |
1662 | } | |
1663 | ||
1664 | memory_region_destroy(&dev->mmio); | |
1665 | ||
1666 | if (munmap(dev->msix_table, MSIX_PAGE_SIZE) == -1) { | |
1667 | error_report("error unmapping msix_table! %s", strerror(errno)); | |
1668 | } | |
1669 | dev->msix_table = NULL; | |
1670 | } | |
1671 | ||
1672 | static const VMStateDescription vmstate_assigned_device = { | |
1673 | .name = "pci-assign", | |
1674 | .unmigratable = 1, | |
1675 | }; | |
1676 | ||
1677 | static void reset_assigned_device(DeviceState *dev) | |
1678 | { | |
1679 | PCIDevice *pci_dev = DO_UPCAST(PCIDevice, qdev, dev); | |
1680 | AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1681 | char reset_file[64]; | |
1682 | const char reset[] = "1"; | |
1683 | int fd, ret; | |
1684 | ||
1685 | /* | |
1686 | * If a guest is reset without being shutdown, MSI/MSI-X can still | |
1687 | * be running. We want to return the device to a known state on | |
1688 | * reset, so disable those here. We especially do not want MSI-X | |
1689 | * enabled since it lives in MMIO space, which is about to get | |
1690 | * disabled. | |
1691 | */ | |
1692 | if (adev->assigned_irq_type == ASSIGNED_IRQ_MSIX) { | |
1693 | uint16_t ctrl = pci_get_word(pci_dev->config + | |
1694 | pci_dev->msix_cap + PCI_MSIX_FLAGS); | |
1695 | ||
1696 | pci_set_word(pci_dev->config + pci_dev->msix_cap + PCI_MSIX_FLAGS, | |
1697 | ctrl & ~PCI_MSIX_FLAGS_ENABLE); | |
1698 | assigned_dev_update_msix(pci_dev); | |
1699 | } else if (adev->assigned_irq_type == ASSIGNED_IRQ_MSI) { | |
1700 | uint8_t ctrl = pci_get_byte(pci_dev->config + | |
1701 | pci_dev->msi_cap + PCI_MSI_FLAGS); | |
1702 | ||
1703 | pci_set_byte(pci_dev->config + pci_dev->msi_cap + PCI_MSI_FLAGS, | |
1704 | ctrl & ~PCI_MSI_FLAGS_ENABLE); | |
1705 | assigned_dev_update_msi(pci_dev); | |
1706 | } | |
1707 | ||
1708 | snprintf(reset_file, sizeof(reset_file), | |
1709 | "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/reset", | |
1710 | adev->host.domain, adev->host.bus, adev->host.slot, | |
1711 | adev->host.function); | |
1712 | ||
1713 | /* | |
1714 | * Issue a device reset via pci-sysfs. Note that we use write(2) here | |
1715 | * and ignore the return value because some kernels have a bug that | |
1716 | * returns 0 rather than bytes written on success, sending us into an | |
1717 | * infinite retry loop using other write mechanisms. | |
1718 | */ | |
1719 | fd = open(reset_file, O_WRONLY); | |
1720 | if (fd != -1) { | |
1721 | ret = write(fd, reset, strlen(reset)); | |
1722 | (void)ret; | |
1723 | close(fd); | |
1724 | } | |
1725 | ||
1726 | /* | |
1727 | * When a 0 is written to the bus master register, the device is logically | |
1728 | * disconnected from the PCI bus. This avoids further DMA transfers. | |
1729 | */ | |
1730 | assigned_dev_pci_write_config(pci_dev, PCI_COMMAND, 0, 1); | |
1731 | } | |
1732 | ||
1733 | static int assigned_initfn(struct PCIDevice *pci_dev) | |
1734 | { | |
1735 | AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1736 | uint8_t e_intx; | |
1737 | int r; | |
1738 | ||
1739 | if (!kvm_enabled()) { | |
1740 | error_report("pci-assign: error: requires KVM support"); | |
1741 | return -1; | |
1742 | } | |
1743 | ||
1744 | if (!dev->host.domain && !dev->host.bus && !dev->host.slot && | |
1745 | !dev->host.function) { | |
1746 | error_report("pci-assign: error: no host device specified"); | |
1747 | return -1; | |
1748 | } | |
1749 | ||
1750 | /* | |
1751 | * Set up basic config space access control. Will be further refined during | |
1752 | * device initialization. | |
1753 | */ | |
1754 | assigned_dev_emulate_config_read(dev, 0, PCI_CONFIG_SPACE_SIZE); | |
1755 | assigned_dev_direct_config_read(dev, PCI_STATUS, 2); | |
1756 | assigned_dev_direct_config_read(dev, PCI_REVISION_ID, 1); | |
1757 | assigned_dev_direct_config_read(dev, PCI_CLASS_PROG, 3); | |
1758 | assigned_dev_direct_config_read(dev, PCI_CACHE_LINE_SIZE, 1); | |
1759 | assigned_dev_direct_config_read(dev, PCI_LATENCY_TIMER, 1); | |
1760 | assigned_dev_direct_config_read(dev, PCI_BIST, 1); | |
1761 | assigned_dev_direct_config_read(dev, PCI_CARDBUS_CIS, 4); | |
1762 | assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_VENDOR_ID, 2); | |
1763 | assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_ID, 2); | |
1764 | assigned_dev_direct_config_read(dev, PCI_CAPABILITY_LIST + 1, 7); | |
1765 | assigned_dev_direct_config_read(dev, PCI_MIN_GNT, 1); | |
1766 | assigned_dev_direct_config_read(dev, PCI_MAX_LAT, 1); | |
1767 | memcpy(dev->emulate_config_write, dev->emulate_config_read, | |
1768 | sizeof(dev->emulate_config_read)); | |
1769 | ||
1770 | if (get_real_device(dev, dev->host.domain, dev->host.bus, | |
1771 | dev->host.slot, dev->host.function)) { | |
1772 | error_report("pci-assign: Error: Couldn't get real device (%s)!", | |
1773 | dev->dev.qdev.id); | |
1774 | goto out; | |
1775 | } | |
1776 | ||
1777 | if (assigned_device_pci_cap_init(pci_dev) < 0) { | |
1778 | goto out; | |
1779 | } | |
1780 | ||
1781 | /* intercept MSI-X entry page in the MMIO */ | |
1782 | if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) { | |
1783 | if (assigned_dev_register_msix_mmio(dev)) { | |
1784 | goto out; | |
1785 | } | |
1786 | } | |
1787 | ||
1788 | /* handle real device's MMIO/PIO BARs */ | |
1789 | if (assigned_dev_register_regions(dev->real_device.regions, | |
1790 | dev->real_device.region_number, | |
1791 | dev)) { | |
1792 | goto out; | |
1793 | } | |
1794 | ||
1795 | /* handle interrupt routing */ | |
1796 | e_intx = dev->dev.config[PCI_INTERRUPT_PIN] - 1; | |
1797 | dev->intpin = e_intx; | |
1798 | dev->intx_route.mode = PCI_INTX_DISABLED; | |
1799 | dev->intx_route.irq = -1; | |
1800 | ||
1801 | /* assign device to guest */ | |
1802 | r = assign_device(dev); | |
1803 | if (r < 0) { | |
1804 | goto out; | |
1805 | } | |
1806 | ||
1807 | /* assign legacy INTx to the device */ | |
1808 | r = assign_intx(dev); | |
1809 | if (r < 0) { | |
1810 | goto assigned_out; | |
1811 | } | |
1812 | ||
1813 | assigned_dev_load_option_rom(dev); | |
1814 | ||
1815 | add_boot_device_path(dev->bootindex, &pci_dev->qdev, NULL); | |
1816 | ||
1817 | return 0; | |
1818 | ||
1819 | assigned_out: | |
1820 | deassign_device(dev); | |
1821 | out: | |
1822 | free_assigned_device(dev); | |
1823 | return -1; | |
1824 | } | |
1825 | ||
1826 | static void assigned_exitfn(struct PCIDevice *pci_dev) | |
1827 | { | |
1828 | AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1829 | ||
1830 | deassign_device(dev); | |
1831 | free_assigned_device(dev); | |
1832 | } | |
1833 | ||
1834 | static Property assigned_dev_properties[] = { | |
1835 | DEFINE_PROP_PCI_HOST_DEVADDR("host", AssignedDevice, host), | |
1836 | DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features, | |
1837 | ASSIGNED_DEVICE_PREFER_MSI_BIT, false), | |
1838 | DEFINE_PROP_BIT("share_intx", AssignedDevice, features, | |
1839 | ASSIGNED_DEVICE_SHARE_INTX_BIT, true), | |
1840 | DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1), | |
1841 | DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name), | |
1842 | DEFINE_PROP_END_OF_LIST(), | |
1843 | }; | |
1844 | ||
1845 | static void assign_class_init(ObjectClass *klass, void *data) | |
1846 | { | |
1847 | PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); | |
1848 | DeviceClass *dc = DEVICE_CLASS(klass); | |
1849 | ||
1850 | k->init = assigned_initfn; | |
1851 | k->exit = assigned_exitfn; | |
1852 | k->config_read = assigned_dev_pci_read_config; | |
1853 | k->config_write = assigned_dev_pci_write_config; | |
1854 | dc->props = assigned_dev_properties; | |
1855 | dc->vmsd = &vmstate_assigned_device; | |
1856 | dc->reset = reset_assigned_device; | |
1857 | dc->desc = "KVM-based PCI passthrough"; | |
1858 | } | |
1859 | ||
1860 | static const TypeInfo assign_info = { | |
1861 | .name = "kvm-pci-assign", | |
1862 | .parent = TYPE_PCI_DEVICE, | |
1863 | .instance_size = sizeof(AssignedDevice), | |
1864 | .class_init = assign_class_init, | |
1865 | }; | |
1866 | ||
1867 | static void assign_register_types(void) | |
1868 | { | |
1869 | type_register_static(&assign_info); | |
1870 | } | |
1871 | ||
1872 | type_init(assign_register_types) | |
1873 | ||
1874 | /* | |
1875 | * Scan the assigned devices for the devices that have an option ROM, and then | |
1876 | * load the corresponding ROM data to RAM. If an error occurs while loading an | |
1877 | * option ROM, we just ignore that option ROM and continue with the next one. | |
1878 | */ | |
1879 | static void assigned_dev_load_option_rom(AssignedDevice *dev) | |
1880 | { | |
1881 | char name[32], rom_file[64]; | |
1882 | FILE *fp; | |
1883 | uint8_t val; | |
1884 | struct stat st; | |
1885 | void *ptr; | |
1886 | ||
1887 | /* If loading ROM from file, pci handles it */ | |
1888 | if (dev->dev.romfile || !dev->dev.rom_bar) { | |
1889 | return; | |
1890 | } | |
1891 | ||
1892 | snprintf(rom_file, sizeof(rom_file), | |
1893 | "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom", | |
1894 | dev->host.domain, dev->host.bus, dev->host.slot, | |
1895 | dev->host.function); | |
1896 | ||
1897 | if (stat(rom_file, &st)) { | |
1898 | return; | |
1899 | } | |
1900 | ||
1901 | if (access(rom_file, F_OK)) { | |
1902 | error_report("pci-assign: Insufficient privileges for %s", rom_file); | |
1903 | return; | |
1904 | } | |
1905 | ||
1906 | /* Write "1" to the ROM file to enable it */ | |
1907 | fp = fopen(rom_file, "r+"); | |
1908 | if (fp == NULL) { | |
1909 | return; | |
1910 | } | |
1911 | val = 1; | |
1912 | if (fwrite(&val, 1, 1, fp) != 1) { | |
1913 | goto close_rom; | |
1914 | } | |
1915 | fseek(fp, 0, SEEK_SET); | |
1916 | ||
1917 | snprintf(name, sizeof(name), "%s.rom", | |
1918 | object_get_typename(OBJECT(dev))); | |
2c9b15ca | 1919 | memory_region_init_ram(&dev->dev.rom, NULL, name, st.st_size); |
c3ebd3ba JK |
1920 | vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev); |
1921 | ptr = memory_region_get_ram_ptr(&dev->dev.rom); | |
1922 | memset(ptr, 0xff, st.st_size); | |
1923 | ||
1924 | if (!fread(ptr, 1, st.st_size, fp)) { | |
474c2134 MA |
1925 | error_report("pci-assign: Cannot read from host %s", rom_file); |
1926 | error_printf("Device option ROM contents are probably invalid " | |
1927 | "(check dmesg).\nSkip option ROM probe with rombar=0, " | |
1928 | "or load from file with romfile=\n"); | |
c3ebd3ba JK |
1929 | memory_region_destroy(&dev->dev.rom); |
1930 | goto close_rom; | |
1931 | } | |
1932 | ||
1933 | pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom); | |
1934 | dev->dev.has_rom = true; | |
1935 | close_rom: | |
1936 | /* Write "0" to disable ROM */ | |
1937 | fseek(fp, 0, SEEK_SET); | |
1938 | val = 0; | |
1939 | if (!fwrite(&val, 1, 1, fp)) { | |
1940 | DEBUG("%s\n", "Failed to disable pci-sysfs rom file"); | |
1941 | } | |
1942 | fclose(fp); | |
1943 | } |