]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (c) 2007, Neocleus Corporation. | |
3 | * | |
4 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
5 | * the COPYING file in the top-level directory. | |
6 | * | |
7 | * | |
8 | * Assign a PCI device from the host to a guest VM. | |
9 | * | |
10 | * This implementation uses the classic device assignment interface of KVM | |
11 | * and is only available on x86 hosts. It is expected to be obsoleted by VFIO | |
12 | * based device assignment. | |
13 | * | |
14 | * Adapted for KVM (qemu-kvm) by Qumranet. QEMU version was based on qemu-kvm | |
15 | * revision 4144fe9d48. See its repository for the history. | |
16 | * | |
17 | * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) | |
18 | * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) | |
19 | * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) | |
20 | * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) | |
21 | * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) | |
22 | */ | |
23 | #include <stdio.h> | |
24 | #include <unistd.h> | |
25 | #include <sys/io.h> | |
26 | #include <sys/mman.h> | |
27 | #include <sys/types.h> | |
28 | #include <sys/stat.h> | |
29 | #include "hw/hw.h" | |
30 | #include "hw/i386/pc.h" | |
31 | #include "qemu/error-report.h" | |
32 | #include "ui/console.h" | |
33 | #include "hw/loader.h" | |
34 | #include "monitor/monitor.h" | |
35 | #include "qemu/range.h" | |
36 | #include "sysemu/sysemu.h" | |
37 | #include "hw/pci/pci.h" | |
38 | #include "hw/pci/msi.h" | |
39 | #include "kvm_i386.h" | |
40 | ||
41 | #define MSIX_PAGE_SIZE 0x1000 | |
42 | ||
43 | /* From linux/ioport.h */ | |
44 | #define IORESOURCE_IO 0x00000100 /* Resource type */ | |
45 | #define IORESOURCE_MEM 0x00000200 | |
46 | #define IORESOURCE_IRQ 0x00000400 | |
47 | #define IORESOURCE_DMA 0x00000800 | |
48 | #define IORESOURCE_PREFETCH 0x00002000 /* No side effects */ | |
49 | #define IORESOURCE_MEM_64 0x00100000 | |
50 | ||
51 | //#define DEVICE_ASSIGNMENT_DEBUG | |
52 | ||
53 | #ifdef DEVICE_ASSIGNMENT_DEBUG | |
54 | #define DEBUG(fmt, ...) \ | |
55 | do { \ | |
56 | fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__); \ | |
57 | } while (0) | |
58 | #else | |
59 | #define DEBUG(fmt, ...) | |
60 | #endif | |
61 | ||
62 | typedef struct PCIRegion { | |
63 | int type; /* Memory or port I/O */ | |
64 | int valid; | |
65 | uint64_t base_addr; | |
66 | uint64_t size; /* size of the region */ | |
67 | int resource_fd; | |
68 | } PCIRegion; | |
69 | ||
70 | typedef struct PCIDevRegions { | |
71 | uint8_t bus, dev, func; /* Bus inside domain, device and function */ | |
72 | int irq; /* IRQ number */ | |
73 | uint16_t region_number; /* number of active regions */ | |
74 | ||
75 | /* Port I/O or MMIO Regions */ | |
76 | PCIRegion regions[PCI_NUM_REGIONS - 1]; | |
77 | int config_fd; | |
78 | } PCIDevRegions; | |
79 | ||
80 | typedef struct AssignedDevRegion { | |
81 | MemoryRegion container; | |
82 | MemoryRegion real_iomem; | |
83 | union { | |
84 | uint8_t *r_virtbase; /* mmapped access address for memory regions */ | |
85 | uint32_t r_baseport; /* the base guest port for I/O regions */ | |
86 | } u; | |
87 | pcibus_t e_size; /* emulated size of region in bytes */ | |
88 | pcibus_t r_size; /* real size of region in bytes */ | |
89 | PCIRegion *region; | |
90 | } AssignedDevRegion; | |
91 | ||
92 | #define ASSIGNED_DEVICE_PREFER_MSI_BIT 0 | |
93 | #define ASSIGNED_DEVICE_SHARE_INTX_BIT 1 | |
94 | ||
95 | #define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT) | |
96 | #define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT) | |
97 | ||
98 | typedef struct MSIXTableEntry { | |
99 | uint32_t addr_lo; | |
100 | uint32_t addr_hi; | |
101 | uint32_t data; | |
102 | uint32_t ctrl; | |
103 | } MSIXTableEntry; | |
104 | ||
105 | typedef enum AssignedIRQType { | |
106 | ASSIGNED_IRQ_NONE = 0, | |
107 | ASSIGNED_IRQ_INTX_HOST_INTX, | |
108 | ASSIGNED_IRQ_INTX_HOST_MSI, | |
109 | ASSIGNED_IRQ_MSI, | |
110 | ASSIGNED_IRQ_MSIX | |
111 | } AssignedIRQType; | |
112 | ||
113 | typedef struct AssignedDevice { | |
114 | PCIDevice dev; | |
115 | PCIHostDeviceAddress host; | |
116 | uint32_t dev_id; | |
117 | uint32_t features; | |
118 | int intpin; | |
119 | AssignedDevRegion v_addrs[PCI_NUM_REGIONS - 1]; | |
120 | PCIDevRegions real_device; | |
121 | PCIINTxRoute intx_route; | |
122 | AssignedIRQType assigned_irq_type; | |
123 | struct { | |
124 | #define ASSIGNED_DEVICE_CAP_MSI (1 << 0) | |
125 | #define ASSIGNED_DEVICE_CAP_MSIX (1 << 1) | |
126 | uint32_t available; | |
127 | #define ASSIGNED_DEVICE_MSI_ENABLED (1 << 0) | |
128 | #define ASSIGNED_DEVICE_MSIX_ENABLED (1 << 1) | |
129 | #define ASSIGNED_DEVICE_MSIX_MASKED (1 << 2) | |
130 | uint32_t state; | |
131 | } cap; | |
132 | uint8_t emulate_config_read[PCI_CONFIG_SPACE_SIZE]; | |
133 | uint8_t emulate_config_write[PCI_CONFIG_SPACE_SIZE]; | |
134 | int msi_virq_nr; | |
135 | int *msi_virq; | |
136 | MSIXTableEntry *msix_table; | |
137 | hwaddr msix_table_addr; | |
138 | uint16_t msix_max; | |
139 | MemoryRegion mmio; | |
140 | char *configfd_name; | |
141 | int32_t bootindex; | |
142 | } AssignedDevice; | |
143 | ||
144 | static void assigned_dev_update_irq_routing(PCIDevice *dev); | |
145 | ||
146 | static void assigned_dev_load_option_rom(AssignedDevice *dev); | |
147 | ||
148 | static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev); | |
149 | ||
150 | static uint64_t assigned_dev_ioport_rw(AssignedDevRegion *dev_region, | |
151 | hwaddr addr, int size, | |
152 | uint64_t *data) | |
153 | { | |
154 | uint64_t val = 0; | |
155 | int fd = dev_region->region->resource_fd; | |
156 | ||
157 | if (fd >= 0) { | |
158 | if (data) { | |
159 | DEBUG("pwrite data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
160 | ", addr="TARGET_FMT_plx"\n", *data, size, addr, addr); | |
161 | if (pwrite(fd, data, size, addr) != size) { | |
162 | error_report("%s - pwrite failed %s", | |
163 | __func__, strerror(errno)); | |
164 | } | |
165 | } else { | |
166 | if (pread(fd, &val, size, addr) != size) { | |
167 | error_report("%s - pread failed %s", | |
168 | __func__, strerror(errno)); | |
169 | val = (1UL << (size * 8)) - 1; | |
170 | } | |
171 | DEBUG("pread val=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
172 | ", addr=" TARGET_FMT_plx "\n", val, size, addr, addr); | |
173 | } | |
174 | } else { | |
175 | uint32_t port = addr + dev_region->u.r_baseport; | |
176 | ||
177 | if (data) { | |
178 | DEBUG("out data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
179 | ", host=%x\n", *data, size, addr, port); | |
180 | switch (size) { | |
181 | case 1: | |
182 | outb(*data, port); | |
183 | break; | |
184 | case 2: | |
185 | outw(*data, port); | |
186 | break; | |
187 | case 4: | |
188 | outl(*data, port); | |
189 | break; | |
190 | } | |
191 | } else { | |
192 | switch (size) { | |
193 | case 1: | |
194 | val = inb(port); | |
195 | break; | |
196 | case 2: | |
197 | val = inw(port); | |
198 | break; | |
199 | case 4: | |
200 | val = inl(port); | |
201 | break; | |
202 | } | |
203 | DEBUG("in data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx | |
204 | ", host=%x\n", val, size, addr, port); | |
205 | } | |
206 | } | |
207 | return val; | |
208 | } | |
209 | ||
210 | static void assigned_dev_ioport_write(void *opaque, hwaddr addr, | |
211 | uint64_t data, unsigned size) | |
212 | { | |
213 | assigned_dev_ioport_rw(opaque, addr, size, &data); | |
214 | } | |
215 | ||
216 | static uint64_t assigned_dev_ioport_read(void *opaque, | |
217 | hwaddr addr, unsigned size) | |
218 | { | |
219 | return assigned_dev_ioport_rw(opaque, addr, size, NULL); | |
220 | } | |
221 | ||
222 | static uint32_t slow_bar_readb(void *opaque, hwaddr addr) | |
223 | { | |
224 | AssignedDevRegion *d = opaque; | |
225 | uint8_t *in = d->u.r_virtbase + addr; | |
226 | uint32_t r; | |
227 | ||
228 | r = *in; | |
229 | DEBUG("addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); | |
230 | ||
231 | return r; | |
232 | } | |
233 | ||
234 | static uint32_t slow_bar_readw(void *opaque, hwaddr addr) | |
235 | { | |
236 | AssignedDevRegion *d = opaque; | |
237 | uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr); | |
238 | uint32_t r; | |
239 | ||
240 | r = *in; | |
241 | DEBUG("addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); | |
242 | ||
243 | return r; | |
244 | } | |
245 | ||
246 | static uint32_t slow_bar_readl(void *opaque, hwaddr addr) | |
247 | { | |
248 | AssignedDevRegion *d = opaque; | |
249 | uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr); | |
250 | uint32_t r; | |
251 | ||
252 | r = *in; | |
253 | DEBUG("addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r); | |
254 | ||
255 | return r; | |
256 | } | |
257 | ||
258 | static void slow_bar_writeb(void *opaque, hwaddr addr, uint32_t val) | |
259 | { | |
260 | AssignedDevRegion *d = opaque; | |
261 | uint8_t *out = d->u.r_virtbase + addr; | |
262 | ||
263 | DEBUG("addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr, val); | |
264 | *out = val; | |
265 | } | |
266 | ||
267 | static void slow_bar_writew(void *opaque, hwaddr addr, uint32_t val) | |
268 | { | |
269 | AssignedDevRegion *d = opaque; | |
270 | uint16_t *out = (uint16_t *)(d->u.r_virtbase + addr); | |
271 | ||
272 | DEBUG("addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr, val); | |
273 | *out = val; | |
274 | } | |
275 | ||
276 | static void slow_bar_writel(void *opaque, hwaddr addr, uint32_t val) | |
277 | { | |
278 | AssignedDevRegion *d = opaque; | |
279 | uint32_t *out = (uint32_t *)(d->u.r_virtbase + addr); | |
280 | ||
281 | DEBUG("addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, val); | |
282 | *out = val; | |
283 | } | |
284 | ||
285 | static const MemoryRegionOps slow_bar_ops = { | |
286 | .old_mmio = { | |
287 | .read = { slow_bar_readb, slow_bar_readw, slow_bar_readl, }, | |
288 | .write = { slow_bar_writeb, slow_bar_writew, slow_bar_writel, }, | |
289 | }, | |
290 | .endianness = DEVICE_NATIVE_ENDIAN, | |
291 | }; | |
292 | ||
293 | static void assigned_dev_iomem_setup(PCIDevice *pci_dev, int region_num, | |
294 | pcibus_t e_size) | |
295 | { | |
296 | AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
297 | AssignedDevRegion *region = &r_dev->v_addrs[region_num]; | |
298 | PCIRegion *real_region = &r_dev->real_device.regions[region_num]; | |
299 | ||
300 | if (e_size > 0) { | |
301 | memory_region_init(®ion->container, OBJECT(pci_dev), | |
302 | "assigned-dev-container", e_size); | |
303 | memory_region_add_subregion(®ion->container, 0, ®ion->real_iomem); | |
304 | ||
305 | /* deal with MSI-X MMIO page */ | |
306 | if (real_region->base_addr <= r_dev->msix_table_addr && | |
307 | real_region->base_addr + real_region->size > | |
308 | r_dev->msix_table_addr) { | |
309 | uint64_t offset = r_dev->msix_table_addr - real_region->base_addr; | |
310 | ||
311 | memory_region_add_subregion_overlap(®ion->container, | |
312 | offset, | |
313 | &r_dev->mmio, | |
314 | 1); | |
315 | } | |
316 | } | |
317 | } | |
318 | ||
319 | static const MemoryRegionOps assigned_dev_ioport_ops = { | |
320 | .read = assigned_dev_ioport_read, | |
321 | .write = assigned_dev_ioport_write, | |
322 | .endianness = DEVICE_NATIVE_ENDIAN, | |
323 | }; | |
324 | ||
325 | static void assigned_dev_ioport_setup(PCIDevice *pci_dev, int region_num, | |
326 | pcibus_t size) | |
327 | { | |
328 | AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
329 | AssignedDevRegion *region = &r_dev->v_addrs[region_num]; | |
330 | ||
331 | region->e_size = size; | |
332 | memory_region_init(®ion->container, OBJECT(pci_dev), | |
333 | "assigned-dev-container", size); | |
334 | memory_region_init_io(®ion->real_iomem, OBJECT(pci_dev), | |
335 | &assigned_dev_ioport_ops, r_dev->v_addrs + region_num, | |
336 | "assigned-dev-iomem", size); | |
337 | memory_region_add_subregion(®ion->container, 0, ®ion->real_iomem); | |
338 | } | |
339 | ||
340 | static uint32_t assigned_dev_pci_read(PCIDevice *d, int pos, int len) | |
341 | { | |
342 | AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d); | |
343 | uint32_t val; | |
344 | ssize_t ret; | |
345 | int fd = pci_dev->real_device.config_fd; | |
346 | ||
347 | again: | |
348 | ret = pread(fd, &val, len, pos); | |
349 | if (ret != len) { | |
350 | if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) { | |
351 | goto again; | |
352 | } | |
353 | ||
354 | hw_error("pci read failed, ret = %zd errno = %d\n", ret, errno); | |
355 | } | |
356 | ||
357 | return val; | |
358 | } | |
359 | ||
360 | static uint8_t assigned_dev_pci_read_byte(PCIDevice *d, int pos) | |
361 | { | |
362 | return (uint8_t)assigned_dev_pci_read(d, pos, 1); | |
363 | } | |
364 | ||
365 | static void assigned_dev_pci_write(PCIDevice *d, int pos, uint32_t val, int len) | |
366 | { | |
367 | AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d); | |
368 | ssize_t ret; | |
369 | int fd = pci_dev->real_device.config_fd; | |
370 | ||
371 | again: | |
372 | ret = pwrite(fd, &val, len, pos); | |
373 | if (ret != len) { | |
374 | if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) { | |
375 | goto again; | |
376 | } | |
377 | ||
378 | hw_error("pci write failed, ret = %zd errno = %d\n", ret, errno); | |
379 | } | |
380 | } | |
381 | ||
382 | static void assigned_dev_emulate_config_read(AssignedDevice *dev, | |
383 | uint32_t offset, uint32_t len) | |
384 | { | |
385 | memset(dev->emulate_config_read + offset, 0xff, len); | |
386 | } | |
387 | ||
388 | static void assigned_dev_direct_config_read(AssignedDevice *dev, | |
389 | uint32_t offset, uint32_t len) | |
390 | { | |
391 | memset(dev->emulate_config_read + offset, 0, len); | |
392 | } | |
393 | ||
394 | static void assigned_dev_direct_config_write(AssignedDevice *dev, | |
395 | uint32_t offset, uint32_t len) | |
396 | { | |
397 | memset(dev->emulate_config_write + offset, 0, len); | |
398 | } | |
399 | ||
400 | static uint8_t pci_find_cap_offset(PCIDevice *d, uint8_t cap, uint8_t start) | |
401 | { | |
402 | int id; | |
403 | int max_cap = 48; | |
404 | int pos = start ? start : PCI_CAPABILITY_LIST; | |
405 | int status; | |
406 | ||
407 | status = assigned_dev_pci_read_byte(d, PCI_STATUS); | |
408 | if ((status & PCI_STATUS_CAP_LIST) == 0) { | |
409 | return 0; | |
410 | } | |
411 | ||
412 | while (max_cap--) { | |
413 | pos = assigned_dev_pci_read_byte(d, pos); | |
414 | if (pos < 0x40) { | |
415 | break; | |
416 | } | |
417 | ||
418 | pos &= ~3; | |
419 | id = assigned_dev_pci_read_byte(d, pos + PCI_CAP_LIST_ID); | |
420 | ||
421 | if (id == 0xff) { | |
422 | break; | |
423 | } | |
424 | if (id == cap) { | |
425 | return pos; | |
426 | } | |
427 | ||
428 | pos += PCI_CAP_LIST_NEXT; | |
429 | } | |
430 | return 0; | |
431 | } | |
432 | ||
433 | static int assigned_dev_register_regions(PCIRegion *io_regions, | |
434 | unsigned long regions_num, | |
435 | AssignedDevice *pci_dev) | |
436 | { | |
437 | uint32_t i; | |
438 | PCIRegion *cur_region = io_regions; | |
439 | ||
440 | for (i = 0; i < regions_num; i++, cur_region++) { | |
441 | if (!cur_region->valid) { | |
442 | continue; | |
443 | } | |
444 | ||
445 | /* handle memory io regions */ | |
446 | if (cur_region->type & IORESOURCE_MEM) { | |
447 | int t = PCI_BASE_ADDRESS_SPACE_MEMORY; | |
448 | if (cur_region->type & IORESOURCE_PREFETCH) { | |
449 | t |= PCI_BASE_ADDRESS_MEM_PREFETCH; | |
450 | } | |
451 | if (cur_region->type & IORESOURCE_MEM_64) { | |
452 | t |= PCI_BASE_ADDRESS_MEM_TYPE_64; | |
453 | } | |
454 | ||
455 | /* map physical memory */ | |
456 | pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL, cur_region->size, | |
457 | PROT_WRITE | PROT_READ, | |
458 | MAP_SHARED, | |
459 | cur_region->resource_fd, | |
460 | (off_t)0); | |
461 | ||
462 | if (pci_dev->v_addrs[i].u.r_virtbase == MAP_FAILED) { | |
463 | pci_dev->v_addrs[i].u.r_virtbase = NULL; | |
464 | error_report("%s: Error: Couldn't mmap 0x%" PRIx64 "!", | |
465 | __func__, cur_region->base_addr); | |
466 | return -1; | |
467 | } | |
468 | ||
469 | pci_dev->v_addrs[i].r_size = cur_region->size; | |
470 | pci_dev->v_addrs[i].e_size = 0; | |
471 | ||
472 | /* add offset */ | |
473 | pci_dev->v_addrs[i].u.r_virtbase += | |
474 | (cur_region->base_addr & 0xFFF); | |
475 | ||
476 | if (cur_region->size & 0xFFF) { | |
477 | error_report("PCI region %d at address 0x%" PRIx64 " has " | |
478 | "size 0x%" PRIx64 ", which is not a multiple of " | |
479 | "4K. You might experience some performance hit " | |
480 | "due to that.", | |
481 | i, cur_region->base_addr, cur_region->size); | |
482 | memory_region_init_io(&pci_dev->v_addrs[i].real_iomem, | |
483 | OBJECT(pci_dev), &slow_bar_ops, | |
484 | &pci_dev->v_addrs[i], | |
485 | "assigned-dev-slow-bar", | |
486 | cur_region->size); | |
487 | } else { | |
488 | void *virtbase = pci_dev->v_addrs[i].u.r_virtbase; | |
489 | char name[32]; | |
490 | snprintf(name, sizeof(name), "%s.bar%d", | |
491 | object_get_typename(OBJECT(pci_dev)), i); | |
492 | memory_region_init_ram_ptr(&pci_dev->v_addrs[i].real_iomem, | |
493 | OBJECT(pci_dev), name, | |
494 | cur_region->size, virtbase); | |
495 | vmstate_register_ram(&pci_dev->v_addrs[i].real_iomem, | |
496 | &pci_dev->dev.qdev); | |
497 | } | |
498 | ||
499 | assigned_dev_iomem_setup(&pci_dev->dev, i, cur_region->size); | |
500 | pci_register_bar((PCIDevice *) pci_dev, i, t, | |
501 | &pci_dev->v_addrs[i].container); | |
502 | continue; | |
503 | } else { | |
504 | /* handle port io regions */ | |
505 | uint32_t val; | |
506 | int ret; | |
507 | ||
508 | /* Test kernel support for ioport resource read/write. Old | |
509 | * kernels return EIO. New kernels only allow 1/2/4 byte reads | |
510 | * so should return EINVAL for a 3 byte read */ | |
511 | ret = pread(pci_dev->v_addrs[i].region->resource_fd, &val, 3, 0); | |
512 | if (ret >= 0) { | |
513 | error_report("Unexpected return from I/O port read: %d", ret); | |
514 | abort(); | |
515 | } else if (errno != EINVAL) { | |
516 | error_report("Kernel doesn't support ioport resource " | |
517 | "access, hiding this region."); | |
518 | close(pci_dev->v_addrs[i].region->resource_fd); | |
519 | cur_region->valid = 0; | |
520 | continue; | |
521 | } | |
522 | ||
523 | pci_dev->v_addrs[i].u.r_baseport = cur_region->base_addr; | |
524 | pci_dev->v_addrs[i].r_size = cur_region->size; | |
525 | pci_dev->v_addrs[i].e_size = 0; | |
526 | ||
527 | assigned_dev_ioport_setup(&pci_dev->dev, i, cur_region->size); | |
528 | pci_register_bar((PCIDevice *) pci_dev, i, | |
529 | PCI_BASE_ADDRESS_SPACE_IO, | |
530 | &pci_dev->v_addrs[i].container); | |
531 | } | |
532 | } | |
533 | ||
534 | /* success */ | |
535 | return 0; | |
536 | } | |
537 | ||
538 | static int get_real_id(const char *devpath, const char *idname, uint16_t *val) | |
539 | { | |
540 | FILE *f; | |
541 | char name[128]; | |
542 | long id; | |
543 | ||
544 | snprintf(name, sizeof(name), "%s%s", devpath, idname); | |
545 | f = fopen(name, "r"); | |
546 | if (f == NULL) { | |
547 | error_report("%s: %s: %m", __func__, name); | |
548 | return -1; | |
549 | } | |
550 | if (fscanf(f, "%li\n", &id) == 1) { | |
551 | *val = id; | |
552 | } else { | |
553 | fclose(f); | |
554 | return -1; | |
555 | } | |
556 | fclose(f); | |
557 | ||
558 | return 0; | |
559 | } | |
560 | ||
561 | static int get_real_vendor_id(const char *devpath, uint16_t *val) | |
562 | { | |
563 | return get_real_id(devpath, "vendor", val); | |
564 | } | |
565 | ||
566 | static int get_real_device_id(const char *devpath, uint16_t *val) | |
567 | { | |
568 | return get_real_id(devpath, "device", val); | |
569 | } | |
570 | ||
571 | static int get_real_device(AssignedDevice *pci_dev, uint16_t r_seg, | |
572 | uint8_t r_bus, uint8_t r_dev, uint8_t r_func) | |
573 | { | |
574 | char dir[128], name[128]; | |
575 | int fd, r = 0, v; | |
576 | FILE *f; | |
577 | uint64_t start, end, size, flags; | |
578 | uint16_t id; | |
579 | PCIRegion *rp; | |
580 | PCIDevRegions *dev = &pci_dev->real_device; | |
581 | ||
582 | dev->region_number = 0; | |
583 | ||
584 | snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%x/", | |
585 | r_seg, r_bus, r_dev, r_func); | |
586 | ||
587 | snprintf(name, sizeof(name), "%sconfig", dir); | |
588 | ||
589 | if (pci_dev->configfd_name && *pci_dev->configfd_name) { | |
590 | dev->config_fd = monitor_handle_fd_param(cur_mon, pci_dev->configfd_name); | |
591 | if (dev->config_fd < 0) { | |
592 | return 1; | |
593 | } | |
594 | } else { | |
595 | dev->config_fd = open(name, O_RDWR); | |
596 | ||
597 | if (dev->config_fd == -1) { | |
598 | error_report("%s: %s: %m", __func__, name); | |
599 | return 1; | |
600 | } | |
601 | } | |
602 | again: | |
603 | r = read(dev->config_fd, pci_dev->dev.config, | |
604 | pci_config_size(&pci_dev->dev)); | |
605 | if (r < 0) { | |
606 | if (errno == EINTR || errno == EAGAIN) { | |
607 | goto again; | |
608 | } | |
609 | error_report("%s: read failed, errno = %d", __func__, errno); | |
610 | } | |
611 | ||
612 | /* Restore or clear multifunction, this is always controlled by qemu */ | |
613 | if (pci_dev->dev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) { | |
614 | pci_dev->dev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION; | |
615 | } else { | |
616 | pci_dev->dev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION; | |
617 | } | |
618 | ||
619 | /* Clear host resource mapping info. If we choose not to register a | |
620 | * BAR, such as might be the case with the option ROM, we can get | |
621 | * confusing, unwritable, residual addresses from the host here. */ | |
622 | memset(&pci_dev->dev.config[PCI_BASE_ADDRESS_0], 0, 24); | |
623 | memset(&pci_dev->dev.config[PCI_ROM_ADDRESS], 0, 4); | |
624 | ||
625 | snprintf(name, sizeof(name), "%sresource", dir); | |
626 | ||
627 | f = fopen(name, "r"); | |
628 | if (f == NULL) { | |
629 | error_report("%s: %s: %m", __func__, name); | |
630 | return 1; | |
631 | } | |
632 | ||
633 | for (r = 0; r < PCI_ROM_SLOT; r++) { | |
634 | if (fscanf(f, "%" SCNi64 " %" SCNi64 " %" SCNi64 "\n", | |
635 | &start, &end, &flags) != 3) { | |
636 | break; | |
637 | } | |
638 | ||
639 | rp = dev->regions + r; | |
640 | rp->valid = 0; | |
641 | rp->resource_fd = -1; | |
642 | size = end - start + 1; | |
643 | flags &= IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH | |
644 | | IORESOURCE_MEM_64; | |
645 | if (size == 0 || (flags & ~IORESOURCE_PREFETCH) == 0) { | |
646 | continue; | |
647 | } | |
648 | if (flags & IORESOURCE_MEM) { | |
649 | flags &= ~IORESOURCE_IO; | |
650 | } else { | |
651 | flags &= ~IORESOURCE_PREFETCH; | |
652 | } | |
653 | snprintf(name, sizeof(name), "%sresource%d", dir, r); | |
654 | fd = open(name, O_RDWR); | |
655 | if (fd == -1) { | |
656 | continue; | |
657 | } | |
658 | rp->resource_fd = fd; | |
659 | ||
660 | rp->type = flags; | |
661 | rp->valid = 1; | |
662 | rp->base_addr = start; | |
663 | rp->size = size; | |
664 | pci_dev->v_addrs[r].region = rp; | |
665 | DEBUG("region %d size %" PRIu64 " start 0x%" PRIx64 | |
666 | " type %d resource_fd %d\n", | |
667 | r, rp->size, start, rp->type, rp->resource_fd); | |
668 | } | |
669 | ||
670 | fclose(f); | |
671 | ||
672 | /* read and fill vendor ID */ | |
673 | v = get_real_vendor_id(dir, &id); | |
674 | if (v) { | |
675 | return 1; | |
676 | } | |
677 | pci_dev->dev.config[0] = id & 0xff; | |
678 | pci_dev->dev.config[1] = (id & 0xff00) >> 8; | |
679 | ||
680 | /* read and fill device ID */ | |
681 | v = get_real_device_id(dir, &id); | |
682 | if (v) { | |
683 | return 1; | |
684 | } | |
685 | pci_dev->dev.config[2] = id & 0xff; | |
686 | pci_dev->dev.config[3] = (id & 0xff00) >> 8; | |
687 | ||
688 | pci_word_test_and_clear_mask(pci_dev->emulate_config_write + PCI_COMMAND, | |
689 | PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE); | |
690 | ||
691 | dev->region_number = r; | |
692 | return 0; | |
693 | } | |
694 | ||
695 | static void free_msi_virqs(AssignedDevice *dev) | |
696 | { | |
697 | int i; | |
698 | ||
699 | for (i = 0; i < dev->msi_virq_nr; i++) { | |
700 | if (dev->msi_virq[i] >= 0) { | |
701 | kvm_irqchip_release_virq(kvm_state, dev->msi_virq[i]); | |
702 | dev->msi_virq[i] = -1; | |
703 | } | |
704 | } | |
705 | g_free(dev->msi_virq); | |
706 | dev->msi_virq = NULL; | |
707 | dev->msi_virq_nr = 0; | |
708 | } | |
709 | ||
710 | static void free_assigned_device(AssignedDevice *dev) | |
711 | { | |
712 | int i; | |
713 | ||
714 | if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) { | |
715 | assigned_dev_unregister_msix_mmio(dev); | |
716 | } | |
717 | for (i = 0; i < dev->real_device.region_number; i++) { | |
718 | PCIRegion *pci_region = &dev->real_device.regions[i]; | |
719 | AssignedDevRegion *region = &dev->v_addrs[i]; | |
720 | ||
721 | if (!pci_region->valid) { | |
722 | continue; | |
723 | } | |
724 | if (pci_region->type & IORESOURCE_IO) { | |
725 | if (region->u.r_baseport) { | |
726 | memory_region_del_subregion(®ion->container, | |
727 | ®ion->real_iomem); | |
728 | memory_region_destroy(®ion->real_iomem); | |
729 | memory_region_destroy(®ion->container); | |
730 | } | |
731 | } else if (pci_region->type & IORESOURCE_MEM) { | |
732 | if (region->u.r_virtbase) { | |
733 | memory_region_del_subregion(®ion->container, | |
734 | ®ion->real_iomem); | |
735 | ||
736 | /* Remove MSI-X table subregion */ | |
737 | if (pci_region->base_addr <= dev->msix_table_addr && | |
738 | pci_region->base_addr + pci_region->size > | |
739 | dev->msix_table_addr) { | |
740 | memory_region_del_subregion(®ion->container, | |
741 | &dev->mmio); | |
742 | } | |
743 | ||
744 | memory_region_destroy(®ion->real_iomem); | |
745 | memory_region_destroy(®ion->container); | |
746 | if (munmap(region->u.r_virtbase, | |
747 | (pci_region->size + 0xFFF) & 0xFFFFF000)) { | |
748 | error_report("Failed to unmap assigned device region: %s", | |
749 | strerror(errno)); | |
750 | } | |
751 | } | |
752 | } | |
753 | if (pci_region->resource_fd >= 0) { | |
754 | close(pci_region->resource_fd); | |
755 | } | |
756 | } | |
757 | ||
758 | if (dev->real_device.config_fd >= 0) { | |
759 | close(dev->real_device.config_fd); | |
760 | } | |
761 | ||
762 | free_msi_virqs(dev); | |
763 | } | |
764 | ||
765 | static void assign_failed_examine(AssignedDevice *dev) | |
766 | { | |
767 | char name[PATH_MAX], dir[PATH_MAX], driver[PATH_MAX] = {}, *ns; | |
768 | uint16_t vendor_id, device_id; | |
769 | int r; | |
770 | ||
771 | snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/", | |
772 | dev->host.domain, dev->host.bus, dev->host.slot, | |
773 | dev->host.function); | |
774 | ||
775 | snprintf(name, sizeof(name), "%sdriver", dir); | |
776 | ||
777 | r = readlink(name, driver, sizeof(driver)); | |
778 | if ((r <= 0) || r >= sizeof(driver)) { | |
779 | goto fail; | |
780 | } | |
781 | ||
782 | ns = strrchr(driver, '/'); | |
783 | if (!ns) { | |
784 | goto fail; | |
785 | } | |
786 | ||
787 | ns++; | |
788 | ||
789 | if (get_real_vendor_id(dir, &vendor_id) || | |
790 | get_real_device_id(dir, &device_id)) { | |
791 | goto fail; | |
792 | } | |
793 | ||
794 | error_report("*** The driver '%s' is occupying your device " | |
795 | "%04x:%02x:%02x.%x.", | |
796 | ns, dev->host.domain, dev->host.bus, dev->host.slot, | |
797 | dev->host.function); | |
798 | error_report("***"); | |
799 | error_report("*** You can try the following commands to free it:"); | |
800 | error_report("***"); | |
801 | error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub/" | |
802 | "new_id", vendor_id, device_id); | |
803 | error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/" | |
804 | "%s/unbind", | |
805 | dev->host.domain, dev->host.bus, dev->host.slot, | |
806 | dev->host.function, ns); | |
807 | error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/" | |
808 | "pci-stub/bind", | |
809 | dev->host.domain, dev->host.bus, dev->host.slot, | |
810 | dev->host.function); | |
811 | error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub" | |
812 | "/remove_id", vendor_id, device_id); | |
813 | error_report("***"); | |
814 | ||
815 | return; | |
816 | ||
817 | fail: | |
818 | error_report("Couldn't find out why."); | |
819 | } | |
820 | ||
821 | static int assign_device(AssignedDevice *dev) | |
822 | { | |
823 | uint32_t flags = KVM_DEV_ASSIGN_ENABLE_IOMMU; | |
824 | int r; | |
825 | ||
826 | /* Only pass non-zero PCI segment to capable module */ | |
827 | if (!kvm_check_extension(kvm_state, KVM_CAP_PCI_SEGMENT) && | |
828 | dev->host.domain) { | |
829 | error_report("Can't assign device inside non-zero PCI segment " | |
830 | "as this KVM module doesn't support it."); | |
831 | return -ENODEV; | |
832 | } | |
833 | ||
834 | if (!kvm_check_extension(kvm_state, KVM_CAP_IOMMU)) { | |
835 | error_report("No IOMMU found. Unable to assign device \"%s\"", | |
836 | dev->dev.qdev.id); | |
837 | return -ENODEV; | |
838 | } | |
839 | ||
840 | if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK && | |
841 | kvm_has_intx_set_mask()) { | |
842 | flags |= KVM_DEV_ASSIGN_PCI_2_3; | |
843 | } | |
844 | ||
845 | r = kvm_device_pci_assign(kvm_state, &dev->host, flags, &dev->dev_id); | |
846 | if (r < 0) { | |
847 | error_report("Failed to assign device \"%s\" : %s", | |
848 | dev->dev.qdev.id, strerror(-r)); | |
849 | ||
850 | switch (r) { | |
851 | case -EBUSY: | |
852 | assign_failed_examine(dev); | |
853 | break; | |
854 | default: | |
855 | break; | |
856 | } | |
857 | } | |
858 | return r; | |
859 | } | |
860 | ||
861 | static bool check_irqchip_in_kernel(void) | |
862 | { | |
863 | if (kvm_irqchip_in_kernel()) { | |
864 | return true; | |
865 | } | |
866 | error_report("pci-assign: error: requires KVM with in-kernel irqchip " | |
867 | "enabled"); | |
868 | return false; | |
869 | } | |
870 | ||
871 | static int assign_intx(AssignedDevice *dev) | |
872 | { | |
873 | AssignedIRQType new_type; | |
874 | PCIINTxRoute intx_route; | |
875 | bool intx_host_msi; | |
876 | int r; | |
877 | ||
878 | /* Interrupt PIN 0 means don't use INTx */ | |
879 | if (assigned_dev_pci_read_byte(&dev->dev, PCI_INTERRUPT_PIN) == 0) { | |
880 | pci_device_set_intx_routing_notifier(&dev->dev, NULL); | |
881 | return 0; | |
882 | } | |
883 | ||
884 | if (!check_irqchip_in_kernel()) { | |
885 | return -ENOTSUP; | |
886 | } | |
887 | ||
888 | pci_device_set_intx_routing_notifier(&dev->dev, | |
889 | assigned_dev_update_irq_routing); | |
890 | ||
891 | intx_route = pci_device_route_intx_to_irq(&dev->dev, dev->intpin); | |
892 | assert(intx_route.mode != PCI_INTX_INVERTED); | |
893 | ||
894 | if (!pci_intx_route_changed(&dev->intx_route, &intx_route)) { | |
895 | return 0; | |
896 | } | |
897 | ||
898 | switch (dev->assigned_irq_type) { | |
899 | case ASSIGNED_IRQ_INTX_HOST_INTX: | |
900 | case ASSIGNED_IRQ_INTX_HOST_MSI: | |
901 | intx_host_msi = dev->assigned_irq_type == ASSIGNED_IRQ_INTX_HOST_MSI; | |
902 | r = kvm_device_intx_deassign(kvm_state, dev->dev_id, intx_host_msi); | |
903 | break; | |
904 | case ASSIGNED_IRQ_MSI: | |
905 | r = kvm_device_msi_deassign(kvm_state, dev->dev_id); | |
906 | break; | |
907 | case ASSIGNED_IRQ_MSIX: | |
908 | r = kvm_device_msix_deassign(kvm_state, dev->dev_id); | |
909 | break; | |
910 | default: | |
911 | r = 0; | |
912 | break; | |
913 | } | |
914 | if (r) { | |
915 | perror("assign_intx: deassignment of previous interrupt failed"); | |
916 | } | |
917 | dev->assigned_irq_type = ASSIGNED_IRQ_NONE; | |
918 | ||
919 | if (intx_route.mode == PCI_INTX_DISABLED) { | |
920 | dev->intx_route = intx_route; | |
921 | return 0; | |
922 | } | |
923 | ||
924 | retry: | |
925 | if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK && | |
926 | dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) { | |
927 | intx_host_msi = true; | |
928 | new_type = ASSIGNED_IRQ_INTX_HOST_MSI; | |
929 | } else { | |
930 | intx_host_msi = false; | |
931 | new_type = ASSIGNED_IRQ_INTX_HOST_INTX; | |
932 | } | |
933 | ||
934 | r = kvm_device_intx_assign(kvm_state, dev->dev_id, intx_host_msi, | |
935 | intx_route.irq); | |
936 | if (r < 0) { | |
937 | if (r == -EIO && !(dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK) && | |
938 | dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) { | |
939 | /* Retry with host-side MSI. There might be an IRQ conflict and | |
940 | * either the kernel or the device doesn't support sharing. */ | |
941 | error_report("Host-side INTx sharing not supported, " | |
942 | "using MSI instead"); | |
943 | error_printf("Some devices do not work properly in this mode.\n"); | |
944 | dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK; | |
945 | goto retry; | |
946 | } | |
947 | error_report("Failed to assign irq for \"%s\": %s", | |
948 | dev->dev.qdev.id, strerror(-r)); | |
949 | error_report("Perhaps you are assigning a device " | |
950 | "that shares an IRQ with another device?"); | |
951 | return r; | |
952 | } | |
953 | ||
954 | dev->intx_route = intx_route; | |
955 | dev->assigned_irq_type = new_type; | |
956 | return r; | |
957 | } | |
958 | ||
959 | static void deassign_device(AssignedDevice *dev) | |
960 | { | |
961 | int r; | |
962 | ||
963 | r = kvm_device_pci_deassign(kvm_state, dev->dev_id); | |
964 | assert(r == 0); | |
965 | } | |
966 | ||
967 | /* The pci config space got updated. Check if irq numbers have changed | |
968 | * for our devices | |
969 | */ | |
970 | static void assigned_dev_update_irq_routing(PCIDevice *dev) | |
971 | { | |
972 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, dev); | |
973 | Error *err = NULL; | |
974 | int r; | |
975 | ||
976 | r = assign_intx(assigned_dev); | |
977 | if (r < 0) { | |
978 | qdev_unplug(&dev->qdev, &err); | |
979 | assert(!err); | |
980 | } | |
981 | } | |
982 | ||
983 | static void assigned_dev_update_msi(PCIDevice *pci_dev) | |
984 | { | |
985 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
986 | uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap + | |
987 | PCI_MSI_FLAGS); | |
988 | int r; | |
989 | ||
990 | /* Some guests gratuitously disable MSI even if they're not using it, | |
991 | * try to catch this by only deassigning irqs if the guest is using | |
992 | * MSI or intends to start. */ | |
993 | if (assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSI || | |
994 | (ctrl_byte & PCI_MSI_FLAGS_ENABLE)) { | |
995 | r = kvm_device_msi_deassign(kvm_state, assigned_dev->dev_id); | |
996 | /* -ENXIO means no assigned irq */ | |
997 | if (r && r != -ENXIO) { | |
998 | perror("assigned_dev_update_msi: deassign irq"); | |
999 | } | |
1000 | ||
1001 | free_msi_virqs(assigned_dev); | |
1002 | ||
1003 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE; | |
1004 | pci_device_set_intx_routing_notifier(pci_dev, NULL); | |
1005 | } | |
1006 | ||
1007 | if (ctrl_byte & PCI_MSI_FLAGS_ENABLE) { | |
1008 | MSIMessage msg = msi_get_message(pci_dev, 0); | |
1009 | int virq; | |
1010 | ||
1011 | virq = kvm_irqchip_add_msi_route(kvm_state, msg); | |
1012 | if (virq < 0) { | |
1013 | perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route"); | |
1014 | return; | |
1015 | } | |
1016 | ||
1017 | assigned_dev->msi_virq = g_malloc(sizeof(*assigned_dev->msi_virq)); | |
1018 | assigned_dev->msi_virq_nr = 1; | |
1019 | assigned_dev->msi_virq[0] = virq; | |
1020 | if (kvm_device_msi_assign(kvm_state, assigned_dev->dev_id, virq) < 0) { | |
1021 | perror("assigned_dev_update_msi: kvm_device_msi_assign"); | |
1022 | } | |
1023 | ||
1024 | assigned_dev->intx_route.mode = PCI_INTX_DISABLED; | |
1025 | assigned_dev->intx_route.irq = -1; | |
1026 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSI; | |
1027 | } else { | |
1028 | assign_intx(assigned_dev); | |
1029 | } | |
1030 | } | |
1031 | ||
1032 | static void assigned_dev_update_msi_msg(PCIDevice *pci_dev) | |
1033 | { | |
1034 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1035 | uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap + | |
1036 | PCI_MSI_FLAGS); | |
1037 | ||
1038 | if (assigned_dev->assigned_irq_type != ASSIGNED_IRQ_MSI || | |
1039 | !(ctrl_byte & PCI_MSI_FLAGS_ENABLE)) { | |
1040 | return; | |
1041 | } | |
1042 | ||
1043 | kvm_irqchip_update_msi_route(kvm_state, assigned_dev->msi_virq[0], | |
1044 | msi_get_message(pci_dev, 0)); | |
1045 | } | |
1046 | ||
1047 | static bool assigned_dev_msix_masked(MSIXTableEntry *entry) | |
1048 | { | |
1049 | return (entry->ctrl & cpu_to_le32(0x1)) != 0; | |
1050 | } | |
1051 | ||
1052 | /* | |
1053 | * When MSI-X is first enabled the vector table typically has all the | |
1054 | * vectors masked, so we can't use that as the obvious test to figure out | |
1055 | * how many vectors to initially enable. Instead we look at the data field | |
1056 | * because this is what worked for pci-assign for a long time. This makes | |
1057 | * sure the physical MSI-X state tracks the guest's view, which is important | |
1058 | * for some VF/PF and PF/fw communication channels. | |
1059 | */ | |
1060 | static bool assigned_dev_msix_skipped(MSIXTableEntry *entry) | |
1061 | { | |
1062 | return !entry->data; | |
1063 | } | |
1064 | ||
1065 | static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev) | |
1066 | { | |
1067 | AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1068 | uint16_t entries_nr = 0; | |
1069 | int i, r = 0; | |
1070 | MSIXTableEntry *entry = adev->msix_table; | |
1071 | MSIMessage msg; | |
1072 | ||
1073 | /* Get the usable entry number for allocating */ | |
1074 | for (i = 0; i < adev->msix_max; i++, entry++) { | |
1075 | if (assigned_dev_msix_skipped(entry)) { | |
1076 | continue; | |
1077 | } | |
1078 | entries_nr++; | |
1079 | } | |
1080 | ||
1081 | DEBUG("MSI-X entries: %d\n", entries_nr); | |
1082 | ||
1083 | /* It's valid to enable MSI-X with all entries masked */ | |
1084 | if (!entries_nr) { | |
1085 | return 0; | |
1086 | } | |
1087 | ||
1088 | r = kvm_device_msix_init_vectors(kvm_state, adev->dev_id, entries_nr); | |
1089 | if (r != 0) { | |
1090 | error_report("fail to set MSI-X entry number for MSIX! %s", | |
1091 | strerror(-r)); | |
1092 | return r; | |
1093 | } | |
1094 | ||
1095 | free_msi_virqs(adev); | |
1096 | ||
1097 | adev->msi_virq_nr = adev->msix_max; | |
1098 | adev->msi_virq = g_malloc(adev->msix_max * sizeof(*adev->msi_virq)); | |
1099 | ||
1100 | entry = adev->msix_table; | |
1101 | for (i = 0; i < adev->msix_max; i++, entry++) { | |
1102 | adev->msi_virq[i] = -1; | |
1103 | ||
1104 | if (assigned_dev_msix_skipped(entry)) { | |
1105 | continue; | |
1106 | } | |
1107 | ||
1108 | msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32); | |
1109 | msg.data = entry->data; | |
1110 | r = kvm_irqchip_add_msi_route(kvm_state, msg); | |
1111 | if (r < 0) { | |
1112 | return r; | |
1113 | } | |
1114 | adev->msi_virq[i] = r; | |
1115 | ||
1116 | DEBUG("MSI-X vector %d, gsi %d, addr %08x_%08x, data %08x\n", i, | |
1117 | r, entry->addr_hi, entry->addr_lo, entry->data); | |
1118 | ||
1119 | r = kvm_device_msix_set_vector(kvm_state, adev->dev_id, i, | |
1120 | adev->msi_virq[i]); | |
1121 | if (r) { | |
1122 | error_report("fail to set MSI-X entry! %s", strerror(-r)); | |
1123 | break; | |
1124 | } | |
1125 | } | |
1126 | ||
1127 | return r; | |
1128 | } | |
1129 | ||
1130 | static void assigned_dev_update_msix(PCIDevice *pci_dev) | |
1131 | { | |
1132 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1133 | uint16_t ctrl_word = pci_get_word(pci_dev->config + pci_dev->msix_cap + | |
1134 | PCI_MSIX_FLAGS); | |
1135 | int r; | |
1136 | ||
1137 | /* Some guests gratuitously disable MSIX even if they're not using it, | |
1138 | * try to catch this by only deassigning irqs if the guest is using | |
1139 | * MSIX or intends to start. */ | |
1140 | if ((assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSIX) || | |
1141 | (ctrl_word & PCI_MSIX_FLAGS_ENABLE)) { | |
1142 | r = kvm_device_msix_deassign(kvm_state, assigned_dev->dev_id); | |
1143 | /* -ENXIO means no assigned irq */ | |
1144 | if (r && r != -ENXIO) { | |
1145 | perror("assigned_dev_update_msix: deassign irq"); | |
1146 | } | |
1147 | ||
1148 | free_msi_virqs(assigned_dev); | |
1149 | ||
1150 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE; | |
1151 | pci_device_set_intx_routing_notifier(pci_dev, NULL); | |
1152 | } | |
1153 | ||
1154 | if (ctrl_word & PCI_MSIX_FLAGS_ENABLE) { | |
1155 | if (assigned_dev_update_msix_mmio(pci_dev) < 0) { | |
1156 | perror("assigned_dev_update_msix_mmio"); | |
1157 | return; | |
1158 | } | |
1159 | ||
1160 | if (assigned_dev->msi_virq_nr > 0) { | |
1161 | if (kvm_device_msix_assign(kvm_state, assigned_dev->dev_id) < 0) { | |
1162 | perror("assigned_dev_enable_msix: assign irq"); | |
1163 | return; | |
1164 | } | |
1165 | } | |
1166 | assigned_dev->intx_route.mode = PCI_INTX_DISABLED; | |
1167 | assigned_dev->intx_route.irq = -1; | |
1168 | assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSIX; | |
1169 | } else { | |
1170 | assign_intx(assigned_dev); | |
1171 | } | |
1172 | } | |
1173 | ||
1174 | static uint32_t assigned_dev_pci_read_config(PCIDevice *pci_dev, | |
1175 | uint32_t address, int len) | |
1176 | { | |
1177 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1178 | uint32_t virt_val = pci_default_read_config(pci_dev, address, len); | |
1179 | uint32_t real_val, emulate_mask, full_emulation_mask; | |
1180 | ||
1181 | emulate_mask = 0; | |
1182 | memcpy(&emulate_mask, assigned_dev->emulate_config_read + address, len); | |
1183 | emulate_mask = le32_to_cpu(emulate_mask); | |
1184 | ||
1185 | full_emulation_mask = 0xffffffff >> (32 - len * 8); | |
1186 | ||
1187 | if (emulate_mask != full_emulation_mask) { | |
1188 | real_val = assigned_dev_pci_read(pci_dev, address, len); | |
1189 | return (virt_val & emulate_mask) | (real_val & ~emulate_mask); | |
1190 | } else { | |
1191 | return virt_val; | |
1192 | } | |
1193 | } | |
1194 | ||
1195 | static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address, | |
1196 | uint32_t val, int len) | |
1197 | { | |
1198 | AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1199 | uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND); | |
1200 | uint32_t emulate_mask, full_emulation_mask; | |
1201 | int ret; | |
1202 | ||
1203 | pci_default_write_config(pci_dev, address, val, len); | |
1204 | ||
1205 | if (kvm_has_intx_set_mask() && | |
1206 | range_covers_byte(address, len, PCI_COMMAND + 1)) { | |
1207 | bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) & | |
1208 | PCI_COMMAND_INTX_DISABLE); | |
1209 | ||
1210 | if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) { | |
1211 | ret = kvm_device_intx_set_mask(kvm_state, assigned_dev->dev_id, | |
1212 | intx_masked); | |
1213 | if (ret) { | |
1214 | perror("assigned_dev_pci_write_config: set intx mask"); | |
1215 | } | |
1216 | } | |
1217 | } | |
1218 | if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) { | |
1219 | if (range_covers_byte(address, len, | |
1220 | pci_dev->msi_cap + PCI_MSI_FLAGS)) { | |
1221 | assigned_dev_update_msi(pci_dev); | |
1222 | } else if (ranges_overlap(address, len, /* 32bit MSI only */ | |
1223 | pci_dev->msi_cap + PCI_MSI_ADDRESS_LO, 6)) { | |
1224 | assigned_dev_update_msi_msg(pci_dev); | |
1225 | } | |
1226 | } | |
1227 | if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) { | |
1228 | if (range_covers_byte(address, len, | |
1229 | pci_dev->msix_cap + PCI_MSIX_FLAGS + 1)) { | |
1230 | assigned_dev_update_msix(pci_dev); | |
1231 | } | |
1232 | } | |
1233 | ||
1234 | emulate_mask = 0; | |
1235 | memcpy(&emulate_mask, assigned_dev->emulate_config_write + address, len); | |
1236 | emulate_mask = le32_to_cpu(emulate_mask); | |
1237 | ||
1238 | full_emulation_mask = 0xffffffff >> (32 - len * 8); | |
1239 | ||
1240 | if (emulate_mask != full_emulation_mask) { | |
1241 | if (emulate_mask) { | |
1242 | val &= ~emulate_mask; | |
1243 | val |= assigned_dev_pci_read(pci_dev, address, len) & emulate_mask; | |
1244 | } | |
1245 | assigned_dev_pci_write(pci_dev, address, val, len); | |
1246 | } | |
1247 | } | |
1248 | ||
1249 | static void assigned_dev_setup_cap_read(AssignedDevice *dev, uint32_t offset, | |
1250 | uint32_t len) | |
1251 | { | |
1252 | assigned_dev_direct_config_read(dev, offset, len); | |
1253 | assigned_dev_emulate_config_read(dev, offset + PCI_CAP_LIST_NEXT, 1); | |
1254 | } | |
1255 | ||
1256 | static int assigned_device_pci_cap_init(PCIDevice *pci_dev) | |
1257 | { | |
1258 | AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1259 | PCIRegion *pci_region = dev->real_device.regions; | |
1260 | int ret, pos; | |
1261 | ||
1262 | /* Clear initial capabilities pointer and status copied from hw */ | |
1263 | pci_set_byte(pci_dev->config + PCI_CAPABILITY_LIST, 0); | |
1264 | pci_set_word(pci_dev->config + PCI_STATUS, | |
1265 | pci_get_word(pci_dev->config + PCI_STATUS) & | |
1266 | ~PCI_STATUS_CAP_LIST); | |
1267 | ||
1268 | /* Expose MSI capability | |
1269 | * MSI capability is the 1st capability in capability config */ | |
1270 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSI, 0); | |
1271 | if (pos != 0 && kvm_check_extension(kvm_state, KVM_CAP_ASSIGN_DEV_IRQ)) { | |
1272 | if (!check_irqchip_in_kernel()) { | |
1273 | return -ENOTSUP; | |
1274 | } | |
1275 | dev->cap.available |= ASSIGNED_DEVICE_CAP_MSI; | |
1276 | /* Only 32-bit/no-mask currently supported */ | |
1277 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSI, pos, 10); | |
1278 | if (ret < 0) { | |
1279 | return ret; | |
1280 | } | |
1281 | pci_dev->msi_cap = pos; | |
1282 | ||
1283 | pci_set_word(pci_dev->config + pos + PCI_MSI_FLAGS, | |
1284 | pci_get_word(pci_dev->config + pos + PCI_MSI_FLAGS) & | |
1285 | PCI_MSI_FLAGS_QMASK); | |
1286 | pci_set_long(pci_dev->config + pos + PCI_MSI_ADDRESS_LO, 0); | |
1287 | pci_set_word(pci_dev->config + pos + PCI_MSI_DATA_32, 0); | |
1288 | ||
1289 | /* Set writable fields */ | |
1290 | pci_set_word(pci_dev->wmask + pos + PCI_MSI_FLAGS, | |
1291 | PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); | |
1292 | pci_set_long(pci_dev->wmask + pos + PCI_MSI_ADDRESS_LO, 0xfffffffc); | |
1293 | pci_set_word(pci_dev->wmask + pos + PCI_MSI_DATA_32, 0xffff); | |
1294 | } | |
1295 | /* Expose MSI-X capability */ | |
1296 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSIX, 0); | |
1297 | if (pos != 0 && kvm_device_msix_supported(kvm_state)) { | |
1298 | int bar_nr; | |
1299 | uint32_t msix_table_entry; | |
1300 | ||
1301 | if (!check_irqchip_in_kernel()) { | |
1302 | return -ENOTSUP; | |
1303 | } | |
1304 | dev->cap.available |= ASSIGNED_DEVICE_CAP_MSIX; | |
1305 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSIX, pos, 12); | |
1306 | if (ret < 0) { | |
1307 | return ret; | |
1308 | } | |
1309 | pci_dev->msix_cap = pos; | |
1310 | ||
1311 | pci_set_word(pci_dev->config + pos + PCI_MSIX_FLAGS, | |
1312 | pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS) & | |
1313 | PCI_MSIX_FLAGS_QSIZE); | |
1314 | ||
1315 | /* Only enable and function mask bits are writable */ | |
1316 | pci_set_word(pci_dev->wmask + pos + PCI_MSIX_FLAGS, | |
1317 | PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); | |
1318 | ||
1319 | msix_table_entry = pci_get_long(pci_dev->config + pos + PCI_MSIX_TABLE); | |
1320 | bar_nr = msix_table_entry & PCI_MSIX_FLAGS_BIRMASK; | |
1321 | msix_table_entry &= ~PCI_MSIX_FLAGS_BIRMASK; | |
1322 | dev->msix_table_addr = pci_region[bar_nr].base_addr + msix_table_entry; | |
1323 | dev->msix_max = pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS); | |
1324 | dev->msix_max &= PCI_MSIX_FLAGS_QSIZE; | |
1325 | dev->msix_max += 1; | |
1326 | } | |
1327 | ||
1328 | /* Minimal PM support, nothing writable, device appears to NAK changes */ | |
1329 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PM, 0); | |
1330 | if (pos) { | |
1331 | uint16_t pmc; | |
1332 | ||
1333 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, pos, PCI_PM_SIZEOF); | |
1334 | if (ret < 0) { | |
1335 | return ret; | |
1336 | } | |
1337 | ||
1338 | assigned_dev_setup_cap_read(dev, pos, PCI_PM_SIZEOF); | |
1339 | ||
1340 | pmc = pci_get_word(pci_dev->config + pos + PCI_CAP_FLAGS); | |
1341 | pmc &= (PCI_PM_CAP_VER_MASK | PCI_PM_CAP_DSI); | |
1342 | pci_set_word(pci_dev->config + pos + PCI_CAP_FLAGS, pmc); | |
1343 | ||
1344 | /* assign_device will bring the device up to D0, so we don't need | |
1345 | * to worry about doing that ourselves here. */ | |
1346 | pci_set_word(pci_dev->config + pos + PCI_PM_CTRL, | |
1347 | PCI_PM_CTRL_NO_SOFT_RESET); | |
1348 | ||
1349 | pci_set_byte(pci_dev->config + pos + PCI_PM_PPB_EXTENSIONS, 0); | |
1350 | pci_set_byte(pci_dev->config + pos + PCI_PM_DATA_REGISTER, 0); | |
1351 | } | |
1352 | ||
1353 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_EXP, 0); | |
1354 | if (pos) { | |
1355 | uint8_t version, size = 0; | |
1356 | uint16_t type, devctl, lnksta; | |
1357 | uint32_t devcap, lnkcap; | |
1358 | ||
1359 | version = pci_get_byte(pci_dev->config + pos + PCI_EXP_FLAGS); | |
1360 | version &= PCI_EXP_FLAGS_VERS; | |
1361 | if (version == 1) { | |
1362 | size = 0x14; | |
1363 | } else if (version == 2) { | |
1364 | /* | |
1365 | * Check for non-std size, accept reduced size to 0x34, | |
1366 | * which is what bcm5761 implemented, violating the | |
1367 | * PCIe v3.0 spec that regs should exist and be read as 0, | |
1368 | * not optionally provided and shorten the struct size. | |
1369 | */ | |
1370 | size = MIN(0x3c, PCI_CONFIG_SPACE_SIZE - pos); | |
1371 | if (size < 0x34) { | |
1372 | error_report("%s: Invalid size PCIe cap-id 0x%x", | |
1373 | __func__, PCI_CAP_ID_EXP); | |
1374 | return -EINVAL; | |
1375 | } else if (size != 0x3c) { | |
1376 | error_report("WARNING, %s: PCIe cap-id 0x%x has " | |
1377 | "non-standard size 0x%x; std size should be 0x3c", | |
1378 | __func__, PCI_CAP_ID_EXP, size); | |
1379 | } | |
1380 | } else if (version == 0) { | |
1381 | uint16_t vid, did; | |
1382 | vid = pci_get_word(pci_dev->config + PCI_VENDOR_ID); | |
1383 | did = pci_get_word(pci_dev->config + PCI_DEVICE_ID); | |
1384 | if (vid == PCI_VENDOR_ID_INTEL && did == 0x10ed) { | |
1385 | /* | |
1386 | * quirk for Intel 82599 VF with invalid PCIe capability | |
1387 | * version, should really be version 2 (same as PF) | |
1388 | */ | |
1389 | size = 0x3c; | |
1390 | } | |
1391 | } | |
1392 | ||
1393 | if (size == 0) { | |
1394 | error_report("%s: Unsupported PCI express capability version %d", | |
1395 | __func__, version); | |
1396 | return -EINVAL; | |
1397 | } | |
1398 | ||
1399 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_EXP, pos, size); | |
1400 | if (ret < 0) { | |
1401 | return ret; | |
1402 | } | |
1403 | ||
1404 | assigned_dev_setup_cap_read(dev, pos, size); | |
1405 | ||
1406 | type = pci_get_word(pci_dev->config + pos + PCI_EXP_FLAGS); | |
1407 | type = (type & PCI_EXP_FLAGS_TYPE) >> 4; | |
1408 | if (type != PCI_EXP_TYPE_ENDPOINT && | |
1409 | type != PCI_EXP_TYPE_LEG_END && type != PCI_EXP_TYPE_RC_END) { | |
1410 | error_report("Device assignment only supports endpoint assignment," | |
1411 | " device type %d", type); | |
1412 | return -EINVAL; | |
1413 | } | |
1414 | ||
1415 | /* capabilities, pass existing read-only copy | |
1416 | * PCI_EXP_FLAGS_IRQ: updated by hardware, should be direct read */ | |
1417 | ||
1418 | /* device capabilities: hide FLR */ | |
1419 | devcap = pci_get_long(pci_dev->config + pos + PCI_EXP_DEVCAP); | |
1420 | devcap &= ~PCI_EXP_DEVCAP_FLR; | |
1421 | pci_set_long(pci_dev->config + pos + PCI_EXP_DEVCAP, devcap); | |
1422 | ||
1423 | /* device control: clear all error reporting enable bits, leaving | |
1424 | * only a few host values. Note, these are | |
1425 | * all writable, but not passed to hw. | |
1426 | */ | |
1427 | devctl = pci_get_word(pci_dev->config + pos + PCI_EXP_DEVCTL); | |
1428 | devctl = (devctl & (PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_PAYLOAD)) | | |
1429 | PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN; | |
1430 | pci_set_word(pci_dev->config + pos + PCI_EXP_DEVCTL, devctl); | |
1431 | devctl = PCI_EXP_DEVCTL_BCR_FLR | PCI_EXP_DEVCTL_AUX_PME; | |
1432 | pci_set_word(pci_dev->wmask + pos + PCI_EXP_DEVCTL, ~devctl); | |
1433 | ||
1434 | /* Clear device status */ | |
1435 | pci_set_word(pci_dev->config + pos + PCI_EXP_DEVSTA, 0); | |
1436 | ||
1437 | /* Link capabilities, expose links and latencues, clear reporting */ | |
1438 | lnkcap = pci_get_long(pci_dev->config + pos + PCI_EXP_LNKCAP); | |
1439 | lnkcap &= (PCI_EXP_LNKCAP_SLS | PCI_EXP_LNKCAP_MLW | | |
1440 | PCI_EXP_LNKCAP_ASPMS | PCI_EXP_LNKCAP_L0SEL | | |
1441 | PCI_EXP_LNKCAP_L1EL); | |
1442 | pci_set_long(pci_dev->config + pos + PCI_EXP_LNKCAP, lnkcap); | |
1443 | ||
1444 | /* Link control, pass existing read-only copy. Should be writable? */ | |
1445 | ||
1446 | /* Link status, only expose current speed and width */ | |
1447 | lnksta = pci_get_word(pci_dev->config + pos + PCI_EXP_LNKSTA); | |
1448 | lnksta &= (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW); | |
1449 | pci_set_word(pci_dev->config + pos + PCI_EXP_LNKSTA, lnksta); | |
1450 | ||
1451 | if (version >= 2) { | |
1452 | /* Slot capabilities, control, status - not needed for endpoints */ | |
1453 | pci_set_long(pci_dev->config + pos + PCI_EXP_SLTCAP, 0); | |
1454 | pci_set_word(pci_dev->config + pos + PCI_EXP_SLTCTL, 0); | |
1455 | pci_set_word(pci_dev->config + pos + PCI_EXP_SLTSTA, 0); | |
1456 | ||
1457 | /* Root control, capabilities, status - not needed for endpoints */ | |
1458 | pci_set_word(pci_dev->config + pos + PCI_EXP_RTCTL, 0); | |
1459 | pci_set_word(pci_dev->config + pos + PCI_EXP_RTCAP, 0); | |
1460 | pci_set_long(pci_dev->config + pos + PCI_EXP_RTSTA, 0); | |
1461 | ||
1462 | /* Device capabilities/control 2, pass existing read-only copy */ | |
1463 | /* Link control 2, pass existing read-only copy */ | |
1464 | } | |
1465 | } | |
1466 | ||
1467 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PCIX, 0); | |
1468 | if (pos) { | |
1469 | uint16_t cmd; | |
1470 | uint32_t status; | |
1471 | ||
1472 | /* Only expose the minimum, 8 byte capability */ | |
1473 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_PCIX, pos, 8); | |
1474 | if (ret < 0) { | |
1475 | return ret; | |
1476 | } | |
1477 | ||
1478 | assigned_dev_setup_cap_read(dev, pos, 8); | |
1479 | ||
1480 | /* Command register, clear upper bits, including extended modes */ | |
1481 | cmd = pci_get_word(pci_dev->config + pos + PCI_X_CMD); | |
1482 | cmd &= (PCI_X_CMD_DPERR_E | PCI_X_CMD_ERO | PCI_X_CMD_MAX_READ | | |
1483 | PCI_X_CMD_MAX_SPLIT); | |
1484 | pci_set_word(pci_dev->config + pos + PCI_X_CMD, cmd); | |
1485 | ||
1486 | /* Status register, update with emulated PCI bus location, clear | |
1487 | * error bits, leave the rest. */ | |
1488 | status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS); | |
1489 | status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN); | |
1490 | status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn; | |
1491 | status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL | | |
1492 | PCI_X_STATUS_SPL_ERR); | |
1493 | pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status); | |
1494 | } | |
1495 | ||
1496 | pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VPD, 0); | |
1497 | if (pos) { | |
1498 | /* Direct R/W passthrough */ | |
1499 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_VPD, pos, 8); | |
1500 | if (ret < 0) { | |
1501 | return ret; | |
1502 | } | |
1503 | ||
1504 | assigned_dev_setup_cap_read(dev, pos, 8); | |
1505 | ||
1506 | /* direct write for cap content */ | |
1507 | assigned_dev_direct_config_write(dev, pos + 2, 6); | |
1508 | } | |
1509 | ||
1510 | /* Devices can have multiple vendor capabilities, get them all */ | |
1511 | for (pos = 0; (pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VNDR, pos)); | |
1512 | pos += PCI_CAP_LIST_NEXT) { | |
1513 | uint8_t len = pci_get_byte(pci_dev->config + pos + PCI_CAP_FLAGS); | |
1514 | /* Direct R/W passthrough */ | |
1515 | ret = pci_add_capability(pci_dev, PCI_CAP_ID_VNDR, pos, len); | |
1516 | if (ret < 0) { | |
1517 | return ret; | |
1518 | } | |
1519 | ||
1520 | assigned_dev_setup_cap_read(dev, pos, len); | |
1521 | ||
1522 | /* direct write for cap content */ | |
1523 | assigned_dev_direct_config_write(dev, pos + 2, len - 2); | |
1524 | } | |
1525 | ||
1526 | /* If real and virtual capability list status bits differ, virtualize the | |
1527 | * access. */ | |
1528 | if ((pci_get_word(pci_dev->config + PCI_STATUS) & PCI_STATUS_CAP_LIST) != | |
1529 | (assigned_dev_pci_read_byte(pci_dev, PCI_STATUS) & | |
1530 | PCI_STATUS_CAP_LIST)) { | |
1531 | dev->emulate_config_read[PCI_STATUS] |= PCI_STATUS_CAP_LIST; | |
1532 | } | |
1533 | ||
1534 | return 0; | |
1535 | } | |
1536 | ||
1537 | static uint64_t | |
1538 | assigned_dev_msix_mmio_read(void *opaque, hwaddr addr, | |
1539 | unsigned size) | |
1540 | { | |
1541 | AssignedDevice *adev = opaque; | |
1542 | uint64_t val; | |
1543 | ||
1544 | memcpy(&val, (void *)((uint8_t *)adev->msix_table + addr), size); | |
1545 | ||
1546 | return val; | |
1547 | } | |
1548 | ||
1549 | static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr, | |
1550 | uint64_t val, unsigned size) | |
1551 | { | |
1552 | AssignedDevice *adev = opaque; | |
1553 | PCIDevice *pdev = &adev->dev; | |
1554 | uint16_t ctrl; | |
1555 | MSIXTableEntry orig; | |
1556 | int i = addr >> 4; | |
1557 | ||
1558 | if (i >= adev->msix_max) { | |
1559 | return; /* Drop write */ | |
1560 | } | |
1561 | ||
1562 | ctrl = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS); | |
1563 | ||
1564 | DEBUG("write to MSI-X table offset 0x%lx, val 0x%lx\n", addr, val); | |
1565 | ||
1566 | if (ctrl & PCI_MSIX_FLAGS_ENABLE) { | |
1567 | orig = adev->msix_table[i]; | |
1568 | } | |
1569 | ||
1570 | memcpy((uint8_t *)adev->msix_table + addr, &val, size); | |
1571 | ||
1572 | if (ctrl & PCI_MSIX_FLAGS_ENABLE) { | |
1573 | MSIXTableEntry *entry = &adev->msix_table[i]; | |
1574 | ||
1575 | if (!assigned_dev_msix_masked(&orig) && | |
1576 | assigned_dev_msix_masked(entry)) { | |
1577 | /* | |
1578 | * Vector masked, disable it | |
1579 | * | |
1580 | * XXX It's not clear if we can or should actually attempt | |
1581 | * to mask or disable the interrupt. KVM doesn't have | |
1582 | * support for pending bits and kvm_assign_set_msix_entry | |
1583 | * doesn't modify the device hardware mask. Interrupts | |
1584 | * while masked are simply not injected to the guest, so | |
1585 | * are lost. Can we get away with always injecting an | |
1586 | * interrupt on unmask? | |
1587 | */ | |
1588 | } else if (assigned_dev_msix_masked(&orig) && | |
1589 | !assigned_dev_msix_masked(entry)) { | |
1590 | /* Vector unmasked */ | |
1591 | if (i >= adev->msi_virq_nr || adev->msi_virq[i] < 0) { | |
1592 | /* Previously unassigned vector, start from scratch */ | |
1593 | assigned_dev_update_msix(pdev); | |
1594 | return; | |
1595 | } else { | |
1596 | /* Update an existing, previously masked vector */ | |
1597 | MSIMessage msg; | |
1598 | int ret; | |
1599 | ||
1600 | msg.address = entry->addr_lo | | |
1601 | ((uint64_t)entry->addr_hi << 32); | |
1602 | msg.data = entry->data; | |
1603 | ||
1604 | ret = kvm_irqchip_update_msi_route(kvm_state, | |
1605 | adev->msi_virq[i], msg); | |
1606 | if (ret) { | |
1607 | error_report("Error updating irq routing entry (%d)", ret); | |
1608 | } | |
1609 | } | |
1610 | } | |
1611 | } | |
1612 | } | |
1613 | ||
1614 | static const MemoryRegionOps assigned_dev_msix_mmio_ops = { | |
1615 | .read = assigned_dev_msix_mmio_read, | |
1616 | .write = assigned_dev_msix_mmio_write, | |
1617 | .endianness = DEVICE_NATIVE_ENDIAN, | |
1618 | .valid = { | |
1619 | .min_access_size = 4, | |
1620 | .max_access_size = 8, | |
1621 | }, | |
1622 | .impl = { | |
1623 | .min_access_size = 4, | |
1624 | .max_access_size = 8, | |
1625 | }, | |
1626 | }; | |
1627 | ||
1628 | static void assigned_dev_msix_reset(AssignedDevice *dev) | |
1629 | { | |
1630 | MSIXTableEntry *entry; | |
1631 | int i; | |
1632 | ||
1633 | if (!dev->msix_table) { | |
1634 | return; | |
1635 | } | |
1636 | ||
1637 | memset(dev->msix_table, 0, MSIX_PAGE_SIZE); | |
1638 | ||
1639 | for (i = 0, entry = dev->msix_table; i < dev->msix_max; i++, entry++) { | |
1640 | entry->ctrl = cpu_to_le32(0x1); /* Masked */ | |
1641 | } | |
1642 | } | |
1643 | ||
1644 | static int assigned_dev_register_msix_mmio(AssignedDevice *dev) | |
1645 | { | |
1646 | dev->msix_table = mmap(NULL, MSIX_PAGE_SIZE, PROT_READ|PROT_WRITE, | |
1647 | MAP_ANONYMOUS|MAP_PRIVATE, 0, 0); | |
1648 | if (dev->msix_table == MAP_FAILED) { | |
1649 | error_report("fail allocate msix_table! %s", strerror(errno)); | |
1650 | return -EFAULT; | |
1651 | } | |
1652 | ||
1653 | assigned_dev_msix_reset(dev); | |
1654 | ||
1655 | memory_region_init_io(&dev->mmio, OBJECT(dev), &assigned_dev_msix_mmio_ops, | |
1656 | dev, "assigned-dev-msix", MSIX_PAGE_SIZE); | |
1657 | return 0; | |
1658 | } | |
1659 | ||
1660 | static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev) | |
1661 | { | |
1662 | if (!dev->msix_table) { | |
1663 | return; | |
1664 | } | |
1665 | ||
1666 | memory_region_destroy(&dev->mmio); | |
1667 | ||
1668 | if (munmap(dev->msix_table, MSIX_PAGE_SIZE) == -1) { | |
1669 | error_report("error unmapping msix_table! %s", strerror(errno)); | |
1670 | } | |
1671 | dev->msix_table = NULL; | |
1672 | } | |
1673 | ||
1674 | static const VMStateDescription vmstate_assigned_device = { | |
1675 | .name = "pci-assign", | |
1676 | .unmigratable = 1, | |
1677 | }; | |
1678 | ||
1679 | static void reset_assigned_device(DeviceState *dev) | |
1680 | { | |
1681 | PCIDevice *pci_dev = DO_UPCAST(PCIDevice, qdev, dev); | |
1682 | AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1683 | char reset_file[64]; | |
1684 | const char reset[] = "1"; | |
1685 | int fd, ret; | |
1686 | ||
1687 | /* | |
1688 | * If a guest is reset without being shutdown, MSI/MSI-X can still | |
1689 | * be running. We want to return the device to a known state on | |
1690 | * reset, so disable those here. We especially do not want MSI-X | |
1691 | * enabled since it lives in MMIO space, which is about to get | |
1692 | * disabled. | |
1693 | */ | |
1694 | if (adev->assigned_irq_type == ASSIGNED_IRQ_MSIX) { | |
1695 | uint16_t ctrl = pci_get_word(pci_dev->config + | |
1696 | pci_dev->msix_cap + PCI_MSIX_FLAGS); | |
1697 | ||
1698 | pci_set_word(pci_dev->config + pci_dev->msix_cap + PCI_MSIX_FLAGS, | |
1699 | ctrl & ~PCI_MSIX_FLAGS_ENABLE); | |
1700 | assigned_dev_update_msix(pci_dev); | |
1701 | } else if (adev->assigned_irq_type == ASSIGNED_IRQ_MSI) { | |
1702 | uint8_t ctrl = pci_get_byte(pci_dev->config + | |
1703 | pci_dev->msi_cap + PCI_MSI_FLAGS); | |
1704 | ||
1705 | pci_set_byte(pci_dev->config + pci_dev->msi_cap + PCI_MSI_FLAGS, | |
1706 | ctrl & ~PCI_MSI_FLAGS_ENABLE); | |
1707 | assigned_dev_update_msi(pci_dev); | |
1708 | } | |
1709 | ||
1710 | snprintf(reset_file, sizeof(reset_file), | |
1711 | "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/reset", | |
1712 | adev->host.domain, adev->host.bus, adev->host.slot, | |
1713 | adev->host.function); | |
1714 | ||
1715 | /* | |
1716 | * Issue a device reset via pci-sysfs. Note that we use write(2) here | |
1717 | * and ignore the return value because some kernels have a bug that | |
1718 | * returns 0 rather than bytes written on success, sending us into an | |
1719 | * infinite retry loop using other write mechanisms. | |
1720 | */ | |
1721 | fd = open(reset_file, O_WRONLY); | |
1722 | if (fd != -1) { | |
1723 | ret = write(fd, reset, strlen(reset)); | |
1724 | (void)ret; | |
1725 | close(fd); | |
1726 | } | |
1727 | ||
1728 | /* | |
1729 | * When a 0 is written to the bus master register, the device is logically | |
1730 | * disconnected from the PCI bus. This avoids further DMA transfers. | |
1731 | */ | |
1732 | assigned_dev_pci_write_config(pci_dev, PCI_COMMAND, 0, 1); | |
1733 | } | |
1734 | ||
1735 | static int assigned_initfn(struct PCIDevice *pci_dev) | |
1736 | { | |
1737 | AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1738 | uint8_t e_intx; | |
1739 | int r; | |
1740 | ||
1741 | if (!kvm_enabled()) { | |
1742 | error_report("pci-assign: error: requires KVM support"); | |
1743 | return -1; | |
1744 | } | |
1745 | ||
1746 | if (!dev->host.domain && !dev->host.bus && !dev->host.slot && | |
1747 | !dev->host.function) { | |
1748 | error_report("pci-assign: error: no host device specified"); | |
1749 | return -1; | |
1750 | } | |
1751 | ||
1752 | /* | |
1753 | * Set up basic config space access control. Will be further refined during | |
1754 | * device initialization. | |
1755 | */ | |
1756 | assigned_dev_emulate_config_read(dev, 0, PCI_CONFIG_SPACE_SIZE); | |
1757 | assigned_dev_direct_config_read(dev, PCI_STATUS, 2); | |
1758 | assigned_dev_direct_config_read(dev, PCI_REVISION_ID, 1); | |
1759 | assigned_dev_direct_config_read(dev, PCI_CLASS_PROG, 3); | |
1760 | assigned_dev_direct_config_read(dev, PCI_CACHE_LINE_SIZE, 1); | |
1761 | assigned_dev_direct_config_read(dev, PCI_LATENCY_TIMER, 1); | |
1762 | assigned_dev_direct_config_read(dev, PCI_BIST, 1); | |
1763 | assigned_dev_direct_config_read(dev, PCI_CARDBUS_CIS, 4); | |
1764 | assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_VENDOR_ID, 2); | |
1765 | assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_ID, 2); | |
1766 | assigned_dev_direct_config_read(dev, PCI_CAPABILITY_LIST + 1, 7); | |
1767 | assigned_dev_direct_config_read(dev, PCI_MIN_GNT, 1); | |
1768 | assigned_dev_direct_config_read(dev, PCI_MAX_LAT, 1); | |
1769 | memcpy(dev->emulate_config_write, dev->emulate_config_read, | |
1770 | sizeof(dev->emulate_config_read)); | |
1771 | ||
1772 | if (get_real_device(dev, dev->host.domain, dev->host.bus, | |
1773 | dev->host.slot, dev->host.function)) { | |
1774 | error_report("pci-assign: Error: Couldn't get real device (%s)!", | |
1775 | dev->dev.qdev.id); | |
1776 | goto out; | |
1777 | } | |
1778 | ||
1779 | if (assigned_device_pci_cap_init(pci_dev) < 0) { | |
1780 | goto out; | |
1781 | } | |
1782 | ||
1783 | /* intercept MSI-X entry page in the MMIO */ | |
1784 | if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) { | |
1785 | if (assigned_dev_register_msix_mmio(dev)) { | |
1786 | goto out; | |
1787 | } | |
1788 | } | |
1789 | ||
1790 | /* handle real device's MMIO/PIO BARs */ | |
1791 | if (assigned_dev_register_regions(dev->real_device.regions, | |
1792 | dev->real_device.region_number, | |
1793 | dev)) { | |
1794 | goto out; | |
1795 | } | |
1796 | ||
1797 | /* handle interrupt routing */ | |
1798 | e_intx = dev->dev.config[PCI_INTERRUPT_PIN] - 1; | |
1799 | dev->intpin = e_intx; | |
1800 | dev->intx_route.mode = PCI_INTX_DISABLED; | |
1801 | dev->intx_route.irq = -1; | |
1802 | ||
1803 | /* assign device to guest */ | |
1804 | r = assign_device(dev); | |
1805 | if (r < 0) { | |
1806 | goto out; | |
1807 | } | |
1808 | ||
1809 | /* assign legacy INTx to the device */ | |
1810 | r = assign_intx(dev); | |
1811 | if (r < 0) { | |
1812 | goto assigned_out; | |
1813 | } | |
1814 | ||
1815 | assigned_dev_load_option_rom(dev); | |
1816 | ||
1817 | add_boot_device_path(dev->bootindex, &pci_dev->qdev, NULL); | |
1818 | ||
1819 | return 0; | |
1820 | ||
1821 | assigned_out: | |
1822 | deassign_device(dev); | |
1823 | out: | |
1824 | free_assigned_device(dev); | |
1825 | return -1; | |
1826 | } | |
1827 | ||
1828 | static void assigned_exitfn(struct PCIDevice *pci_dev) | |
1829 | { | |
1830 | AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev); | |
1831 | ||
1832 | deassign_device(dev); | |
1833 | free_assigned_device(dev); | |
1834 | } | |
1835 | ||
1836 | static Property assigned_dev_properties[] = { | |
1837 | DEFINE_PROP_PCI_HOST_DEVADDR("host", AssignedDevice, host), | |
1838 | DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features, | |
1839 | ASSIGNED_DEVICE_PREFER_MSI_BIT, false), | |
1840 | DEFINE_PROP_BIT("share_intx", AssignedDevice, features, | |
1841 | ASSIGNED_DEVICE_SHARE_INTX_BIT, true), | |
1842 | DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1), | |
1843 | DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name), | |
1844 | DEFINE_PROP_END_OF_LIST(), | |
1845 | }; | |
1846 | ||
1847 | static void assign_class_init(ObjectClass *klass, void *data) | |
1848 | { | |
1849 | PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); | |
1850 | DeviceClass *dc = DEVICE_CLASS(klass); | |
1851 | ||
1852 | k->init = assigned_initfn; | |
1853 | k->exit = assigned_exitfn; | |
1854 | k->config_read = assigned_dev_pci_read_config; | |
1855 | k->config_write = assigned_dev_pci_write_config; | |
1856 | dc->props = assigned_dev_properties; | |
1857 | dc->vmsd = &vmstate_assigned_device; | |
1858 | dc->reset = reset_assigned_device; | |
1859 | dc->desc = "KVM-based PCI passthrough"; | |
1860 | } | |
1861 | ||
1862 | static const TypeInfo assign_info = { | |
1863 | .name = "kvm-pci-assign", | |
1864 | .parent = TYPE_PCI_DEVICE, | |
1865 | .instance_size = sizeof(AssignedDevice), | |
1866 | .class_init = assign_class_init, | |
1867 | }; | |
1868 | ||
1869 | static void assign_register_types(void) | |
1870 | { | |
1871 | type_register_static(&assign_info); | |
1872 | } | |
1873 | ||
1874 | type_init(assign_register_types) | |
1875 | ||
1876 | /* | |
1877 | * Scan the assigned devices for the devices that have an option ROM, and then | |
1878 | * load the corresponding ROM data to RAM. If an error occurs while loading an | |
1879 | * option ROM, we just ignore that option ROM and continue with the next one. | |
1880 | */ | |
1881 | static void assigned_dev_load_option_rom(AssignedDevice *dev) | |
1882 | { | |
1883 | char name[32], rom_file[64]; | |
1884 | FILE *fp; | |
1885 | uint8_t val; | |
1886 | struct stat st; | |
1887 | void *ptr; | |
1888 | ||
1889 | /* If loading ROM from file, pci handles it */ | |
1890 | if (dev->dev.romfile || !dev->dev.rom_bar) { | |
1891 | return; | |
1892 | } | |
1893 | ||
1894 | snprintf(rom_file, sizeof(rom_file), | |
1895 | "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom", | |
1896 | dev->host.domain, dev->host.bus, dev->host.slot, | |
1897 | dev->host.function); | |
1898 | ||
1899 | if (stat(rom_file, &st)) { | |
1900 | return; | |
1901 | } | |
1902 | ||
1903 | if (access(rom_file, F_OK)) { | |
1904 | error_report("pci-assign: Insufficient privileges for %s", rom_file); | |
1905 | return; | |
1906 | } | |
1907 | ||
1908 | /* Write "1" to the ROM file to enable it */ | |
1909 | fp = fopen(rom_file, "r+"); | |
1910 | if (fp == NULL) { | |
1911 | return; | |
1912 | } | |
1913 | val = 1; | |
1914 | if (fwrite(&val, 1, 1, fp) != 1) { | |
1915 | goto close_rom; | |
1916 | } | |
1917 | fseek(fp, 0, SEEK_SET); | |
1918 | ||
1919 | snprintf(name, sizeof(name), "%s.rom", | |
1920 | object_get_typename(OBJECT(dev))); | |
1921 | memory_region_init_ram(&dev->dev.rom, OBJECT(dev), name, st.st_size); | |
1922 | vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev); | |
1923 | ptr = memory_region_get_ram_ptr(&dev->dev.rom); | |
1924 | memset(ptr, 0xff, st.st_size); | |
1925 | ||
1926 | if (!fread(ptr, 1, st.st_size, fp)) { | |
1927 | error_report("pci-assign: Cannot read from host %s", rom_file); | |
1928 | error_printf("Device option ROM contents are probably invalid " | |
1929 | "(check dmesg).\nSkip option ROM probe with rombar=0, " | |
1930 | "or load from file with romfile=\n"); | |
1931 | memory_region_destroy(&dev->dev.rom); | |
1932 | goto close_rom; | |
1933 | } | |
1934 | ||
1935 | pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom); | |
1936 | dev->dev.has_rom = true; | |
1937 | close_rom: | |
1938 | /* Write "0" to disable ROM */ | |
1939 | fseek(fp, 0, SEEK_SET); | |
1940 | val = 0; | |
1941 | if (!fwrite(&val, 1, 1, fp)) { | |
1942 | DEBUG("%s\n", "Failed to disable pci-sysfs rom file"); | |
1943 | } | |
1944 | fclose(fp); | |
1945 | } |