4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
13 * Contributions after 2012-01-13 are licensed under the terms of the
14 * GNU GPL, version 2 or (at your option) any later version.
17 #include "qemu/osdep.h"
18 #include "hw/pci/msi.h"
19 #include "hw/pci/msix.h"
20 #include "hw/pci/pci.h"
21 #include "hw/xen/xen.h"
22 #include "sysemu/xen.h"
23 #include "migration/qemu-file-types.h"
24 #include "migration/vmstate.h"
25 #include "qemu/range.h"
26 #include "qapi/error.h"
29 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
30 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
31 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
32 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
34 static MSIMessage
msix_prepare_message(PCIDevice
*dev
, unsigned vector
)
36 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
39 msg
.address
= pci_get_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
40 msg
.data
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_DATA
);
44 MSIMessage
msix_get_message(PCIDevice
*dev
, unsigned vector
)
46 return dev
->msix_prepare_message(dev
, vector
);
50 * Special API for POWER to configure the vectors through
51 * a side channel. Should never be used by devices.
53 void msix_set_message(PCIDevice
*dev
, int vector
, struct MSIMessage msg
)
55 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
57 pci_set_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
, msg
.address
);
58 pci_set_long(table_entry
+ PCI_MSIX_ENTRY_DATA
, msg
.data
);
59 table_entry
[PCI_MSIX_ENTRY_VECTOR_CTRL
] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT
;
62 static uint8_t msix_pending_mask(int vector
)
64 return 1 << (vector
% 8);
67 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
69 return dev
->msix_pba
+ vector
/ 8;
72 static int msix_is_pending(PCIDevice
*dev
, int vector
)
74 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
77 void msix_set_pending(PCIDevice
*dev
, unsigned int vector
)
79 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
82 void msix_clr_pending(PCIDevice
*dev
, int vector
)
84 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
87 static bool msix_vector_masked(PCIDevice
*dev
, unsigned int vector
, bool fmask
)
89 unsigned offset
= vector
* PCI_MSIX_ENTRY_SIZE
;
90 uint8_t *data
= &dev
->msix_table
[offset
+ PCI_MSIX_ENTRY_DATA
];
91 /* MSIs on Xen can be remapped into pirqs. In those cases, masking
92 * and unmasking go through the PV evtchn path. */
93 if (xen_enabled() && xen_is_pirq_msi(pci_get_long(data
))) {
96 return fmask
|| dev
->msix_table
[offset
+ PCI_MSIX_ENTRY_VECTOR_CTRL
] &
97 PCI_MSIX_ENTRY_CTRL_MASKBIT
;
100 bool msix_is_masked(PCIDevice
*dev
, unsigned int vector
)
102 return msix_vector_masked(dev
, vector
, dev
->msix_function_masked
);
105 static void msix_fire_vector_notifier(PCIDevice
*dev
,
106 unsigned int vector
, bool is_masked
)
111 if (!dev
->msix_vector_use_notifier
) {
115 dev
->msix_vector_release_notifier(dev
, vector
);
117 msg
= msix_get_message(dev
, vector
);
118 ret
= dev
->msix_vector_use_notifier(dev
, vector
, msg
);
123 static void msix_handle_mask_update(PCIDevice
*dev
, int vector
, bool was_masked
)
125 bool is_masked
= msix_is_masked(dev
, vector
);
127 if (is_masked
== was_masked
) {
131 msix_fire_vector_notifier(dev
, vector
, is_masked
);
133 if (!is_masked
&& msix_is_pending(dev
, vector
)) {
134 msix_clr_pending(dev
, vector
);
135 msix_notify(dev
, vector
);
139 void msix_set_mask(PCIDevice
*dev
, int vector
, bool mask
)
144 assert(vector
< dev
->msix_entries_nr
);
146 offset
= vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
148 was_masked
= msix_is_masked(dev
, vector
);
151 dev
->msix_table
[offset
] |= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
153 dev
->msix_table
[offset
] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT
;
156 msix_handle_mask_update(dev
, vector
, was_masked
);
159 static bool msix_masked(PCIDevice
*dev
)
161 return dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] & MSIX_MASKALL_MASK
;
164 static void msix_update_function_masked(PCIDevice
*dev
)
166 dev
->msix_function_masked
= !msix_enabled(dev
) || msix_masked(dev
);
169 /* Handle MSI-X capability config write. */
170 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
171 uint32_t val
, int len
)
173 unsigned enable_pos
= dev
->msix_cap
+ MSIX_CONTROL_OFFSET
;
177 if (!msix_present(dev
) || !range_covers_byte(addr
, len
, enable_pos
)) {
181 trace_msix_write_config(dev
->name
, msix_enabled(dev
), msix_masked(dev
));
183 was_masked
= dev
->msix_function_masked
;
184 msix_update_function_masked(dev
);
186 if (!msix_enabled(dev
)) {
190 pci_device_deassert_intx(dev
);
192 if (dev
->msix_function_masked
== was_masked
) {
196 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
197 msix_handle_mask_update(dev
, vector
,
198 msix_vector_masked(dev
, vector
, was_masked
));
202 static uint64_t msix_table_mmio_read(void *opaque
, hwaddr addr
,
205 PCIDevice
*dev
= opaque
;
207 assert(addr
+ size
<= dev
->msix_entries_nr
* PCI_MSIX_ENTRY_SIZE
);
208 return pci_get_long(dev
->msix_table
+ addr
);
211 static void msix_table_mmio_write(void *opaque
, hwaddr addr
,
212 uint64_t val
, unsigned size
)
214 PCIDevice
*dev
= opaque
;
215 int vector
= addr
/ PCI_MSIX_ENTRY_SIZE
;
218 assert(addr
+ size
<= dev
->msix_entries_nr
* PCI_MSIX_ENTRY_SIZE
);
220 was_masked
= msix_is_masked(dev
, vector
);
221 pci_set_long(dev
->msix_table
+ addr
, val
);
222 msix_handle_mask_update(dev
, vector
, was_masked
);
225 static const MemoryRegionOps msix_table_mmio_ops
= {
226 .read
= msix_table_mmio_read
,
227 .write
= msix_table_mmio_write
,
228 .endianness
= DEVICE_LITTLE_ENDIAN
,
230 .min_access_size
= 4,
231 .max_access_size
= 8,
234 .max_access_size
= 4,
238 static uint64_t msix_pba_mmio_read(void *opaque
, hwaddr addr
,
241 PCIDevice
*dev
= opaque
;
242 if (dev
->msix_vector_poll_notifier
) {
243 unsigned vector_start
= addr
* 8;
244 unsigned vector_end
= MIN(addr
+ size
* 8, dev
->msix_entries_nr
);
245 dev
->msix_vector_poll_notifier(dev
, vector_start
, vector_end
);
248 return pci_get_long(dev
->msix_pba
+ addr
);
251 static void msix_pba_mmio_write(void *opaque
, hwaddr addr
,
252 uint64_t val
, unsigned size
)
256 static const MemoryRegionOps msix_pba_mmio_ops
= {
257 .read
= msix_pba_mmio_read
,
258 .write
= msix_pba_mmio_write
,
259 .endianness
= DEVICE_LITTLE_ENDIAN
,
261 .min_access_size
= 4,
262 .max_access_size
= 8,
265 .max_access_size
= 4,
269 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
273 for (vector
= 0; vector
< nentries
; ++vector
) {
275 vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
276 bool was_masked
= msix_is_masked(dev
, vector
);
278 dev
->msix_table
[offset
] |= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
279 msix_handle_mask_update(dev
, vector
, was_masked
);
284 * Make PCI device @dev MSI-X capable
285 * @nentries is the max number of MSI-X vectors that the device support.
286 * @table_bar is the MemoryRegion that MSI-X table structure resides.
287 * @table_bar_nr is number of base address register corresponding to @table_bar.
288 * @table_offset indicates the offset that the MSI-X table structure starts with
290 * @pba_bar is the MemoryRegion that the Pending Bit Array structure resides.
291 * @pba_bar_nr is number of base address register corresponding to @pba_bar.
292 * @pba_offset indicates the offset that the Pending Bit Array structure
293 * starts with in @pba_bar.
294 * Non-zero @cap_pos puts capability MSI-X at that offset in PCI config space.
295 * @errp is for returning errors.
297 * Return 0 on success; set @errp and return -errno on error:
298 * -ENOTSUP means lacking msi support for a msi-capable platform.
299 * -EINVAL means capability overlap, happens when @cap_pos is non-zero,
300 * also means a programming error, except device assignment, which can check
301 * if a real HW is broken.
303 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
304 MemoryRegion
*table_bar
, uint8_t table_bar_nr
,
305 unsigned table_offset
, MemoryRegion
*pba_bar
,
306 uint8_t pba_bar_nr
, unsigned pba_offset
, uint8_t cap_pos
,
310 unsigned table_size
, pba_size
;
313 /* Nothing to do if MSI is not supported by interrupt controller */
314 if (!msi_nonbroken
) {
315 error_setg(errp
, "MSI-X is not supported by interrupt controller");
319 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
320 error_setg(errp
, "The number of MSI-X vectors is invalid");
324 table_size
= nentries
* PCI_MSIX_ENTRY_SIZE
;
325 pba_size
= QEMU_ALIGN_UP(nentries
, 64) / 8;
327 /* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
328 if ((table_bar_nr
== pba_bar_nr
&&
329 ranges_overlap(table_offset
, table_size
, pba_offset
, pba_size
)) ||
330 table_offset
+ table_size
> memory_region_size(table_bar
) ||
331 pba_offset
+ pba_size
> memory_region_size(pba_bar
) ||
332 (table_offset
| pba_offset
) & PCI_MSIX_FLAGS_BIRMASK
) {
333 error_setg(errp
, "table & pba overlap, or they don't fit in BARs,"
338 cap
= pci_add_capability(dev
, PCI_CAP_ID_MSIX
,
339 cap_pos
, MSIX_CAP_LENGTH
, errp
);
345 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
346 config
= dev
->config
+ cap
;
348 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
349 dev
->msix_entries_nr
= nentries
;
350 dev
->msix_function_masked
= true;
352 pci_set_long(config
+ PCI_MSIX_TABLE
, table_offset
| table_bar_nr
);
353 pci_set_long(config
+ PCI_MSIX_PBA
, pba_offset
| pba_bar_nr
);
355 /* Make flags bit writable. */
356 dev
->wmask
[cap
+ MSIX_CONTROL_OFFSET
] |= MSIX_ENABLE_MASK
|
359 dev
->msix_table
= g_malloc0(table_size
);
360 dev
->msix_pba
= g_malloc0(pba_size
);
361 dev
->msix_entry_used
= g_malloc0(nentries
* sizeof *dev
->msix_entry_used
);
363 msix_mask_all(dev
, nentries
);
365 memory_region_init_io(&dev
->msix_table_mmio
, OBJECT(dev
), &msix_table_mmio_ops
, dev
,
366 "msix-table", table_size
);
367 memory_region_add_subregion(table_bar
, table_offset
, &dev
->msix_table_mmio
);
368 memory_region_init_io(&dev
->msix_pba_mmio
, OBJECT(dev
), &msix_pba_mmio_ops
, dev
,
369 "msix-pba", pba_size
);
370 memory_region_add_subregion(pba_bar
, pba_offset
, &dev
->msix_pba_mmio
);
372 dev
->msix_prepare_message
= msix_prepare_message
;
377 int msix_init_exclusive_bar(PCIDevice
*dev
, unsigned short nentries
,
378 uint8_t bar_nr
, Error
**errp
)
382 uint32_t bar_size
= 4096;
383 uint32_t bar_pba_offset
= bar_size
/ 2;
384 uint32_t bar_pba_size
= QEMU_ALIGN_UP(nentries
, 64) / 8;
387 * Migration compatibility dictates that this remains a 4k
388 * BAR with the vector table in the lower half and PBA in
389 * the upper half for nentries which is lower or equal to 128.
390 * No need to care about using more than 65 entries for legacy
391 * machine types who has at most 64 queues.
393 if (nentries
* PCI_MSIX_ENTRY_SIZE
> bar_pba_offset
) {
394 bar_pba_offset
= nentries
* PCI_MSIX_ENTRY_SIZE
;
397 if (bar_pba_offset
+ bar_pba_size
> 4096) {
398 bar_size
= bar_pba_offset
+ bar_pba_size
;
401 bar_size
= pow2ceil(bar_size
);
403 name
= g_strdup_printf("%s-msix", dev
->name
);
404 memory_region_init(&dev
->msix_exclusive_bar
, OBJECT(dev
), name
, bar_size
);
407 ret
= msix_init(dev
, nentries
, &dev
->msix_exclusive_bar
, bar_nr
,
408 0, &dev
->msix_exclusive_bar
,
409 bar_nr
, bar_pba_offset
,
415 pci_register_bar(dev
, bar_nr
, PCI_BASE_ADDRESS_SPACE_MEMORY
,
416 &dev
->msix_exclusive_bar
);
421 static void msix_free_irq_entries(PCIDevice
*dev
)
425 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
426 dev
->msix_entry_used
[vector
] = 0;
427 msix_clr_pending(dev
, vector
);
431 static void msix_clear_all_vectors(PCIDevice
*dev
)
435 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
436 msix_clr_pending(dev
, vector
);
440 /* Clean up resources for the device. */
441 void msix_uninit(PCIDevice
*dev
, MemoryRegion
*table_bar
, MemoryRegion
*pba_bar
)
443 if (!msix_present(dev
)) {
446 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
448 msix_free_irq_entries(dev
);
449 dev
->msix_entries_nr
= 0;
450 memory_region_del_subregion(pba_bar
, &dev
->msix_pba_mmio
);
451 g_free(dev
->msix_pba
);
452 dev
->msix_pba
= NULL
;
453 memory_region_del_subregion(table_bar
, &dev
->msix_table_mmio
);
454 g_free(dev
->msix_table
);
455 dev
->msix_table
= NULL
;
456 g_free(dev
->msix_entry_used
);
457 dev
->msix_entry_used
= NULL
;
458 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
459 dev
->msix_prepare_message
= NULL
;
462 void msix_uninit_exclusive_bar(PCIDevice
*dev
)
464 if (msix_present(dev
)) {
465 msix_uninit(dev
, &dev
->msix_exclusive_bar
, &dev
->msix_exclusive_bar
);
469 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
471 unsigned n
= dev
->msix_entries_nr
;
473 if (!msix_present(dev
)) {
477 qemu_put_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
478 qemu_put_buffer(f
, dev
->msix_pba
, DIV_ROUND_UP(n
, 8));
481 /* Should be called after restoring the config space. */
482 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
484 unsigned n
= dev
->msix_entries_nr
;
487 if (!msix_present(dev
)) {
491 msix_clear_all_vectors(dev
);
492 qemu_get_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
493 qemu_get_buffer(f
, dev
->msix_pba
, DIV_ROUND_UP(n
, 8));
494 msix_update_function_masked(dev
);
496 for (vector
= 0; vector
< n
; vector
++) {
497 msix_handle_mask_update(dev
, vector
, true);
501 /* Does device support MSI-X? */
502 int msix_present(PCIDevice
*dev
)
504 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
507 /* Is MSI-X enabled? */
508 int msix_enabled(PCIDevice
*dev
)
510 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
511 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
515 /* Send an MSI-X message */
516 void msix_notify(PCIDevice
*dev
, unsigned vector
)
520 assert(vector
< dev
->msix_entries_nr
);
522 if (!dev
->msix_entry_used
[vector
]) {
526 if (msix_is_masked(dev
, vector
)) {
527 msix_set_pending(dev
, vector
);
531 msg
= msix_get_message(dev
, vector
);
533 msi_send_message(dev
, msg
);
536 void msix_reset(PCIDevice
*dev
)
538 if (!msix_present(dev
)) {
541 msix_clear_all_vectors(dev
);
542 dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &=
543 ~dev
->wmask
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
];
544 memset(dev
->msix_table
, 0, dev
->msix_entries_nr
* PCI_MSIX_ENTRY_SIZE
);
545 memset(dev
->msix_pba
, 0, QEMU_ALIGN_UP(dev
->msix_entries_nr
, 64) / 8);
546 msix_mask_all(dev
, dev
->msix_entries_nr
);
549 /* PCI spec suggests that devices make it possible for software to configure
550 * less vectors than supported by the device, but does not specify a standard
551 * mechanism for devices to do so.
553 * We support this by asking devices to declare vectors software is going to
554 * actually use, and checking this on the notification path. Devices that
555 * don't want to follow the spec suggestion can declare all vectors as used. */
557 /* Mark vector as used. */
558 void msix_vector_use(PCIDevice
*dev
, unsigned vector
)
560 assert(vector
< dev
->msix_entries_nr
);
561 dev
->msix_entry_used
[vector
]++;
564 /* Mark vector as unused. */
565 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
567 assert(vector
< dev
->msix_entries_nr
);
568 if (!dev
->msix_entry_used
[vector
]) {
571 if (--dev
->msix_entry_used
[vector
]) {
574 msix_clr_pending(dev
, vector
);
577 void msix_unuse_all_vectors(PCIDevice
*dev
)
579 if (!msix_present(dev
)) {
582 msix_free_irq_entries(dev
);
585 unsigned int msix_nr_vectors_allocated(const PCIDevice
*dev
)
587 return dev
->msix_entries_nr
;
590 static int msix_set_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
594 if (msix_is_masked(dev
, vector
)) {
597 msg
= msix_get_message(dev
, vector
);
598 return dev
->msix_vector_use_notifier(dev
, vector
, msg
);
601 static void msix_unset_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
603 if (msix_is_masked(dev
, vector
)) {
606 dev
->msix_vector_release_notifier(dev
, vector
);
609 int msix_set_vector_notifiers(PCIDevice
*dev
,
610 MSIVectorUseNotifier use_notifier
,
611 MSIVectorReleaseNotifier release_notifier
,
612 MSIVectorPollNotifier poll_notifier
)
616 assert(use_notifier
&& release_notifier
);
618 dev
->msix_vector_use_notifier
= use_notifier
;
619 dev
->msix_vector_release_notifier
= release_notifier
;
620 dev
->msix_vector_poll_notifier
= poll_notifier
;
622 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
623 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
624 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
625 ret
= msix_set_notifier_for_vector(dev
, vector
);
631 if (dev
->msix_vector_poll_notifier
) {
632 dev
->msix_vector_poll_notifier(dev
, 0, dev
->msix_entries_nr
);
637 while (--vector
>= 0) {
638 msix_unset_notifier_for_vector(dev
, vector
);
640 dev
->msix_vector_use_notifier
= NULL
;
641 dev
->msix_vector_release_notifier
= NULL
;
645 void msix_unset_vector_notifiers(PCIDevice
*dev
)
649 assert(dev
->msix_vector_use_notifier
&&
650 dev
->msix_vector_release_notifier
);
652 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
653 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
654 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
655 msix_unset_notifier_for_vector(dev
, vector
);
658 dev
->msix_vector_use_notifier
= NULL
;
659 dev
->msix_vector_release_notifier
= NULL
;
660 dev
->msix_vector_poll_notifier
= NULL
;
663 static int put_msix_state(QEMUFile
*f
, void *pv
, size_t size
,
664 const VMStateField
*field
, JSONWriter
*vmdesc
)
671 static int get_msix_state(QEMUFile
*f
, void *pv
, size_t size
,
672 const VMStateField
*field
)
678 static VMStateInfo vmstate_info_msix
= {
679 .name
= "msix state",
680 .get
= get_msix_state
,
681 .put
= put_msix_state
,
684 const VMStateDescription vmstate_msix
= {
686 .fields
= (VMStateField
[]) {
690 .field_exists
= NULL
,
691 .size
= 0, /* ouch */
692 .info
= &vmstate_info_msix
,
696 VMSTATE_END_OF_LIST()