4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
13 * Contributions after 2012-01-13 are licensed under the terms of the
14 * GNU GPL, version 2 or (at your option) any later version.
18 #include "hw/pci/msi.h"
19 #include "hw/pci/msix.h"
20 #include "hw/pci/pci.h"
23 #define MSIX_CAP_LENGTH 12
25 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
26 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
27 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
28 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
30 static MSIMessage
msix_get_message(PCIDevice
*dev
, unsigned vector
)
32 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
35 msg
.address
= pci_get_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
36 msg
.data
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_DATA
);
41 * Special API for POWER to configure the vectors through
42 * a side channel. Should never be used by devices.
44 void msix_set_message(PCIDevice
*dev
, int vector
, struct MSIMessage msg
)
46 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
48 pci_set_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
, msg
.address
);
49 pci_set_long(table_entry
+ PCI_MSIX_ENTRY_DATA
, msg
.data
);
50 table_entry
[PCI_MSIX_ENTRY_VECTOR_CTRL
] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT
;
53 static uint8_t msix_pending_mask(int vector
)
55 return 1 << (vector
% 8);
58 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
60 return dev
->msix_pba
+ vector
/ 8;
63 static int msix_is_pending(PCIDevice
*dev
, int vector
)
65 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
68 static void msix_set_pending(PCIDevice
*dev
, int vector
)
70 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
73 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
75 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
78 static bool msix_vector_masked(PCIDevice
*dev
, int vector
, bool fmask
)
80 unsigned offset
= vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
81 return fmask
|| dev
->msix_table
[offset
] & PCI_MSIX_ENTRY_CTRL_MASKBIT
;
84 static bool msix_is_masked(PCIDevice
*dev
, int vector
)
86 return msix_vector_masked(dev
, vector
, dev
->msix_function_masked
);
89 static void msix_fire_vector_notifier(PCIDevice
*dev
,
90 unsigned int vector
, bool is_masked
)
95 if (!dev
->msix_vector_use_notifier
) {
99 dev
->msix_vector_release_notifier(dev
, vector
);
101 msg
= msix_get_message(dev
, vector
);
102 ret
= dev
->msix_vector_use_notifier(dev
, vector
, msg
);
107 static void msix_handle_mask_update(PCIDevice
*dev
, int vector
, bool was_masked
)
109 bool is_masked
= msix_is_masked(dev
, vector
);
111 if (is_masked
== was_masked
) {
115 msix_fire_vector_notifier(dev
, vector
, is_masked
);
117 if (!is_masked
&& msix_is_pending(dev
, vector
)) {
118 msix_clr_pending(dev
, vector
);
119 msix_notify(dev
, vector
);
123 static void msix_update_function_masked(PCIDevice
*dev
)
125 dev
->msix_function_masked
= !msix_enabled(dev
) ||
126 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] & MSIX_MASKALL_MASK
);
129 /* Handle MSI-X capability config write. */
130 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
131 uint32_t val
, int len
)
133 unsigned enable_pos
= dev
->msix_cap
+ MSIX_CONTROL_OFFSET
;
137 if (!msix_present(dev
) || !range_covers_byte(addr
, len
, enable_pos
)) {
141 was_masked
= dev
->msix_function_masked
;
142 msix_update_function_masked(dev
);
144 if (!msix_enabled(dev
)) {
148 pci_device_deassert_intx(dev
);
150 if (dev
->msix_function_masked
== was_masked
) {
154 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
155 msix_handle_mask_update(dev
, vector
,
156 msix_vector_masked(dev
, vector
, was_masked
));
160 static uint64_t msix_table_mmio_read(void *opaque
, hwaddr addr
,
163 PCIDevice
*dev
= opaque
;
165 return pci_get_long(dev
->msix_table
+ addr
);
168 static void msix_table_mmio_write(void *opaque
, hwaddr addr
,
169 uint64_t val
, unsigned size
)
171 PCIDevice
*dev
= opaque
;
172 int vector
= addr
/ PCI_MSIX_ENTRY_SIZE
;
175 was_masked
= msix_is_masked(dev
, vector
);
176 pci_set_long(dev
->msix_table
+ addr
, val
);
177 msix_handle_mask_update(dev
, vector
, was_masked
);
180 static const MemoryRegionOps msix_table_mmio_ops
= {
181 .read
= msix_table_mmio_read
,
182 .write
= msix_table_mmio_write
,
183 .endianness
= DEVICE_LITTLE_ENDIAN
,
185 .min_access_size
= 4,
186 .max_access_size
= 4,
190 static uint64_t msix_pba_mmio_read(void *opaque
, hwaddr addr
,
193 PCIDevice
*dev
= opaque
;
195 return pci_get_long(dev
->msix_pba
+ addr
);
198 static const MemoryRegionOps msix_pba_mmio_ops
= {
199 .read
= msix_pba_mmio_read
,
200 .endianness
= DEVICE_LITTLE_ENDIAN
,
202 .min_access_size
= 4,
203 .max_access_size
= 4,
207 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
211 for (vector
= 0; vector
< nentries
; ++vector
) {
213 vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
214 bool was_masked
= msix_is_masked(dev
, vector
);
216 dev
->msix_table
[offset
] |= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
217 msix_handle_mask_update(dev
, vector
, was_masked
);
221 /* Initialize the MSI-X structures */
222 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
223 MemoryRegion
*table_bar
, uint8_t table_bar_nr
,
224 unsigned table_offset
, MemoryRegion
*pba_bar
,
225 uint8_t pba_bar_nr
, unsigned pba_offset
, uint8_t cap_pos
)
228 unsigned table_size
, pba_size
;
231 /* Nothing to do if MSI is not supported by interrupt controller */
232 if (!msi_supported
) {
236 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
240 table_size
= nentries
* PCI_MSIX_ENTRY_SIZE
;
241 pba_size
= QEMU_ALIGN_UP(nentries
, 64) / 8;
243 /* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
244 if ((table_bar_nr
== pba_bar_nr
&&
245 ranges_overlap(table_offset
, table_size
, pba_offset
, pba_size
)) ||
246 table_offset
+ table_size
> memory_region_size(table_bar
) ||
247 pba_offset
+ pba_size
> memory_region_size(pba_bar
) ||
248 (table_offset
| pba_offset
) & PCI_MSIX_FLAGS_BIRMASK
) {
252 cap
= pci_add_capability(dev
, PCI_CAP_ID_MSIX
, cap_pos
, MSIX_CAP_LENGTH
);
258 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
259 config
= dev
->config
+ cap
;
261 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
262 dev
->msix_entries_nr
= nentries
;
263 dev
->msix_function_masked
= true;
265 pci_set_long(config
+ PCI_MSIX_TABLE
, table_offset
| table_bar_nr
);
266 pci_set_long(config
+ PCI_MSIX_PBA
, pba_offset
| pba_bar_nr
);
268 /* Make flags bit writable. */
269 dev
->wmask
[cap
+ MSIX_CONTROL_OFFSET
] |= MSIX_ENABLE_MASK
|
272 dev
->msix_table
= g_malloc0(table_size
);
273 dev
->msix_pba
= g_malloc0(pba_size
);
274 dev
->msix_entry_used
= g_malloc0(nentries
* sizeof *dev
->msix_entry_used
);
276 msix_mask_all(dev
, nentries
);
278 memory_region_init_io(&dev
->msix_table_mmio
, &msix_table_mmio_ops
, dev
,
279 "msix-table", table_size
);
280 memory_region_add_subregion(table_bar
, table_offset
, &dev
->msix_table_mmio
);
281 memory_region_init_io(&dev
->msix_pba_mmio
, &msix_pba_mmio_ops
, dev
,
282 "msix-pba", pba_size
);
283 memory_region_add_subregion(pba_bar
, pba_offset
, &dev
->msix_pba_mmio
);
288 int msix_init_exclusive_bar(PCIDevice
*dev
, unsigned short nentries
,
295 * Migration compatibility dictates that this remains a 4k
296 * BAR with the vector table in the lower half and PBA in
297 * the upper half. Do not use these elsewhere!
299 #define MSIX_EXCLUSIVE_BAR_SIZE 4096
300 #define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
301 #define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
302 #define MSIX_EXCLUSIVE_CAP_OFFSET 0
304 if (nentries
* PCI_MSIX_ENTRY_SIZE
> MSIX_EXCLUSIVE_BAR_PBA_OFFSET
) {
308 name
= g_strdup_printf("%s-msix", dev
->name
);
309 memory_region_init(&dev
->msix_exclusive_bar
, name
, MSIX_EXCLUSIVE_BAR_SIZE
);
312 ret
= msix_init(dev
, nentries
, &dev
->msix_exclusive_bar
, bar_nr
,
313 MSIX_EXCLUSIVE_BAR_TABLE_OFFSET
, &dev
->msix_exclusive_bar
,
314 bar_nr
, MSIX_EXCLUSIVE_BAR_PBA_OFFSET
,
315 MSIX_EXCLUSIVE_CAP_OFFSET
);
317 memory_region_destroy(&dev
->msix_exclusive_bar
);
321 pci_register_bar(dev
, bar_nr
, PCI_BASE_ADDRESS_SPACE_MEMORY
,
322 &dev
->msix_exclusive_bar
);
327 static void msix_free_irq_entries(PCIDevice
*dev
)
331 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
332 dev
->msix_entry_used
[vector
] = 0;
333 msix_clr_pending(dev
, vector
);
337 static void msix_clear_all_vectors(PCIDevice
*dev
)
341 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
342 msix_clr_pending(dev
, vector
);
346 /* Clean up resources for the device. */
347 void msix_uninit(PCIDevice
*dev
, MemoryRegion
*table_bar
, MemoryRegion
*pba_bar
)
349 if (!msix_present(dev
)) {
352 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
354 msix_free_irq_entries(dev
);
355 dev
->msix_entries_nr
= 0;
356 memory_region_del_subregion(pba_bar
, &dev
->msix_pba_mmio
);
357 memory_region_destroy(&dev
->msix_pba_mmio
);
358 g_free(dev
->msix_pba
);
359 dev
->msix_pba
= NULL
;
360 memory_region_del_subregion(table_bar
, &dev
->msix_table_mmio
);
361 memory_region_destroy(&dev
->msix_table_mmio
);
362 g_free(dev
->msix_table
);
363 dev
->msix_table
= NULL
;
364 g_free(dev
->msix_entry_used
);
365 dev
->msix_entry_used
= NULL
;
366 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
369 void msix_uninit_exclusive_bar(PCIDevice
*dev
)
371 if (msix_present(dev
)) {
372 msix_uninit(dev
, &dev
->msix_exclusive_bar
, &dev
->msix_exclusive_bar
);
373 memory_region_destroy(&dev
->msix_exclusive_bar
);
377 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
379 unsigned n
= dev
->msix_entries_nr
;
381 if (!msix_present(dev
)) {
385 qemu_put_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
386 qemu_put_buffer(f
, dev
->msix_pba
, (n
+ 7) / 8);
389 /* Should be called after restoring the config space. */
390 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
392 unsigned n
= dev
->msix_entries_nr
;
395 if (!msix_present(dev
)) {
399 msix_clear_all_vectors(dev
);
400 qemu_get_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
401 qemu_get_buffer(f
, dev
->msix_pba
, (n
+ 7) / 8);
402 msix_update_function_masked(dev
);
404 for (vector
= 0; vector
< n
; vector
++) {
405 msix_handle_mask_update(dev
, vector
, true);
409 /* Does device support MSI-X? */
410 int msix_present(PCIDevice
*dev
)
412 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
415 /* Is MSI-X enabled? */
416 int msix_enabled(PCIDevice
*dev
)
418 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
419 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
423 /* Send an MSI-X message */
424 void msix_notify(PCIDevice
*dev
, unsigned vector
)
428 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
430 if (msix_is_masked(dev
, vector
)) {
431 msix_set_pending(dev
, vector
);
435 msg
= msix_get_message(dev
, vector
);
437 stl_le_phys(msg
.address
, msg
.data
);
440 void msix_reset(PCIDevice
*dev
)
442 if (!msix_present(dev
)) {
445 msix_clear_all_vectors(dev
);
446 dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &=
447 ~dev
->wmask
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
];
448 memset(dev
->msix_table
, 0, dev
->msix_entries_nr
* PCI_MSIX_ENTRY_SIZE
);
449 memset(dev
->msix_pba
, 0, QEMU_ALIGN_UP(dev
->msix_entries_nr
, 64) / 8);
450 msix_mask_all(dev
, dev
->msix_entries_nr
);
453 /* PCI spec suggests that devices make it possible for software to configure
454 * less vectors than supported by the device, but does not specify a standard
455 * mechanism for devices to do so.
457 * We support this by asking devices to declare vectors software is going to
458 * actually use, and checking this on the notification path. Devices that
459 * don't want to follow the spec suggestion can declare all vectors as used. */
461 /* Mark vector as used. */
462 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
464 if (vector
>= dev
->msix_entries_nr
)
466 dev
->msix_entry_used
[vector
]++;
470 /* Mark vector as unused. */
471 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
473 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
476 if (--dev
->msix_entry_used
[vector
]) {
479 msix_clr_pending(dev
, vector
);
482 void msix_unuse_all_vectors(PCIDevice
*dev
)
484 if (!msix_present(dev
)) {
487 msix_free_irq_entries(dev
);
490 unsigned int msix_nr_vectors_allocated(const PCIDevice
*dev
)
492 return dev
->msix_entries_nr
;
495 static int msix_set_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
499 if (msix_is_masked(dev
, vector
)) {
502 msg
= msix_get_message(dev
, vector
);
503 return dev
->msix_vector_use_notifier(dev
, vector
, msg
);
506 static void msix_unset_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
508 if (msix_is_masked(dev
, vector
)) {
511 dev
->msix_vector_release_notifier(dev
, vector
);
514 int msix_set_vector_notifiers(PCIDevice
*dev
,
515 MSIVectorUseNotifier use_notifier
,
516 MSIVectorReleaseNotifier release_notifier
)
520 assert(use_notifier
&& release_notifier
);
522 dev
->msix_vector_use_notifier
= use_notifier
;
523 dev
->msix_vector_release_notifier
= release_notifier
;
525 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
526 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
527 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
528 ret
= msix_set_notifier_for_vector(dev
, vector
);
537 while (--vector
>= 0) {
538 msix_unset_notifier_for_vector(dev
, vector
);
540 dev
->msix_vector_use_notifier
= NULL
;
541 dev
->msix_vector_release_notifier
= NULL
;
545 void msix_unset_vector_notifiers(PCIDevice
*dev
)
549 assert(dev
->msix_vector_use_notifier
&&
550 dev
->msix_vector_release_notifier
);
552 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
553 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
554 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
555 msix_unset_notifier_for_vector(dev
, vector
);
558 dev
->msix_vector_use_notifier
= NULL
;
559 dev
->msix_vector_release_notifier
= NULL
;