4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
18 /* Declaration from linux/pci_regs.h */
19 #define PCI_CAP_ID_MSIX 0x11 /* MSI-X */
20 #define PCI_MSIX_FLAGS 2 /* Table at lower 11 bits */
21 #define PCI_MSIX_FLAGS_QSIZE 0x7FF
22 #define PCI_MSIX_FLAGS_ENABLE (1 << 15)
23 #define PCI_MSIX_FLAGS_BIRMASK (7 << 0)
25 /* MSI-X capability structure */
26 #define MSIX_TABLE_OFFSET 4
27 #define MSIX_PBA_OFFSET 8
28 #define MSIX_CAP_LENGTH 12
30 /* MSI enable bit is in byte 1 in FLAGS register */
31 #define MSIX_ENABLE_OFFSET (PCI_MSIX_FLAGS + 1)
32 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
34 /* MSI-X table format */
35 #define MSIX_MSG_ADDR 0
36 #define MSIX_MSG_UPPER_ADDR 4
37 #define MSIX_MSG_DATA 8
38 #define MSIX_VECTOR_CTRL 12
39 #define MSIX_ENTRY_SIZE 16
40 #define MSIX_VECTOR_MASK 0x1
42 /* How much space does an MSIX table need. */
43 /* The spec requires giving the table structure
44 * a 4K aligned region all by itself. */
45 #define MSIX_PAGE_SIZE 0x1000
46 /* Reserve second half of the page for pending bits */
47 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
48 #define MSIX_MAX_ENTRIES 32
52 #define DEBUG(fmt, ...) \
54 fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__); \
57 #define DEBUG(fmt, ...) do { } while(0)
60 /* Flag for interrupt controller to declare MSI-X support */
63 /* Add MSI-X capability to the config space for the device. */
64 /* Given a bar and its size, add MSI-X table on top of it
65 * and fill MSI-X capability in the config space.
66 * Original bar size must be a power of 2 or 0.
67 * New bar size is returned. */
68 static int msix_add_config(struct PCIDevice
*pdev
, unsigned short nentries
,
69 unsigned bar_nr
, unsigned bar_size
)
75 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1)
77 if (bar_size
> 0x80000000)
80 /* Add space for MSI-X structures */
82 new_size
= MSIX_PAGE_SIZE
;
83 } else if (bar_size
< MSIX_PAGE_SIZE
) {
84 bar_size
= MSIX_PAGE_SIZE
;
85 new_size
= MSIX_PAGE_SIZE
* 2;
87 new_size
= bar_size
* 2;
90 pdev
->msix_bar_size
= new_size
;
91 config_offset
= pci_add_capability(pdev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
92 if (config_offset
< 0)
94 config
= pdev
->config
+ config_offset
;
96 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
97 /* Table on top of BAR */
98 pci_set_long(config
+ MSIX_TABLE_OFFSET
, bar_size
| bar_nr
);
99 /* Pending bits on top of that */
100 pci_set_long(config
+ MSIX_PBA_OFFSET
, (bar_size
+ MSIX_PAGE_PENDING
) |
102 pdev
->msix_cap
= config_offset
;
103 /* Make flags bit writeable. */
104 pdev
->wmask
[config_offset
+ MSIX_ENABLE_OFFSET
] |= MSIX_ENABLE_MASK
;
108 /* Handle MSI-X capability config write. */
109 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
110 uint32_t val
, int len
)
112 unsigned enable_pos
= dev
->msix_cap
+ MSIX_ENABLE_OFFSET
;
113 if (addr
+ len
<= enable_pos
|| addr
> enable_pos
)
116 if (msix_enabled(dev
))
117 qemu_set_irq(dev
->irq
[0], 0);
120 static uint32_t msix_mmio_readl(void *opaque
, target_phys_addr_t addr
)
122 PCIDevice
*dev
= opaque
;
123 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1) & ~0x3;
124 void *page
= dev
->msix_table_page
;
126 return pci_get_long(page
+ offset
);
129 static uint32_t msix_mmio_read_unallowed(void *opaque
, target_phys_addr_t addr
)
131 fprintf(stderr
, "MSI-X: only dword read is allowed!\n");
135 static uint8_t msix_pending_mask(int vector
)
137 return 1 << (vector
% 8);
140 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
142 return dev
->msix_table_page
+ MSIX_PAGE_PENDING
+ vector
/ 8;
145 static int msix_is_pending(PCIDevice
*dev
, int vector
)
147 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
150 static void msix_set_pending(PCIDevice
*dev
, int vector
)
152 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
155 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
157 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
160 static int msix_is_masked(PCIDevice
*dev
, int vector
)
162 unsigned offset
= vector
* MSIX_ENTRY_SIZE
+ MSIX_VECTOR_CTRL
;
163 return dev
->msix_table_page
[offset
] & MSIX_VECTOR_MASK
;
166 static void msix_mmio_writel(void *opaque
, target_phys_addr_t addr
,
169 PCIDevice
*dev
= opaque
;
170 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1) & ~0x3;
171 int vector
= offset
/ MSIX_ENTRY_SIZE
;
172 pci_set_long(dev
->msix_table_page
+ offset
, val
);
173 if (!msix_is_masked(dev
, vector
) && msix_is_pending(dev
, vector
)) {
174 msix_clr_pending(dev
, vector
);
175 msix_notify(dev
, vector
);
179 static void msix_mmio_write_unallowed(void *opaque
, target_phys_addr_t addr
,
182 fprintf(stderr
, "MSI-X: only dword write is allowed!\n");
185 static CPUWriteMemoryFunc
* const msix_mmio_write
[] = {
186 msix_mmio_write_unallowed
, msix_mmio_write_unallowed
, msix_mmio_writel
189 static CPUReadMemoryFunc
* const msix_mmio_read
[] = {
190 msix_mmio_read_unallowed
, msix_mmio_read_unallowed
, msix_mmio_readl
193 /* Should be called from device's map method. */
194 void msix_mmio_map(PCIDevice
*d
, int region_num
,
195 pcibus_t addr
, pcibus_t size
, int type
)
197 uint8_t *config
= d
->config
+ d
->msix_cap
;
198 uint32_t table
= pci_get_long(config
+ MSIX_TABLE_OFFSET
);
199 uint32_t offset
= table
& ~(MSIX_PAGE_SIZE
- 1);
200 /* TODO: for assigned devices, we'll want to make it possible to map
201 * pending bits separately in case they are in a separate bar. */
202 int table_bir
= table
& PCI_MSIX_FLAGS_BIRMASK
;
204 if (table_bir
!= region_num
)
208 cpu_register_physical_memory(addr
+ offset
, size
- offset
,
212 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
215 for (vector
= 0; vector
< nentries
; ++vector
) {
216 unsigned offset
= vector
* MSIX_ENTRY_SIZE
+ MSIX_VECTOR_CTRL
;
217 dev
->msix_table_page
[offset
] |= MSIX_VECTOR_MASK
;
221 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
222 * modified, it should be retrieved with msix_bar_size. */
223 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
224 unsigned bar_nr
, unsigned bar_size
)
227 /* Nothing to do if MSI is not supported by interrupt controller */
231 if (nentries
> MSIX_MAX_ENTRIES
)
234 dev
->msix_entry_used
= qemu_mallocz(MSIX_MAX_ENTRIES
*
235 sizeof *dev
->msix_entry_used
);
237 dev
->msix_table_page
= qemu_mallocz(MSIX_PAGE_SIZE
);
238 msix_mask_all(dev
, nentries
);
240 dev
->msix_mmio_index
= cpu_register_io_memory(msix_mmio_read
,
241 msix_mmio_write
, dev
);
242 if (dev
->msix_mmio_index
== -1) {
247 dev
->msix_entries_nr
= nentries
;
248 ret
= msix_add_config(dev
, nentries
, bar_nr
, bar_size
);
252 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
256 dev
->msix_entries_nr
= 0;
257 cpu_unregister_io_memory(dev
->msix_mmio_index
);
259 qemu_free(dev
->msix_table_page
);
260 dev
->msix_table_page
= NULL
;
261 qemu_free(dev
->msix_entry_used
);
262 dev
->msix_entry_used
= NULL
;
266 static void msix_free_irq_entries(PCIDevice
*dev
)
270 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
271 dev
->msix_entry_used
[vector
] = 0;
272 msix_clr_pending(dev
, vector
);
276 /* Clean up resources for the device. */
277 int msix_uninit(PCIDevice
*dev
)
279 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
281 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
283 msix_free_irq_entries(dev
);
284 dev
->msix_entries_nr
= 0;
285 cpu_unregister_io_memory(dev
->msix_mmio_index
);
286 qemu_free(dev
->msix_table_page
);
287 dev
->msix_table_page
= NULL
;
288 qemu_free(dev
->msix_entry_used
);
289 dev
->msix_entry_used
= NULL
;
290 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
294 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
296 unsigned n
= dev
->msix_entries_nr
;
298 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
302 qemu_put_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
303 qemu_put_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
306 /* Should be called after restoring the config space. */
307 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
309 unsigned n
= dev
->msix_entries_nr
;
311 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
315 msix_free_irq_entries(dev
);
316 qemu_get_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
317 qemu_get_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
320 /* Does device support MSI-X? */
321 int msix_present(PCIDevice
*dev
)
323 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
326 /* Is MSI-X enabled? */
327 int msix_enabled(PCIDevice
*dev
)
329 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
330 (dev
->config
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
] &
334 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
335 uint32_t msix_bar_size(PCIDevice
*dev
)
337 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) ?
338 dev
->msix_bar_size
: 0;
341 /* Send an MSI-X message */
342 void msix_notify(PCIDevice
*dev
, unsigned vector
)
344 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
348 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
350 if (msix_is_masked(dev
, vector
)) {
351 msix_set_pending(dev
, vector
);
355 address
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
356 address
= (address
<< 32) | pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
357 data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
358 stl_phys(address
, data
);
361 void msix_reset(PCIDevice
*dev
)
363 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
365 msix_free_irq_entries(dev
);
366 dev
->config
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
] &=
367 ~dev
->wmask
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
];
368 memset(dev
->msix_table_page
, 0, MSIX_PAGE_SIZE
);
369 msix_mask_all(dev
, dev
->msix_entries_nr
);
372 /* PCI spec suggests that devices make it possible for software to configure
373 * less vectors than supported by the device, but does not specify a standard
374 * mechanism for devices to do so.
376 * We support this by asking devices to declare vectors software is going to
377 * actually use, and checking this on the notification path. Devices that
378 * don't want to follow the spec suggestion can declare all vectors as used. */
380 /* Mark vector as used. */
381 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
383 if (vector
>= dev
->msix_entries_nr
)
385 dev
->msix_entry_used
[vector
]++;
389 /* Mark vector as unused. */
390 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
392 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
395 if (--dev
->msix_entry_used
[vector
]) {
398 msix_clr_pending(dev
, vector
);
401 void msix_unuse_all_vectors(PCIDevice
*dev
)
403 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
405 msix_free_irq_entries(dev
);