[mirror_qemu.git] / hw / pci / msix.c

/*
 * MSI-X device support
 *
 * This module includes support for MSI-X in pci devices.
 *
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 *  Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
 */

#include "hw/hw.h"
#include "hw/pci/msi.h"
#include "hw/pci/msix.h"
#include "hw/pci/pci.h"
#include "range.h"

#define MSIX_CAP_LENGTH 12

/* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
#define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)

static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
{
    uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
    MSIMessage msg;

    msg.address = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
    msg.data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
    return msg;
}

/*
 * Special API for POWER to configure the vectors through
 * a side channel. Should never be used by devices.
 */
void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)
{
    uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;

    pci_set_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR, msg.address);
    pci_set_long(table_entry + PCI_MSIX_ENTRY_DATA, msg.data);
    table_entry[PCI_MSIX_ENTRY_VECTOR_CTRL] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
}

static uint8_t msix_pending_mask(int vector)
{
    return 1 << (vector % 8);
}

static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)
{
    return dev->msix_pba + vector / 8;
}

static int msix_is_pending(PCIDevice *dev, int vector)
{
    return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
}

static void msix_set_pending(PCIDevice *dev, int vector)
{
    *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
}

static void msix_clr_pending(PCIDevice *dev, int vector)
{
    *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
}

static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
{
    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
    return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
}

static bool msix_is_masked(PCIDevice *dev, int vector)
{
    return msix_vector_masked(dev, vector, dev->msix_function_masked);
}

static void msix_fire_vector_notifier(PCIDevice *dev,
                                      unsigned int vector, bool is_masked)
{
    MSIMessage msg;
    int ret;

    if (!dev->msix_vector_use_notifier) {
        return;
    }
    if (is_masked) {
        dev->msix_vector_release_notifier(dev, vector);
    } else {
        msg = msix_get_message(dev, vector);
        ret = dev->msix_vector_use_notifier(dev, vector, msg);
        assert(ret >= 0);
    }
}

static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
{
    bool is_masked = msix_is_masked(dev, vector);

    if (is_masked == was_masked) {
        return;
    }

    msix_fire_vector_notifier(dev, vector, is_masked);

    if (!is_masked && msix_is_pending(dev, vector)) {
        msix_clr_pending(dev, vector);
        msix_notify(dev, vector);
    }
}

static void msix_update_function_masked(PCIDevice *dev)
{
    dev->msix_function_masked = !msix_enabled(dev) ||
        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
}

/* Handle MSI-X capability config write. */
void msix_write_config(PCIDevice *dev, uint32_t addr,
                       uint32_t val, int len)
{
    unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
    int vector;
    bool was_masked;

    if (!msix_present(dev) || !range_covers_byte(addr, len, enable_pos)) {
        return;
    }

    was_masked = dev->msix_function_masked;
    msix_update_function_masked(dev);

    if (!msix_enabled(dev)) {
        return;
    }

    pci_device_deassert_intx(dev);

    if (dev->msix_function_masked == was_masked) {
        return;
    }

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        msix_handle_mask_update(dev, vector,
                                msix_vector_masked(dev, vector, was_masked));
    }
}

static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
                                     unsigned size)
{
    PCIDevice *dev = opaque;

    return pci_get_long(dev->msix_table + addr);
}

static void msix_table_mmio_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
{
    PCIDevice *dev = opaque;
    int vector = addr / PCI_MSIX_ENTRY_SIZE;
    bool was_masked;

    was_masked = msix_is_masked(dev, vector);
    pci_set_long(dev->msix_table + addr, val);
    msix_handle_mask_update(dev, vector, was_masked);
}

static const MemoryRegionOps msix_table_mmio_ops = {
    .read = msix_table_mmio_read,
    .write = msix_table_mmio_write,
    .endianness = DEVICE_LITTLE_ENDIAN,
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
};

static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
                                   unsigned size)
{
    PCIDevice *dev = opaque;

    return pci_get_long(dev->msix_pba + addr);
}

static const MemoryRegionOps msix_pba_mmio_ops = {
    .read = msix_pba_mmio_read,
    .endianness = DEVICE_LITTLE_ENDIAN,
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
};

static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
{
    int vector;

    for (vector = 0; vector < nentries; ++vector) {
        unsigned offset =
            vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
        bool was_masked = msix_is_masked(dev, vector);

        dev->msix_table[offset] |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
        msix_handle_mask_update(dev, vector, was_masked);
    }
}

/* Initialize the MSI-X structures */
int msix_init(struct PCIDevice *dev, unsigned short nentries,
              MemoryRegion *table_bar, uint8_t table_bar_nr,
              unsigned table_offset, MemoryRegion *pba_bar,
              uint8_t pba_bar_nr, unsigned pba_offset, uint8_t cap_pos)
{
    int cap;
    unsigned table_size, pba_size;
    uint8_t *config;

    /* Nothing to do if MSI is not supported by interrupt controller */
    if (!msi_supported) {
        return -ENOTSUP;
    }

    if (nentries < 1 || nentries > PCI_MSIX_FLAGS_QSIZE + 1) {
        return -EINVAL;
    }

    table_size = nentries * PCI_MSIX_ENTRY_SIZE;
    pba_size = QEMU_ALIGN_UP(nentries, 64) / 8;

    /* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
    if ((table_bar_nr == pba_bar_nr &&
         ranges_overlap(table_offset, table_size, pba_offset, pba_size)) ||
        table_offset + table_size > memory_region_size(table_bar) ||
        pba_offset + pba_size > memory_region_size(pba_bar) ||
        (table_offset | pba_offset) & PCI_MSIX_FLAGS_BIRMASK) {
        return -EINVAL;
    }

    cap = pci_add_capability(dev, PCI_CAP_ID_MSIX, cap_pos, MSIX_CAP_LENGTH);
    if (cap < 0) {
        return cap;
    }

    dev->msix_cap = cap;
    dev->cap_present |= QEMU_PCI_CAP_MSIX;
    config = dev->config + cap;

    pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
    dev->msix_entries_nr = nentries;
    dev->msix_function_masked = true;

    pci_set_long(config + PCI_MSIX_TABLE, table_offset | table_bar_nr);
    pci_set_long(config + PCI_MSIX_PBA, pba_offset | pba_bar_nr);

    /* Make flags bit writable. */
    dev->wmask[cap + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
                                             MSIX_MASKALL_MASK;

    dev->msix_table = g_malloc0(table_size);
    dev->msix_pba = g_malloc0(pba_size);
    dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);

    msix_mask_all(dev, nentries);

    memory_region_init_io(&dev->msix_table_mmio, &msix_table_mmio_ops, dev,
                          "msix-table", table_size);
    memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);
    memory_region_init_io(&dev->msix_pba_mmio, &msix_pba_mmio_ops, dev,
                          "msix-pba", pba_size);
    memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);

    return 0;
}

int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
                            uint8_t bar_nr)
{
    int ret;
    char *name;

    /*
     * Migration compatibility dictates that this remains a 4k
     * BAR with the vector table in the lower half and PBA in
     * the upper half.  Do not use these elsewhere!
     */
#define MSIX_EXCLUSIVE_BAR_SIZE 4096
#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
#define MSIX_EXCLUSIVE_CAP_OFFSET 0

    if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
        return -EINVAL;
    }

    name = g_strdup_printf("%s-msix", dev->name);
    memory_region_init(&dev->msix_exclusive_bar, name, MSIX_EXCLUSIVE_BAR_SIZE);
    g_free(name);

    ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
                    MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
                    bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
                    MSIX_EXCLUSIVE_CAP_OFFSET);
    if (ret) {
        memory_region_destroy(&dev->msix_exclusive_bar);
        return ret;
    }

    pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,
                     &dev->msix_exclusive_bar);

    return 0;
}

static void msix_free_irq_entries(PCIDevice *dev)
{
    int vector;

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        dev->msix_entry_used[vector] = 0;
        msix_clr_pending(dev, vector);
    }
}

static void msix_clear_all_vectors(PCIDevice *dev)
{
    int vector;

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        msix_clr_pending(dev, vector);
    }
}

/* Clean up resources for the device. */
void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)
{
    if (!msix_present(dev)) {
        return;
    }
    pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
    dev->msix_cap = 0;
    msix_free_irq_entries(dev);
    dev->msix_entries_nr = 0;
    memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
    memory_region_destroy(&dev->msix_pba_mmio);
    g_free(dev->msix_pba);
    dev->msix_pba = NULL;
    memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
    memory_region_destroy(&dev->msix_table_mmio);
    g_free(dev->msix_table);
    dev->msix_table = NULL;
    g_free(dev->msix_entry_used);
    dev->msix_entry_used = NULL;
    dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
}

void msix_uninit_exclusive_bar(PCIDevice *dev)
{
    if (msix_present(dev)) {
        msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
        memory_region_destroy(&dev->msix_exclusive_bar);
    }
}

void msix_save(PCIDevice *dev, QEMUFile *f)
{
    unsigned n = dev->msix_entries_nr;

    if (!msix_present(dev)) {
        return;
    }

    qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
    qemu_put_buffer(f, dev->msix_pba, (n + 7) / 8);
}

/* Should be called after restoring the config space. */
void msix_load(PCIDevice *dev, QEMUFile *f)
{
    unsigned n = dev->msix_entries_nr;
    unsigned int vector;

    if (!msix_present(dev)) {
        return;
    }

    msix_clear_all_vectors(dev);
    qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
    qemu_get_buffer(f, dev->msix_pba, (n + 7) / 8);
    msix_update_function_masked(dev);

    for (vector = 0; vector < n; vector++) {
        msix_handle_mask_update(dev, vector, true);
    }
}

/* Does device support MSI-X? */
int msix_present(PCIDevice *dev)
{
    return dev->cap_present & QEMU_PCI_CAP_MSIX;
}

/* Is MSI-X enabled? */
int msix_enabled(PCIDevice *dev)
{
    return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
         MSIX_ENABLE_MASK);
}

/* Send an MSI-X message */
void msix_notify(PCIDevice *dev, unsigned vector)
{
    MSIMessage msg;

    if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector])
        return;
    if (msix_is_masked(dev, vector)) {
        msix_set_pending(dev, vector);
        return;
    }

    msg = msix_get_message(dev, vector);

    stl_le_phys(msg.address, msg.data);
}

void msix_reset(PCIDevice *dev)
{
    if (!msix_present(dev)) {
        return;
    }
    msix_clear_all_vectors(dev);
    dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
	    ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
    memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
    memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
    msix_mask_all(dev, dev->msix_entries_nr);
}

/* PCI spec suggests that devices make it possible for software to configure
 * less vectors than supported by the device, but does not specify a standard
 * mechanism for devices to do so.
 *
 * We support this by asking devices to declare vectors software is going to
 * actually use, and checking this on the notification path. Devices that
 * don't want to follow the spec suggestion can declare all vectors as used. */

/* Mark vector as used. */
int msix_vector_use(PCIDevice *dev, unsigned vector)
{
    if (vector >= dev->msix_entries_nr)
        return -EINVAL;
    dev->msix_entry_used[vector]++;
    return 0;
}

/* Mark vector as unused. */
void msix_vector_unuse(PCIDevice *dev, unsigned vector)
{
    if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) {
        return;
    }
    if (--dev->msix_entry_used[vector]) {
        return;
    }
    msix_clr_pending(dev, vector);
}

void msix_unuse_all_vectors(PCIDevice *dev)
{
    if (!msix_present(dev)) {
        return;
    }
    msix_free_irq_entries(dev);
}

unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)
{
    return dev->msix_entries_nr;
}

static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)
{
    MSIMessage msg;

    if (msix_is_masked(dev, vector)) {
        return 0;
    }
    msg = msix_get_message(dev, vector);
    return dev->msix_vector_use_notifier(dev, vector, msg);
}

static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
{
    if (msix_is_masked(dev, vector)) {
        return;
    }
    dev->msix_vector_release_notifier(dev, vector);
}

int msix_set_vector_notifiers(PCIDevice *dev,
                              MSIVectorUseNotifier use_notifier,
                              MSIVectorReleaseNotifier release_notifier)
{
    int vector, ret;

    assert(use_notifier && release_notifier);

    dev->msix_vector_use_notifier = use_notifier;
    dev->msix_vector_release_notifier = release_notifier;

    if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
        (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
        for (vector = 0; vector < dev->msix_entries_nr; vector++) {
            ret = msix_set_notifier_for_vector(dev, vector);
            if (ret < 0) {
                goto undo;
            }
        }
    }
    return 0;

undo:
    while (--vector >= 0) {
        msix_unset_notifier_for_vector(dev, vector);
    }
    dev->msix_vector_use_notifier = NULL;
    dev->msix_vector_release_notifier = NULL;
    return ret;
}

void msix_unset_vector_notifiers(PCIDevice *dev)
{
    int vector;

    assert(dev->msix_vector_use_notifier &&
           dev->msix_vector_release_notifier);

    if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
        (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
        for (vector = 0; vector < dev->msix_entries_nr; vector++) {
            msix_unset_notifier_for_vector(dev, vector);
        }
    }
    dev->msix_vector_use_notifier = NULL;
    dev->msix_vector_release_notifier = NULL;
}
Commit	Line	Data
02eb84d0 MT	1	/*
	2	* MSI-X device support
	3	*
	4	* This module includes support for MSI-X in pci devices.
	5	*
	6	* Author: Michael S. Tsirkin <mst@redhat.com>
	7	*
	8	* Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
	9	*
	10	* This work is licensed under the terms of the GNU GPL, version 2. See
	11	* the COPYING file in the top-level directory.
6b620ca3 PB	12	*
	13	* Contributions after 2012-01-13 are licensed under the terms of the
	14	* GNU GPL, version 2 or (at your option) any later version.
02eb84d0 MT	15	*/
02eb84d0 MT	16
c759b24f MT	17	#include "hw/hw.h"
	18	#include "hw/pci/msi.h"
	19	#include "hw/pci/msix.h"
	20	#include "hw/pci/pci.h"
bf1b0071	21	#include "range.h"
02eb84d0	22
02eb84d0 MT	23	#define MSIX_CAP_LENGTH 12
02eb84d0 MT	24
2760952b MT	25	/* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
2760952b MT	26	#define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
02eb84d0	27	#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
5b5cb086	28	#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
02eb84d0	29
bc4caf49 JK	30	static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
bc4caf49 JK	31	{
d35e428c	32	uint8_t table_entry = dev->msix_table + vector PCI_MSIX_ENTRY_SIZE;
bc4caf49 JK	33	MSIMessage msg;
	34
	35	msg.address = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
	36	msg.data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
	37	return msg;
	38	}
02eb84d0	39
932d4a42 AK	40	/*
	41	* Special API for POWER to configure the vectors through
	42	* a side channel. Should never be used by devices.
	43	*/
	44	void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)
	45	{
	46	uint8_t table_entry = dev->msix_table + vector PCI_MSIX_ENTRY_SIZE;
	47
	48	pci_set_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR, msg.address);
	49	pci_set_long(table_entry + PCI_MSIX_ENTRY_DATA, msg.data);
	50	table_entry[PCI_MSIX_ENTRY_VECTOR_CTRL] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
	51	}
	52
02eb84d0 MT	53	static uint8_t msix_pending_mask(int vector)
	54	{
	55	return 1 << (vector % 8);
	56	}
	57
	58	static uint8_t msix_pending_byte(PCIDevice dev, int vector)
	59	{
d35e428c	60	return dev->msix_pba + vector / 8;
02eb84d0 MT	61	}
	62
	63	static int msix_is_pending(PCIDevice *dev, int vector)
	64	{
	65	return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
	66	}
	67
	68	static void msix_set_pending(PCIDevice *dev, int vector)
	69	{
	70	*msix_pending_byte(dev, vector) \|= msix_pending_mask(vector);
	71	}
	72
	73	static void msix_clr_pending(PCIDevice *dev, int vector)
	74	{
	75	*msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
	76	}
	77
ae392c41	78	static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
02eb84d0	79	{
ae392c41	80	unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
d35e428c	81	return fmask \|\| dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
5b5cb086 MT	82	}
5b5cb086 MT	83
ae392c41	84	static bool msix_is_masked(PCIDevice *dev, int vector)
5b5cb086	85	{
ae392c41 MT	86	return msix_vector_masked(dev, vector, dev->msix_function_masked);
	87	}
	88
2cdfe53c JK	89	static void msix_fire_vector_notifier(PCIDevice *dev,
	90	unsigned int vector, bool is_masked)
	91	{
	92	MSIMessage msg;
	93	int ret;
	94
	95	if (!dev->msix_vector_use_notifier) {
	96	return;
	97	}
	98	if (is_masked) {
	99	dev->msix_vector_release_notifier(dev, vector);
	100	} else {
	101	msg = msix_get_message(dev, vector);
	102	ret = dev->msix_vector_use_notifier(dev, vector, msg);
	103	assert(ret >= 0);
	104	}
	105	}
	106
ae392c41 MT	107	static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
	108	{
	109	bool is_masked = msix_is_masked(dev, vector);
2cdfe53c	110
ae392c41 MT	111	if (is_masked == was_masked) {
	112	return;
	113	}
	114
2cdfe53c JK	115	msix_fire_vector_notifier(dev, vector, is_masked);
2cdfe53c JK	116
ae392c41	117	if (!is_masked && msix_is_pending(dev, vector)) {
5b5cb086 MT	118	msix_clr_pending(dev, vector);
	119	msix_notify(dev, vector);
	120	}
	121	}
	122
50322249 MT	123	static void msix_update_function_masked(PCIDevice *dev)
	124	{
	125	dev->msix_function_masked = !msix_enabled(dev) \|\|
	126	(dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
	127	}
	128
5b5cb086 MT	129	/* Handle MSI-X capability config write. */
	130	void msix_write_config(PCIDevice *dev, uint32_t addr,
	131	uint32_t val, int len)
	132	{
	133	unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
	134	int vector;
50322249	135	bool was_masked;
5b5cb086	136
7c9958b0	137	if (!msix_present(dev) \|\| !range_covers_byte(addr, len, enable_pos)) {
5b5cb086 MT	138	return;
	139	}
	140
50322249 MT	141	was_masked = dev->msix_function_masked;
	142	msix_update_function_masked(dev);
	143
5b5cb086 MT	144	if (!msix_enabled(dev)) {
	145	return;
	146	}
	147
e407bf13	148	pci_device_deassert_intx(dev);
5b5cb086	149
50322249	150	if (dev->msix_function_masked == was_masked) {
5b5cb086 MT	151	return;
	152	}
	153
	154	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
ae392c41 MT	155	msix_handle_mask_update(dev, vector,
ae392c41 MT	156	msix_vector_masked(dev, vector, was_masked));
5b5cb086	157	}
02eb84d0 MT	158	}
02eb84d0 MT	159
a8170e5e	160	static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
d35e428c	161	unsigned size)
eebcb0a7 AW	162	{
eebcb0a7 AW	163	PCIDevice *dev = opaque;
eebcb0a7	164
d35e428c	165	return pci_get_long(dev->msix_table + addr);
eebcb0a7 AW	166	}
eebcb0a7 AW	167
a8170e5e	168	static void msix_table_mmio_write(void *opaque, hwaddr addr,
d35e428c	169	uint64_t val, unsigned size)
02eb84d0 MT	170	{
02eb84d0 MT	171	PCIDevice *dev = opaque;
d35e428c	172	int vector = addr / PCI_MSIX_ENTRY_SIZE;
ae392c41	173	bool was_masked;
9a93b617	174
ae392c41	175	was_masked = msix_is_masked(dev, vector);
d35e428c	176	pci_set_long(dev->msix_table + addr, val);
ae392c41	177	msix_handle_mask_update(dev, vector, was_masked);
02eb84d0 MT	178	}
02eb84d0 MT	179
d35e428c AW	180	static const MemoryRegionOps msix_table_mmio_ops = {
	181	.read = msix_table_mmio_read,
	182	.write = msix_table_mmio_write,
68d1e1f5	183	.endianness = DEVICE_LITTLE_ENDIAN,
d35e428c AW	184	.valid = {
	185	.min_access_size = 4,
	186	.max_access_size = 4,
	187	},
	188	};
	189
a8170e5e	190	static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
d35e428c AW	191	unsigned size)
	192	{
	193	PCIDevice *dev = opaque;
	194
	195	return pci_get_long(dev->msix_pba + addr);
	196	}
	197
	198	static const MemoryRegionOps msix_pba_mmio_ops = {
	199	.read = msix_pba_mmio_read,
68d1e1f5	200	.endianness = DEVICE_LITTLE_ENDIAN,
95524ae8 AK	201	.valid = {
	202	.min_access_size = 4,
	203	.max_access_size = 4,
	204	},
02eb84d0 MT	205	};
02eb84d0 MT	206
ae1be0bb MT	207	static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
	208	{
	209	int vector;
5b5f1330	210
ae1be0bb	211	for (vector = 0; vector < nentries; ++vector) {
01731cfb JK	212	unsigned offset =
01731cfb JK	213	vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
5b5f1330 JK	214	bool was_masked = msix_is_masked(dev, vector);
5b5f1330 JK	215
d35e428c	216	dev->msix_table[offset] \|= PCI_MSIX_ENTRY_CTRL_MASKBIT;
5b5f1330	217	msix_handle_mask_update(dev, vector, was_masked);
ae1be0bb MT	218	}
	219	}
	220
5a2c2029	221	/* Initialize the MSI-X structures */
02eb84d0	222	int msix_init(struct PCIDevice *dev, unsigned short nentries,
5a2c2029 AW	223	MemoryRegion *table_bar, uint8_t table_bar_nr,
	224	unsigned table_offset, MemoryRegion *pba_bar,
	225	uint8_t pba_bar_nr, unsigned pba_offset, uint8_t cap_pos)
02eb84d0	226	{
5a2c2029	227	int cap;
d35e428c	228	unsigned table_size, pba_size;
5a2c2029	229	uint8_t *config;
60ba3cc2	230
02eb84d0	231	/* Nothing to do if MSI is not supported by interrupt controller */
60ba3cc2	232	if (!msi_supported) {
02eb84d0	233	return -ENOTSUP;
60ba3cc2	234	}
5a2c2029 AW	235
5a2c2029 AW	236	if (nentries < 1 \|\| nentries > PCI_MSIX_FLAGS_QSIZE + 1) {
02eb84d0	237	return -EINVAL;
5a2c2029	238	}
02eb84d0	239
d35e428c AW	240	table_size = nentries * PCI_MSIX_ENTRY_SIZE;
	241	pba_size = QEMU_ALIGN_UP(nentries, 64) / 8;
	242
5a2c2029 AW	243	/* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
	244	if ((table_bar_nr == pba_bar_nr &&
	245	ranges_overlap(table_offset, table_size, pba_offset, pba_size)) \|\|
	246	table_offset + table_size > memory_region_size(table_bar) \|\|
	247	pba_offset + pba_size > memory_region_size(pba_bar) \|\|
	248	(table_offset \| pba_offset) & PCI_MSIX_FLAGS_BIRMASK) {
	249	return -EINVAL;
	250	}
	251
	252	cap = pci_add_capability(dev, PCI_CAP_ID_MSIX, cap_pos, MSIX_CAP_LENGTH);
	253	if (cap < 0) {
	254	return cap;
	255	}
	256
	257	dev->msix_cap = cap;
	258	dev->cap_present \|= QEMU_PCI_CAP_MSIX;
	259	config = dev->config + cap;
	260
	261	pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
	262	dev->msix_entries_nr = nentries;
	263	dev->msix_function_masked = true;
	264
	265	pci_set_long(config + PCI_MSIX_TABLE, table_offset \| table_bar_nr);
	266	pci_set_long(config + PCI_MSIX_PBA, pba_offset \| pba_bar_nr);
	267
	268	/* Make flags bit writable. */
	269	dev->wmask[cap + MSIX_CONTROL_OFFSET] \|= MSIX_ENABLE_MASK \|
	270	MSIX_MASKALL_MASK;
02eb84d0	271
d35e428c AW	272	dev->msix_table = g_malloc0(table_size);
d35e428c AW	273	dev->msix_pba = g_malloc0(pba_size);
5a2c2029 AW	274	dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);
5a2c2029 AW	275
ae1be0bb	276	msix_mask_all(dev, nentries);
02eb84d0	277
d35e428c AW	278	memory_region_init_io(&dev->msix_table_mmio, &msix_table_mmio_ops, dev,
d35e428c AW	279	"msix-table", table_size);
5a2c2029	280	memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);
d35e428c AW	281	memory_region_init_io(&dev->msix_pba_mmio, &msix_pba_mmio_ops, dev,
d35e428c AW	282	"msix-pba", pba_size);
5a2c2029	283	memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);
02eb84d0	284
02eb84d0	285	return 0;
02eb84d0 MT	286	}
02eb84d0 MT	287
53f94925 AW	288	int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
	289	uint8_t bar_nr)
	290	{
	291	int ret;
	292	char *name;
	293
	294	/*
	295	* Migration compatibility dictates that this remains a 4k
	296	* BAR with the vector table in the lower half and PBA in
	297	* the upper half. Do not use these elsewhere!
	298	*/
	299	#define MSIX_EXCLUSIVE_BAR_SIZE 4096
5a2c2029	300	#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
53f94925	301	#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
5a2c2029	302	#define MSIX_EXCLUSIVE_CAP_OFFSET 0
53f94925 AW	303
	304	if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
	305	return -EINVAL;
	306	}
	307
5f893b4e	308	name = g_strdup_printf("%s-msix", dev->name);
53f94925	309	memory_region_init(&dev->msix_exclusive_bar, name, MSIX_EXCLUSIVE_BAR_SIZE);
5f893b4e	310	g_free(name);
53f94925 AW	311
53f94925 AW	312	ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
5a2c2029 AW	313	MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
	314	bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
	315	MSIX_EXCLUSIVE_CAP_OFFSET);
53f94925 AW	316	if (ret) {
	317	memory_region_destroy(&dev->msix_exclusive_bar);
	318	return ret;
	319	}
	320
	321	pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,
	322	&dev->msix_exclusive_bar);
	323
	324	return 0;
	325	}
	326
98304c84 MT	327	static void msix_free_irq_entries(PCIDevice *dev)
	328	{
	329	int vector;
	330
	331	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
	332	dev->msix_entry_used[vector] = 0;
	333	msix_clr_pending(dev, vector);
	334	}
	335	}
	336
3cac001e MT	337	static void msix_clear_all_vectors(PCIDevice *dev)
	338	{
	339	int vector;
	340
	341	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
	342	msix_clr_pending(dev, vector);
	343	}
	344	}
	345
02eb84d0	346	/* Clean up resources for the device. */
572992ee	347	void msix_uninit(PCIDevice dev, MemoryRegion table_bar, MemoryRegion *pba_bar)
02eb84d0	348	{
44701ab7	349	if (!msix_present(dev)) {
572992ee	350	return;
44701ab7	351	}
02eb84d0 MT	352	pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
	353	dev->msix_cap = 0;
	354	msix_free_irq_entries(dev);
	355	dev->msix_entries_nr = 0;
5a2c2029	356	memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
d35e428c AW	357	memory_region_destroy(&dev->msix_pba_mmio);
	358	g_free(dev->msix_pba);
	359	dev->msix_pba = NULL;
5a2c2029	360	memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
d35e428c AW	361	memory_region_destroy(&dev->msix_table_mmio);
	362	g_free(dev->msix_table);
	363	dev->msix_table = NULL;
7267c094	364	g_free(dev->msix_entry_used);
02eb84d0 MT	365	dev->msix_entry_used = NULL;
02eb84d0 MT	366	dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
02eb84d0 MT	367	}
02eb84d0 MT	368
53f94925 AW	369	void msix_uninit_exclusive_bar(PCIDevice *dev)
	370	{
	371	if (msix_present(dev)) {
5a2c2029	372	msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
53f94925 AW	373	memory_region_destroy(&dev->msix_exclusive_bar);
	374	}
	375	}
	376
02eb84d0 MT	377	void msix_save(PCIDevice dev, QEMUFile f)
02eb84d0 MT	378	{
9a3e12c8 MT	379	unsigned n = dev->msix_entries_nr;
9a3e12c8 MT	380
44701ab7	381	if (!msix_present(dev)) {
9a3e12c8	382	return;
72755a70	383	}
9a3e12c8	384
d35e428c AW	385	qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	386	qemu_put_buffer(f, dev->msix_pba, (n + 7) / 8);
02eb84d0 MT	387	}
	388
	389	/* Should be called after restoring the config space. */
	390	void msix_load(PCIDevice dev, QEMUFile f)
	391	{
	392	unsigned n = dev->msix_entries_nr;
2cdfe53c	393	unsigned int vector;
02eb84d0	394
44701ab7	395	if (!msix_present(dev)) {
02eb84d0	396	return;
98846d73	397	}
02eb84d0	398
3cac001e	399	msix_clear_all_vectors(dev);
d35e428c AW	400	qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	401	qemu_get_buffer(f, dev->msix_pba, (n + 7) / 8);
50322249	402	msix_update_function_masked(dev);
2cdfe53c JK	403
	404	for (vector = 0; vector < n; vector++) {
	405	msix_handle_mask_update(dev, vector, true);
	406	}
02eb84d0 MT	407	}
	408
	409	/* Does device support MSI-X? */
	410	int msix_present(PCIDevice *dev)
	411	{
	412	return dev->cap_present & QEMU_PCI_CAP_MSIX;
	413	}
	414
	415	/* Is MSI-X enabled? */
	416	int msix_enabled(PCIDevice *dev)
	417	{
	418	return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
2760952b	419	(dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
02eb84d0 MT	420	MSIX_ENABLE_MASK);
	421	}
	422
02eb84d0 MT	423	/* Send an MSI-X message */
	424	void msix_notify(PCIDevice *dev, unsigned vector)
	425	{
bc4caf49	426	MSIMessage msg;
02eb84d0 MT	427
	428	if (vector >= dev->msix_entries_nr \|\| !dev->msix_entry_used[vector])
	429	return;
	430	if (msix_is_masked(dev, vector)) {
	431	msix_set_pending(dev, vector);
	432	return;
	433	}
	434
bc4caf49 JK	435	msg = msix_get_message(dev, vector);
	436
	437	stl_le_phys(msg.address, msg.data);
02eb84d0 MT	438	}
	439
	440	void msix_reset(PCIDevice *dev)
	441	{
44701ab7	442	if (!msix_present(dev)) {
02eb84d0	443	return;
44701ab7	444	}
3cac001e	445	msix_clear_all_vectors(dev);
2760952b MT	446	dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
2760952b MT	447	~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
d35e428c AW	448	memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	449	memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
ae1be0bb	450	msix_mask_all(dev, dev->msix_entries_nr);
02eb84d0 MT	451	}
	452
	453	/* PCI spec suggests that devices make it possible for software to configure
	454	* less vectors than supported by the device, but does not specify a standard
	455	* mechanism for devices to do so.
	456	*
	457	* We support this by asking devices to declare vectors software is going to
	458	* actually use, and checking this on the notification path. Devices that
	459	* don't want to follow the spec suggestion can declare all vectors as used. */
	460
	461	/* Mark vector as used. */
	462	int msix_vector_use(PCIDevice *dev, unsigned vector)
	463	{
	464	if (vector >= dev->msix_entries_nr)
	465	return -EINVAL;
	466	dev->msix_entry_used[vector]++;
	467	return 0;
	468	}
	469
	470	/* Mark vector as unused. */
	471	void msix_vector_unuse(PCIDevice *dev, unsigned vector)
	472	{
98304c84 MT	473	if (vector >= dev->msix_entries_nr \|\| !dev->msix_entry_used[vector]) {
	474	return;
	475	}
	476	if (--dev->msix_entry_used[vector]) {
	477	return;
	478	}
	479	msix_clr_pending(dev, vector);
02eb84d0	480	}
b5f28bca MT	481
	482	void msix_unuse_all_vectors(PCIDevice *dev)
	483	{
44701ab7	484	if (!msix_present(dev)) {
b5f28bca	485	return;
44701ab7	486	}
b5f28bca MT	487	msix_free_irq_entries(dev);
b5f28bca MT	488	}
2cdfe53c	489
cb697aaa JK	490	unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)
	491	{
	492	return dev->msix_entries_nr;
	493	}
	494
2cdfe53c JK	495	static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)
	496	{
	497	MSIMessage msg;
	498
	499	if (msix_is_masked(dev, vector)) {
	500	return 0;
	501	}
	502	msg = msix_get_message(dev, vector);
	503	return dev->msix_vector_use_notifier(dev, vector, msg);
	504	}
	505
	506	static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
	507	{
	508	if (msix_is_masked(dev, vector)) {
	509	return;
	510	}
	511	dev->msix_vector_release_notifier(dev, vector);
	512	}
	513
	514	int msix_set_vector_notifiers(PCIDevice *dev,
	515	MSIVectorUseNotifier use_notifier,
	516	MSIVectorReleaseNotifier release_notifier)
	517	{
	518	int vector, ret;
	519
	520	assert(use_notifier && release_notifier);
	521
	522	dev->msix_vector_use_notifier = use_notifier;
	523	dev->msix_vector_release_notifier = release_notifier;
	524
	525	if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
	526	(MSIX_ENABLE_MASK \| MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
	527	for (vector = 0; vector < dev->msix_entries_nr; vector++) {
	528	ret = msix_set_notifier_for_vector(dev, vector);
	529	if (ret < 0) {
	530	goto undo;
	531	}
	532	}
	533	}
	534	return 0;
	535
	536	undo:
	537	while (--vector >= 0) {
	538	msix_unset_notifier_for_vector(dev, vector);
	539	}
	540	dev->msix_vector_use_notifier = NULL;
	541	dev->msix_vector_release_notifier = NULL;
	542	return ret;
	543	}
	544
	545	void msix_unset_vector_notifiers(PCIDevice *dev)
	546	{
	547	int vector;
	548
	549	assert(dev->msix_vector_use_notifier &&
	550	dev->msix_vector_release_notifier);
	551
	552	if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
	553	(MSIX_ENABLE_MASK \| MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
	554	for (vector = 0; vector < dev->msix_entries_nr; vector++) {
	555	msix_unset_notifier_for_vector(dev, vector);
	556	}
	557	}
	558	dev->msix_vector_use_notifier = NULL;
559	dev->msix_vector_release_notifier = NULL;
560	}