[qemu.git] / hw / msix.c

/*
 * MSI-X device support
 *
 * This module includes support for MSI-X in pci devices.
 *
 * Author: Michael S. Tsirkin <mst@redhat.com>
 *
 *  Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
 */

#include "hw.h"
#include "msi.h"
#include "msix.h"
#include "pci.h"
#include "range.h"

#define MSIX_CAP_LENGTH 12

/* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
#define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)

static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
{
    uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
    MSIMessage msg;

    msg.address = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
    msg.data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
    return msg;
}

/*
 * Special API for POWER to configure the vectors through
 * a side channel. Should never be used by devices.
 */
void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)
{
    uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;

    pci_set_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR, msg.address);
    pci_set_long(table_entry + PCI_MSIX_ENTRY_DATA, msg.data);
    table_entry[PCI_MSIX_ENTRY_VECTOR_CTRL] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
}

static uint8_t msix_pending_mask(int vector)
{
    return 1 << (vector % 8);
}

static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)
{
    return dev->msix_pba + vector / 8;
}

static int msix_is_pending(PCIDevice *dev, int vector)
{
    return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
}

static void msix_set_pending(PCIDevice *dev, int vector)
{
    *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
}

static void msix_clr_pending(PCIDevice *dev, int vector)
{
    *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
}

static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
{
    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
    return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
}

static bool msix_is_masked(PCIDevice *dev, int vector)
{
    return msix_vector_masked(dev, vector, dev->msix_function_masked);
}

static void msix_fire_vector_notifier(PCIDevice *dev,
                                      unsigned int vector, bool is_masked)
{
    MSIMessage msg;
    int ret;

    if (!dev->msix_vector_use_notifier) {
        return;
    }
    if (is_masked) {
        dev->msix_vector_release_notifier(dev, vector);
    } else {
        msg = msix_get_message(dev, vector);
        ret = dev->msix_vector_use_notifier(dev, vector, msg);
        assert(ret >= 0);
    }
}

static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
{
    bool is_masked = msix_is_masked(dev, vector);

    if (is_masked == was_masked) {
        return;
    }

    msix_fire_vector_notifier(dev, vector, is_masked);

    if (!is_masked && msix_is_pending(dev, vector)) {
        msix_clr_pending(dev, vector);
        msix_notify(dev, vector);
    }
}

static void msix_update_function_masked(PCIDevice *dev)
{
    dev->msix_function_masked = !msix_enabled(dev) ||
        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
}

/* Handle MSI-X capability config write. */
void msix_write_config(PCIDevice *dev, uint32_t addr,
                       uint32_t val, int len)
{
    unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
    int vector;
    bool was_masked;

    if (!msix_present(dev) || !range_covers_byte(addr, len, enable_pos)) {
        return;
    }

    was_masked = dev->msix_function_masked;
    msix_update_function_masked(dev);

    if (!msix_enabled(dev)) {
        return;
    }

    pci_device_deassert_intx(dev);

    if (dev->msix_function_masked == was_masked) {
        return;
    }

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        msix_handle_mask_update(dev, vector,
                                msix_vector_masked(dev, vector, was_masked));
    }
}

static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
                                     unsigned size)
{
    PCIDevice *dev = opaque;

    return pci_get_long(dev->msix_table + addr);
}

static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
                                  uint64_t val, unsigned size)
{
    PCIDevice *dev = opaque;
    int vector = addr / PCI_MSIX_ENTRY_SIZE;
    bool was_masked;

    was_masked = msix_is_masked(dev, vector);
    pci_set_long(dev->msix_table + addr, val);
    msix_handle_mask_update(dev, vector, was_masked);
}

static const MemoryRegionOps msix_table_mmio_ops = {
    .read = msix_table_mmio_read,
    .write = msix_table_mmio_write,
    /* TODO: MSIX should be LITTLE_ENDIAN. */
    .endianness = DEVICE_NATIVE_ENDIAN,
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
};

static uint64_t msix_pba_mmio_read(void *opaque, target_phys_addr_t addr,
                                   unsigned size)
{
    PCIDevice *dev = opaque;

    return pci_get_long(dev->msix_pba + addr);
}

static const MemoryRegionOps msix_pba_mmio_ops = {
    .read = msix_pba_mmio_read,
    /* TODO: MSIX should be LITTLE_ENDIAN. */
    .endianness = DEVICE_NATIVE_ENDIAN,
    .valid = {
        .min_access_size = 4,
        .max_access_size = 4,
    },
};

static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
{
    int vector;

    for (vector = 0; vector < nentries; ++vector) {
        unsigned offset =
            vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
        bool was_masked = msix_is_masked(dev, vector);

        dev->msix_table[offset] |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
        msix_handle_mask_update(dev, vector, was_masked);
    }
}

/* Initialize the MSI-X structures */
int msix_init(struct PCIDevice *dev, unsigned short nentries,
              MemoryRegion *table_bar, uint8_t table_bar_nr,
              unsigned table_offset, MemoryRegion *pba_bar,
              uint8_t pba_bar_nr, unsigned pba_offset, uint8_t cap_pos)
{
    int cap;
    unsigned table_size, pba_size;
    uint8_t *config;

    /* Nothing to do if MSI is not supported by interrupt controller */
    if (!msi_supported) {
        return -ENOTSUP;
    }

    if (nentries < 1 || nentries > PCI_MSIX_FLAGS_QSIZE + 1) {
        return -EINVAL;
    }

    table_size = nentries * PCI_MSIX_ENTRY_SIZE;
    pba_size = QEMU_ALIGN_UP(nentries, 64) / 8;

    /* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
    if ((table_bar_nr == pba_bar_nr &&
         ranges_overlap(table_offset, table_size, pba_offset, pba_size)) ||
        table_offset + table_size > memory_region_size(table_bar) ||
        pba_offset + pba_size > memory_region_size(pba_bar) ||
        (table_offset | pba_offset) & PCI_MSIX_FLAGS_BIRMASK) {
        return -EINVAL;
    }

    cap = pci_add_capability(dev, PCI_CAP_ID_MSIX, cap_pos, MSIX_CAP_LENGTH);
    if (cap < 0) {
        return cap;
    }

    dev->msix_cap = cap;
    dev->cap_present |= QEMU_PCI_CAP_MSIX;
    config = dev->config + cap;

    pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
    dev->msix_entries_nr = nentries;
    dev->msix_function_masked = true;

    pci_set_long(config + PCI_MSIX_TABLE, table_offset | table_bar_nr);
    pci_set_long(config + PCI_MSIX_PBA, pba_offset | pba_bar_nr);

    /* Make flags bit writable. */
    dev->wmask[cap + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
                                             MSIX_MASKALL_MASK;

    dev->msix_table = g_malloc0(table_size);
    dev->msix_pba = g_malloc0(pba_size);
    dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);

    msix_mask_all(dev, nentries);

    memory_region_init_io(&dev->msix_table_mmio, &msix_table_mmio_ops, dev,
                          "msix-table", table_size);
    memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);
    memory_region_init_io(&dev->msix_pba_mmio, &msix_pba_mmio_ops, dev,
                          "msix-pba", pba_size);
    memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);

    return 0;
}

int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
                            uint8_t bar_nr)
{
    int ret;
    char *name;

    /*
     * Migration compatibility dictates that this remains a 4k
     * BAR with the vector table in the lower half and PBA in
     * the upper half.  Do not use these elsewhere!
     */
#define MSIX_EXCLUSIVE_BAR_SIZE 4096
#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
#define MSIX_EXCLUSIVE_CAP_OFFSET 0

    if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
        return -EINVAL;
    }

    if (asprintf(&name, "%s-msix", dev->name) == -1) {
        return -ENOMEM;
    }

    memory_region_init(&dev->msix_exclusive_bar, name, MSIX_EXCLUSIVE_BAR_SIZE);

    free(name);

    ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
                    MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
                    bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
                    MSIX_EXCLUSIVE_CAP_OFFSET);
    if (ret) {
        memory_region_destroy(&dev->msix_exclusive_bar);
        return ret;
    }

    pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,
                     &dev->msix_exclusive_bar);

    return 0;
}

static void msix_free_irq_entries(PCIDevice *dev)
{
    int vector;

    for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
        dev->msix_entry_used[vector] = 0;
        msix_clr_pending(dev, vector);
    }
}

/* Clean up resources for the device. */
void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)
{
    if (!msix_present(dev)) {
        return;
    }
    pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
    dev->msix_cap = 0;
    msix_free_irq_entries(dev);
    dev->msix_entries_nr = 0;
    memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
    memory_region_destroy(&dev->msix_pba_mmio);
    g_free(dev->msix_pba);
    dev->msix_pba = NULL;
    memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
    memory_region_destroy(&dev->msix_table_mmio);
    g_free(dev->msix_table);
    dev->msix_table = NULL;
    g_free(dev->msix_entry_used);
    dev->msix_entry_used = NULL;
    dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
    return;
}

void msix_uninit_exclusive_bar(PCIDevice *dev)
{
    if (msix_present(dev)) {
        msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
        memory_region_destroy(&dev->msix_exclusive_bar);
    }
}

void msix_save(PCIDevice *dev, QEMUFile *f)
{
    unsigned n = dev->msix_entries_nr;

    if (!msix_present(dev)) {
        return;
    }

    qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
    qemu_put_buffer(f, dev->msix_pba, (n + 7) / 8);
}

/* Should be called after restoring the config space. */
void msix_load(PCIDevice *dev, QEMUFile *f)
{
    unsigned n = dev->msix_entries_nr;
    unsigned int vector;

    if (!msix_present(dev)) {
        return;
    }

    msix_free_irq_entries(dev);
    qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
    qemu_get_buffer(f, dev->msix_pba, (n + 7) / 8);
    msix_update_function_masked(dev);

    for (vector = 0; vector < n; vector++) {
        msix_handle_mask_update(dev, vector, true);
    }
}

/* Does device support MSI-X? */
int msix_present(PCIDevice *dev)
{
    return dev->cap_present & QEMU_PCI_CAP_MSIX;
}

/* Is MSI-X enabled? */
int msix_enabled(PCIDevice *dev)
{
    return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
        (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
         MSIX_ENABLE_MASK);
}

/* Send an MSI-X message */
void msix_notify(PCIDevice *dev, unsigned vector)
{
    MSIMessage msg;

    if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector])
        return;
    if (msix_is_masked(dev, vector)) {
        msix_set_pending(dev, vector);
        return;
    }

    msg = msix_get_message(dev, vector);

    stl_le_phys(msg.address, msg.data);
}

void msix_reset(PCIDevice *dev)
{
    if (!msix_present(dev)) {
        return;
    }
    msix_free_irq_entries(dev);
    dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
	    ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
    memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
    memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
    msix_mask_all(dev, dev->msix_entries_nr);
}

/* PCI spec suggests that devices make it possible for software to configure
 * less vectors than supported by the device, but does not specify a standard
 * mechanism for devices to do so.
 *
 * We support this by asking devices to declare vectors software is going to
 * actually use, and checking this on the notification path. Devices that
 * don't want to follow the spec suggestion can declare all vectors as used. */

/* Mark vector as used. */
int msix_vector_use(PCIDevice *dev, unsigned vector)
{
    if (vector >= dev->msix_entries_nr)
        return -EINVAL;
    dev->msix_entry_used[vector]++;
    return 0;
}

/* Mark vector as unused. */
void msix_vector_unuse(PCIDevice *dev, unsigned vector)
{
    if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) {
        return;
    }
    if (--dev->msix_entry_used[vector]) {
        return;
    }
    msix_clr_pending(dev, vector);
}

void msix_unuse_all_vectors(PCIDevice *dev)
{
    if (!msix_present(dev)) {
        return;
    }
    msix_free_irq_entries(dev);
}

unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)
{
    return dev->msix_entries_nr;
}

static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)
{
    MSIMessage msg;

    if (msix_is_masked(dev, vector)) {
        return 0;
    }
    msg = msix_get_message(dev, vector);
    return dev->msix_vector_use_notifier(dev, vector, msg);
}

static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
{
    if (msix_is_masked(dev, vector)) {
        return;
    }
    dev->msix_vector_release_notifier(dev, vector);
}

int msix_set_vector_notifiers(PCIDevice *dev,
                              MSIVectorUseNotifier use_notifier,
                              MSIVectorReleaseNotifier release_notifier)
{
    int vector, ret;

    assert(use_notifier && release_notifier);

    dev->msix_vector_use_notifier = use_notifier;
    dev->msix_vector_release_notifier = release_notifier;

    if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
        (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
        for (vector = 0; vector < dev->msix_entries_nr; vector++) {
            ret = msix_set_notifier_for_vector(dev, vector);
            if (ret < 0) {
                goto undo;
            }
        }
    }
    return 0;

undo:
    while (--vector >= 0) {
        msix_unset_notifier_for_vector(dev, vector);
    }
    dev->msix_vector_use_notifier = NULL;
    dev->msix_vector_release_notifier = NULL;
    return ret;
}

void msix_unset_vector_notifiers(PCIDevice *dev)
{
    int vector;

    assert(dev->msix_vector_use_notifier &&
           dev->msix_vector_release_notifier);

    if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
        (MSIX_ENABLE_MASK | MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
        for (vector = 0; vector < dev->msix_entries_nr; vector++) {
            msix_unset_notifier_for_vector(dev, vector);
        }
    }
    dev->msix_vector_use_notifier = NULL;
    dev->msix_vector_release_notifier = NULL;
}
Commit	Line	Data
02eb84d0 MT	1	/*
	2	* MSI-X device support
	3	*
	4	* This module includes support for MSI-X in pci devices.
	5	*
	6	* Author: Michael S. Tsirkin <mst@redhat.com>
	7	*
	8	* Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
	9	*
	10	* This work is licensed under the terms of the GNU GPL, version 2. See
	11	* the COPYING file in the top-level directory.
6b620ca3 PB	12	*
	13	* Contributions after 2012-01-13 are licensed under the terms of the
	14	* GNU GPL, version 2 or (at your option) any later version.
02eb84d0 MT	15	*/
	16
	17	#include "hw.h"
60ba3cc2	18	#include "msi.h"
02eb84d0 MT	19	#include "msix.h"
02eb84d0 MT	20	#include "pci.h"
bf1b0071	21	#include "range.h"
02eb84d0	22
02eb84d0 MT	23	#define MSIX_CAP_LENGTH 12
02eb84d0 MT	24
2760952b MT	25	/* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
2760952b MT	26	#define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
02eb84d0	27	#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
5b5cb086	28	#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
02eb84d0	29
bc4caf49 JK	30	static MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)
bc4caf49 JK	31	{
d35e428c	32	uint8_t table_entry = dev->msix_table + vector PCI_MSIX_ENTRY_SIZE;
bc4caf49 JK	33	MSIMessage msg;
	34
	35	msg.address = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
	36	msg.data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
	37	return msg;
	38	}
02eb84d0	39
932d4a42 AK	40	/*
	41	* Special API for POWER to configure the vectors through
	42	* a side channel. Should never be used by devices.
	43	*/
	44	void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)
	45	{
	46	uint8_t table_entry = dev->msix_table + vector PCI_MSIX_ENTRY_SIZE;
	47
	48	pci_set_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR, msg.address);
	49	pci_set_long(table_entry + PCI_MSIX_ENTRY_DATA, msg.data);
	50	table_entry[PCI_MSIX_ENTRY_VECTOR_CTRL] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
	51	}
	52
02eb84d0 MT	53	static uint8_t msix_pending_mask(int vector)
	54	{
	55	return 1 << (vector % 8);
	56	}
	57
	58	static uint8_t msix_pending_byte(PCIDevice dev, int vector)
	59	{
d35e428c	60	return dev->msix_pba + vector / 8;
02eb84d0 MT	61	}
	62
	63	static int msix_is_pending(PCIDevice *dev, int vector)
	64	{
	65	return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
	66	}
	67
	68	static void msix_set_pending(PCIDevice *dev, int vector)
	69	{
	70	*msix_pending_byte(dev, vector) \|= msix_pending_mask(vector);
	71	}
	72
	73	static void msix_clr_pending(PCIDevice *dev, int vector)
	74	{
	75	*msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
	76	}
	77
ae392c41	78	static bool msix_vector_masked(PCIDevice *dev, int vector, bool fmask)
02eb84d0	79	{
ae392c41	80	unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
d35e428c	81	return fmask \|\| dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
5b5cb086 MT	82	}
5b5cb086 MT	83
ae392c41	84	static bool msix_is_masked(PCIDevice *dev, int vector)
5b5cb086	85	{
ae392c41 MT	86	return msix_vector_masked(dev, vector, dev->msix_function_masked);
	87	}
	88
2cdfe53c JK	89	static void msix_fire_vector_notifier(PCIDevice *dev,
	90	unsigned int vector, bool is_masked)
	91	{
	92	MSIMessage msg;
	93	int ret;
	94
	95	if (!dev->msix_vector_use_notifier) {
	96	return;
	97	}
	98	if (is_masked) {
	99	dev->msix_vector_release_notifier(dev, vector);
	100	} else {
	101	msg = msix_get_message(dev, vector);
	102	ret = dev->msix_vector_use_notifier(dev, vector, msg);
	103	assert(ret >= 0);
	104	}
	105	}
	106
ae392c41 MT	107	static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)
	108	{
	109	bool is_masked = msix_is_masked(dev, vector);
2cdfe53c	110
ae392c41 MT	111	if (is_masked == was_masked) {
	112	return;
	113	}
	114
2cdfe53c JK	115	msix_fire_vector_notifier(dev, vector, is_masked);
2cdfe53c JK	116
ae392c41	117	if (!is_masked && msix_is_pending(dev, vector)) {
5b5cb086 MT	118	msix_clr_pending(dev, vector);
	119	msix_notify(dev, vector);
	120	}
	121	}
	122
50322249 MT	123	static void msix_update_function_masked(PCIDevice *dev)
	124	{
	125	dev->msix_function_masked = !msix_enabled(dev) \|\|
	126	(dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK);
	127	}
	128
5b5cb086 MT	129	/* Handle MSI-X capability config write. */
	130	void msix_write_config(PCIDevice *dev, uint32_t addr,
	131	uint32_t val, int len)
	132	{
	133	unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
	134	int vector;
50322249	135	bool was_masked;
5b5cb086	136
7c9958b0	137	if (!msix_present(dev) \|\| !range_covers_byte(addr, len, enable_pos)) {
5b5cb086 MT	138	return;
	139	}
	140
50322249 MT	141	was_masked = dev->msix_function_masked;
	142	msix_update_function_masked(dev);
	143
5b5cb086 MT	144	if (!msix_enabled(dev)) {
	145	return;
	146	}
	147
e407bf13	148	pci_device_deassert_intx(dev);
5b5cb086	149
50322249	150	if (dev->msix_function_masked == was_masked) {
5b5cb086 MT	151	return;
	152	}
	153
	154	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
ae392c41 MT	155	msix_handle_mask_update(dev, vector,
ae392c41 MT	156	msix_vector_masked(dev, vector, was_masked));
5b5cb086	157	}
02eb84d0 MT	158	}
02eb84d0 MT	159
d35e428c AW	160	static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
d35e428c AW	161	unsigned size)
eebcb0a7 AW	162	{
eebcb0a7 AW	163	PCIDevice *dev = opaque;
eebcb0a7	164
d35e428c	165	return pci_get_long(dev->msix_table + addr);
eebcb0a7 AW	166	}
eebcb0a7 AW	167
d35e428c AW	168	static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
d35e428c AW	169	uint64_t val, unsigned size)
02eb84d0 MT	170	{
02eb84d0 MT	171	PCIDevice *dev = opaque;
d35e428c	172	int vector = addr / PCI_MSIX_ENTRY_SIZE;
ae392c41	173	bool was_masked;
9a93b617	174
ae392c41	175	was_masked = msix_is_masked(dev, vector);
d35e428c	176	pci_set_long(dev->msix_table + addr, val);
ae392c41	177	msix_handle_mask_update(dev, vector, was_masked);
02eb84d0 MT	178	}
02eb84d0 MT	179
d35e428c AW	180	static const MemoryRegionOps msix_table_mmio_ops = {
	181	.read = msix_table_mmio_read,
	182	.write = msix_table_mmio_write,
	183	/* TODO: MSIX should be LITTLE_ENDIAN. */
	184	.endianness = DEVICE_NATIVE_ENDIAN,
	185	.valid = {
	186	.min_access_size = 4,
	187	.max_access_size = 4,
	188	},
	189	};
	190
	191	static uint64_t msix_pba_mmio_read(void *opaque, target_phys_addr_t addr,
	192	unsigned size)
	193	{
	194	PCIDevice *dev = opaque;
	195
	196	return pci_get_long(dev->msix_pba + addr);
	197	}
	198
	199	static const MemoryRegionOps msix_pba_mmio_ops = {
	200	.read = msix_pba_mmio_read,
2cf62ad7	201	/* TODO: MSIX should be LITTLE_ENDIAN. */
95524ae8 AK	202	.endianness = DEVICE_NATIVE_ENDIAN,
	203	.valid = {
	204	.min_access_size = 4,
	205	.max_access_size = 4,
	206	},
02eb84d0 MT	207	};
02eb84d0 MT	208
ae1be0bb MT	209	static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
	210	{
	211	int vector;
5b5f1330	212
ae1be0bb	213	for (vector = 0; vector < nentries; ++vector) {
01731cfb JK	214	unsigned offset =
01731cfb JK	215	vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
5b5f1330 JK	216	bool was_masked = msix_is_masked(dev, vector);
5b5f1330 JK	217
d35e428c	218	dev->msix_table[offset] \|= PCI_MSIX_ENTRY_CTRL_MASKBIT;
5b5f1330	219	msix_handle_mask_update(dev, vector, was_masked);
ae1be0bb MT	220	}
	221	}
	222
5a2c2029	223	/* Initialize the MSI-X structures */
02eb84d0	224	int msix_init(struct PCIDevice *dev, unsigned short nentries,
5a2c2029 AW	225	MemoryRegion *table_bar, uint8_t table_bar_nr,
	226	unsigned table_offset, MemoryRegion *pba_bar,
	227	uint8_t pba_bar_nr, unsigned pba_offset, uint8_t cap_pos)
02eb84d0	228	{
5a2c2029	229	int cap;
d35e428c	230	unsigned table_size, pba_size;
5a2c2029	231	uint8_t *config;
60ba3cc2	232
02eb84d0	233	/* Nothing to do if MSI is not supported by interrupt controller */
60ba3cc2	234	if (!msi_supported) {
02eb84d0	235	return -ENOTSUP;
60ba3cc2	236	}
5a2c2029 AW	237
5a2c2029 AW	238	if (nentries < 1 \|\| nentries > PCI_MSIX_FLAGS_QSIZE + 1) {
02eb84d0	239	return -EINVAL;
5a2c2029	240	}
02eb84d0	241
d35e428c AW	242	table_size = nentries * PCI_MSIX_ENTRY_SIZE;
	243	pba_size = QEMU_ALIGN_UP(nentries, 64) / 8;
	244
5a2c2029 AW	245	/* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
	246	if ((table_bar_nr == pba_bar_nr &&
	247	ranges_overlap(table_offset, table_size, pba_offset, pba_size)) \|\|
	248	table_offset + table_size > memory_region_size(table_bar) \|\|
	249	pba_offset + pba_size > memory_region_size(pba_bar) \|\|
	250	(table_offset \| pba_offset) & PCI_MSIX_FLAGS_BIRMASK) {
	251	return -EINVAL;
	252	}
	253
	254	cap = pci_add_capability(dev, PCI_CAP_ID_MSIX, cap_pos, MSIX_CAP_LENGTH);
	255	if (cap < 0) {
	256	return cap;
	257	}
	258
	259	dev->msix_cap = cap;
	260	dev->cap_present \|= QEMU_PCI_CAP_MSIX;
	261	config = dev->config + cap;
	262
	263	pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
	264	dev->msix_entries_nr = nentries;
	265	dev->msix_function_masked = true;
	266
	267	pci_set_long(config + PCI_MSIX_TABLE, table_offset \| table_bar_nr);
	268	pci_set_long(config + PCI_MSIX_PBA, pba_offset \| pba_bar_nr);
	269
	270	/* Make flags bit writable. */
	271	dev->wmask[cap + MSIX_CONTROL_OFFSET] \|= MSIX_ENABLE_MASK \|
	272	MSIX_MASKALL_MASK;
02eb84d0	273
d35e428c AW	274	dev->msix_table = g_malloc0(table_size);
d35e428c AW	275	dev->msix_pba = g_malloc0(pba_size);
5a2c2029 AW	276	dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);
5a2c2029 AW	277
ae1be0bb	278	msix_mask_all(dev, nentries);
02eb84d0	279
d35e428c AW	280	memory_region_init_io(&dev->msix_table_mmio, &msix_table_mmio_ops, dev,
d35e428c AW	281	"msix-table", table_size);
5a2c2029	282	memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);
d35e428c AW	283	memory_region_init_io(&dev->msix_pba_mmio, &msix_pba_mmio_ops, dev,
d35e428c AW	284	"msix-pba", pba_size);
5a2c2029	285	memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);
02eb84d0	286
02eb84d0	287	return 0;
02eb84d0 MT	288	}
02eb84d0 MT	289
53f94925 AW	290	int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
	291	uint8_t bar_nr)
	292	{
	293	int ret;
	294	char *name;
	295
	296	/*
	297	* Migration compatibility dictates that this remains a 4k
	298	* BAR with the vector table in the lower half and PBA in
	299	* the upper half. Do not use these elsewhere!
	300	*/
	301	#define MSIX_EXCLUSIVE_BAR_SIZE 4096
5a2c2029	302	#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
53f94925	303	#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
5a2c2029	304	#define MSIX_EXCLUSIVE_CAP_OFFSET 0
53f94925 AW	305
	306	if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
	307	return -EINVAL;
	308	}
	309
	310	if (asprintf(&name, "%s-msix", dev->name) == -1) {
	311	return -ENOMEM;
	312	}
	313
	314	memory_region_init(&dev->msix_exclusive_bar, name, MSIX_EXCLUSIVE_BAR_SIZE);
	315
	316	free(name);
	317
	318	ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
5a2c2029 AW	319	MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
	320	bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
	321	MSIX_EXCLUSIVE_CAP_OFFSET);
53f94925 AW	322	if (ret) {
	323	memory_region_destroy(&dev->msix_exclusive_bar);
	324	return ret;
	325	}
	326
	327	pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,
	328	&dev->msix_exclusive_bar);
	329
	330	return 0;
	331	}
	332
98304c84 MT	333	static void msix_free_irq_entries(PCIDevice *dev)
	334	{
	335	int vector;
	336
	337	for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
	338	dev->msix_entry_used[vector] = 0;
	339	msix_clr_pending(dev, vector);
	340	}
	341	}
	342
02eb84d0	343	/* Clean up resources for the device. */
572992ee	344	void msix_uninit(PCIDevice dev, MemoryRegion table_bar, MemoryRegion *pba_bar)
02eb84d0	345	{
44701ab7	346	if (!msix_present(dev)) {
572992ee	347	return;
44701ab7	348	}
02eb84d0 MT	349	pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
	350	dev->msix_cap = 0;
	351	msix_free_irq_entries(dev);
	352	dev->msix_entries_nr = 0;
5a2c2029	353	memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);
d35e428c AW	354	memory_region_destroy(&dev->msix_pba_mmio);
	355	g_free(dev->msix_pba);
	356	dev->msix_pba = NULL;
5a2c2029	357	memory_region_del_subregion(table_bar, &dev->msix_table_mmio);
d35e428c AW	358	memory_region_destroy(&dev->msix_table_mmio);
	359	g_free(dev->msix_table);
	360	dev->msix_table = NULL;
7267c094	361	g_free(dev->msix_entry_used);
02eb84d0 MT	362	dev->msix_entry_used = NULL;
02eb84d0 MT	363	dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
572992ee	364	return;
02eb84d0 MT	365	}
02eb84d0 MT	366
53f94925 AW	367	void msix_uninit_exclusive_bar(PCIDevice *dev)
	368	{
	369	if (msix_present(dev)) {
5a2c2029	370	msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);
53f94925 AW	371	memory_region_destroy(&dev->msix_exclusive_bar);
	372	}
	373	}
	374
02eb84d0 MT	375	void msix_save(PCIDevice dev, QEMUFile f)
02eb84d0 MT	376	{
9a3e12c8 MT	377	unsigned n = dev->msix_entries_nr;
9a3e12c8 MT	378
44701ab7	379	if (!msix_present(dev)) {
9a3e12c8	380	return;
72755a70	381	}
9a3e12c8	382
d35e428c AW	383	qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	384	qemu_put_buffer(f, dev->msix_pba, (n + 7) / 8);
02eb84d0 MT	385	}
	386
	387	/* Should be called after restoring the config space. */
	388	void msix_load(PCIDevice dev, QEMUFile f)
	389	{
	390	unsigned n = dev->msix_entries_nr;
2cdfe53c	391	unsigned int vector;
02eb84d0	392
44701ab7	393	if (!msix_present(dev)) {
02eb84d0	394	return;
98846d73	395	}
02eb84d0	396
4bfd1712	397	msix_free_irq_entries(dev);
d35e428c AW	398	qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	399	qemu_get_buffer(f, dev->msix_pba, (n + 7) / 8);
50322249	400	msix_update_function_masked(dev);
2cdfe53c JK	401
	402	for (vector = 0; vector < n; vector++) {
	403	msix_handle_mask_update(dev, vector, true);
	404	}
02eb84d0 MT	405	}
	406
	407	/* Does device support MSI-X? */
	408	int msix_present(PCIDevice *dev)
	409	{
	410	return dev->cap_present & QEMU_PCI_CAP_MSIX;
	411	}
	412
	413	/* Is MSI-X enabled? */
	414	int msix_enabled(PCIDevice *dev)
	415	{
	416	return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
2760952b	417	(dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
02eb84d0 MT	418	MSIX_ENABLE_MASK);
	419	}
	420
02eb84d0 MT	421	/* Send an MSI-X message */
	422	void msix_notify(PCIDevice *dev, unsigned vector)
	423	{
bc4caf49	424	MSIMessage msg;
02eb84d0 MT	425
	426	if (vector >= dev->msix_entries_nr \|\| !dev->msix_entry_used[vector])
	427	return;
	428	if (msix_is_masked(dev, vector)) {
	429	msix_set_pending(dev, vector);
	430	return;
	431	}
	432
bc4caf49 JK	433	msg = msix_get_message(dev, vector);
	434
	435	stl_le_phys(msg.address, msg.data);
02eb84d0 MT	436	}
	437
	438	void msix_reset(PCIDevice *dev)
	439	{
44701ab7	440	if (!msix_present(dev)) {
02eb84d0	441	return;
44701ab7	442	}
02eb84d0	443	msix_free_irq_entries(dev);
2760952b MT	444	dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
2760952b MT	445	~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
d35e428c AW	446	memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
d35e428c AW	447	memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
ae1be0bb	448	msix_mask_all(dev, dev->msix_entries_nr);
02eb84d0 MT	449	}
	450
	451	/* PCI spec suggests that devices make it possible for software to configure
	452	* less vectors than supported by the device, but does not specify a standard
	453	* mechanism for devices to do so.
	454	*
	455	* We support this by asking devices to declare vectors software is going to
	456	* actually use, and checking this on the notification path. Devices that
	457	* don't want to follow the spec suggestion can declare all vectors as used. */
	458
	459	/* Mark vector as used. */
	460	int msix_vector_use(PCIDevice *dev, unsigned vector)
	461	{
	462	if (vector >= dev->msix_entries_nr)
	463	return -EINVAL;
	464	dev->msix_entry_used[vector]++;
	465	return 0;
	466	}
	467
	468	/* Mark vector as unused. */
	469	void msix_vector_unuse(PCIDevice *dev, unsigned vector)
	470	{
98304c84 MT	471	if (vector >= dev->msix_entries_nr \|\| !dev->msix_entry_used[vector]) {
	472	return;
	473	}
	474	if (--dev->msix_entry_used[vector]) {
	475	return;
	476	}
	477	msix_clr_pending(dev, vector);
02eb84d0	478	}
b5f28bca MT	479
	480	void msix_unuse_all_vectors(PCIDevice *dev)
	481	{
44701ab7	482	if (!msix_present(dev)) {
b5f28bca	483	return;
44701ab7	484	}
b5f28bca MT	485	msix_free_irq_entries(dev);
b5f28bca MT	486	}
2cdfe53c	487
cb697aaa JK	488	unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)
	489	{
	490	return dev->msix_entries_nr;
	491	}
	492
2cdfe53c JK	493	static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)
	494	{
	495	MSIMessage msg;
	496
	497	if (msix_is_masked(dev, vector)) {
	498	return 0;
	499	}
	500	msg = msix_get_message(dev, vector);
	501	return dev->msix_vector_use_notifier(dev, vector, msg);
	502	}
	503
	504	static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)
	505	{
	506	if (msix_is_masked(dev, vector)) {
	507	return;
	508	}
	509	dev->msix_vector_release_notifier(dev, vector);
	510	}
	511
	512	int msix_set_vector_notifiers(PCIDevice *dev,
	513	MSIVectorUseNotifier use_notifier,
	514	MSIVectorReleaseNotifier release_notifier)
	515	{
	516	int vector, ret;
	517
	518	assert(use_notifier && release_notifier);
	519
	520	dev->msix_vector_use_notifier = use_notifier;
	521	dev->msix_vector_release_notifier = release_notifier;
	522
	523	if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
	524	(MSIX_ENABLE_MASK \| MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
	525	for (vector = 0; vector < dev->msix_entries_nr; vector++) {
	526	ret = msix_set_notifier_for_vector(dev, vector);
	527	if (ret < 0) {
	528	goto undo;
	529	}
	530	}
	531	}
	532	return 0;
	533
	534	undo:
	535	while (--vector >= 0) {
	536	msix_unset_notifier_for_vector(dev, vector);
	537	}
	538	dev->msix_vector_use_notifier = NULL;
	539	dev->msix_vector_release_notifier = NULL;
	540	return ret;
	541	}
	542
	543	void msix_unset_vector_notifiers(PCIDevice *dev)
	544	{
	545	int vector;
	546
	547	assert(dev->msix_vector_use_notifier &&
	548	dev->msix_vector_release_notifier);
	549
	550	if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
	551	(MSIX_ENABLE_MASK \| MSIX_MASKALL_MASK)) == MSIX_ENABLE_MASK) {
	552	for (vector = 0; vector < dev->msix_entries_nr; vector++) {
	553	msix_unset_notifier_for_vector(dev, vector);
	554	}
	555	}
	556	dev->msix_vector_use_notifier = NULL;
557	dev->msix_vector_release_notifier = NULL;
558	}