[mirror_ubuntu-bionic-kernel.git] / drivers / virtio / virtio_pci.c

/*
 * Virtio PCI driver
 *
 * This module allows virtio devices to be used over a virtual PCI device.
 * This can be used with QEMU based VMMs like KVM or Xen.
 *
 * Copyright IBM Corp. 2007
 *
 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

#include <linux/module.h>
#include <linux/list.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/virtio.h>
#include <linux/virtio_config.h>
#include <linux/virtio_ring.h>
#include <linux/virtio_pci.h>
#include <linux/highmem.h>
#include <linux/spinlock.h>

MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
MODULE_DESCRIPTION("virtio-pci");
MODULE_LICENSE("GPL");
MODULE_VERSION("1");

/* Our device structure */
struct virtio_pci_device
{
	struct virtio_device vdev;
	struct pci_dev *pci_dev;

	/* the IO mapping for the PCI config space */
	void __iomem *ioaddr;

	/* a list of queues so we can dispatch IRQs */
	spinlock_t lock;
	struct list_head virtqueues;
};

struct virtio_pci_vq_info
{
	/* the actual virtqueue */
	struct virtqueue *vq;

	/* the number of entries in the queue */
	int num;

	/* the index of the queue */
	int queue_index;

	/* the virtual address of the ring queue */
	void *queue;

	/* the list node for the virtqueues list */
	struct list_head node;
};

/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
static struct pci_device_id virtio_pci_id_table[] = {
	{ 0x1af4, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
	{ 0 },
};

MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);

/* A PCI device has it's own struct device and so does a virtio device so
 * we create a place for the virtio devices to show up in sysfs.  I think it
 * would make more sense for virtio to not insist on having it's own device. */
static struct device *virtio_pci_root;

/* Convert a generic virtio device to our structure */
static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
{
	return container_of(vdev, struct virtio_pci_device, vdev);
}

/* virtio config->get_features() implementation */
static u32 vp_get_features(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);

	/* When someone needs more than 32 feature bits, we'll need to
	 * steal a bit to indicate that the rest are somewhere else. */
	return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
}

/* virtio config->finalize_features() implementation */
static void vp_finalize_features(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);

	/* Give virtio_ring a chance to accept features. */
	vring_transport_features(vdev);

	/* We only support 32 feature bits. */
	BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
	iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
}

/* virtio config->get() implementation */
static void vp_get(struct virtio_device *vdev, unsigned offset,
		   void *buf, unsigned len)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	void __iomem *ioaddr = vp_dev->ioaddr + VIRTIO_PCI_CONFIG + offset;
	u8 *ptr = buf;
	int i;

	for (i = 0; i < len; i++)
		ptr[i] = ioread8(ioaddr + i);
}

/* the config->set() implementation.  it's symmetric to the config->get()
 * implementation */
static void vp_set(struct virtio_device *vdev, unsigned offset,
		   const void *buf, unsigned len)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	void __iomem *ioaddr = vp_dev->ioaddr + VIRTIO_PCI_CONFIG + offset;
	const u8 *ptr = buf;
	int i;

	for (i = 0; i < len; i++)
		iowrite8(ptr[i], ioaddr + i);
}

/* config->{get,set}_status() implementations */
static u8 vp_get_status(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
}

static void vp_set_status(struct virtio_device *vdev, u8 status)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	/* We should never be setting status to 0. */
	BUG_ON(status == 0);
	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
}

static void vp_reset(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	/* 0 status means a reset. */
	iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
}

/* the notify function used when creating a virt queue */
static void vp_notify(struct virtqueue *vq)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	struct virtio_pci_vq_info *info = vq->priv;

	/* we write the queue's selector into the notification register to
	 * signal the other end */
	iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
}

/* A small wrapper to also acknowledge the interrupt when it's handled.
 * I really need an EIO hook for the vring so I can ack the interrupt once we
 * know that we'll be handling the IRQ but before we invoke the callback since
 * the callback may notify the host which results in the host attempting to
 * raise an interrupt that we would then mask once we acknowledged the
 * interrupt. */
static irqreturn_t vp_interrupt(int irq, void *opaque)
{
	struct virtio_pci_device *vp_dev = opaque;
	struct virtio_pci_vq_info *info;
	irqreturn_t ret = IRQ_NONE;
	unsigned long flags;
	u8 isr;

	/* reading the ISR has the effect of also clearing it so it's very
	 * important to save off the value. */
	isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);

	/* It's definitely not us if the ISR was not high */
	if (!isr)
		return IRQ_NONE;

	/* Configuration change?  Tell driver if it wants to know. */
	if (isr & VIRTIO_PCI_ISR_CONFIG) {
		struct virtio_driver *drv;
		drv = container_of(vp_dev->vdev.dev.driver,
				   struct virtio_driver, driver);

		if (drv && drv->config_changed)
			drv->config_changed(&vp_dev->vdev);
	}

	spin_lock_irqsave(&vp_dev->lock, flags);
	list_for_each_entry(info, &vp_dev->virtqueues, node) {
		if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
			ret = IRQ_HANDLED;
	}
	spin_unlock_irqrestore(&vp_dev->lock, flags);

	return ret;
}

/* the config->find_vq() implementation */
static struct virtqueue *vp_find_vq(struct virtio_device *vdev, unsigned index,
				    void (*callback)(struct virtqueue *vq),
				    const char *name)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	struct virtio_pci_vq_info *info;
	struct virtqueue *vq;
	unsigned long flags, size;
	u16 num;
	int err;

	/* Select the queue we're interested in */
	iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);

	/* Check if queue is either not available or already active. */
	num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
	if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
		return ERR_PTR(-ENOENT);

	/* allocate and fill out our structure the represents an active
	 * queue */
	info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
	if (!info)
		return ERR_PTR(-ENOMEM);

	info->queue_index = index;
	info->num = num;

	size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
	info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
	if (info->queue == NULL) {
		err = -ENOMEM;
		goto out_info;
	}

	/* activate the queue */
	iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
		  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);

	/* create the vring */
	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
				 vdev, info->queue, vp_notify, callback, name);
	if (!vq) {
		err = -ENOMEM;
		goto out_activate_queue;
	}

	vq->priv = info;
	info->vq = vq;

	spin_lock_irqsave(&vp_dev->lock, flags);
	list_add(&info->node, &vp_dev->virtqueues);
	spin_unlock_irqrestore(&vp_dev->lock, flags);

	return vq;

out_activate_queue:
	iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
	free_pages_exact(info->queue, size);
out_info:
	kfree(info);
	return ERR_PTR(err);
}

/* the config->del_vq() implementation */
static void vp_del_vq(struct virtqueue *vq)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	struct virtio_pci_vq_info *info = vq->priv;
	unsigned long size;

	vring_del_virtqueue(vq);

	/* Select and deactivate the queue */
	iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
	iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);

	size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
	free_pages_exact(info->queue, size);
	kfree(info);
}

static void vp_del_vqs(struct virtio_device *vdev)
{
	struct virtqueue *vq, *n;

	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
		vp_del_vq(vq);
}

static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
		       struct virtqueue *vqs[],
		       vq_callback_t *callbacks[],
		       const char *names[])
{
	int i;

	for (i = 0; i < nvqs; ++i) {
		vqs[i] = vp_find_vq(vdev, i, callbacks[i], names[i]);
		if (IS_ERR(vqs[i]))
			goto error;
	}
	return 0;

error:
	vp_del_vqs(vdev);
	return PTR_ERR(vqs[i]);
}

static struct virtio_config_ops virtio_pci_config_ops = {
	.get		= vp_get,
	.set		= vp_set,
	.get_status	= vp_get_status,
	.set_status	= vp_set_status,
	.reset		= vp_reset,
	.find_vqs	= vp_find_vqs,
	.del_vqs	= vp_del_vqs,
	.get_features	= vp_get_features,
	.finalize_features = vp_finalize_features,
};

static void virtio_pci_release_dev(struct device *_d)
{
	struct virtio_device *dev = container_of(_d, struct virtio_device, dev);
	struct virtio_pci_device *vp_dev = to_vp_device(dev);
	struct pci_dev *pci_dev = vp_dev->pci_dev;

	free_irq(pci_dev->irq, vp_dev);
	pci_set_drvdata(pci_dev, NULL);
	pci_iounmap(pci_dev, vp_dev->ioaddr);
	pci_release_regions(pci_dev);
	pci_disable_device(pci_dev);
	kfree(vp_dev);
}

/* the PCI probing function */
static int __devinit virtio_pci_probe(struct pci_dev *pci_dev,
				      const struct pci_device_id *id)
{
	struct virtio_pci_device *vp_dev;
	int err;

	/* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
	if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
		return -ENODEV;

	if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
		printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
		       VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
		return -ENODEV;
	}

	/* allocate our structure and fill it out */
	vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
	if (vp_dev == NULL)
		return -ENOMEM;

	vp_dev->vdev.dev.parent = virtio_pci_root;
	vp_dev->vdev.dev.release = virtio_pci_release_dev;
	vp_dev->vdev.config = &virtio_pci_config_ops;
	vp_dev->pci_dev = pci_dev;
	INIT_LIST_HEAD(&vp_dev->virtqueues);
	spin_lock_init(&vp_dev->lock);

	/* enable the device */
	err = pci_enable_device(pci_dev);
	if (err)
		goto out;

	err = pci_request_regions(pci_dev, "virtio-pci");
	if (err)
		goto out_enable_device;

	vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
	if (vp_dev->ioaddr == NULL)
		goto out_req_regions;

	pci_set_drvdata(pci_dev, vp_dev);

	/* we use the subsystem vendor/device id as the virtio vendor/device
	 * id.  this allows us to use the same PCI vendor/device id for all
	 * virtio devices and to identify the particular virtio driver by
	 * the subsytem ids */
	vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
	vp_dev->vdev.id.device = pci_dev->subsystem_device;

	/* register a handler for the queue with the PCI device's interrupt */
	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
			  dev_name(&vp_dev->vdev.dev), vp_dev);
	if (err)
		goto out_set_drvdata;

	/* finally register the virtio device */
	err = register_virtio_device(&vp_dev->vdev);
	if (err)
		goto out_req_irq;

	return 0;

out_req_irq:
	free_irq(pci_dev->irq, vp_dev);
out_set_drvdata:
	pci_set_drvdata(pci_dev, NULL);
	pci_iounmap(pci_dev, vp_dev->ioaddr);
out_req_regions:
	pci_release_regions(pci_dev);
out_enable_device:
	pci_disable_device(pci_dev);
out:
	kfree(vp_dev);
	return err;
}

static void __devexit virtio_pci_remove(struct pci_dev *pci_dev)
{
	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);

	unregister_virtio_device(&vp_dev->vdev);
}

#ifdef CONFIG_PM
static int virtio_pci_suspend(struct pci_dev *pci_dev, pm_message_t state)
{
	pci_save_state(pci_dev);
	pci_set_power_state(pci_dev, PCI_D3hot);
	return 0;
}

static int virtio_pci_resume(struct pci_dev *pci_dev)
{
	pci_restore_state(pci_dev);
	pci_set_power_state(pci_dev, PCI_D0);
	return 0;
}
#endif

static struct pci_driver virtio_pci_driver = {
	.name		= "virtio-pci",
	.id_table	= virtio_pci_id_table,
	.probe		= virtio_pci_probe,
	.remove		= virtio_pci_remove,
#ifdef CONFIG_PM
	.suspend	= virtio_pci_suspend,
	.resume		= virtio_pci_resume,
#endif
};

static int __init virtio_pci_init(void)
{
	int err;

	virtio_pci_root = root_device_register("virtio-pci");
	if (IS_ERR(virtio_pci_root))
		return PTR_ERR(virtio_pci_root);

	err = pci_register_driver(&virtio_pci_driver);
	if (err)
		device_unregister(virtio_pci_root);

	return err;
}

module_init(virtio_pci_init);

static void __exit virtio_pci_exit(void)
{
	pci_unregister_driver(&virtio_pci_driver);
	root_device_unregister(virtio_pci_root);
}

module_exit(virtio_pci_exit);
Commit	Line	Data
3343660d AL	1	/*
	2	* Virtio PCI driver
	3	*
	4	* This module allows virtio devices to be used over a virtual PCI device.
	5	* This can be used with QEMU based VMMs like KVM or Xen.
	6	*
	7	* Copyright IBM Corp. 2007
	8	*
	9	* Authors:
	10	* Anthony Liguori <aliguori@us.ibm.com>
	11	*
	12	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	13	* See the COPYING file in the top-level directory.
	14	*
	15	*/
	16
	17	#include <linux/module.h>
	18	#include <linux/list.h>
	19	#include <linux/pci.h>
	20	#include <linux/interrupt.h>
	21	#include <linux/virtio.h>
	22	#include <linux/virtio_config.h>
	23	#include <linux/virtio_ring.h>
	24	#include <linux/virtio_pci.h>
	25	#include <linux/highmem.h>
	26	#include <linux/spinlock.h>
	27
	28	MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
	29	MODULE_DESCRIPTION("virtio-pci");
	30	MODULE_LICENSE("GPL");
	31	MODULE_VERSION("1");
	32
	33	/* Our device structure */
	34	struct virtio_pci_device
	35	{
	36	struct virtio_device vdev;
	37	struct pci_dev *pci_dev;
	38
	39	/* the IO mapping for the PCI config space */
97968358	40	void __iomem *ioaddr;
3343660d AL	41
	42	/* a list of queues so we can dispatch IRQs */
	43	spinlock_t lock;
	44	struct list_head virtqueues;
	45	};
	46
	47	struct virtio_pci_vq_info
	48	{
	49	/* the actual virtqueue */
	50	struct virtqueue *vq;
	51
	52	/* the number of entries in the queue */
	53	int num;
	54
	55	/* the index of the queue */
	56	int queue_index;
	57
	58	/* the virtual address of the ring queue */
	59	void *queue;
	60
	61	/* the list node for the virtqueues list */
	62	struct list_head node;
	63	};
	64
	65	/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
	66	static struct pci_device_id virtio_pci_id_table[] = {
	67	{ 0x1af4, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
	68	{ 0 },
	69	};
	70
	71	MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
	72
	73	/* A PCI device has it's own struct device and so does a virtio device so
	74	* we create a place for the virtio devices to show up in sysfs. I think it
	75	* would make more sense for virtio to not insist on having it's own device. */
63d12556	76	static struct device *virtio_pci_root;
3343660d	77
3343660d AL	78	/* Convert a generic virtio device to our structure */
	79	static struct virtio_pci_device to_vp_device(struct virtio_device vdev)
	80	{
	81	return container_of(vdev, struct virtio_pci_device, vdev);
	82	}
	83
c45a6816 RR	84	/* virtio config->get_features() implementation */
	85	static u32 vp_get_features(struct virtio_device *vdev)
	86	{
	87	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	88
	89	/* When someone needs more than 32 feature bits, we'll need to
	90	* steal a bit to indicate that the rest are somewhere else. */
	91	return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
	92	}
	93
c624896e RR	94	/* virtio config->finalize_features() implementation */
c624896e RR	95	static void vp_finalize_features(struct virtio_device *vdev)
3343660d AL	96	{
3343660d AL	97	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
3343660d	98
e34f8725 RR	99	/* Give virtio_ring a chance to accept features. */
	100	vring_transport_features(vdev);
	101
c624896e RR	102	/* We only support 32 feature bits. */
	103	BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
	104	iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
3343660d AL	105	}
	106
	107	/* virtio config->get() implementation */
	108	static void vp_get(struct virtio_device *vdev, unsigned offset,
	109	void *buf, unsigned len)
	110	{
	111	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
97968358	112	void __iomem *ioaddr = vp_dev->ioaddr + VIRTIO_PCI_CONFIG + offset;
3343660d AL	113	u8 *ptr = buf;
	114	int i;
	115
	116	for (i = 0; i < len; i++)
	117	ptr[i] = ioread8(ioaddr + i);
	118	}
	119
	120	/* the config->set() implementation. it's symmetric to the config->get()
	121	* implementation */
	122	static void vp_set(struct virtio_device *vdev, unsigned offset,
	123	const void *buf, unsigned len)
	124	{
	125	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
97968358	126	void __iomem *ioaddr = vp_dev->ioaddr + VIRTIO_PCI_CONFIG + offset;
3343660d AL	127	const u8 *ptr = buf;
	128	int i;
	129
	130	for (i = 0; i < len; i++)
	131	iowrite8(ptr[i], ioaddr + i);
	132	}
	133
	134	/* config->{get,set}_status() implementations */
	135	static u8 vp_get_status(struct virtio_device *vdev)
	136	{
	137	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	138	return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
	139	}
	140
	141	static void vp_set_status(struct virtio_device *vdev, u8 status)
	142	{
	143	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	144	/* We should never be setting status to 0. */
	145	BUG_ON(status == 0);
597d56e4	146	iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
3343660d AL	147	}
	148
	149	static void vp_reset(struct virtio_device *vdev)
	150	{
	151	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	152	/* 0 status means a reset. */
597d56e4	153	iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
3343660d AL	154	}
	155
	156	/* the notify function used when creating a virt queue */
	157	static void vp_notify(struct virtqueue *vq)
	158	{
	159	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	160	struct virtio_pci_vq_info *info = vq->priv;
	161
	162	/* we write the queue's selector into the notification register to
	163	* signal the other end */
	164	iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
	165	}
	166
	167	/* A small wrapper to also acknowledge the interrupt when it's handled.
	168	* I really need an EIO hook for the vring so I can ack the interrupt once we
	169	* know that we'll be handling the IRQ but before we invoke the callback since
	170	* the callback may notify the host which results in the host attempting to
	171	* raise an interrupt that we would then mask once we acknowledged the
	172	* interrupt. */
	173	static irqreturn_t vp_interrupt(int irq, void *opaque)
	174	{
	175	struct virtio_pci_device *vp_dev = opaque;
	176	struct virtio_pci_vq_info *info;
	177	irqreturn_t ret = IRQ_NONE;
27ebe308	178	unsigned long flags;
3343660d AL	179	u8 isr;
	180
	181	/* reading the ISR has the effect of also clearing it so it's very
	182	* important to save off the value. */
	183	isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
	184
	185	/* It's definitely not us if the ISR was not high */
	186	if (!isr)
	187	return IRQ_NONE;
	188
	189	/* Configuration change? Tell driver if it wants to know. */
	190	if (isr & VIRTIO_PCI_ISR_CONFIG) {
	191	struct virtio_driver *drv;
	192	drv = container_of(vp_dev->vdev.dev.driver,
	193	struct virtio_driver, driver);
	194
3fff0179	195	if (drv && drv->config_changed)
3343660d AL	196	drv->config_changed(&vp_dev->vdev);
	197	}
	198
27ebe308	199	spin_lock_irqsave(&vp_dev->lock, flags);
3343660d AL	200	list_for_each_entry(info, &vp_dev->virtqueues, node) {
	201	if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
	202	ret = IRQ_HANDLED;
	203	}
27ebe308	204	spin_unlock_irqrestore(&vp_dev->lock, flags);
3343660d AL	205
	206	return ret;
	207	}
	208
	209	/* the config->find_vq() implementation */
	210	static struct virtqueue vp_find_vq(struct virtio_device vdev, unsigned index,
9499f5e7 RR	211	void (callback)(struct virtqueue vq),
9499f5e7 RR	212	const char *name)
3343660d AL	213	{
	214	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	215	struct virtio_pci_vq_info *info;
	216	struct virtqueue *vq;
13b1eb33	217	unsigned long flags, size;
3343660d AL	218	u16 num;
	219	int err;
	220
	221	/* Select the queue we're interested in */
	222	iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
	223
	224	/* Check if queue is either not available or already active. */
	225	num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
	226	if (!num \|\| ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
	227	return ERR_PTR(-ENOENT);
	228
	229	/* allocate and fill out our structure the represents an active
	230	* queue */
	231	info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
	232	if (!info)
	233	return ERR_PTR(-ENOMEM);
	234
	235	info->queue_index = index;
	236	info->num = num;
	237
498af147	238	size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
13b1eb33	239	info->queue = alloc_pages_exact(size, GFP_KERNEL\|__GFP_ZERO);
3343660d AL	240	if (info->queue == NULL) {
	241	err = -ENOMEM;
	242	goto out_info;
	243	}
	244
	245	/* activate the queue */
480daab4	246	iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
3343660d AL	247	vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
	248
	249	/* create the vring */
87c7d57c	250	vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
9499f5e7	251	vdev, info->queue, vp_notify, callback, name);
3343660d AL	252	if (!vq) {
	253	err = -ENOMEM;
	254	goto out_activate_queue;
	255	}
	256
	257	vq->priv = info;
	258	info->vq = vq;
	259
27ebe308	260	spin_lock_irqsave(&vp_dev->lock, flags);
3343660d	261	list_add(&info->node, &vp_dev->virtqueues);
27ebe308	262	spin_unlock_irqrestore(&vp_dev->lock, flags);
3343660d AL	263
	264	return vq;
	265
	266	out_activate_queue:
	267	iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
13b1eb33	268	free_pages_exact(info->queue, size);
3343660d AL	269	out_info:
	270	kfree(info);
	271	return ERR_PTR(err);
	272	}
	273
	274	/* the config->del_vq() implementation */
	275	static void vp_del_vq(struct virtqueue *vq)
	276	{
	277	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	278	struct virtio_pci_vq_info *info = vq->priv;
d2a7ddda	279	unsigned long size;
3343660d AL	280
	281	vring_del_virtqueue(vq);
	282
	283	/* Select and deactivate the queue */
	284	iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
	285	iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
	286
498af147	287	size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
13b1eb33	288	free_pages_exact(info->queue, size);
3343660d AL	289	kfree(info);
	290	}
	291
d2a7ddda MT	292	static void vp_del_vqs(struct virtio_device *vdev)
	293	{
	294	struct virtqueue vq, n;
	295
	296	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
	297	vp_del_vq(vq);
	298	}
	299
	300	static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
	301	struct virtqueue *vqs[],
	302	vq_callback_t *callbacks[],
	303	const char *names[])
	304	{
	305	int i;
	306
	307	for (i = 0; i < nvqs; ++i) {
	308	vqs[i] = vp_find_vq(vdev, i, callbacks[i], names[i]);
	309	if (IS_ERR(vqs[i]))
	310	goto error;
	311	}
	312	return 0;
	313
	314	error:
	315	vp_del_vqs(vdev);
	316	return PTR_ERR(vqs[i]);
	317	}
	318
3343660d	319	static struct virtio_config_ops virtio_pci_config_ops = {
3343660d AL	320	.get = vp_get,
	321	.set = vp_set,
	322	.get_status = vp_get_status,
	323	.set_status = vp_set_status,
	324	.reset = vp_reset,
d2a7ddda MT	325	.find_vqs = vp_find_vqs,
d2a7ddda MT	326	.del_vqs = vp_del_vqs,
c45a6816	327	.get_features = vp_get_features,
c624896e	328	.finalize_features = vp_finalize_features,
3343660d AL	329	};
3343660d AL	330
29f9f12e MM	331	static void virtio_pci_release_dev(struct device *_d)
	332	{
	333	struct virtio_device *dev = container_of(_d, struct virtio_device, dev);
	334	struct virtio_pci_device *vp_dev = to_vp_device(dev);
	335	struct pci_dev *pci_dev = vp_dev->pci_dev;
	336
	337	free_irq(pci_dev->irq, vp_dev);
	338	pci_set_drvdata(pci_dev, NULL);
	339	pci_iounmap(pci_dev, vp_dev->ioaddr);
	340	pci_release_regions(pci_dev);
	341	pci_disable_device(pci_dev);
	342	kfree(vp_dev);
	343	}
	344
3343660d AL	345	/* the PCI probing function */
	346	static int __devinit virtio_pci_probe(struct pci_dev *pci_dev,
	347	const struct pci_device_id *id)
	348	{
	349	struct virtio_pci_device *vp_dev;
	350	int err;
	351
	352	/* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
	353	if (pci_dev->device < 0x1000 \|\| pci_dev->device > 0x103f)
	354	return -ENODEV;
	355
55a7c066 AL	356	if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
	357	printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
	358	VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
	359	return -ENODEV;
	360	}
	361
3343660d AL	362	/* allocate our structure and fill it out */
	363	vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
	364	if (vp_dev == NULL)
	365	return -ENOMEM;
	366
63d12556	367	vp_dev->vdev.dev.parent = virtio_pci_root;
29f9f12e	368	vp_dev->vdev.dev.release = virtio_pci_release_dev;
3343660d AL	369	vp_dev->vdev.config = &virtio_pci_config_ops;
	370	vp_dev->pci_dev = pci_dev;
	371	INIT_LIST_HEAD(&vp_dev->virtqueues);
	372	spin_lock_init(&vp_dev->lock);
	373
	374	/* enable the device */
	375	err = pci_enable_device(pci_dev);
	376	if (err)
	377	goto out;
	378
	379	err = pci_request_regions(pci_dev, "virtio-pci");
	380	if (err)
	381	goto out_enable_device;
	382
	383	vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
	384	if (vp_dev->ioaddr == NULL)
	385	goto out_req_regions;
	386
	387	pci_set_drvdata(pci_dev, vp_dev);
	388
	389	/* we use the subsystem vendor/device id as the virtio vendor/device
	390	* id. this allows us to use the same PCI vendor/device id for all
	391	* virtio devices and to identify the particular virtio driver by
	392	* the subsytem ids */
	393	vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
	394	vp_dev->vdev.id.device = pci_dev->subsystem_device;
	395
	396	/* register a handler for the queue with the PCI device's interrupt */
	397	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
99e0b6c8	398	dev_name(&vp_dev->vdev.dev), vp_dev);
3343660d AL	399	if (err)
	400	goto out_set_drvdata;
	401
	402	/* finally register the virtio device */
	403	err = register_virtio_device(&vp_dev->vdev);
	404	if (err)
	405	goto out_req_irq;
	406
	407	return 0;
	408
	409	out_req_irq:
	410	free_irq(pci_dev->irq, vp_dev);
	411	out_set_drvdata:
	412	pci_set_drvdata(pci_dev, NULL);
	413	pci_iounmap(pci_dev, vp_dev->ioaddr);
	414	out_req_regions:
	415	pci_release_regions(pci_dev);
	416	out_enable_device:
	417	pci_disable_device(pci_dev);
	418	out:
	419	kfree(vp_dev);
	420	return err;
	421	}
	422
	423	static void __devexit virtio_pci_remove(struct pci_dev *pci_dev)
	424	{
	425	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
	426
bd6c2690	427	unregister_virtio_device(&vp_dev->vdev);
3343660d AL	428	}
	429
	430	#ifdef CONFIG_PM
	431	static int virtio_pci_suspend(struct pci_dev *pci_dev, pm_message_t state)
	432	{
	433	pci_save_state(pci_dev);
	434	pci_set_power_state(pci_dev, PCI_D3hot);
	435	return 0;
	436	}
	437
	438	static int virtio_pci_resume(struct pci_dev *pci_dev)
	439	{
	440	pci_restore_state(pci_dev);
	441	pci_set_power_state(pci_dev, PCI_D0);
	442	return 0;
	443	}
	444	#endif
	445
	446	static struct pci_driver virtio_pci_driver = {
	447	.name = "virtio-pci",
	448	.id_table = virtio_pci_id_table,
	449	.probe = virtio_pci_probe,
	450	.remove = virtio_pci_remove,
	451	#ifdef CONFIG_PM
	452	.suspend = virtio_pci_suspend,
	453	.resume = virtio_pci_resume,
	454	#endif
	455	};
	456
	457	static int __init virtio_pci_init(void)
	458	{
	459	int err;
	460
63d12556 MM	461	virtio_pci_root = root_device_register("virtio-pci");
	462	if (IS_ERR(virtio_pci_root))
	463	return PTR_ERR(virtio_pci_root);
3343660d AL	464
	465	err = pci_register_driver(&virtio_pci_driver);
	466	if (err)
63d12556	467	device_unregister(virtio_pci_root);
3343660d AL	468
	469	return err;
	470	}
	471
	472	module_init(virtio_pci_init);
	473
	474	static void __exit virtio_pci_exit(void)
	475	{
3343660d	476	pci_unregister_driver(&virtio_pci_driver);
63d12556	477	root_device_unregister(virtio_pci_root);
3343660d AL	478	}
	479
	480	module_exit(virtio_pci_exit);