[mirror_ubuntu-jammy-kernel.git] / drivers / iommu / s390-iommu.c

// SPDX-License-Identifier: GPL-2.0
/*
 * IOMMU API for s390 PCI devices
 *
 * Copyright IBM Corp. 2015
 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
 */

#include <linux/pci.h>
#include <linux/iommu.h>
#include <linux/iommu-helper.h>
#include <linux/sizes.h>
#include <asm/pci_dma.h>

/*
 * Physically contiguous memory regions can be mapped with 4 KiB alignment,
 * we allow all page sizes that are an order of 4KiB (no special large page
 * support so far).
 */
#define S390_IOMMU_PGSIZES	(~0xFFFUL)

static const struct iommu_ops s390_iommu_ops;

struct s390_domain {
	struct iommu_domain	domain;
	struct list_head	devices;
	unsigned long		*dma_table;
	spinlock_t		dma_table_lock;
	spinlock_t		list_lock;
};

struct s390_domain_device {
	struct list_head	list;
	struct zpci_dev		*zdev;
};

static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
{
	return container_of(dom, struct s390_domain, domain);
}

static bool s390_iommu_capable(enum iommu_cap cap)
{
	switch (cap) {
	case IOMMU_CAP_CACHE_COHERENCY:
		return true;
	case IOMMU_CAP_INTR_REMAP:
		return true;
	default:
		return false;
	}
}

static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
{
	struct s390_domain *s390_domain;

	if (domain_type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

	s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL);
	if (!s390_domain)
		return NULL;

	s390_domain->dma_table = dma_alloc_cpu_table();
	if (!s390_domain->dma_table) {
		kfree(s390_domain);
		return NULL;
	}

	spin_lock_init(&s390_domain->dma_table_lock);
	spin_lock_init(&s390_domain->list_lock);
	INIT_LIST_HEAD(&s390_domain->devices);

	return &s390_domain->domain;
}

static void s390_domain_free(struct iommu_domain *domain)
{
	struct s390_domain *s390_domain = to_s390_domain(domain);

	dma_cleanup_tables(s390_domain->dma_table);
	kfree(s390_domain);
}

static int s390_iommu_attach_device(struct iommu_domain *domain,
				    struct device *dev)
{
	struct s390_domain *s390_domain = to_s390_domain(domain);
	struct zpci_dev *zdev = to_zpci_dev(dev);
	struct s390_domain_device *domain_device;
	unsigned long flags;
	int cc, rc;

	if (!zdev)
		return -ENODEV;

	domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
	if (!domain_device)
		return -ENOMEM;

	if (zdev->dma_table) {
		cc = zpci_dma_exit_device(zdev);
		if (cc) {
			rc = -EIO;
			goto out_free;
		}
	}

	zdev->dma_table = s390_domain->dma_table;
	cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
				(u64) zdev->dma_table);
	if (cc) {
		rc = -EIO;
		goto out_restore;
	}

	spin_lock_irqsave(&s390_domain->list_lock, flags);
	/* First device defines the DMA range limits */
	if (list_empty(&s390_domain->devices)) {
		domain->geometry.aperture_start = zdev->start_dma;
		domain->geometry.aperture_end = zdev->end_dma;
		domain->geometry.force_aperture = true;
	/* Allow only devices with identical DMA range limits */
	} else if (domain->geometry.aperture_start != zdev->start_dma ||
		   domain->geometry.aperture_end != zdev->end_dma) {
		rc = -EINVAL;
		spin_unlock_irqrestore(&s390_domain->list_lock, flags);
		goto out_restore;
	}
	domain_device->zdev = zdev;
	zdev->s390_domain = s390_domain;
	list_add(&domain_device->list, &s390_domain->devices);
	spin_unlock_irqrestore(&s390_domain->list_lock, flags);

	return 0;

out_restore:
	zpci_dma_init_device(zdev);
out_free:
	kfree(domain_device);

	return rc;
}

static void s390_iommu_detach_device(struct iommu_domain *domain,
				     struct device *dev)
{
	struct s390_domain *s390_domain = to_s390_domain(domain);
	struct zpci_dev *zdev = to_zpci_dev(dev);
	struct s390_domain_device *domain_device, *tmp;
	unsigned long flags;
	int found = 0;

	if (!zdev)
		return;

	spin_lock_irqsave(&s390_domain->list_lock, flags);
	list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
				 list) {
		if (domain_device->zdev == zdev) {
			list_del(&domain_device->list);
			kfree(domain_device);
			found = 1;
			break;
		}
	}
	spin_unlock_irqrestore(&s390_domain->list_lock, flags);

	if (found) {
		zdev->s390_domain = NULL;
		zpci_unregister_ioat(zdev, 0);
		zpci_dma_init_device(zdev);
	}
}

static struct iommu_device *s390_iommu_probe_device(struct device *dev)
{
	struct zpci_dev *zdev = to_zpci_dev(dev);

	return &zdev->iommu_dev;
}

static void s390_iommu_release_device(struct device *dev)
{
	struct zpci_dev *zdev = to_zpci_dev(dev);
	struct iommu_domain *domain;

	/*
	 * This is a workaround for a scenario where the IOMMU API common code
	 * "forgets" to call the detach_dev callback: After binding a device
	 * to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
	 * the attach_dev), removing the device via
	 * "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
	 * only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
	 * notifier.
	 *
	 * So let's call detach_dev from here if it hasn't been called before.
	 */
	if (zdev && zdev->s390_domain) {
		domain = iommu_get_domain_for_dev(dev);
		if (domain)
			s390_iommu_detach_device(domain, dev);
	}
}

static int s390_iommu_update_trans(struct s390_domain *s390_domain,
				   unsigned long pa, dma_addr_t dma_addr,
				   size_t size, int flags)
{
	struct s390_domain_device *domain_device;
	u8 *page_addr = (u8 *) (pa & PAGE_MASK);
	dma_addr_t start_dma_addr = dma_addr;
	unsigned long irq_flags, nr_pages, i;
	unsigned long *entry;
	int rc = 0;

	if (dma_addr < s390_domain->domain.geometry.aperture_start ||
	    dma_addr + size > s390_domain->domain.geometry.aperture_end)
		return -EINVAL;

	nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
	if (!nr_pages)
		return 0;

	spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
	for (i = 0; i < nr_pages; i++) {
		entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
		if (!entry) {
			rc = -ENOMEM;
			goto undo_cpu_trans;
		}
		dma_update_cpu_trans(entry, page_addr, flags);
		page_addr += PAGE_SIZE;
		dma_addr += PAGE_SIZE;
	}

	spin_lock(&s390_domain->list_lock);
	list_for_each_entry(domain_device, &s390_domain->devices, list) {
		rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
					start_dma_addr, nr_pages * PAGE_SIZE);
		if (rc)
			break;
	}
	spin_unlock(&s390_domain->list_lock);

undo_cpu_trans:
	if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
		flags = ZPCI_PTE_INVALID;
		while (i-- > 0) {
			page_addr -= PAGE_SIZE;
			dma_addr -= PAGE_SIZE;
			entry = dma_walk_cpu_trans(s390_domain->dma_table,
						   dma_addr);
			if (!entry)
				break;
			dma_update_cpu_trans(entry, page_addr, flags);
		}
	}
	spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);

	return rc;
}

static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
			  phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
	struct s390_domain *s390_domain = to_s390_domain(domain);
	int flags = ZPCI_PTE_VALID, rc = 0;

	if (!(prot & IOMMU_READ))
		return -EINVAL;

	if (!(prot & IOMMU_WRITE))
		flags |= ZPCI_TABLE_PROTECTED;

	rc = s390_iommu_update_trans(s390_domain, (unsigned long) paddr, iova,
				     size, flags);

	return rc;
}

static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
					   dma_addr_t iova)
{
	struct s390_domain *s390_domain = to_s390_domain(domain);
	unsigned long *sto, *pto, *rto, flags;
	unsigned int rtx, sx, px;
	phys_addr_t phys = 0;

	if (iova < domain->geometry.aperture_start ||
	    iova > domain->geometry.aperture_end)
		return 0;

	rtx = calc_rtx(iova);
	sx = calc_sx(iova);
	px = calc_px(iova);
	rto = s390_domain->dma_table;

	spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
	if (rto && reg_entry_isvalid(rto[rtx])) {
		sto = get_rt_sto(rto[rtx]);
		if (sto && reg_entry_isvalid(sto[sx])) {
			pto = get_st_pto(sto[sx]);
			if (pto && pt_entry_isvalid(pto[px]))
				phys = pto[px] & ZPCI_PTE_ADDR_MASK;
		}
	}
	spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);

	return phys;
}

static size_t s390_iommu_unmap(struct iommu_domain *domain,
			       unsigned long iova, size_t size,
			       struct iommu_iotlb_gather *gather)
{
	struct s390_domain *s390_domain = to_s390_domain(domain);
	int flags = ZPCI_PTE_INVALID;
	phys_addr_t paddr;
	int rc;

	paddr = s390_iommu_iova_to_phys(domain, iova);
	if (!paddr)
		return 0;

	rc = s390_iommu_update_trans(s390_domain, (unsigned long) paddr, iova,
				     size, flags);
	if (rc)
		return 0;

	return size;
}

int zpci_init_iommu(struct zpci_dev *zdev)
{
	int rc = 0;

	rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
				    "s390-iommu.%08x", zdev->fid);
	if (rc)
		goto out_err;

	rc = iommu_device_register(&zdev->iommu_dev, &s390_iommu_ops, NULL);
	if (rc)
		goto out_sysfs;

	return 0;

out_sysfs:
	iommu_device_sysfs_remove(&zdev->iommu_dev);

out_err:
	return rc;
}

void zpci_destroy_iommu(struct zpci_dev *zdev)
{
	iommu_device_unregister(&zdev->iommu_dev);
	iommu_device_sysfs_remove(&zdev->iommu_dev);
}

static const struct iommu_ops s390_iommu_ops = {
	.capable = s390_iommu_capable,
	.domain_alloc = s390_domain_alloc,
	.domain_free = s390_domain_free,
	.attach_dev = s390_iommu_attach_device,
	.detach_dev = s390_iommu_detach_device,
	.map = s390_iommu_map,
	.unmap = s390_iommu_unmap,
	.iova_to_phys = s390_iommu_iova_to_phys,
	.probe_device = s390_iommu_probe_device,
	.release_device = s390_iommu_release_device,
	.device_group = generic_device_group,
	.pgsize_bitmap = S390_IOMMU_PGSIZES,
};

static int __init s390_iommu_init(void)
{
	return bus_set_iommu(&pci_bus_type, &s390_iommu_ops);
}
subsys_initcall(s390_iommu_init);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
8128f23c GS	2	/*
	3	* IOMMU API for s390 PCI devices
	4	*
	5	* Copyright IBM Corp. 2015
	6	* Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
	7	*/
	8
	9	#include <linux/pci.h>
	10	#include <linux/iommu.h>
	11	#include <linux/iommu-helper.h>
8128f23c GS	12	#include <linux/sizes.h>
	13	#include <asm/pci_dma.h>
	14
	15	/*
	16	* Physically contiguous memory regions can be mapped with 4 KiB alignment,
	17	* we allow all page sizes that are an order of 4KiB (no special large page
	18	* support so far).
	19	*/
	20	#define S390_IOMMU_PGSIZES (~0xFFFUL)
	21
cceb8451	22	static const struct iommu_ops s390_iommu_ops;
f42c2235	23
8128f23c GS	24	struct s390_domain {
	25	struct iommu_domain domain;
	26	struct list_head devices;
	27	unsigned long *dma_table;
	28	spinlock_t dma_table_lock;
	29	spinlock_t list_lock;
	30	};
	31
	32	struct s390_domain_device {
	33	struct list_head list;
	34	struct zpci_dev *zdev;
	35	};
	36
	37	static struct s390_domain to_s390_domain(struct iommu_domain dom)
	38	{
	39	return container_of(dom, struct s390_domain, domain);
	40	}
	41
	42	static bool s390_iommu_capable(enum iommu_cap cap)
	43	{
	44	switch (cap) {
	45	case IOMMU_CAP_CACHE_COHERENCY:
	46	return true;
	47	case IOMMU_CAP_INTR_REMAP:
	48	return true;
	49	default:
	50	return false;
	51	}
	52	}
	53
7cd75787	54	static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
8128f23c GS	55	{
	56	struct s390_domain *s390_domain;
	57
	58	if (domain_type != IOMMU_DOMAIN_UNMANAGED)
	59	return NULL;
	60
	61	s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL);
	62	if (!s390_domain)
	63	return NULL;
	64
	65	s390_domain->dma_table = dma_alloc_cpu_table();
	66	if (!s390_domain->dma_table) {
	67	kfree(s390_domain);
	68	return NULL;
	69	}
	70
	71	spin_lock_init(&s390_domain->dma_table_lock);
	72	spin_lock_init(&s390_domain->list_lock);
	73	INIT_LIST_HEAD(&s390_domain->devices);
	74
	75	return &s390_domain->domain;
	76	}
	77
7cd75787	78	static void s390_domain_free(struct iommu_domain *domain)
8128f23c GS	79	{
	80	struct s390_domain *s390_domain = to_s390_domain(domain);
	81
	82	dma_cleanup_tables(s390_domain->dma_table);
	83	kfree(s390_domain);
	84	}
	85
	86	static int s390_iommu_attach_device(struct iommu_domain *domain,
	87	struct device *dev)
	88	{
	89	struct s390_domain *s390_domain = to_s390_domain(domain);
d08d6f5d	90	struct zpci_dev *zdev = to_zpci_dev(dev);
8128f23c GS	91	struct s390_domain_device *domain_device;
8128f23c GS	92	unsigned long flags;
1f3f7681	93	int cc, rc;
8128f23c GS	94
	95	if (!zdev)
	96	return -ENODEV;
	97
	98	domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
	99	if (!domain_device)
	100	return -ENOMEM;
	101
1f3f7681 NS	102	if (zdev->dma_table) {
	103	cc = zpci_dma_exit_device(zdev);
	104	if (cc) {
	105	rc = -EIO;
	106	goto out_free;
	107	}
	108	}
8128f23c GS	109
8128f23c GS	110	zdev->dma_table = s390_domain->dma_table;
1f3f7681	111	cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
8128f23c	112	(u64) zdev->dma_table);
1f3f7681 NS	113	if (cc) {
1f3f7681 NS	114	rc = -EIO;
8128f23c	115	goto out_restore;
1f3f7681	116	}
8128f23c GS	117
	118	spin_lock_irqsave(&s390_domain->list_lock, flags);
	119	/* First device defines the DMA range limits */
	120	if (list_empty(&s390_domain->devices)) {
	121	domain->geometry.aperture_start = zdev->start_dma;
	122	domain->geometry.aperture_end = zdev->end_dma;
	123	domain->geometry.force_aperture = true;
	124	/* Allow only devices with identical DMA range limits */
	125	} else if (domain->geometry.aperture_start != zdev->start_dma \|\|
	126	domain->geometry.aperture_end != zdev->end_dma) {
	127	rc = -EINVAL;
	128	spin_unlock_irqrestore(&s390_domain->list_lock, flags);
	129	goto out_restore;
	130	}
	131	domain_device->zdev = zdev;
	132	zdev->s390_domain = s390_domain;
	133	list_add(&domain_device->list, &s390_domain->devices);
	134	spin_unlock_irqrestore(&s390_domain->list_lock, flags);
	135
	136	return 0;
	137
	138	out_restore:
	139	zpci_dma_init_device(zdev);
1f3f7681	140	out_free:
8128f23c GS	141	kfree(domain_device);
	142
	143	return rc;
	144	}
	145
	146	static void s390_iommu_detach_device(struct iommu_domain *domain,
	147	struct device *dev)
	148	{
	149	struct s390_domain *s390_domain = to_s390_domain(domain);
d08d6f5d	150	struct zpci_dev *zdev = to_zpci_dev(dev);
8128f23c GS	151	struct s390_domain_device domain_device, tmp;
	152	unsigned long flags;
	153	int found = 0;
	154
	155	if (!zdev)
	156	return;
	157
	158	spin_lock_irqsave(&s390_domain->list_lock, flags);
	159	list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
	160	list) {
	161	if (domain_device->zdev == zdev) {
	162	list_del(&domain_device->list);
	163	kfree(domain_device);
	164	found = 1;
	165	break;
	166	}
	167	}
	168	spin_unlock_irqrestore(&s390_domain->list_lock, flags);
	169
	170	if (found) {
	171	zdev->s390_domain = NULL;
	172	zpci_unregister_ioat(zdev, 0);
	173	zpci_dma_init_device(zdev);
	174	}
	175	}
	176
522af649	177	static struct iommu_device s390_iommu_probe_device(struct device dev)
8128f23c	178	{
d08d6f5d	179	struct zpci_dev *zdev = to_zpci_dev(dev);
8128f23c	180
522af649	181	return &zdev->iommu_dev;
8128f23c GS	182	}
8128f23c GS	183
522af649	184	static void s390_iommu_release_device(struct device *dev)
8128f23c	185	{
d08d6f5d	186	struct zpci_dev *zdev = to_zpci_dev(dev);
8128f23c GS	187	struct iommu_domain *domain;
	188
	189	/*
	190	* This is a workaround for a scenario where the IOMMU API common code
	191	* "forgets" to call the detach_dev callback: After binding a device
	192	* to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
	193	* the attach_dev), removing the device via
	194	* "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
522af649	195	* only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
8128f23c GS	196	* notifier.
	197	*
	198	* So let's call detach_dev from here if it hasn't been called before.
	199	*/
	200	if (zdev && zdev->s390_domain) {
	201	domain = iommu_get_domain_for_dev(dev);
	202	if (domain)
	203	s390_iommu_detach_device(domain, dev);
	204	}
8128f23c GS	205	}
	206
	207	static int s390_iommu_update_trans(struct s390_domain *s390_domain,
	208	unsigned long pa, dma_addr_t dma_addr,
	209	size_t size, int flags)
	210	{
	211	struct s390_domain_device *domain_device;
	212	u8 page_addr = (u8 ) (pa & PAGE_MASK);
	213	dma_addr_t start_dma_addr = dma_addr;
	214	unsigned long irq_flags, nr_pages, i;
66728eee	215	unsigned long *entry;
8128f23c GS	216	int rc = 0;
	217
	218	if (dma_addr < s390_domain->domain.geometry.aperture_start \|\|
	219	dma_addr + size > s390_domain->domain.geometry.aperture_end)
	220	return -EINVAL;
	221
	222	nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
	223	if (!nr_pages)
	224	return 0;
	225
	226	spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
	227	for (i = 0; i < nr_pages; i++) {
66728eee SO	228	entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
	229	if (!entry) {
	230	rc = -ENOMEM;
	231	goto undo_cpu_trans;
	232	}
	233	dma_update_cpu_trans(entry, page_addr, flags);
8128f23c GS	234	page_addr += PAGE_SIZE;
	235	dma_addr += PAGE_SIZE;
	236	}
	237
	238	spin_lock(&s390_domain->list_lock);
	239	list_for_each_entry(domain_device, &s390_domain->devices, list) {
	240	rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
	241	start_dma_addr, nr_pages * PAGE_SIZE);
	242	if (rc)
	243	break;
	244	}
	245	spin_unlock(&s390_domain->list_lock);
66728eee SO	246
	247	undo_cpu_trans:
	248	if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
	249	flags = ZPCI_PTE_INVALID;
	250	while (i-- > 0) {
	251	page_addr -= PAGE_SIZE;
	252	dma_addr -= PAGE_SIZE;
	253	entry = dma_walk_cpu_trans(s390_domain->dma_table,
	254	dma_addr);
	255	if (!entry)
	256	break;
	257	dma_update_cpu_trans(entry, page_addr, flags);
	258	}
	259	}
8128f23c GS	260	spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);
	261
	262	return rc;
	263	}
	264
	265	static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
781ca2de	266	phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
8128f23c GS	267	{
	268	struct s390_domain *s390_domain = to_s390_domain(domain);
	269	int flags = ZPCI_PTE_VALID, rc = 0;
	270
	271	if (!(prot & IOMMU_READ))
	272	return -EINVAL;
	273
	274	if (!(prot & IOMMU_WRITE))
	275	flags \|= ZPCI_TABLE_PROTECTED;
	276
	277	rc = s390_iommu_update_trans(s390_domain, (unsigned long) paddr, iova,
	278	size, flags);
	279
	280	return rc;
	281	}
	282
	283	static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
	284	dma_addr_t iova)
	285	{
	286	struct s390_domain *s390_domain = to_s390_domain(domain);
	287	unsigned long sto, pto, *rto, flags;
	288	unsigned int rtx, sx, px;
	289	phys_addr_t phys = 0;
	290
	291	if (iova < domain->geometry.aperture_start \|\|
	292	iova > domain->geometry.aperture_end)
	293	return 0;
	294
	295	rtx = calc_rtx(iova);
	296	sx = calc_sx(iova);
	297	px = calc_px(iova);
	298	rto = s390_domain->dma_table;
	299
	300	spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
	301	if (rto && reg_entry_isvalid(rto[rtx])) {
	302	sto = get_rt_sto(rto[rtx]);
	303	if (sto && reg_entry_isvalid(sto[sx])) {
	304	pto = get_st_pto(sto[sx]);
	305	if (pto && pt_entry_isvalid(pto[px]))
	306	phys = pto[px] & ZPCI_PTE_ADDR_MASK;
	307	}
	308	}
	309	spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);
	310
	311	return phys;
	312	}
	313
	314	static size_t s390_iommu_unmap(struct iommu_domain *domain,
56f8af5e WD	315	unsigned long iova, size_t size,
56f8af5e WD	316	struct iommu_iotlb_gather *gather)
8128f23c GS	317	{
	318	struct s390_domain *s390_domain = to_s390_domain(domain);
	319	int flags = ZPCI_PTE_INVALID;
	320	phys_addr_t paddr;
	321	int rc;
	322
	323	paddr = s390_iommu_iova_to_phys(domain, iova);
	324	if (!paddr)
	325	return 0;
	326
	327	rc = s390_iommu_update_trans(s390_domain, (unsigned long) paddr, iova,
	328	size, flags);
	329	if (rc)
	330	return 0;
	331
	332	return size;
	333	}
	334
f42c2235 JR	335	int zpci_init_iommu(struct zpci_dev *zdev)
	336	{
	337	int rc = 0;
	338
	339	rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
	340	"s390-iommu.%08x", zdev->fid);
	341	if (rc)
	342	goto out_err;
	343
2d471b20	344	rc = iommu_device_register(&zdev->iommu_dev, &s390_iommu_ops, NULL);
f42c2235 JR	345	if (rc)
	346	goto out_sysfs;
	347
	348	return 0;
	349
	350	out_sysfs:
	351	iommu_device_sysfs_remove(&zdev->iommu_dev);
	352
	353	out_err:
	354	return rc;
	355	}
	356
	357	void zpci_destroy_iommu(struct zpci_dev *zdev)
	358	{
	359	iommu_device_unregister(&zdev->iommu_dev);
	360	iommu_device_sysfs_remove(&zdev->iommu_dev);
	361	}
	362
cceb8451	363	static const struct iommu_ops s390_iommu_ops = {
8128f23c GS	364	.capable = s390_iommu_capable,
	365	.domain_alloc = s390_domain_alloc,
	366	.domain_free = s390_domain_free,
	367	.attach_dev = s390_iommu_attach_device,
	368	.detach_dev = s390_iommu_detach_device,
	369	.map = s390_iommu_map,
	370	.unmap = s390_iommu_unmap,
	371	.iova_to_phys = s390_iommu_iova_to_phys,
522af649 JR	372	.probe_device = s390_iommu_probe_device,
522af649 JR	373	.release_device = s390_iommu_release_device,
0929deca	374	.device_group = generic_device_group,
8128f23c GS	375	.pgsize_bitmap = S390_IOMMU_PGSIZES,
	376	};
	377
	378	static int __init s390_iommu_init(void)
	379	{
	380	return bus_set_iommu(&pci_bus_type, &s390_iommu_ops);
	381	}
	382	subsys_initcall(s390_iommu_init);