[mirror_ubuntu-artful-kernel.git] / arch / powerpc / kvm / book3s_64_vio.c

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 *
 * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 * Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
 * Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com>
 */

#include <linux/types.h>
#include <linux/string.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/highmem.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>
#include <linux/hugetlb.h>
#include <linux/list.h>
#include <linux/anon_inodes.h>
#include <linux/iommu.h>
#include <linux/file.h>

#include <asm/tlbflush.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <asm/book3s/64/mmu-hash.h>
#include <asm/hvcall.h>
#include <asm/synch.h>
#include <asm/ppc-opcode.h>
#include <asm/kvm_host.h>
#include <asm/udbg.h>
#include <asm/iommu.h>
#include <asm/tce.h>
#include <asm/mmu_context.h>

static unsigned long kvmppc_tce_pages(unsigned long iommu_pages)
{
	return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
}

static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
{
	unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) +
			(tce_pages * sizeof(struct page *));

	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
}

static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
{
	long ret = 0;

	if (!current || !current->mm)
		return ret; /* process exited */

	down_write(&current->mm->mmap_sem);

	if (inc) {
		unsigned long locked, lock_limit;

		locked = current->mm->locked_vm + stt_pages;
		lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
			ret = -ENOMEM;
		else
			current->mm->locked_vm += stt_pages;
	} else {
		if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
			stt_pages = current->mm->locked_vm;

		current->mm->locked_vm -= stt_pages;
	}

	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
			inc ? '+' : '-',
			stt_pages << PAGE_SHIFT,
			current->mm->locked_vm << PAGE_SHIFT,
			rlimit(RLIMIT_MEMLOCK),
			ret ? " - exceeded" : "");

	up_write(&current->mm->mmap_sem);

	return ret;
}

static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
{
	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
			struct kvmppc_spapr_tce_iommu_table, rcu);

	iommu_tce_table_put(stit->tbl);

	kfree(stit);
}

static void kvm_spapr_tce_liobn_put(struct kref *kref)
{
	struct kvmppc_spapr_tce_iommu_table *stit = container_of(kref,
			struct kvmppc_spapr_tce_iommu_table, kref);

	list_del_rcu(&stit->next);

	call_rcu(&stit->rcu, kvm_spapr_tce_iommu_table_free);
}

extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm,
		struct iommu_group *grp)
{
	int i;
	struct kvmppc_spapr_tce_table *stt;
	struct kvmppc_spapr_tce_iommu_table *stit, *tmp;
	struct iommu_table_group *table_group = NULL;

	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {

		table_group = iommu_group_get_iommudata(grp);
		if (WARN_ON(!table_group))
			continue;

		list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {
			for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
				if (table_group->tables[i] != stit->tbl)
					continue;

				kref_put(&stit->kref, kvm_spapr_tce_liobn_put);
				return;
			}
		}
	}
}

extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd,
		struct iommu_group *grp)
{
	struct kvmppc_spapr_tce_table *stt = NULL;
	bool found = false;
	struct iommu_table *tbl = NULL;
	struct iommu_table_group *table_group;
	long i;
	struct kvmppc_spapr_tce_iommu_table *stit;
	struct fd f;

	f = fdget(tablefd);
	if (!f.file)
		return -EBADF;

	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
		if (stt == f.file->private_data) {
			found = true;
			break;
		}
	}

	fdput(f);

	if (!found)
		return -EINVAL;

	table_group = iommu_group_get_iommudata(grp);
	if (WARN_ON(!table_group))
		return -EFAULT;

	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
		struct iommu_table *tbltmp = table_group->tables[i];

		if (!tbltmp)
			continue;
		/*
		 * Make sure hardware table parameters are exactly the same;
		 * this is used in the TCE handlers where boundary checks
		 * use only the first attached table.
		 */
		if ((tbltmp->it_page_shift == stt->page_shift) &&
				(tbltmp->it_offset == stt->offset) &&
				(tbltmp->it_size == stt->size)) {
			/*
			 * Reference the table to avoid races with
			 * add/remove DMA windows.
			 */
			tbl = iommu_tce_table_get(tbltmp);
			break;
		}
	}
	if (!tbl)
		return -EINVAL;

	list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {
		if (tbl != stit->tbl)
			continue;

		if (!kref_get_unless_zero(&stit->kref)) {
			/* stit is being destroyed */
			iommu_tce_table_put(tbl);
			return -ENOTTY;
		}
		/*
		 * The table is already known to this KVM, we just increased
		 * its KVM reference counter and can return.
		 */
		return 0;
	}

	stit = kzalloc(sizeof(*stit), GFP_KERNEL);
	if (!stit) {
		iommu_tce_table_put(tbl);
		return -ENOMEM;
	}

	stit->tbl = tbl;
	kref_init(&stit->kref);

	list_add_rcu(&stit->next, &stt->iommu_tables);

	return 0;
}

static void release_spapr_tce_table(struct rcu_head *head)
{
	struct kvmppc_spapr_tce_table *stt = container_of(head,
			struct kvmppc_spapr_tce_table, rcu);
	unsigned long i, npages = kvmppc_tce_pages(stt->size);

	for (i = 0; i < npages; i++)
		__free_page(stt->pages[i]);

	kfree(stt);
}

static int kvm_spapr_tce_fault(struct vm_fault *vmf)
{
	struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data;
	struct page *page;

	if (vmf->pgoff >= kvmppc_tce_pages(stt->size))
		return VM_FAULT_SIGBUS;

	page = stt->pages[vmf->pgoff];
	get_page(page);
	vmf->page = page;
	return 0;
}

static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
	.fault = kvm_spapr_tce_fault,
};

static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
{
	vma->vm_ops = &kvm_spapr_tce_vm_ops;
	return 0;
}

static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
{
	struct kvmppc_spapr_tce_table *stt = filp->private_data;
	struct kvmppc_spapr_tce_iommu_table *stit, *tmp;

	list_del_rcu(&stt->list);

	list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {
		WARN_ON(!kref_read(&stit->kref));
		while (1) {
			if (kref_put(&stit->kref, kvm_spapr_tce_liobn_put))
				break;
		}
	}

	kvm_put_kvm(stt->kvm);

	kvmppc_account_memlimit(
		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
	call_rcu(&stt->rcu, release_spapr_tce_table);

	return 0;
}

static const struct file_operations kvm_spapr_tce_fops = {
	.mmap           = kvm_spapr_tce_mmap,
	.release	= kvm_spapr_tce_release,
};

long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
				   struct kvm_create_spapr_tce_64 *args)
{
	struct kvmppc_spapr_tce_table *stt = NULL;
	struct kvmppc_spapr_tce_table *siter;
	unsigned long npages, size;
	int ret = -ENOMEM;
	int i;
	int fd = -1;

	if (!args->size)
		return -EINVAL;

	size = _ALIGN_UP(args->size, PAGE_SIZE >> 3);
	npages = kvmppc_tce_pages(size);
	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
	if (ret)
		return ret;

	ret = -ENOMEM;
	stt = kzalloc(sizeof(*stt) + npages * sizeof(struct page *),
		      GFP_KERNEL);
	if (!stt)
		goto fail_acct;

	stt->liobn = args->liobn;
	stt->page_shift = args->page_shift;
	stt->offset = args->offset;
	stt->size = size;
	stt->kvm = kvm;
	INIT_LIST_HEAD_RCU(&stt->iommu_tables);

	for (i = 0; i < npages; i++) {
		stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!stt->pages[i])
			goto fail;
	}

	ret = fd = anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
				    stt, O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto fail;

	mutex_lock(&kvm->lock);

	/* Check this LIOBN hasn't been previously allocated */
	ret = 0;
	list_for_each_entry(siter, &kvm->arch.spapr_tce_tables, list) {
		if (siter->liobn == args->liobn) {
			ret = -EBUSY;
			break;
		}
	}

	if (!ret) {
		list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);
		kvm_get_kvm(kvm);
	}

	mutex_unlock(&kvm->lock);

	if (!ret)
		return fd;

	put_unused_fd(fd);

 fail:
	for (i = 0; i < npages; i++)
		if (stt->pages[i])
			__free_page(stt->pages[i]);

	kfree(stt);
 fail_acct:
	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
	return ret;
}

static void kvmppc_clear_tce(struct iommu_table *tbl, unsigned long entry)
{
	unsigned long hpa = 0;
	enum dma_data_direction dir = DMA_NONE;

	iommu_tce_xchg(tbl, entry, &hpa, &dir);
}

static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
		struct iommu_table *tbl, unsigned long entry)
{
	struct mm_iommu_table_group_mem_t *mem = NULL;
	const unsigned long pgsize = 1ULL << tbl->it_page_shift;
	unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);

	if (!pua)
		/* it_userspace allocation might be delayed */
		return H_TOO_HARD;

	mem = mm_iommu_lookup(kvm->mm, *pua, pgsize);
	if (!mem)
		return H_TOO_HARD;

	mm_iommu_mapped_dec(mem);

	*pua = 0;

	return H_SUCCESS;
}

static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
		struct iommu_table *tbl, unsigned long entry)
{
	enum dma_data_direction dir = DMA_NONE;
	unsigned long hpa = 0;
	long ret;

	if (WARN_ON_ONCE(iommu_tce_xchg(tbl, entry, &hpa, &dir)))
		return H_HARDWARE;

	if (dir == DMA_NONE)
		return H_SUCCESS;

	ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
	if (ret != H_SUCCESS)
		iommu_tce_xchg(tbl, entry, &hpa, &dir);

	return ret;
}

long kvmppc_tce_iommu_map(struct kvm *kvm, struct iommu_table *tbl,
		unsigned long entry, unsigned long ua,
		enum dma_data_direction dir)
{
	long ret;
	unsigned long hpa, *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
	struct mm_iommu_table_group_mem_t *mem;

	if (!pua)
		/* it_userspace allocation might be delayed */
		return H_TOO_HARD;

	mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift);
	if (!mem)
		/* This only handles v2 IOMMU type, v1 is handled via ioctl() */
		return H_TOO_HARD;

	if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, &hpa)))
		return H_HARDWARE;

	if (mm_iommu_mapped_inc(mem))
		return H_CLOSED;

	ret = iommu_tce_xchg(tbl, entry, &hpa, &dir);
	if (WARN_ON_ONCE(ret)) {
		mm_iommu_mapped_dec(mem);
		return H_HARDWARE;
	}

	if (dir != DMA_NONE)
		kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);

	*pua = ua;

	return 0;
}

long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
		      unsigned long ioba, unsigned long tce)
{
	struct kvmppc_spapr_tce_table *stt;
	long ret, idx;
	struct kvmppc_spapr_tce_iommu_table *stit;
	unsigned long entry, ua = 0;
	enum dma_data_direction dir;

	/* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
	/* 	    liobn, ioba, tce); */

	stt = kvmppc_find_table(vcpu->kvm, liobn);
	if (!stt)
		return H_TOO_HARD;

	ret = kvmppc_ioba_validate(stt, ioba, 1);
	if (ret != H_SUCCESS)
		return ret;

	ret = kvmppc_tce_validate(stt, tce);
	if (ret != H_SUCCESS)
		return ret;

	dir = iommu_tce_direction(tce);
	if ((dir != DMA_NONE) && kvmppc_gpa_to_ua(vcpu->kvm,
			tce & ~(TCE_PCI_READ | TCE_PCI_WRITE), &ua, NULL))
		return H_PARAMETER;

	entry = ioba >> stt->page_shift;

	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
		if (dir == DMA_NONE) {
			ret = kvmppc_tce_iommu_unmap(vcpu->kvm,
					stit->tbl, entry);
		} else {
			idx = srcu_read_lock(&vcpu->kvm->srcu);
			ret = kvmppc_tce_iommu_map(vcpu->kvm, stit->tbl,
					entry, ua, dir);
			srcu_read_unlock(&vcpu->kvm->srcu, idx);
		}

		if (ret == H_SUCCESS)
			continue;

		if (ret == H_TOO_HARD)
			return ret;

		WARN_ON_ONCE(1);
		kvmppc_clear_tce(stit->tbl, entry);
	}

	kvmppc_tce_put(stt, entry, tce);

	return H_SUCCESS;
}
EXPORT_SYMBOL_GPL(kvmppc_h_put_tce);

long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
		unsigned long liobn, unsigned long ioba,
		unsigned long tce_list, unsigned long npages)
{
	struct kvmppc_spapr_tce_table *stt;
	long i, ret = H_SUCCESS, idx;
	unsigned long entry, ua = 0;
	u64 __user *tces;
	u64 tce;
	struct kvmppc_spapr_tce_iommu_table *stit;

	stt = kvmppc_find_table(vcpu->kvm, liobn);
	if (!stt)
		return H_TOO_HARD;

	entry = ioba >> stt->page_shift;
	/*
	 * SPAPR spec says that the maximum size of the list is 512 TCEs
	 * so the whole table fits in 4K page
	 */
	if (npages > 512)
		return H_PARAMETER;

	if (tce_list & (SZ_4K - 1))
		return H_PARAMETER;

	ret = kvmppc_ioba_validate(stt, ioba, npages);
	if (ret != H_SUCCESS)
		return ret;

	idx = srcu_read_lock(&vcpu->kvm->srcu);
	if (kvmppc_gpa_to_ua(vcpu->kvm, tce_list, &ua, NULL)) {
		ret = H_TOO_HARD;
		goto unlock_exit;
	}
	tces = (u64 __user *) ua;

	for (i = 0; i < npages; ++i) {
		if (get_user(tce, tces + i)) {
			ret = H_TOO_HARD;
			goto unlock_exit;
		}
		tce = be64_to_cpu(tce);

		ret = kvmppc_tce_validate(stt, tce);
		if (ret != H_SUCCESS)
			goto unlock_exit;

		if (kvmppc_gpa_to_ua(vcpu->kvm,
				tce & ~(TCE_PCI_READ | TCE_PCI_WRITE),
				&ua, NULL))
			return H_PARAMETER;

		list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
			ret = kvmppc_tce_iommu_map(vcpu->kvm,
					stit->tbl, entry + i, ua,
					iommu_tce_direction(tce));

			if (ret == H_SUCCESS)
				continue;

			if (ret == H_TOO_HARD)
				goto unlock_exit;

			WARN_ON_ONCE(1);
			kvmppc_clear_tce(stit->tbl, entry);
		}

		kvmppc_tce_put(stt, entry + i, tce);
	}

unlock_exit:
	srcu_read_unlock(&vcpu->kvm->srcu, idx);

	return ret;
}
EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect);

long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
		unsigned long liobn, unsigned long ioba,
		unsigned long tce_value, unsigned long npages)
{
	struct kvmppc_spapr_tce_table *stt;
	long i, ret;
	struct kvmppc_spapr_tce_iommu_table *stit;

	stt = kvmppc_find_table(vcpu->kvm, liobn);
	if (!stt)
		return H_TOO_HARD;

	ret = kvmppc_ioba_validate(stt, ioba, npages);
	if (ret != H_SUCCESS)
		return ret;

	/* Check permission bits only to allow userspace poison TCE for debug */
	if (tce_value & (TCE_PCI_WRITE | TCE_PCI_READ))
		return H_PARAMETER;

	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
		unsigned long entry = ioba >> stit->tbl->it_page_shift;

		for (i = 0; i < npages; ++i) {
			ret = kvmppc_tce_iommu_unmap(vcpu->kvm,
					stit->tbl, entry + i);

			if (ret == H_SUCCESS)
				continue;

			if (ret == H_TOO_HARD)
				return ret;

			WARN_ON_ONCE(1);
			kvmppc_clear_tce(stit->tbl, entry);
		}
	}

	for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
		kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);

	return H_SUCCESS;
}
EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);
Commit	Line	Data
f31e65e1 BH	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License, version 2, as
	4	* published by the Free Software Foundation.
	5	*
	6	* This program is distributed in the hope that it will be useful,
	7	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	8	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	9	* GNU General Public License for more details.
	10	*
	11	* You should have received a copy of the GNU General Public License
	12	* along with this program; if not, write to the Free Software
	13	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	14	*
	15	* Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
	16	* Copyright 2011 David Gibson, IBM Corporation <dwg@au1.ibm.com>
d3695aa4	17	* Copyright 2016 Alexey Kardashevskiy, IBM Corporation <aik@au1.ibm.com>
f31e65e1 BH	18	*/
	19
	20	#include <linux/types.h>
	21	#include <linux/string.h>
	22	#include <linux/kvm.h>
	23	#include <linux/kvm_host.h>
	24	#include <linux/highmem.h>
	25	#include <linux/gfp.h>
	26	#include <linux/slab.h>
3f07c014	27	#include <linux/sched/signal.h>
f31e65e1 BH	28	#include <linux/hugetlb.h>
	29	#include <linux/list.h>
	30	#include <linux/anon_inodes.h>
121f80ba AK	31	#include <linux/iommu.h>
121f80ba AK	32	#include <linux/file.h>
f31e65e1 BH	33
	34	#include <asm/tlbflush.h>
	35	#include <asm/kvm_ppc.h>
	36	#include <asm/kvm_book3s.h>
f64e8084	37	#include <asm/book3s/64/mmu-hash.h>
f31e65e1 BH	38	#include <asm/hvcall.h>
	39	#include <asm/synch.h>
	40	#include <asm/ppc-opcode.h>
	41	#include <asm/kvm_host.h>
	42	#include <asm/udbg.h>
462ee11e	43	#include <asm/iommu.h>
d3695aa4	44	#include <asm/tce.h>
121f80ba	45	#include <asm/mmu_context.h>
f31e65e1	46
fe26e527	47	static unsigned long kvmppc_tce_pages(unsigned long iommu_pages)
f31e65e1	48	{
fe26e527	49	return ALIGN(iommu_pages * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
f31e65e1 BH	50	}
f31e65e1 BH	51
f8626985 AK	52	static unsigned long kvmppc_stt_pages(unsigned long tce_pages)
	53	{
	54	unsigned long stt_bytes = sizeof(struct kvmppc_spapr_tce_table) +
	55	(tce_pages * sizeof(struct page *));
	56
	57	return tce_pages + ALIGN(stt_bytes, PAGE_SIZE) / PAGE_SIZE;
	58	}
	59
	60	static long kvmppc_account_memlimit(unsigned long stt_pages, bool inc)
	61	{
	62	long ret = 0;
	63
	64	if (!current \|\| !current->mm)
	65	return ret; /* process exited */
	66
	67	down_write(&current->mm->mmap_sem);
	68
	69	if (inc) {
	70	unsigned long locked, lock_limit;
	71
	72	locked = current->mm->locked_vm + stt_pages;
	73	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	74	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
	75	ret = -ENOMEM;
	76	else
	77	current->mm->locked_vm += stt_pages;
	78	} else {
	79	if (WARN_ON_ONCE(stt_pages > current->mm->locked_vm))
	80	stt_pages = current->mm->locked_vm;
	81
	82	current->mm->locked_vm -= stt_pages;
	83	}
	84
	85	pr_debug("[%d] RLIMIT_MEMLOCK KVM %c%ld %ld/%ld%s\n", current->pid,
	86	inc ? '+' : '-',
	87	stt_pages << PAGE_SHIFT,
	88	current->mm->locked_vm << PAGE_SHIFT,
	89	rlimit(RLIMIT_MEMLOCK),
	90	ret ? " - exceeded" : "");
	91
	92	up_write(&current->mm->mmap_sem);
	93
	94	return ret;
	95	}
	96
121f80ba AK	97	static void kvm_spapr_tce_iommu_table_free(struct rcu_head *head)
	98	{
	99	struct kvmppc_spapr_tce_iommu_table *stit = container_of(head,
	100	struct kvmppc_spapr_tce_iommu_table, rcu);
	101
	102	iommu_tce_table_put(stit->tbl);
	103
	104	kfree(stit);
	105	}
	106
	107	static void kvm_spapr_tce_liobn_put(struct kref *kref)
	108	{
	109	struct kvmppc_spapr_tce_iommu_table *stit = container_of(kref,
	110	struct kvmppc_spapr_tce_iommu_table, kref);
	111
	112	list_del_rcu(&stit->next);
	113
	114	call_rcu(&stit->rcu, kvm_spapr_tce_iommu_table_free);
	115	}
	116
	117	extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm,
	118	struct iommu_group *grp)
	119	{
	120	int i;
	121	struct kvmppc_spapr_tce_table *stt;
	122	struct kvmppc_spapr_tce_iommu_table stit, tmp;
	123	struct iommu_table_group *table_group = NULL;
	124
	125	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
	126
	127	table_group = iommu_group_get_iommudata(grp);
	128	if (WARN_ON(!table_group))
	129	continue;
	130
	131	list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {
	132	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
	133	if (table_group->tables[i] != stit->tbl)
	134	continue;
	135
	136	kref_put(&stit->kref, kvm_spapr_tce_liobn_put);
	137	return;
	138	}
	139	}
	140	}
	141	}
	142
	143	extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd,
	144	struct iommu_group *grp)
	145	{
	146	struct kvmppc_spapr_tce_table *stt = NULL;
	147	bool found = false;
	148	struct iommu_table *tbl = NULL;
	149	struct iommu_table_group *table_group;
	150	long i;
	151	struct kvmppc_spapr_tce_iommu_table *stit;
	152	struct fd f;
	153
	154	f = fdget(tablefd);
	155	if (!f.file)
	156	return -EBADF;
	157
	158	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {
	159	if (stt == f.file->private_data) {
	160	found = true;
161	break;
162	}
163	}
164
165	fdput(f);
166
167	if (!found)
168	return -EINVAL;
169
170	table_group = iommu_group_get_iommudata(grp);
171	if (WARN_ON(!table_group))
172	return -EFAULT;
173
174	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
175	struct iommu_table *tbltmp = table_group->tables[i];
176
177	if (!tbltmp)
178	continue;
179	/*
180	* Make sure hardware table parameters are exactly the same;
181	* this is used in the TCE handlers where boundary checks
182	* use only the first attached table.
183	*/
184	if ((tbltmp->it_page_shift == stt->page_shift) &&
185	(tbltmp->it_offset == stt->offset) &&
186	(tbltmp->it_size == stt->size)) {
187	/*
188	* Reference the table to avoid races with
189	* add/remove DMA windows.
190	*/
191	tbl = iommu_tce_table_get(tbltmp);
192	break;
193	}
194	}
195	if (!tbl)
196	return -EINVAL;
197
198	list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {
199	if (tbl != stit->tbl)
200	continue;
201
202	if (!kref_get_unless_zero(&stit->kref)) {
203	/* stit is being destroyed */
204	iommu_tce_table_put(tbl);
205	return -ENOTTY;
206	}
207	/*
208	* The table is already known to this KVM, we just increased
209	* its KVM reference counter and can return.
210	*/
211	return 0;
212	}
213
214	stit = kzalloc(sizeof(*stit), GFP_KERNEL);
215	if (!stit) {
216	iommu_tce_table_put(tbl);
217	return -ENOMEM;
218	}
219
220	stit->tbl = tbl;
221	kref_init(&stit->kref);
222
223	list_add_rcu(&stit->next, &stt->iommu_tables);
224
225	return 0;
226	}
227
366baf28	228	static void release_spapr_tce_table(struct rcu_head *head)
f31e65e1	229	{
366baf28 AK	230	struct kvmppc_spapr_tce_table *stt = container_of(head,
366baf28 AK	231	struct kvmppc_spapr_tce_table, rcu);
fe26e527	232	unsigned long i, npages = kvmppc_tce_pages(stt->size);
f31e65e1	233
f8626985	234	for (i = 0; i < npages; i++)
f31e65e1	235	__free_page(stt->pages[i]);
f31e65e1	236
366baf28	237	kfree(stt);
f31e65e1 BH	238	}
f31e65e1 BH	239
11bac800	240	static int kvm_spapr_tce_fault(struct vm_fault *vmf)
f31e65e1	241	{
11bac800	242	struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data;
f31e65e1 BH	243	struct page *page;
f31e65e1 BH	244
fe26e527	245	if (vmf->pgoff >= kvmppc_tce_pages(stt->size))
f31e65e1 BH	246	return VM_FAULT_SIGBUS;
	247
	248	page = stt->pages[vmf->pgoff];
	249	get_page(page);
	250	vmf->page = page;
	251	return 0;
	252	}
	253
	254	static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
	255	.fault = kvm_spapr_tce_fault,
	256	};
	257
	258	static int kvm_spapr_tce_mmap(struct file file, struct vm_area_struct vma)
	259	{
	260	vma->vm_ops = &kvm_spapr_tce_vm_ops;
	261	return 0;
	262	}
	263
	264	static int kvm_spapr_tce_release(struct inode inode, struct file filp)
	265	{
	266	struct kvmppc_spapr_tce_table *stt = filp->private_data;
121f80ba	267	struct kvmppc_spapr_tce_iommu_table stit, tmp;
f31e65e1	268
366baf28 AK	269	list_del_rcu(&stt->list);
366baf28 AK	270
121f80ba AK	271	list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {
	272	WARN_ON(!kref_read(&stit->kref));
	273	while (1) {
	274	if (kref_put(&stit->kref, kvm_spapr_tce_liobn_put))
	275	break;
	276	}
	277	}
	278
366baf28 AK	279	kvm_put_kvm(stt->kvm);
366baf28 AK	280
f8626985	281	kvmppc_account_memlimit(
fe26e527	282	kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);
366baf28 AK	283	call_rcu(&stt->rcu, release_spapr_tce_table);
366baf28 AK	284
f31e65e1 BH	285	return 0;
	286	}
	287
75ef9de1	288	static const struct file_operations kvm_spapr_tce_fops = {
f31e65e1 BH	289	.mmap = kvm_spapr_tce_mmap,
	290	.release = kvm_spapr_tce_release,
	291	};
	292
	293	long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
58ded420	294	struct kvm_create_spapr_tce_64 *args)
f31e65e1 BH	295	{
f31e65e1 BH	296	struct kvmppc_spapr_tce_table *stt = NULL;
47c5310a	297	struct kvmppc_spapr_tce_table *siter;
fe26e527	298	unsigned long npages, size;
f31e65e1 BH	299	int ret = -ENOMEM;
f31e65e1 BH	300	int i;
47c5310a	301	int fd = -1;
f31e65e1	302
58ded420 AK	303	if (!args->size)
	304	return -EINVAL;
	305
3762d45a	306	size = _ALIGN_UP(args->size, PAGE_SIZE >> 3);
fe26e527	307	npages = kvmppc_tce_pages(size);
f8626985	308	ret = kvmppc_account_memlimit(kvmppc_stt_pages(npages), true);
47c5310a PM	309	if (ret)
47c5310a PM	310	return ret;
f31e65e1	311
5982f084	312	ret = -ENOMEM;
f31e65e1 BH	313	stt = kzalloc(sizeof(stt) + npages sizeof(struct page *),
	314	GFP_KERNEL);
	315	if (!stt)
47c5310a	316	goto fail_acct;
f31e65e1 BH	317
f31e65e1 BH	318	stt->liobn = args->liobn;
58ded420 AK	319	stt->page_shift = args->page_shift;
58ded420 AK	320	stt->offset = args->offset;
fe26e527	321	stt->size = size;
f31e65e1	322	stt->kvm = kvm;
121f80ba	323	INIT_LIST_HEAD_RCU(&stt->iommu_tables);
f31e65e1 BH	324
	325	for (i = 0; i < npages; i++) {
	326	stt->pages[i] = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	327	if (!stt->pages[i])
	328	goto fail;
	329	}
	330
47c5310a PM	331	ret = fd = anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
	332	stt, O_RDWR \| O_CLOEXEC);
	333	if (ret < 0)
	334	goto fail;
f31e65e1 BH	335
f31e65e1 BH	336	mutex_lock(&kvm->lock);
47c5310a PM	337
	338	/* Check this LIOBN hasn't been previously allocated */
	339	ret = 0;
	340	list_for_each_entry(siter, &kvm->arch.spapr_tce_tables, list) {
	341	if (siter->liobn == args->liobn) {
	342	ret = -EBUSY;
	343	break;
	344	}
	345	}
	346
	347	if (!ret) {
	348	list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);
	349	kvm_get_kvm(kvm);
	350	}
f31e65e1 BH	351
	352	mutex_unlock(&kvm->lock);
	353
47c5310a PM	354	if (!ret)
47c5310a PM	355	return fd;
f31e65e1	356
47c5310a	357	put_unused_fd(fd);
f31e65e1	358
47c5310a PM	359	fail:
	360	for (i = 0; i < npages; i++)
	361	if (stt->pages[i])
	362	__free_page(stt->pages[i]);
	363
	364	kfree(stt);
	365	fail_acct:
	366	kvmppc_account_memlimit(kvmppc_stt_pages(npages), false);
f31e65e1 BH	367	return ret;
f31e65e1 BH	368	}
d3695aa4	369
121f80ba AK	370	static void kvmppc_clear_tce(struct iommu_table *tbl, unsigned long entry)
	371	{
	372	unsigned long hpa = 0;
	373	enum dma_data_direction dir = DMA_NONE;
	374
	375	iommu_tce_xchg(tbl, entry, &hpa, &dir);
	376	}
	377
	378	static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
	379	struct iommu_table *tbl, unsigned long entry)
	380	{
	381	struct mm_iommu_table_group_mem_t *mem = NULL;
	382	const unsigned long pgsize = 1ULL << tbl->it_page_shift;
	383	unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
	384
	385	if (!pua)
	386	/* it_userspace allocation might be delayed */
	387	return H_TOO_HARD;
	388
	389	mem = mm_iommu_lookup(kvm->mm, *pua, pgsize);
	390	if (!mem)
	391	return H_TOO_HARD;
	392
	393	mm_iommu_mapped_dec(mem);
	394
	395	*pua = 0;
	396
	397	return H_SUCCESS;
	398	}
	399
	400	static long kvmppc_tce_iommu_unmap(struct kvm *kvm,
	401	struct iommu_table *tbl, unsigned long entry)
	402	{
	403	enum dma_data_direction dir = DMA_NONE;
	404	unsigned long hpa = 0;
	405	long ret;
	406
	407	if (WARN_ON_ONCE(iommu_tce_xchg(tbl, entry, &hpa, &dir)))
	408	return H_HARDWARE;
	409
	410	if (dir == DMA_NONE)
	411	return H_SUCCESS;
	412
	413	ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
	414	if (ret != H_SUCCESS)
	415	iommu_tce_xchg(tbl, entry, &hpa, &dir);
	416
	417	return ret;
	418	}
	419
	420	long kvmppc_tce_iommu_map(struct kvm kvm, struct iommu_table tbl,
	421	unsigned long entry, unsigned long ua,
	422	enum dma_data_direction dir)
	423	{
	424	long ret;
	425	unsigned long hpa, *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
	426	struct mm_iommu_table_group_mem_t *mem;
	427
	428	if (!pua)
	429	/* it_userspace allocation might be delayed */
	430	return H_TOO_HARD;
	431
	432	mem = mm_iommu_lookup(kvm->mm, ua, 1ULL << tbl->it_page_shift);
	433	if (!mem)
434	/* This only handles v2 IOMMU type, v1 is handled via ioctl() */
435	return H_TOO_HARD;
436
437	if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, &hpa)))
438	return H_HARDWARE;
439
440	if (mm_iommu_mapped_inc(mem))
441	return H_CLOSED;
442
443	ret = iommu_tce_xchg(tbl, entry, &hpa, &dir);
444	if (WARN_ON_ONCE(ret)) {
445	mm_iommu_mapped_dec(mem);
446	return H_HARDWARE;
447	}
448
449	if (dir != DMA_NONE)
450	kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
451
452	*pua = ua;
453
454	return 0;
455	}
456
31217db7 AK	457	long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
	458	unsigned long ioba, unsigned long tce)
	459	{
503bfcbe	460	struct kvmppc_spapr_tce_table *stt;
121f80ba AK	461	long ret, idx;
	462	struct kvmppc_spapr_tce_iommu_table *stit;
	463	unsigned long entry, ua = 0;
	464	enum dma_data_direction dir;
31217db7 AK	465
	466	/* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */
	467	/* liobn, ioba, tce); */
	468
503bfcbe	469	stt = kvmppc_find_table(vcpu->kvm, liobn);
31217db7 AK	470	if (!stt)
	471	return H_TOO_HARD;
	472
	473	ret = kvmppc_ioba_validate(stt, ioba, 1);
	474	if (ret != H_SUCCESS)
	475	return ret;
	476
	477	ret = kvmppc_tce_validate(stt, tce);
	478	if (ret != H_SUCCESS)
	479	return ret;
	480
121f80ba AK	481	dir = iommu_tce_direction(tce);
	482	if ((dir != DMA_NONE) && kvmppc_gpa_to_ua(vcpu->kvm,
	483	tce & ~(TCE_PCI_READ \| TCE_PCI_WRITE), &ua, NULL))
	484	return H_PARAMETER;
	485
	486	entry = ioba >> stt->page_shift;
	487
	488	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
	489	if (dir == DMA_NONE) {
	490	ret = kvmppc_tce_iommu_unmap(vcpu->kvm,
	491	stit->tbl, entry);
	492	} else {
	493	idx = srcu_read_lock(&vcpu->kvm->srcu);
	494	ret = kvmppc_tce_iommu_map(vcpu->kvm, stit->tbl,
	495	entry, ua, dir);
	496	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	497	}
	498
	499	if (ret == H_SUCCESS)
	500	continue;
	501
	502	if (ret == H_TOO_HARD)
	503	return ret;
	504
	505	WARN_ON_ONCE(1);
	506	kvmppc_clear_tce(stit->tbl, entry);
	507	}
	508
	509	kvmppc_tce_put(stt, entry, tce);
31217db7 AK	510
	511	return H_SUCCESS;
	512	}
	513	EXPORT_SYMBOL_GPL(kvmppc_h_put_tce);
	514
d3695aa4 AK	515	long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
	516	unsigned long liobn, unsigned long ioba,
	517	unsigned long tce_list, unsigned long npages)
	518	{
	519	struct kvmppc_spapr_tce_table *stt;
	520	long i, ret = H_SUCCESS, idx;
	521	unsigned long entry, ua = 0;
f8750513 DA	522	u64 __user *tces;
f8750513 DA	523	u64 tce;
121f80ba	524	struct kvmppc_spapr_tce_iommu_table *stit;
d3695aa4	525
503bfcbe	526	stt = kvmppc_find_table(vcpu->kvm, liobn);
d3695aa4 AK	527	if (!stt)
	528	return H_TOO_HARD;
	529
fe26e527	530	entry = ioba >> stt->page_shift;
d3695aa4 AK	531	/*
	532	* SPAPR spec says that the maximum size of the list is 512 TCEs
	533	* so the whole table fits in 4K page
	534	*/
	535	if (npages > 512)
	536	return H_PARAMETER;
	537
	538	if (tce_list & (SZ_4K - 1))
	539	return H_PARAMETER;
	540
	541	ret = kvmppc_ioba_validate(stt, ioba, npages);
	542	if (ret != H_SUCCESS)
	543	return ret;
	544
	545	idx = srcu_read_lock(&vcpu->kvm->srcu);
	546	if (kvmppc_gpa_to_ua(vcpu->kvm, tce_list, &ua, NULL)) {
	547	ret = H_TOO_HARD;
	548	goto unlock_exit;
	549	}
	550	tces = (u64 __user *) ua;
	551
	552	for (i = 0; i < npages; ++i) {
	553	if (get_user(tce, tces + i)) {
	554	ret = H_TOO_HARD;
	555	goto unlock_exit;
	556	}
	557	tce = be64_to_cpu(tce);
	558
	559	ret = kvmppc_tce_validate(stt, tce);
	560	if (ret != H_SUCCESS)
	561	goto unlock_exit;
	562
121f80ba AK	563	if (kvmppc_gpa_to_ua(vcpu->kvm,
	564	tce & ~(TCE_PCI_READ \| TCE_PCI_WRITE),
	565	&ua, NULL))
	566	return H_PARAMETER;
	567
	568	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
	569	ret = kvmppc_tce_iommu_map(vcpu->kvm,
	570	stit->tbl, entry + i, ua,
	571	iommu_tce_direction(tce));
	572
	573	if (ret == H_SUCCESS)
	574	continue;
	575
	576	if (ret == H_TOO_HARD)
	577	goto unlock_exit;
	578
	579	WARN_ON_ONCE(1);
	580	kvmppc_clear_tce(stit->tbl, entry);
	581	}
	582
d3695aa4 AK	583	kvmppc_tce_put(stt, entry + i, tce);
	584	}
	585
	586	unlock_exit:
	587	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	588
	589	return ret;
	590	}
	591	EXPORT_SYMBOL_GPL(kvmppc_h_put_tce_indirect);
31217db7 AK	592
	593	long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
	594	unsigned long liobn, unsigned long ioba,
	595	unsigned long tce_value, unsigned long npages)
	596	{
	597	struct kvmppc_spapr_tce_table *stt;
	598	long i, ret;
121f80ba	599	struct kvmppc_spapr_tce_iommu_table *stit;
31217db7	600
503bfcbe	601	stt = kvmppc_find_table(vcpu->kvm, liobn);
31217db7 AK	602	if (!stt)
	603	return H_TOO_HARD;
	604
	605	ret = kvmppc_ioba_validate(stt, ioba, npages);
	606	if (ret != H_SUCCESS)
	607	return ret;
	608
	609	/* Check permission bits only to allow userspace poison TCE for debug */
	610	if (tce_value & (TCE_PCI_WRITE \| TCE_PCI_READ))
	611	return H_PARAMETER;
	612
121f80ba AK	613	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
	614	unsigned long entry = ioba >> stit->tbl->it_page_shift;
	615
	616	for (i = 0; i < npages; ++i) {
	617	ret = kvmppc_tce_iommu_unmap(vcpu->kvm,
	618	stit->tbl, entry + i);
	619
	620	if (ret == H_SUCCESS)
	621	continue;
	622
	623	if (ret == H_TOO_HARD)
	624	return ret;
	625
	626	WARN_ON_ONCE(1);
	627	kvmppc_clear_tce(stit->tbl, entry);
	628	}
	629	}
	630
31217db7 AK	631	for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
	632	kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);
	633
	634	return H_SUCCESS;
	635	}
	636	EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);