[mirror_ubuntu-zesty-kernel.git] / drivers / infiniband / hw / ipath / ipath_user_pages.c

/*
 * Copyright (c) 2006 QLogic, Inc. All rights reserved.
 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/mm.h>
#include <linux/device.h>

#include "ipath_kernel.h"

static void __ipath_release_user_pages(struct page **p, size_t num_pages,
				   int dirty)
{
	size_t i;

	for (i = 0; i < num_pages; i++) {
		ipath_cdbg(MM, "%lu/%lu put_page %p\n", (unsigned long) i,
			   (unsigned long) num_pages, p[i]);
		if (dirty)
			set_page_dirty_lock(p[i]);
		put_page(p[i]);
	}
}

/* call with current->mm->mmap_sem held */
static int __get_user_pages(unsigned long start_page, size_t num_pages,
			struct page **p, struct vm_area_struct **vma)
{
	unsigned long lock_limit;
	size_t got;
	int ret;

	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >>
		PAGE_SHIFT;

	if (num_pages > lock_limit) {
		ret = -ENOMEM;
		goto bail;
	}

	ipath_cdbg(VERBOSE, "pin %lx pages from vaddr %lx\n",
		   (unsigned long) num_pages, start_page);

	for (got = 0; got < num_pages; got += ret) {
		ret = get_user_pages(current, current->mm,
				     start_page + got * PAGE_SIZE,
				     num_pages - got, 1, 1,
				     p + got, vma);
		if (ret < 0)
			goto bail_release;
	}

	current->mm->locked_vm += num_pages;

	ret = 0;
	goto bail;

bail_release:
	__ipath_release_user_pages(p, got, 0);
bail:
	return ret;
}

/**
 * ipath_map_page - a safety wrapper around pci_map_page()
 *
 * A dma_addr of all 0's is interpreted by the chip as "disabled".
 * Unfortunately, it can also be a valid dma_addr returned on some
 * architectures.
 *
 * The powerpc iommu assigns dma_addrs in ascending order, so we don't
 * have to bother with retries or mapping a dummy page to insure we
 * don't just get the same mapping again.
 *
 * I'm sure we won't be so lucky with other iommu's, so FIXME.
 */
dma_addr_t ipath_map_page(struct pci_dev *hwdev, struct page *page,
	unsigned long offset, size_t size, int direction)
{
	dma_addr_t phys;

	phys = pci_map_page(hwdev, page, offset, size, direction);

	if (phys == 0) {
		pci_unmap_page(hwdev, phys, size, direction);
		phys = pci_map_page(hwdev, page, offset, size, direction);
		/*
		 * FIXME: If we get 0 again, we should keep this page,
		 * map another, then free the 0 page.
		 */
	}

	return phys;
}

/**
 * ipath_map_single - a safety wrapper around pci_map_single()
 *
 * Same idea as ipath_map_page().
 */
dma_addr_t ipath_map_single(struct pci_dev *hwdev, void *ptr, size_t size,
	int direction)
{
	dma_addr_t phys;

	phys = pci_map_single(hwdev, ptr, size, direction);

	if (phys == 0) {
		pci_unmap_single(hwdev, phys, size, direction);
		phys = pci_map_single(hwdev, ptr, size, direction);
		/*
		 * FIXME: If we get 0 again, we should keep this page,
		 * map another, then free the 0 page.
		 */
	}

	return phys;
}

/**
 * ipath_get_user_pages - lock user pages into memory
 * @start_page: the start page
 * @num_pages: the number of pages
 * @p: the output page structures
 *
 * This function takes a given start page (page aligned user virtual
 * address) and pins it and the following specified number of pages.  For
 * now, num_pages is always 1, but that will probably change at some point
 * (because caller is doing expected sends on a single virtually contiguous
 * buffer, so we can do all pages at once).
 */
int ipath_get_user_pages(unsigned long start_page, size_t num_pages,
			 struct page **p)
{
	int ret;

	down_write(&current->mm->mmap_sem);

	ret = __get_user_pages(start_page, num_pages, p, NULL);

	up_write(&current->mm->mmap_sem);

	return ret;
}

/**
 * ipath_get_user_pages_nocopy - lock a single page for I/O and mark shared
 * @start_page: the page to lock
 * @p: the output page structure
 *
 * This is similar to ipath_get_user_pages, but it's always one page, and we
 * mark the page as locked for I/O, and shared.  This is used for the user
 * process page that contains the destination address for the rcvhdrq tail
 * update, so we need to have the vma. If we don't do this, the page can be
 * taken away from us on fork, even if the child never touches it, and then
 * the user process never sees the tail register updates.
 */
int ipath_get_user_pages_nocopy(unsigned long page, struct page **p)
{
	struct vm_area_struct *vma;
	int ret;

	down_write(&current->mm->mmap_sem);

	ret = __get_user_pages(page, 1, p, &vma);

	up_write(&current->mm->mmap_sem);

	return ret;
}

void ipath_release_user_pages(struct page **p, size_t num_pages)
{
	down_write(&current->mm->mmap_sem);

	__ipath_release_user_pages(p, num_pages, 1);

	current->mm->locked_vm -= num_pages;

	up_write(&current->mm->mmap_sem);
}

struct ipath_user_pages_work {
	struct work_struct work;
	struct mm_struct *mm;
	unsigned long num_pages;
};

static void user_pages_account(struct work_struct *_work)
{
	struct ipath_user_pages_work *work =
		container_of(_work, struct ipath_user_pages_work, work);

	down_write(&work->mm->mmap_sem);
	work->mm->locked_vm -= work->num_pages;
	up_write(&work->mm->mmap_sem);
	mmput(work->mm);
	kfree(work);
}

void ipath_release_user_pages_on_close(struct page **p, size_t num_pages)
{
	struct ipath_user_pages_work *work;
	struct mm_struct *mm;

	__ipath_release_user_pages(p, num_pages, 1);

	mm = get_task_mm(current);
	if (!mm)
		goto bail;

	work = kmalloc(sizeof(*work), GFP_KERNEL);
	if (!work)
		goto bail_mm;

	goto bail;

	INIT_WORK(&work->work, user_pages_account);
	work->mm = mm;
	work->num_pages = num_pages;

bail_mm:
	mmput(mm);
bail:
	return;
}
Commit	Line	Data
7f510b46	1	/*
759d5768	2	* Copyright (c) 2006 QLogic, Inc. All rights reserved.
7f510b46 BS	3	* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
	4	*
	5	* This software is available to you under a choice of one of two
	6	* licenses. You may choose to be licensed under the terms of the GNU
	7	* General Public License (GPL) Version 2, available from the file
	8	* COPYING in the main directory of this source tree, or the
	9	* OpenIB.org BSD license below:
	10	*
	11	* Redistribution and use in source and binary forms, with or
	12	* without modification, are permitted provided that the following
	13	* conditions are met:
	14	*
	15	* - Redistributions of source code must retain the above
	16	* copyright notice, this list of conditions and the following
	17	* disclaimer.
	18	*
	19	* - Redistributions in binary form must reproduce the above
	20	* copyright notice, this list of conditions and the following
	21	* disclaimer in the documentation and/or other materials
	22	* provided with the distribution.
	23	*
	24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	31	* SOFTWARE.
	32	*/
	33
	34	#include <linux/mm.h>
	35	#include <linux/device.h>
	36
	37	#include "ipath_kernel.h"
	38
	39	static void __ipath_release_user_pages(struct page **p, size_t num_pages,
	40	int dirty)
	41	{
	42	size_t i;
	43
	44	for (i = 0; i < num_pages; i++) {
	45	ipath_cdbg(MM, "%lu/%lu put_page %p\n", (unsigned long) i,
	46	(unsigned long) num_pages, p[i]);
	47	if (dirty)
	48	set_page_dirty_lock(p[i]);
	49	put_page(p[i]);
	50	}
	51	}
	52
	53	/* call with current->mm->mmap_sem held */
	54	static int __get_user_pages(unsigned long start_page, size_t num_pages,
	55	struct page p, struct vm_area_struct vma)
	56	{
	57	unsigned long lock_limit;
	58	size_t got;
	59	int ret;
	60
7f510b46 BS	61	lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >>
	62	PAGE_SHIFT;
	63
	64	if (num_pages > lock_limit) {
	65	ret = -ENOMEM;
	66	goto bail;
	67	}
	68
	69	ipath_cdbg(VERBOSE, "pin %lx pages from vaddr %lx\n",
	70	(unsigned long) num_pages, start_page);
	71
	72	for (got = 0; got < num_pages; got += ret) {
	73	ret = get_user_pages(current, current->mm,
	74	start_page + got * PAGE_SIZE,
	75	num_pages - got, 1, 1,
	76	p + got, vma);
	77	if (ret < 0)
	78	goto bail_release;
	79	}
	80
	81	current->mm->locked_vm += num_pages;
	82
	83	ret = 0;
	84	goto bail;
	85
	86	bail_release:
	87	__ipath_release_user_pages(p, got, 0);
	88	bail:
	89	return ret;
	90	}
	91
1fd3b40f BS	92	/**
	93	* ipath_map_page - a safety wrapper around pci_map_page()
	94	*
	95	* A dma_addr of all 0's is interpreted by the chip as "disabled".
	96	* Unfortunately, it can also be a valid dma_addr returned on some
	97	* architectures.
	98	*
	99	* The powerpc iommu assigns dma_addrs in ascending order, so we don't
	100	* have to bother with retries or mapping a dummy page to insure we
	101	* don't just get the same mapping again.
	102	*
	103	* I'm sure we won't be so lucky with other iommu's, so FIXME.
	104	*/
	105	dma_addr_t ipath_map_page(struct pci_dev hwdev, struct page page,
	106	unsigned long offset, size_t size, int direction)
	107	{
	108	dma_addr_t phys;
	109
	110	phys = pci_map_page(hwdev, page, offset, size, direction);
	111
	112	if (phys == 0) {
	113	pci_unmap_page(hwdev, phys, size, direction);
	114	phys = pci_map_page(hwdev, page, offset, size, direction);
	115	/*
	116	* FIXME: If we get 0 again, we should keep this page,
	117	* map another, then free the 0 page.
	118	*/
	119	}
	120
	121	return phys;
	122	}
	123
	124	/**
	125	* ipath_map_single - a safety wrapper around pci_map_single()
	126	*
	127	* Same idea as ipath_map_page().
	128	*/
	129	dma_addr_t ipath_map_single(struct pci_dev hwdev, void ptr, size_t size,
	130	int direction)
	131	{
	132	dma_addr_t phys;
	133
	134	phys = pci_map_single(hwdev, ptr, size, direction);
	135
	136	if (phys == 0) {
	137	pci_unmap_single(hwdev, phys, size, direction);
	138	phys = pci_map_single(hwdev, ptr, size, direction);
	139	/*
	140	* FIXME: If we get 0 again, we should keep this page,
	141	* map another, then free the 0 page.
	142	*/
	143	}
	144
	145	return phys;
	146	}
	147
7f510b46 BS	148	/**
	149	* ipath_get_user_pages - lock user pages into memory
	150	* @start_page: the start page
	151	* @num_pages: the number of pages
	152	* @p: the output page structures
	153	*
	154	* This function takes a given start page (page aligned user virtual
	155	* address) and pins it and the following specified number of pages. For
	156	* now, num_pages is always 1, but that will probably change at some point
	157	* (because caller is doing expected sends on a single virtually contiguous
	158	* buffer, so we can do all pages at once).
	159	*/
	160	int ipath_get_user_pages(unsigned long start_page, size_t num_pages,
	161	struct page **p)
	162	{
	163	int ret;
	164
	165	down_write(&current->mm->mmap_sem);
	166
	167	ret = __get_user_pages(start_page, num_pages, p, NULL);
	168
	169	up_write(&current->mm->mmap_sem);
	170
	171	return ret;
	172	}
	173
	174	/**
	175	* ipath_get_user_pages_nocopy - lock a single page for I/O and mark shared
	176	* @start_page: the page to lock
	177	* @p: the output page structure
	178	*
	179	* This is similar to ipath_get_user_pages, but it's always one page, and we
	180	* mark the page as locked for I/O, and shared. This is used for the user
	181	* process page that contains the destination address for the rcvhdrq tail
	182	* update, so we need to have the vma. If we don't do this, the page can be
	183	* taken away from us on fork, even if the child never touches it, and then
	184	* the user process never sees the tail register updates.
	185	*/
	186	int ipath_get_user_pages_nocopy(unsigned long page, struct page **p)
	187	{
	188	struct vm_area_struct *vma;
	189	int ret;
	190
	191	down_write(&current->mm->mmap_sem);
	192
	193	ret = __get_user_pages(page, 1, p, &vma);
	194
	195	up_write(&current->mm->mmap_sem);
	196
	197	return ret;
	198	}
	199
	200	void ipath_release_user_pages(struct page **p, size_t num_pages)
	201	{
	202	down_write(&current->mm->mmap_sem);
	203
	204	__ipath_release_user_pages(p, num_pages, 1);
	205
	206	current->mm->locked_vm -= num_pages;
	207
	208	up_write(&current->mm->mmap_sem);
	209	}
	210
	211	struct ipath_user_pages_work {
212	struct work_struct work;
213	struct mm_struct *mm;
214	unsigned long num_pages;
215	};
216
c4028958	217	static void user_pages_account(struct work_struct *_work)
7f510b46	218	{
c4028958 DH	219	struct ipath_user_pages_work *work =
c4028958 DH	220	container_of(_work, struct ipath_user_pages_work, work);
7f510b46 BS	221
	222	down_write(&work->mm->mmap_sem);
	223	work->mm->locked_vm -= work->num_pages;
	224	up_write(&work->mm->mmap_sem);
	225	mmput(work->mm);
	226	kfree(work);
	227	}
	228
	229	void ipath_release_user_pages_on_close(struct page **p, size_t num_pages)
	230	{
	231	struct ipath_user_pages_work *work;
	232	struct mm_struct *mm;
	233
	234	__ipath_release_user_pages(p, num_pages, 1);
	235
	236	mm = get_task_mm(current);
	237	if (!mm)
	238	goto bail;
	239
	240	work = kmalloc(sizeof(*work), GFP_KERNEL);
	241	if (!work)
	242	goto bail_mm;
	243
	244	goto bail;
	245
c4028958	246	INIT_WORK(&work->work, user_pages_account);
7f510b46 BS	247	work->mm = mm;
	248	work->num_pages = num_pages;
	249
	250	bail_mm:
	251	mmput(mm);
	252	bail:
	253	return;
	254	}