[mirror_ubuntu-artful-kernel.git] / drivers / infiniband / hw / ipath / ipath_mr.c

/*
 * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/slab.h>

#include <rdma/ib_umem.h>
#include <rdma/ib_pack.h>
#include <rdma/ib_smi.h>

#include "ipath_verbs.h"

/* Fast memory region */
struct ipath_fmr {
	struct ib_fmr ibfmr;
	u8 page_shift;
	struct ipath_mregion mr;        /* must be last */
};

static inline struct ipath_fmr *to_ifmr(struct ib_fmr *ibfmr)
{
	return container_of(ibfmr, struct ipath_fmr, ibfmr);
}

/**
 * ipath_get_dma_mr - get a DMA memory region
 * @pd: protection domain for this memory region
 * @acc: access flags
 *
 * Returns the memory region on success, otherwise returns an errno.
 * Note that all DMA addresses should be created via the
 * struct ib_dma_mapping_ops functions (see ipath_dma.c).
 */
struct ib_mr *ipath_get_dma_mr(struct ib_pd *pd, int acc)
{
	struct ipath_mr *mr;
	struct ib_mr *ret;

	mr = kzalloc(sizeof *mr, GFP_KERNEL);
	if (!mr) {
		ret = ERR_PTR(-ENOMEM);
		goto bail;
	}

	mr->mr.access_flags = acc;
	ret = &mr->ibmr;

bail:
	return ret;
}

static struct ipath_mr *alloc_mr(int count,
				 struct ipath_lkey_table *lk_table)
{
	struct ipath_mr *mr;
	int m, i = 0;

	/* Allocate struct plus pointers to first level page tables. */
	m = (count + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
	mr = kmalloc(sizeof *mr + m * sizeof mr->mr.map[0], GFP_KERNEL);
	if (!mr)
		goto done;

	/* Allocate first level page tables. */
	for (; i < m; i++) {
		mr->mr.map[i] = kmalloc(sizeof *mr->mr.map[0], GFP_KERNEL);
		if (!mr->mr.map[i])
			goto bail;
	}
	mr->mr.mapsz = m;

	/*
	 * ib_reg_phys_mr() will initialize mr->ibmr except for
	 * lkey and rkey.
	 */
	if (!ipath_alloc_lkey(lk_table, &mr->mr))
		goto bail;
	mr->ibmr.rkey = mr->ibmr.lkey = mr->mr.lkey;

	goto done;

bail:
	while (i) {
		i--;
		kfree(mr->mr.map[i]);
	}
	kfree(mr);
	mr = NULL;

done:
	return mr;
}

/**
 * ipath_reg_phys_mr - register a physical memory region
 * @pd: protection domain for this memory region
 * @buffer_list: pointer to the list of physical buffers to register
 * @num_phys_buf: the number of physical buffers to register
 * @iova_start: the starting address passed over IB which maps to this MR
 *
 * Returns the memory region on success, otherwise returns an errno.
 */
struct ib_mr *ipath_reg_phys_mr(struct ib_pd *pd,
				struct ib_phys_buf *buffer_list,
				int num_phys_buf, int acc, u64 *iova_start)
{
	struct ipath_mr *mr;
	int n, m, i;
	struct ib_mr *ret;

	mr = alloc_mr(num_phys_buf, &to_idev(pd->device)->lk_table);
	if (mr == NULL) {
		ret = ERR_PTR(-ENOMEM);
		goto bail;
	}

	mr->mr.pd = pd;
	mr->mr.user_base = *iova_start;
	mr->mr.iova = *iova_start;
	mr->mr.length = 0;
	mr->mr.offset = 0;
	mr->mr.access_flags = acc;
	mr->mr.max_segs = num_phys_buf;
	mr->umem = NULL;

	m = 0;
	n = 0;
	for (i = 0; i < num_phys_buf; i++) {
		mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr;
		mr->mr.map[m]->segs[n].length = buffer_list[i].size;
		mr->mr.length += buffer_list[i].size;
		n++;
		if (n == IPATH_SEGSZ) {
			m++;
			n = 0;
		}
	}

	ret = &mr->ibmr;

bail:
	return ret;
}

/**
 * ipath_reg_user_mr - register a userspace memory region
 * @pd: protection domain for this memory region
 * @start: starting userspace address
 * @length: length of region to register
 * @virt_addr: virtual address to use (from HCA's point of view)
 * @mr_access_flags: access flags for this memory region
 * @udata: unused by the InfiniPath driver
 *
 * Returns the memory region on success, otherwise returns an errno.
 */
struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
				u64 virt_addr, int mr_access_flags,
				struct ib_udata *udata)
{
	struct ipath_mr *mr;
	struct ib_umem *umem;
	struct ib_umem_chunk *chunk;
	int n, m, i;
	struct ib_mr *ret;

	if (length == 0) {
		ret = ERR_PTR(-EINVAL);
		goto bail;
	}

	umem = ib_umem_get(pd->uobject->context, start, length,
			   mr_access_flags, 0);
	if (IS_ERR(umem))
		return (void *) umem;

	n = 0;
	list_for_each_entry(chunk, &umem->chunk_list, list)
		n += chunk->nents;

	mr = alloc_mr(n, &to_idev(pd->device)->lk_table);
	if (!mr) {
		ret = ERR_PTR(-ENOMEM);
		ib_umem_release(umem);
		goto bail;
	}

	mr->mr.pd = pd;
	mr->mr.user_base = start;
	mr->mr.iova = virt_addr;
	mr->mr.length = length;
	mr->mr.offset = umem->offset;
	mr->mr.access_flags = mr_access_flags;
	mr->mr.max_segs = n;
	mr->umem = umem;

	m = 0;
	n = 0;
	list_for_each_entry(chunk, &umem->chunk_list, list) {
		for (i = 0; i < chunk->nents; i++) {
			void *vaddr;

			vaddr = page_address(sg_page(&chunk->page_list[i]));
			if (!vaddr) {
				ret = ERR_PTR(-EINVAL);
				goto bail;
			}
			mr->mr.map[m]->segs[n].vaddr = vaddr;
			mr->mr.map[m]->segs[n].length = umem->page_size;
			n++;
			if (n == IPATH_SEGSZ) {
				m++;
				n = 0;
			}
		}
	}
	ret = &mr->ibmr;

bail:
	return ret;
}

/**
 * ipath_dereg_mr - unregister and free a memory region
 * @ibmr: the memory region to free
 *
 * Returns 0 on success.
 *
 * Note that this is called to free MRs created by ipath_get_dma_mr()
 * or ipath_reg_user_mr().
 */
int ipath_dereg_mr(struct ib_mr *ibmr)
{
	struct ipath_mr *mr = to_imr(ibmr);
	int i;

	ipath_free_lkey(&to_idev(ibmr->device)->lk_table, ibmr->lkey);
	i = mr->mr.mapsz;
	while (i) {
		i--;
		kfree(mr->mr.map[i]);
	}

	if (mr->umem)
		ib_umem_release(mr->umem);

	kfree(mr);
	return 0;
}

/**
 * ipath_alloc_fmr - allocate a fast memory region
 * @pd: the protection domain for this memory region
 * @mr_access_flags: access flags for this memory region
 * @fmr_attr: fast memory region attributes
 *
 * Returns the memory region on success, otherwise returns an errno.
 */
struct ib_fmr *ipath_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
			       struct ib_fmr_attr *fmr_attr)
{
	struct ipath_fmr *fmr;
	int m, i = 0;
	struct ib_fmr *ret;

	/* Allocate struct plus pointers to first level page tables. */
	m = (fmr_attr->max_pages + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
	fmr = kmalloc(sizeof *fmr + m * sizeof fmr->mr.map[0], GFP_KERNEL);
	if (!fmr)
		goto bail;

	/* Allocate first level page tables. */
	for (; i < m; i++) {
		fmr->mr.map[i] = kmalloc(sizeof *fmr->mr.map[0],
					 GFP_KERNEL);
		if (!fmr->mr.map[i])
			goto bail;
	}
	fmr->mr.mapsz = m;

	/*
	 * ib_alloc_fmr() will initialize fmr->ibfmr except for lkey &
	 * rkey.
	 */
	if (!ipath_alloc_lkey(&to_idev(pd->device)->lk_table, &fmr->mr))
		goto bail;
	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mr.lkey;
	/*
	 * Resources are allocated but no valid mapping (RKEY can't be
	 * used).
	 */
	fmr->mr.pd = pd;
	fmr->mr.user_base = 0;
	fmr->mr.iova = 0;
	fmr->mr.length = 0;
	fmr->mr.offset = 0;
	fmr->mr.access_flags = mr_access_flags;
	fmr->mr.max_segs = fmr_attr->max_pages;
	fmr->page_shift = fmr_attr->page_shift;

	ret = &fmr->ibfmr;
	goto done;

bail:
	while (i)
		kfree(fmr->mr.map[--i]);
	kfree(fmr);
	ret = ERR_PTR(-ENOMEM);

done:
	return ret;
}

/**
 * ipath_map_phys_fmr - set up a fast memory region
 * @ibmfr: the fast memory region to set up
 * @page_list: the list of pages to associate with the fast memory region
 * @list_len: the number of pages to associate with the fast memory region
 * @iova: the virtual address of the start of the fast memory region
 *
 * This may be called from interrupt context.
 */

int ipath_map_phys_fmr(struct ib_fmr *ibfmr, u64 * page_list,
		       int list_len, u64 iova)
{
	struct ipath_fmr *fmr = to_ifmr(ibfmr);
	struct ipath_lkey_table *rkt;
	unsigned long flags;
	int m, n, i;
	u32 ps;
	int ret;

	if (list_len > fmr->mr.max_segs) {
		ret = -EINVAL;
		goto bail;
	}
	rkt = &to_idev(ibfmr->device)->lk_table;
	spin_lock_irqsave(&rkt->lock, flags);
	fmr->mr.user_base = iova;
	fmr->mr.iova = iova;
	ps = 1 << fmr->page_shift;
	fmr->mr.length = list_len * ps;
	m = 0;
	n = 0;
	ps = 1 << fmr->page_shift;
	for (i = 0; i < list_len; i++) {
		fmr->mr.map[m]->segs[n].vaddr = (void *) page_list[i];
		fmr->mr.map[m]->segs[n].length = ps;
		if (++n == IPATH_SEGSZ) {
			m++;
			n = 0;
		}
	}
	spin_unlock_irqrestore(&rkt->lock, flags);
	ret = 0;

bail:
	return ret;
}

/**
 * ipath_unmap_fmr - unmap fast memory regions
 * @fmr_list: the list of fast memory regions to unmap
 *
 * Returns 0 on success.
 */
int ipath_unmap_fmr(struct list_head *fmr_list)
{
	struct ipath_fmr *fmr;
	struct ipath_lkey_table *rkt;
	unsigned long flags;

	list_for_each_entry(fmr, fmr_list, ibfmr.list) {
		rkt = &to_idev(fmr->ibfmr.device)->lk_table;
		spin_lock_irqsave(&rkt->lock, flags);
		fmr->mr.user_base = 0;
		fmr->mr.iova = 0;
		fmr->mr.length = 0;
		spin_unlock_irqrestore(&rkt->lock, flags);
	}
	return 0;
}

/**
 * ipath_dealloc_fmr - deallocate a fast memory region
 * @ibfmr: the fast memory region to deallocate
 *
 * Returns 0 on success.
 */
int ipath_dealloc_fmr(struct ib_fmr *ibfmr)
{
	struct ipath_fmr *fmr = to_ifmr(ibfmr);
	int i;

	ipath_free_lkey(&to_idev(ibfmr->device)->lk_table, ibfmr->lkey);
	i = fmr->mr.mapsz;
	while (i)
		kfree(fmr->mr.map[--i]);
	kfree(fmr);
	return 0;
}
Commit	Line	Data
cef1cce5	1	/*
87427da5	2	* Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved.
cef1cce5 BS	3	* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
	4	*
	5	* This software is available to you under a choice of one of two
	6	* licenses. You may choose to be licensed under the terms of the GNU
	7	* General Public License (GPL) Version 2, available from the file
	8	* COPYING in the main directory of this source tree, or the
	9	* OpenIB.org BSD license below:
	10	*
	11	* Redistribution and use in source and binary forms, with or
	12	* without modification, are permitted provided that the following
	13	* conditions are met:
	14	*
	15	* - Redistributions of source code must retain the above
	16	* copyright notice, this list of conditions and the following
	17	* disclaimer.
	18	*
	19	* - Redistributions in binary form must reproduce the above
	20	* copyright notice, this list of conditions and the following
	21	* disclaimer in the documentation and/or other materials
	22	* provided with the distribution.
	23	*
	24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	31	* SOFTWARE.
	32	*/
	33
5a0e3ad6 TH	34	#include <linux/slab.h>
5a0e3ad6 TH	35
f7c6a7b5	36	#include <rdma/ib_umem.h>
cef1cce5 BS	37	#include <rdma/ib_pack.h>
	38	#include <rdma/ib_smi.h>
	39
	40	#include "ipath_verbs.h"
	41
34b2aafe BS	42	/* Fast memory region */
	43	struct ipath_fmr {
	44	struct ib_fmr ibfmr;
	45	u8 page_shift;
	46	struct ipath_mregion mr; /* must be last */
	47	};
	48
	49	static inline struct ipath_fmr to_ifmr(struct ib_fmr ibfmr)
	50	{
	51	return container_of(ibfmr, struct ipath_fmr, ibfmr);
	52	}
	53
cef1cce5 BS	54	/**
	55	* ipath_get_dma_mr - get a DMA memory region
	56	* @pd: protection domain for this memory region
	57	* @acc: access flags
	58	*
	59	* Returns the memory region on success, otherwise returns an errno.
f2cbb660 RC	60	* Note that all DMA addresses should be created via the
f2cbb660 RC	61	* struct ib_dma_mapping_ops functions (see ipath_dma.c).
cef1cce5 BS	62	*/
	63	struct ib_mr ipath_get_dma_mr(struct ib_pd pd, int acc)
	64	{
	65	struct ipath_mr *mr;
	66	struct ib_mr *ret;
	67
	68	mr = kzalloc(sizeof *mr, GFP_KERNEL);
	69	if (!mr) {
	70	ret = ERR_PTR(-ENOMEM);
	71	goto bail;
	72	}
	73
	74	mr->mr.access_flags = acc;
	75	ret = &mr->ibmr;
	76
	77	bail:
	78	return ret;
	79	}
	80
	81	static struct ipath_mr *alloc_mr(int count,
	82	struct ipath_lkey_table *lk_table)
	83	{
	84	struct ipath_mr *mr;
	85	int m, i = 0;
	86
	87	/* Allocate struct plus pointers to first level page tables. */
	88	m = (count + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
	89	mr = kmalloc(sizeof mr + m sizeof mr->mr.map[0], GFP_KERNEL);
	90	if (!mr)
	91	goto done;
	92
	93	/* Allocate first level page tables. */
	94	for (; i < m; i++) {
	95	mr->mr.map[i] = kmalloc(sizeof *mr->mr.map[0], GFP_KERNEL);
	96	if (!mr->mr.map[i])
	97	goto bail;
	98	}
	99	mr->mr.mapsz = m;
	100
	101	/*
	102	* ib_reg_phys_mr() will initialize mr->ibmr except for
	103	* lkey and rkey.
	104	*/
	105	if (!ipath_alloc_lkey(lk_table, &mr->mr))
	106	goto bail;
	107	mr->ibmr.rkey = mr->ibmr.lkey = mr->mr.lkey;
	108
	109	goto done;
	110
	111	bail:
	112	while (i) {
	113	i--;
	114	kfree(mr->mr.map[i]);
	115	}
	116	kfree(mr);
	117	mr = NULL;
	118
	119	done:
	120	return mr;
	121	}
	122
	123	/**
	124	* ipath_reg_phys_mr - register a physical memory region
	125	* @pd: protection domain for this memory region
126	* @buffer_list: pointer to the list of physical buffers to register
127	* @num_phys_buf: the number of physical buffers to register
128	* @iova_start: the starting address passed over IB which maps to this MR
129	*
130	* Returns the memory region on success, otherwise returns an errno.
131	*/
132	struct ib_mr ipath_reg_phys_mr(struct ib_pd pd,
133	struct ib_phys_buf *buffer_list,
134	int num_phys_buf, int acc, u64 *iova_start)
135	{
136	struct ipath_mr *mr;
137	int n, m, i;
138	struct ib_mr *ret;
139
140	mr = alloc_mr(num_phys_buf, &to_idev(pd->device)->lk_table);
141	if (mr == NULL) {
142	ret = ERR_PTR(-ENOMEM);
143	goto bail;
144	}
145
6a553af2	146	mr->mr.pd = pd;
cef1cce5 BS	147	mr->mr.user_base = *iova_start;
	148	mr->mr.iova = *iova_start;
	149	mr->mr.length = 0;
	150	mr->mr.offset = 0;
	151	mr->mr.access_flags = acc;
	152	mr->mr.max_segs = num_phys_buf;
f7c6a7b5	153	mr->umem = NULL;
cef1cce5 BS	154
	155	m = 0;
	156	n = 0;
	157	for (i = 0; i < num_phys_buf; i++) {
f2cbb660	158	mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr;
cef1cce5 BS	159	mr->mr.map[m]->segs[n].length = buffer_list[i].size;
	160	mr->mr.length += buffer_list[i].size;
	161	n++;
	162	if (n == IPATH_SEGSZ) {
	163	m++;
	164	n = 0;
	165	}
	166	}
	167
	168	ret = &mr->ibmr;
	169
	170	bail:
	171	return ret;
	172	}
	173
	174	/**
	175	* ipath_reg_user_mr - register a userspace memory region
	176	* @pd: protection domain for this memory region
f7c6a7b5 RD	177	* @start: starting userspace address
	178	* @length: length of region to register
	179	* @virt_addr: virtual address to use (from HCA's point of view)
cef1cce5 BS	180	* @mr_access_flags: access flags for this memory region
	181	* @udata: unused by the InfiniPath driver
	182	*
	183	* Returns the memory region on success, otherwise returns an errno.
	184	*/
f7c6a7b5 RD	185	struct ib_mr ipath_reg_user_mr(struct ib_pd pd, u64 start, u64 length,
	186	u64 virt_addr, int mr_access_flags,
	187	struct ib_udata *udata)
cef1cce5 BS	188	{
cef1cce5 BS	189	struct ipath_mr *mr;
f7c6a7b5	190	struct ib_umem *umem;
cef1cce5 BS	191	struct ib_umem_chunk *chunk;
	192	int n, m, i;
	193	struct ib_mr *ret;
	194
f7c6a7b5	195	if (length == 0) {
4a45b7d4 BS	196	ret = ERR_PTR(-EINVAL);
	197	goto bail;
	198	}
	199
cb9fbc5c AK	200	umem = ib_umem_get(pd->uobject->context, start, length,
cb9fbc5c AK	201	mr_access_flags, 0);
f7c6a7b5 RD	202	if (IS_ERR(umem))
	203	return (void *) umem;
	204
cef1cce5	205	n = 0;
f7c6a7b5	206	list_for_each_entry(chunk, &umem->chunk_list, list)
cef1cce5 BS	207	n += chunk->nents;
	208
	209	mr = alloc_mr(n, &to_idev(pd->device)->lk_table);
	210	if (!mr) {
	211	ret = ERR_PTR(-ENOMEM);
f7c6a7b5	212	ib_umem_release(umem);
cef1cce5 BS	213	goto bail;
	214	}
	215
6a553af2	216	mr->mr.pd = pd;
f7c6a7b5 RD	217	mr->mr.user_base = start;
	218	mr->mr.iova = virt_addr;
	219	mr->mr.length = length;
	220	mr->mr.offset = umem->offset;
cef1cce5 BS	221	mr->mr.access_flags = mr_access_flags;
cef1cce5 BS	222	mr->mr.max_segs = n;
f7c6a7b5	223	mr->umem = umem;
cef1cce5 BS	224
	225	m = 0;
	226	n = 0;
f7c6a7b5	227	list_for_each_entry(chunk, &umem->chunk_list, list) {
165c552c BS	228	for (i = 0; i < chunk->nents; i++) {
	229	void *vaddr;
	230
45711f1a	231	vaddr = page_address(sg_page(&chunk->page_list[i]));
165c552c BS	232	if (!vaddr) {
	233	ret = ERR_PTR(-EINVAL);
	234	goto bail;
	235	}
	236	mr->mr.map[m]->segs[n].vaddr = vaddr;
f7c6a7b5	237	mr->mr.map[m]->segs[n].length = umem->page_size;
cef1cce5 BS	238	n++;
	239	if (n == IPATH_SEGSZ) {
	240	m++;
	241	n = 0;
	242	}
	243	}
	244	}
	245	ret = &mr->ibmr;
	246
	247	bail:
	248	return ret;
	249	}
	250
	251	/**
	252	* ipath_dereg_mr - unregister and free a memory region
	253	* @ibmr: the memory region to free
	254	*
	255	* Returns 0 on success.
	256	*
	257	* Note that this is called to free MRs created by ipath_get_dma_mr()
	258	* or ipath_reg_user_mr().
	259	*/
	260	int ipath_dereg_mr(struct ib_mr *ibmr)
	261	{
	262	struct ipath_mr *mr = to_imr(ibmr);
	263	int i;
	264
	265	ipath_free_lkey(&to_idev(ibmr->device)->lk_table, ibmr->lkey);
	266	i = mr->mr.mapsz;
	267	while (i) {
	268	i--;
	269	kfree(mr->mr.map[i]);
	270	}
f7c6a7b5 RD	271
	272	if (mr->umem)
	273	ib_umem_release(mr->umem);
	274
cef1cce5 BS	275	kfree(mr);
	276	return 0;
	277	}
	278
	279	/**
	280	* ipath_alloc_fmr - allocate a fast memory region
	281	* @pd: the protection domain for this memory region
	282	* @mr_access_flags: access flags for this memory region
	283	* @fmr_attr: fast memory region attributes
	284	*
	285	* Returns the memory region on success, otherwise returns an errno.
	286	*/
	287	struct ib_fmr ipath_alloc_fmr(struct ib_pd pd, int mr_access_flags,
	288	struct ib_fmr_attr *fmr_attr)
	289	{
	290	struct ipath_fmr *fmr;
	291	int m, i = 0;
	292	struct ib_fmr *ret;
	293
	294	/* Allocate struct plus pointers to first level page tables. */
	295	m = (fmr_attr->max_pages + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
	296	fmr = kmalloc(sizeof fmr + m sizeof fmr->mr.map[0], GFP_KERNEL);
	297	if (!fmr)
	298	goto bail;
	299
	300	/* Allocate first level page tables. */
	301	for (; i < m; i++) {
	302	fmr->mr.map[i] = kmalloc(sizeof *fmr->mr.map[0],
	303	GFP_KERNEL);
	304	if (!fmr->mr.map[i])
	305	goto bail;
	306	}
	307	fmr->mr.mapsz = m;
	308
	309	/*
	310	* ib_alloc_fmr() will initialize fmr->ibfmr except for lkey &
	311	* rkey.
	312	*/
	313	if (!ipath_alloc_lkey(&to_idev(pd->device)->lk_table, &fmr->mr))
	314	goto bail;
	315	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mr.lkey;
	316	/*
	317	* Resources are allocated but no valid mapping (RKEY can't be
	318	* used).
	319	*/
6a553af2	320	fmr->mr.pd = pd;
cef1cce5 BS	321	fmr->mr.user_base = 0;
	322	fmr->mr.iova = 0;
	323	fmr->mr.length = 0;
	324	fmr->mr.offset = 0;
	325	fmr->mr.access_flags = mr_access_flags;
	326	fmr->mr.max_segs = fmr_attr->max_pages;
	327	fmr->page_shift = fmr_attr->page_shift;
	328
	329	ret = &fmr->ibfmr;
	330	goto done;
	331
	332	bail:
	333	while (i)
	334	kfree(fmr->mr.map[--i]);
	335	kfree(fmr);
	336	ret = ERR_PTR(-ENOMEM);
	337
	338	done:
	339	return ret;
	340	}
	341
	342	/**
	343	* ipath_map_phys_fmr - set up a fast memory region
	344	* @ibmfr: the fast memory region to set up
	345	* @page_list: the list of pages to associate with the fast memory region
	346	* @list_len: the number of pages to associate with the fast memory region
	347	* @iova: the virtual address of the start of the fast memory region
	348	*
	349	* This may be called from interrupt context.
	350	*/
	351
	352	int ipath_map_phys_fmr(struct ib_fmr ibfmr, u64 page_list,
	353	int list_len, u64 iova)
	354	{
	355	struct ipath_fmr *fmr = to_ifmr(ibfmr);
	356	struct ipath_lkey_table *rkt;
	357	unsigned long flags;
	358	int m, n, i;
	359	u32 ps;
	360	int ret;
	361
	362	if (list_len > fmr->mr.max_segs) {
	363	ret = -EINVAL;
	364	goto bail;
	365	}
	366	rkt = &to_idev(ibfmr->device)->lk_table;
	367	spin_lock_irqsave(&rkt->lock, flags);
	368	fmr->mr.user_base = iova;
	369	fmr->mr.iova = iova;
	370	ps = 1 << fmr->page_shift;
	371	fmr->mr.length = list_len * ps;
	372	m = 0;
	373	n = 0;
	374	ps = 1 << fmr->page_shift;
	375	for (i = 0; i < list_len; i++) {
f2cbb660	376	fmr->mr.map[m]->segs[n].vaddr = (void *) page_list[i];
cef1cce5 BS	377	fmr->mr.map[m]->segs[n].length = ps;
	378	if (++n == IPATH_SEGSZ) {
	379	m++;
	380	n = 0;
	381	}
	382	}
	383	spin_unlock_irqrestore(&rkt->lock, flags);
	384	ret = 0;
	385
	386	bail:
	387	return ret;
	388	}
	389
	390	/**
	391	* ipath_unmap_fmr - unmap fast memory regions
	392	* @fmr_list: the list of fast memory regions to unmap
	393	*
	394	* Returns 0 on success.
	395	*/
	396	int ipath_unmap_fmr(struct list_head *fmr_list)
	397	{
	398	struct ipath_fmr *fmr;
	399	struct ipath_lkey_table *rkt;
	400	unsigned long flags;
	401
	402	list_for_each_entry(fmr, fmr_list, ibfmr.list) {
	403	rkt = &to_idev(fmr->ibfmr.device)->lk_table;
	404	spin_lock_irqsave(&rkt->lock, flags);
	405	fmr->mr.user_base = 0;
	406	fmr->mr.iova = 0;
	407	fmr->mr.length = 0;
	408	spin_unlock_irqrestore(&rkt->lock, flags);
	409	}
	410	return 0;
	411	}
	412
	413	/**
	414	* ipath_dealloc_fmr - deallocate a fast memory region
	415	* @ibfmr: the fast memory region to deallocate
	416	*
	417	* Returns 0 on success.
	418	*/
	419	int ipath_dealloc_fmr(struct ib_fmr *ibfmr)
	420	{
	421	struct ipath_fmr *fmr = to_ifmr(ibfmr);
	422	int i;
	423
	424	ipath_free_lkey(&to_idev(ibfmr->device)->lk_table, ibfmr->lkey);
	425	i = fmr->mr.mapsz;
	426	while (i)
	427	kfree(fmr->mr.map[--i]);
	428	kfree(fmr);
	429	return 0;
	430	}