[mirror_ubuntu-artful-kernel.git] / drivers / net / mlx4 / alloc.c

/*
 * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/bitmap.h>
#include <linux/dma-mapping.h>
#include <linux/vmalloc.h>

#include "mlx4.h"

u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
{
	u32 obj;

	spin_lock(&bitmap->lock);

	obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
	if (obj >= bitmap->max) {
		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
				& bitmap->mask;
		obj = find_first_zero_bit(bitmap->table, bitmap->max);
	}

	if (obj < bitmap->max) {
		set_bit(obj, bitmap->table);
		bitmap->last = (obj + 1);
		if (bitmap->last == bitmap->max)
			bitmap->last = 0;
		obj |= bitmap->top;
	} else
		obj = -1;

	spin_unlock(&bitmap->lock);

	return obj;
}

void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj)
{
	mlx4_bitmap_free_range(bitmap, obj, 1);
}

u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
{
	u32 obj;

	if (likely(cnt == 1 && align == 1))
		return mlx4_bitmap_alloc(bitmap);

	spin_lock(&bitmap->lock);

	obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
				bitmap->last, cnt, align - 1);
	if (obj >= bitmap->max) {
		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
				& bitmap->mask;
		obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
						0, cnt, align - 1);
	}

	if (obj < bitmap->max) {
		bitmap_set(bitmap->table, obj, cnt);
		if (obj == bitmap->last) {
			bitmap->last = (obj + cnt);
			if (bitmap->last >= bitmap->max)
				bitmap->last = 0;
		}
		obj |= bitmap->top;
	} else
		obj = -1;

	spin_unlock(&bitmap->lock);

	return obj;
}

void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt)
{
	obj &= bitmap->max + bitmap->reserved_top - 1;

	spin_lock(&bitmap->lock);
	bitmap_clear(bitmap->table, obj, cnt);
	bitmap->last = min(bitmap->last, obj);
	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
			& bitmap->mask;
	spin_unlock(&bitmap->lock);
}

int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
		     u32 reserved_bot, u32 reserved_top)
{
	/* num must be a power of 2 */
	if (num != roundup_pow_of_two(num))
		return -EINVAL;

	bitmap->last = 0;
	bitmap->top  = 0;
	bitmap->max  = num - reserved_top;
	bitmap->mask = mask;
	bitmap->reserved_top = reserved_top;
	spin_lock_init(&bitmap->lock);
	bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
				sizeof (long), GFP_KERNEL);
	if (!bitmap->table)
		return -ENOMEM;

	bitmap_set(bitmap->table, 0, reserved_bot);

	return 0;
}

void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
{
	kfree(bitmap->table);
}

/*
 * Handling for queue buffers -- we allocate a bunch of memory and
 * register it in a memory region at HCA virtual address 0.  If the
 * requested size is > max_direct, we split the allocation into
 * multiple pages, so we don't require too much contiguous memory.
 */

int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
		   struct mlx4_buf *buf)
{
	dma_addr_t t;

	if (size <= max_direct) {
		buf->nbufs        = 1;
		buf->npages       = 1;
		buf->page_shift   = get_order(size) + PAGE_SHIFT;
		buf->direct.buf   = dma_alloc_coherent(&dev->pdev->dev,
						       size, &t, GFP_KERNEL);
		if (!buf->direct.buf)
			return -ENOMEM;

		buf->direct.map = t;

		while (t & ((1 << buf->page_shift) - 1)) {
			--buf->page_shift;
			buf->npages *= 2;
		}

		memset(buf->direct.buf, 0, size);
	} else {
		int i;

		buf->nbufs       = (size + PAGE_SIZE - 1) / PAGE_SIZE;
		buf->npages      = buf->nbufs;
		buf->page_shift  = PAGE_SHIFT;
		buf->page_list   = kcalloc(buf->nbufs, sizeof(*buf->page_list),
					   GFP_KERNEL);
		if (!buf->page_list)
			return -ENOMEM;

		for (i = 0; i < buf->nbufs; ++i) {
			buf->page_list[i].buf =
				dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
						   &t, GFP_KERNEL);
			if (!buf->page_list[i].buf)
				goto err_free;

			buf->page_list[i].map = t;

			memset(buf->page_list[i].buf, 0, PAGE_SIZE);
		}

		if (BITS_PER_LONG == 64) {
			struct page **pages;
			pages = kmalloc(sizeof *pages * buf->nbufs, GFP_KERNEL);
			if (!pages)
				goto err_free;
			for (i = 0; i < buf->nbufs; ++i)
				pages[i] = virt_to_page(buf->page_list[i].buf);
			buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
			kfree(pages);
			if (!buf->direct.buf)
				goto err_free;
		}
	}

	return 0;

err_free:
	mlx4_buf_free(dev, size, buf);

	return -ENOMEM;
}
EXPORT_SYMBOL_GPL(mlx4_buf_alloc);

void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
{
	int i;

	if (buf->nbufs == 1)
		dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,
				  buf->direct.map);
	else {
		if (BITS_PER_LONG == 64)
			vunmap(buf->direct.buf);

		for (i = 0; i < buf->nbufs; ++i)
			if (buf->page_list[i].buf)
				dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
						  buf->page_list[i].buf,
						  buf->page_list[i].map);
		kfree(buf->page_list);
	}
}
EXPORT_SYMBOL_GPL(mlx4_buf_free);

static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device)
{
	struct mlx4_db_pgdir *pgdir;

	pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL);
	if (!pgdir)
		return NULL;

	bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
	pgdir->bits[0] = pgdir->order0;
	pgdir->bits[1] = pgdir->order1;
	pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
					    &pgdir->db_dma, GFP_KERNEL);
	if (!pgdir->db_page) {
		kfree(pgdir);
		return NULL;
	}

	return pgdir;
}

static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
				    struct mlx4_db *db, int order)
{
	int o;
	int i;

	for (o = order; o <= 1; ++o) {
		i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o);
		if (i < MLX4_DB_PER_PAGE >> o)
			goto found;
	}

	return -ENOMEM;

found:
	clear_bit(i, pgdir->bits[o]);

	i <<= o;

	if (o > order)
		set_bit(i ^ 1, pgdir->bits[order]);

	db->u.pgdir = pgdir;
	db->index   = i;
	db->db      = pgdir->db_page + db->index;
	db->dma     = pgdir->db_dma  + db->index * 4;
	db->order   = order;

	return 0;
}

int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order)
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_db_pgdir *pgdir;
	int ret = 0;

	mutex_lock(&priv->pgdir_mutex);

	list_for_each_entry(pgdir, &priv->pgdir_list, list)
		if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
			goto out;

	pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev));
	if (!pgdir) {
		ret = -ENOMEM;
		goto out;
	}

	list_add(&pgdir->list, &priv->pgdir_list);

	/* This should never fail -- we just allocated an empty page: */
	WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order));

out:
	mutex_unlock(&priv->pgdir_mutex);

	return ret;
}
EXPORT_SYMBOL_GPL(mlx4_db_alloc);

void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db)
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	int o;
	int i;

	mutex_lock(&priv->pgdir_mutex);

	o = db->order;
	i = db->index;

	if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
		clear_bit(i ^ 1, db->u.pgdir->order0);
		++o;
	}
	i >>= o;
	set_bit(i, db->u.pgdir->bits[o]);

	if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
		dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
				  db->u.pgdir->db_page, db->u.pgdir->db_dma);
		list_del(&db->u.pgdir->list);
		kfree(db->u.pgdir);
	}

	mutex_unlock(&priv->pgdir_mutex);
}
EXPORT_SYMBOL_GPL(mlx4_db_free);

int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
		       int size, int max_direct)
{
	int err;

	err = mlx4_db_alloc(dev, &wqres->db, 1);
	if (err)
		return err;

	*wqres->db.db = 0;

	err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf);
	if (err)
		goto err_db;

	err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,
			    &wqres->mtt);
	if (err)
		goto err_buf;

	err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf);
	if (err)
		goto err_mtt;

	return 0;

err_mtt:
	mlx4_mtt_cleanup(dev, &wqres->mtt);
err_buf:
	mlx4_buf_free(dev, size, &wqres->buf);
err_db:
	mlx4_db_free(dev, &wqres->db);

	return err;
}
EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res);

void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
		       int size)
{
	mlx4_mtt_cleanup(dev, &wqres->mtt);
	mlx4_buf_free(dev, size, &wqres->buf);
	mlx4_db_free(dev, &wqres->db);
}
EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);
Commit	Line	Data
225c7b1f RD	1	/*
225c7b1f RD	2	* Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
51a379d0	3	* Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
225c7b1f RD	4	*
	5	* This software is available to you under a choice of one of two
	6	* licenses. You may choose to be licensed under the terms of the GNU
	7	* General Public License (GPL) Version 2, available from the file
	8	* COPYING in the main directory of this source tree, or the
	9	* OpenIB.org BSD license below:
	10	*
	11	* Redistribution and use in source and binary forms, with or
	12	* without modification, are permitted provided that the following
	13	* conditions are met:
	14	*
	15	* - Redistributions of source code must retain the above
	16	* copyright notice, this list of conditions and the following
	17	* disclaimer.
	18	*
	19	* - Redistributions in binary form must reproduce the above
	20	* copyright notice, this list of conditions and the following
	21	* disclaimer in the documentation and/or other materials
	22	* provided with the distribution.
	23	*
	24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	31	* SOFTWARE.
	32	*/
	33
	34	#include <linux/errno.h>
	35	#include <linux/slab.h>
6526128c	36	#include <linux/mm.h>
225c7b1f	37	#include <linux/bitmap.h>
9cbe05c7	38	#include <linux/dma-mapping.h>
29c27112	39	#include <linux/vmalloc.h>
225c7b1f RD	40
	41	#include "mlx4.h"
	42
	43	u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
	44	{
	45	u32 obj;
	46
	47	spin_lock(&bitmap->lock);
	48
	49	obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
	50	if (obj >= bitmap->max) {
93fc9e1b YP	51	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
93fc9e1b YP	52	& bitmap->mask;
225c7b1f RD	53	obj = find_first_zero_bit(bitmap->table, bitmap->max);
	54	}
	55
	56	if (obj < bitmap->max) {
	57	set_bit(obj, bitmap->table);
93fc9e1b YP	58	bitmap->last = (obj + 1);
	59	if (bitmap->last == bitmap->max)
	60	bitmap->last = 0;
225c7b1f	61	obj \|= bitmap->top;
225c7b1f RD	62	} else
	63	obj = -1;
	64
	65	spin_unlock(&bitmap->lock);
	66
	67	return obj;
	68	}
	69
	70	void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj)
	71	{
a3cdcbfa YP	72	mlx4_bitmap_free_range(bitmap, obj, 1);
	73	}
	74
a3cdcbfa YP	75	u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align)
a3cdcbfa YP	76	{
e27cd4f8	77	u32 obj;
a3cdcbfa YP	78
	79	if (likely(cnt == 1 && align == 1))
	80	return mlx4_bitmap_alloc(bitmap);
	81
	82	spin_lock(&bitmap->lock);
	83
43ff8b60 AM	84	obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
43ff8b60 AM	85	bitmap->last, cnt, align - 1);
a3cdcbfa	86	if (obj >= bitmap->max) {
93fc9e1b YP	87	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
93fc9e1b YP	88	& bitmap->mask;
43ff8b60 AM	89	obj = bitmap_find_next_zero_area(bitmap->table, bitmap->max,
43ff8b60 AM	90	0, cnt, align - 1);
a3cdcbfa YP	91	}
	92
	93	if (obj < bitmap->max) {
e27cd4f8	94	bitmap_set(bitmap->table, obj, cnt);
a3cdcbfa YP	95	if (obj == bitmap->last) {
	96	bitmap->last = (obj + cnt);
	97	if (bitmap->last >= bitmap->max)
	98	bitmap->last = 0;
	99	}
	100	obj \|= bitmap->top;
	101	} else
	102	obj = -1;
	103
	104	spin_unlock(&bitmap->lock);
	105
	106	return obj;
	107	}
	108
	109	void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt)
	110	{
93fc9e1b	111	obj &= bitmap->max + bitmap->reserved_top - 1;
225c7b1f RD	112
225c7b1f RD	113	spin_lock(&bitmap->lock);
e27cd4f8	114	bitmap_clear(bitmap->table, obj, cnt);
225c7b1f	115	bitmap->last = min(bitmap->last, obj);
93fc9e1b YP	116	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
93fc9e1b YP	117	& bitmap->mask;
225c7b1f RD	118	spin_unlock(&bitmap->lock);
	119	}
	120
93fc9e1b YP	121	int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
93fc9e1b YP	122	u32 reserved_bot, u32 reserved_top)
225c7b1f	123	{
225c7b1f RD	124	/* num must be a power of 2 */
	125	if (num != roundup_pow_of_two(num))
	126	return -EINVAL;
	127
	128	bitmap->last = 0;
	129	bitmap->top = 0;
93fc9e1b	130	bitmap->max = num - reserved_top;
225c7b1f	131	bitmap->mask = mask;
93fc9e1b	132	bitmap->reserved_top = reserved_top;
225c7b1f	133	spin_lock_init(&bitmap->lock);
93fc9e1b YP	134	bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
93fc9e1b YP	135	sizeof (long), GFP_KERNEL);
225c7b1f RD	136	if (!bitmap->table)
	137	return -ENOMEM;
	138
e27cd4f8	139	bitmap_set(bitmap->table, 0, reserved_bot);
225c7b1f RD	140
	141	return 0;
	142	}
	143
	144	void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
	145	{
	146	kfree(bitmap->table);
	147	}
	148
	149	/*
	150	* Handling for queue buffers -- we allocate a bunch of memory and
	151	* register it in a memory region at HCA virtual address 0. If the
	152	* requested size is > max_direct, we split the allocation into
	153	* multiple pages, so we don't require too much contiguous memory.
	154	*/
	155
	156	int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
	157	struct mlx4_buf *buf)
	158	{
	159	dma_addr_t t;
	160
	161	if (size <= max_direct) {
	162	buf->nbufs = 1;
	163	buf->npages = 1;
	164	buf->page_shift = get_order(size) + PAGE_SHIFT;
b57aacfa	165	buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev,
225c7b1f	166	size, &t, GFP_KERNEL);
b57aacfa	167	if (!buf->direct.buf)
225c7b1f RD	168	return -ENOMEM;
225c7b1f RD	169
b57aacfa	170	buf->direct.map = t;
225c7b1f RD	171
	172	while (t & ((1 << buf->page_shift) - 1)) {
	173	--buf->page_shift;
	174	buf->npages *= 2;
	175	}
	176
b57aacfa	177	memset(buf->direct.buf, 0, size);
225c7b1f RD	178	} else {
	179	int i;
	180
	181	buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
	182	buf->npages = buf->nbufs;
	183	buf->page_shift = PAGE_SHIFT;
baeb2ffa	184	buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),
225c7b1f	185	GFP_KERNEL);
b57aacfa	186	if (!buf->page_list)
225c7b1f RD	187	return -ENOMEM;
	188
	189	for (i = 0; i < buf->nbufs; ++i) {
b57aacfa	190	buf->page_list[i].buf =
225c7b1f RD	191	dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
225c7b1f RD	192	&t, GFP_KERNEL);
b57aacfa	193	if (!buf->page_list[i].buf)
225c7b1f RD	194	goto err_free;
225c7b1f RD	195
b57aacfa	196	buf->page_list[i].map = t;
225c7b1f	197
b57aacfa	198	memset(buf->page_list[i].buf, 0, PAGE_SIZE);
225c7b1f	199	}
313abe55 JM	200
	201	if (BITS_PER_LONG == 64) {
	202	struct page **pages;
	203	pages = kmalloc(sizeof pages buf->nbufs, GFP_KERNEL);
	204	if (!pages)
	205	goto err_free;
	206	for (i = 0; i < buf->nbufs; ++i)
b57aacfa RD	207	pages[i] = virt_to_page(buf->page_list[i].buf);
b57aacfa RD	208	buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
313abe55	209	kfree(pages);
b57aacfa	210	if (!buf->direct.buf)
313abe55 JM	211	goto err_free;
313abe55 JM	212	}
225c7b1f RD	213	}
	214
	215	return 0;
	216
	217	err_free:
	218	mlx4_buf_free(dev, size, buf);
	219
	220	return -ENOMEM;
	221	}
	222	EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
	223
	224	void mlx4_buf_free(struct mlx4_dev dev, int size, struct mlx4_buf buf)
	225	{
	226	int i;
	227
	228	if (buf->nbufs == 1)
b57aacfa RD	229	dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,
b57aacfa RD	230	buf->direct.map);
225c7b1f	231	else {
313abe55	232	if (BITS_PER_LONG == 64)
b57aacfa	233	vunmap(buf->direct.buf);
313abe55	234
225c7b1f	235	for (i = 0; i < buf->nbufs; ++i)
b57aacfa	236	if (buf->page_list[i].buf)
3bba11e5	237	dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
b57aacfa RD	238	buf->page_list[i].buf,
	239	buf->page_list[i].map);
	240	kfree(buf->page_list);
225c7b1f RD	241	}
	242	}
	243	EXPORT_SYMBOL_GPL(mlx4_buf_free);
6296883c YP	244
	245	static struct mlx4_db_pgdir mlx4_alloc_db_pgdir(struct device dma_device)
	246	{
	247	struct mlx4_db_pgdir *pgdir;
	248
	249	pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL);
	250	if (!pgdir)
	251	return NULL;
	252
	253	bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
	254	pgdir->bits[0] = pgdir->order0;
	255	pgdir->bits[1] = pgdir->order1;
	256	pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
	257	&pgdir->db_dma, GFP_KERNEL);
	258	if (!pgdir->db_page) {
	259	kfree(pgdir);
	260	return NULL;
	261	}
	262
	263	return pgdir;
	264	}
	265
	266	static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
	267	struct mlx4_db *db, int order)
	268	{
	269	int o;
	270	int i;
	271
	272	for (o = order; o <= 1; ++o) {
	273	i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o);
	274	if (i < MLX4_DB_PER_PAGE >> o)
	275	goto found;
	276	}
	277
	278	return -ENOMEM;
	279
	280	found:
	281	clear_bit(i, pgdir->bits[o]);
	282
	283	i <<= o;
	284
	285	if (o > order)
	286	set_bit(i ^ 1, pgdir->bits[order]);
	287
	288	db->u.pgdir = pgdir;
	289	db->index = i;
	290	db->db = pgdir->db_page + db->index;
	291	db->dma = pgdir->db_dma + db->index * 4;
	292	db->order = order;
	293
	294	return 0;
	295	}
	296
	297	int mlx4_db_alloc(struct mlx4_dev dev, struct mlx4_db db, int order)
	298	{
	299	struct mlx4_priv *priv = mlx4_priv(dev);
	300	struct mlx4_db_pgdir *pgdir;
	301	int ret = 0;
	302
	303	mutex_lock(&priv->pgdir_mutex);
	304
	305	list_for_each_entry(pgdir, &priv->pgdir_list, list)
	306	if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
	307	goto out;
308
309	pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev));
310	if (!pgdir) {
311	ret = -ENOMEM;
312	goto out;
313	}
314
315	list_add(&pgdir->list, &priv->pgdir_list);
316
317	/* This should never fail -- we just allocated an empty page: */
318	WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order));
319
320	out:
321	mutex_unlock(&priv->pgdir_mutex);
322
323	return ret;
324	}
325	EXPORT_SYMBOL_GPL(mlx4_db_alloc);
326
327	void mlx4_db_free(struct mlx4_dev dev, struct mlx4_db db)
328	{
329	struct mlx4_priv *priv = mlx4_priv(dev);
330	int o;
331	int i;
332
333	mutex_lock(&priv->pgdir_mutex);
334
335	o = db->order;
336	i = db->index;
337
338	if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
339	clear_bit(i ^ 1, db->u.pgdir->order0);
340	++o;
341	}
342	i >>= o;
343	set_bit(i, db->u.pgdir->bits[o]);
344
345	if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
346	dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
347	db->u.pgdir->db_page, db->u.pgdir->db_dma);
348	list_del(&db->u.pgdir->list);
349	kfree(db->u.pgdir);
350	}
351
352	mutex_unlock(&priv->pgdir_mutex);
353	}
354	EXPORT_SYMBOL_GPL(mlx4_db_free);
38ae6a53 YP	355
	356	int mlx4_alloc_hwq_res(struct mlx4_dev dev, struct mlx4_hwq_resources wqres,
	357	int size, int max_direct)
	358	{
	359	int err;
	360
	361	err = mlx4_db_alloc(dev, &wqres->db, 1);
	362	if (err)
	363	return err;
	364
	365	*wqres->db.db = 0;
	366
	367	err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf);
	368	if (err)
	369	goto err_db;
	370
	371	err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,
	372	&wqres->mtt);
	373	if (err)
	374	goto err_buf;
	375
	376	err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf);
	377	if (err)
	378	goto err_mtt;
	379
	380	return 0;
	381
	382	err_mtt:
	383	mlx4_mtt_cleanup(dev, &wqres->mtt);
	384	err_buf:
	385	mlx4_buf_free(dev, size, &wqres->buf);
	386	err_db:
	387	mlx4_db_free(dev, &wqres->db);
	388
	389	return err;
	390	}
	391	EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res);
	392
	393	void mlx4_free_hwq_res(struct mlx4_dev dev, struct mlx4_hwq_resources wqres,
	394	int size)
	395	{
	396	mlx4_mtt_cleanup(dev, &wqres->mtt);
	397	mlx4_buf_free(dev, size, &wqres->buf);
	398	mlx4_db_free(dev, &wqres->db);
	399	}
	400	EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);