[mirror_ubuntu-jammy-kernel.git] / drivers / infiniband / hw / ehca / ehca_cq.c

/*
 *  IBM eServer eHCA Infiniband device driver for Linux on POWER
 *
 *  Completion queue handling
 *
 *  Authors: Waleri Fomin <fomin@de.ibm.com>
 *           Khadija Souissi <souissi@de.ibm.com>
 *           Reinhard Ernst <rernst@de.ibm.com>
 *           Heiko J Schick <schickhj@de.ibm.com>
 *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
 *
 *
 *  Copyright (c) 2005 IBM Corporation
 *
 *  All rights reserved.
 *
 *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
 *  BSD.
 *
 * OpenIB BSD License
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials
 * provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "ehca_iverbs.h"
#include "ehca_classes.h"
#include "ehca_irq.h"
#include "hcp_if.h"

static struct kmem_cache *cq_cache;

int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
{
	unsigned int qp_num = qp->real_qp_num;
	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
	unsigned long flags;

	spin_lock_irqsave(&cq->spinlock, flags);
	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
	spin_unlock_irqrestore(&cq->spinlock, flags);

	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
		 cq->cq_number, qp_num);

	return 0;
}

int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
{
	int ret = -EINVAL;
	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	struct hlist_node *iter;
	struct ehca_qp *qp;
	unsigned long flags;

	spin_lock_irqsave(&cq->spinlock, flags);
	hlist_for_each(iter, &cq->qp_hashtab[key]) {
		qp = hlist_entry(iter, struct ehca_qp, list_entries);
		if (qp->real_qp_num == real_qp_num) {
			hlist_del(iter);
			ehca_dbg(cq->ib_cq.device,
				 "removed qp from cq .cq_num=%x real_qp_num=%x",
				 cq->cq_number, real_qp_num);
			ret = 0;
			break;
		}
	}
	spin_unlock_irqrestore(&cq->spinlock, flags);
	if (ret)
		ehca_err(cq->ib_cq.device,
			 "qp not found cq_num=%x real_qp_num=%x",
			 cq->cq_number, real_qp_num);

	return ret;
}

struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
{
	struct ehca_qp *ret = NULL;
	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	struct hlist_node *iter;
	struct ehca_qp *qp;
	hlist_for_each(iter, &cq->qp_hashtab[key]) {
		qp = hlist_entry(iter, struct ehca_qp, list_entries);
		if (qp->real_qp_num == real_qp_num) {
			ret = qp;
			break;
		}
	}
	return ret;
}

struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
			     struct ib_ucontext *context,
			     struct ib_udata *udata)
{
	static const u32 additional_cqe = 20;
	struct ib_cq *cq;
	struct ehca_cq *my_cq;
	struct ehca_shca *shca =
		container_of(device, struct ehca_shca, ib_device);
	struct ipz_adapter_handle adapter_handle;
	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
	struct h_galpa gal;
	void *vpage;
	u32 counter;
	u64 rpage, cqx_fec, h_ret;
	int ipz_rc, ret, i;
	unsigned long flags;

	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
		return ERR_PTR(-EINVAL);

	if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
		ehca_err(device, "Unable to create CQ, max number of %i "
			"CQs reached.", shca->max_num_cqs);
		ehca_err(device, "To increase the maximum number of CQs "
			"use the number_of_cqs module parameter.\n");
		return ERR_PTR(-ENOSPC);
	}

	my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
	if (!my_cq) {
		ehca_err(device, "Out of memory for ehca_cq struct device=%p",
			 device);
		atomic_dec(&shca->num_cqs);
		return ERR_PTR(-ENOMEM);
	}

	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));

	spin_lock_init(&my_cq->spinlock);
	spin_lock_init(&my_cq->cb_lock);
	spin_lock_init(&my_cq->task_lock);
	atomic_set(&my_cq->nr_events, 0);
	init_waitqueue_head(&my_cq->wait_completion);

	cq = &my_cq->ib_cq;

	adapter_handle = shca->ipz_hca_handle;
	param.eq_handle = shca->eq.ipz_eq_handle;

	do {
		if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) {
			cq = ERR_PTR(-ENOMEM);
			ehca_err(device, "Can't reserve idr nr. device=%p",
				 device);
			goto create_cq_exit1;
		}

		write_lock_irqsave(&ehca_cq_idr_lock, flags);
		ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token);
		write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
	} while (ret == -EAGAIN);

	if (ret) {
		cq = ERR_PTR(-ENOMEM);
		ehca_err(device, "Can't allocate new idr entry. device=%p",
			 device);
		goto create_cq_exit1;
	}

	if (my_cq->token > 0x1FFFFFF) {
		cq = ERR_PTR(-ENOMEM);
		ehca_err(device, "Invalid number of cq. device=%p", device);
		goto create_cq_exit2;
	}

	/*
	 * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
	 * for receiving errors CQEs.
	 */
	param.nr_cqe = cqe + additional_cqe;
	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);

	if (h_ret != H_SUCCESS) {
		ehca_err(device, "hipz_h_alloc_resource_cq() failed "
			 "h_ret=%li device=%p", h_ret, device);
		cq = ERR_PTR(ehca2ib_return_code(h_ret));
		goto create_cq_exit2;
	}

	ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
				EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
	if (!ipz_rc) {
		ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
			 ipz_rc, device);
		cq = ERR_PTR(-EINVAL);
		goto create_cq_exit3;
	}

	for (counter = 0; counter < param.act_pages; counter++) {
		vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
		if (!vpage) {
			ehca_err(device, "ipz_qpageit_get_inc() "
				 "returns NULL device=%p", device);
			cq = ERR_PTR(-EAGAIN);
			goto create_cq_exit4;
		}
		rpage = virt_to_abs(vpage);

		h_ret = hipz_h_register_rpage_cq(adapter_handle,
						 my_cq->ipz_cq_handle,
						 &my_cq->pf,
						 0,
						 0,
						 rpage,
						 1,
						 my_cq->galpas.
						 kernel);

		if (h_ret < H_SUCCESS) {
			ehca_err(device, "hipz_h_register_rpage_cq() failed "
				 "ehca_cq=%p cq_num=%x h_ret=%li counter=%i "
				 "act_pages=%i", my_cq, my_cq->cq_number,
				 h_ret, counter, param.act_pages);
			cq = ERR_PTR(-EINVAL);
			goto create_cq_exit4;
		}

		if (counter == (param.act_pages - 1)) {
			vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
			if ((h_ret != H_SUCCESS) || vpage) {
				ehca_err(device, "Registration of pages not "
					 "complete ehca_cq=%p cq_num=%x "
					 "h_ret=%li", my_cq, my_cq->cq_number,
					 h_ret);
				cq = ERR_PTR(-EAGAIN);
				goto create_cq_exit4;
			}
		} else {
			if (h_ret != H_PAGE_REGISTERED) {
				ehca_err(device, "Registration of page failed "
					 "ehca_cq=%p cq_num=%x h_ret=%li "
					 "counter=%i act_pages=%i",
					 my_cq, my_cq->cq_number,
					 h_ret, counter, param.act_pages);
				cq = ERR_PTR(-ENOMEM);
				goto create_cq_exit4;
			}
		}
	}

	ipz_qeit_reset(&my_cq->ipz_queue);

	gal = my_cq->galpas.kernel;
	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx",
		 my_cq, my_cq->cq_number, cqx_fec);

	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
		param.act_nr_of_entries - additional_cqe;
	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;

	for (i = 0; i < QP_HASHTAB_LEN; i++)
		INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);

	INIT_LIST_HEAD(&my_cq->sqp_err_list);
	INIT_LIST_HEAD(&my_cq->rqp_err_list);

	if (context) {
		struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
		struct ehca_create_cq_resp resp;
		memset(&resp, 0, sizeof(resp));
		resp.cq_number = my_cq->cq_number;
		resp.token = my_cq->token;
		resp.ipz_queue.qe_size = ipz_queue->qe_size;
		resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
		resp.ipz_queue.queue_length = ipz_queue->queue_length;
		resp.ipz_queue.pagesize = ipz_queue->pagesize;
		resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
		resp.fw_handle_ofs = (u32)
			(my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
			ehca_err(device, "Copy to udata failed.");
			goto create_cq_exit4;
		}
	}

	return cq;

create_cq_exit4:
	ipz_queue_dtor(NULL, &my_cq->ipz_queue);

create_cq_exit3:
	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	if (h_ret != H_SUCCESS)
		ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
			 "cq_num=%x h_ret=%li", my_cq, my_cq->cq_number, h_ret);

create_cq_exit2:
	write_lock_irqsave(&ehca_cq_idr_lock, flags);
	idr_remove(&ehca_cq_idr, my_cq->token);
	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);

create_cq_exit1:
	kmem_cache_free(cq_cache, my_cq);

	atomic_dec(&shca->num_cqs);
	return cq;
}

int ehca_destroy_cq(struct ib_cq *cq)
{
	u64 h_ret;
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	int cq_num = my_cq->cq_number;
	struct ib_device *device = cq->device;
	struct ehca_shca *shca = container_of(device, struct ehca_shca,
					      ib_device);
	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
	unsigned long flags;

	if (cq->uobject) {
		if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
			ehca_err(device, "Resources still referenced in "
				 "user space cq_num=%x", my_cq->cq_number);
			return -EINVAL;
		}
	}

	/*
	 * remove the CQ from the idr first to make sure
	 * no more interrupt tasklets will touch this CQ
	 */
	write_lock_irqsave(&ehca_cq_idr_lock, flags);
	idr_remove(&ehca_cq_idr, my_cq->token);
	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);

	/* now wait until all pending events have completed */
	wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));

	/* nobody's using our CQ any longer -- we can destroy it */
	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
	if (h_ret == H_R_STATE) {
		/* cq in err: read err data and destroy it forcibly */
		ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err "
			 "state. Try to delete it forcibly.",
			 my_cq, cq_num, my_cq->ipz_cq_handle.handle);
		ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
		h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
		if (h_ret == H_SUCCESS)
			ehca_dbg(device, "cq_num=%x deleted successfully.",
				 cq_num);
	}
	if (h_ret != H_SUCCESS) {
		ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%li "
			 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
		return ehca2ib_return_code(h_ret);
	}
	ipz_queue_dtor(NULL, &my_cq->ipz_queue);
	kmem_cache_free(cq_cache, my_cq);

	atomic_dec(&shca->num_cqs);
	return 0;
}

int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
{
	/* TODO: proper resize needs to be done */
	ehca_err(cq->device, "not implemented yet");

	return -EFAULT;
}

int ehca_init_cq_cache(void)
{
	cq_cache = kmem_cache_create("ehca_cache_cq",
				     sizeof(struct ehca_cq), 0,
				     SLAB_HWCACHE_ALIGN,
				     NULL);
	if (!cq_cache)
		return -ENOMEM;
	return 0;
}

void ehca_cleanup_cq_cache(void)
{
	if (cq_cache)
		kmem_cache_destroy(cq_cache);
}
Commit	Line	Data
fab97220 HS	1	/*
	2	* IBM eServer eHCA Infiniband device driver for Linux on POWER
	3	*
	4	* Completion queue handling
	5	*
	6	* Authors: Waleri Fomin <fomin@de.ibm.com>
	7	* Khadija Souissi <souissi@de.ibm.com>
	8	* Reinhard Ernst <rernst@de.ibm.com>
	9	* Heiko J Schick <schickhj@de.ibm.com>
	10	* Hoang-Nam Nguyen <hnguyen@de.ibm.com>
	11	*
	12	*
	13	* Copyright (c) 2005 IBM Corporation
	14	*
	15	* All rights reserved.
	16	*
	17	* This source code is distributed under a dual license of GPL v2.0 and OpenIB
	18	* BSD.
	19	*
	20	* OpenIB BSD License
	21	*
	22	* Redistribution and use in source and binary forms, with or without
	23	* modification, are permitted provided that the following conditions are met:
	24	*
	25	* Redistributions of source code must retain the above copyright notice, this
	26	* list of conditions and the following disclaimer.
	27	*
	28	* Redistributions in binary form must reproduce the above copyright notice,
	29	* this list of conditions and the following disclaimer in the documentation
	30	* and/or other materials
	31	* provided with the distribution.
	32	*
	33	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	34	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	35	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	36	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
	37	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	38	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	39	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
	40	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
	41	* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	42	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	43	* POSSIBILITY OF SUCH DAMAGE.
	44	*/
	45
fab97220 HS	46	#include "ehca_iverbs.h"
	47	#include "ehca_classes.h"
	48	#include "ehca_irq.h"
	49	#include "hcp_if.h"
	50
	51	static struct kmem_cache *cq_cache;
	52
	53	int ehca_cq_assign_qp(struct ehca_cq cq, struct ehca_qp qp)
	54	{
	55	unsigned int qp_num = qp->real_qp_num;
	56	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
9844b71b	57	unsigned long flags;
fab97220	58
9844b71b	59	spin_lock_irqsave(&cq->spinlock, flags);
fab97220	60	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
9844b71b	61	spin_unlock_irqrestore(&cq->spinlock, flags);
fab97220 HS	62
	63	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
	64	cq->cq_number, qp_num);
	65
	66	return 0;
	67	}
	68
	69	int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
	70	{
	71	int ret = -EINVAL;
	72	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	73	struct hlist_node *iter;
	74	struct ehca_qp *qp;
9844b71b	75	unsigned long flags;
fab97220	76
9844b71b	77	spin_lock_irqsave(&cq->spinlock, flags);
fab97220 HS	78	hlist_for_each(iter, &cq->qp_hashtab[key]) {
	79	qp = hlist_entry(iter, struct ehca_qp, list_entries);
	80	if (qp->real_qp_num == real_qp_num) {
	81	hlist_del(iter);
	82	ehca_dbg(cq->ib_cq.device,
	83	"removed qp from cq .cq_num=%x real_qp_num=%x",
	84	cq->cq_number, real_qp_num);
	85	ret = 0;
	86	break;
	87	}
	88	}
9844b71b	89	spin_unlock_irqrestore(&cq->spinlock, flags);
fab97220 HS	90	if (ret)
	91	ehca_err(cq->ib_cq.device,
	92	"qp not found cq_num=%x real_qp_num=%x",
	93	cq->cq_number, real_qp_num);
	94
	95	return ret;
	96	}
	97
2b94397a	98	struct ehca_qp ehca_cq_get_qp(struct ehca_cq cq, int real_qp_num)
fab97220 HS	99	{
	100	struct ehca_qp *ret = NULL;
	101	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	102	struct hlist_node *iter;
	103	struct ehca_qp *qp;
	104	hlist_for_each(iter, &cq->qp_hashtab[key]) {
	105	qp = hlist_entry(iter, struct ehca_qp, list_entries);
	106	if (qp->real_qp_num == real_qp_num) {
	107	ret = qp;
	108	break;
	109	}
	110	}
	111	return ret;
	112	}
	113
f4fd0b22	114	struct ib_cq ehca_create_cq(struct ib_device device, int cqe, int comp_vector,
fab97220 HS	115	struct ib_ucontext *context,
	116	struct ib_udata *udata)
	117	{
	118	static const u32 additional_cqe = 20;
	119	struct ib_cq *cq;
	120	struct ehca_cq *my_cq;
	121	struct ehca_shca *shca =
	122	container_of(device, struct ehca_shca, ib_device);
	123	struct ipz_adapter_handle adapter_handle;
	124	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
	125	struct h_galpa gal;
	126	void *vpage;
	127	u32 counter;
	128	u64 rpage, cqx_fec, h_ret;
	129	int ipz_rc, ret, i;
	130	unsigned long flags;
	131
	132	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
	133	return ERR_PTR(-EINVAL);
	134
19f42821	135	if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
d227fa72	136	ehca_err(device, "Unable to create CQ, max number of %i "
19f42821	137	"CQs reached.", shca->max_num_cqs);
d227fa72 SR	138	ehca_err(device, "To increase the maximum number of CQs "
	139	"use the number_of_cqs module parameter.\n");
	140	return ERR_PTR(-ENOSPC);
	141	}
	142
c3762229	143	my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
fab97220 HS	144	if (!my_cq) {
	145	ehca_err(device, "Out of memory for ehca_cq struct device=%p",
	146	device);
d227fa72	147	atomic_dec(&shca->num_cqs);
fab97220 HS	148	return ERR_PTR(-ENOMEM);
	149	}
	150
fab97220 HS	151	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));
	152
	153	spin_lock_init(&my_cq->spinlock);
	154	spin_lock_init(&my_cq->cb_lock);
	155	spin_lock_init(&my_cq->task_lock);
28db6beb	156	atomic_set(&my_cq->nr_events, 0);
31726798	157	init_waitqueue_head(&my_cq->wait_completion);
fab97220 HS	158
	159	cq = &my_cq->ib_cq;
	160
	161	adapter_handle = shca->ipz_hca_handle;
	162	param.eq_handle = shca->eq.ipz_eq_handle;
	163
	164	do {
	165	if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) {
	166	cq = ERR_PTR(-ENOMEM);
	167	ehca_err(device, "Can't reserve idr nr. device=%p",
	168	device);
	169	goto create_cq_exit1;
	170	}
	171
26ed687f	172	write_lock_irqsave(&ehca_cq_idr_lock, flags);
fab97220	173	ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token);
26ed687f	174	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
fab97220 HS	175	} while (ret == -EAGAIN);
	176
	177	if (ret) {
	178	cq = ERR_PTR(-ENOMEM);
	179	ehca_err(device, "Can't allocate new idr entry. device=%p",
	180	device);
	181	goto create_cq_exit1;
	182	}
	183
5281a4b8 SR	184	if (my_cq->token > 0x1FFFFFF) {
	185	cq = ERR_PTR(-ENOMEM);
	186	ehca_err(device, "Invalid number of cq. device=%p", device);
	187	goto create_cq_exit2;
	188	}
	189
fab97220 HS	190	/*
	191	* CQs maximum depth is 4GB-64, but we need additional 20 as buffer
	192	* for receiving errors CQEs.
	193	*/
	194	param.nr_cqe = cqe + additional_cqe;
	195	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);
	196
	197	if (h_ret != H_SUCCESS) {
	198	ehca_err(device, "hipz_h_alloc_resource_cq() failed "
e3722192	199	"h_ret=%li device=%p", h_ret, device);
fab97220 HS	200	cq = ERR_PTR(ehca2ib_return_code(h_ret));
	201	goto create_cq_exit2;
	202	}
	203
e2f81daf SR	204	ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
e2f81daf SR	205	EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
fab97220	206	if (!ipz_rc) {
e3722192	207	ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
fab97220 HS	208	ipz_rc, device);
	209	cq = ERR_PTR(-EINVAL);
	210	goto create_cq_exit3;
	211	}
	212
	213	for (counter = 0; counter < param.act_pages; counter++) {
	214	vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
	215	if (!vpage) {
	216	ehca_err(device, "ipz_qpageit_get_inc() "
	217	"returns NULL device=%p", device);
	218	cq = ERR_PTR(-EAGAIN);
	219	goto create_cq_exit4;
	220	}
	221	rpage = virt_to_abs(vpage);
	222
	223	h_ret = hipz_h_register_rpage_cq(adapter_handle,
	224	my_cq->ipz_cq_handle,
	225	&my_cq->pf,
	226	0,
	227	0,
	228	rpage,
	229	1,
	230	my_cq->galpas.
	231	kernel);
	232
	233	if (h_ret < H_SUCCESS) {
	234	ehca_err(device, "hipz_h_register_rpage_cq() failed "
e3722192	235	"ehca_cq=%p cq_num=%x h_ret=%li counter=%i "
fab97220 HS	236	"act_pages=%i", my_cq, my_cq->cq_number,
	237	h_ret, counter, param.act_pages);
	238	cq = ERR_PTR(-EINVAL);
	239	goto create_cq_exit4;
	240	}
	241
	242	if (counter == (param.act_pages - 1)) {
	243	vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
	244	if ((h_ret != H_SUCCESS) \|\| vpage) {
	245	ehca_err(device, "Registration of pages not "
	246	"complete ehca_cq=%p cq_num=%x "
e3722192	247	"h_ret=%li", my_cq, my_cq->cq_number,
fab97220 HS	248	h_ret);
	249	cq = ERR_PTR(-EAGAIN);
	250	goto create_cq_exit4;
	251	}
	252	} else {
	253	if (h_ret != H_PAGE_REGISTERED) {
	254	ehca_err(device, "Registration of page failed "
908cf9a5	255	"ehca_cq=%p cq_num=%x h_ret=%li "
fab97220 HS	256	"counter=%i act_pages=%i",
	257	my_cq, my_cq->cq_number,
	258	h_ret, counter, param.act_pages);
	259	cq = ERR_PTR(-ENOMEM);
	260	goto create_cq_exit4;
	261	}
	262	}
	263	}
	264
	265	ipz_qeit_reset(&my_cq->ipz_queue);
	266
	267	gal = my_cq->galpas.kernel;
	268	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
	269	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx",
	270	my_cq, my_cq->cq_number, cqx_fec);
	271
	272	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
	273	param.act_nr_of_entries - additional_cqe;
	274	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
	275
	276	for (i = 0; i < QP_HASHTAB_LEN; i++)
	277	INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
	278
b9012e0a AS	279	INIT_LIST_HEAD(&my_cq->sqp_err_list);
	280	INIT_LIST_HEAD(&my_cq->rqp_err_list);
	281
fab97220 HS	282	if (context) {
	283	struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
	284	struct ehca_create_cq_resp resp;
fab97220 HS	285	memset(&resp, 0, sizeof(resp));
	286	resp.cq_number = my_cq->cq_number;
	287	resp.token = my_cq->token;
	288	resp.ipz_queue.qe_size = ipz_queue->qe_size;
	289	resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
	290	resp.ipz_queue.queue_length = ipz_queue->queue_length;
	291	resp.ipz_queue.pagesize = ipz_queue->pagesize;
	292	resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
e390d3b5 HNN	293	resp.fw_handle_ofs = (u32)
e390d3b5 HNN	294	(my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
fab97220 HS	295	if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
fab97220 HS	296	ehca_err(device, "Copy to udata failed.");
4c34bdf5	297	goto create_cq_exit4;
fab97220 HS	298	}
	299	}
	300
	301	return cq;
	302
fab97220	303	create_cq_exit4:
e2f81daf	304	ipz_queue_dtor(NULL, &my_cq->ipz_queue);
fab97220 HS	305
	306	create_cq_exit3:
	307	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	308	if (h_ret != H_SUCCESS)
	309	ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
e3722192	310	"cq_num=%x h_ret=%li", my_cq, my_cq->cq_number, h_ret);
fab97220 HS	311
fab97220 HS	312	create_cq_exit2:
26ed687f	313	write_lock_irqsave(&ehca_cq_idr_lock, flags);
fab97220	314	idr_remove(&ehca_cq_idr, my_cq->token);
26ed687f	315	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
fab97220 HS	316
	317	create_cq_exit1:
	318	kmem_cache_free(cq_cache, my_cq);
	319
d227fa72	320	atomic_dec(&shca->num_cqs);
fab97220 HS	321	return cq;
	322	}
	323
	324	int ehca_destroy_cq(struct ib_cq *cq)
	325	{
	326	u64 h_ret;
fab97220 HS	327	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	328	int cq_num = my_cq->cq_number;
	329	struct ib_device *device = cq->device;
	330	struct ehca_shca *shca = container_of(device, struct ehca_shca,
	331	ib_device);
	332	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
fab97220 HS	333	unsigned long flags;
fab97220 HS	334
4c34bdf5 HNN	335	if (cq->uobject) {
	336	if (my_cq->mm_count_galpa \|\| my_cq->mm_count_queue) {
	337	ehca_err(device, "Resources still referenced in "
	338	"user space cq_num=%x", my_cq->cq_number);
	339	return -EINVAL;
	340	}
4c34bdf5 HNN	341	}
4c34bdf5 HNN	342
28db6beb JF	343	/*
	344	* remove the CQ from the idr first to make sure
	345	* no more interrupt tasklets will touch this CQ
	346	*/
26ed687f	347	write_lock_irqsave(&ehca_cq_idr_lock, flags);
fab97220	348	idr_remove(&ehca_cq_idr, my_cq->token);
26ed687f	349	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
fab97220	350
28db6beb JF	351	/* now wait until all pending events have completed */
	352	wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
	353
	354	/* nobody's using our CQ any longer -- we can destroy it */
fab97220 HS	355	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
	356	if (h_ret == H_R_STATE) {
	357	/* cq in err: read err data and destroy it forcibly */
	358	ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err "
	359	"state. Try to delete it forcibly.",
	360	my_cq, cq_num, my_cq->ipz_cq_handle.handle);
	361	ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
	362	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	363	if (h_ret == H_SUCCESS)
	364	ehca_dbg(device, "cq_num=%x deleted successfully.",
	365	cq_num);
	366	}
	367	if (h_ret != H_SUCCESS) {
e3722192	368	ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%li "
fab97220 HS	369	"ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
	370	return ehca2ib_return_code(h_ret);
	371	}
e2f81daf	372	ipz_queue_dtor(NULL, &my_cq->ipz_queue);
fab97220 HS	373	kmem_cache_free(cq_cache, my_cq);
fab97220 HS	374
d227fa72	375	atomic_dec(&shca->num_cqs);
fab97220 HS	376	return 0;
	377	}
	378
	379	int ehca_resize_cq(struct ib_cq cq, int cqe, struct ib_udata udata)
	380	{
fab97220 HS	381	/* TODO: proper resize needs to be done */
	382	ehca_err(cq->device, "not implemented yet");
	383
	384	return -EFAULT;
	385	}
	386
	387	int ehca_init_cq_cache(void)
	388	{
	389	cq_cache = kmem_cache_create("ehca_cache_cq",
	390	sizeof(struct ehca_cq), 0,
	391	SLAB_HWCACHE_ALIGN,
20c2df83	392	NULL);
fab97220 HS	393	if (!cq_cache)
	394	return -ENOMEM;
	395	return 0;
	396	}
	397
	398	void ehca_cleanup_cq_cache(void)
	399	{
	400	if (cq_cache)
	401	kmem_cache_destroy(cq_cache);
	402	}