2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4 * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved.
5 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
6 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 * $Id: mthca_cq.c 1369 2004-12-20 16:17:07Z roland $
39 #include <linux/hardirq.h>
43 #include <rdma/ib_pack.h>
45 #include "mthca_dev.h"
46 #include "mthca_cmd.h"
47 #include "mthca_memfree.h"
50 MTHCA_MAX_DIRECT_CQ_SIZE
= 4 * PAGE_SIZE
54 MTHCA_CQ_ENTRY_SIZE
= 0x20
58 MTHCA_ATOMIC_BYTE_LEN
= 8
62 * Must be packed because start is 64 bits but only aligned to 32 bits.
64 struct mthca_cq_context
{
67 __be32 logsize_usrpage
;
68 __be32 error_eqn
; /* Tavor only */
72 __be32 last_notified_index
;
73 __be32 solicit_producer_index
;
74 __be32 consumer_index
;
75 __be32 producer_index
;
77 __be32 ci_db
; /* Arbel only */
78 __be32 state_db
; /* Arbel only */
80 } __attribute__((packed
));
82 #define MTHCA_CQ_STATUS_OK ( 0 << 28)
83 #define MTHCA_CQ_STATUS_OVERFLOW ( 9 << 28)
84 #define MTHCA_CQ_STATUS_WRITE_FAIL (10 << 28)
85 #define MTHCA_CQ_FLAG_TR ( 1 << 18)
86 #define MTHCA_CQ_FLAG_OI ( 1 << 17)
87 #define MTHCA_CQ_STATE_DISARMED ( 0 << 8)
88 #define MTHCA_CQ_STATE_ARMED ( 1 << 8)
89 #define MTHCA_CQ_STATE_ARMED_SOL ( 4 << 8)
90 #define MTHCA_EQ_STATE_FIRED (10 << 8)
93 MTHCA_ERROR_CQE_OPCODE_MASK
= 0xfe
97 SYNDROME_LOCAL_LENGTH_ERR
= 0x01,
98 SYNDROME_LOCAL_QP_OP_ERR
= 0x02,
99 SYNDROME_LOCAL_EEC_OP_ERR
= 0x03,
100 SYNDROME_LOCAL_PROT_ERR
= 0x04,
101 SYNDROME_WR_FLUSH_ERR
= 0x05,
102 SYNDROME_MW_BIND_ERR
= 0x06,
103 SYNDROME_BAD_RESP_ERR
= 0x10,
104 SYNDROME_LOCAL_ACCESS_ERR
= 0x11,
105 SYNDROME_REMOTE_INVAL_REQ_ERR
= 0x12,
106 SYNDROME_REMOTE_ACCESS_ERR
= 0x13,
107 SYNDROME_REMOTE_OP_ERR
= 0x14,
108 SYNDROME_RETRY_EXC_ERR
= 0x15,
109 SYNDROME_RNR_RETRY_EXC_ERR
= 0x16,
110 SYNDROME_LOCAL_RDD_VIOL_ERR
= 0x20,
111 SYNDROME_REMOTE_INVAL_RD_REQ_ERR
= 0x21,
112 SYNDROME_REMOTE_ABORTED_ERR
= 0x22,
113 SYNDROME_INVAL_EECN_ERR
= 0x23,
114 SYNDROME_INVAL_EEC_STATE_ERR
= 0x24
123 __be32 imm_etype_pkey_eec
;
132 struct mthca_err_cqe
{
145 #define MTHCA_CQ_ENTRY_OWNER_SW (0 << 7)
146 #define MTHCA_CQ_ENTRY_OWNER_HW (1 << 7)
148 #define MTHCA_TAVOR_CQ_DB_INC_CI (1 << 24)
149 #define MTHCA_TAVOR_CQ_DB_REQ_NOT (2 << 24)
150 #define MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL (3 << 24)
151 #define MTHCA_TAVOR_CQ_DB_SET_CI (4 << 24)
152 #define MTHCA_TAVOR_CQ_DB_REQ_NOT_MULT (5 << 24)
154 #define MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL (1 << 24)
155 #define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24)
156 #define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24)
158 static inline struct mthca_cqe
*get_cqe_from_buf(struct mthca_cq_buf
*buf
,
162 return buf
->queue
.direct
.buf
+ (entry
* MTHCA_CQ_ENTRY_SIZE
);
164 return buf
->queue
.page_list
[entry
* MTHCA_CQ_ENTRY_SIZE
/ PAGE_SIZE
].buf
165 + (entry
* MTHCA_CQ_ENTRY_SIZE
) % PAGE_SIZE
;
168 static inline struct mthca_cqe
*get_cqe(struct mthca_cq
*cq
, int entry
)
170 return get_cqe_from_buf(&cq
->buf
, entry
);
173 static inline struct mthca_cqe
*cqe_sw(struct mthca_cqe
*cqe
)
175 return MTHCA_CQ_ENTRY_OWNER_HW
& cqe
->owner
? NULL
: cqe
;
178 static inline struct mthca_cqe
*next_cqe_sw(struct mthca_cq
*cq
)
180 return cqe_sw(get_cqe(cq
, cq
->cons_index
& cq
->ibcq
.cqe
));
183 static inline void set_cqe_hw(struct mthca_cqe
*cqe
)
185 cqe
->owner
= MTHCA_CQ_ENTRY_OWNER_HW
;
188 static void dump_cqe(struct mthca_dev
*dev
, void *cqe_ptr
)
190 __be32
*cqe
= cqe_ptr
;
192 (void) cqe
; /* avoid warning if mthca_dbg compiled away... */
193 mthca_dbg(dev
, "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
194 be32_to_cpu(cqe
[0]), be32_to_cpu(cqe
[1]), be32_to_cpu(cqe
[2]),
195 be32_to_cpu(cqe
[3]), be32_to_cpu(cqe
[4]), be32_to_cpu(cqe
[5]),
196 be32_to_cpu(cqe
[6]), be32_to_cpu(cqe
[7]));
200 * incr is ignored in native Arbel (mem-free) mode, so cq->cons_index
201 * should be correct before calling update_cons_index().
203 static inline void update_cons_index(struct mthca_dev
*dev
, struct mthca_cq
*cq
,
208 if (mthca_is_memfree(dev
)) {
209 *cq
->set_ci_db
= cpu_to_be32(cq
->cons_index
);
212 doorbell
[0] = cpu_to_be32(MTHCA_TAVOR_CQ_DB_INC_CI
| cq
->cqn
);
213 doorbell
[1] = cpu_to_be32(incr
- 1);
215 mthca_write64(doorbell
,
216 dev
->kar
+ MTHCA_CQ_DOORBELL
,
217 MTHCA_GET_DOORBELL_LOCK(&dev
->doorbell_lock
));
219 * Make sure doorbells don't leak out of CQ spinlock
220 * and reach the HCA out of order:
226 void mthca_cq_completion(struct mthca_dev
*dev
, u32 cqn
)
230 cq
= mthca_array_get(&dev
->cq_table
.cq
, cqn
& (dev
->limits
.num_cqs
- 1));
233 mthca_warn(dev
, "Completion event for bogus CQ %08x\n", cqn
);
239 cq
->ibcq
.comp_handler(&cq
->ibcq
, cq
->ibcq
.cq_context
);
242 void mthca_cq_event(struct mthca_dev
*dev
, u32 cqn
,
243 enum ib_event_type event_type
)
246 struct ib_event event
;
248 spin_lock(&dev
->cq_table
.lock
);
250 cq
= mthca_array_get(&dev
->cq_table
.cq
, cqn
& (dev
->limits
.num_cqs
- 1));
254 spin_unlock(&dev
->cq_table
.lock
);
257 mthca_warn(dev
, "Async event for bogus CQ %08x\n", cqn
);
261 event
.device
= &dev
->ib_dev
;
262 event
.event
= event_type
;
263 event
.element
.cq
= &cq
->ibcq
;
264 if (cq
->ibcq
.event_handler
)
265 cq
->ibcq
.event_handler(&event
, cq
->ibcq
.cq_context
);
267 spin_lock(&dev
->cq_table
.lock
);
270 spin_unlock(&dev
->cq_table
.lock
);
273 static inline int is_recv_cqe(struct mthca_cqe
*cqe
)
275 if ((cqe
->opcode
& MTHCA_ERROR_CQE_OPCODE_MASK
) ==
276 MTHCA_ERROR_CQE_OPCODE_MASK
)
277 return !(cqe
->opcode
& 0x01);
279 return !(cqe
->is_send
& 0x80);
282 void mthca_cq_clean(struct mthca_dev
*dev
, struct mthca_cq
*cq
, u32 qpn
,
283 struct mthca_srq
*srq
)
285 struct mthca_cqe
*cqe
;
289 spin_lock_irq(&cq
->lock
);
292 * First we need to find the current producer index, so we
293 * know where to start cleaning from. It doesn't matter if HW
294 * adds new entries after this loop -- the QP we're worried
295 * about is already in RESET, so the new entries won't come
296 * from our QP and therefore don't need to be checked.
298 for (prod_index
= cq
->cons_index
;
299 cqe_sw(get_cqe(cq
, prod_index
& cq
->ibcq
.cqe
));
301 if (prod_index
== cq
->cons_index
+ cq
->ibcq
.cqe
)
305 mthca_dbg(dev
, "Cleaning QPN %06x from CQN %06x; ci %d, pi %d\n",
306 qpn
, cq
->cqn
, cq
->cons_index
, prod_index
);
309 * Now sweep backwards through the CQ, removing CQ entries
310 * that match our QP by copying older entries on top of them.
312 while ((int) --prod_index
- (int) cq
->cons_index
>= 0) {
313 cqe
= get_cqe(cq
, prod_index
& cq
->ibcq
.cqe
);
314 if (cqe
->my_qpn
== cpu_to_be32(qpn
)) {
315 if (srq
&& is_recv_cqe(cqe
))
316 mthca_free_srq_wqe(srq
, be32_to_cpu(cqe
->wqe
));
319 memcpy(get_cqe(cq
, (prod_index
+ nfreed
) & cq
->ibcq
.cqe
),
320 cqe
, MTHCA_CQ_ENTRY_SIZE
);
324 for (i
= 0; i
< nfreed
; ++i
)
325 set_cqe_hw(get_cqe(cq
, (cq
->cons_index
+ i
) & cq
->ibcq
.cqe
));
327 cq
->cons_index
+= nfreed
;
328 update_cons_index(dev
, cq
, nfreed
);
331 spin_unlock_irq(&cq
->lock
);
334 void mthca_cq_resize_copy_cqes(struct mthca_cq
*cq
)
339 * In Tavor mode, the hardware keeps the consumer and producer
340 * indices mod the CQ size. Since we might be making the CQ
341 * bigger, we need to deal with the case where the producer
342 * index wrapped around before the CQ was resized.
344 if (!mthca_is_memfree(to_mdev(cq
->ibcq
.device
)) &&
345 cq
->ibcq
.cqe
< cq
->resize_buf
->cqe
) {
346 cq
->cons_index
&= cq
->ibcq
.cqe
;
347 if (cqe_sw(get_cqe(cq
, cq
->ibcq
.cqe
)))
348 cq
->cons_index
-= cq
->ibcq
.cqe
+ 1;
351 for (i
= cq
->cons_index
; cqe_sw(get_cqe(cq
, i
& cq
->ibcq
.cqe
)); ++i
)
352 memcpy(get_cqe_from_buf(&cq
->resize_buf
->buf
,
353 i
& cq
->resize_buf
->cqe
),
354 get_cqe(cq
, i
& cq
->ibcq
.cqe
), MTHCA_CQ_ENTRY_SIZE
);
357 int mthca_alloc_cq_buf(struct mthca_dev
*dev
, struct mthca_cq_buf
*buf
, int nent
)
362 ret
= mthca_buf_alloc(dev
, nent
* MTHCA_CQ_ENTRY_SIZE
,
363 MTHCA_MAX_DIRECT_CQ_SIZE
,
364 &buf
->queue
, &buf
->is_direct
,
365 &dev
->driver_pd
, 1, &buf
->mr
);
369 for (i
= 0; i
< nent
; ++i
)
370 set_cqe_hw(get_cqe_from_buf(buf
, i
));
375 void mthca_free_cq_buf(struct mthca_dev
*dev
, struct mthca_cq_buf
*buf
, int cqe
)
377 mthca_buf_free(dev
, (cqe
+ 1) * MTHCA_CQ_ENTRY_SIZE
, &buf
->queue
,
378 buf
->is_direct
, &buf
->mr
);
381 static void handle_error_cqe(struct mthca_dev
*dev
, struct mthca_cq
*cq
,
382 struct mthca_qp
*qp
, int wqe_index
, int is_send
,
383 struct mthca_err_cqe
*cqe
,
384 struct ib_wc
*entry
, int *free_cqe
)
389 if (cqe
->syndrome
== SYNDROME_LOCAL_QP_OP_ERR
) {
390 mthca_dbg(dev
, "local QP operation err "
391 "(QPN %06x, WQE @ %08x, CQN %06x, index %d)\n",
392 be32_to_cpu(cqe
->my_qpn
), be32_to_cpu(cqe
->wqe
),
393 cq
->cqn
, cq
->cons_index
);
398 * For completions in error, only work request ID, status, vendor error
399 * (and freed resource count for RD) have to be set.
401 switch (cqe
->syndrome
) {
402 case SYNDROME_LOCAL_LENGTH_ERR
:
403 entry
->status
= IB_WC_LOC_LEN_ERR
;
405 case SYNDROME_LOCAL_QP_OP_ERR
:
406 entry
->status
= IB_WC_LOC_QP_OP_ERR
;
408 case SYNDROME_LOCAL_EEC_OP_ERR
:
409 entry
->status
= IB_WC_LOC_EEC_OP_ERR
;
411 case SYNDROME_LOCAL_PROT_ERR
:
412 entry
->status
= IB_WC_LOC_PROT_ERR
;
414 case SYNDROME_WR_FLUSH_ERR
:
415 entry
->status
= IB_WC_WR_FLUSH_ERR
;
417 case SYNDROME_MW_BIND_ERR
:
418 entry
->status
= IB_WC_MW_BIND_ERR
;
420 case SYNDROME_BAD_RESP_ERR
:
421 entry
->status
= IB_WC_BAD_RESP_ERR
;
423 case SYNDROME_LOCAL_ACCESS_ERR
:
424 entry
->status
= IB_WC_LOC_ACCESS_ERR
;
426 case SYNDROME_REMOTE_INVAL_REQ_ERR
:
427 entry
->status
= IB_WC_REM_INV_REQ_ERR
;
429 case SYNDROME_REMOTE_ACCESS_ERR
:
430 entry
->status
= IB_WC_REM_ACCESS_ERR
;
432 case SYNDROME_REMOTE_OP_ERR
:
433 entry
->status
= IB_WC_REM_OP_ERR
;
435 case SYNDROME_RETRY_EXC_ERR
:
436 entry
->status
= IB_WC_RETRY_EXC_ERR
;
438 case SYNDROME_RNR_RETRY_EXC_ERR
:
439 entry
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
441 case SYNDROME_LOCAL_RDD_VIOL_ERR
:
442 entry
->status
= IB_WC_LOC_RDD_VIOL_ERR
;
444 case SYNDROME_REMOTE_INVAL_RD_REQ_ERR
:
445 entry
->status
= IB_WC_REM_INV_RD_REQ_ERR
;
447 case SYNDROME_REMOTE_ABORTED_ERR
:
448 entry
->status
= IB_WC_REM_ABORT_ERR
;
450 case SYNDROME_INVAL_EECN_ERR
:
451 entry
->status
= IB_WC_INV_EECN_ERR
;
453 case SYNDROME_INVAL_EEC_STATE_ERR
:
454 entry
->status
= IB_WC_INV_EEC_STATE_ERR
;
457 entry
->status
= IB_WC_GENERAL_ERR
;
461 entry
->vendor_err
= cqe
->vendor_err
;
464 * Mem-free HCAs always generate one CQE per WQE, even in the
465 * error case, so we don't have to check the doorbell count, etc.
467 if (mthca_is_memfree(dev
))
470 mthca_free_err_wqe(dev
, qp
, is_send
, wqe_index
, &dbd
, &new_wqe
);
473 * If we're at the end of the WQE chain, or we've used up our
474 * doorbell count, free the CQE. Otherwise just update it for
475 * the next poll operation.
477 if (!(new_wqe
& cpu_to_be32(0x3f)) || (!cqe
->db_cnt
&& dbd
))
480 cqe
->db_cnt
= cpu_to_be16(be16_to_cpu(cqe
->db_cnt
) - dbd
);
482 cqe
->syndrome
= SYNDROME_WR_FLUSH_ERR
;
487 static inline int mthca_poll_one(struct mthca_dev
*dev
,
489 struct mthca_qp
**cur_qp
,
494 struct mthca_cqe
*cqe
;
501 cqe
= next_cqe_sw(cq
);
506 * Make sure we read CQ entry contents after we've checked the
512 mthca_dbg(dev
, "%x/%d: CQE -> QPN %06x, WQE @ %08x\n",
513 cq
->cqn
, cq
->cons_index
, be32_to_cpu(cqe
->my_qpn
),
514 be32_to_cpu(cqe
->wqe
));
518 is_error
= (cqe
->opcode
& MTHCA_ERROR_CQE_OPCODE_MASK
) ==
519 MTHCA_ERROR_CQE_OPCODE_MASK
;
520 is_send
= is_error
? cqe
->opcode
& 0x01 : cqe
->is_send
& 0x80;
522 if (!*cur_qp
|| be32_to_cpu(cqe
->my_qpn
) != (*cur_qp
)->qpn
) {
524 * We do not have to take the QP table lock here,
525 * because CQs will be locked while QPs are removed
528 *cur_qp
= mthca_array_get(&dev
->qp_table
.qp
,
529 be32_to_cpu(cqe
->my_qpn
) &
530 (dev
->limits
.num_qps
- 1));
532 mthca_warn(dev
, "CQ entry for unknown QP %06x\n",
533 be32_to_cpu(cqe
->my_qpn
) & 0xffffff);
539 entry
->qp
= &(*cur_qp
)->ibqp
;
543 wqe_index
= ((be32_to_cpu(cqe
->wqe
) - (*cur_qp
)->send_wqe_offset
)
545 entry
->wr_id
= (*cur_qp
)->wrid
[wqe_index
+
547 } else if ((*cur_qp
)->ibqp
.srq
) {
548 struct mthca_srq
*srq
= to_msrq((*cur_qp
)->ibqp
.srq
);
549 u32 wqe
= be32_to_cpu(cqe
->wqe
);
551 wqe_index
= wqe
>> srq
->wqe_shift
;
552 entry
->wr_id
= srq
->wrid
[wqe_index
];
553 mthca_free_srq_wqe(srq
, wqe
);
557 wqe
= be32_to_cpu(cqe
->wqe
);
558 wqe_index
= wqe
>> wq
->wqe_shift
;
560 * WQE addr == base - 1 might be reported in receive completion
561 * with error instead of (rq size - 1) by Sinai FW 1.0.800 and
562 * Arbel FW 5.1.400. This bug should be fixed in later FW revs.
564 if (unlikely(wqe_index
< 0))
565 wqe_index
= wq
->max
- 1;
566 entry
->wr_id
= (*cur_qp
)->wrid
[wqe_index
];
570 if (wq
->last_comp
< wqe_index
)
571 wq
->tail
+= wqe_index
- wq
->last_comp
;
573 wq
->tail
+= wqe_index
+ wq
->max
- wq
->last_comp
;
575 wq
->last_comp
= wqe_index
;
579 handle_error_cqe(dev
, cq
, *cur_qp
, wqe_index
, is_send
,
580 (struct mthca_err_cqe
*) cqe
,
587 switch (cqe
->opcode
) {
588 case MTHCA_OPCODE_RDMA_WRITE
:
589 entry
->opcode
= IB_WC_RDMA_WRITE
;
591 case MTHCA_OPCODE_RDMA_WRITE_IMM
:
592 entry
->opcode
= IB_WC_RDMA_WRITE
;
593 entry
->wc_flags
|= IB_WC_WITH_IMM
;
595 case MTHCA_OPCODE_SEND
:
596 entry
->opcode
= IB_WC_SEND
;
598 case MTHCA_OPCODE_SEND_IMM
:
599 entry
->opcode
= IB_WC_SEND
;
600 entry
->wc_flags
|= IB_WC_WITH_IMM
;
602 case MTHCA_OPCODE_RDMA_READ
:
603 entry
->opcode
= IB_WC_RDMA_READ
;
604 entry
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
606 case MTHCA_OPCODE_ATOMIC_CS
:
607 entry
->opcode
= IB_WC_COMP_SWAP
;
608 entry
->byte_len
= MTHCA_ATOMIC_BYTE_LEN
;
610 case MTHCA_OPCODE_ATOMIC_FA
:
611 entry
->opcode
= IB_WC_FETCH_ADD
;
612 entry
->byte_len
= MTHCA_ATOMIC_BYTE_LEN
;
614 case MTHCA_OPCODE_BIND_MW
:
615 entry
->opcode
= IB_WC_BIND_MW
;
618 entry
->opcode
= MTHCA_OPCODE_INVALID
;
622 entry
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
623 switch (cqe
->opcode
& 0x1f) {
624 case IB_OPCODE_SEND_LAST_WITH_IMMEDIATE
:
625 case IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE
:
626 entry
->wc_flags
= IB_WC_WITH_IMM
;
627 entry
->imm_data
= cqe
->imm_etype_pkey_eec
;
628 entry
->opcode
= IB_WC_RECV
;
630 case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE
:
631 case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE
:
632 entry
->wc_flags
= IB_WC_WITH_IMM
;
633 entry
->imm_data
= cqe
->imm_etype_pkey_eec
;
634 entry
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
638 entry
->opcode
= IB_WC_RECV
;
641 entry
->slid
= be16_to_cpu(cqe
->rlid
);
642 entry
->sl
= be16_to_cpu(cqe
->sl_g_mlpath
) >> 12;
643 entry
->src_qp
= be32_to_cpu(cqe
->rqpn
) & 0xffffff;
644 entry
->dlid_path_bits
= be16_to_cpu(cqe
->sl_g_mlpath
) & 0x7f;
645 entry
->pkey_index
= be32_to_cpu(cqe
->imm_etype_pkey_eec
) >> 16;
646 entry
->wc_flags
|= be16_to_cpu(cqe
->sl_g_mlpath
) & 0x80 ?
650 entry
->status
= IB_WC_SUCCESS
;
653 if (likely(free_cqe
)) {
662 int mthca_poll_cq(struct ib_cq
*ibcq
, int num_entries
,
665 struct mthca_dev
*dev
= to_mdev(ibcq
->device
);
666 struct mthca_cq
*cq
= to_mcq(ibcq
);
667 struct mthca_qp
*qp
= NULL
;
673 spin_lock_irqsave(&cq
->lock
, flags
);
677 while (npolled
< num_entries
) {
678 err
= mthca_poll_one(dev
, cq
, &qp
,
679 &freed
, entry
+ npolled
);
687 update_cons_index(dev
, cq
, freed
);
691 * If a CQ resize is in progress and we discovered that the
692 * old buffer is empty, then peek in the new buffer, and if
693 * it's not empty, switch to the new buffer and continue
696 if (unlikely(err
== -EAGAIN
&& cq
->resize_buf
&&
697 cq
->resize_buf
->state
== CQ_RESIZE_READY
)) {
699 * In Tavor mode, the hardware keeps the producer
700 * index modulo the CQ size. Since we might be making
701 * the CQ bigger, we need to mask our consumer index
702 * using the size of the old CQ buffer before looking
703 * in the new CQ buffer.
705 if (!mthca_is_memfree(dev
))
706 cq
->cons_index
&= cq
->ibcq
.cqe
;
708 if (cqe_sw(get_cqe_from_buf(&cq
->resize_buf
->buf
,
709 cq
->cons_index
& cq
->resize_buf
->cqe
))) {
710 struct mthca_cq_buf tbuf
;
715 cq
->buf
= cq
->resize_buf
->buf
;
716 cq
->ibcq
.cqe
= cq
->resize_buf
->cqe
;
718 cq
->resize_buf
->buf
= tbuf
;
719 cq
->resize_buf
->cqe
= tcqe
;
720 cq
->resize_buf
->state
= CQ_RESIZE_SWAPPED
;
726 spin_unlock_irqrestore(&cq
->lock
, flags
);
728 return err
== 0 || err
== -EAGAIN
? npolled
: err
;
731 int mthca_tavor_arm_cq(struct ib_cq
*cq
, enum ib_cq_notify_flags flags
)
735 doorbell
[0] = cpu_to_be32(((flags
& IB_CQ_SOLICITED_MASK
) ==
737 MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL
:
738 MTHCA_TAVOR_CQ_DB_REQ_NOT
) |
740 doorbell
[1] = (__force __be32
) 0xffffffff;
742 mthca_write64(doorbell
,
743 to_mdev(cq
->device
)->kar
+ MTHCA_CQ_DOORBELL
,
744 MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq
->device
)->doorbell_lock
));
749 int mthca_arbel_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
751 struct mthca_cq
*cq
= to_mcq(ibcq
);
757 ci
= cpu_to_be32(cq
->cons_index
);
760 doorbell
[1] = cpu_to_be32((cq
->cqn
<< 8) | (2 << 5) | (sn
<< 3) |
761 ((flags
& IB_CQ_SOLICITED_MASK
) ==
762 IB_CQ_SOLICITED
? 1 : 2));
764 mthca_write_db_rec(doorbell
, cq
->arm_db
);
767 * Make sure that the doorbell record in host memory is
768 * written before ringing the doorbell via PCI MMIO.
772 doorbell
[0] = cpu_to_be32((sn
<< 28) |
773 ((flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
?
774 MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL
:
775 MTHCA_ARBEL_CQ_DB_REQ_NOT
) |
779 mthca_write64(doorbell
,
780 to_mdev(ibcq
->device
)->kar
+ MTHCA_CQ_DOORBELL
,
781 MTHCA_GET_DOORBELL_LOCK(&to_mdev(ibcq
->device
)->doorbell_lock
));
786 int mthca_init_cq(struct mthca_dev
*dev
, int nent
,
787 struct mthca_ucontext
*ctx
, u32 pdn
,
790 struct mthca_mailbox
*mailbox
;
791 struct mthca_cq_context
*cq_context
;
795 cq
->ibcq
.cqe
= nent
- 1;
796 cq
->is_kernel
= !ctx
;
798 cq
->cqn
= mthca_alloc(&dev
->cq_table
.alloc
);
802 if (mthca_is_memfree(dev
)) {
803 err
= mthca_table_get(dev
, dev
->cq_table
.table
, cq
->cqn
);
812 cq
->set_ci_db_index
= mthca_alloc_db(dev
, MTHCA_DB_TYPE_CQ_SET_CI
,
813 cq
->cqn
, &cq
->set_ci_db
);
814 if (cq
->set_ci_db_index
< 0)
817 cq
->arm_db_index
= mthca_alloc_db(dev
, MTHCA_DB_TYPE_CQ_ARM
,
818 cq
->cqn
, &cq
->arm_db
);
819 if (cq
->arm_db_index
< 0)
824 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
828 cq_context
= mailbox
->buf
;
831 err
= mthca_alloc_cq_buf(dev
, &cq
->buf
, nent
);
833 goto err_out_mailbox
;
836 spin_lock_init(&cq
->lock
);
838 init_waitqueue_head(&cq
->wait
);
839 mutex_init(&cq
->mutex
);
841 memset(cq_context
, 0, sizeof *cq_context
);
842 cq_context
->flags
= cpu_to_be32(MTHCA_CQ_STATUS_OK
|
843 MTHCA_CQ_STATE_DISARMED
|
845 cq_context
->logsize_usrpage
= cpu_to_be32((ffs(nent
) - 1) << 24);
847 cq_context
->logsize_usrpage
|= cpu_to_be32(ctx
->uar
.index
);
849 cq_context
->logsize_usrpage
|= cpu_to_be32(dev
->driver_uar
.index
);
850 cq_context
->error_eqn
= cpu_to_be32(dev
->eq_table
.eq
[MTHCA_EQ_ASYNC
].eqn
);
851 cq_context
->comp_eqn
= cpu_to_be32(dev
->eq_table
.eq
[MTHCA_EQ_COMP
].eqn
);
852 cq_context
->pd
= cpu_to_be32(pdn
);
853 cq_context
->lkey
= cpu_to_be32(cq
->buf
.mr
.ibmr
.lkey
);
854 cq_context
->cqn
= cpu_to_be32(cq
->cqn
);
856 if (mthca_is_memfree(dev
)) {
857 cq_context
->ci_db
= cpu_to_be32(cq
->set_ci_db_index
);
858 cq_context
->state_db
= cpu_to_be32(cq
->arm_db_index
);
861 err
= mthca_SW2HW_CQ(dev
, mailbox
, cq
->cqn
, &status
);
863 mthca_warn(dev
, "SW2HW_CQ failed (%d)\n", err
);
864 goto err_out_free_mr
;
868 mthca_warn(dev
, "SW2HW_CQ returned status 0x%02x\n",
871 goto err_out_free_mr
;
874 spin_lock_irq(&dev
->cq_table
.lock
);
875 if (mthca_array_set(&dev
->cq_table
.cq
,
876 cq
->cqn
& (dev
->limits
.num_cqs
- 1),
878 spin_unlock_irq(&dev
->cq_table
.lock
);
879 goto err_out_free_mr
;
881 spin_unlock_irq(&dev
->cq_table
.lock
);
885 mthca_free_mailbox(dev
, mailbox
);
891 mthca_free_cq_buf(dev
, &cq
->buf
, cq
->ibcq
.cqe
);
894 mthca_free_mailbox(dev
, mailbox
);
897 if (cq
->is_kernel
&& mthca_is_memfree(dev
))
898 mthca_free_db(dev
, MTHCA_DB_TYPE_CQ_ARM
, cq
->arm_db_index
);
901 if (cq
->is_kernel
&& mthca_is_memfree(dev
))
902 mthca_free_db(dev
, MTHCA_DB_TYPE_CQ_SET_CI
, cq
->set_ci_db_index
);
905 mthca_table_put(dev
, dev
->cq_table
.table
, cq
->cqn
);
908 mthca_free(&dev
->cq_table
.alloc
, cq
->cqn
);
913 static inline int get_cq_refcount(struct mthca_dev
*dev
, struct mthca_cq
*cq
)
917 spin_lock_irq(&dev
->cq_table
.lock
);
919 spin_unlock_irq(&dev
->cq_table
.lock
);
924 void mthca_free_cq(struct mthca_dev
*dev
,
927 struct mthca_mailbox
*mailbox
;
931 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
932 if (IS_ERR(mailbox
)) {
933 mthca_warn(dev
, "No memory for mailbox to free CQ.\n");
937 err
= mthca_HW2SW_CQ(dev
, mailbox
, cq
->cqn
, &status
);
939 mthca_warn(dev
, "HW2SW_CQ failed (%d)\n", err
);
941 mthca_warn(dev
, "HW2SW_CQ returned status 0x%02x\n", status
);
944 __be32
*ctx
= mailbox
->buf
;
947 printk(KERN_ERR
"context for CQN %x (cons index %x, next sw %d)\n",
948 cq
->cqn
, cq
->cons_index
,
949 cq
->is_kernel
? !!next_cqe_sw(cq
) : 0);
950 for (j
= 0; j
< 16; ++j
)
951 printk(KERN_ERR
"[%2x] %08x\n", j
* 4, be32_to_cpu(ctx
[j
]));
954 spin_lock_irq(&dev
->cq_table
.lock
);
955 mthca_array_clear(&dev
->cq_table
.cq
,
956 cq
->cqn
& (dev
->limits
.num_cqs
- 1));
958 spin_unlock_irq(&dev
->cq_table
.lock
);
960 if (dev
->mthca_flags
& MTHCA_FLAG_MSI_X
)
961 synchronize_irq(dev
->eq_table
.eq
[MTHCA_EQ_COMP
].msi_x_vector
);
963 synchronize_irq(dev
->pdev
->irq
);
965 wait_event(cq
->wait
, !get_cq_refcount(dev
, cq
));
968 mthca_free_cq_buf(dev
, &cq
->buf
, cq
->ibcq
.cqe
);
969 if (mthca_is_memfree(dev
)) {
970 mthca_free_db(dev
, MTHCA_DB_TYPE_CQ_ARM
, cq
->arm_db_index
);
971 mthca_free_db(dev
, MTHCA_DB_TYPE_CQ_SET_CI
, cq
->set_ci_db_index
);
975 mthca_table_put(dev
, dev
->cq_table
.table
, cq
->cqn
);
976 mthca_free(&dev
->cq_table
.alloc
, cq
->cqn
);
977 mthca_free_mailbox(dev
, mailbox
);
980 int mthca_init_cq_table(struct mthca_dev
*dev
)
984 spin_lock_init(&dev
->cq_table
.lock
);
986 err
= mthca_alloc_init(&dev
->cq_table
.alloc
,
989 dev
->limits
.reserved_cqs
);
993 err
= mthca_array_init(&dev
->cq_table
.cq
,
994 dev
->limits
.num_cqs
);
996 mthca_alloc_cleanup(&dev
->cq_table
.alloc
);
1001 void mthca_cleanup_cq_table(struct mthca_dev
*dev
)
1003 mthca_array_cleanup(&dev
->cq_table
.cq
, dev
->limits
.num_cqs
);
1004 mthca_alloc_cleanup(&dev
->cq_table
.alloc
);