2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kref.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_user_verbs.h>
36 #include <rdma/ib_cache.h>
39 static void mlx5_ib_cq_comp(struct mlx5_core_cq
*cq
)
41 struct ib_cq
*ibcq
= &to_mibcq(cq
)->ibcq
;
43 ibcq
->comp_handler(ibcq
, ibcq
->cq_context
);
46 static void mlx5_ib_cq_event(struct mlx5_core_cq
*mcq
, enum mlx5_event type
)
48 struct mlx5_ib_cq
*cq
= container_of(mcq
, struct mlx5_ib_cq
, mcq
);
49 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
50 struct ib_cq
*ibcq
= &cq
->ibcq
;
51 struct ib_event event
;
53 if (type
!= MLX5_EVENT_TYPE_CQ_ERROR
) {
54 mlx5_ib_warn(dev
, "Unexpected event type %d on CQ %06x\n",
59 if (ibcq
->event_handler
) {
60 event
.device
= &dev
->ib_dev
;
61 event
.event
= IB_EVENT_CQ_ERR
;
62 event
.element
.cq
= ibcq
;
63 ibcq
->event_handler(&event
, ibcq
->cq_context
);
67 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf
*buf
, int n
, int size
)
69 return mlx5_buf_offset(&buf
->buf
, n
* size
);
72 static void *get_cqe(struct mlx5_ib_cq
*cq
, int n
)
74 return get_cqe_from_buf(&cq
->buf
, n
, cq
->mcq
.cqe_sz
);
77 static u8
sw_ownership_bit(int n
, int nent
)
79 return (n
& nent
) ? 1 : 0;
82 static void *get_sw_cqe(struct mlx5_ib_cq
*cq
, int n
)
84 void *cqe
= get_cqe(cq
, n
& cq
->ibcq
.cqe
);
85 struct mlx5_cqe64
*cqe64
;
87 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
89 if (likely((cqe64
->op_own
) >> 4 != MLX5_CQE_INVALID
) &&
90 !((cqe64
->op_own
& MLX5_CQE_OWNER_MASK
) ^ !!(n
& (cq
->ibcq
.cqe
+ 1)))) {
97 static void *next_cqe_sw(struct mlx5_ib_cq
*cq
)
99 return get_sw_cqe(cq
, cq
->mcq
.cons_index
);
102 static enum ib_wc_opcode
get_umr_comp(struct mlx5_ib_wq
*wq
, int idx
)
104 switch (wq
->wr_data
[idx
]) {
108 case IB_WR_LOCAL_INV
:
109 return IB_WC_LOCAL_INV
;
115 pr_warn("unknown completion status\n");
120 static void handle_good_req(struct ib_wc
*wc
, struct mlx5_cqe64
*cqe
,
121 struct mlx5_ib_wq
*wq
, int idx
)
124 switch (be32_to_cpu(cqe
->sop_drop_qpn
) >> 24) {
125 case MLX5_OPCODE_RDMA_WRITE_IMM
:
126 wc
->wc_flags
|= IB_WC_WITH_IMM
;
127 case MLX5_OPCODE_RDMA_WRITE
:
128 wc
->opcode
= IB_WC_RDMA_WRITE
;
130 case MLX5_OPCODE_SEND_IMM
:
131 wc
->wc_flags
|= IB_WC_WITH_IMM
;
132 case MLX5_OPCODE_SEND
:
133 case MLX5_OPCODE_SEND_INVAL
:
134 wc
->opcode
= IB_WC_SEND
;
136 case MLX5_OPCODE_RDMA_READ
:
137 wc
->opcode
= IB_WC_RDMA_READ
;
138 wc
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
140 case MLX5_OPCODE_ATOMIC_CS
:
141 wc
->opcode
= IB_WC_COMP_SWAP
;
144 case MLX5_OPCODE_ATOMIC_FA
:
145 wc
->opcode
= IB_WC_FETCH_ADD
;
148 case MLX5_OPCODE_ATOMIC_MASKED_CS
:
149 wc
->opcode
= IB_WC_MASKED_COMP_SWAP
;
152 case MLX5_OPCODE_ATOMIC_MASKED_FA
:
153 wc
->opcode
= IB_WC_MASKED_FETCH_ADD
;
156 case MLX5_OPCODE_UMR
:
157 wc
->opcode
= get_umr_comp(wq
, idx
);
163 MLX5_GRH_IN_BUFFER
= 1,
167 static void handle_responder(struct ib_wc
*wc
, struct mlx5_cqe64
*cqe
,
168 struct mlx5_ib_qp
*qp
)
170 enum rdma_link_layer ll
= rdma_port_get_link_layer(qp
->ibqp
.device
, 1);
171 struct mlx5_ib_dev
*dev
= to_mdev(qp
->ibqp
.device
);
172 struct mlx5_ib_srq
*srq
;
173 struct mlx5_ib_wq
*wq
;
177 if (qp
->ibqp
.srq
|| qp
->ibqp
.xrcd
) {
178 struct mlx5_core_srq
*msrq
= NULL
;
181 msrq
= mlx5_core_get_srq(dev
->mdev
,
182 be32_to_cpu(cqe
->srqn
));
183 srq
= to_mibsrq(msrq
);
185 srq
= to_msrq(qp
->ibqp
.srq
);
188 wqe_ctr
= be16_to_cpu(cqe
->wqe_counter
);
189 wc
->wr_id
= srq
->wrid
[wqe_ctr
];
190 mlx5_ib_free_srq_wqe(srq
, wqe_ctr
);
191 if (msrq
&& atomic_dec_and_test(&msrq
->refcount
))
192 complete(&msrq
->free
);
196 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
199 wc
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
201 switch (cqe
->op_own
>> 4) {
202 case MLX5_CQE_RESP_WR_IMM
:
203 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
204 wc
->wc_flags
= IB_WC_WITH_IMM
;
205 wc
->ex
.imm_data
= cqe
->imm_inval_pkey
;
207 case MLX5_CQE_RESP_SEND
:
208 wc
->opcode
= IB_WC_RECV
;
209 wc
->wc_flags
= IB_WC_IP_CSUM_OK
;
210 if (unlikely(!((cqe
->hds_ip_ext
& CQE_L3_OK
) &&
211 (cqe
->hds_ip_ext
& CQE_L4_OK
))))
214 case MLX5_CQE_RESP_SEND_IMM
:
215 wc
->opcode
= IB_WC_RECV
;
216 wc
->wc_flags
= IB_WC_WITH_IMM
;
217 wc
->ex
.imm_data
= cqe
->imm_inval_pkey
;
219 case MLX5_CQE_RESP_SEND_INV
:
220 wc
->opcode
= IB_WC_RECV
;
221 wc
->wc_flags
= IB_WC_WITH_INVALIDATE
;
222 wc
->ex
.invalidate_rkey
= be32_to_cpu(cqe
->imm_inval_pkey
);
225 wc
->slid
= be16_to_cpu(cqe
->slid
);
226 wc
->sl
= (be32_to_cpu(cqe
->flags_rqpn
) >> 24) & 0xf;
227 wc
->src_qp
= be32_to_cpu(cqe
->flags_rqpn
) & 0xffffff;
228 wc
->dlid_path_bits
= cqe
->ml_path
;
229 g
= (be32_to_cpu(cqe
->flags_rqpn
) >> 28) & 3;
230 wc
->wc_flags
|= g
? IB_WC_GRH
: 0;
231 if (unlikely(is_qp1(qp
->ibqp
.qp_type
))) {
232 u16 pkey
= be32_to_cpu(cqe
->imm_inval_pkey
) & 0xffff;
234 ib_find_cached_pkey(&dev
->ib_dev
, qp
->port
, pkey
,
240 if (ll
!= IB_LINK_LAYER_ETHERNET
)
243 switch (wc
->sl
& 0x3) {
244 case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH
:
245 wc
->network_hdr_type
= RDMA_NETWORK_IB
;
247 case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6
:
248 wc
->network_hdr_type
= RDMA_NETWORK_IPV6
;
250 case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV4
:
251 wc
->network_hdr_type
= RDMA_NETWORK_IPV4
;
254 wc
->wc_flags
|= IB_WC_WITH_NETWORK_HDR_TYPE
;
257 static void dump_cqe(struct mlx5_ib_dev
*dev
, struct mlx5_err_cqe
*cqe
)
259 __be32
*p
= (__be32
*)cqe
;
262 mlx5_ib_warn(dev
, "dump error cqe\n");
263 for (i
= 0; i
< sizeof(*cqe
) / 16; i
++, p
+= 4)
264 pr_info("%08x %08x %08x %08x\n", be32_to_cpu(p
[0]),
265 be32_to_cpu(p
[1]), be32_to_cpu(p
[2]),
269 static void mlx5_handle_error_cqe(struct mlx5_ib_dev
*dev
,
270 struct mlx5_err_cqe
*cqe
,
275 switch (cqe
->syndrome
) {
276 case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR
:
277 wc
->status
= IB_WC_LOC_LEN_ERR
;
279 case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR
:
280 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
282 case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR
:
283 wc
->status
= IB_WC_LOC_PROT_ERR
;
285 case MLX5_CQE_SYNDROME_WR_FLUSH_ERR
:
287 wc
->status
= IB_WC_WR_FLUSH_ERR
;
289 case MLX5_CQE_SYNDROME_MW_BIND_ERR
:
290 wc
->status
= IB_WC_MW_BIND_ERR
;
292 case MLX5_CQE_SYNDROME_BAD_RESP_ERR
:
293 wc
->status
= IB_WC_BAD_RESP_ERR
;
295 case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR
:
296 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
298 case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR
:
299 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
301 case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR
:
302 wc
->status
= IB_WC_REM_ACCESS_ERR
;
304 case MLX5_CQE_SYNDROME_REMOTE_OP_ERR
:
305 wc
->status
= IB_WC_REM_OP_ERR
;
307 case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR
:
308 wc
->status
= IB_WC_RETRY_EXC_ERR
;
311 case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR
:
312 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
315 case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR
:
316 wc
->status
= IB_WC_REM_ABORT_ERR
;
319 wc
->status
= IB_WC_GENERAL_ERR
;
323 wc
->vendor_err
= cqe
->vendor_err_synd
;
328 static int is_atomic_response(struct mlx5_ib_qp
*qp
, uint16_t idx
)
330 /* TBD: waiting decision
335 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp
*qp
, uint16_t idx
)
337 struct mlx5_wqe_data_seg
*dpseg
;
340 dpseg
= mlx5_get_send_wqe(qp
, idx
) + sizeof(struct mlx5_wqe_ctrl_seg
) +
341 sizeof(struct mlx5_wqe_raddr_seg
) +
342 sizeof(struct mlx5_wqe_atomic_seg
);
343 addr
= (void *)(unsigned long)be64_to_cpu(dpseg
->addr
);
347 static void handle_atomic(struct mlx5_ib_qp
*qp
, struct mlx5_cqe64
*cqe64
,
354 if (!is_atomic_response(qp
, idx
))
357 byte_count
= be32_to_cpu(cqe64
->byte_cnt
);
358 addr
= mlx5_get_atomic_laddr(qp
, idx
);
360 if (byte_count
== 4) {
361 *(uint32_t *)addr
= be32_to_cpu(*((__be32
*)addr
));
363 for (i
= 0; i
< byte_count
; i
+= 8) {
364 *(uint64_t *)addr
= be64_to_cpu(*((__be64
*)addr
));
372 static void handle_atomics(struct mlx5_ib_qp
*qp
, struct mlx5_cqe64
*cqe64
,
378 idx
= tail
& (qp
->sq
.wqe_cnt
- 1);
379 handle_atomic(qp
, cqe64
, idx
);
383 tail
= qp
->sq
.w_list
[idx
].next
;
385 tail
= qp
->sq
.w_list
[idx
].next
;
386 qp
->sq
.last_poll
= tail
;
389 static void free_cq_buf(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq_buf
*buf
)
391 mlx5_buf_free(dev
->mdev
, &buf
->buf
);
394 static void get_sig_err_item(struct mlx5_sig_err_cqe
*cqe
,
395 struct ib_sig_err
*item
)
397 u16 syndrome
= be16_to_cpu(cqe
->syndrome
);
399 #define GUARD_ERR (1 << 13)
400 #define APPTAG_ERR (1 << 12)
401 #define REFTAG_ERR (1 << 11)
403 if (syndrome
& GUARD_ERR
) {
404 item
->err_type
= IB_SIG_BAD_GUARD
;
405 item
->expected
= be32_to_cpu(cqe
->expected_trans_sig
) >> 16;
406 item
->actual
= be32_to_cpu(cqe
->actual_trans_sig
) >> 16;
408 if (syndrome
& REFTAG_ERR
) {
409 item
->err_type
= IB_SIG_BAD_REFTAG
;
410 item
->expected
= be32_to_cpu(cqe
->expected_reftag
);
411 item
->actual
= be32_to_cpu(cqe
->actual_reftag
);
413 if (syndrome
& APPTAG_ERR
) {
414 item
->err_type
= IB_SIG_BAD_APPTAG
;
415 item
->expected
= be32_to_cpu(cqe
->expected_trans_sig
) & 0xffff;
416 item
->actual
= be32_to_cpu(cqe
->actual_trans_sig
) & 0xffff;
418 pr_err("Got signature completion error with bad syndrome %04x\n",
422 item
->sig_err_offset
= be64_to_cpu(cqe
->err_offset
);
423 item
->key
= be32_to_cpu(cqe
->mkey
);
426 static void sw_send_comp(struct mlx5_ib_qp
*qp
, int num_entries
,
427 struct ib_wc
*wc
, int *npolled
)
429 struct mlx5_ib_wq
*wq
;
436 cur
= wq
->head
- wq
->tail
;
442 for (i
= 0; i
< cur
&& np
< num_entries
; i
++) {
443 idx
= wq
->last_poll
& (wq
->wqe_cnt
- 1);
444 wc
->wr_id
= wq
->wrid
[idx
];
445 wc
->status
= IB_WC_WR_FLUSH_ERR
;
446 wc
->vendor_err
= MLX5_CQE_SYNDROME_WR_FLUSH_ERR
;
451 wq
->last_poll
= wq
->w_list
[idx
].next
;
456 static void sw_recv_comp(struct mlx5_ib_qp
*qp
, int num_entries
,
457 struct ib_wc
*wc
, int *npolled
)
459 struct mlx5_ib_wq
*wq
;
465 cur
= wq
->head
- wq
->tail
;
471 for (i
= 0; i
< cur
&& np
< num_entries
; i
++) {
472 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
473 wc
->status
= IB_WC_WR_FLUSH_ERR
;
474 wc
->vendor_err
= MLX5_CQE_SYNDROME_WR_FLUSH_ERR
;
483 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq
*cq
, int num_entries
,
484 struct ib_wc
*wc
, int *npolled
)
486 struct mlx5_ib_qp
*qp
;
489 /* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
490 list_for_each_entry(qp
, &cq
->list_send_qp
, cq_send_list
) {
491 sw_send_comp(qp
, num_entries
, wc
+ *npolled
, npolled
);
492 if (*npolled
>= num_entries
)
496 list_for_each_entry(qp
, &cq
->list_recv_qp
, cq_recv_list
) {
497 sw_recv_comp(qp
, num_entries
, wc
+ *npolled
, npolled
);
498 if (*npolled
>= num_entries
)
503 static int mlx5_poll_one(struct mlx5_ib_cq
*cq
,
504 struct mlx5_ib_qp
**cur_qp
,
507 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
508 struct mlx5_err_cqe
*err_cqe
;
509 struct mlx5_cqe64
*cqe64
;
510 struct mlx5_core_qp
*mqp
;
511 struct mlx5_ib_wq
*wq
;
512 struct mlx5_sig_err_cqe
*sig_err_cqe
;
513 struct mlx5_core_mkey
*mmkey
;
514 struct mlx5_ib_mr
*mr
;
522 cqe
= next_cqe_sw(cq
);
526 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
528 ++cq
->mcq
.cons_index
;
530 /* Make sure we read CQ entry contents after we've checked the
535 opcode
= cqe64
->op_own
>> 4;
536 if (unlikely(opcode
== MLX5_CQE_RESIZE_CQ
)) {
537 if (likely(cq
->resize_buf
)) {
538 free_cq_buf(dev
, &cq
->buf
);
539 cq
->buf
= *cq
->resize_buf
;
540 kfree(cq
->resize_buf
);
541 cq
->resize_buf
= NULL
;
544 mlx5_ib_warn(dev
, "unexpected resize cqe\n");
548 qpn
= ntohl(cqe64
->sop_drop_qpn
) & 0xffffff;
549 if (!*cur_qp
|| (qpn
!= (*cur_qp
)->ibqp
.qp_num
)) {
550 /* We do not have to take the QP table lock here,
551 * because CQs will be locked while QPs are removed
554 mqp
= __mlx5_qp_lookup(dev
->mdev
, qpn
);
555 *cur_qp
= to_mibqp(mqp
);
558 wc
->qp
= &(*cur_qp
)->ibqp
;
562 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
563 idx
= wqe_ctr
& (wq
->wqe_cnt
- 1);
564 handle_good_req(wc
, cqe64
, wq
, idx
);
565 handle_atomics(*cur_qp
, cqe64
, wq
->last_poll
, idx
);
566 wc
->wr_id
= wq
->wrid
[idx
];
567 wq
->tail
= wq
->wqe_head
[idx
] + 1;
568 wc
->status
= IB_WC_SUCCESS
;
570 case MLX5_CQE_RESP_WR_IMM
:
571 case MLX5_CQE_RESP_SEND
:
572 case MLX5_CQE_RESP_SEND_IMM
:
573 case MLX5_CQE_RESP_SEND_INV
:
574 handle_responder(wc
, cqe64
, *cur_qp
);
575 wc
->status
= IB_WC_SUCCESS
;
577 case MLX5_CQE_RESIZE_CQ
:
579 case MLX5_CQE_REQ_ERR
:
580 case MLX5_CQE_RESP_ERR
:
581 err_cqe
= (struct mlx5_err_cqe
*)cqe64
;
582 mlx5_handle_error_cqe(dev
, err_cqe
, wc
);
583 mlx5_ib_dbg(dev
, "%s error cqe on cqn 0x%x:\n",
584 opcode
== MLX5_CQE_REQ_ERR
?
585 "Requestor" : "Responder", cq
->mcq
.cqn
);
586 mlx5_ib_dbg(dev
, "syndrome 0x%x, vendor syndrome 0x%x\n",
587 err_cqe
->syndrome
, err_cqe
->vendor_err_synd
);
588 if (opcode
== MLX5_CQE_REQ_ERR
) {
590 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
591 idx
= wqe_ctr
& (wq
->wqe_cnt
- 1);
592 wc
->wr_id
= wq
->wrid
[idx
];
593 wq
->tail
= wq
->wqe_head
[idx
] + 1;
595 struct mlx5_ib_srq
*srq
;
597 if ((*cur_qp
)->ibqp
.srq
) {
598 srq
= to_msrq((*cur_qp
)->ibqp
.srq
);
599 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
600 wc
->wr_id
= srq
->wrid
[wqe_ctr
];
601 mlx5_ib_free_srq_wqe(srq
, wqe_ctr
);
604 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
609 case MLX5_CQE_SIG_ERR
:
610 sig_err_cqe
= (struct mlx5_sig_err_cqe
*)cqe64
;
612 read_lock(&dev
->mdev
->priv
.mkey_table
.lock
);
613 mmkey
= __mlx5_mr_lookup(dev
->mdev
,
614 mlx5_base_mkey(be32_to_cpu(sig_err_cqe
->mkey
)));
615 mr
= to_mibmr(mmkey
);
616 get_sig_err_item(sig_err_cqe
, &mr
->sig
->err_item
);
617 mr
->sig
->sig_err_exists
= true;
618 mr
->sig
->sigerr_count
++;
620 mlx5_ib_warn(dev
, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",
621 cq
->mcq
.cqn
, mr
->sig
->err_item
.key
,
622 mr
->sig
->err_item
.err_type
,
623 mr
->sig
->err_item
.sig_err_offset
,
624 mr
->sig
->err_item
.expected
,
625 mr
->sig
->err_item
.actual
);
627 read_unlock(&dev
->mdev
->priv
.mkey_table
.lock
);
634 static int poll_soft_wc(struct mlx5_ib_cq
*cq
, int num_entries
,
637 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
638 struct mlx5_ib_wc
*soft_wc
, *next
;
641 list_for_each_entry_safe(soft_wc
, next
, &cq
->wc_list
, list
) {
642 if (npolled
>= num_entries
)
645 mlx5_ib_dbg(dev
, "polled software generated completion on CQ 0x%x\n",
648 wc
[npolled
++] = soft_wc
->wc
;
649 list_del(&soft_wc
->list
);
656 int mlx5_ib_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
658 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
659 struct mlx5_ib_qp
*cur_qp
= NULL
;
660 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
661 struct mlx5_core_dev
*mdev
= dev
->mdev
;
666 spin_lock_irqsave(&cq
->lock
, flags
);
667 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
) {
668 mlx5_ib_poll_sw_comp(cq
, num_entries
, wc
, &npolled
);
672 if (unlikely(!list_empty(&cq
->wc_list
)))
673 soft_polled
= poll_soft_wc(cq
, num_entries
, wc
);
675 for (npolled
= 0; npolled
< num_entries
- soft_polled
; npolled
++) {
676 if (mlx5_poll_one(cq
, &cur_qp
, wc
+ soft_polled
+ npolled
))
681 mlx5_cq_set_ci(&cq
->mcq
);
683 spin_unlock_irqrestore(&cq
->lock
, flags
);
685 return soft_polled
+ npolled
;
688 int mlx5_ib_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
690 struct mlx5_core_dev
*mdev
= to_mdev(ibcq
->device
)->mdev
;
691 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
692 void __iomem
*uar_page
= mdev
->priv
.uuari
.uars
[0].map
;
693 unsigned long irq_flags
;
696 spin_lock_irqsave(&cq
->lock
, irq_flags
);
697 if (cq
->notify_flags
!= IB_CQ_NEXT_COMP
)
698 cq
->notify_flags
= flags
& IB_CQ_SOLICITED_MASK
;
700 if ((flags
& IB_CQ_REPORT_MISSED_EVENTS
) && !list_empty(&cq
->wc_list
))
702 spin_unlock_irqrestore(&cq
->lock
, irq_flags
);
704 mlx5_cq_arm(&cq
->mcq
,
705 (flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
?
706 MLX5_CQ_DB_REQ_NOT_SOL
: MLX5_CQ_DB_REQ_NOT
,
708 MLX5_GET_DOORBELL_LOCK(&mdev
->priv
.cq_uar_lock
),
709 to_mcq(ibcq
)->mcq
.cons_index
);
714 static int alloc_cq_buf(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq_buf
*buf
,
715 int nent
, int cqe_size
)
719 err
= mlx5_buf_alloc(dev
->mdev
, nent
* cqe_size
, &buf
->buf
);
723 buf
->cqe_size
= cqe_size
;
729 static int create_cq_user(struct mlx5_ib_dev
*dev
, struct ib_udata
*udata
,
730 struct ib_ucontext
*context
, struct mlx5_ib_cq
*cq
,
731 int entries
, u32
**cqb
,
732 int *cqe_size
, int *index
, int *inlen
)
734 struct mlx5_ib_create_cq ucmd
= {};
744 (udata
->inlen
- sizeof(struct ib_uverbs_cmd_hdr
) <
745 sizeof(ucmd
)) ? (sizeof(ucmd
) -
746 sizeof(ucmd
.reserved
)) : sizeof(ucmd
);
748 if (ib_copy_from_udata(&ucmd
, udata
, ucmdlen
))
751 if (ucmdlen
== sizeof(ucmd
) &&
755 if (ucmd
.cqe_size
!= 64 && ucmd
.cqe_size
!= 128)
758 *cqe_size
= ucmd
.cqe_size
;
760 cq
->buf
.umem
= ib_umem_get(context
, ucmd
.buf_addr
,
761 entries
* ucmd
.cqe_size
,
762 IB_ACCESS_LOCAL_WRITE
, 1);
763 if (IS_ERR(cq
->buf
.umem
)) {
764 err
= PTR_ERR(cq
->buf
.umem
);
768 err
= mlx5_ib_db_map_user(to_mucontext(context
), ucmd
.db_addr
,
773 mlx5_ib_cont_pages(cq
->buf
.umem
, ucmd
.buf_addr
, 0, &npages
, &page_shift
,
775 mlx5_ib_dbg(dev
, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
776 ucmd
.buf_addr
, entries
* ucmd
.cqe_size
, npages
, page_shift
, ncont
);
778 *inlen
= MLX5_ST_SZ_BYTES(create_cq_in
) +
779 MLX5_FLD_SZ_BYTES(create_cq_in
, pas
[0]) * ncont
;
780 *cqb
= mlx5_vzalloc(*inlen
);
786 pas
= (__be64
*)MLX5_ADDR_OF(create_cq_in
, *cqb
, pas
);
787 mlx5_ib_populate_pas(dev
, cq
->buf
.umem
, page_shift
, pas
, 0);
789 cqc
= MLX5_ADDR_OF(create_cq_in
, *cqb
, cq_context
);
790 MLX5_SET(cqc
, cqc
, log_page_size
,
791 page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
793 *index
= to_mucontext(context
)->uuari
.uars
[0].index
;
795 if (ucmd
.cqe_comp_en
== 1) {
796 if (unlikely((*cqe_size
!= 64) ||
797 !MLX5_CAP_GEN(dev
->mdev
, cqe_compression
))) {
799 mlx5_ib_warn(dev
, "CQE compression is not supported for size %d!\n",
804 if (unlikely(!ucmd
.cqe_comp_res_format
||
805 !(ucmd
.cqe_comp_res_format
<
806 MLX5_IB_CQE_RES_RESERVED
) ||
807 (ucmd
.cqe_comp_res_format
&
808 (ucmd
.cqe_comp_res_format
- 1)))) {
810 mlx5_ib_warn(dev
, "CQE compression res format %d is not supported!\n",
811 ucmd
.cqe_comp_res_format
);
815 MLX5_SET(cqc
, cqc
, cqe_comp_en
, 1);
816 MLX5_SET(cqc
, cqc
, mini_cqe_res_format
,
817 ilog2(ucmd
.cqe_comp_res_format
));
826 mlx5_ib_db_unmap_user(to_mucontext(context
), &cq
->db
);
829 ib_umem_release(cq
->buf
.umem
);
833 static void destroy_cq_user(struct mlx5_ib_cq
*cq
, struct ib_ucontext
*context
)
835 mlx5_ib_db_unmap_user(to_mucontext(context
), &cq
->db
);
836 ib_umem_release(cq
->buf
.umem
);
839 static void init_cq_buf(struct mlx5_ib_cq
*cq
, struct mlx5_ib_cq_buf
*buf
)
843 struct mlx5_cqe64
*cqe64
;
845 for (i
= 0; i
< buf
->nent
; i
++) {
846 cqe
= get_cqe_from_buf(buf
, i
, buf
->cqe_size
);
847 cqe64
= buf
->cqe_size
== 64 ? cqe
: cqe
+ 64;
848 cqe64
->op_own
= MLX5_CQE_INVALID
<< 4;
852 static int create_cq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
853 int entries
, int cqe_size
,
854 u32
**cqb
, int *index
, int *inlen
)
860 err
= mlx5_db_alloc(dev
->mdev
, &cq
->db
);
864 cq
->mcq
.set_ci_db
= cq
->db
.db
;
865 cq
->mcq
.arm_db
= cq
->db
.db
+ 1;
866 cq
->mcq
.cqe_sz
= cqe_size
;
868 err
= alloc_cq_buf(dev
, &cq
->buf
, entries
, cqe_size
);
872 init_cq_buf(cq
, &cq
->buf
);
874 *inlen
= MLX5_ST_SZ_BYTES(create_cq_in
) +
875 MLX5_FLD_SZ_BYTES(create_cq_in
, pas
[0]) * cq
->buf
.buf
.npages
;
876 *cqb
= mlx5_vzalloc(*inlen
);
882 pas
= (__be64
*)MLX5_ADDR_OF(create_cq_in
, *cqb
, pas
);
883 mlx5_fill_page_array(&cq
->buf
.buf
, pas
);
885 cqc
= MLX5_ADDR_OF(create_cq_in
, *cqb
, cq_context
);
886 MLX5_SET(cqc
, cqc
, log_page_size
,
887 cq
->buf
.buf
.page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
889 *index
= dev
->mdev
->priv
.uuari
.uars
[0].index
;
894 free_cq_buf(dev
, &cq
->buf
);
897 mlx5_db_free(dev
->mdev
, &cq
->db
);
901 static void destroy_cq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
)
903 free_cq_buf(dev
, &cq
->buf
);
904 mlx5_db_free(dev
->mdev
, &cq
->db
);
907 static void notify_soft_wc_handler(struct work_struct
*work
)
909 struct mlx5_ib_cq
*cq
= container_of(work
, struct mlx5_ib_cq
,
912 cq
->ibcq
.comp_handler(&cq
->ibcq
, cq
->ibcq
.cq_context
);
915 struct ib_cq
*mlx5_ib_create_cq(struct ib_device
*ibdev
,
916 const struct ib_cq_init_attr
*attr
,
917 struct ib_ucontext
*context
,
918 struct ib_udata
*udata
)
920 int entries
= attr
->cqe
;
921 int vector
= attr
->comp_vector
;
922 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
923 struct mlx5_ib_cq
*cq
;
924 int uninitialized_var(index
);
925 int uninitialized_var(inlen
);
934 (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
))))
935 return ERR_PTR(-EINVAL
);
937 if (check_cq_create_flags(attr
->flags
))
938 return ERR_PTR(-EOPNOTSUPP
);
940 entries
= roundup_pow_of_two(entries
+ 1);
941 if (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
)))
942 return ERR_PTR(-EINVAL
);
944 cq
= kzalloc(sizeof(*cq
), GFP_KERNEL
);
946 return ERR_PTR(-ENOMEM
);
948 cq
->ibcq
.cqe
= entries
- 1;
949 mutex_init(&cq
->resize_mutex
);
950 spin_lock_init(&cq
->lock
);
951 cq
->resize_buf
= NULL
;
952 cq
->resize_umem
= NULL
;
953 cq
->create_flags
= attr
->flags
;
954 INIT_LIST_HEAD(&cq
->list_send_qp
);
955 INIT_LIST_HEAD(&cq
->list_recv_qp
);
958 err
= create_cq_user(dev
, udata
, context
, cq
, entries
,
959 &cqb
, &cqe_size
, &index
, &inlen
);
963 cqe_size
= cache_line_size() == 128 ? 128 : 64;
964 err
= create_cq_kernel(dev
, cq
, entries
, cqe_size
, &cqb
,
969 INIT_WORK(&cq
->notify_work
, notify_soft_wc_handler
);
972 err
= mlx5_vector2eqn(dev
->mdev
, vector
, &eqn
, &irqn
);
976 cq
->cqe_size
= cqe_size
;
978 cqc
= MLX5_ADDR_OF(create_cq_in
, cqb
, cq_context
);
979 MLX5_SET(cqc
, cqc
, cqe_sz
, cqe_sz_to_mlx_sz(cqe_size
));
980 MLX5_SET(cqc
, cqc
, log_cq_size
, ilog2(entries
));
981 MLX5_SET(cqc
, cqc
, uar_page
, index
);
982 MLX5_SET(cqc
, cqc
, c_eqn
, eqn
);
983 MLX5_SET64(cqc
, cqc
, dbr_addr
, cq
->db
.dma
);
984 if (cq
->create_flags
& IB_CQ_FLAGS_IGNORE_OVERRUN
)
985 MLX5_SET(cqc
, cqc
, oi
, 1);
987 err
= mlx5_core_create_cq(dev
->mdev
, &cq
->mcq
, cqb
, inlen
);
991 mlx5_ib_dbg(dev
, "cqn 0x%x\n", cq
->mcq
.cqn
);
994 cq
->mcq
.tasklet_ctx
.comp
= mlx5_ib_cq_comp
;
996 cq
->mcq
.comp
= mlx5_ib_cq_comp
;
997 cq
->mcq
.event
= mlx5_ib_cq_event
;
999 INIT_LIST_HEAD(&cq
->wc_list
);
1002 if (ib_copy_to_udata(udata
, &cq
->mcq
.cqn
, sizeof(__u32
))) {
1012 mlx5_core_destroy_cq(dev
->mdev
, &cq
->mcq
);
1017 destroy_cq_user(cq
, context
);
1019 destroy_cq_kernel(dev
, cq
);
1024 return ERR_PTR(err
);
1028 int mlx5_ib_destroy_cq(struct ib_cq
*cq
)
1030 struct mlx5_ib_dev
*dev
= to_mdev(cq
->device
);
1031 struct mlx5_ib_cq
*mcq
= to_mcq(cq
);
1032 struct ib_ucontext
*context
= NULL
;
1035 context
= cq
->uobject
->context
;
1037 mlx5_core_destroy_cq(dev
->mdev
, &mcq
->mcq
);
1039 destroy_cq_user(mcq
, context
);
1041 destroy_cq_kernel(dev
, mcq
);
1048 static int is_equal_rsn(struct mlx5_cqe64
*cqe64
, u32 rsn
)
1050 return rsn
== (ntohl(cqe64
->sop_drop_qpn
) & 0xffffff);
1053 void __mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 rsn
, struct mlx5_ib_srq
*srq
)
1055 struct mlx5_cqe64
*cqe64
, *dest64
;
1064 /* First we need to find the current producer index, so we
1065 * know where to start cleaning from. It doesn't matter if HW
1066 * adds new entries after this loop -- the QP we're worried
1067 * about is already in RESET, so the new entries won't come
1068 * from our QP and therefore don't need to be checked.
1070 for (prod_index
= cq
->mcq
.cons_index
; get_sw_cqe(cq
, prod_index
); prod_index
++)
1071 if (prod_index
== cq
->mcq
.cons_index
+ cq
->ibcq
.cqe
)
1074 /* Now sweep backwards through the CQ, removing CQ entries
1075 * that match our QP by copying older entries on top of them.
1077 while ((int) --prod_index
- (int) cq
->mcq
.cons_index
>= 0) {
1078 cqe
= get_cqe(cq
, prod_index
& cq
->ibcq
.cqe
);
1079 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
1080 if (is_equal_rsn(cqe64
, rsn
)) {
1081 if (srq
&& (ntohl(cqe64
->srqn
) & 0xffffff))
1082 mlx5_ib_free_srq_wqe(srq
, be16_to_cpu(cqe64
->wqe_counter
));
1084 } else if (nfreed
) {
1085 dest
= get_cqe(cq
, (prod_index
+ nfreed
) & cq
->ibcq
.cqe
);
1086 dest64
= (cq
->mcq
.cqe_sz
== 64) ? dest
: dest
+ 64;
1087 owner_bit
= dest64
->op_own
& MLX5_CQE_OWNER_MASK
;
1088 memcpy(dest
, cqe
, cq
->mcq
.cqe_sz
);
1089 dest64
->op_own
= owner_bit
|
1090 (dest64
->op_own
& ~MLX5_CQE_OWNER_MASK
);
1095 cq
->mcq
.cons_index
+= nfreed
;
1096 /* Make sure update of buffer contents is done before
1097 * updating consumer index.
1100 mlx5_cq_set_ci(&cq
->mcq
);
1104 void mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 qpn
, struct mlx5_ib_srq
*srq
)
1109 spin_lock_irq(&cq
->lock
);
1110 __mlx5_ib_cq_clean(cq
, qpn
, srq
);
1111 spin_unlock_irq(&cq
->lock
);
1114 int mlx5_ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
1116 struct mlx5_ib_dev
*dev
= to_mdev(cq
->device
);
1117 struct mlx5_ib_cq
*mcq
= to_mcq(cq
);
1120 if (!MLX5_CAP_GEN(dev
->mdev
, cq_moderation
))
1123 err
= mlx5_core_modify_cq_moderation(dev
->mdev
, &mcq
->mcq
,
1124 cq_period
, cq_count
);
1126 mlx5_ib_warn(dev
, "modify cq 0x%x failed\n", mcq
->mcq
.cqn
);
1131 static int resize_user(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
1132 int entries
, struct ib_udata
*udata
, int *npas
,
1133 int *page_shift
, int *cqe_size
)
1135 struct mlx5_ib_resize_cq ucmd
;
1136 struct ib_umem
*umem
;
1139 struct ib_ucontext
*context
= cq
->buf
.umem
->context
;
1141 err
= ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
));
1145 if (ucmd
.reserved0
|| ucmd
.reserved1
)
1148 umem
= ib_umem_get(context
, ucmd
.buf_addr
, entries
* ucmd
.cqe_size
,
1149 IB_ACCESS_LOCAL_WRITE
, 1);
1151 err
= PTR_ERR(umem
);
1155 mlx5_ib_cont_pages(umem
, ucmd
.buf_addr
, 0, &npages
, page_shift
,
1158 cq
->resize_umem
= umem
;
1159 *cqe_size
= ucmd
.cqe_size
;
1164 static void un_resize_user(struct mlx5_ib_cq
*cq
)
1166 ib_umem_release(cq
->resize_umem
);
1169 static int resize_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
1170 int entries
, int cqe_size
)
1174 cq
->resize_buf
= kzalloc(sizeof(*cq
->resize_buf
), GFP_KERNEL
);
1175 if (!cq
->resize_buf
)
1178 err
= alloc_cq_buf(dev
, cq
->resize_buf
, entries
, cqe_size
);
1182 init_cq_buf(cq
, cq
->resize_buf
);
1187 kfree(cq
->resize_buf
);
1191 static void un_resize_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
)
1193 free_cq_buf(dev
, cq
->resize_buf
);
1194 cq
->resize_buf
= NULL
;
1197 static int copy_resize_cqes(struct mlx5_ib_cq
*cq
)
1199 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
1200 struct mlx5_cqe64
*scqe64
;
1201 struct mlx5_cqe64
*dcqe64
;
1210 ssize
= cq
->buf
.cqe_size
;
1211 dsize
= cq
->resize_buf
->cqe_size
;
1212 if (ssize
!= dsize
) {
1213 mlx5_ib_warn(dev
, "resize from different cqe size is not supported\n");
1217 i
= cq
->mcq
.cons_index
;
1218 scqe
= get_sw_cqe(cq
, i
);
1219 scqe64
= ssize
== 64 ? scqe
: scqe
+ 64;
1222 mlx5_ib_warn(dev
, "expected cqe in sw ownership\n");
1226 while ((scqe64
->op_own
>> 4) != MLX5_CQE_RESIZE_CQ
) {
1227 dcqe
= get_cqe_from_buf(cq
->resize_buf
,
1228 (i
+ 1) & (cq
->resize_buf
->nent
),
1230 dcqe64
= dsize
== 64 ? dcqe
: dcqe
+ 64;
1231 sw_own
= sw_ownership_bit(i
+ 1, cq
->resize_buf
->nent
);
1232 memcpy(dcqe
, scqe
, dsize
);
1233 dcqe64
->op_own
= (dcqe64
->op_own
& ~MLX5_CQE_OWNER_MASK
) | sw_own
;
1236 scqe
= get_sw_cqe(cq
, i
);
1237 scqe64
= ssize
== 64 ? scqe
: scqe
+ 64;
1239 mlx5_ib_warn(dev
, "expected cqe in sw ownership\n");
1243 if (scqe
== start_cqe
) {
1244 pr_warn("resize CQ failed to get resize CQE, CQN 0x%x\n",
1249 ++cq
->mcq
.cons_index
;
1253 int mlx5_ib_resize_cq(struct ib_cq
*ibcq
, int entries
, struct ib_udata
*udata
)
1255 struct mlx5_ib_dev
*dev
= to_mdev(ibcq
->device
);
1256 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
1264 int uninitialized_var(cqe_size
);
1265 unsigned long flags
;
1267 if (!MLX5_CAP_GEN(dev
->mdev
, cq_resize
)) {
1268 pr_info("Firmware does not support resize CQ\n");
1273 entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
))) {
1274 mlx5_ib_warn(dev
, "wrong entries number %d, max %d\n",
1276 1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
));
1280 entries
= roundup_pow_of_two(entries
+ 1);
1281 if (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
)) + 1)
1284 if (entries
== ibcq
->cqe
+ 1)
1287 mutex_lock(&cq
->resize_mutex
);
1289 err
= resize_user(dev
, cq
, entries
, udata
, &npas
, &page_shift
,
1293 err
= resize_kernel(dev
, cq
, entries
, cqe_size
);
1295 npas
= cq
->resize_buf
->buf
.npages
;
1296 page_shift
= cq
->resize_buf
->buf
.page_shift
;
1303 inlen
= MLX5_ST_SZ_BYTES(modify_cq_in
) +
1304 MLX5_FLD_SZ_BYTES(modify_cq_in
, pas
[0]) * npas
;
1306 in
= mlx5_vzalloc(inlen
);
1312 pas
= (__be64
*)MLX5_ADDR_OF(modify_cq_in
, in
, pas
);
1314 mlx5_ib_populate_pas(dev
, cq
->resize_umem
, page_shift
,
1317 mlx5_fill_page_array(&cq
->resize_buf
->buf
, pas
);
1319 MLX5_SET(modify_cq_in
, in
,
1320 modify_field_select_resize_field_select
.resize_field_select
.resize_field_select
,
1321 MLX5_MODIFY_CQ_MASK_LOG_SIZE
|
1322 MLX5_MODIFY_CQ_MASK_PG_OFFSET
|
1323 MLX5_MODIFY_CQ_MASK_PG_SIZE
);
1325 cqc
= MLX5_ADDR_OF(modify_cq_in
, in
, cq_context
);
1327 MLX5_SET(cqc
, cqc
, log_page_size
,
1328 page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
1329 MLX5_SET(cqc
, cqc
, cqe_sz
, cqe_sz_to_mlx_sz(cqe_size
));
1330 MLX5_SET(cqc
, cqc
, log_cq_size
, ilog2(entries
));
1332 MLX5_SET(modify_cq_in
, in
, op_mod
, MLX5_CQ_OPMOD_RESIZE
);
1333 MLX5_SET(modify_cq_in
, in
, cqn
, cq
->mcq
.cqn
);
1335 err
= mlx5_core_modify_cq(dev
->mdev
, &cq
->mcq
, in
, inlen
);
1340 cq
->ibcq
.cqe
= entries
- 1;
1341 ib_umem_release(cq
->buf
.umem
);
1342 cq
->buf
.umem
= cq
->resize_umem
;
1343 cq
->resize_umem
= NULL
;
1345 struct mlx5_ib_cq_buf tbuf
;
1348 spin_lock_irqsave(&cq
->lock
, flags
);
1349 if (cq
->resize_buf
) {
1350 err
= copy_resize_cqes(cq
);
1353 cq
->buf
= *cq
->resize_buf
;
1354 kfree(cq
->resize_buf
);
1355 cq
->resize_buf
= NULL
;
1359 cq
->ibcq
.cqe
= entries
- 1;
1360 spin_unlock_irqrestore(&cq
->lock
, flags
);
1362 free_cq_buf(dev
, &tbuf
);
1364 mutex_unlock(&cq
->resize_mutex
);
1376 un_resize_kernel(dev
, cq
);
1378 mutex_unlock(&cq
->resize_mutex
);
1382 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev
*dev
, struct ib_cq
*ibcq
)
1384 struct mlx5_ib_cq
*cq
;
1390 return cq
->cqe_size
;
1393 /* Called from atomic context */
1394 int mlx5_ib_generate_wc(struct ib_cq
*ibcq
, struct ib_wc
*wc
)
1396 struct mlx5_ib_wc
*soft_wc
;
1397 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
1398 unsigned long flags
;
1400 soft_wc
= kmalloc(sizeof(*soft_wc
), GFP_ATOMIC
);
1405 spin_lock_irqsave(&cq
->lock
, flags
);
1406 list_add_tail(&soft_wc
->list
, &cq
->wc_list
);
1407 if (cq
->notify_flags
== IB_CQ_NEXT_COMP
||
1408 wc
->status
!= IB_WC_SUCCESS
) {
1409 cq
->notify_flags
= 0;
1410 schedule_work(&cq
->notify_work
);
1412 spin_unlock_irqrestore(&cq
->lock
, flags
);