2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kref.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_user_verbs.h>
36 #include <rdma/ib_cache.h>
39 static void mlx5_ib_cq_comp(struct mlx5_core_cq
*cq
)
41 struct ib_cq
*ibcq
= &to_mibcq(cq
)->ibcq
;
43 ibcq
->comp_handler(ibcq
, ibcq
->cq_context
);
46 static void mlx5_ib_cq_event(struct mlx5_core_cq
*mcq
, enum mlx5_event type
)
48 struct mlx5_ib_cq
*cq
= container_of(mcq
, struct mlx5_ib_cq
, mcq
);
49 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
50 struct ib_cq
*ibcq
= &cq
->ibcq
;
51 struct ib_event event
;
53 if (type
!= MLX5_EVENT_TYPE_CQ_ERROR
) {
54 mlx5_ib_warn(dev
, "Unexpected event type %d on CQ %06x\n",
59 if (ibcq
->event_handler
) {
60 event
.device
= &dev
->ib_dev
;
61 event
.event
= IB_EVENT_CQ_ERR
;
62 event
.element
.cq
= ibcq
;
63 ibcq
->event_handler(&event
, ibcq
->cq_context
);
67 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf
*buf
, int n
, int size
)
69 return mlx5_buf_offset(&buf
->buf
, n
* size
);
72 static void *get_cqe(struct mlx5_ib_cq
*cq
, int n
)
74 return get_cqe_from_buf(&cq
->buf
, n
, cq
->mcq
.cqe_sz
);
77 static u8
sw_ownership_bit(int n
, int nent
)
79 return (n
& nent
) ? 1 : 0;
82 static void *get_sw_cqe(struct mlx5_ib_cq
*cq
, int n
)
84 void *cqe
= get_cqe(cq
, n
& cq
->ibcq
.cqe
);
85 struct mlx5_cqe64
*cqe64
;
87 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
89 if (likely((cqe64
->op_own
) >> 4 != MLX5_CQE_INVALID
) &&
90 !((cqe64
->op_own
& MLX5_CQE_OWNER_MASK
) ^ !!(n
& (cq
->ibcq
.cqe
+ 1)))) {
97 static void *next_cqe_sw(struct mlx5_ib_cq
*cq
)
99 return get_sw_cqe(cq
, cq
->mcq
.cons_index
);
102 static enum ib_wc_opcode
get_umr_comp(struct mlx5_ib_wq
*wq
, int idx
)
104 switch (wq
->wr_data
[idx
]) {
108 case IB_WR_LOCAL_INV
:
109 return IB_WC_LOCAL_INV
;
115 pr_warn("unknown completion status\n");
120 static void handle_good_req(struct ib_wc
*wc
, struct mlx5_cqe64
*cqe
,
121 struct mlx5_ib_wq
*wq
, int idx
)
124 switch (be32_to_cpu(cqe
->sop_drop_qpn
) >> 24) {
125 case MLX5_OPCODE_RDMA_WRITE_IMM
:
126 wc
->wc_flags
|= IB_WC_WITH_IMM
;
127 case MLX5_OPCODE_RDMA_WRITE
:
128 wc
->opcode
= IB_WC_RDMA_WRITE
;
130 case MLX5_OPCODE_SEND_IMM
:
131 wc
->wc_flags
|= IB_WC_WITH_IMM
;
132 case MLX5_OPCODE_SEND
:
133 case MLX5_OPCODE_SEND_INVAL
:
134 wc
->opcode
= IB_WC_SEND
;
136 case MLX5_OPCODE_RDMA_READ
:
137 wc
->opcode
= IB_WC_RDMA_READ
;
138 wc
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
140 case MLX5_OPCODE_ATOMIC_CS
:
141 wc
->opcode
= IB_WC_COMP_SWAP
;
144 case MLX5_OPCODE_ATOMIC_FA
:
145 wc
->opcode
= IB_WC_FETCH_ADD
;
148 case MLX5_OPCODE_ATOMIC_MASKED_CS
:
149 wc
->opcode
= IB_WC_MASKED_COMP_SWAP
;
152 case MLX5_OPCODE_ATOMIC_MASKED_FA
:
153 wc
->opcode
= IB_WC_MASKED_FETCH_ADD
;
156 case MLX5_OPCODE_UMR
:
157 wc
->opcode
= get_umr_comp(wq
, idx
);
163 MLX5_GRH_IN_BUFFER
= 1,
167 static void handle_responder(struct ib_wc
*wc
, struct mlx5_cqe64
*cqe
,
168 struct mlx5_ib_qp
*qp
)
170 enum rdma_link_layer ll
= rdma_port_get_link_layer(qp
->ibqp
.device
, 1);
171 struct mlx5_ib_dev
*dev
= to_mdev(qp
->ibqp
.device
);
172 struct mlx5_ib_srq
*srq
;
173 struct mlx5_ib_wq
*wq
;
177 if (qp
->ibqp
.srq
|| qp
->ibqp
.xrcd
) {
178 struct mlx5_core_srq
*msrq
= NULL
;
181 msrq
= mlx5_core_get_srq(dev
->mdev
,
182 be32_to_cpu(cqe
->srqn
));
183 srq
= to_mibsrq(msrq
);
185 srq
= to_msrq(qp
->ibqp
.srq
);
188 wqe_ctr
= be16_to_cpu(cqe
->wqe_counter
);
189 wc
->wr_id
= srq
->wrid
[wqe_ctr
];
190 mlx5_ib_free_srq_wqe(srq
, wqe_ctr
);
191 if (msrq
&& atomic_dec_and_test(&msrq
->refcount
))
192 complete(&msrq
->free
);
196 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
199 wc
->byte_len
= be32_to_cpu(cqe
->byte_cnt
);
201 switch (cqe
->op_own
>> 4) {
202 case MLX5_CQE_RESP_WR_IMM
:
203 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
204 wc
->wc_flags
= IB_WC_WITH_IMM
;
205 wc
->ex
.imm_data
= cqe
->imm_inval_pkey
;
207 case MLX5_CQE_RESP_SEND
:
208 wc
->opcode
= IB_WC_RECV
;
209 wc
->wc_flags
= IB_WC_IP_CSUM_OK
;
210 if (unlikely(!((cqe
->hds_ip_ext
& CQE_L3_OK
) &&
211 (cqe
->hds_ip_ext
& CQE_L4_OK
))))
214 case MLX5_CQE_RESP_SEND_IMM
:
215 wc
->opcode
= IB_WC_RECV
;
216 wc
->wc_flags
= IB_WC_WITH_IMM
;
217 wc
->ex
.imm_data
= cqe
->imm_inval_pkey
;
219 case MLX5_CQE_RESP_SEND_INV
:
220 wc
->opcode
= IB_WC_RECV
;
221 wc
->wc_flags
= IB_WC_WITH_INVALIDATE
;
222 wc
->ex
.invalidate_rkey
= be32_to_cpu(cqe
->imm_inval_pkey
);
225 wc
->slid
= be16_to_cpu(cqe
->slid
);
226 wc
->sl
= (be32_to_cpu(cqe
->flags_rqpn
) >> 24) & 0xf;
227 wc
->src_qp
= be32_to_cpu(cqe
->flags_rqpn
) & 0xffffff;
228 wc
->dlid_path_bits
= cqe
->ml_path
;
229 g
= (be32_to_cpu(cqe
->flags_rqpn
) >> 28) & 3;
230 wc
->wc_flags
|= g
? IB_WC_GRH
: 0;
231 if (unlikely(is_qp1(qp
->ibqp
.qp_type
))) {
232 u16 pkey
= be32_to_cpu(cqe
->imm_inval_pkey
) & 0xffff;
234 ib_find_cached_pkey(&dev
->ib_dev
, qp
->port
, pkey
,
240 if (ll
!= IB_LINK_LAYER_ETHERNET
)
243 switch (wc
->sl
& 0x3) {
244 case MLX5_CQE_ROCE_L3_HEADER_TYPE_GRH
:
245 wc
->network_hdr_type
= RDMA_NETWORK_IB
;
247 case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV6
:
248 wc
->network_hdr_type
= RDMA_NETWORK_IPV6
;
250 case MLX5_CQE_ROCE_L3_HEADER_TYPE_IPV4
:
251 wc
->network_hdr_type
= RDMA_NETWORK_IPV4
;
254 wc
->wc_flags
|= IB_WC_WITH_NETWORK_HDR_TYPE
;
257 static void dump_cqe(struct mlx5_ib_dev
*dev
, struct mlx5_err_cqe
*cqe
)
259 __be32
*p
= (__be32
*)cqe
;
262 mlx5_ib_warn(dev
, "dump error cqe\n");
263 for (i
= 0; i
< sizeof(*cqe
) / 16; i
++, p
+= 4)
264 pr_info("%08x %08x %08x %08x\n", be32_to_cpu(p
[0]),
265 be32_to_cpu(p
[1]), be32_to_cpu(p
[2]),
269 static void mlx5_handle_error_cqe(struct mlx5_ib_dev
*dev
,
270 struct mlx5_err_cqe
*cqe
,
275 switch (cqe
->syndrome
) {
276 case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR
:
277 wc
->status
= IB_WC_LOC_LEN_ERR
;
279 case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR
:
280 wc
->status
= IB_WC_LOC_QP_OP_ERR
;
282 case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR
:
283 wc
->status
= IB_WC_LOC_PROT_ERR
;
285 case MLX5_CQE_SYNDROME_WR_FLUSH_ERR
:
287 wc
->status
= IB_WC_WR_FLUSH_ERR
;
289 case MLX5_CQE_SYNDROME_MW_BIND_ERR
:
290 wc
->status
= IB_WC_MW_BIND_ERR
;
292 case MLX5_CQE_SYNDROME_BAD_RESP_ERR
:
293 wc
->status
= IB_WC_BAD_RESP_ERR
;
295 case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR
:
296 wc
->status
= IB_WC_LOC_ACCESS_ERR
;
298 case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR
:
299 wc
->status
= IB_WC_REM_INV_REQ_ERR
;
301 case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR
:
302 wc
->status
= IB_WC_REM_ACCESS_ERR
;
304 case MLX5_CQE_SYNDROME_REMOTE_OP_ERR
:
305 wc
->status
= IB_WC_REM_OP_ERR
;
307 case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR
:
308 wc
->status
= IB_WC_RETRY_EXC_ERR
;
311 case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR
:
312 wc
->status
= IB_WC_RNR_RETRY_EXC_ERR
;
315 case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR
:
316 wc
->status
= IB_WC_REM_ABORT_ERR
;
319 wc
->status
= IB_WC_GENERAL_ERR
;
323 wc
->vendor_err
= cqe
->vendor_err_synd
;
328 static int is_atomic_response(struct mlx5_ib_qp
*qp
, uint16_t idx
)
330 /* TBD: waiting decision
335 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp
*qp
, uint16_t idx
)
337 struct mlx5_wqe_data_seg
*dpseg
;
340 dpseg
= mlx5_get_send_wqe(qp
, idx
) + sizeof(struct mlx5_wqe_ctrl_seg
) +
341 sizeof(struct mlx5_wqe_raddr_seg
) +
342 sizeof(struct mlx5_wqe_atomic_seg
);
343 addr
= (void *)(unsigned long)be64_to_cpu(dpseg
->addr
);
347 static void handle_atomic(struct mlx5_ib_qp
*qp
, struct mlx5_cqe64
*cqe64
,
354 if (!is_atomic_response(qp
, idx
))
357 byte_count
= be32_to_cpu(cqe64
->byte_cnt
);
358 addr
= mlx5_get_atomic_laddr(qp
, idx
);
360 if (byte_count
== 4) {
361 *(uint32_t *)addr
= be32_to_cpu(*((__be32
*)addr
));
363 for (i
= 0; i
< byte_count
; i
+= 8) {
364 *(uint64_t *)addr
= be64_to_cpu(*((__be64
*)addr
));
372 static void handle_atomics(struct mlx5_ib_qp
*qp
, struct mlx5_cqe64
*cqe64
,
378 idx
= tail
& (qp
->sq
.wqe_cnt
- 1);
379 handle_atomic(qp
, cqe64
, idx
);
383 tail
= qp
->sq
.w_list
[idx
].next
;
385 tail
= qp
->sq
.w_list
[idx
].next
;
386 qp
->sq
.last_poll
= tail
;
389 static void free_cq_buf(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq_buf
*buf
)
391 mlx5_buf_free(dev
->mdev
, &buf
->buf
);
394 static void get_sig_err_item(struct mlx5_sig_err_cqe
*cqe
,
395 struct ib_sig_err
*item
)
397 u16 syndrome
= be16_to_cpu(cqe
->syndrome
);
399 #define GUARD_ERR (1 << 13)
400 #define APPTAG_ERR (1 << 12)
401 #define REFTAG_ERR (1 << 11)
403 if (syndrome
& GUARD_ERR
) {
404 item
->err_type
= IB_SIG_BAD_GUARD
;
405 item
->expected
= be32_to_cpu(cqe
->expected_trans_sig
) >> 16;
406 item
->actual
= be32_to_cpu(cqe
->actual_trans_sig
) >> 16;
408 if (syndrome
& REFTAG_ERR
) {
409 item
->err_type
= IB_SIG_BAD_REFTAG
;
410 item
->expected
= be32_to_cpu(cqe
->expected_reftag
);
411 item
->actual
= be32_to_cpu(cqe
->actual_reftag
);
413 if (syndrome
& APPTAG_ERR
) {
414 item
->err_type
= IB_SIG_BAD_APPTAG
;
415 item
->expected
= be32_to_cpu(cqe
->expected_trans_sig
) & 0xffff;
416 item
->actual
= be32_to_cpu(cqe
->actual_trans_sig
) & 0xffff;
418 pr_err("Got signature completion error with bad syndrome %04x\n",
422 item
->sig_err_offset
= be64_to_cpu(cqe
->err_offset
);
423 item
->key
= be32_to_cpu(cqe
->mkey
);
426 static void sw_send_comp(struct mlx5_ib_qp
*qp
, int num_entries
,
427 struct ib_wc
*wc
, int *npolled
)
429 struct mlx5_ib_wq
*wq
;
436 cur
= wq
->head
- wq
->tail
;
442 for (i
= 0; i
< cur
&& np
< num_entries
; i
++) {
443 idx
= wq
->last_poll
& (wq
->wqe_cnt
- 1);
444 wc
->wr_id
= wq
->wrid
[idx
];
445 wc
->status
= IB_WC_WR_FLUSH_ERR
;
446 wc
->vendor_err
= MLX5_CQE_SYNDROME_WR_FLUSH_ERR
;
451 wq
->last_poll
= wq
->w_list
[idx
].next
;
456 static void sw_recv_comp(struct mlx5_ib_qp
*qp
, int num_entries
,
457 struct ib_wc
*wc
, int *npolled
)
459 struct mlx5_ib_wq
*wq
;
465 cur
= wq
->head
- wq
->tail
;
471 for (i
= 0; i
< cur
&& np
< num_entries
; i
++) {
472 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
473 wc
->status
= IB_WC_WR_FLUSH_ERR
;
474 wc
->vendor_err
= MLX5_CQE_SYNDROME_WR_FLUSH_ERR
;
483 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq
*cq
, int num_entries
,
484 struct ib_wc
*wc
, int *npolled
)
486 struct mlx5_ib_qp
*qp
;
489 /* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
490 list_for_each_entry(qp
, &cq
->list_send_qp
, cq_send_list
) {
491 sw_send_comp(qp
, num_entries
, wc
+ *npolled
, npolled
);
492 if (*npolled
>= num_entries
)
496 list_for_each_entry(qp
, &cq
->list_recv_qp
, cq_recv_list
) {
497 sw_recv_comp(qp
, num_entries
, wc
+ *npolled
, npolled
);
498 if (*npolled
>= num_entries
)
503 static int mlx5_poll_one(struct mlx5_ib_cq
*cq
,
504 struct mlx5_ib_qp
**cur_qp
,
507 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
508 struct mlx5_err_cqe
*err_cqe
;
509 struct mlx5_cqe64
*cqe64
;
510 struct mlx5_core_qp
*mqp
;
511 struct mlx5_ib_wq
*wq
;
512 struct mlx5_sig_err_cqe
*sig_err_cqe
;
513 struct mlx5_core_mkey
*mmkey
;
514 struct mlx5_ib_mr
*mr
;
522 cqe
= next_cqe_sw(cq
);
526 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
528 ++cq
->mcq
.cons_index
;
530 /* Make sure we read CQ entry contents after we've checked the
535 opcode
= cqe64
->op_own
>> 4;
536 if (unlikely(opcode
== MLX5_CQE_RESIZE_CQ
)) {
537 if (likely(cq
->resize_buf
)) {
538 free_cq_buf(dev
, &cq
->buf
);
539 cq
->buf
= *cq
->resize_buf
;
540 kfree(cq
->resize_buf
);
541 cq
->resize_buf
= NULL
;
544 mlx5_ib_warn(dev
, "unexpected resize cqe\n");
548 qpn
= ntohl(cqe64
->sop_drop_qpn
) & 0xffffff;
549 if (!*cur_qp
|| (qpn
!= (*cur_qp
)->ibqp
.qp_num
)) {
550 /* We do not have to take the QP table lock here,
551 * because CQs will be locked while QPs are removed
554 mqp
= __mlx5_qp_lookup(dev
->mdev
, qpn
);
555 *cur_qp
= to_mibqp(mqp
);
558 wc
->qp
= &(*cur_qp
)->ibqp
;
562 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
563 idx
= wqe_ctr
& (wq
->wqe_cnt
- 1);
564 handle_good_req(wc
, cqe64
, wq
, idx
);
565 handle_atomics(*cur_qp
, cqe64
, wq
->last_poll
, idx
);
566 wc
->wr_id
= wq
->wrid
[idx
];
567 wq
->tail
= wq
->wqe_head
[idx
] + 1;
568 wc
->status
= IB_WC_SUCCESS
;
570 case MLX5_CQE_RESP_WR_IMM
:
571 case MLX5_CQE_RESP_SEND
:
572 case MLX5_CQE_RESP_SEND_IMM
:
573 case MLX5_CQE_RESP_SEND_INV
:
574 handle_responder(wc
, cqe64
, *cur_qp
);
575 wc
->status
= IB_WC_SUCCESS
;
577 case MLX5_CQE_RESIZE_CQ
:
579 case MLX5_CQE_REQ_ERR
:
580 case MLX5_CQE_RESP_ERR
:
581 err_cqe
= (struct mlx5_err_cqe
*)cqe64
;
582 mlx5_handle_error_cqe(dev
, err_cqe
, wc
);
583 mlx5_ib_dbg(dev
, "%s error cqe on cqn 0x%x:\n",
584 opcode
== MLX5_CQE_REQ_ERR
?
585 "Requestor" : "Responder", cq
->mcq
.cqn
);
586 mlx5_ib_dbg(dev
, "syndrome 0x%x, vendor syndrome 0x%x\n",
587 err_cqe
->syndrome
, err_cqe
->vendor_err_synd
);
588 if (opcode
== MLX5_CQE_REQ_ERR
) {
590 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
591 idx
= wqe_ctr
& (wq
->wqe_cnt
- 1);
592 wc
->wr_id
= wq
->wrid
[idx
];
593 wq
->tail
= wq
->wqe_head
[idx
] + 1;
595 struct mlx5_ib_srq
*srq
;
597 if ((*cur_qp
)->ibqp
.srq
) {
598 srq
= to_msrq((*cur_qp
)->ibqp
.srq
);
599 wqe_ctr
= be16_to_cpu(cqe64
->wqe_counter
);
600 wc
->wr_id
= srq
->wrid
[wqe_ctr
];
601 mlx5_ib_free_srq_wqe(srq
, wqe_ctr
);
604 wc
->wr_id
= wq
->wrid
[wq
->tail
& (wq
->wqe_cnt
- 1)];
609 case MLX5_CQE_SIG_ERR
:
610 sig_err_cqe
= (struct mlx5_sig_err_cqe
*)cqe64
;
612 read_lock(&dev
->mdev
->priv
.mkey_table
.lock
);
613 mmkey
= __mlx5_mr_lookup(dev
->mdev
,
614 mlx5_base_mkey(be32_to_cpu(sig_err_cqe
->mkey
)));
615 mr
= to_mibmr(mmkey
);
616 get_sig_err_item(sig_err_cqe
, &mr
->sig
->err_item
);
617 mr
->sig
->sig_err_exists
= true;
618 mr
->sig
->sigerr_count
++;
620 mlx5_ib_warn(dev
, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",
621 cq
->mcq
.cqn
, mr
->sig
->err_item
.key
,
622 mr
->sig
->err_item
.err_type
,
623 mr
->sig
->err_item
.sig_err_offset
,
624 mr
->sig
->err_item
.expected
,
625 mr
->sig
->err_item
.actual
);
627 read_unlock(&dev
->mdev
->priv
.mkey_table
.lock
);
634 static int poll_soft_wc(struct mlx5_ib_cq
*cq
, int num_entries
,
637 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
638 struct mlx5_ib_wc
*soft_wc
, *next
;
641 list_for_each_entry_safe(soft_wc
, next
, &cq
->wc_list
, list
) {
642 if (npolled
>= num_entries
)
645 mlx5_ib_dbg(dev
, "polled software generated completion on CQ 0x%x\n",
648 wc
[npolled
++] = soft_wc
->wc
;
649 list_del(&soft_wc
->list
);
656 int mlx5_ib_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
)
658 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
659 struct mlx5_ib_qp
*cur_qp
= NULL
;
660 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
661 struct mlx5_core_dev
*mdev
= dev
->mdev
;
666 spin_lock_irqsave(&cq
->lock
, flags
);
667 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
) {
668 mlx5_ib_poll_sw_comp(cq
, num_entries
, wc
, &npolled
);
672 if (unlikely(!list_empty(&cq
->wc_list
)))
673 soft_polled
= poll_soft_wc(cq
, num_entries
, wc
);
675 for (npolled
= 0; npolled
< num_entries
- soft_polled
; npolled
++) {
676 if (mlx5_poll_one(cq
, &cur_qp
, wc
+ soft_polled
+ npolled
))
681 mlx5_cq_set_ci(&cq
->mcq
);
683 spin_unlock_irqrestore(&cq
->lock
, flags
);
685 return soft_polled
+ npolled
;
688 int mlx5_ib_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
)
690 struct mlx5_core_dev
*mdev
= to_mdev(ibcq
->device
)->mdev
;
691 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
692 void __iomem
*uar_page
= mdev
->priv
.uuari
.uars
[0].map
;
693 unsigned long irq_flags
;
696 spin_lock_irqsave(&cq
->lock
, irq_flags
);
697 if (cq
->notify_flags
!= IB_CQ_NEXT_COMP
)
698 cq
->notify_flags
= flags
& IB_CQ_SOLICITED_MASK
;
700 if ((flags
& IB_CQ_REPORT_MISSED_EVENTS
) && !list_empty(&cq
->wc_list
))
702 spin_unlock_irqrestore(&cq
->lock
, irq_flags
);
704 mlx5_cq_arm(&cq
->mcq
,
705 (flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
?
706 MLX5_CQ_DB_REQ_NOT_SOL
: MLX5_CQ_DB_REQ_NOT
,
708 MLX5_GET_DOORBELL_LOCK(&mdev
->priv
.cq_uar_lock
),
709 to_mcq(ibcq
)->mcq
.cons_index
);
714 static int alloc_cq_buf(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq_buf
*buf
,
715 int nent
, int cqe_size
)
719 err
= mlx5_buf_alloc(dev
->mdev
, nent
* cqe_size
, &buf
->buf
);
723 buf
->cqe_size
= cqe_size
;
729 static int create_cq_user(struct mlx5_ib_dev
*dev
, struct ib_udata
*udata
,
730 struct ib_ucontext
*context
, struct mlx5_ib_cq
*cq
,
731 int entries
, u32
**cqb
,
732 int *cqe_size
, int *index
, int *inlen
)
734 struct mlx5_ib_create_cq ucmd
;
744 (udata
->inlen
- sizeof(struct ib_uverbs_cmd_hdr
) <
745 sizeof(ucmd
)) ? (sizeof(ucmd
) -
746 sizeof(ucmd
.reserved
)) : sizeof(ucmd
);
748 if (ib_copy_from_udata(&ucmd
, udata
, ucmdlen
))
751 if (ucmdlen
== sizeof(ucmd
) &&
755 if (ucmd
.cqe_size
!= 64 && ucmd
.cqe_size
!= 128)
758 *cqe_size
= ucmd
.cqe_size
;
760 cq
->buf
.umem
= ib_umem_get(context
, ucmd
.buf_addr
,
761 entries
* ucmd
.cqe_size
,
762 IB_ACCESS_LOCAL_WRITE
, 1);
763 if (IS_ERR(cq
->buf
.umem
)) {
764 err
= PTR_ERR(cq
->buf
.umem
);
768 err
= mlx5_ib_db_map_user(to_mucontext(context
), ucmd
.db_addr
,
773 mlx5_ib_cont_pages(cq
->buf
.umem
, ucmd
.buf_addr
, &npages
, &page_shift
,
775 mlx5_ib_dbg(dev
, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
776 ucmd
.buf_addr
, entries
* ucmd
.cqe_size
, npages
, page_shift
, ncont
);
778 *inlen
= MLX5_ST_SZ_BYTES(create_cq_in
) +
779 MLX5_FLD_SZ_BYTES(create_cq_in
, pas
[0]) * ncont
;
780 *cqb
= mlx5_vzalloc(*inlen
);
786 pas
= (__be64
*)MLX5_ADDR_OF(create_cq_in
, *cqb
, pas
);
787 mlx5_ib_populate_pas(dev
, cq
->buf
.umem
, page_shift
, pas
, 0);
789 cqc
= MLX5_ADDR_OF(create_cq_in
, *cqb
, cq_context
);
790 MLX5_SET(cqc
, cqc
, log_page_size
,
791 page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
793 *index
= to_mucontext(context
)->uuari
.uars
[0].index
;
798 mlx5_ib_db_unmap_user(to_mucontext(context
), &cq
->db
);
801 ib_umem_release(cq
->buf
.umem
);
805 static void destroy_cq_user(struct mlx5_ib_cq
*cq
, struct ib_ucontext
*context
)
807 mlx5_ib_db_unmap_user(to_mucontext(context
), &cq
->db
);
808 ib_umem_release(cq
->buf
.umem
);
811 static void init_cq_buf(struct mlx5_ib_cq
*cq
, struct mlx5_ib_cq_buf
*buf
)
815 struct mlx5_cqe64
*cqe64
;
817 for (i
= 0; i
< buf
->nent
; i
++) {
818 cqe
= get_cqe_from_buf(buf
, i
, buf
->cqe_size
);
819 cqe64
= buf
->cqe_size
== 64 ? cqe
: cqe
+ 64;
820 cqe64
->op_own
= MLX5_CQE_INVALID
<< 4;
824 static int create_cq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
825 int entries
, int cqe_size
,
826 u32
**cqb
, int *index
, int *inlen
)
832 err
= mlx5_db_alloc(dev
->mdev
, &cq
->db
);
836 cq
->mcq
.set_ci_db
= cq
->db
.db
;
837 cq
->mcq
.arm_db
= cq
->db
.db
+ 1;
838 cq
->mcq
.cqe_sz
= cqe_size
;
840 err
= alloc_cq_buf(dev
, &cq
->buf
, entries
, cqe_size
);
844 init_cq_buf(cq
, &cq
->buf
);
846 *inlen
= MLX5_ST_SZ_BYTES(create_cq_in
) +
847 MLX5_FLD_SZ_BYTES(create_cq_in
, pas
[0]) * cq
->buf
.buf
.npages
;
848 *cqb
= mlx5_vzalloc(*inlen
);
854 pas
= (__be64
*)MLX5_ADDR_OF(create_cq_in
, *cqb
, pas
);
855 mlx5_fill_page_array(&cq
->buf
.buf
, pas
);
857 cqc
= MLX5_ADDR_OF(create_cq_in
, *cqb
, cq_context
);
858 MLX5_SET(cqc
, cqc
, log_page_size
,
859 cq
->buf
.buf
.page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
861 *index
= dev
->mdev
->priv
.uuari
.uars
[0].index
;
866 free_cq_buf(dev
, &cq
->buf
);
869 mlx5_db_free(dev
->mdev
, &cq
->db
);
873 static void destroy_cq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
)
875 free_cq_buf(dev
, &cq
->buf
);
876 mlx5_db_free(dev
->mdev
, &cq
->db
);
879 static void notify_soft_wc_handler(struct work_struct
*work
)
881 struct mlx5_ib_cq
*cq
= container_of(work
, struct mlx5_ib_cq
,
884 cq
->ibcq
.comp_handler(&cq
->ibcq
, cq
->ibcq
.cq_context
);
887 struct ib_cq
*mlx5_ib_create_cq(struct ib_device
*ibdev
,
888 const struct ib_cq_init_attr
*attr
,
889 struct ib_ucontext
*context
,
890 struct ib_udata
*udata
)
892 int entries
= attr
->cqe
;
893 int vector
= attr
->comp_vector
;
894 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
895 struct mlx5_ib_cq
*cq
;
896 int uninitialized_var(index
);
897 int uninitialized_var(inlen
);
906 (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
))))
907 return ERR_PTR(-EINVAL
);
909 if (check_cq_create_flags(attr
->flags
))
910 return ERR_PTR(-EOPNOTSUPP
);
912 entries
= roundup_pow_of_two(entries
+ 1);
913 if (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
)))
914 return ERR_PTR(-EINVAL
);
916 cq
= kzalloc(sizeof(*cq
), GFP_KERNEL
);
918 return ERR_PTR(-ENOMEM
);
920 cq
->ibcq
.cqe
= entries
- 1;
921 mutex_init(&cq
->resize_mutex
);
922 spin_lock_init(&cq
->lock
);
923 cq
->resize_buf
= NULL
;
924 cq
->resize_umem
= NULL
;
925 cq
->create_flags
= attr
->flags
;
926 INIT_LIST_HEAD(&cq
->list_send_qp
);
927 INIT_LIST_HEAD(&cq
->list_recv_qp
);
930 err
= create_cq_user(dev
, udata
, context
, cq
, entries
,
931 &cqb
, &cqe_size
, &index
, &inlen
);
935 /* for now choose 64 bytes till we have a proper interface */
937 err
= create_cq_kernel(dev
, cq
, entries
, cqe_size
, &cqb
,
942 INIT_WORK(&cq
->notify_work
, notify_soft_wc_handler
);
945 err
= mlx5_vector2eqn(dev
->mdev
, vector
, &eqn
, &irqn
);
949 cq
->cqe_size
= cqe_size
;
951 cqc
= MLX5_ADDR_OF(create_cq_in
, cqb
, cq_context
);
952 MLX5_SET(cqc
, cqc
, cqe_sz
, cqe_sz_to_mlx_sz(cqe_size
));
953 MLX5_SET(cqc
, cqc
, log_cq_size
, ilog2(entries
));
954 MLX5_SET(cqc
, cqc
, uar_page
, index
);
955 MLX5_SET(cqc
, cqc
, c_eqn
, eqn
);
956 MLX5_SET64(cqc
, cqc
, dbr_addr
, cq
->db
.dma
);
957 if (cq
->create_flags
& IB_CQ_FLAGS_IGNORE_OVERRUN
)
958 MLX5_SET(cqc
, cqc
, oi
, 1);
960 err
= mlx5_core_create_cq(dev
->mdev
, &cq
->mcq
, cqb
, inlen
);
964 mlx5_ib_dbg(dev
, "cqn 0x%x\n", cq
->mcq
.cqn
);
967 cq
->mcq
.tasklet_ctx
.comp
= mlx5_ib_cq_comp
;
969 cq
->mcq
.comp
= mlx5_ib_cq_comp
;
970 cq
->mcq
.event
= mlx5_ib_cq_event
;
972 INIT_LIST_HEAD(&cq
->wc_list
);
975 if (ib_copy_to_udata(udata
, &cq
->mcq
.cqn
, sizeof(__u32
))) {
985 mlx5_core_destroy_cq(dev
->mdev
, &cq
->mcq
);
990 destroy_cq_user(cq
, context
);
992 destroy_cq_kernel(dev
, cq
);
1001 int mlx5_ib_destroy_cq(struct ib_cq
*cq
)
1003 struct mlx5_ib_dev
*dev
= to_mdev(cq
->device
);
1004 struct mlx5_ib_cq
*mcq
= to_mcq(cq
);
1005 struct ib_ucontext
*context
= NULL
;
1008 context
= cq
->uobject
->context
;
1010 mlx5_core_destroy_cq(dev
->mdev
, &mcq
->mcq
);
1012 destroy_cq_user(mcq
, context
);
1014 destroy_cq_kernel(dev
, mcq
);
1021 static int is_equal_rsn(struct mlx5_cqe64
*cqe64
, u32 rsn
)
1023 return rsn
== (ntohl(cqe64
->sop_drop_qpn
) & 0xffffff);
1026 void __mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 rsn
, struct mlx5_ib_srq
*srq
)
1028 struct mlx5_cqe64
*cqe64
, *dest64
;
1037 /* First we need to find the current producer index, so we
1038 * know where to start cleaning from. It doesn't matter if HW
1039 * adds new entries after this loop -- the QP we're worried
1040 * about is already in RESET, so the new entries won't come
1041 * from our QP and therefore don't need to be checked.
1043 for (prod_index
= cq
->mcq
.cons_index
; get_sw_cqe(cq
, prod_index
); prod_index
++)
1044 if (prod_index
== cq
->mcq
.cons_index
+ cq
->ibcq
.cqe
)
1047 /* Now sweep backwards through the CQ, removing CQ entries
1048 * that match our QP by copying older entries on top of them.
1050 while ((int) --prod_index
- (int) cq
->mcq
.cons_index
>= 0) {
1051 cqe
= get_cqe(cq
, prod_index
& cq
->ibcq
.cqe
);
1052 cqe64
= (cq
->mcq
.cqe_sz
== 64) ? cqe
: cqe
+ 64;
1053 if (is_equal_rsn(cqe64
, rsn
)) {
1054 if (srq
&& (ntohl(cqe64
->srqn
) & 0xffffff))
1055 mlx5_ib_free_srq_wqe(srq
, be16_to_cpu(cqe64
->wqe_counter
));
1057 } else if (nfreed
) {
1058 dest
= get_cqe(cq
, (prod_index
+ nfreed
) & cq
->ibcq
.cqe
);
1059 dest64
= (cq
->mcq
.cqe_sz
== 64) ? dest
: dest
+ 64;
1060 owner_bit
= dest64
->op_own
& MLX5_CQE_OWNER_MASK
;
1061 memcpy(dest
, cqe
, cq
->mcq
.cqe_sz
);
1062 dest64
->op_own
= owner_bit
|
1063 (dest64
->op_own
& ~MLX5_CQE_OWNER_MASK
);
1068 cq
->mcq
.cons_index
+= nfreed
;
1069 /* Make sure update of buffer contents is done before
1070 * updating consumer index.
1073 mlx5_cq_set_ci(&cq
->mcq
);
1077 void mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 qpn
, struct mlx5_ib_srq
*srq
)
1082 spin_lock_irq(&cq
->lock
);
1083 __mlx5_ib_cq_clean(cq
, qpn
, srq
);
1084 spin_unlock_irq(&cq
->lock
);
1087 int mlx5_ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
)
1089 struct mlx5_ib_dev
*dev
= to_mdev(cq
->device
);
1090 struct mlx5_ib_cq
*mcq
= to_mcq(cq
);
1093 if (!MLX5_CAP_GEN(dev
->mdev
, cq_moderation
))
1096 err
= mlx5_core_modify_cq_moderation(dev
->mdev
, &mcq
->mcq
,
1097 cq_period
, cq_count
);
1099 mlx5_ib_warn(dev
, "modify cq 0x%x failed\n", mcq
->mcq
.cqn
);
1104 static int resize_user(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
1105 int entries
, struct ib_udata
*udata
, int *npas
,
1106 int *page_shift
, int *cqe_size
)
1108 struct mlx5_ib_resize_cq ucmd
;
1109 struct ib_umem
*umem
;
1112 struct ib_ucontext
*context
= cq
->buf
.umem
->context
;
1114 err
= ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
));
1118 if (ucmd
.reserved0
|| ucmd
.reserved1
)
1121 umem
= ib_umem_get(context
, ucmd
.buf_addr
, entries
* ucmd
.cqe_size
,
1122 IB_ACCESS_LOCAL_WRITE
, 1);
1124 err
= PTR_ERR(umem
);
1128 mlx5_ib_cont_pages(umem
, ucmd
.buf_addr
, &npages
, page_shift
,
1131 cq
->resize_umem
= umem
;
1132 *cqe_size
= ucmd
.cqe_size
;
1137 static void un_resize_user(struct mlx5_ib_cq
*cq
)
1139 ib_umem_release(cq
->resize_umem
);
1142 static int resize_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
,
1143 int entries
, int cqe_size
)
1147 cq
->resize_buf
= kzalloc(sizeof(*cq
->resize_buf
), GFP_KERNEL
);
1148 if (!cq
->resize_buf
)
1151 err
= alloc_cq_buf(dev
, cq
->resize_buf
, entries
, cqe_size
);
1155 init_cq_buf(cq
, cq
->resize_buf
);
1160 kfree(cq
->resize_buf
);
1164 static void un_resize_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_cq
*cq
)
1166 free_cq_buf(dev
, cq
->resize_buf
);
1167 cq
->resize_buf
= NULL
;
1170 static int copy_resize_cqes(struct mlx5_ib_cq
*cq
)
1172 struct mlx5_ib_dev
*dev
= to_mdev(cq
->ibcq
.device
);
1173 struct mlx5_cqe64
*scqe64
;
1174 struct mlx5_cqe64
*dcqe64
;
1183 ssize
= cq
->buf
.cqe_size
;
1184 dsize
= cq
->resize_buf
->cqe_size
;
1185 if (ssize
!= dsize
) {
1186 mlx5_ib_warn(dev
, "resize from different cqe size is not supported\n");
1190 i
= cq
->mcq
.cons_index
;
1191 scqe
= get_sw_cqe(cq
, i
);
1192 scqe64
= ssize
== 64 ? scqe
: scqe
+ 64;
1195 mlx5_ib_warn(dev
, "expected cqe in sw ownership\n");
1199 while ((scqe64
->op_own
>> 4) != MLX5_CQE_RESIZE_CQ
) {
1200 dcqe
= get_cqe_from_buf(cq
->resize_buf
,
1201 (i
+ 1) & (cq
->resize_buf
->nent
),
1203 dcqe64
= dsize
== 64 ? dcqe
: dcqe
+ 64;
1204 sw_own
= sw_ownership_bit(i
+ 1, cq
->resize_buf
->nent
);
1205 memcpy(dcqe
, scqe
, dsize
);
1206 dcqe64
->op_own
= (dcqe64
->op_own
& ~MLX5_CQE_OWNER_MASK
) | sw_own
;
1209 scqe
= get_sw_cqe(cq
, i
);
1210 scqe64
= ssize
== 64 ? scqe
: scqe
+ 64;
1212 mlx5_ib_warn(dev
, "expected cqe in sw ownership\n");
1216 if (scqe
== start_cqe
) {
1217 pr_warn("resize CQ failed to get resize CQE, CQN 0x%x\n",
1222 ++cq
->mcq
.cons_index
;
1226 int mlx5_ib_resize_cq(struct ib_cq
*ibcq
, int entries
, struct ib_udata
*udata
)
1228 struct mlx5_ib_dev
*dev
= to_mdev(ibcq
->device
);
1229 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
1237 int uninitialized_var(cqe_size
);
1238 unsigned long flags
;
1240 if (!MLX5_CAP_GEN(dev
->mdev
, cq_resize
)) {
1241 pr_info("Firmware does not support resize CQ\n");
1246 entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
))) {
1247 mlx5_ib_warn(dev
, "wrong entries number %d, max %d\n",
1249 1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
));
1253 entries
= roundup_pow_of_two(entries
+ 1);
1254 if (entries
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_cq_sz
)) + 1)
1257 if (entries
== ibcq
->cqe
+ 1)
1260 mutex_lock(&cq
->resize_mutex
);
1262 err
= resize_user(dev
, cq
, entries
, udata
, &npas
, &page_shift
,
1266 err
= resize_kernel(dev
, cq
, entries
, cqe_size
);
1268 npas
= cq
->resize_buf
->buf
.npages
;
1269 page_shift
= cq
->resize_buf
->buf
.page_shift
;
1276 inlen
= MLX5_ST_SZ_BYTES(modify_cq_in
) +
1277 MLX5_FLD_SZ_BYTES(modify_cq_in
, pas
[0]) * npas
;
1279 in
= mlx5_vzalloc(inlen
);
1285 pas
= (__be64
*)MLX5_ADDR_OF(modify_cq_in
, in
, pas
);
1287 mlx5_ib_populate_pas(dev
, cq
->resize_umem
, page_shift
,
1290 mlx5_fill_page_array(&cq
->resize_buf
->buf
, pas
);
1292 MLX5_SET(modify_cq_in
, in
,
1293 modify_field_select_resize_field_select
.resize_field_select
.resize_field_select
,
1294 MLX5_MODIFY_CQ_MASK_LOG_SIZE
|
1295 MLX5_MODIFY_CQ_MASK_PG_OFFSET
|
1296 MLX5_MODIFY_CQ_MASK_PG_SIZE
);
1298 cqc
= MLX5_ADDR_OF(modify_cq_in
, in
, cq_context
);
1300 MLX5_SET(cqc
, cqc
, log_page_size
,
1301 page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
1302 MLX5_SET(cqc
, cqc
, cqe_sz
, cqe_sz_to_mlx_sz(cqe_size
));
1303 MLX5_SET(cqc
, cqc
, log_cq_size
, ilog2(entries
));
1305 MLX5_SET(modify_cq_in
, in
, op_mod
, MLX5_CQ_OPMOD_RESIZE
);
1306 MLX5_SET(modify_cq_in
, in
, cqn
, cq
->mcq
.cqn
);
1308 err
= mlx5_core_modify_cq(dev
->mdev
, &cq
->mcq
, in
, inlen
);
1313 cq
->ibcq
.cqe
= entries
- 1;
1314 ib_umem_release(cq
->buf
.umem
);
1315 cq
->buf
.umem
= cq
->resize_umem
;
1316 cq
->resize_umem
= NULL
;
1318 struct mlx5_ib_cq_buf tbuf
;
1321 spin_lock_irqsave(&cq
->lock
, flags
);
1322 if (cq
->resize_buf
) {
1323 err
= copy_resize_cqes(cq
);
1326 cq
->buf
= *cq
->resize_buf
;
1327 kfree(cq
->resize_buf
);
1328 cq
->resize_buf
= NULL
;
1332 cq
->ibcq
.cqe
= entries
- 1;
1333 spin_unlock_irqrestore(&cq
->lock
, flags
);
1335 free_cq_buf(dev
, &tbuf
);
1337 mutex_unlock(&cq
->resize_mutex
);
1349 un_resize_kernel(dev
, cq
);
1351 mutex_unlock(&cq
->resize_mutex
);
1355 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev
*dev
, struct ib_cq
*ibcq
)
1357 struct mlx5_ib_cq
*cq
;
1363 return cq
->cqe_size
;
1366 /* Called from atomic context */
1367 int mlx5_ib_generate_wc(struct ib_cq
*ibcq
, struct ib_wc
*wc
)
1369 struct mlx5_ib_wc
*soft_wc
;
1370 struct mlx5_ib_cq
*cq
= to_mcq(ibcq
);
1371 unsigned long flags
;
1373 soft_wc
= kmalloc(sizeof(*soft_wc
), GFP_ATOMIC
);
1378 spin_lock_irqsave(&cq
->lock
, flags
);
1379 list_add_tail(&soft_wc
->list
, &cq
->wc_list
);
1380 if (cq
->notify_flags
== IB_CQ_NEXT_COMP
||
1381 wc
->status
!= IB_WC_SUCCESS
) {
1382 cq
->notify_flags
= 0;
1383 schedule_work(&cq
->notify_work
);
1385 spin_unlock_irqrestore(&cq
->lock
, flags
);