]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/infiniband/hw/mlx5/qp.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/mlx5: Set ECE options during QP create
[mirror_ubuntu-hirsute-kernel.git] / drivers / infiniband / hw / mlx5 / qp.c
CommitLineData
e126ba97 1/*
6cf0a15f 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/module.h>
34#include <rdma/ib_umem.h>
2811ba51 35#include <rdma/ib_cache.h>
cfb5e088 36#include <rdma/ib_user_verbs.h>
d14133dd 37#include <rdma/rdma_counter.h>
c2e53b2c 38#include <linux/mlx5/fs.h>
e126ba97 39#include "mlx5_ib.h"
b96c9dde 40#include "ib_rep.h"
443c1cf9 41#include "cmd.h"
333fbaa0 42#include "qp.h"
029e88fd 43#include "wr.h"
e126ba97 44
e126ba97
EC		 45enum {
46 MLX5_IB_ACK_REQ_FREQ = 8,
47};
48
49enum {
50 MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83,
51 MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
52 MLX5_IB_LINK_TYPE_IB = 0,
53 MLX5_IB_LINK_TYPE_ETH = 1
54};
55
eb49ab0c
AV		 56enum raw_qp_set_mask_map {
57 MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID = 1UL << 0,
7d29f349 58 MLX5_RAW_QP_RATE_LIMIT = 1UL << 1,
eb49ab0c
AV		 59};
60
0680efa2
AV		 61struct mlx5_modify_raw_qp_param {
62 u16 operation;
eb49ab0c
AV		 63
64 u32 set_mask; /* raw_qp_set_mask_map */
61147f39
BW		 65
66 struct mlx5_rate_limit rl;
67
eb49ab0c 68 u8 rq_q_ctr_id;
d5ed8ac3 69 u16 port;
0680efa2
AV		 70};
71
89ea94a7
MG		 72static void get_cqs(enum ib_qp_type qp_type,
73 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
74 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
75
e126ba97
EC		 76static int is_qp0(enum ib_qp_type qp_type)
77{
78 return qp_type == IB_QPT_SMI;
79}
80
e126ba97
EC		 81static int is_sqp(enum ib_qp_type qp_type)
82{
83 return is_qp0(qp_type) || is_qp1(qp_type);
84}
85
c1395a2a 86/**
fbeb4075
MS		 87 * mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
88 * to kernel buffer
c1395a2a 89 *
fbeb4075
MS		 90 * @umem: User space memory where the WQ is
91 * @buffer: buffer to copy to
92 * @buflen: buffer length
93 * @wqe_index: index of WQE to copy from
94 * @wq_offset: offset to start of WQ
95 * @wq_wqe_cnt: number of WQEs in WQ
96 * @wq_wqe_shift: log2 of WQE size
97 * @bcnt: number of bytes to copy
98 * @bytes_copied: number of bytes to copy (return value)
c1395a2a 99 *
fbeb4075
MS		 100 * Copies from start of WQE bcnt or less bytes.
101 * Does not gurantee to copy the entire WQE.
c1395a2a 102 *
fbeb4075 103 * Return: zero on success, or an error code.
c1395a2a 104 */
da9ee9d8
MS		 105static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,
106 size_t buflen, int wqe_index,
107 int wq_offset, int wq_wqe_cnt,
108 int wq_wqe_shift, int bcnt,
fbeb4075 109 size_t *bytes_copied)
c1395a2a 110{
fbeb4075
MS		 111 size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) << wq_wqe_shift);
112 size_t wq_end = wq_offset + (wq_wqe_cnt << wq_wqe_shift);
113 size_t copy_length;
c1395a2a
HE		 114 int ret;
115
fbeb4075
MS		 116 /* don't copy more than requested, more than buffer length or
117 * beyond WQ end
118 */
119 copy_length = min_t(u32, buflen, wq_end - offset);
120 copy_length = min_t(u32, copy_length, bcnt);
121
122 ret = ib_umem_copy_from(buffer, umem, offset, copy_length);
123 if (ret)
124 return ret;
c1395a2a 125
fbeb4075
MS		 126 if (!ret && bytes_copied)
127 *bytes_copied = copy_length;
c1395a2a 128
fbeb4075
MS		 129 return 0;
130}
c1395a2a 131
da9ee9d8
MS		 132static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
133 void *buffer, size_t buflen, size_t *bc)
134{
135 struct mlx5_wqe_ctrl_seg *ctrl;
136 size_t bytes_copied = 0;
137 size_t wqe_length;
138 void *p;
139 int ds;
140
141 wqe_index = wqe_index & qp->sq.fbc.sz_m1;
142
143 /* read the control segment first */
144 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
145 ctrl = p;
146 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
147 wqe_length = ds * MLX5_WQE_DS_UNITS;
148
149 /* read rest of WQE if it spreads over more than one stride */
150 while (bytes_copied < wqe_length) {
151 size_t copy_length =
152 min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);
153
154 if (!copy_length)
155 break;
156
157 memcpy(buffer + bytes_copied, p, copy_length);
158 bytes_copied += copy_length;
159
160 wqe_index = (wqe_index + 1) & qp->sq.fbc.sz_m1;
161 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
162 }
163 *bc = bytes_copied;
164 return 0;
165}
166
167static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
168 void *buffer, size_t buflen, size_t *bc)
fbeb4075
MS		 169{
170 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
171 struct ib_umem *umem = base->ubuffer.umem;
172 struct mlx5_ib_wq *wq = &qp->sq;
173 struct mlx5_wqe_ctrl_seg *ctrl;
174 size_t bytes_copied;
175 size_t bytes_copied2;
176 size_t wqe_length;
177 int ret;
178 int ds;
179
fbeb4075 180 /* at first read as much as possible */
da9ee9d8
MS		 181 ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
182 wq->offset, wq->wqe_cnt,
183 wq->wqe_shift, buflen,
fbeb4075 184 &bytes_copied);
c1395a2a
HE		 185 if (ret)
186 return ret;
187
fbeb4075
MS		 188 /* we need at least control segment size to proceed */
189 if (bytes_copied < sizeof(*ctrl))
190 return -EINVAL;
191
192 ctrl = buffer;
193 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
194 wqe_length = ds * MLX5_WQE_DS_UNITS;
c1395a2a 195
fbeb4075
MS		 196 /* if we copied enough then we are done */
197 if (bytes_copied >= wqe_length) {
198 *bc = bytes_copied;
199 return 0;
c1395a2a
HE		 200 }
201
fbeb4075
MS		 202 /* otherwise this a wrapped around wqe
203 * so read the remaining bytes starting
204 * from wqe_index 0
205 */
da9ee9d8
MS		 206 ret = mlx5_ib_read_user_wqe_common(umem, buffer + bytes_copied,
207 buflen - bytes_copied, 0, wq->offset,
208 wq->wqe_cnt, wq->wqe_shift,
fbeb4075
MS		 209 wqe_length - bytes_copied,
210 &bytes_copied2);
211
212 if (ret)
213 return ret;
214 *bc = bytes_copied + bytes_copied2;
215 return 0;
216}
217
da9ee9d8
MS		 218int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
219 size_t buflen, size_t *bc)
220{
221 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
222 struct ib_umem *umem = base->ubuffer.umem;
223
224 if (buflen < sizeof(struct mlx5_wqe_ctrl_seg))
225 return -EINVAL;
226
227 if (!umem)
228 return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,
229 buflen, bc);
230
231 return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);
232}
233
234static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
235 void *buffer, size_t buflen, size_t *bc)
fbeb4075
MS		 236{
237 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
238 struct ib_umem *umem = base->ubuffer.umem;
239 struct mlx5_ib_wq *wq = &qp->rq;
240 size_t bytes_copied;
241 int ret;
242
da9ee9d8
MS		 243 ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
244 wq->offset, wq->wqe_cnt,
245 wq->wqe_shift, buflen,
fbeb4075 246 &bytes_copied);
c1395a2a 247
c1395a2a
HE		 248 if (ret)
249 return ret;
fbeb4075
MS		 250 *bc = bytes_copied;
251 return 0;
252}
253
da9ee9d8
MS		 254int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
255 size_t buflen, size_t *bc)
256{
257 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
258 struct ib_umem *umem = base->ubuffer.umem;
259 struct mlx5_ib_wq *wq = &qp->rq;
260 size_t wqe_size = 1 << wq->wqe_shift;
261
262 if (buflen < wqe_size)
263 return -EINVAL;
264
265 if (!umem)
266 return -EOPNOTSUPP;
267
268 return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);
269}
270
271static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
272 void *buffer, size_t buflen, size_t *bc)
fbeb4075
MS		 273{
274 struct ib_umem *umem = srq->umem;
275 size_t bytes_copied;
276 int ret;
c1395a2a 277
da9ee9d8
MS		 278 ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, 0,
279 srq->msrq.max, srq->msrq.wqe_shift,
280 buflen, &bytes_copied);
fbeb4075
MS		 281
282 if (ret)
283 return ret;
284 *bc = bytes_copied;
285 return 0;
c1395a2a
HE		 286}
287
da9ee9d8
MS		 288int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
289 size_t buflen, size_t *bc)
290{
291 struct ib_umem *umem = srq->umem;
292 size_t wqe_size = 1 << srq->msrq.wqe_shift;
293
294 if (buflen < wqe_size)
295 return -EINVAL;
296
297 if (!umem)
298 return -EOPNOTSUPP;
299
300 return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
301}
302
e126ba97
EC		 303static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
304{
305 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
306 struct ib_event event;
307
19098df2 308 if (type == MLX5_EVENT_TYPE_PATH_MIG) {
309 /* This event is only valid for trans_qps */
310 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
311 }
e126ba97
EC		 312
313 if (ibqp->event_handler) {
314 event.device = ibqp->device;
315 event.element.qp = ibqp;
316 switch (type) {
317 case MLX5_EVENT_TYPE_PATH_MIG:
318 event.event = IB_EVENT_PATH_MIG;
319 break;
320 case MLX5_EVENT_TYPE_COMM_EST:
321 event.event = IB_EVENT_COMM_EST;
322 break;
323 case MLX5_EVENT_TYPE_SQ_DRAINED:
324 event.event = IB_EVENT_SQ_DRAINED;
325 break;
326 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
327 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
328 break;
329 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
330 event.event = IB_EVENT_QP_FATAL;
331 break;
332 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
333 event.event = IB_EVENT_PATH_MIG_ERR;
334 break;
335 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
336 event.event = IB_EVENT_QP_REQ_ERR;
337 break;
338 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
339 event.event = IB_EVENT_QP_ACCESS_ERR;
340 break;
341 default:
342 pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
343 return;
344 }
345
346 ibqp->event_handler(&event, ibqp->qp_context);
347 }
348}
349
350static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
351 int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
352{
353 int wqe_size;
354 int wq_size;
355
356 /* Sanity check RQ size before proceeding */
938fe83c 357 if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
e126ba97
EC		 358 return -EINVAL;
359
360 if (!has_rq) {
361 qp->rq.max_gs = 0;
362 qp->rq.wqe_cnt = 0;
363 qp->rq.wqe_shift = 0;
0540d814
NO		 364 cap->max_recv_wr = 0;
365 cap->max_recv_sge = 0;
e126ba97 366 } else {
c95e6d53
LR		 367 int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
368
e126ba97
EC		 369 if (ucmd) {
370 qp->rq.wqe_cnt = ucmd->rq_wqe_count;
002bf228
LR		 371 if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
372 return -EINVAL;
e126ba97 373 qp->rq.wqe_shift = ucmd->rq_wqe_shift;
c95e6d53
LR		 374 if ((1 << qp->rq.wqe_shift) /
375 sizeof(struct mlx5_wqe_data_seg) <
376 wq_sig)
002bf228 377 return -EINVAL;
c95e6d53
LR		 378 qp->rq.max_gs =
379 (1 << qp->rq.wqe_shift) /
380 sizeof(struct mlx5_wqe_data_seg) -
381 wq_sig;
e126ba97
EC		 382 qp->rq.max_post = qp->rq.wqe_cnt;
383 } else {
c95e6d53
LR		 384 wqe_size =
385 wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
386 0;
e126ba97
EC		 387 wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
388 wqe_size = roundup_pow_of_two(wqe_size);
389 wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
390 wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
391 qp->rq.wqe_cnt = wq_size / wqe_size;
938fe83c 392 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
e126ba97
EC		 393 mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
394 wqe_size,
938fe83c
SM		 395 MLX5_CAP_GEN(dev->mdev,
396 max_wqe_sz_rq));
e126ba97
EC		 397 return -EINVAL;
398 }
399 qp->rq.wqe_shift = ilog2(wqe_size);
c95e6d53
LR		 400 qp->rq.max_gs =
401 (1 << qp->rq.wqe_shift) /
402 sizeof(struct mlx5_wqe_data_seg) -
403 wq_sig;
e126ba97
EC		 404 qp->rq.max_post = qp->rq.wqe_cnt;
405 }
406 }
407
408 return 0;
409}
410
f0313965 411static int sq_overhead(struct ib_qp_init_attr *attr)
e126ba97 412{
618af384 413 int size = 0;
e126ba97 414
f0313965 415 switch (attr->qp_type) {
e126ba97 416 case IB_QPT_XRC_INI:
b125a54b 417 size += sizeof(struct mlx5_wqe_xrc_seg);
e126ba97
EC		 418 /* fall through */
419 case IB_QPT_RC:
420 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 421 max(sizeof(struct mlx5_wqe_atomic_seg) +
422 sizeof(struct mlx5_wqe_raddr_seg),
423 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
064e5262
IB		 424 sizeof(struct mlx5_mkey_seg) +
425 MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
426 MLX5_IB_UMR_OCTOWORD);
e126ba97
EC		 427 break;
428
b125a54b
EC		 429 case IB_QPT_XRC_TGT:
430 return 0;
431
e126ba97 432 case IB_QPT_UC:
b125a54b 433 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 434 max(sizeof(struct mlx5_wqe_raddr_seg),
435 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
436 sizeof(struct mlx5_mkey_seg));
e126ba97
EC		 437 break;
438
439 case IB_QPT_UD:
f0313965
ES		 440 if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
441 size += sizeof(struct mlx5_wqe_eth_pad) +
442 sizeof(struct mlx5_wqe_eth_seg);
443 /* fall through */
e126ba97 444 case IB_QPT_SMI:
d16e91da 445 case MLX5_IB_QPT_HW_GSI:
b125a54b 446 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 447 sizeof(struct mlx5_wqe_datagram_seg);
448 break;
449
450 case MLX5_IB_QPT_REG_UMR:
b125a54b 451 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 452 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
453 sizeof(struct mlx5_mkey_seg);
454 break;
455
456 default:
457 return -EINVAL;
458 }
459
460 return size;
461}
462
463static int calc_send_wqe(struct ib_qp_init_attr *attr)
464{
465 int inl_size = 0;
466 int size;
467
f0313965 468 size = sq_overhead(attr);
e126ba97
EC		 469 if (size < 0)
470 return size;
471
472 if (attr->cap.max_inline_data) {
473 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
474 attr->cap.max_inline_data;
475 }
476
477 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
c0a6cbb9 478 if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
e1e66cc2 479 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
c0a6cbb9 480 return MLX5_SIG_WQE_SIZE;
e1e66cc2
SG		 481 else
482 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
e126ba97
EC		 483}
484
288c01b7
EC		 485static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
486{
487 int max_sge;
488
489 if (attr->qp_type == IB_QPT_RC)
490 max_sge = (min_t(int, wqe_size, 512) -
491 sizeof(struct mlx5_wqe_ctrl_seg) -
492 sizeof(struct mlx5_wqe_raddr_seg)) /
493 sizeof(struct mlx5_wqe_data_seg);
494 else if (attr->qp_type == IB_QPT_XRC_INI)
495 max_sge = (min_t(int, wqe_size, 512) -
496 sizeof(struct mlx5_wqe_ctrl_seg) -
497 sizeof(struct mlx5_wqe_xrc_seg) -
498 sizeof(struct mlx5_wqe_raddr_seg)) /
499 sizeof(struct mlx5_wqe_data_seg);
500 else
501 max_sge = (wqe_size - sq_overhead(attr)) /
502 sizeof(struct mlx5_wqe_data_seg);
503
504 return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
505 sizeof(struct mlx5_wqe_data_seg));
506}
507
e126ba97
EC		 508static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
509 struct mlx5_ib_qp *qp)
510{
511 int wqe_size;
512 int wq_size;
513
514 if (!attr->cap.max_send_wr)
515 return 0;
516
517 wqe_size = calc_send_wqe(attr);
518 mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
519 if (wqe_size < 0)
520 return wqe_size;
521
938fe83c 522 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
b125a54b 523 mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
938fe83c 524 wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 525 return -EINVAL;
526 }
527
f0313965
ES		 528 qp->max_inline_data = wqe_size - sq_overhead(attr) -
529 sizeof(struct mlx5_wqe_inline_seg);
e126ba97
EC		 530 attr->cap.max_inline_data = qp->max_inline_data;
531
532 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
533 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
938fe83c 534 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
1974ab9d
BVA		 535 mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
536 attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
938fe83c
SM		 537 qp->sq.wqe_cnt,
538 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
b125a54b
EC		 539 return -ENOMEM;
540 }
e126ba97 541 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
288c01b7
EC		 542 qp->sq.max_gs = get_send_sge(attr, wqe_size);
543 if (qp->sq.max_gs < attr->cap.max_send_sge)
544 return -ENOMEM;
545
546 attr->cap.max_send_sge = qp->sq.max_gs;
b125a54b
EC		 547 qp->sq.max_post = wq_size / wqe_size;
548 attr->cap.max_send_wr = qp->sq.max_post;
e126ba97
EC		 549
550 return wq_size;
551}
552
553static int set_user_buf_size(struct mlx5_ib_dev *dev,
554 struct mlx5_ib_qp *qp,
19098df2 555 struct mlx5_ib_create_qp *ucmd,
0fb2ed66 556 struct mlx5_ib_qp_base *base,
557 struct ib_qp_init_attr *attr)
e126ba97
EC		 558{
559 int desc_sz = 1 << qp->sq.wqe_shift;
560
938fe83c 561 if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
e126ba97 562 mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
938fe83c 563 desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 564 return -EINVAL;
565 }
566
af8b38ed
GP		 567 if (ucmd->sq_wqe_count && !is_power_of_2(ucmd->sq_wqe_count)) {
568 mlx5_ib_warn(dev, "sq_wqe_count %d is not a power of two\n",
569 ucmd->sq_wqe_count);
e126ba97
EC		 570 return -EINVAL;
571 }
572
573 qp->sq.wqe_cnt = ucmd->sq_wqe_count;
574
938fe83c 575 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
e126ba97 576 mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
938fe83c
SM		 577 qp->sq.wqe_cnt,
578 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
e126ba97
EC		 579 return -EINVAL;
580 }
581
c2e53b2c 582 if (attr->qp_type == IB_QPT_RAW_PACKET ||
2be08c30 583 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
0fb2ed66 584 base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
585 qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
586 } else {
587 base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
588 (qp->sq.wqe_cnt << 6);
589 }
e126ba97
EC		 590
591 return 0;
592}
593
594static int qp_has_rq(struct ib_qp_init_attr *attr)
595{
596 if (attr->qp_type == IB_QPT_XRC_INI ||
597 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
598 attr->qp_type == MLX5_IB_QPT_REG_UMR ||
599 !attr->cap.max_recv_wr)
600 return 0;
601
602 return 1;
603}
604
0b80c14f
EC		 605enum {
606 /* this is the first blue flame register in the array of bfregs assigned
607 * to a processes. Since we do not use it for blue flame but rather
608 * regular 64 bit doorbells, we do not need a lock for maintaiing
609 * "odd/even" order
610 */
611 NUM_NON_BLUE_FLAME_BFREGS = 1,
612};
613
b037c29a
EC		 614static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
615{
31a78a5a 616 return get_num_static_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a
EC		 617}
618
619static int num_med_bfreg(struct mlx5_ib_dev *dev,
620 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 621{
622 int n;
623
b037c29a
EC		 624 n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
625 NUM_NON_BLUE_FLAME_BFREGS;
c1be5232
EC		 626
627 return n >= 0 ? n : 0;
628}
629
18b0362e
YH		 630static int first_med_bfreg(struct mlx5_ib_dev *dev,
631 struct mlx5_bfreg_info *bfregi)
632{
633 return num_med_bfreg(dev, bfregi) ? 1 : -ENOMEM;
634}
635
b037c29a
EC		 636static int first_hi_bfreg(struct mlx5_ib_dev *dev,
637 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 638{
639 int med;
c1be5232 640
b037c29a
EC		 641 med = num_med_bfreg(dev, bfregi);
642 return ++med;
c1be5232
EC		 643}
644
b037c29a
EC		 645static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
646 struct mlx5_bfreg_info *bfregi)
e126ba97 647{
e126ba97
EC		 648 int i;
649
b037c29a
EC		 650 for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
651 if (!bfregi->count[i]) {
2f5ff264 652 bfregi->count[i]++;
e126ba97
EC		 653 return i;
654 }
655 }
656
657 return -ENOMEM;
658}
659
b037c29a
EC		 660static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
661 struct mlx5_bfreg_info *bfregi)
e126ba97 662{
18b0362e 663 int minidx = first_med_bfreg(dev, bfregi);
e126ba97
EC		 664 int i;
665
18b0362e
YH		 666 if (minidx < 0)
667 return minidx;
668
669 for (i = minidx; i < first_hi_bfreg(dev, bfregi); i++) {
2f5ff264 670 if (bfregi->count[i] < bfregi->count[minidx])
e126ba97 671 minidx = i;
0b80c14f
EC		 672 if (!bfregi->count[minidx])
673 break;
e126ba97
EC		 674 }
675
2f5ff264 676 bfregi->count[minidx]++;
e126ba97
EC		 677 return minidx;
678}
679
b037c29a 680static int alloc_bfreg(struct mlx5_ib_dev *dev,
ffaf58de 681 struct mlx5_bfreg_info *bfregi)
e126ba97 682{
ffaf58de 683 int bfregn = -ENOMEM;
e126ba97 684
0a2fd01c
YH		 685 if (bfregi->lib_uar_dyn)
686 return -EINVAL;
687
2f5ff264 688 mutex_lock(&bfregi->lock);
ffaf58de
LR		 689 if (bfregi->ver >= 2) {
690 bfregn = alloc_high_class_bfreg(dev, bfregi);
691 if (bfregn < 0)
692 bfregn = alloc_med_class_bfreg(dev, bfregi);
693 }
694
695 if (bfregn < 0) {
0b80c14f 696 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
2f5ff264
EC		 697 bfregn = 0;
698 bfregi->count[bfregn]++;
e126ba97 699 }
2f5ff264 700 mutex_unlock(&bfregi->lock);
e126ba97 701
2f5ff264 702 return bfregn;
e126ba97
EC		 703}
704
4ed131d0 705void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
e126ba97 706{
2f5ff264 707 mutex_lock(&bfregi->lock);
b037c29a 708 bfregi->count[bfregn]--;
2f5ff264 709 mutex_unlock(&bfregi->lock);
e126ba97
EC		 710}
711
712static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
713{
714 switch (state) {
715 case IB_QPS_RESET: return MLX5_QP_STATE_RST;
716 case IB_QPS_INIT: return MLX5_QP_STATE_INIT;
717 case IB_QPS_RTR: return MLX5_QP_STATE_RTR;
718 case IB_QPS_RTS: return MLX5_QP_STATE_RTS;
719 case IB_QPS_SQD: return MLX5_QP_STATE_SQD;
720 case IB_QPS_SQE: return MLX5_QP_STATE_SQER;
721 case IB_QPS_ERR: return MLX5_QP_STATE_ERR;
722 default: return -1;
723 }
724}
725
726static int to_mlx5_st(enum ib_qp_type type)
727{
728 switch (type) {
729 case IB_QPT_RC: return MLX5_QP_ST_RC;
730 case IB_QPT_UC: return MLX5_QP_ST_UC;
731 case IB_QPT_UD: return MLX5_QP_ST_UD;
732 case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR;
733 case IB_QPT_XRC_INI:
734 case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC;
735 case IB_QPT_SMI: return MLX5_QP_ST_QP0;
d16e91da 736 case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1;
c32a4f29 737 case MLX5_IB_QPT_DCI: return MLX5_QP_ST_DCI;
3ae7e66a 738 case IB_QPT_RAW_PACKET: return MLX5_QP_ST_RAW_ETHERTYPE;
e126ba97
EC		 739 default: return -EINVAL;
740 }
741}
742
89ea94a7
MG		 743static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
744 struct mlx5_ib_cq *recv_cq);
745static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
746 struct mlx5_ib_cq *recv_cq);
747
7c043e90 748int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
05f58ceb 749 struct mlx5_bfreg_info *bfregi, u32 bfregn,
7c043e90 750 bool dyn_bfreg)
e126ba97 751{
05f58ceb
LR		 752 unsigned int bfregs_per_sys_page;
753 u32 index_of_sys_page;
754 u32 offset;
b037c29a 755
0a2fd01c
YH		 756 if (bfregi->lib_uar_dyn)
757 return -EINVAL;
758
b037c29a
EC		 759 bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
760 MLX5_NON_FP_BFREGS_PER_UAR;
761 index_of_sys_page = bfregn / bfregs_per_sys_page;
762
1ee47ab3
YH		 763 if (dyn_bfreg) {
764 index_of_sys_page += bfregi->num_static_sys_pages;
05f58ceb
LR		 765
766 if (index_of_sys_page >= bfregi->num_sys_pages)
767 return -EINVAL;
768
1ee47ab3
YH		 769 if (bfregn > bfregi->num_dyn_bfregs ||
770 bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
771 mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
772 return -EINVAL;
773 }
774 }
b037c29a 775
1ee47ab3 776 offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a 777 return bfregi->sys_pages[index_of_sys_page] + offset;
e126ba97
EC		 778}
779
b0ea0fa5 780static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev, struct ib_udata *udata,
19098df2 781 unsigned long addr, size_t size,
b0ea0fa5
JG		 782 struct ib_umem **umem, int *npages, int *page_shift,
783 int *ncont, u32 *offset)
19098df2 784{
785 int err;
786
c320e527 787 *umem = ib_umem_get(&dev->ib_dev, addr, size, 0);
19098df2 788 if (IS_ERR(*umem)) {
789 mlx5_ib_dbg(dev, "umem_get failed\n");
790 return PTR_ERR(*umem);
791 }
792
762f899a 793 mlx5_ib_cont_pages(*umem, addr, 0, npages, page_shift, ncont, NULL);
19098df2 794
795 err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
796 if (err) {
797 mlx5_ib_warn(dev, "bad offset\n");
798 goto err_umem;
799 }
800
801 mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
802 addr, size, *npages, *page_shift, *ncont, *offset);
803
804 return 0;
805
806err_umem:
807 ib_umem_release(*umem);
808 *umem = NULL;
809
810 return err;
811}
812
fe248c3a 813static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
bdeacabd 814 struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
79b20a6c 815{
bdeacabd
SR		 816 struct mlx5_ib_ucontext *context =
817 rdma_udata_to_drv_context(
818 udata,
819 struct mlx5_ib_ucontext,
820 ibucontext);
79b20a6c 821
fe248c3a
MG		 822 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
823 atomic_dec(&dev->delay_drop.rqs_cnt);
824
79b20a6c 825 mlx5_ib_db_unmap_user(context, &rwq->db);
836a0fbb 826 ib_umem_release(rwq->umem);
79b20a6c
YH		 827}
828
829static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
b0ea0fa5 830 struct ib_udata *udata, struct mlx5_ib_rwq *rwq,
79b20a6c
YH		 831 struct mlx5_ib_create_wq *ucmd)
832{
89944450
SR		 833 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
834 udata, struct mlx5_ib_ucontext, ibucontext);
79b20a6c
YH		 835 int page_shift = 0;
836 int npages;
837 u32 offset = 0;
838 int ncont = 0;
839 int err;
840
841 if (!ucmd->buf_addr)
842 return -EINVAL;
843
c320e527 844 rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0);
79b20a6c
YH		 845 if (IS_ERR(rwq->umem)) {
846 mlx5_ib_dbg(dev, "umem_get failed\n");
847 err = PTR_ERR(rwq->umem);
848 return err;
849 }
850
762f899a 851 mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,
79b20a6c
YH		 852 &ncont, NULL);
853 err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
854 &rwq->rq_page_offset);
855 if (err) {
856 mlx5_ib_warn(dev, "bad offset\n");
857 goto err_umem;
858 }
859
860 rwq->rq_num_pas = ncont;
861 rwq->page_shift = page_shift;
862 rwq->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
863 rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
864
865 mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
866 (unsigned long long)ucmd->buf_addr, rwq->buf_size,
867 npages, page_shift, ncont, offset);
868
89944450 869 err = mlx5_ib_db_map_user(ucontext, udata, ucmd->db_addr, &rwq->db);
79b20a6c
YH		 870 if (err) {
871 mlx5_ib_dbg(dev, "map failed\n");
872 goto err_umem;
873 }
874
79b20a6c
YH		 875 return 0;
876
877err_umem:
878 ib_umem_release(rwq->umem);
879 return err;
880}
881
b037c29a
EC		 882static int adjust_bfregn(struct mlx5_ib_dev *dev,
883 struct mlx5_bfreg_info *bfregi, int bfregn)
884{
885 return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
886 bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
887}
888
98fc1126
LR		 889static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
890 struct mlx5_ib_qp *qp, struct ib_udata *udata,
891 struct ib_qp_init_attr *attr, u32 **in,
892 struct mlx5_ib_create_qp_resp *resp, int *inlen,
893 struct mlx5_ib_qp_base *base,
894 struct mlx5_ib_create_qp *ucmd)
e126ba97
EC		 895{
896 struct mlx5_ib_ucontext *context;
19098df2 897 struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
9e9c47d0 898 int page_shift = 0;
1ee47ab3 899 int uar_index = 0;
e126ba97 900 int npages;
9e9c47d0 901 u32 offset = 0;
2f5ff264 902 int bfregn;
9e9c47d0 903 int ncont = 0;
09a7d9ec
SM		 904 __be64 *pas;
905 void *qpc;
e126ba97 906 int err;
5aa3771d 907 u16 uid;
ac42a5ee 908 u32 uar_flags;
e126ba97 909
89944450
SR		 910 context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
911 ibucontext);
76883a6c
LR		 912 uar_flags = qp->flags_en &
913 (MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);
ac42a5ee
YH		 914 switch (uar_flags) {
915 case MLX5_QP_FLAG_UAR_PAGE_INDEX:
76883a6c 916 uar_index = ucmd->bfreg_index;
ac42a5ee
YH		 917 bfregn = MLX5_IB_INVALID_BFREG;
918 break;
919 case MLX5_QP_FLAG_BFREG_INDEX:
1ee47ab3 920 uar_index = bfregn_to_uar_index(dev, &context->bfregi,
76883a6c 921 ucmd->bfreg_index, true);
1ee47ab3
YH		 922 if (uar_index < 0)
923 return uar_index;
1ee47ab3 924 bfregn = MLX5_IB_INVALID_BFREG;
ac42a5ee
YH		 925 break;
926 case 0:
2be08c30 927 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
ac42a5ee 928 return -EINVAL;
ffaf58de
LR		 929 bfregn = alloc_bfreg(dev, &context->bfregi);
930 if (bfregn < 0)
931 return bfregn;
ac42a5ee
YH		 932 break;
933 default:
934 return -EINVAL;
e126ba97
EC		 935 }
936
2f5ff264 937 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
1ee47ab3
YH		 938 if (bfregn != MLX5_IB_INVALID_BFREG)
939 uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
940 false);
e126ba97 941
48fea837
HE		 942 qp->rq.offset = 0;
943 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
944 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
945
76883a6c 946 err = set_user_buf_size(dev, qp, ucmd, base, attr);
e126ba97 947 if (err)
2f5ff264 948 goto err_bfreg;
e126ba97 949
76883a6c
LR		 950 if (ucmd->buf_addr && ubuffer->buf_size) {
951 ubuffer->buf_addr = ucmd->buf_addr;
b0ea0fa5
JG		 952 err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr,
953 ubuffer->buf_size, &ubuffer->umem,
954 &npages, &page_shift, &ncont, &offset);
19098df2 955 if (err)
2f5ff264 956 goto err_bfreg;
9e9c47d0 957 } else {
19098df2 958 ubuffer->umem = NULL;
e126ba97 959 }
e126ba97 960
09a7d9ec
SM		 961 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
962 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1b9a07ee 963 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 964 if (!*in) {
965 err = -ENOMEM;
966 goto err_umem;
967 }
09a7d9ec 968
04bcc1c2 969 uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
5aa3771d 970 MLX5_SET(create_qp_in, *in, uid, uid);
09a7d9ec 971 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
19098df2 972 if (ubuffer->umem)
09a7d9ec
SM		 973 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
974
975 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
976
977 MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
978 MLX5_SET(qpc, qpc, page_offset, offset);
e126ba97 979
09a7d9ec 980 MLX5_SET(qpc, qpc, uar_page, uar_index);
1ee47ab3
YH		 981 if (bfregn != MLX5_IB_INVALID_BFREG)
982 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
983 else
984 resp->bfreg_index = MLX5_IB_INVALID_BFREG;
2f5ff264 985 qp->bfregn = bfregn;
e126ba97 986
76883a6c 987 err = mlx5_ib_db_map_user(context, udata, ucmd->db_addr, &qp->db);
e126ba97
EC		 988 if (err) {
989 mlx5_ib_dbg(dev, "map failed\n");
990 goto err_free;
991 }
992
e126ba97
EC		 993 return 0;
994
e126ba97 995err_free:
479163f4 996 kvfree(*in);
e126ba97
EC		 997
998err_umem:
836a0fbb 999 ib_umem_release(ubuffer->umem);
e126ba97 1000
2f5ff264 1001err_bfreg:
1ee47ab3
YH		 1002 if (bfregn != MLX5_IB_INVALID_BFREG)
1003 mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
e126ba97
EC		 1004 return err;
1005}
1006
747c519c
LR		 1007static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1008 struct mlx5_ib_qp_base *base, struct ib_udata *udata)
e126ba97 1009{
747c519c
LR		 1010 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1011 udata, struct mlx5_ib_ucontext, ibucontext);
e126ba97 1012
747c519c
LR		 1013 if (udata) {
1014 /* User QP */
1015 mlx5_ib_db_unmap_user(context, &qp->db);
1016 ib_umem_release(base->ubuffer.umem);
1017
1018 /*
1019 * Free only the BFREGs which are handled by the kernel.
1020 * BFREGs of UARs allocated dynamically are handled by user.
1021 */
1022 if (qp->bfregn != MLX5_IB_INVALID_BFREG)
1023 mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
1024 return;
1025 }
1ee47ab3 1026
747c519c
LR		 1027 /* Kernel QP */
1028 kvfree(qp->sq.wqe_head);
1029 kvfree(qp->sq.w_list);
1030 kvfree(qp->sq.wrid);
1031 kvfree(qp->sq.wr_data);
1032 kvfree(qp->rq.wrid);
1033 if (qp->db.db)
1034 mlx5_db_free(dev->mdev, &qp->db);
1035 if (qp->buf.frags)
1036 mlx5_frag_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1037}
1038
98fc1126
LR		 1039static int _create_kernel_qp(struct mlx5_ib_dev *dev,
1040 struct ib_qp_init_attr *init_attr,
1041 struct mlx5_ib_qp *qp, u32 **in, int *inlen,
1042 struct mlx5_ib_qp_base *base)
e126ba97 1043{
e126ba97 1044 int uar_index;
09a7d9ec 1045 void *qpc;
e126ba97
EC		 1046 int err;
1047
e126ba97 1048 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
5fe9dec0 1049 qp->bf.bfreg = &dev->fp_bfreg;
2978975c 1050 else if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
11f552e2 1051 qp->bf.bfreg = &dev->wc_bfreg;
5fe9dec0
EC		 1052 else
1053 qp->bf.bfreg = &dev->bfreg;
e126ba97 1054
d8030b0d
EC		 1055 /* We need to divide by two since each register is comprised of
1056 * two buffers of identical size, namely odd and even
1057 */
1058 qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
5fe9dec0 1059 uar_index = qp->bf.bfreg->index;
e126ba97
EC		 1060
1061 err = calc_sq_size(dev, init_attr, qp);
1062 if (err < 0) {
1063 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 1064 return err;
e126ba97
EC		 1065 }
1066
1067 qp->rq.offset = 0;
1068 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
19098df2 1069 base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
e126ba97 1070
34f4c955
GL		 1071 err = mlx5_frag_buf_alloc_node(dev->mdev, base->ubuffer.buf_size,
1072 &qp->buf, dev->mdev->priv.numa_node);
e126ba97
EC		 1073 if (err) {
1074 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 1075 return err;
e126ba97
EC		 1076 }
1077
34f4c955
GL		 1078 if (qp->rq.wqe_cnt)
1079 mlx5_init_fbc(qp->buf.frags, qp->rq.wqe_shift,
1080 ilog2(qp->rq.wqe_cnt), &qp->rq.fbc);
1081
1082 if (qp->sq.wqe_cnt) {
1083 int sq_strides_offset = (qp->sq.offset & (PAGE_SIZE - 1)) /
1084 MLX5_SEND_WQE_BB;
1085 mlx5_init_fbc_offset(qp->buf.frags +
1086 (qp->sq.offset / PAGE_SIZE),
1087 ilog2(MLX5_SEND_WQE_BB),
1088 ilog2(qp->sq.wqe_cnt),
1089 sq_strides_offset, &qp->sq.fbc);
1090
1091 qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
1092 }
1093
09a7d9ec
SM		 1094 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1095 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1b9a07ee 1096 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 1097 if (!*in) {
1098 err = -ENOMEM;
1099 goto err_buf;
1100 }
09a7d9ec
SM		 1101
1102 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1103 MLX5_SET(qpc, qpc, uar_page, uar_index);
1104 MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1105
e126ba97 1106 /* Set "fast registration enabled" for all kernel QPs */
09a7d9ec
SM		 1107 MLX5_SET(qpc, qpc, fre, 1);
1108 MLX5_SET(qpc, qpc, rlky, 1);
e126ba97 1109
2978975c 1110 if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
09a7d9ec 1111 MLX5_SET(qpc, qpc, deth_sqpn, 1);
b11a4f9c 1112
34f4c955
GL		 1113 mlx5_fill_page_frag_array(&qp->buf,
1114 (__be64 *)MLX5_ADDR_OF(create_qp_in,
1115 *in, pas));
e126ba97 1116
9603b61d 1117 err = mlx5_db_alloc(dev->mdev, &qp->db);
e126ba97
EC		 1118 if (err) {
1119 mlx5_ib_dbg(dev, "err %d\n", err);
1120 goto err_free;
1121 }
1122
b5883008
LD		 1123 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1124 sizeof(*qp->sq.wrid), GFP_KERNEL);
1125 qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1126 sizeof(*qp->sq.wr_data), GFP_KERNEL);
1127 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1128 sizeof(*qp->rq.wrid), GFP_KERNEL);
1129 qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1130 sizeof(*qp->sq.w_list), GFP_KERNEL);
1131 qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1132 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
e126ba97
EC		 1133
1134 if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1135 !qp->sq.w_list || !qp->sq.wqe_head) {
1136 err = -ENOMEM;
1137 goto err_wrid;
1138 }
e126ba97
EC		 1139
1140 return 0;
1141
1142err_wrid:
b5883008
LD		 1143 kvfree(qp->sq.wqe_head);
1144 kvfree(qp->sq.w_list);
1145 kvfree(qp->sq.wrid);
1146 kvfree(qp->sq.wr_data);
1147 kvfree(qp->rq.wrid);
f4044dac 1148 mlx5_db_free(dev->mdev, &qp->db);
e126ba97
EC		 1149
1150err_free:
479163f4 1151 kvfree(*in);
e126ba97
EC		 1152
1153err_buf:
34f4c955 1154 mlx5_frag_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1155 return err;
1156}
1157
09a7d9ec 1158static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
e126ba97 1159{
7aede1a2
LR		 1160 if (attr->srq || (qp->type == IB_QPT_XRC_TGT) ||
1161 (qp->type == MLX5_IB_QPT_DCI) || (qp->type == IB_QPT_XRC_INI))
09a7d9ec 1162 return MLX5_SRQ_RQ;
e126ba97 1163 else if (!qp->has_rq)
09a7d9ec 1164 return MLX5_ZERO_LEN_RQ;
7aede1a2
LR		 1165
1166 return MLX5_NON_ZERO_RQ;
e126ba97
EC		 1167}
1168
0fb2ed66 1169static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
c2e53b2c 1170 struct mlx5_ib_qp *qp,
1cd6dbd3
YH		 1171 struct mlx5_ib_sq *sq, u32 tdn,
1172 struct ib_pd *pd)
0fb2ed66 1173{
e0b4b472 1174 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
0fb2ed66 1175 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1176
1cd6dbd3 1177 MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1178 MLX5_SET(tisc, tisc, transport_domain, tdn);
2be08c30 1179 if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
c2e53b2c
YH		 1180 MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1181
e0b4b472 1182 return mlx5_core_create_tis(dev->mdev, in, &sq->tisn);
0fb2ed66 1183}
1184
1185static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1cd6dbd3 1186 struct mlx5_ib_sq *sq, struct ib_pd *pd)
0fb2ed66 1187{
1cd6dbd3 1188 mlx5_cmd_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
0fb2ed66 1189}
1190
d5ed8ac3 1191static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
b96c9dde
MB		 1192{
1193 if (sq->flow_rule)
1194 mlx5_del_flow_rules(sq->flow_rule);
d5ed8ac3 1195 sq->flow_rule = NULL;
b96c9dde
MB		 1196}
1197
0fb2ed66 1198static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
b0ea0fa5 1199 struct ib_udata *udata,
0fb2ed66 1200 struct mlx5_ib_sq *sq, void *qpin,
1201 struct ib_pd *pd)
1202{
1203 struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1204 __be64 *pas;
1205 void *in;
1206 void *sqc;
1207 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1208 void *wq;
1209 int inlen;
1210 int err;
1211 int page_shift = 0;
1212 int npages;
1213 int ncont = 0;
1214 u32 offset = 0;
1215
b0ea0fa5
JG		 1216 err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr, ubuffer->buf_size,
1217 &sq->ubuffer.umem, &npages, &page_shift, &ncont,
1218 &offset);
0fb2ed66 1219 if (err)
1220 return err;
1221
1222 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;
1b9a07ee 1223 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1224 if (!in) {
1225 err = -ENOMEM;
1226 goto err_umem;
1227 }
1228
c14003f0 1229 MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1230 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1231 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
795b609c
BW		 1232 if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1233 MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
0fb2ed66 1234 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1235 MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1236 MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1237 MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1238 MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
96dc3fc5
NO		 1239 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1240 MLX5_CAP_ETH(dev->mdev, swp))
1241 MLX5_SET(sqc, sqc, allow_swp, 1);
0fb2ed66 1242
1243 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1244 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1245 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1246 MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1247 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1248 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1249 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1250 MLX5_SET(wq, wq, log_wq_pg_sz, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1251 MLX5_SET(wq, wq, page_offset, offset);
1252
1253 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1254 mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
1255
333fbaa0 1256 err = mlx5_core_create_sq_tracked(dev, in, inlen, &sq->base.mqp);
0fb2ed66 1257
1258 kvfree(in);
1259
1260 if (err)
1261 goto err_umem;
1262
1263 return 0;
1264
1265err_umem:
1266 ib_umem_release(sq->ubuffer.umem);
1267 sq->ubuffer.umem = NULL;
1268
1269 return err;
1270}
1271
1272static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1273 struct mlx5_ib_sq *sq)
1274{
d5ed8ac3 1275 destroy_flow_rule_vport_sq(sq);
333fbaa0 1276 mlx5_core_destroy_sq_tracked(dev, &sq->base.mqp);
0fb2ed66 1277 ib_umem_release(sq->ubuffer.umem);
1278}
1279
2c292dbb 1280static size_t get_rq_pas_size(void *qpc)
0fb2ed66 1281{
1282 u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
1283 u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
1284 u32 log_rq_size = MLX5_GET(qpc, qpc, log_rq_size);
1285 u32 page_offset = MLX5_GET(qpc, qpc, page_offset);
1286 u32 po_quanta = 1 << (log_page_size - 6);
1287 u32 rq_sz = 1 << (log_rq_size + 4 + log_rq_stride);
1288 u32 page_size = 1 << log_page_size;
1289 u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
1290 u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size;
1291
1292 return rq_num_pas * sizeof(u64);
1293}
1294
1295static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2c292dbb 1296 struct mlx5_ib_rq *rq, void *qpin,
34d57585 1297 size_t qpinlen, struct ib_pd *pd)
0fb2ed66 1298{
358e42ea 1299 struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
0fb2ed66 1300 __be64 *pas;
1301 __be64 *qp_pas;
1302 void *in;
1303 void *rqc;
1304 void *wq;
1305 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
2c292dbb
BP		 1306 size_t rq_pas_size = get_rq_pas_size(qpc);
1307 size_t inlen;
0fb2ed66 1308 int err;
2c292dbb
BP		 1309
1310 if (qpinlen < rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas))
1311 return -EINVAL;
0fb2ed66 1312
1313 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
1b9a07ee 1314 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1315 if (!in)
1316 return -ENOMEM;
1317
34d57585 1318 MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1319 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
e4cc4fa7
NO		 1320 if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1321 MLX5_SET(rqc, rqc, vsd, 1);
0fb2ed66 1322 MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1323 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1324 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1325 MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1326 MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1327
2be08c30 1328 if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
358e42ea
MD		 1329 MLX5_SET(rqc, rqc, scatter_fcs, 1);
1330
0fb2ed66 1331 wq = MLX5_ADDR_OF(rqc, rqc, wq);
1332 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 1333 if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1334 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
0fb2ed66 1335 MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));
1336 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1337 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1338 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1339 MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));
1340 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1341
1342 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1343 qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
1344 memcpy(pas, qp_pas, rq_pas_size);
1345
333fbaa0 1346 err = mlx5_core_create_rq_tracked(dev, in, inlen, &rq->base.mqp);
0fb2ed66 1347
1348 kvfree(in);
1349
1350 return err;
1351}
1352
1353static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1354 struct mlx5_ib_rq *rq)
1355{
333fbaa0 1356 mlx5_core_destroy_rq_tracked(dev, &rq->base.mqp);
0fb2ed66 1357}
1358
0042f9e4
MB		 1359static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1360 struct mlx5_ib_rq *rq,
443c1cf9
YH		 1361 u32 qp_flags_en,
1362 struct ib_pd *pd)
0042f9e4
MB		 1363{
1364 if (qp_flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1365 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1366 mlx5_ib_disable_lb(dev, false, true);
443c1cf9 1367 mlx5_cmd_destroy_tir(dev->mdev, rq->tirn, to_mpd(pd)->uid);
0042f9e4
MB		 1368}
1369
0fb2ed66 1370static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
f95ef6cb 1371 struct mlx5_ib_rq *rq, u32 tdn,
e0b4b472
LR		 1372 u32 *qp_flags_en, struct ib_pd *pd,
1373 u32 *out)
0fb2ed66 1374{
175edba8 1375 u8 lb_flag = 0;
0fb2ed66 1376 u32 *in;
1377 void *tirc;
1378 int inlen;
1379 int err;
1380
1381 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 1382 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1383 if (!in)
1384 return -ENOMEM;
1385
443c1cf9 1386 MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1387 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1388 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1389 MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1390 MLX5_SET(tirc, tirc, transport_domain, tdn);
175edba8 1391 if (*qp_flags_en & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
f95ef6cb 1392 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
0fb2ed66 1393
175edba8
MB		 1394 if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1395 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1396
1397 if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1398 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1399
6a4d00be 1400 if (dev->is_rep) {
175edba8
MB		 1401 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1402 *qp_flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1403 }
1404
1405 MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
e0b4b472
LR		 1406 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1407 err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1f1d6abb 1408 rq->tirn = MLX5_GET(create_tir_out, out, tirn);
0042f9e4
MB		 1409 if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1410 err = mlx5_ib_enable_lb(dev, false, true);
1411
1412 if (err)
443c1cf9 1413 destroy_raw_packet_qp_tir(dev, rq, 0, pd);
0042f9e4 1414 }
0fb2ed66 1415 kvfree(in);
1416
1417 return err;
1418}
1419
0fb2ed66 1420static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2c292dbb 1421 u32 *in, size_t inlen,
7f72052c
YH		 1422 struct ib_pd *pd,
1423 struct ib_udata *udata,
1424 struct mlx5_ib_create_qp_resp *resp)
0fb2ed66 1425{
1426 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1427 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1428 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
89944450
SR		 1429 struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1430 udata, struct mlx5_ib_ucontext, ibucontext);
0fb2ed66 1431 int err;
1432 u32 tdn = mucontext->tdn;
7f72052c 1433 u16 uid = to_mpd(pd)->uid;
1f1d6abb 1434 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
0fb2ed66 1435
0eacc574
AL		 1436 if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
1437 return -EINVAL;
0fb2ed66 1438 if (qp->sq.wqe_cnt) {
1cd6dbd3 1439 err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
0fb2ed66 1440 if (err)
1441 return err;
1442
b0ea0fa5 1443 err = create_raw_packet_qp_sq(dev, udata, sq, in, pd);
0fb2ed66 1444 if (err)
1445 goto err_destroy_tis;
1446
7f72052c
YH		 1447 if (uid) {
1448 resp->tisn = sq->tisn;
1449 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TISN;
1450 resp->sqn = sq->base.mqp.qpn;
1451 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
1452 }
1453
0fb2ed66 1454 sq->base.container_mibqp = qp;
1d31e9c0 1455 sq->base.mqp.event = mlx5_ib_qp_event;
0fb2ed66 1456 }
1457
1458 if (qp->rq.wqe_cnt) {
358e42ea
MD		 1459 rq->base.container_mibqp = qp;
1460
2be08c30 1461 if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
e4cc4fa7 1462 rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
2be08c30 1463 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
b1383aa6 1464 rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
34d57585 1465 err = create_raw_packet_qp_rq(dev, rq, in, inlen, pd);
0fb2ed66 1466 if (err)
1467 goto err_destroy_sq;
1468
e0b4b472
LR		 1469 err = create_raw_packet_qp_tir(dev, rq, tdn, &qp->flags_en, pd,
1470 out);
0fb2ed66 1471 if (err)
1472 goto err_destroy_rq;
7f72052c
YH		 1473
1474 if (uid) {
1475 resp->rqn = rq->base.mqp.qpn;
1476 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
1477 resp->tirn = rq->tirn;
1478 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1f1d6abb
AL		 1479 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) {
1480 resp->tir_icm_addr = MLX5_GET(
1481 create_tir_out, out, icm_address_31_0);
1482 resp->tir_icm_addr |=
1483 (u64)MLX5_GET(create_tir_out, out,
1484 icm_address_39_32)
1485 << 32;
1486 resp->tir_icm_addr |=
1487 (u64)MLX5_GET(create_tir_out, out,
1488 icm_address_63_40)
1489 << 40;
1490 resp->comp_mask |=
1491 MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1492 }
7f72052c 1493 }
0fb2ed66 1494 }
1495
1496 qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1497 rq->base.mqp.qpn;
0fb2ed66 1498 return 0;
1499
1500err_destroy_rq:
1501 destroy_raw_packet_qp_rq(dev, rq);
1502err_destroy_sq:
1503 if (!qp->sq.wqe_cnt)
1504 return err;
1505 destroy_raw_packet_qp_sq(dev, sq);
1506err_destroy_tis:
1cd6dbd3 1507 destroy_raw_packet_qp_tis(dev, sq, pd);
0fb2ed66 1508
1509 return err;
1510}
1511
1512static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1513 struct mlx5_ib_qp *qp)
1514{
1515 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1516 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1517 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1518
1519 if (qp->rq.wqe_cnt) {
443c1cf9 1520 destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, qp->ibqp.pd);
0fb2ed66 1521 destroy_raw_packet_qp_rq(dev, rq);
1522 }
1523
1524 if (qp->sq.wqe_cnt) {
1525 destroy_raw_packet_qp_sq(dev, sq);
1cd6dbd3 1526 destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd);
0fb2ed66 1527 }
1528}
1529
1530static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1531 struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1532{
1533 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1534 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1535
1536 sq->sq = &qp->sq;
1537 rq->rq = &qp->rq;
1538 sq->doorbell = &qp->db;
1539 rq->doorbell = &qp->db;
1540}
1541
28d61370
YH		 1542static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1543{
0042f9e4
MB		 1544 if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1545 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1546 mlx5_ib_disable_lb(dev, false, true);
443c1cf9
YH		 1547 mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1548 to_mpd(qp->ibqp.pd)->uid);
28d61370
YH		 1549}
1550
f78d358c
LR		 1551struct mlx5_create_qp_params {
1552 struct ib_udata *udata;
1553 size_t inlen;
6f2cf76e 1554 size_t outlen;
e383085c 1555 size_t ucmd_size;
f78d358c
LR		 1556 void *ucmd;
1557 u8 is_rss_raw : 1;
1558 struct ib_qp_init_attr *attr;
1559 u32 uidx;
08d53976 1560 struct mlx5_ib_create_qp_resp resp;
f78d358c
LR		 1561};
1562
1563static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1564 struct mlx5_ib_qp *qp,
1565 struct mlx5_create_qp_params *params)
28d61370 1566{
f78d358c
LR		 1567 struct ib_qp_init_attr *init_attr = params->attr;
1568 struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
1569 struct ib_udata *udata = params->udata;
89944450
SR		 1570 struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1571 udata, struct mlx5_ib_ucontext, ibucontext);
28d61370 1572 int inlen;
1f1d6abb 1573 int outlen;
28d61370
YH		 1574 int err;
1575 u32 *in;
1f1d6abb 1576 u32 *out;
28d61370
YH		 1577 void *tirc;
1578 void *hfso;
1579 u32 selected_fields = 0;
2d93fc85 1580 u32 outer_l4;
28d61370 1581 u32 tdn = mucontext->tdn;
175edba8 1582 u8 lb_flag = 0;
28d61370 1583
5ce0592b 1584 if (ucmd->comp_mask) {
28d61370
YH		 1585 mlx5_ib_dbg(dev, "invalid comp mask\n");
1586 return -EOPNOTSUPP;
1587 }
1588
5ce0592b
LR		 1589 if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1590 !(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
309fa347
MG		 1591 mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1592 return -EOPNOTSUPP;
1593 }
1594
37518fa4 1595 if (dev->is_rep)
175edba8 1596 qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
175edba8 1597
37518fa4
LR		 1598 if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1599 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1600
1601 if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
175edba8 1602 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
175edba8 1603
28d61370 1604 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1f1d6abb
AL		 1605 outlen = MLX5_ST_SZ_BYTES(create_tir_out);
1606 in = kvzalloc(inlen + outlen, GFP_KERNEL);
28d61370
YH		 1607 if (!in)
1608 return -ENOMEM;
1609
1f1d6abb 1610 out = in + MLX5_ST_SZ_DW(create_tir_in);
443c1cf9 1611 MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
28d61370
YH		 1612 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1613 MLX5_SET(tirc, tirc, disp_type,
1614 MLX5_TIRC_DISP_TYPE_INDIRECT);
1615 MLX5_SET(tirc, tirc, indirect_table,
1616 init_attr->rwq_ind_tbl->ind_tbl_num);
1617 MLX5_SET(tirc, tirc, transport_domain, tdn);
1618
1619 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
f95ef6cb 1620
5ce0592b 1621 if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
f95ef6cb
MG		 1622 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1623
175edba8
MB		 1624 MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1625
5ce0592b 1626 if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
309fa347
MG		 1627 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1628 else
1629 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1630
5ce0592b 1631 switch (ucmd->rx_hash_function) {
28d61370
YH		 1632 case MLX5_RX_HASH_FUNC_TOEPLITZ:
1633 {
1634 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1635 size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1636
5ce0592b 1637 if (len != ucmd->rx_key_len) {
28d61370
YH		 1638 err = -EINVAL;
1639 goto err;
1640 }
1641
1642 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
5ce0592b 1643 memcpy(rss_key, ucmd->rx_hash_key, len);
28d61370
YH		 1644 break;
1645 }
1646 default:
1647 err = -EOPNOTSUPP;
1648 goto err;
1649 }
1650
5ce0592b 1651 if (!ucmd->rx_hash_fields_mask) {
28d61370
YH		 1652 /* special case when this TIR serves as steering entry without hashing */
1653 if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1654 goto create_tir;
1655 err = -EINVAL;
1656 goto err;
1657 }
1658
5ce0592b
LR		 1659 if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1660 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1661 ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1662 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
28d61370
YH		 1663 err = -EINVAL;
1664 goto err;
1665 }
1666
1667 /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
5ce0592b
LR		 1668 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1669 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
28d61370
YH		 1670 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1671 MLX5_L3_PROT_TYPE_IPV4);
5ce0592b
LR		 1672 else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1673 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
28d61370
YH		 1674 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1675 MLX5_L3_PROT_TYPE_IPV6);
1676
5ce0592b
LR		 1677 outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1678 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1679 << 0 |
1680 ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1681 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1682 << 1 |
1683 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
2d93fc85
MB		 1684
1685 /* Check that only one l4 protocol is set */
1686 if (outer_l4 & (outer_l4 - 1)) {
28d61370
YH		 1687 err = -EINVAL;
1688 goto err;
1689 }
1690
1691 /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
5ce0592b
LR		 1692 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1693 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
28d61370
YH		 1694 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1695 MLX5_L4_PROT_TYPE_TCP);
5ce0592b
LR		 1696 else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1697 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
28d61370
YH		 1698 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1699 MLX5_L4_PROT_TYPE_UDP);
1700
5ce0592b
LR		 1701 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1702 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
28d61370
YH		 1703 selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1704
5ce0592b
LR		 1705 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1706 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
28d61370
YH		 1707 selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1708
5ce0592b
LR		 1709 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1710 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
28d61370
YH		 1711 selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1712
5ce0592b
LR		 1713 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1714 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
28d61370
YH		 1715 selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1716
5ce0592b 1717 if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
2d93fc85
MB		 1718 selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1719
28d61370
YH		 1720 MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1721
1722create_tir:
e0b4b472
LR		 1723 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1724 err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
28d61370 1725
1f1d6abb 1726 qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
0042f9e4
MB		 1727 if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1728 err = mlx5_ib_enable_lb(dev, false, true);
1729
1730 if (err)
443c1cf9
YH		 1731 mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1732 to_mpd(pd)->uid);
0042f9e4
MB		 1733 }
1734
28d61370
YH		 1735 if (err)
1736 goto err;
1737
7f72052c 1738 if (mucontext->devx_uid) {
08d53976
LR		 1739 params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1740 params->resp.tirn = qp->rss_qp.tirn;
1f1d6abb 1741 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) {
08d53976 1742 params->resp.tir_icm_addr =
1f1d6abb 1743 MLX5_GET(create_tir_out, out, icm_address_31_0);
08d53976
LR		 1744 params->resp.tir_icm_addr |=
1745 (u64)MLX5_GET(create_tir_out, out,
1746 icm_address_39_32)
1747 << 32;
1748 params->resp.tir_icm_addr |=
1749 (u64)MLX5_GET(create_tir_out, out,
1750 icm_address_63_40)
1751 << 40;
1752 params->resp.comp_mask |=
1f1d6abb
AL		 1753 MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1754 }
7f72052c
YH		 1755 }
1756
28d61370
YH		 1757 kvfree(in);
1758 /* qpn is reserved for that QP */
1759 qp->trans_qp.base.mqp.qpn = 0;
2be08c30 1760 qp->is_rss = true;
28d61370
YH		 1761 return 0;
1762
1763err:
1764 kvfree(in);
1765 return err;
1766}
1767
5d6ff1ba
YC		 1768static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
1769 struct ib_qp_init_attr *init_attr,
6f4bc0ea 1770 struct mlx5_ib_create_qp *ucmd,
5d6ff1ba
YC		 1771 void *qpc)
1772{
5d6ff1ba 1773 int scqe_sz;
2ab367a7 1774 bool allow_scat_cqe = false;
5d6ff1ba 1775
6f4bc0ea
YC		 1776 if (ucmd)
1777 allow_scat_cqe = ucmd->flags & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
1778
1779 if (!allow_scat_cqe && init_attr->sq_sig_type != IB_SIGNAL_ALL_WR)
5d6ff1ba
YC		 1780 return;
1781
1782 scqe_sz = mlx5_ib_get_cqe_size(init_attr->send_cq);
1783 if (scqe_sz == 128) {
1784 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1785 return;
1786 }
1787
1788 if (init_attr->qp_type != MLX5_IB_QPT_DCI ||
1789 MLX5_CAP_GEN(dev->mdev, dc_req_scat_data_cqe))
1790 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1791}
1792
a60109dc
YC		 1793static int atomic_size_to_mode(int size_mask)
1794{
1795 /* driver does not support atomic_size > 256B
1796 * and does not know how to translate bigger sizes
1797 */
1798 int supported_size_mask = size_mask & 0x1ff;
1799 int log_max_size;
1800
1801 if (!supported_size_mask)
1802 return -EOPNOTSUPP;
1803
1804 log_max_size = __fls(supported_size_mask);
1805
1806 if (log_max_size > 3)
1807 return log_max_size;
1808
1809 return MLX5_ATOMIC_MODE_8B;
1810}
1811
1812static int get_atomic_mode(struct mlx5_ib_dev *dev,
1813 enum ib_qp_type qp_type)
1814{
1815 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
1816 u8 atomic = MLX5_CAP_GEN(dev->mdev, atomic);
1817 int atomic_mode = -EOPNOTSUPP;
1818 int atomic_size_mask;
1819
1820 if (!atomic)
1821 return -EOPNOTSUPP;
1822
1823 if (qp_type == MLX5_IB_QPT_DCT)
1824 atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
1825 else
1826 atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
1827
1828 if ((atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP) ||
1829 (atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD))
1830 atomic_mode = atomic_size_to_mode(atomic_size_mask);
1831
1832 if (atomic_mode <= 0 &&
1833 (atomic_operations & MLX5_ATOMIC_OPS_CMP_SWAP &&
1834 atomic_operations & MLX5_ATOMIC_OPS_FETCH_ADD))
1835 atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;
1836
1837 return atomic_mode;
1838}
1839
f78d358c
LR		 1840static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1841 struct mlx5_create_qp_params *params)
04bcc1c2 1842{
e383085c 1843 struct mlx5_ib_create_qp *ucmd = params->ucmd;
f78d358c 1844 struct ib_qp_init_attr *attr = params->attr;
f78d358c 1845 u32 uidx = params->uidx;
04bcc1c2 1846 struct mlx5_ib_resources *devr = &dev->devr;
3e09a427 1847 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
04bcc1c2
LR		 1848 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
1849 struct mlx5_core_dev *mdev = dev->mdev;
1850 struct mlx5_ib_qp_base *base;
1851 unsigned long flags;
1852 void *qpc;
1853 u32 *in;
1854 int err;
1855
1856 mutex_init(&qp->mutex);
1857
1858 if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1859 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
1860
1861 in = kvzalloc(inlen, GFP_KERNEL);
1862 if (!in)
1863 return -ENOMEM;
1864
e383085c
LR		 1865 if (MLX5_CAP_GEN(mdev, ece_support))
1866 MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
04bcc1c2
LR		 1867 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1868
1869 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
1870 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
1871 MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
1872
1873 if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
1874 MLX5_SET(qpc, qpc, block_lb_mc, 1);
1875 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
1876 MLX5_SET(qpc, qpc, cd_master, 1);
1877 if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
1878 MLX5_SET(qpc, qpc, cd_slave_send, 1);
1879 if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
1880 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
1881
1882 MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
1883 MLX5_SET(qpc, qpc, no_sq, 1);
1884 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1885 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
1886 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1887 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
1888 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
1889
1890 /* 0xffffff means we ask to work with cqe version 0 */
1891 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
1892 MLX5_SET(qpc, qpc, user_index, uidx);
1893
1894 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
1895 MLX5_SET(qpc, qpc, end_padding_mode,
1896 MLX5_WQ_END_PAD_MODE_ALIGN);
1897 /* Special case to clean flag */
1898 qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
1899 }
1900
1901 base = &qp->trans_qp.base;
3e09a427 1902 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
04bcc1c2 1903 kvfree(in);
6367da46 1904 if (err)
04bcc1c2 1905 return err;
04bcc1c2
LR		 1906
1907 base->container_mibqp = qp;
1908 base->mqp.event = mlx5_ib_qp_event;
3e09a427 1909 params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
04bcc1c2
LR		 1910
1911 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1912 list_add_tail(&qp->qps_list, &dev->qp_list);
1913 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1914
968f0b6f 1915 qp->trans_qp.xrcdn = to_mxrcd(attr->xrcd)->xrcdn;
04bcc1c2
LR		 1916 return 0;
1917}
1918
98fc1126 1919static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
f78d358c
LR		 1920 struct mlx5_ib_qp *qp,
1921 struct mlx5_create_qp_params *params)
e126ba97 1922{
f78d358c
LR		 1923 struct ib_qp_init_attr *init_attr = params->attr;
1924 struct mlx5_ib_create_qp *ucmd = params->ucmd;
3e09a427 1925 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
f78d358c
LR		 1926 struct ib_udata *udata = params->udata;
1927 u32 uidx = params->uidx;
e126ba97 1928 struct mlx5_ib_resources *devr = &dev->devr;
09a7d9ec 1929 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
938fe83c 1930 struct mlx5_core_dev *mdev = dev->mdev;
89ea94a7
MG		 1931 struct mlx5_ib_cq *send_cq;
1932 struct mlx5_ib_cq *recv_cq;
1933 unsigned long flags;
09a7d9ec 1934 struct mlx5_ib_qp_base *base;
e7b169f3 1935 int mlx5_st;
cfb5e088 1936 void *qpc;
09a7d9ec
SM		 1937 u32 *in;
1938 int err;
e126ba97
EC		 1939
1940 mutex_init(&qp->mutex);
1941 spin_lock_init(&qp->sq.lock);
1942 spin_lock_init(&qp->rq.lock);
1943
7aede1a2 1944 mlx5_st = to_mlx5_st(qp->type);
e7b169f3
NO		 1945 if (mlx5_st < 0)
1946 return -EINVAL;
1947
e126ba97
EC		 1948 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1949 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
1950
2978975c
LR		 1951 if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
1952 qp->underlay_qpn = init_attr->source_qpn;
1953
c2e53b2c 1954 base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2be08c30 1955 qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
c2e53b2c
YH		 1956 &qp->raw_packet_qp.rq.base :
1957 &qp->trans_qp.base;
1958
e126ba97 1959 qp->has_rq = qp_has_rq(init_attr);
2dfac92d 1960 err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
e126ba97
EC		 1961 if (err) {
1962 mlx5_ib_dbg(dev, "err %d\n", err);
1963 return err;
1964 }
1965
98fc1126
LR		 1966 if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
1967 ucmd->rq_wqe_count != qp->rq.wqe_cnt)
1968 return -EINVAL;
04bcc1c2 1969
98fc1126
LR		 1970 if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
1971 return -EINVAL;
e126ba97 1972
08d53976
LR		 1973 err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
1974 &inlen, base, ucmd);
04bcc1c2
LR		 1975 if (err)
1976 return err;
e126ba97
EC		 1977
1978 if (is_sqp(init_attr->qp_type))
1979 qp->port = init_attr->port_num;
1980
e383085c
LR		 1981 if (MLX5_CAP_GEN(mdev, ece_support))
1982 MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
09a7d9ec
SM		 1983 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1984
e7b169f3 1985 MLX5_SET(qpc, qpc, st, mlx5_st);
09a7d9ec 1986 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
98fc1126 1987 MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
e126ba97 1988
c95e6d53 1989 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
09a7d9ec 1990 MLX5_SET(qpc, qpc, wq_signature, 1);
e126ba97 1991
2be08c30 1992 if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
09a7d9ec 1993 MLX5_SET(qpc, qpc, block_lb_mc, 1);
f360d88a 1994
2be08c30 1995 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
09a7d9ec 1996 MLX5_SET(qpc, qpc, cd_master, 1);
2be08c30 1997 if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
09a7d9ec 1998 MLX5_SET(qpc, qpc, cd_slave_send, 1);
2be08c30 1999 if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
09a7d9ec 2000 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2be08c30 2001 if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
569c6651 2002 MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
90ecb37a
LR		 2003 if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2004 (init_attr->qp_type == IB_QPT_RC ||
2005 init_attr->qp_type == IB_QPT_UC)) {
52c81f47 2006 int rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
8bde2c50
LR		 2007
2008 MLX5_SET(qpc, qpc, cs_res,
2009 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
2010 MLX5_RES_SCAT_DATA32_CQE);
2011 }
90ecb37a 2012 if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
7aede1a2 2013 (qp->type == MLX5_IB_QPT_DCI || qp->type == IB_QPT_RC))
2dfac92d 2014 configure_requester_scat_cqe(dev, init_attr, ucmd, qpc);
e126ba97
EC		 2015
2016 if (qp->rq.wqe_cnt) {
09a7d9ec
SM		 2017 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2018 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
e126ba97
EC		 2019 }
2020
09a7d9ec 2021 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
e126ba97 2022
3fd3307e 2023 if (qp->sq.wqe_cnt) {
09a7d9ec 2024 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
3fd3307e 2025 } else {
09a7d9ec 2026 MLX5_SET(qpc, qpc, no_sq, 1);
3fd3307e
AK		 2027 if (init_attr->srq &&
2028 init_attr->srq->srq_type == IB_SRQT_TM)
2029 MLX5_SET(qpc, qpc, offload_type,
2030 MLX5_QPC_OFFLOAD_TYPE_RNDV);
2031 }
e126ba97
EC		 2032
2033 /* Set default resources */
2034 switch (init_attr->qp_type) {
e126ba97 2035 case IB_QPT_XRC_INI:
09a7d9ec
SM		 2036 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2037 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
2038 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
e126ba97
EC		 2039 break;
2040 default:
2041 if (init_attr->srq) {
09a7d9ec
SM		 2042 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
2043 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
e126ba97 2044 } else {
09a7d9ec
SM		 2045 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
2046 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
e126ba97
EC		 2047 }
2048 }
2049
2050 if (init_attr->send_cq)
09a7d9ec 2051 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
e126ba97
EC		 2052
2053 if (init_attr->recv_cq)
09a7d9ec 2054 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
e126ba97 2055
09a7d9ec 2056 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
e126ba97 2057
09a7d9ec
SM		 2058 /* 0xffffff means we ask to work with cqe version 0 */
2059 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
cfb5e088 2060 MLX5_SET(qpc, qpc, user_index, uidx);
09a7d9ec 2061
2978975c
LR		 2062 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
2063 init_attr->qp_type != IB_QPT_RAW_PACKET) {
2064 MLX5_SET(qpc, qpc, end_padding_mode,
2065 MLX5_WQ_END_PAD_MODE_ALIGN);
2066 /* Special case to clean flag */
2067 qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
b1383aa6
NO		 2068 }
2069
c2e53b2c 2070 if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2be08c30 2071 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2dfac92d 2072 qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
0fb2ed66 2073 raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
7f72052c 2074 err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
08d53976 2075 &params->resp);
04bcc1c2 2076 } else
3e09a427 2077 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
e126ba97 2078
479163f4 2079 kvfree(in);
04bcc1c2
LR		 2080 if (err)
2081 goto err_create;
e126ba97 2082
19098df2 2083 base->container_mibqp = qp;
2084 base->mqp.event = mlx5_ib_qp_event;
3e09a427 2085 params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
e126ba97 2086
7aede1a2 2087 get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
89ea94a7
MG		 2088 &send_cq, &recv_cq);
2089 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2090 mlx5_ib_lock_cqs(send_cq, recv_cq);
2091 /* Maintain device to QPs access, needed for further handling via reset
2092 * flow
2093 */
2094 list_add_tail(&qp->qps_list, &dev->qp_list);
2095 /* Maintain CQ to QPs access, needed for further handling via reset flow
2096 */
2097 if (send_cq)
2098 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2099 if (recv_cq)
2100 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2101 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2102 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2103
e126ba97
EC		 2104 return 0;
2105
2106err_create:
747c519c 2107 destroy_qp(dev, qp, base, udata);
e126ba97
EC		 2108 return err;
2109}
2110
98fc1126 2111static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
f78d358c
LR		 2112 struct mlx5_ib_qp *qp,
2113 struct mlx5_create_qp_params *params)
98fc1126 2114{
f78d358c
LR		 2115 struct ib_qp_init_attr *attr = params->attr;
2116 u32 uidx = params->uidx;
98fc1126 2117 struct mlx5_ib_resources *devr = &dev->devr;
3e09a427 2118 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
98fc1126
LR		 2119 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2120 struct mlx5_core_dev *mdev = dev->mdev;
2121 struct mlx5_ib_cq *send_cq;
2122 struct mlx5_ib_cq *recv_cq;
2123 unsigned long flags;
2124 struct mlx5_ib_qp_base *base;
2125 int mlx5_st;
2126 void *qpc;
2127 u32 *in;
2128 int err;
2129
2130 mutex_init(&qp->mutex);
2131 spin_lock_init(&qp->sq.lock);
2132 spin_lock_init(&qp->rq.lock);
2133
2134 mlx5_st = to_mlx5_st(qp->type);
2135 if (mlx5_st < 0)
2136 return -EINVAL;
2137
2138 if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2139 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2140
2141 base = &qp->trans_qp.base;
2142
2143 qp->has_rq = qp_has_rq(attr);
2144 err = set_rq_size(dev, &attr->cap, qp->has_rq, qp, NULL);
2145 if (err) {
2146 mlx5_ib_dbg(dev, "err %d\n", err);
2147 return err;
2148 }
2149
2150 err = _create_kernel_qp(dev, attr, qp, &in, &inlen, base);
2151 if (err)
2152 return err;
2153
2154 if (is_sqp(attr->qp_type))
2155 qp->port = attr->port_num;
2156
2157 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2158
2159 MLX5_SET(qpc, qpc, st, mlx5_st);
2160 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2161
2162 if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
2163 MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
2164 else
2165 MLX5_SET(qpc, qpc, latency_sensitive, 1);
2166
2167
2168 if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2169 MLX5_SET(qpc, qpc, block_lb_mc, 1);
2170
2171 if (qp->rq.wqe_cnt) {
2172 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2173 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2174 }
2175
2176 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
2177
2178 if (qp->sq.wqe_cnt)
2179 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2180 else
2181 MLX5_SET(qpc, qpc, no_sq, 1);
2182
2183 if (attr->srq) {
2184 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
2185 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2186 to_msrq(attr->srq)->msrq.srqn);
2187 } else {
2188 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
2189 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2190 to_msrq(devr->s1)->msrq.srqn);
2191 }
2192
2193 if (attr->send_cq)
2194 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
2195
2196 if (attr->recv_cq)
2197 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
2198
2199 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2200
2201 /* 0xffffff means we ask to work with cqe version 0 */
2202 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2203 MLX5_SET(qpc, qpc, user_index, uidx);
2204
2205 /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
2206 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
2207 MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
2208
3e09a427 2209 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
98fc1126
LR		 2210 kvfree(in);
2211 if (err)
2212 goto err_create;
2213
2214 base->container_mibqp = qp;
2215 base->mqp.event = mlx5_ib_qp_event;
2216
2217 get_cqs(qp->type, attr->send_cq, attr->recv_cq,
2218 &send_cq, &recv_cq);
2219 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2220 mlx5_ib_lock_cqs(send_cq, recv_cq);
2221 /* Maintain device to QPs access, needed for further handling via reset
2222 * flow
2223 */
2224 list_add_tail(&qp->qps_list, &dev->qp_list);
2225 /* Maintain CQ to QPs access, needed for further handling via reset flow
2226 */
2227 if (send_cq)
2228 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2229 if (recv_cq)
2230 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2231 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2232 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2233
2234 return 0;
2235
2236err_create:
747c519c 2237 destroy_qp(dev, qp, base, NULL);
98fc1126
LR		 2238 return err;
2239}
2240
e126ba97
EC		 2241static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2242 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
2243{
2244 if (send_cq) {
2245 if (recv_cq) {
2246 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
89ea94a7 2247 spin_lock(&send_cq->lock);
e126ba97
EC		 2248 spin_lock_nested(&recv_cq->lock,
2249 SINGLE_DEPTH_NESTING);
2250 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
89ea94a7 2251 spin_lock(&send_cq->lock);
e126ba97
EC		 2252 __acquire(&recv_cq->lock);
2253 } else {
89ea94a7 2254 spin_lock(&recv_cq->lock);
e126ba97
EC		 2255 spin_lock_nested(&send_cq->lock,
2256 SINGLE_DEPTH_NESTING);
2257 }
2258 } else {
89ea94a7 2259 spin_lock(&send_cq->lock);
6a4f139a 2260 __acquire(&recv_cq->lock);
e126ba97
EC		 2261 }
2262 } else if (recv_cq) {
89ea94a7 2263 spin_lock(&recv_cq->lock);
6a4f139a
EC		 2264 __acquire(&send_cq->lock);
2265 } else {
2266 __acquire(&send_cq->lock);
2267 __acquire(&recv_cq->lock);
e126ba97
EC		 2268 }
2269}
2270
2271static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2272 __releases(&send_cq->lock) __releases(&recv_cq->lock)
2273{
2274 if (send_cq) {
2275 if (recv_cq) {
2276 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
2277 spin_unlock(&recv_cq->lock);
89ea94a7 2278 spin_unlock(&send_cq->lock);
e126ba97
EC		 2279 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2280 __release(&recv_cq->lock);
89ea94a7 2281 spin_unlock(&send_cq->lock);
e126ba97
EC		 2282 } else {
2283 spin_unlock(&send_cq->lock);
89ea94a7 2284 spin_unlock(&recv_cq->lock);
e126ba97
EC		 2285 }
2286 } else {
6a4f139a 2287 __release(&recv_cq->lock);
89ea94a7 2288 spin_unlock(&send_cq->lock);
e126ba97
EC		 2289 }
2290 } else if (recv_cq) {
6a4f139a 2291 __release(&send_cq->lock);
89ea94a7 2292 spin_unlock(&recv_cq->lock);
6a4f139a
EC		 2293 } else {
2294 __release(&recv_cq->lock);
2295 __release(&send_cq->lock);
e126ba97
EC		 2296 }
2297}
2298
89ea94a7
MG		 2299static void get_cqs(enum ib_qp_type qp_type,
2300 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
e126ba97
EC		 2301 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2302{
89ea94a7 2303 switch (qp_type) {
e126ba97
EC		 2304 case IB_QPT_XRC_TGT:
2305 *send_cq = NULL;
2306 *recv_cq = NULL;
2307 break;
2308 case MLX5_IB_QPT_REG_UMR:
2309 case IB_QPT_XRC_INI:
89ea94a7 2310 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
e126ba97
EC		 2311 *recv_cq = NULL;
2312 break;
2313
2314 case IB_QPT_SMI:
d16e91da 2315 case MLX5_IB_QPT_HW_GSI:
e126ba97
EC		 2316 case IB_QPT_RC:
2317 case IB_QPT_UC:
2318 case IB_QPT_UD:
0fb2ed66 2319 case IB_QPT_RAW_PACKET:
89ea94a7
MG		 2320 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2321 *recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
e126ba97 2322 break;
e126ba97
EC		 2323 default:
2324 *send_cq = NULL;
2325 *recv_cq = NULL;
2326 break;
2327 }
2328}
2329
ad5f8e96 2330static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 2331 const struct mlx5_modify_raw_qp_param *raw_qp_param,
2332 u8 lag_tx_affinity);
ad5f8e96 2333
bdeacabd
SR		 2334static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2335 struct ib_udata *udata)
e126ba97
EC		 2336{
2337 struct mlx5_ib_cq *send_cq, *recv_cq;
c2e53b2c 2338 struct mlx5_ib_qp_base *base;
89ea94a7 2339 unsigned long flags;
e126ba97
EC		 2340 int err;
2341
28d61370
YH		 2342 if (qp->ibqp.rwq_ind_tbl) {
2343 destroy_rss_raw_qp_tir(dev, qp);
2344 return;
2345 }
2346
c2e53b2c 2347 base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2be08c30 2348 qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
0fb2ed66 2349 &qp->raw_packet_qp.rq.base :
2350 &qp->trans_qp.base;
2351
6aec21f6 2352 if (qp->state != IB_QPS_RESET) {
c2e53b2c 2353 if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET &&
2be08c30 2354 !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
333fbaa0 2355 err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,
1a412fb1 2356 NULL, &base->mqp);
ad5f8e96 2357 } else {
0680efa2
AV		 2358 struct mlx5_modify_raw_qp_param raw_qp_param = {
2359 .operation = MLX5_CMD_OP_2RST_QP
2360 };
2361
13eab21f 2362 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
ad5f8e96 2363 }
2364 if (err)
427c1e7b 2365 mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
19098df2 2366 base->mqp.qpn);
6aec21f6 2367 }
e126ba97 2368
89ea94a7
MG		 2369 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
2370 &send_cq, &recv_cq);
2371
2372 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2373 mlx5_ib_lock_cqs(send_cq, recv_cq);
2374 /* del from lists under both locks above to protect reset flow paths */
2375 list_del(&qp->qps_list);
2376 if (send_cq)
2377 list_del(&qp->cq_send_list);
2378
2379 if (recv_cq)
2380 list_del(&qp->cq_recv_list);
e126ba97 2381
03c4077b 2382 if (!udata) {
19098df2 2383 __mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 2384 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2385 if (send_cq != recv_cq)
19098df2 2386 __mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
2387 NULL);
e126ba97 2388 }
89ea94a7
MG		 2389 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2390 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
e126ba97 2391
c2e53b2c 2392 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2be08c30 2393 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
0fb2ed66 2394 destroy_raw_packet_qp(dev, qp);
2395 } else {
333fbaa0 2396 err = mlx5_core_destroy_qp(dev, &base->mqp);
0fb2ed66 2397 if (err)
2398 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2399 base->mqp.qpn);
2400 }
e126ba97 2401
747c519c 2402 destroy_qp(dev, qp, base, udata);
e126ba97
EC		 2403}
2404
47c80612 2405static int create_dct(struct ib_pd *pd, struct mlx5_ib_qp *qp,
f78d358c 2406 struct mlx5_create_qp_params *params)
b4aaa1f0 2407{
f78d358c
LR		 2408 struct ib_qp_init_attr *attr = params->attr;
2409 struct mlx5_ib_create_qp *ucmd = params->ucmd;
2410 u32 uidx = params->uidx;
b4aaa1f0
MS		 2411 void *dctc;
2412
b4aaa1f0 2413 qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
9c2ba4ed 2414 if (!qp->dct.in)
47c80612 2415 return -ENOMEM;
b4aaa1f0 2416
a01a5860 2417 MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
b4aaa1f0 2418 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
b4aaa1f0
MS		 2419 MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2420 MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2421 MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2422 MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2423 MLX5_SET(dctc, dctc, user_index, uidx);
2424
37518fa4 2425 if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
fd9dab7e
LR		 2426 int rcqe_sz = mlx5_ib_get_cqe_size(attr->recv_cq);
2427
2428 if (rcqe_sz == 128)
2429 MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
2430 }
5d6ff1ba 2431
b4aaa1f0
MS		 2432 qp->state = IB_QPS_RESET;
2433
47c80612 2434 return 0;
b4aaa1f0
MS		 2435}
2436
7aede1a2
LR		 2437static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
2438 enum ib_qp_type *type)
6eb7edff
LR		 2439{
2440 if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
2441 goto out;
2442
2443 switch (attr->qp_type) {
2444 case IB_QPT_XRC_TGT:
2445 case IB_QPT_XRC_INI:
2446 if (!MLX5_CAP_GEN(dev->mdev, xrc))
2447 goto out;
2448 fallthrough;
6eb7edff
LR		 2449 case IB_QPT_RC:
2450 case IB_QPT_UC:
6eb7edff
LR		 2451 case IB_QPT_SMI:
2452 case MLX5_IB_QPT_HW_GSI:
6eb7edff
LR		 2453 case IB_QPT_DRIVER:
2454 case IB_QPT_GSI:
42caf9cb
MB		 2455 if (dev->profile == &raw_eth_profile)
2456 goto out;
2457 case IB_QPT_RAW_PACKET:
2458 case IB_QPT_UD:
2459 case MLX5_IB_QPT_REG_UMR:
7aede1a2 2460 break;
6eb7edff
LR		 2461 default:
2462 goto out;
b4aaa1f0
MS		 2463 }
2464
7aede1a2 2465 *type = attr->qp_type;
b4aaa1f0 2466 return 0;
6eb7edff
LR		 2467
2468out:
2469 mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
2470 return -EOPNOTSUPP;
b4aaa1f0
MS		 2471}
2472
2242cc25
LR		 2473static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2474 struct ib_qp_init_attr *attr,
2475 struct ib_udata *udata)
2476{
2477 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2478 udata, struct mlx5_ib_ucontext, ibucontext);
2479
2480 if (!udata) {
2481 /* Kernel create_qp callers */
2482 if (attr->rwq_ind_tbl)
2483 return -EOPNOTSUPP;
2484
2485 switch (attr->qp_type) {
2486 case IB_QPT_RAW_PACKET:
2487 case IB_QPT_DRIVER:
2488 return -EOPNOTSUPP;
2489 default:
2490 return 0;
2491 }
2492 }
2493
2494 /* Userspace create_qp callers */
2495 if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
2496 mlx5_ib_dbg(dev,
2497 "Raw Packet QP is only supported for CQE version > 0\n");
2498 return -EINVAL;
2499 }
2500
2501 if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
2502 mlx5_ib_dbg(dev,
2503 "Wrong QP type %d for the RWQ indirect table\n",
2504 attr->qp_type);
2505 return -EINVAL;
2506 }
2507
2508 switch (attr->qp_type) {
2509 case IB_QPT_SMI:
2510 case MLX5_IB_QPT_HW_GSI:
2511 case MLX5_IB_QPT_REG_UMR:
2512 case IB_QPT_GSI:
2513 mlx5_ib_dbg(dev, "Kernel doesn't support QP type %d\n",
2514 attr->qp_type);
2515 return -EINVAL;
2516 default:
2517 break;
2518 }
2519
2520 /*
2521 * We don't need to see this warning, it means that kernel code
2522 * missing ib_pd. Placed here to catch developer's mistakes.
2523 */
2524 WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
2525 "There is a missing PD pointer assignment\n");
2526 return 0;
2527}
2528
37518fa4
LR		 2529static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2530 bool cond, struct mlx5_ib_qp *qp)
2531{
2532 if (!(*flags & flag))
2533 return;
2534
2535 if (cond) {
2536 qp->flags_en |= flag;
2537 *flags &= ~flag;
2538 return;
2539 }
2540
2541 if (flag == MLX5_QP_FLAG_SCATTER_CQE) {
2542 /*
2543 * We don't return error if this flag was provided,
2544 * and mlx5 doesn't have right capability.
2545 */
2546 *flags &= ~MLX5_QP_FLAG_SCATTER_CQE;
2547 return;
2548 }
2549 mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
2550}
2551
2552static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
5ce0592b 2553 void *ucmd, struct ib_qp_init_attr *attr)
2fdddbd5 2554{
37518fa4 2555 struct mlx5_core_dev *mdev = dev->mdev;
37518fa4 2556 bool cond;
5ce0592b
LR		 2557 int flags;
2558
2559 if (attr->rwq_ind_tbl)
2560 flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
2561 else
2562 flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
37518fa4
LR		 2563
2564 switch (flags & (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {
2fdddbd5 2565 case MLX5_QP_FLAG_TYPE_DCI:
7aede1a2 2566 qp->type = MLX5_IB_QPT_DCI;
2fdddbd5
LR		 2567 break;
2568 case MLX5_QP_FLAG_TYPE_DCT:
7aede1a2 2569 qp->type = MLX5_IB_QPT_DCT;
37518fa4 2570 break;
7aede1a2
LR		 2571 default:
2572 if (qp->type != IB_QPT_DRIVER)
2573 break;
2574 /*
2575 * It is IB_QPT_DRIVER and or no subtype or
2576 * wrong subtype were provided.
2577 */
2fdddbd5 2578 return -EINVAL;
7aede1a2 2579 }
37518fa4
LR		 2580
2581 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);
2582 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);
2583
2584 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SIGNATURE, true, qp);
2585 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SCATTER_CQE,
2586 MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2587
7aede1a2 2588 if (qp->type == IB_QPT_RAW_PACKET) {
37518fa4
LR		 2589 cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||
2590 MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||
2591 MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
2592 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,
2593 cond, qp);
2594 process_vendor_flag(dev, &flags,
2595 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,
2596 qp);
2597 process_vendor_flag(dev, &flags,
2598 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,
2599 qp);
2fdddbd5
LR		 2600 }
2601
7aede1a2 2602 if (qp->type == IB_QPT_RC)
37518fa4
LR		 2603 process_vendor_flag(dev, &flags,
2604 MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
2605 MLX5_CAP_GEN(mdev, qp_packet_based), qp);
2606
76883a6c
LR		 2607 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);
2608 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);
2609
5d6fffed
LR		 2610 cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
2611 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
2612 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
2613 if (attr->rwq_ind_tbl && cond) {
2614 mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
2615 cond);
2616 return -EINVAL;
2617 }
2618
37518fa4
LR		 2619 if (flags)
2620 mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
2621
2622 return (flags) ? -EINVAL : 0;
5d6fffed 2623 }
2fdddbd5 2624
2978975c
LR		 2625static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2626 bool cond, struct mlx5_ib_qp *qp)
2627{
2628 if (!(*flags & flag))
2629 return;
2630
2631 if (cond) {
2632 qp->flags |= flag;
2633 *flags &= ~flag;
2634 return;
2635 }
2636
2637 if (flag == MLX5_IB_QP_CREATE_WC_TEST) {
2638 /*
2639 * Special case, if condition didn't meet, it won't be error,
2640 * just different in-kernel flow.
2641 */
2642 *flags &= ~MLX5_IB_QP_CREATE_WC_TEST;
2643 return;
2644 }
2645 mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
2646}
2647
2648static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2649 struct ib_qp_init_attr *attr)
2650{
7aede1a2 2651 enum ib_qp_type qp_type = qp->type;
2978975c
LR		 2652 struct mlx5_core_dev *mdev = dev->mdev;
2653 int create_flags = attr->create_flags;
2654 bool cond;
2655
42caf9cb
MB		 2656 if (qp->type == IB_QPT_UD && dev->profile == &raw_eth_profile)
2657 if (create_flags & ~MLX5_IB_QP_CREATE_WC_TEST)
2658 return -EINVAL;
2659
7aede1a2 2660 if (qp_type == MLX5_IB_QPT_DCT)
2978975c
LR		 2661 return (create_flags) ? -EINVAL : 0;
2662
2663 if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
2664 return (create_flags) ? -EINVAL : 0;
2665
2666 process_create_flag(dev, &create_flags,
2667 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
2668 MLX5_CAP_GEN(mdev, block_lb_mc), qp);
2669 process_create_flag(dev, &create_flags, IB_QP_CREATE_CROSS_CHANNEL,
2670 MLX5_CAP_GEN(mdev, cd), qp);
2671 process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_SEND,
2672 MLX5_CAP_GEN(mdev, cd), qp);
2673 process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_RECV,
2674 MLX5_CAP_GEN(mdev, cd), qp);
2675
2676 if (qp_type == IB_QPT_UD) {
2677 process_create_flag(dev, &create_flags,
2678 IB_QP_CREATE_IPOIB_UD_LSO,
2679 MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
2680 qp);
2681 cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
2682 process_create_flag(dev, &create_flags, IB_QP_CREATE_SOURCE_QPN,
2683 cond, qp);
2684 }
2685
2686 if (qp_type == IB_QPT_RAW_PACKET) {
2687 cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
2688 MLX5_CAP_ETH(mdev, scatter_fcs);
2689 process_create_flag(dev, &create_flags,
2690 IB_QP_CREATE_SCATTER_FCS, cond, qp);
2691
2692 cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
2693 MLX5_CAP_ETH(mdev, vlan_cap);
2694 process_create_flag(dev, &create_flags,
2695 IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
2696 }
2697
2698 process_create_flag(dev, &create_flags,
2699 IB_QP_CREATE_PCI_WRITE_END_PADDING,
2700 MLX5_CAP_GEN(mdev, end_pad), qp);
2701
2702 process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_WC_TEST,
2703 qp_type != MLX5_IB_QPT_REG_UMR, qp);
2704 process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
2705 true, qp);
2706
2707 if (create_flags)
2708 mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
2709 create_flags);
2710
2711 return (create_flags) ? -EINVAL : 0;
2712}
2713
6f2cf76e
LR		 2714static int process_udata_size(struct mlx5_ib_dev *dev,
2715 struct mlx5_create_qp_params *params)
2fdddbd5
LR		 2716{
2717 size_t ucmd = sizeof(struct mlx5_ib_create_qp);
6f2cf76e
LR		 2718 struct ib_udata *udata = params->udata;
2719 size_t outlen = udata->outlen;
5ce0592b 2720 size_t inlen = udata->inlen;
2fdddbd5 2721
6f2cf76e 2722 params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
e383085c 2723 params->ucmd_size = ucmd;
6f2cf76e 2724 if (!params->is_rss_raw) {
e383085c
LR		 2725 /* User has old rdma-core, which doesn't support ECE */
2726 size_t min_inlen =
2727 offsetof(struct mlx5_ib_create_qp, ece_options);
2728
2729 /*
2730 * We will check in check_ucmd_data() that user
2731 * cleared everything after inlen.
2732 */
2733 params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
6f2cf76e
LR		 2734 goto out;
2735 }
5ce0592b 2736
6f2cf76e 2737 /* RSS RAW QP */
5ce0592b 2738 if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
6f2cf76e
LR		 2739 return -EINVAL;
2740
2741 if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
2742 return -EINVAL;
5ce0592b
LR		 2743
2744 ucmd = sizeof(struct mlx5_ib_create_qp_rss);
e383085c 2745 params->ucmd_size = ucmd;
5ce0592b 2746 if (inlen > ucmd && !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))
6f2cf76e
LR		 2747 return -EINVAL;
2748
2749 params->inlen = min(ucmd, inlen);
2750out:
2751 if (!params->inlen)
e383085c 2752 mlx5_ib_dbg(dev, "udata is too small\n");
2dfac92d 2753
6f2cf76e 2754 return (params->inlen) ? 0 : -EINVAL;
2fdddbd5
LR		 2755}
2756
968f0b6f
LR		 2757static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2758 struct mlx5_ib_qp *qp,
2759 struct mlx5_create_qp_params *params)
5d0dc3d9 2760{
968f0b6f
LR		 2761 int err;
2762
2763 if (params->is_rss_raw) {
2764 err = create_rss_raw_qp_tir(dev, pd, qp, params);
2765 goto out;
2766 }
2767
2768 if (qp->type == MLX5_IB_QPT_DCT) {
2769 err = create_dct(pd, qp, params);
2770 goto out;
2771 }
2772
2773 if (qp->type == IB_QPT_XRC_TGT) {
2774 err = create_xrc_tgt_qp(dev, qp, params);
2775 goto out;
2776 }
5d0dc3d9 2777
968f0b6f
LR		 2778 if (params->udata)
2779 err = create_user_qp(dev, pd, qp, params);
2780 else
2781 err = create_kernel_qp(dev, pd, qp, params);
2782
2783out:
2784 if (err) {
2785 mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
2786 return err;
2787 }
2788
2789 if (is_qp0(qp->type))
2790 qp->ibqp.qp_num = 0;
2791 else if (is_qp1(qp->type))
2792 qp->ibqp.qp_num = 1;
2793 else
2794 qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
2795
2796 mlx5_ib_dbg(dev,
3e09a427 2797 "QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
968f0b6f
LR		 2798 qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
2799 params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
2800 -1,
2801 params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
3e09a427
LR		 2802 -1,
2803 params->resp.ece_options);
968f0b6f
LR		 2804
2805 return 0;
5d0dc3d9
LR		 2806}
2807
7aede1a2
LR		 2808static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2809 struct ib_qp_init_attr *attr)
2810{
2811 int ret = 0;
2812
2813 switch (qp->type) {
2814 case MLX5_IB_QPT_DCT:
2815 ret = (!attr->srq || !attr->recv_cq) ? -EINVAL : 0;
2816 break;
2817 case MLX5_IB_QPT_DCI:
2818 ret = (attr->cap.max_recv_wr || attr->cap.max_recv_sge) ?
2819 -EINVAL :
2820 0;
2821 break;
266424eb
LR		 2822 case IB_QPT_RAW_PACKET:
2823 ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
2824 break;
7aede1a2
LR		 2825 default:
2826 break;
2827 }
2828
2829 if (ret)
2830 mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
2831
2832 return ret;
2833}
2834
f78d358c
LR		 2835static int get_qp_uidx(struct mlx5_ib_qp *qp,
2836 struct mlx5_create_qp_params *params)
21aad80b 2837{
f78d358c
LR		 2838 struct mlx5_ib_create_qp *ucmd = params->ucmd;
2839 struct ib_udata *udata = params->udata;
21aad80b
LR		 2840 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2841 udata, struct mlx5_ib_ucontext, ibucontext);
2842
f78d358c 2843 if (params->is_rss_raw)
21aad80b
LR		 2844 return 0;
2845
f78d358c 2846 return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &params->uidx);
21aad80b
LR		 2847}
2848
08d53976
LR		 2849static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
2850{
2851 struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
2852
2853 if (mqp->state == IB_QPS_RTR) {
2854 int err;
2855
2856 err = mlx5_core_destroy_dct(dev, &mqp->dct.mdct);
2857 if (err) {
2858 mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
2859 return err;
2860 }
2861 }
2862
2863 kfree(mqp->dct.in);
2864 kfree(mqp);
2865 return 0;
2866}
2867
e383085c
LR		 2868static int check_ucmd_data(struct mlx5_ib_dev *dev,
2869 struct mlx5_create_qp_params *params)
2870{
2871 struct ib_qp_init_attr *attr = params->attr;
2872 struct ib_udata *udata = params->udata;
2873 size_t size, last;
2874 int ret;
2875
2876 if (params->is_rss_raw)
2877 /*
2878 * These QPs don't have "reserved" field in their
2879 * create_qp input struct, so their data is always valid.
2880 */
2881 last = sizeof(struct mlx5_ib_create_qp_rss);
2882 else
2883 /* IB_QPT_RAW_PACKET and IB_QPT_DRIVER don't have ECE data */
2884 switch (attr->qp_type) {
2885 case IB_QPT_DRIVER:
2886 case IB_QPT_RAW_PACKET:
2887 last = offsetof(struct mlx5_ib_create_qp, ece_options);
2888 break;
2889 default:
2890 last = offsetof(struct mlx5_ib_create_qp, reserved);
2891 }
2892
2893 if (udata->inlen <= last)
2894 return 0;
2895
2896 /*
2897 * User provides different create_qp structures based on the
2898 * flow and we need to know if he cleared memory after our
2899 * struct create_qp ends.
2900 */
2901 size = udata->inlen - last;
2902 ret = ib_is_udata_cleared(params->udata, last, size);
2903 if (!ret)
2904 mlx5_ib_dbg(
2905 dev,
2906 "udata is not cleared, inlen = %lu, ucmd = %lu, last = %lu, size = %lu\n",
2907 udata->inlen, params->ucmd_size, last, size);
2908 return ret ? 0 : -EINVAL;
2909}
2910
f78d358c 2911struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr,
e126ba97
EC		 2912 struct ib_udata *udata)
2913{
f78d358c 2914 struct mlx5_create_qp_params params = {};
e126ba97
EC		 2915 struct mlx5_ib_dev *dev;
2916 struct mlx5_ib_qp *qp;
7aede1a2 2917 enum ib_qp_type type;
e126ba97
EC		 2918 int err;
2919
6eb7edff 2920 dev = pd ? to_mdev(pd->device) :
f78d358c 2921 to_mdev(to_mxrcd(attr->xrcd)->ibxrcd.device);
0fb2ed66 2922
f78d358c
LR		 2923 err = check_qp_type(dev, attr, &type);
2924 if (err)
6eb7edff 2925 return ERR_PTR(err);
6eb7edff 2926
f78d358c 2927 err = check_valid_flow(dev, pd, attr, udata);
2242cc25
LR		 2928 if (err)
2929 return ERR_PTR(err);
e126ba97 2930
f78d358c
LR		 2931 if (attr->qp_type == IB_QPT_GSI)
2932 return mlx5_ib_gsi_create_qp(pd, attr);
9c2ba4ed 2933
f78d358c
LR		 2934 params.udata = udata;
2935 params.uidx = MLX5_IB_DEFAULT_UIDX;
2936 params.attr = attr;
2937 params.is_rss_raw = !!attr->rwq_ind_tbl;
2fdddbd5 2938
f78d358c 2939 if (udata) {
6f2cf76e
LR		 2940 err = process_udata_size(dev, &params);
2941 if (err)
2942 return ERR_PTR(err);
2fdddbd5 2943
e383085c
LR		 2944 err = check_ucmd_data(dev, &params);
2945 if (err)
2946 return ERR_PTR(err);
2947
2948 params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
f78d358c 2949 if (!params.ucmd)
5ce0592b
LR		 2950 return ERR_PTR(-ENOMEM);
2951
f78d358c 2952 err = ib_copy_from_udata(params.ucmd, udata, params.inlen);
2fdddbd5 2953 if (err)
5ce0592b 2954 goto free_ucmd;
2fdddbd5
LR		 2955 }
2956
9c2ba4ed 2957 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
5ce0592b
LR		 2958 if (!qp) {
2959 err = -ENOMEM;
2960 goto free_ucmd;
2961 }
9c2ba4ed 2962
7aede1a2 2963 qp->type = type;
37518fa4 2964 if (udata) {
f78d358c 2965 err = process_vendor_flags(dev, qp, params.ucmd, attr);
b4aaa1f0 2966 if (err)
9c2ba4ed 2967 goto free_qp;
21aad80b 2968
f78d358c 2969 err = get_qp_uidx(qp, &params);
21aad80b
LR		 2970 if (err)
2971 goto free_qp;
b4aaa1f0 2972 }
f78d358c 2973 err = process_create_flags(dev, qp, attr);
2978975c
LR		 2974 if (err)
2975 goto free_qp;
b4aaa1f0 2976
f78d358c 2977 err = check_qp_attr(dev, qp, attr);
7aede1a2
LR		 2978 if (err)
2979 goto free_qp;
2980
968f0b6f
LR		 2981 err = create_qp(dev, pd, qp, &params);
2982 if (err)
9c2ba4ed 2983 goto free_qp;
e126ba97 2984
f78d358c 2985 kfree(params.ucmd);
08d53976 2986 params.ucmd = NULL;
5ce0592b 2987
08d53976
LR		 2988 if (udata)
2989 /*
2990 * It is safe to copy response for all user create QP flows,
2991 * including MLX5_IB_QPT_DCT, which doesn't need it.
2992 * In that case, resp will be filled with zeros.
2993 */
2994 err = ib_copy_to_udata(udata, &params.resp, params.outlen);
2995 if (err)
2996 goto destroy_qp;
2997
e126ba97 2998 return &qp->ibqp;
9c2ba4ed 2999
08d53976
LR		 3000destroy_qp:
3001 if (qp->type == MLX5_IB_QPT_DCT)
3002 mlx5_ib_destroy_dct(qp);
3003 else
3004 destroy_qp_common(dev, qp, udata);
3005 qp = NULL;
9c2ba4ed
LR		 3006free_qp:
3007 kfree(qp);
5ce0592b 3008free_ucmd:
f78d358c 3009 kfree(params.ucmd);
9c2ba4ed 3010 return ERR_PTR(err);
e126ba97
EC		 3011}
3012
c4367a26 3013int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
e126ba97
EC		 3014{
3015 struct mlx5_ib_dev *dev = to_mdev(qp->device);
3016 struct mlx5_ib_qp *mqp = to_mqp(qp);
3017
d16e91da
HE		 3018 if (unlikely(qp->qp_type == IB_QPT_GSI))
3019 return mlx5_ib_gsi_destroy_qp(qp);
3020
7aede1a2 3021 if (mqp->type == MLX5_IB_QPT_DCT)
776a3906
MS		 3022 return mlx5_ib_destroy_dct(mqp);
3023
bdeacabd 3024 destroy_qp_common(dev, mqp, udata);
e126ba97
EC		 3025
3026 kfree(mqp);
3027
3028 return 0;
3029}
3030
a60109dc
YC		 3031static int to_mlx5_access_flags(struct mlx5_ib_qp *qp,
3032 const struct ib_qp_attr *attr,
bf3b4f06 3033 int attr_mask, __be32 *hw_access_flags_be)
e126ba97 3034{
e126ba97 3035 u8 dest_rd_atomic;
bf3b4f06 3036 u32 access_flags, hw_access_flags = 0;
e126ba97 3037
a60109dc
YC		 3038 struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
3039
e126ba97
EC		 3040 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3041 dest_rd_atomic = attr->max_dest_rd_atomic;
3042 else
19098df2 3043 dest_rd_atomic = qp->trans_qp.resp_depth;
e126ba97
EC		 3044
3045 if (attr_mask & IB_QP_ACCESS_FLAGS)
3046 access_flags = attr->qp_access_flags;
3047 else
19098df2 3048 access_flags = qp->trans_qp.atomic_rd_en;
e126ba97
EC		 3049
3050 if (!dest_rd_atomic)
3051 access_flags &= IB_ACCESS_REMOTE_WRITE;
3052
3053 if (access_flags & IB_ACCESS_REMOTE_READ)
bf3b4f06 3054 hw_access_flags |= MLX5_QP_BIT_RRE;
13f8d9c1 3055 if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
a60109dc
YC		 3056 int atomic_mode;
3057
3058 atomic_mode = get_atomic_mode(dev, qp->ibqp.qp_type);
3059 if (atomic_mode < 0)
3060 return -EOPNOTSUPP;
3061
bf3b4f06
BVA		 3062 hw_access_flags |= MLX5_QP_BIT_RAE;
3063 hw_access_flags |= atomic_mode << MLX5_ATOMIC_MODE_OFFSET;
a60109dc
YC		 3064 }
3065
e126ba97 3066 if (access_flags & IB_ACCESS_REMOTE_WRITE)
bf3b4f06 3067 hw_access_flags |= MLX5_QP_BIT_RWE;
a60109dc 3068
bf3b4f06 3069 *hw_access_flags_be = cpu_to_be32(hw_access_flags);
e126ba97 3070
a60109dc 3071 return 0;
e126ba97
EC		 3072}
3073
3074enum {
3075 MLX5_PATH_FLAG_FL = 1 << 0,
3076 MLX5_PATH_FLAG_FREE_AR = 1 << 1,
3077 MLX5_PATH_FLAG_COUNTER = 1 << 2,
3078};
3079
3080static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
3081{
4f32ac2e 3082 if (rate == IB_RATE_PORT_CURRENT)
e126ba97 3083 return 0;
4f32ac2e 3084
a5a5d199 3085 if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_600_GBPS)
e126ba97 3086 return -EINVAL;
e126ba97 3087
4f32ac2e
DG		 3088 while (rate != IB_RATE_PORT_CURRENT &&
3089 !(1 << (rate + MLX5_STAT_RATE_OFFSET) &
3090 MLX5_CAP_GEN(dev->mdev, stat_rate_support)))
3091 --rate;
3092
3093 return rate ? rate + MLX5_STAT_RATE_OFFSET : rate;
e126ba97
EC		 3094}
3095
75850d0b 3096static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
1cd6dbd3
YH		 3097 struct mlx5_ib_sq *sq, u8 sl,
3098 struct ib_pd *pd)
75850d0b 3099{
3100 void *in;
3101 void *tisc;
3102 int inlen;
3103 int err;
3104
3105 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 3106 in = kvzalloc(inlen, GFP_KERNEL);
75850d0b 3107 if (!in)
3108 return -ENOMEM;
3109
3110 MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
1cd6dbd3 3111 MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
75850d0b 3112
3113 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3114 MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
3115
e0b4b472 3116 err = mlx5_core_modify_tis(dev, sq->tisn, in);
75850d0b 3117
3118 kvfree(in);
3119
3120 return err;
3121}
3122
13eab21f 3123static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
1cd6dbd3
YH		 3124 struct mlx5_ib_sq *sq, u8 tx_affinity,
3125 struct ib_pd *pd)
13eab21f
AH		 3126{
3127 void *in;
3128 void *tisc;
3129 int inlen;
3130 int err;
3131
3132 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 3133 in = kvzalloc(inlen, GFP_KERNEL);
13eab21f
AH		 3134 if (!in)
3135 return -ENOMEM;
3136
3137 MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
1cd6dbd3 3138 MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
13eab21f
AH		 3139
3140 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3141 MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
3142
e0b4b472 3143 err = mlx5_core_modify_tis(dev, sq->tisn, in);
13eab21f
AH		 3144
3145 kvfree(in);
3146
3147 return err;
3148}
3149
2b880b2e
MZ		 3150static void mlx5_set_path_udp_sport(struct mlx5_qp_path *path,
3151 const struct rdma_ah_attr *ah,
3152 u32 lqpn, u32 rqpn)
3153
3154{
3155 u32 fl = ah->grh.flow_label;
3156 u16 sport;
3157
3158 if (!fl)
3159 fl = rdma_calc_flow_label(lqpn, rqpn);
3160
3161 sport = rdma_flow_label_to_udp_sport(fl);
3162 path->udp_sport = cpu_to_be16(sport);
3163}
3164
75850d0b 3165static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
90898850 3166 const struct rdma_ah_attr *ah,
e126ba97 3167 struct mlx5_qp_path *path, u8 port, int attr_mask,
f879ee8d
AS		 3168 u32 path_flags, const struct ib_qp_attr *attr,
3169 bool alt)
e126ba97 3170{
d8966fcd 3171 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
e126ba97 3172 int err;
ed88451e 3173 enum ib_gid_type gid_type;
d8966fcd
DC		 3174 u8 ah_flags = rdma_ah_get_ah_flags(ah);
3175 u8 sl = rdma_ah_get_sl(ah);
e126ba97 3176
e126ba97 3177 if (attr_mask & IB_QP_PKEY_INDEX)
f879ee8d
AS		 3178 path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index :
3179 attr->pkey_index);
e126ba97 3180
d8966fcd
DC		 3181 if (ah_flags & IB_AH_GRH) {
3182 if (grh->sgid_index >=
938fe83c 3183 dev->mdev->port_caps[port - 1].gid_table_len) {
f4f01b54 3184 pr_err("sgid_index (%u) too large. max is %d\n",
d8966fcd 3185 grh->sgid_index,
938fe83c 3186 dev->mdev->port_caps[port - 1].gid_table_len);
f83b4263
EC		 3187 return -EINVAL;
3188 }
2811ba51 3189 }
44c58487
DC		 3190
3191 if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
d8966fcd 3192 if (!(ah_flags & IB_AH_GRH))
2811ba51 3193 return -EINVAL;
47ec3866 3194
44c58487 3195 memcpy(path->rmac, ah->roce.dmac, sizeof(ah->roce.dmac));
2b880b2e
MZ		 3196 if ((qp->ibqp.qp_type == IB_QPT_RC ||
3197 qp->ibqp.qp_type == IB_QPT_UC ||
3198 qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3199 qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&
3200 (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
3201 (attr_mask & IB_QP_DEST_QPN))
3202 mlx5_set_path_udp_sport(path, ah,
3203 qp->ibqp.qp_num,
3204 attr->dest_qp_num);
d8966fcd 3205 path->dci_cfi_prio_sl = (sl & 0x7) << 4;
47ec3866 3206 gid_type = ah->grh.sgid_attr->gid_type;
ed88451e 3207 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
d8966fcd 3208 path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f;
2811ba51 3209 } else {
d3ae2bde
NO		 3210 path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
3211 path->fl_free_ar |=
3212 (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;
d8966fcd
DC		 3213 path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
3214 path->grh_mlid = rdma_ah_get_path_bits(ah) & 0x7f;
3215 if (ah_flags & IB_AH_GRH)
2811ba51 3216 path->grh_mlid |= 1 << 7;
d8966fcd 3217 path->dci_cfi_prio_sl = sl & 0xf;
2811ba51
AS		 3218 }
3219
d8966fcd
DC		 3220 if (ah_flags & IB_AH_GRH) {
3221 path->mgid_index = grh->sgid_index;
3222 path->hop_limit = grh->hop_limit;
e126ba97 3223 path->tclass_flowlabel =
d8966fcd
DC		 3224 cpu_to_be32((grh->traffic_class << 20) |
3225 (grh->flow_label));
3226 memcpy(path->rgid, grh->dgid.raw, 16);
e126ba97
EC		 3227 }
3228
d8966fcd 3229 err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
e126ba97
EC		 3230 if (err < 0)
3231 return err;
3232 path->static_rate = err;
3233 path->port = port;
3234
e126ba97 3235 if (attr_mask & IB_QP_TIMEOUT)
f879ee8d 3236 path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3;
e126ba97 3237
75850d0b 3238 if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
3239 return modify_raw_packet_eth_prio(dev->mdev,
3240 &qp->raw_packet_qp.sq,
1cd6dbd3 3241 sl & 0xf, qp->ibqp.pd);
75850d0b 3242
e126ba97
EC		 3243 return 0;
3244}
3245
3246static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
3247 [MLX5_QP_STATE_INIT] = {
3248 [MLX5_QP_STATE_INIT] = {
3249 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
3250 MLX5_QP_OPTPAR_RAE |
3251 MLX5_QP_OPTPAR_RWE |
3252 MLX5_QP_OPTPAR_PKEY_INDEX |
cfc1a89e
MG		 3253 MLX5_QP_OPTPAR_PRI_PORT |
3254 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3255 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
3256 MLX5_QP_OPTPAR_PKEY_INDEX |
cfc1a89e
MG		 3257 MLX5_QP_OPTPAR_PRI_PORT |
3258 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3259 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
3260 MLX5_QP_OPTPAR_Q_KEY |
3261 MLX5_QP_OPTPAR_PRI_PORT,
8f4426aa
JM		 3262 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE |
3263 MLX5_QP_OPTPAR_RAE |
3264 MLX5_QP_OPTPAR_RWE |
3265 MLX5_QP_OPTPAR_PKEY_INDEX |
cfc1a89e
MG		 3266 MLX5_QP_OPTPAR_PRI_PORT |
3267 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3268 },
3269 [MLX5_QP_STATE_RTR] = {
3270 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3271 MLX5_QP_OPTPAR_RRE |
3272 MLX5_QP_OPTPAR_RAE |
3273 MLX5_QP_OPTPAR_RWE |
cfc1a89e
MG		 3274 MLX5_QP_OPTPAR_PKEY_INDEX |
3275 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3276 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3277 MLX5_QP_OPTPAR_RWE |
cfc1a89e
MG		 3278 MLX5_QP_OPTPAR_PKEY_INDEX |
3279 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3280 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
3281 MLX5_QP_OPTPAR_Q_KEY,
3282 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX |
3283 MLX5_QP_OPTPAR_Q_KEY,
a4774e90
EC		 3284 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3285 MLX5_QP_OPTPAR_RRE |
3286 MLX5_QP_OPTPAR_RAE |
3287 MLX5_QP_OPTPAR_RWE |
cfc1a89e
MG		 3288 MLX5_QP_OPTPAR_PKEY_INDEX |
3289 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3290 },
3291 },
3292 [MLX5_QP_STATE_RTR] = {
3293 [MLX5_QP_STATE_RTS] = {
3294 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3295 MLX5_QP_OPTPAR_RRE |
3296 MLX5_QP_OPTPAR_RAE |
3297 MLX5_QP_OPTPAR_RWE |
3298 MLX5_QP_OPTPAR_PM_STATE |
3299 MLX5_QP_OPTPAR_RNR_TIMEOUT,
3300 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3301 MLX5_QP_OPTPAR_RWE |
3302 MLX5_QP_OPTPAR_PM_STATE,
3303 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
8f4426aa
JM		 3304 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3305 MLX5_QP_OPTPAR_RRE |
3306 MLX5_QP_OPTPAR_RAE |
3307 MLX5_QP_OPTPAR_RWE |
3308 MLX5_QP_OPTPAR_PM_STATE |
3309 MLX5_QP_OPTPAR_RNR_TIMEOUT,
e126ba97
EC		 3310 },
3311 },
3312 [MLX5_QP_STATE_RTS] = {
3313 [MLX5_QP_STATE_RTS] = {
3314 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
3315 MLX5_QP_OPTPAR_RAE |
3316 MLX5_QP_OPTPAR_RWE |
3317 MLX5_QP_OPTPAR_RNR_TIMEOUT |
c2a3431e
EC		 3318 MLX5_QP_OPTPAR_PM_STATE |
3319 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97 3320 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
c2a3431e
EC		 3321 MLX5_QP_OPTPAR_PM_STATE |
3322 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97
EC		 3323 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY |
3324 MLX5_QP_OPTPAR_SRQN |
3325 MLX5_QP_OPTPAR_CQN_RCV,
8f4426aa
JM		 3326 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE |
3327 MLX5_QP_OPTPAR_RAE |
3328 MLX5_QP_OPTPAR_RWE |
3329 MLX5_QP_OPTPAR_RNR_TIMEOUT |
3330 MLX5_QP_OPTPAR_PM_STATE |
3331 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97
EC		 3332 },
3333 },
3334 [MLX5_QP_STATE_SQER] = {
3335 [MLX5_QP_STATE_RTS] = {
3336 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3337 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
75959f56 3338 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
a4774e90
EC		 3339 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3340 MLX5_QP_OPTPAR_RWE |
3341 MLX5_QP_OPTPAR_RAE |
3342 MLX5_QP_OPTPAR_RRE,
8f4426aa
JM		 3343 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3344 MLX5_QP_OPTPAR_RWE |
3345 MLX5_QP_OPTPAR_RAE |
3346 MLX5_QP_OPTPAR_RRE,
e126ba97
EC		 3347 },
3348 },
3349};
3350
3351static int ib_nr_to_mlx5_nr(int ib_mask)
3352{
3353 switch (ib_mask) {
3354 case IB_QP_STATE:
3355 return 0;
3356 case IB_QP_CUR_STATE:
3357 return 0;
3358 case IB_QP_EN_SQD_ASYNC_NOTIFY:
3359 return 0;
3360 case IB_QP_ACCESS_FLAGS:
3361 return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
3362 MLX5_QP_OPTPAR_RAE;
3363 case IB_QP_PKEY_INDEX:
3364 return MLX5_QP_OPTPAR_PKEY_INDEX;
3365 case IB_QP_PORT:
3366 return MLX5_QP_OPTPAR_PRI_PORT;
3367 case IB_QP_QKEY:
3368 return MLX5_QP_OPTPAR_Q_KEY;
3369 case IB_QP_AV:
3370 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
3371 MLX5_QP_OPTPAR_PRI_PORT;
3372 case IB_QP_PATH_MTU:
3373 return 0;
3374 case IB_QP_TIMEOUT:
3375 return MLX5_QP_OPTPAR_ACK_TIMEOUT;
3376 case IB_QP_RETRY_CNT:
3377 return MLX5_QP_OPTPAR_RETRY_COUNT;
3378 case IB_QP_RNR_RETRY:
3379 return MLX5_QP_OPTPAR_RNR_RETRY;
3380 case IB_QP_RQ_PSN:
3381 return 0;
3382 case IB_QP_MAX_QP_RD_ATOMIC:
3383 return MLX5_QP_OPTPAR_SRA_MAX;
3384 case IB_QP_ALT_PATH:
3385 return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
3386 case IB_QP_MIN_RNR_TIMER:
3387 return MLX5_QP_OPTPAR_RNR_TIMEOUT;
3388 case IB_QP_SQ_PSN:
3389 return 0;
3390 case IB_QP_MAX_DEST_RD_ATOMIC:
3391 return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
3392 MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
3393 case IB_QP_PATH_MIG_STATE:
3394 return MLX5_QP_OPTPAR_PM_STATE;
3395 case IB_QP_CAP:
3396 return 0;
3397 case IB_QP_DEST_QPN:
3398 return 0;
3399 }
3400 return 0;
3401}
3402
3403static int ib_mask_to_mlx5_opt(int ib_mask)
3404{
3405 int result = 0;
3406 int i;
3407
3408 for (i = 0; i < 8 * sizeof(int); i++) {
3409 if ((1 << i) & ib_mask)
3410 result |= ib_nr_to_mlx5_nr(1 << i);
3411 }
3412
3413 return result;
3414}
3415
34d57585
YH		 3416static int modify_raw_packet_qp_rq(
3417 struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
3418 const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
ad5f8e96 3419{
3420 void *in;
3421 void *rqc;
3422 int inlen;
3423 int err;
3424
3425 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 3426 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 3427 if (!in)
3428 return -ENOMEM;
3429
3430 MLX5_SET(modify_rq_in, in, rq_state, rq->state);
34d57585 3431 MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
ad5f8e96 3432
3433 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
3434 MLX5_SET(rqc, rqc, state, new_state);
3435
eb49ab0c
AV		 3436 if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
3437 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
3438 MLX5_SET64(modify_rq_in, in, modify_bitmask,
23a6964e 3439 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
eb49ab0c
AV		 3440 MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
3441 } else
5a738b5d
JG		 3442 dev_info_once(
3443 &dev->ib_dev.dev,
3444 "RAW PACKET QP counters are not supported on current FW\n");
eb49ab0c
AV		 3445 }
3446
e0b4b472 3447 err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in);
ad5f8e96 3448 if (err)
3449 goto out;
3450
3451 rq->state = new_state;
3452
3453out:
3454 kvfree(in);
3455 return err;
3456}
3457
c14003f0
YH		 3458static int modify_raw_packet_qp_sq(
3459 struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state,
3460 const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
ad5f8e96 3461{
7d29f349 3462 struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
61147f39
BW		 3463 struct mlx5_rate_limit old_rl = ibqp->rl;
3464 struct mlx5_rate_limit new_rl = old_rl;
3465 bool new_rate_added = false;
7d29f349 3466 u16 rl_index = 0;
ad5f8e96 3467 void *in;
3468 void *sqc;
3469 int inlen;
3470 int err;
3471
3472 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 3473 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 3474 if (!in)
3475 return -ENOMEM;
3476
c14003f0 3477 MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
ad5f8e96 3478 MLX5_SET(modify_sq_in, in, sq_state, sq->state);
3479
3480 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
3481 MLX5_SET(sqc, sqc, state, new_state);
3482
7d29f349
BW		 3483 if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
3484 if (new_state != MLX5_SQC_STATE_RDY)
3485 pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
3486 __func__);
3487 else
61147f39 3488 new_rl = raw_qp_param->rl;
7d29f349
BW		 3489 }
3490
61147f39
BW		 3491 if (!mlx5_rl_are_equal(&old_rl, &new_rl)) {
3492 if (new_rl.rate) {
3493 err = mlx5_rl_add_rate(dev, &rl_index, &new_rl);
7d29f349 3494 if (err) {
61147f39
BW		 3495 pr_err("Failed configuring rate limit(err %d): \
3496 rate %u, max_burst_sz %u, typical_pkt_sz %u\n",
3497 err, new_rl.rate, new_rl.max_burst_sz,
3498 new_rl.typical_pkt_sz);
3499
7d29f349
BW		 3500 goto out;
3501 }
61147f39 3502 new_rate_added = true;
7d29f349
BW		 3503 }
3504
3505 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
61147f39 3506 /* index 0 means no limit */
7d29f349
BW		 3507 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
3508 }
3509
e0b4b472 3510 err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in);
7d29f349
BW		 3511 if (err) {
3512 /* Remove new rate from table if failed */
61147f39
BW		 3513 if (new_rate_added)
3514 mlx5_rl_remove_rate(dev, &new_rl);
ad5f8e96 3515 goto out;
7d29f349
BW		 3516 }
3517
3518 /* Only remove the old rate after new rate was set */
c8973df2
RW		 3519 if ((old_rl.rate && !mlx5_rl_are_equal(&old_rl, &new_rl)) ||
3520 (new_state != MLX5_SQC_STATE_RDY)) {
61147f39 3521 mlx5_rl_remove_rate(dev, &old_rl);
c8973df2
RW		 3522 if (new_state != MLX5_SQC_STATE_RDY)
3523 memset(&new_rl, 0, sizeof(new_rl));
3524 }
ad5f8e96 3525
61147f39 3526 ibqp->rl = new_rl;
ad5f8e96 3527 sq->state = new_state;
3528
3529out:
3530 kvfree(in);
3531 return err;
3532}
3533
3534static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 3535 const struct mlx5_modify_raw_qp_param *raw_qp_param,
3536 u8 tx_affinity)
ad5f8e96 3537{
3538 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
3539 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
3540 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
7d29f349
BW		 3541 int modify_rq = !!qp->rq.wqe_cnt;
3542 int modify_sq = !!qp->sq.wqe_cnt;
ad5f8e96 3543 int rq_state;
3544 int sq_state;
3545 int err;
3546
0680efa2 3547 switch (raw_qp_param->operation) {
ad5f8e96 3548 case MLX5_CMD_OP_RST2INIT_QP:
3549 rq_state = MLX5_RQC_STATE_RDY;
3550 sq_state = MLX5_SQC_STATE_RDY;
3551 break;
3552 case MLX5_CMD_OP_2ERR_QP:
3553 rq_state = MLX5_RQC_STATE_ERR;
3554 sq_state = MLX5_SQC_STATE_ERR;
3555 break;
3556 case MLX5_CMD_OP_2RST_QP:
3557 rq_state = MLX5_RQC_STATE_RST;
3558 sq_state = MLX5_SQC_STATE_RST;
3559 break;
ad5f8e96 3560 case MLX5_CMD_OP_RTR2RTS_QP:
3561 case MLX5_CMD_OP_RTS2RTS_QP:
7d29f349
BW		 3562 if (raw_qp_param->set_mask ==
3563 MLX5_RAW_QP_RATE_LIMIT) {
3564 modify_rq = 0;
3565 sq_state = sq->state;
3566 } else {
3567 return raw_qp_param->set_mask ? -EINVAL : 0;
3568 }
3569 break;
3570 case MLX5_CMD_OP_INIT2INIT_QP:
3571 case MLX5_CMD_OP_INIT2RTR_QP:
eb49ab0c
AV		 3572 if (raw_qp_param->set_mask)
3573 return -EINVAL;
3574 else
3575 return 0;
ad5f8e96 3576 default:
3577 WARN_ON(1);
3578 return -EINVAL;
3579 }
3580
7d29f349 3581 if (modify_rq) {
34d57585
YH		 3582 err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param,
3583 qp->ibqp.pd);
ad5f8e96 3584 if (err)
3585 return err;
3586 }
3587
7d29f349 3588 if (modify_sq) {
d5ed8ac3
MB		 3589 struct mlx5_flow_handle *flow_rule;
3590
13eab21f
AH		 3591 if (tx_affinity) {
3592 err = modify_raw_packet_tx_affinity(dev->mdev, sq,
1cd6dbd3
YH		 3593 tx_affinity,
3594 qp->ibqp.pd);
13eab21f
AH		 3595 if (err)
3596 return err;
3597 }
3598
d5ed8ac3
MB		 3599 flow_rule = create_flow_rule_vport_sq(dev, sq,
3600 raw_qp_param->port);
3601 if (IS_ERR(flow_rule))
1db86318 3602 return PTR_ERR(flow_rule);
d5ed8ac3
MB		 3603
3604 err = modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
3605 raw_qp_param, qp->ibqp.pd);
3606 if (err) {
3607 if (flow_rule)
3608 mlx5_del_flow_rules(flow_rule);
3609 return err;
3610 }
3611
3612 if (flow_rule) {
3613 destroy_flow_rule_vport_sq(sq);
3614 sq->flow_rule = flow_rule;
3615 }
3616
3617 return err;
13eab21f 3618 }
ad5f8e96 3619
3620 return 0;
3621}
3622
5163b274
MG		 3623static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
3624 struct ib_udata *udata)
c6a21c38 3625{
89944450
SR		 3626 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3627 udata, struct mlx5_ib_ucontext, ibucontext);
5163b274
MG		 3628 u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3629 atomic_t *tx_port_affinity;
c6a21c38 3630
5163b274
MG		 3631 if (ucontext)
3632 tx_port_affinity = &ucontext->tx_port_affinity;
3633 else
3634 tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
3635
3636 return (unsigned int)atomic_add_return(1, tx_port_affinity) %
3637 MLX5_MAX_PORTS + 1;
3638}
3639
3640static bool qp_supports_affinity(struct ib_qp *qp)
3641{
5163b274 3642 if ((qp->qp_type == IB_QPT_RC) ||
cfc1a89e 3643 (qp->qp_type == IB_QPT_UD) ||
5163b274
MG		 3644 (qp->qp_type == IB_QPT_UC) ||
3645 (qp->qp_type == IB_QPT_RAW_PACKET) ||
3646 (qp->qp_type == IB_QPT_XRC_INI) ||
3647 (qp->qp_type == IB_QPT_XRC_TGT))
3648 return true;
3649 return false;
3650}
3651
cfc1a89e
MG		 3652static unsigned int get_tx_affinity(struct ib_qp *qp,
3653 const struct ib_qp_attr *attr,
3654 int attr_mask, u8 init,
5163b274
MG		 3655 struct ib_udata *udata)
3656{
3657 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3658 udata, struct mlx5_ib_ucontext, ibucontext);
3659 struct mlx5_ib_dev *dev = to_mdev(qp->device);
3660 struct mlx5_ib_qp *mqp = to_mqp(qp);
3661 struct mlx5_ib_qp_base *qp_base;
3662 unsigned int tx_affinity;
3663
cfc1a89e 3664 if (!(dev->lag_active && qp_supports_affinity(qp)))
5163b274
MG		 3665 return 0;
3666
cfc1a89e
MG		 3667 if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
3668 tx_affinity = mqp->gsi_lag_port;
3669 else if (init)
3670 tx_affinity = get_tx_affinity_rr(dev, udata);
3671 else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
3672 tx_affinity =
3673 mlx5_lag_get_slave_port(dev->mdev, attr->xmit_slave);
3674 else
3675 return 0;
5163b274
MG		 3676
3677 qp_base = &mqp->trans_qp.base;
3678 if (ucontext)
c6a21c38 3679 mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
5163b274
MG		 3680 tx_affinity, qp_base->mqp.qpn, ucontext);
3681 else
c6a21c38 3682 mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
5163b274
MG		 3683 tx_affinity, qp_base->mqp.qpn);
3684 return tx_affinity;
c6a21c38
MD		 3685}
3686
d14133dd
MZ		 3687static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
3688 struct rdma_counter *counter)
3689{
3690 struct mlx5_ib_dev *dev = to_mdev(qp->device);
3691 struct mlx5_ib_qp *mqp = to_mqp(qp);
3692 struct mlx5_qp_context context = {};
d14133dd
MZ		 3693 struct mlx5_ib_qp_base *base;
3694 u32 set_id;
3695
3e1f000f 3696 if (counter)
d14133dd 3697 set_id = counter->id;
3e1f000f
PP		 3698 else
3699 set_id = mlx5_ib_get_counters_id(dev, mqp->port - 1);
d14133dd
MZ		 3700
3701 base = &mqp->trans_qp.base;
3702 context.qp_counter_set_usr_page &= cpu_to_be32(0xffffff);
3703 context.qp_counter_set_usr_page |= cpu_to_be32(set_id << 24);
333fbaa0
LR		 3704 return mlx5_core_qp_modify(dev, MLX5_CMD_OP_RTS2RTS_QP,
3705 MLX5_QP_OPTPAR_COUNTER_SET_ID, &context,
3706 &base->mqp);
d14133dd
MZ		 3707}
3708
e126ba97
EC		 3709static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
3710 const struct ib_qp_attr *attr, int attr_mask,
89944450
SR		 3711 enum ib_qp_state cur_state,
3712 enum ib_qp_state new_state,
3713 const struct mlx5_ib_modify_qp *ucmd,
3714 struct ib_udata *udata)
e126ba97 3715{
427c1e7b 3716 static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
3717 [MLX5_QP_STATE_RST] = {
3718 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3719 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3720 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_RST2INIT_QP,
3721 },
3722 [MLX5_QP_STATE_INIT] = {
3723 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3724 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3725 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_INIT2INIT_QP,
3726 [MLX5_QP_STATE_RTR] = MLX5_CMD_OP_INIT2RTR_QP,
3727 },
3728 [MLX5_QP_STATE_RTR] = {
3729 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3730 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3731 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTR2RTS_QP,
3732 },
3733 [MLX5_QP_STATE_RTS] = {
3734 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3735 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3736 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTS2RTS_QP,
3737 },
3738 [MLX5_QP_STATE_SQD] = {
3739 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3740 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3741 },
3742 [MLX5_QP_STATE_SQER] = {
3743 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3744 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3745 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQERR2RTS_QP,
3746 },
3747 [MLX5_QP_STATE_ERR] = {
3748 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
3749 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
3750 }
3751 };
3752
e126ba97
EC		 3753 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3754 struct mlx5_ib_qp *qp = to_mqp(ibqp);
19098df2 3755 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
e126ba97
EC		 3756 struct mlx5_ib_cq *send_cq, *recv_cq;
3757 struct mlx5_qp_context *context;
e126ba97
EC		 3758 struct mlx5_ib_pd *pd;
3759 enum mlx5_qp_state mlx5_cur, mlx5_new;
cfc1a89e 3760 enum mlx5_qp_optpar optpar = 0;
d14133dd 3761 u32 set_id = 0;
e126ba97
EC		 3762 int mlx5_st;
3763 int err;
427c1e7b 3764 u16 op;
13eab21f 3765 u8 tx_affinity = 0;
e126ba97 3766
7aede1a2 3767 mlx5_st = to_mlx5_st(qp->type);
55de9a77
LR		 3768 if (mlx5_st < 0)
3769 return -EINVAL;
3770
1a412fb1
SM		 3771 context = kzalloc(sizeof(*context), GFP_KERNEL);
3772 if (!context)
e126ba97
EC		 3773 return -ENOMEM;
3774
029e88fd 3775 pd = to_mpd(qp->ibqp.pd);
55de9a77 3776 context->flags = cpu_to_be32(mlx5_st << 16);
e126ba97
EC		 3777
3778 if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
3779 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
3780 } else {
3781 switch (attr->path_mig_state) {
3782 case IB_MIG_MIGRATED:
3783 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
3784 break;
3785 case IB_MIG_REARM:
3786 context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11);
3787 break;
3788 case IB_MIG_ARMED:
3789 context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11);
3790 break;
3791 }
3792 }
3793
cfc1a89e 3794 tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,
5163b274
MG		 3795 cur_state == IB_QPS_RESET &&
3796 new_state == IB_QPS_INIT, udata);
cfc1a89e
MG		 3797 if (tx_affinity) {
3798 context->flags |= cpu_to_be32(tx_affinity << 24);
3799 if (new_state == IB_QPS_RTR &&
3800 MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
3801 optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;
3802 }
13eab21f 3803
d16e91da 3804 if (is_sqp(ibqp->qp_type)) {
e126ba97 3805 context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
c2e53b2c 3806 } else if ((ibqp->qp_type == IB_QPT_UD &&
2be08c30 3807 !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) ||
e126ba97
EC		 3808 ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
3809 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
3810 } else if (attr_mask & IB_QP_PATH_MTU) {
3811 if (attr->path_mtu < IB_MTU_256 ||
3812 attr->path_mtu > IB_MTU_4096) {
3813 mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
3814 err = -EINVAL;
3815 goto out;
3816 }
938fe83c
SM		 3817 context->mtu_msgmax = (attr->path_mtu << 5) |
3818 (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg);
e126ba97
EC		 3819 }
3820
3821 if (attr_mask & IB_QP_DEST_QPN)
3822 context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num);
3823
3824 if (attr_mask & IB_QP_PKEY_INDEX)
d3ae2bde 3825 context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index);
e126ba97
EC		 3826
3827 /* todo implement counter_index functionality */
3828
3829 if (is_sqp(ibqp->qp_type))
3830 context->pri_path.port = qp->port;
3831
3832 if (attr_mask & IB_QP_PORT)
3833 context->pri_path.port = attr->port_num;
3834
3835 if (attr_mask & IB_QP_AV) {
75850d0b 3836 err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
e126ba97 3837 attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
f879ee8d 3838 attr_mask, 0, attr, false);
e126ba97
EC		 3839 if (err)
3840 goto out;
3841 }
3842
3843 if (attr_mask & IB_QP_TIMEOUT)
3844 context->pri_path.ackto_lt |= attr->timeout << 3;
3845
3846 if (attr_mask & IB_QP_ALT_PATH) {
75850d0b 3847 err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
3848 &context->alt_path,
f879ee8d
AS		 3849 attr->alt_port_num,
3850 attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT,
3851 0, attr, true);
e126ba97
EC		 3852 if (err)
3853 goto out;
3854 }
3855
89ea94a7
MG		 3856 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
3857 &send_cq, &recv_cq);
e126ba97
EC		 3858
3859 context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
3860 context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0;
3861 context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0;
3862 context->params1 = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28);
3863
3864 if (attr_mask & IB_QP_RNR_RETRY)
3865 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
3866
3867 if (attr_mask & IB_QP_RETRY_CNT)
3868 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
3869
3870 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3871 if (attr->max_rd_atomic)
3872 context->params1 |=
3873 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
3874 }
3875
3876 if (attr_mask & IB_QP_SQ_PSN)
3877 context->next_send_psn = cpu_to_be32(attr->sq_psn);
3878
3879 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3880 if (attr->max_dest_rd_atomic)
3881 context->params2 |=
3882 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
3883 }
3884
a60109dc 3885 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
bf3b4f06 3886 __be32 access_flags;
a60109dc
YC		 3887
3888 err = to_mlx5_access_flags(qp, attr, attr_mask, &access_flags);
3889 if (err)
3890 goto out;
3891
3892 context->params2 |= access_flags;
3893 }
e126ba97
EC		 3894
3895 if (attr_mask & IB_QP_MIN_RNR_TIMER)
3896 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
3897
3898 if (attr_mask & IB_QP_RQ_PSN)
3899 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
3900
3901 if (attr_mask & IB_QP_QKEY)
3902 context->qkey = cpu_to_be32(attr->qkey);
3903
3904 if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3905 context->db_rec_addr = cpu_to_be64(qp->db.dma);
3906
0837e86a
MB		 3907 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3908 u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
3909 qp->port) - 1;
c2e53b2c
YH		 3910
3911 /* Underlay port should be used - index 0 function per port */
2be08c30 3912 if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
c2e53b2c
YH		 3913 port_num = 0;
3914
d14133dd
MZ		 3915 if (ibqp->counter)
3916 set_id = ibqp->counter->id;
3917 else
3e1f000f 3918 set_id = mlx5_ib_get_counters_id(dev, port_num);
0837e86a 3919 context->qp_counter_set_usr_page |=
d14133dd 3920 cpu_to_be32(set_id << 24);
0837e86a
MB		 3921 }
3922
e126ba97
EC		 3923 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3924 context->sq_crq_size |= cpu_to_be16(1 << 4);
3925
2be08c30 3926 if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
b11a4f9c 3927 context->deth_sqpn = cpu_to_be32(1);
e126ba97
EC		 3928
3929 mlx5_cur = to_mlx5_state(cur_state);
3930 mlx5_new = to_mlx5_state(new_state);
e126ba97 3931
427c1e7b 3932 if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
5d414b17
DC		 3933 !optab[mlx5_cur][mlx5_new]) {
3934 err = -EINVAL;
427c1e7b 3935 goto out;
5d414b17 3936 }
427c1e7b 3937
3938 op = optab[mlx5_cur][mlx5_new];
cfc1a89e 3939 optpar |= ib_mask_to_mlx5_opt(attr_mask);
e126ba97 3940 optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
ad5f8e96 3941
c2e53b2c 3942 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2be08c30 3943 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
0680efa2
AV		 3944 struct mlx5_modify_raw_qp_param raw_qp_param = {};
3945
3946 raw_qp_param.operation = op;
eb49ab0c 3947 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
d14133dd 3948 raw_qp_param.rq_q_ctr_id = set_id;
eb49ab0c
AV		 3949 raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
3950 }
7d29f349 3951
d5ed8ac3
MB		 3952 if (attr_mask & IB_QP_PORT)
3953 raw_qp_param.port = attr->port_num;
3954
7d29f349 3955 if (attr_mask & IB_QP_RATE_LIMIT) {
61147f39
BW		 3956 raw_qp_param.rl.rate = attr->rate_limit;
3957
3958 if (ucmd->burst_info.max_burst_sz) {
3959 if (attr->rate_limit &&
3960 MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) {
3961 raw_qp_param.rl.max_burst_sz =
3962 ucmd->burst_info.max_burst_sz;
3963 } else {
3964 err = -EINVAL;
3965 goto out;
3966 }
3967 }
3968
3969 if (ucmd->burst_info.typical_pkt_sz) {
3970 if (attr->rate_limit &&
3971 MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) {
3972 raw_qp_param.rl.typical_pkt_sz =
3973 ucmd->burst_info.typical_pkt_sz;
3974 } else {
3975 err = -EINVAL;
3976 goto out;
3977 }
3978 }
3979
7d29f349
BW		 3980 raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
3981 }
3982
13eab21f 3983 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
0680efa2 3984 } else {
333fbaa0 3985 err = mlx5_core_qp_modify(dev, op, optpar, context, &base->mqp);
0680efa2
AV		 3986 }
3987
e126ba97
EC		 3988 if (err)
3989 goto out;
3990
3991 qp->state = new_state;
3992
3993 if (attr_mask & IB_QP_ACCESS_FLAGS)
19098df2 3994 qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
e126ba97 3995 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
19098df2 3996 qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
e126ba97
EC		 3997 if (attr_mask & IB_QP_PORT)
3998 qp->port = attr->port_num;
3999 if (attr_mask & IB_QP_ALT_PATH)
19098df2 4000 qp->trans_qp.alt_port = attr->alt_port_num;
e126ba97
EC		 4001
4002 /*
4003 * If we moved a kernel QP to RESET, clean up all old CQ
4004 * entries and reinitialize the QP.
4005 */
75a45982
LR		 4006 if (new_state == IB_QPS_RESET &&
4007 !ibqp->uobject && ibqp->qp_type != IB_QPT_XRC_TGT) {
19098df2 4008 mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 4009 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
4010 if (send_cq != recv_cq)
19098df2 4011 mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
e126ba97
EC		 4012
4013 qp->rq.head = 0;
4014 qp->rq.tail = 0;
4015 qp->sq.head = 0;
4016 qp->sq.tail = 0;
4017 qp->sq.cur_post = 0;
34f4c955
GL		 4018 if (qp->sq.wqe_cnt)
4019 qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
950bf4f1 4020 qp->sq.last_poll = 0;
e126ba97
EC		 4021 qp->db.db[MLX5_RCV_DBR] = 0;
4022 qp->db.db[MLX5_SND_DBR] = 0;
4023 }
4024
d14133dd
MZ		 4025 if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
4026 err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter);
4027 if (!err)
4028 qp->counter_pending = 0;
4029 }
4030
e126ba97 4031out:
1a412fb1 4032 kfree(context);
e126ba97
EC		 4033 return err;
4034}
4035
c32a4f29
MS		 4036static inline bool is_valid_mask(int mask, int req, int opt)
4037{
4038 if ((mask & req) != req)
4039 return false;
4040
4041 if (mask & ~(req | opt))
4042 return false;
4043
4044 return true;
4045}
4046
4047/* check valid transition for driver QP types
4048 * for now the only QP type that this function supports is DCI
4049 */
4050static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
4051 enum ib_qp_attr_mask attr_mask)
4052{
4053 int req = IB_QP_STATE;
4054 int opt = 0;
4055
99ed748e
MS		 4056 if (new_state == IB_QPS_RESET) {
4057 return is_valid_mask(attr_mask, req, opt);
4058 } else if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
c32a4f29
MS		 4059 req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
4060 return is_valid_mask(attr_mask, req, opt);
4061 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4062 opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
4063 return is_valid_mask(attr_mask, req, opt);
4064 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4065 req |= IB_QP_PATH_MTU;
5ec0304c 4066 opt = IB_QP_PKEY_INDEX | IB_QP_AV;
c32a4f29
MS		 4067 return is_valid_mask(attr_mask, req, opt);
4068 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4069 req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4070 IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
4071 opt = IB_QP_MIN_RNR_TIMER;
4072 return is_valid_mask(attr_mask, req, opt);
4073 } else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
4074 opt = IB_QP_MIN_RNR_TIMER;
4075 return is_valid_mask(attr_mask, req, opt);
4076 } else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
4077 return is_valid_mask(attr_mask, req, opt);
4078 }
4079 return false;
4080}
4081
776a3906
MS		 4082/* mlx5_ib_modify_dct: modify a DCT QP
4083 * valid transitions are:
4084 * RESET to INIT: must set access_flags, pkey_index and port
4085 * INIT to RTR : must set min_rnr_timer, tclass, flow_label,
4086 * mtu, gid_index and hop_limit
4087 * Other transitions and attributes are illegal
4088 */
4089static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4090 int attr_mask, struct ib_udata *udata)
4091{
4092 struct mlx5_ib_qp *qp = to_mqp(ibqp);
4093 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4094 enum ib_qp_state cur_state, new_state;
4095 int err = 0;
4096 int required = IB_QP_STATE;
4097 void *dctc;
4098
4099 if (!(attr_mask & IB_QP_STATE))
4100 return -EINVAL;
4101
4102 cur_state = qp->state;
4103 new_state = attr->qp_state;
4104
4105 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
4106 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3e1f000f
PP		 4107 u16 set_id;
4108
776a3906
MS		 4109 required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
4110 if (!is_valid_mask(attr_mask, required, 0))
4111 return -EINVAL;
4112
4113 if (attr->port_num == 0 ||
4114 attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports)) {
4115 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4116 attr->port_num, dev->num_ports);
4117 return -EINVAL;
4118 }
4119 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
4120 MLX5_SET(dctc, dctc, rre, 1);
4121 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
4122 MLX5_SET(dctc, dctc, rwe, 1);
4123 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
a60109dc
YC		 4124 int atomic_mode;
4125
4126 atomic_mode = get_atomic_mode(dev, MLX5_IB_QPT_DCT);
4127 if (atomic_mode < 0)
776a3906 4128 return -EOPNOTSUPP;
a60109dc
YC		 4129
4130 MLX5_SET(dctc, dctc, atomic_mode, atomic_mode);
776a3906 4131 MLX5_SET(dctc, dctc, rae, 1);
776a3906
MS		 4132 }
4133 MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
4134 MLX5_SET(dctc, dctc, port, attr->port_num);
3e1f000f
PP		 4135
4136 set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1);
4137 MLX5_SET(dctc, dctc, counter_set_id, set_id);
776a3906
MS		 4138
4139 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4140 struct mlx5_ib_modify_qp_resp resp = {};
c5ae1954 4141 u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {0};
776a3906
MS		 4142 u32 min_resp_len = offsetof(typeof(resp), dctn) +
4143 sizeof(resp.dctn);
4144
4145 if (udata->outlen < min_resp_len)
4146 return -EINVAL;
4147 resp.response_length = min_resp_len;
4148
4149 required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
4150 if (!is_valid_mask(attr_mask, required, 0))
4151 return -EINVAL;
4152 MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
4153 MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
4154 MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
4155 MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
4156 MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
4157 MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
4158
333fbaa0 4159 err = mlx5_core_create_dct(dev, &qp->dct.mdct, qp->dct.in,
c5ae1954
YH		 4160 MLX5_ST_SZ_BYTES(create_dct_in), out,
4161 sizeof(out));
776a3906
MS		 4162 if (err)
4163 return err;
4164 resp.dctn = qp->dct.mdct.mqp.qpn;
4165 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4166 if (err) {
333fbaa0 4167 mlx5_core_destroy_dct(dev, &qp->dct.mdct);
776a3906
MS		 4168 return err;
4169 }
4170 } else {
4171 mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
4172 return -EINVAL;
4173 }
4174 if (err)
4175 qp->state = IB_QPS_ERR;
4176 else
4177 qp->state = new_state;
4178 return err;
4179}
4180
e126ba97
EC		 4181int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4182 int attr_mask, struct ib_udata *udata)
4183{
4184 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4185 struct mlx5_ib_qp *qp = to_mqp(ibqp);
61147f39 4186 struct mlx5_ib_modify_qp ucmd = {};
d16e91da 4187 enum ib_qp_type qp_type;
e126ba97 4188 enum ib_qp_state cur_state, new_state;
61147f39 4189 size_t required_cmd_sz;
e126ba97
EC		 4190 int err = -EINVAL;
4191 int port;
4192
28d61370
YH		 4193 if (ibqp->rwq_ind_tbl)
4194 return -ENOSYS;
4195
61147f39
BW		 4196 if (udata && udata->inlen) {
4197 required_cmd_sz = offsetof(typeof(ucmd), reserved) +
4198 sizeof(ucmd.reserved);
4199 if (udata->inlen < required_cmd_sz)
4200 return -EINVAL;
4201
4202 if (udata->inlen > sizeof(ucmd) &&
4203 !ib_is_udata_cleared(udata, sizeof(ucmd),
4204 udata->inlen - sizeof(ucmd)))
4205 return -EOPNOTSUPP;
4206
4207 if (ib_copy_from_udata(&ucmd, udata,
4208 min(udata->inlen, sizeof(ucmd))))
4209 return -EFAULT;
4210
4211 if (ucmd.comp_mask ||
4212 memchr_inv(&ucmd.reserved, 0, sizeof(ucmd.reserved)) ||
4213 memchr_inv(&ucmd.burst_info.reserved, 0,
4214 sizeof(ucmd.burst_info.reserved)))
4215 return -EOPNOTSUPP;
4216 }
4217
d16e91da
HE		 4218 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4219 return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
4220
7aede1a2
LR		 4221 qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ? IB_QPT_GSI :
4222 qp->type;
c32a4f29 4223
776a3906
MS		 4224 if (qp_type == MLX5_IB_QPT_DCT)
4225 return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);
d16e91da 4226
e126ba97
EC		 4227 mutex_lock(&qp->mutex);
4228
4229 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
4230 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
4231
2811ba51
AS		 4232 if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
4233 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2811ba51
AS		 4234 }
4235
2be08c30 4236 if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
c2e53b2c
YH		 4237 if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
4238 mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
4239 attr_mask);
4240 goto out;
4241 }
4242 } else if (qp_type != MLX5_IB_QPT_REG_UMR &&
c32a4f29 4243 qp_type != MLX5_IB_QPT_DCI &&
d31131bb
KH		 4244 !ib_modify_qp_is_ok(cur_state, new_state, qp_type,
4245 attr_mask)) {
158abf86
HE		 4246 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4247 cur_state, new_state, ibqp->qp_type, attr_mask);
e126ba97 4248 goto out;
c32a4f29
MS		 4249 } else if (qp_type == MLX5_IB_QPT_DCI &&
4250 !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
4251 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4252 cur_state, new_state, qp_type, attr_mask);
4253 goto out;
158abf86 4254 }
e126ba97
EC		 4255
4256 if ((attr_mask & IB_QP_PORT) &&
938fe83c 4257 (attr->port_num == 0 ||
508562d6 4258 attr->port_num > dev->num_ports)) {
158abf86
HE		 4259 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4260 attr->port_num, dev->num_ports);
e126ba97 4261 goto out;
158abf86 4262 }
e126ba97
EC		 4263
4264 if (attr_mask & IB_QP_PKEY_INDEX) {
4265 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
938fe83c 4266 if (attr->pkey_index >=
158abf86
HE		 4267 dev->mdev->port_caps[port - 1].pkey_table_len) {
4268 mlx5_ib_dbg(dev, "invalid pkey index %d\n",
4269 attr->pkey_index);
e126ba97 4270 goto out;
158abf86 4271 }
e126ba97
EC		 4272 }
4273
4274 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
938fe83c 4275 attr->max_rd_atomic >
158abf86
HE		 4276 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) {
4277 mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
4278 attr->max_rd_atomic);
e126ba97 4279 goto out;
158abf86 4280 }
e126ba97
EC		 4281
4282 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
938fe83c 4283 attr->max_dest_rd_atomic >
158abf86
HE		 4284 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) {
4285 mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
4286 attr->max_dest_rd_atomic);
e126ba97 4287 goto out;
158abf86 4288 }
e126ba97
EC		 4289
4290 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
4291 err = 0;
4292 goto out;
4293 }
4294
61147f39 4295 err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
89944450 4296 new_state, &ucmd, udata);
e126ba97
EC		 4297
4298out:
4299 mutex_unlock(&qp->mutex);
4300 return err;
4301}
4302
e126ba97
EC		 4303static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4304{
4305 switch (mlx5_state) {
4306 case MLX5_QP_STATE_RST: return IB_QPS_RESET;
4307 case MLX5_QP_STATE_INIT: return IB_QPS_INIT;
4308 case MLX5_QP_STATE_RTR: return IB_QPS_RTR;
4309 case MLX5_QP_STATE_RTS: return IB_QPS_RTS;
4310 case MLX5_QP_STATE_SQ_DRAINING:
4311 case MLX5_QP_STATE_SQD: return IB_QPS_SQD;
4312 case MLX5_QP_STATE_SQER: return IB_QPS_SQE;
4313 case MLX5_QP_STATE_ERR: return IB_QPS_ERR;
4314 default: return -1;
4315 }
4316}
4317
4318static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4319{
4320 switch (mlx5_mig_state) {
4321 case MLX5_QP_PM_ARMED: return IB_MIG_ARMED;
4322 case MLX5_QP_PM_REARM: return IB_MIG_REARM;
4323 case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
4324 default: return -1;
4325 }
4326}
4327
4328static int to_ib_qp_access_flags(int mlx5_flags)
4329{
4330 int ib_flags = 0;
4331
4332 if (mlx5_flags & MLX5_QP_BIT_RRE)
4333 ib_flags |= IB_ACCESS_REMOTE_READ;
4334 if (mlx5_flags & MLX5_QP_BIT_RWE)
4335 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4336 if (mlx5_flags & MLX5_QP_BIT_RAE)
4337 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4338
4339 return ib_flags;
4340}
4341
38349389 4342static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
d8966fcd 4343 struct rdma_ah_attr *ah_attr,
38349389 4344 struct mlx5_qp_path *path)
e126ba97 4345{
e126ba97 4346
d8966fcd 4347 memset(ah_attr, 0, sizeof(*ah_attr));
e126ba97 4348
e7996a9a 4349 if (!path->port || path->port > ibdev->num_ports)
e126ba97
EC		 4350 return;
4351
ae59c3f0
LR		 4352 ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, path->port);
4353
d8966fcd
DC		 4354 rdma_ah_set_port_num(ah_attr, path->port);
4355 rdma_ah_set_sl(ah_attr, path->dci_cfi_prio_sl & 0xf);
4356
4357 rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4358 rdma_ah_set_path_bits(ah_attr, path->grh_mlid & 0x7f);
4359 rdma_ah_set_static_rate(ah_attr,
4360 path->static_rate ? path->static_rate - 5 : 0);
2d7e3ff7
AL		 4361
4362 if (path->grh_mlid & (1 << 7) ||
4363 ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
d8966fcd
DC		 4364 u32 tc_fl = be32_to_cpu(path->tclass_flowlabel);
4365
4366 rdma_ah_set_grh(ah_attr, NULL,
4367 tc_fl & 0xfffff,
4368 path->mgid_index,
4369 path->hop_limit,
4370 (tc_fl >> 20) & 0xff);
4371 rdma_ah_set_dgid_raw(ah_attr, path->rgid);
e126ba97
EC		 4372 }
4373}
4374
6d2f89df 4375static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4376 struct mlx5_ib_sq *sq,
4377 u8 *sq_state)
4378{
6d2f89df 4379 int err;
4380
28160771 4381 err = mlx5_core_query_sq_state(dev->mdev, sq->base.mqp.qpn, sq_state);
6d2f89df 4382 if (err)
4383 goto out;
6d2f89df 4384 sq->state = *sq_state;
4385
4386out:
6d2f89df 4387 return err;
4388}
4389
4390static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4391 struct mlx5_ib_rq *rq,
4392 u8 *rq_state)
4393{
4394 void *out;
4395 void *rqc;
4396 int inlen;
4397 int err;
4398
4399 inlen = MLX5_ST_SZ_BYTES(query_rq_out);
1b9a07ee 4400 out = kvzalloc(inlen, GFP_KERNEL);
6d2f89df 4401 if (!out)
4402 return -ENOMEM;
4403
4404 err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4405 if (err)
4406 goto out;
4407
4408 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4409 *rq_state = MLX5_GET(rqc, rqc, state);
4410 rq->state = *rq_state;
4411
4412out:
4413 kvfree(out);
4414 return err;
4415}
4416
4417static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4418 struct mlx5_ib_qp *qp, u8 *qp_state)
4419{
4420 static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4421 [MLX5_RQC_STATE_RST] = {
4422 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4423 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4424 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE_BAD,
4425 [MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4426 },
4427 [MLX5_RQC_STATE_RDY] = {
4428 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4429 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4430 [MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4431 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
4432 },
4433 [MLX5_RQC_STATE_ERR] = {
4434 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4435 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4436 [MLX5_SQC_STATE_ERR] = IB_QPS_ERR,
4437 [MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4438 },
4439 [MLX5_RQ_STATE_NA] = {
4440 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4441 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4442 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4443 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
4444 },
4445 };
4446
4447 *qp_state = sqrq_trans[rq_state][sq_state];
4448
4449 if (*qp_state == MLX5_QP_STATE_BAD) {
4450 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4451 qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4452 qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4453 return -EINVAL;
4454 }
4455
4456 if (*qp_state == MLX5_QP_STATE)
4457 *qp_state = qp->state;
4458
4459 return 0;
4460}
4461
4462static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4463 struct mlx5_ib_qp *qp,
4464 u8 *raw_packet_qp_state)
4465{
4466 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4467 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4468 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4469 int err;
4470 u8 sq_state = MLX5_SQ_STATE_NA;
4471 u8 rq_state = MLX5_RQ_STATE_NA;
4472
4473 if (qp->sq.wqe_cnt) {
4474 err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4475 if (err)
4476 return err;
4477 }
4478
4479 if (qp->rq.wqe_cnt) {
4480 err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4481 if (err)
4482 return err;
4483 }
4484
4485 return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4486 raw_packet_qp_state);
4487}
4488
4489static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4490 struct ib_qp_attr *qp_attr)
e126ba97 4491{
09a7d9ec 4492 int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
e126ba97
EC		 4493 struct mlx5_qp_context *context;
4494 int mlx5_state;
09a7d9ec 4495 u32 *outb;
e126ba97
EC		 4496 int err = 0;
4497
09a7d9ec 4498 outb = kzalloc(outlen, GFP_KERNEL);
6d2f89df 4499 if (!outb)
4500 return -ENOMEM;
4501
333fbaa0 4502 err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen);
e126ba97 4503 if (err)
6d2f89df 4504 goto out;
e126ba97 4505
09a7d9ec
SM		 4506 /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
4507 context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
4508
e126ba97
EC		 4509 mlx5_state = be32_to_cpu(context->flags) >> 28;
4510
4511 qp->state = to_ib_qp_state(mlx5_state);
e126ba97
EC		 4512 qp_attr->path_mtu = context->mtu_msgmax >> 5;
4513 qp_attr->path_mig_state =
4514 to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
4515 qp_attr->qkey = be32_to_cpu(context->qkey);
4516 qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
4517 qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff;
4518 qp_attr->dest_qp_num = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff;
4519 qp_attr->qp_access_flags =
4520 to_ib_qp_access_flags(be32_to_cpu(context->params2));
4521
4522 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
38349389
DC		 4523 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
4524 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
d3ae2bde
NO		 4525 qp_attr->alt_pkey_index =
4526 be16_to_cpu(context->alt_path.pkey_index);
d8966fcd
DC		 4527 qp_attr->alt_port_num =
4528 rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
e126ba97
EC		 4529 }
4530
d3ae2bde 4531 qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index);
e126ba97
EC		 4532 qp_attr->port_num = context->pri_path.port;
4533
4534 /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4535 qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;
4536
4537 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
4538
4539 qp_attr->max_dest_rd_atomic =
4540 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
4541 qp_attr->min_rnr_timer =
4542 (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
4543 qp_attr->timeout = context->pri_path.ackto_lt >> 3;
4544 qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7;
4545 qp_attr->rnr_retry = (be32_to_cpu(context->params1) >> 13) & 0x7;
4546 qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3;
6d2f89df 4547
4548out:
4549 kfree(outb);
4550 return err;
4551}
4552
776a3906
MS		 4553static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
4554 struct ib_qp_attr *qp_attr, int qp_attr_mask,
4555 struct ib_qp_init_attr *qp_init_attr)
4556{
4557 struct mlx5_core_dct *dct = &mqp->dct.mdct;
4558 u32 *out;
4559 u32 access_flags = 0;
4560 int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
4561 void *dctc;
4562 int err;
4563 int supported_mask = IB_QP_STATE |
4564 IB_QP_ACCESS_FLAGS |
4565 IB_QP_PORT |
4566 IB_QP_MIN_RNR_TIMER |
4567 IB_QP_AV |
4568 IB_QP_PATH_MTU |
4569 IB_QP_PKEY_INDEX;
4570
4571 if (qp_attr_mask & ~supported_mask)
4572 return -EINVAL;
4573 if (mqp->state != IB_QPS_RTR)
4574 return -EINVAL;
4575
4576 out = kzalloc(outlen, GFP_KERNEL);
4577 if (!out)
4578 return -ENOMEM;
4579
333fbaa0 4580 err = mlx5_core_dct_query(dev, dct, out, outlen);
776a3906
MS		 4581 if (err)
4582 goto out;
4583
4584 dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
4585
4586 if (qp_attr_mask & IB_QP_STATE)
4587 qp_attr->qp_state = IB_QPS_RTR;
4588
4589 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
4590 if (MLX5_GET(dctc, dctc, rre))
4591 access_flags |= IB_ACCESS_REMOTE_READ;
4592 if (MLX5_GET(dctc, dctc, rwe))
4593 access_flags |= IB_ACCESS_REMOTE_WRITE;
4594 if (MLX5_GET(dctc, dctc, rae))
4595 access_flags |= IB_ACCESS_REMOTE_ATOMIC;
4596 qp_attr->qp_access_flags = access_flags;
4597 }
4598
4599 if (qp_attr_mask & IB_QP_PORT)
4600 qp_attr->port_num = MLX5_GET(dctc, dctc, port);
4601 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
4602 qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
4603 if (qp_attr_mask & IB_QP_AV) {
4604 qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
4605 qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
4606 qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
4607 qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
4608 }
4609 if (qp_attr_mask & IB_QP_PATH_MTU)
4610 qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
4611 if (qp_attr_mask & IB_QP_PKEY_INDEX)
4612 qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
4613out:
4614 kfree(out);
4615 return err;
4616}
4617
6d2f89df 4618int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4619 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
4620{
4621 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4622 struct mlx5_ib_qp *qp = to_mqp(ibqp);
4623 int err = 0;
4624 u8 raw_packet_qp_state;
4625
28d61370
YH		 4626 if (ibqp->rwq_ind_tbl)
4627 return -ENOSYS;
4628
d16e91da
HE		 4629 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4630 return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
4631 qp_init_attr);
4632
c2e53b2c
YH		 4633 /* Not all of output fields are applicable, make sure to zero them */
4634 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4635 memset(qp_attr, 0, sizeof(*qp_attr));
4636
7aede1a2 4637 if (unlikely(qp->type == MLX5_IB_QPT_DCT))
776a3906
MS		 4638 return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
4639 qp_attr_mask, qp_init_attr);
4640
6d2f89df 4641 mutex_lock(&qp->mutex);
4642
c2e53b2c 4643 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2be08c30 4644 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
6d2f89df 4645 err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
4646 if (err)
4647 goto out;
4648 qp->state = raw_packet_qp_state;
4649 qp_attr->port_num = 1;
4650 } else {
4651 err = query_qp_attr(dev, qp, qp_attr);
4652 if (err)
4653 goto out;
4654 }
4655
4656 qp_attr->qp_state = qp->state;
e126ba97
EC		 4657 qp_attr->cur_qp_state = qp_attr->qp_state;
4658 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
4659 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
4660
4661 if (!ibqp->uobject) {
0540d814 4662 qp_attr->cap.max_send_wr = qp->sq.max_post;
e126ba97 4663 qp_attr->cap.max_send_sge = qp->sq.max_gs;
0540d814 4664 qp_init_attr->qp_context = ibqp->qp_context;
e126ba97
EC		 4665 } else {
4666 qp_attr->cap.max_send_wr = 0;
4667 qp_attr->cap.max_send_sge = 0;
4668 }
4669
0540d814
NO		 4670 qp_init_attr->qp_type = ibqp->qp_type;
4671 qp_init_attr->recv_cq = ibqp->recv_cq;
4672 qp_init_attr->send_cq = ibqp->send_cq;
4673 qp_init_attr->srq = ibqp->srq;
4674 qp_attr->cap.max_inline_data = qp->max_inline_data;
e126ba97
EC		 4675
4676 qp_init_attr->cap = qp_attr->cap;
4677
a8f3ea61 4678 qp_init_attr->create_flags = qp->flags;
051f2630 4679
e126ba97
EC		 4680 qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
4681 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
4682
e126ba97
EC		 4683out:
4684 mutex_unlock(&qp->mutex);
4685 return err;
4686}
4687
4688struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
ff23dfa1 4689 struct ib_udata *udata)
e126ba97
EC		 4690{
4691 struct mlx5_ib_dev *dev = to_mdev(ibdev);
4692 struct mlx5_ib_xrcd *xrcd;
4693 int err;
4694
938fe83c 4695 if (!MLX5_CAP_GEN(dev->mdev, xrc))
e126ba97
EC		 4696 return ERR_PTR(-ENOSYS);
4697
4698 xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
4699 if (!xrcd)
4700 return ERR_PTR(-ENOMEM);
4701
5aa3771d 4702 err = mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
e126ba97
EC		 4703 if (err) {
4704 kfree(xrcd);
4705 return ERR_PTR(-ENOMEM);
4706 }
4707
4708 return &xrcd->ibxrcd;
4709}
4710
c4367a26 4711int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
e126ba97
EC		 4712{
4713 struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
4714 u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
4715 int err;
4716
5aa3771d 4717 err = mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
b081808a 4718 if (err)
e126ba97 4719 mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
e126ba97
EC		 4720
4721 kfree(xrcd);
e126ba97
EC		 4722 return 0;
4723}
79b20a6c 4724
350d0e4c
YH		 4725static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
4726{
4727 struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
4728 struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
4729 struct ib_event event;
4730
4731 if (rwq->ibwq.event_handler) {
4732 event.device = rwq->ibwq.device;
4733 event.element.wq = &rwq->ibwq;
4734 switch (type) {
4735 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
4736 event.event = IB_EVENT_WQ_FATAL;
4737 break;
4738 default:
4739 mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
4740 return;
4741 }
4742
4743 rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
4744 }
4745}
4746
03404e8a
MG		 4747static int set_delay_drop(struct mlx5_ib_dev *dev)
4748{
4749 int err = 0;
4750
4751 mutex_lock(&dev->delay_drop.lock);
4752 if (dev->delay_drop.activate)
4753 goto out;
4754
333fbaa0 4755 err = mlx5_core_set_delay_drop(dev, dev->delay_drop.timeout);
03404e8a
MG		 4756 if (err)
4757 goto out;
4758
4759 dev->delay_drop.activate = true;
4760out:
4761 mutex_unlock(&dev->delay_drop.lock);
fe248c3a
MG		 4762
4763 if (!err)
4764 atomic_inc(&dev->delay_drop.rqs_cnt);
03404e8a
MG		 4765 return err;
4766}
4767
79b20a6c
YH		 4768static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
4769 struct ib_wq_init_attr *init_attr)
4770{
4771 struct mlx5_ib_dev *dev;
4be6da1e 4772 int has_net_offloads;
79b20a6c
YH		 4773 __be64 *rq_pas0;
4774 void *in;
4775 void *rqc;
4776 void *wq;
4777 int inlen;
4778 int err;
4779
4780 dev = to_mdev(pd->device);
4781
4782 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
1b9a07ee 4783 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 4784 if (!in)
4785 return -ENOMEM;
4786
34d57585 4787 MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
79b20a6c
YH		 4788 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
4789 MLX5_SET(rqc, rqc, mem_rq_type,
4790 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
4791 MLX5_SET(rqc, rqc, user_index, rwq->user_index);
4792 MLX5_SET(rqc, rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
4793 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
4794 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
4795 wq = MLX5_ADDR_OF(rqc, rqc, wq);
ccc87087
NO		 4796 MLX5_SET(wq, wq, wq_type,
4797 rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
4798 MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 4799 if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
4800 if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
4801 mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
4802 err = -EOPNOTSUPP;
4803 goto out;
4804 } else {
4805 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
4806 }
4807 }
79b20a6c 4808 MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
ccc87087 4809 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
c16339b6
MZ		 4810 /*
4811 * In Firmware number of strides in each WQE is:
4812 * "512 * 2^single_wqe_log_num_of_strides"
4813 * Values 3 to 8 are accepted as 10 to 15, 9 to 18 are
4814 * accepted as 0 to 9
4815 */
4816 static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,
4817 2, 3, 4, 5, 6, 7, 8, 9 };
ccc87087
NO		 4818 MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
4819 MLX5_SET(wq, wq, log_wqe_stride_size,
4820 rwq->single_stride_log_num_of_bytes -
4821 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
c16339b6
MZ		 4822 MLX5_SET(wq, wq, log_wqe_num_of_strides,
4823 fw_map[rwq->log_num_strides -
4824 MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);
ccc87087 4825 }
79b20a6c
YH		 4826 MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
4827 MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
4828 MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
4829 MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
4830 MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
4831 MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
4be6da1e 4832 has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
b1f74a84 4833 if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
4be6da1e 4834 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
b1f74a84
NO		 4835 mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
4836 err = -EOPNOTSUPP;
4837 goto out;
4838 }
4839 } else {
4840 MLX5_SET(rqc, rqc, vsd, 1);
4841 }
4be6da1e
NO		 4842 if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
4843 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
4844 mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
4845 err = -EOPNOTSUPP;
4846 goto out;
4847 }
4848 MLX5_SET(rqc, rqc, scatter_fcs, 1);
4849 }
03404e8a
MG		 4850 if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
4851 if (!(dev->ib_dev.attrs.raw_packet_caps &
4852 IB_RAW_PACKET_CAP_DELAY_DROP)) {
4853 mlx5_ib_dbg(dev, "Delay drop is not supported\n");
4854 err = -EOPNOTSUPP;
4855 goto out;
4856 }
4857 MLX5_SET(rqc, rqc, delay_drop_en, 1);
4858 }
79b20a6c
YH		 4859 rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
4860 mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
333fbaa0 4861 err = mlx5_core_create_rq_tracked(dev, in, inlen, &rwq->core_qp);
03404e8a
MG		 4862 if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
4863 err = set_delay_drop(dev);
4864 if (err) {
4865 mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
4866 err);
333fbaa0 4867 mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
03404e8a
MG		 4868 } else {
4869 rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
4870 }
4871 }
b1f74a84 4872out:
79b20a6c
YH		 4873 kvfree(in);
4874 return err;
4875}
4876
4877static int set_user_rq_size(struct mlx5_ib_dev *dev,
4878 struct ib_wq_init_attr *wq_init_attr,
4879 struct mlx5_ib_create_wq *ucmd,
4880 struct mlx5_ib_rwq *rwq)
4881{
4882 /* Sanity check RQ size before proceeding */
4883 if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
4884 return -EINVAL;
4885
4886 if (!ucmd->rq_wqe_count)
4887 return -EINVAL;
4888
4889 rwq->wqe_count = ucmd->rq_wqe_count;
4890 rwq->wqe_shift = ucmd->rq_wqe_shift;
0dfe4522
LR		 4891 if (check_shl_overflow(rwq->wqe_count, rwq->wqe_shift, &rwq->buf_size))
4892 return -EINVAL;
4893
79b20a6c
YH		 4894 rwq->log_rq_stride = rwq->wqe_shift;
4895 rwq->log_rq_size = ilog2(rwq->wqe_count);
4896 return 0;
4897}
4898
c16339b6
MZ		 4899static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
4900{
4901 if ((log_num_strides > MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
4902 (log_num_strides < MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
4903 return false;
4904
4905 if (!MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) &&
4906 (log_num_strides < MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
4907 return false;
4908
4909 return true;
4910}
4911
79b20a6c
YH		 4912static int prepare_user_rq(struct ib_pd *pd,
4913 struct ib_wq_init_attr *init_attr,
4914 struct ib_udata *udata,
4915 struct mlx5_ib_rwq *rwq)
4916{
4917 struct mlx5_ib_dev *dev = to_mdev(pd->device);
4918 struct mlx5_ib_create_wq ucmd = {};
4919 int err;
4920 size_t required_cmd_sz;
4921
ccc87087
NO		 4922 required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes)
4923 + sizeof(ucmd.single_stride_log_num_of_bytes);
79b20a6c
YH		 4924 if (udata->inlen < required_cmd_sz) {
4925 mlx5_ib_dbg(dev, "invalid inlen\n");
4926 return -EINVAL;
4927 }
4928
4929 if (udata->inlen > sizeof(ucmd) &&
4930 !ib_is_udata_cleared(udata, sizeof(ucmd),
4931 udata->inlen - sizeof(ucmd))) {
4932 mlx5_ib_dbg(dev, "inlen is not supported\n");
4933 return -EOPNOTSUPP;
4934 }
4935
4936 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
4937 mlx5_ib_dbg(dev, "copy failed\n");
4938 return -EFAULT;
4939 }
4940
ccc87087 4941 if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
79b20a6c
YH		 4942 mlx5_ib_dbg(dev, "invalid comp mask\n");
4943 return -EOPNOTSUPP;
ccc87087
NO		 4944 } else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
4945 if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
4946 mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
4947 return -EOPNOTSUPP;
4948 }
4949 if ((ucmd.single_stride_log_num_of_bytes <
4950 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
4951 (ucmd.single_stride_log_num_of_bytes >
4952 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
4953 mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
4954 ucmd.single_stride_log_num_of_bytes,
4955 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
4956 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
4957 return -EINVAL;
4958 }
c16339b6
MZ		 4959 if (!log_of_strides_valid(dev,
4960 ucmd.single_wqe_log_num_of_strides)) {
4961 mlx5_ib_dbg(
4962 dev,
4963 "Invalid log num strides (%u. Range is %u - %u)\n",
4964 ucmd.single_wqe_log_num_of_strides,
4965 MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) ?
4966 MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :
4967 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
4968 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
ccc87087
NO		 4969 return -EINVAL;
4970 }
4971 rwq->single_stride_log_num_of_bytes =
4972 ucmd.single_stride_log_num_of_bytes;
4973 rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
4974 rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
4975 rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
79b20a6c
YH		 4976 }
4977
4978 err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
4979 if (err) {
4980 mlx5_ib_dbg(dev, "err %d\n", err);
4981 return err;
4982 }
4983
b0ea0fa5 4984 err = create_user_rq(dev, pd, udata, rwq, &ucmd);
79b20a6c
YH		 4985 if (err) {
4986 mlx5_ib_dbg(dev, "err %d\n", err);
645ba597 4987 return err;
79b20a6c
YH		 4988 }
4989
4990 rwq->user_index = ucmd.user_index;
4991 return 0;
4992}
4993
4994struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
4995 struct ib_wq_init_attr *init_attr,
4996 struct ib_udata *udata)
4997{
4998 struct mlx5_ib_dev *dev;
4999 struct mlx5_ib_rwq *rwq;
5000 struct mlx5_ib_create_wq_resp resp = {};
5001 size_t min_resp_len;
5002 int err;
5003
5004 if (!udata)
5005 return ERR_PTR(-ENOSYS);
5006
5007 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
5008 if (udata->outlen && udata->outlen < min_resp_len)
5009 return ERR_PTR(-EINVAL);
5010
ba80013f
MG		 5011 if (!capable(CAP_SYS_RAWIO) &&
5012 init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP)
5013 return ERR_PTR(-EPERM);
5014
79b20a6c
YH		 5015 dev = to_mdev(pd->device);
5016 switch (init_attr->wq_type) {
5017 case IB_WQT_RQ:
5018 rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5019 if (!rwq)
5020 return ERR_PTR(-ENOMEM);
5021 err = prepare_user_rq(pd, init_attr, udata, rwq);
5022 if (err)
5023 goto err;
5024 err = create_rq(rwq, pd, init_attr);
5025 if (err)
5026 goto err_user_rq;
5027 break;
5028 default:
5029 mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5030 init_attr->wq_type);
5031 return ERR_PTR(-EINVAL);
5032 }
5033
350d0e4c 5034 rwq->ibwq.wq_num = rwq->core_qp.qpn;
79b20a6c
YH		 5035 rwq->ibwq.state = IB_WQS_RESET;
5036 if (udata->outlen) {
5037 resp.response_length = offsetof(typeof(resp), response_length) +
5038 sizeof(resp.response_length);
5039 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5040 if (err)
5041 goto err_copy;
5042 }
5043
350d0e4c
YH		 5044 rwq->core_qp.event = mlx5_ib_wq_event;
5045 rwq->ibwq.event_handler = init_attr->event_handler;
79b20a6c
YH		 5046 return &rwq->ibwq;
5047
5048err_copy:
333fbaa0 5049 mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
79b20a6c 5050err_user_rq:
bdeacabd 5051 destroy_user_rq(dev, pd, rwq, udata);
79b20a6c
YH		 5052err:
5053 kfree(rwq);
5054 return ERR_PTR(err);
5055}
5056
a49b1dc7 5057void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
79b20a6c
YH		 5058{
5059 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5060 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5061
333fbaa0 5062 mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
bdeacabd 5063 destroy_user_rq(dev, wq->pd, rwq, udata);
79b20a6c 5064 kfree(rwq);
79b20a6c
YH		 5065}
5066
c5f90929
YH		 5067struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
5068 struct ib_rwq_ind_table_init_attr *init_attr,
5069 struct ib_udata *udata)
5070{
5071 struct mlx5_ib_dev *dev = to_mdev(device);
5072 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl;
5073 int sz = 1 << init_attr->log_ind_tbl_size;
5074 struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5075 size_t min_resp_len;
5076 int inlen;
5077 int err;
5078 int i;
5079 u32 *in;
5080 void *rqtc;
5081
5082 if (udata->inlen > 0 &&
5083 !ib_is_udata_cleared(udata, 0,
5084 udata->inlen))
5085 return ERR_PTR(-EOPNOTSUPP);
5086
efd7f400
MG		 5087 if (init_attr->log_ind_tbl_size >
5088 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5089 mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5090 init_attr->log_ind_tbl_size,
5091 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5092 return ERR_PTR(-EINVAL);
5093 }
5094
c5f90929
YH		 5095 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
5096 if (udata->outlen && udata->outlen < min_resp_len)
5097 return ERR_PTR(-EINVAL);
5098
5099 rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL);
5100 if (!rwq_ind_tbl)
5101 return ERR_PTR(-ENOMEM);
5102
5103 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1b9a07ee 5104 in = kvzalloc(inlen, GFP_KERNEL);
c5f90929
YH		 5105 if (!in) {
5106 err = -ENOMEM;
5107 goto err;
5108 }
5109
5110 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5111
5112 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5113 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5114
5115 for (i = 0; i < sz; i++)
5116 MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5117
5deba86e
YH		 5118 rwq_ind_tbl->uid = to_mpd(init_attr->ind_tbl[0]->pd)->uid;
5119 MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
5120
c5f90929
YH		 5121 err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5122 kvfree(in);
5123
5124 if (err)
5125 goto err;
5126
5127 rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5128 if (udata->outlen) {
5129 resp.response_length = offsetof(typeof(resp), response_length) +
5130 sizeof(resp.response_length);
5131 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5132 if (err)
5133 goto err_copy;
5134 }
5135
5136 return &rwq_ind_tbl->ib_rwq_ind_tbl;
5137
5138err_copy:
5deba86e 5139 mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
c5f90929
YH		 5140err:
5141 kfree(rwq_ind_tbl);
5142 return ERR_PTR(err);
5143}
5144
5145int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5146{
5147 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5148 struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5149
5deba86e 5150 mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
c5f90929
YH		 5151
5152 kfree(rwq_ind_tbl);
5153 return 0;
5154}
5155
79b20a6c
YH		 5156int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5157 u32 wq_attr_mask, struct ib_udata *udata)
5158{
5159 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5160 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5161 struct mlx5_ib_modify_wq ucmd = {};
5162 size_t required_cmd_sz;
5163 int curr_wq_state;
5164 int wq_state;
5165 int inlen;
5166 int err;
5167 void *rqc;
5168 void *in;
5169
5170 required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);
5171 if (udata->inlen < required_cmd_sz)
5172 return -EINVAL;
5173
5174 if (udata->inlen > sizeof(ucmd) &&
5175 !ib_is_udata_cleared(udata, sizeof(ucmd),
5176 udata->inlen - sizeof(ucmd)))
5177 return -EOPNOTSUPP;
5178
5179 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5180 return -EFAULT;
5181
5182 if (ucmd.comp_mask || ucmd.reserved)
5183 return -EOPNOTSUPP;
5184
5185 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 5186 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5187 if (!in)
5188 return -ENOMEM;
5189
5190 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5191
5192 curr_wq_state = (wq_attr_mask & IB_WQ_CUR_STATE) ?
5193 wq_attr->curr_wq_state : wq->state;
5194 wq_state = (wq_attr_mask & IB_WQ_STATE) ?
5195 wq_attr->wq_state : curr_wq_state;
5196 if (curr_wq_state == IB_WQS_ERR)
5197 curr_wq_state = MLX5_RQC_STATE_ERR;
5198 if (wq_state == IB_WQS_ERR)
5199 wq_state = MLX5_RQC_STATE_ERR;
5200 MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
34d57585 5201 MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
79b20a6c
YH		 5202 MLX5_SET(rqc, rqc, state, wq_state);
5203
b1f74a84
NO		 5204 if (wq_attr_mask & IB_WQ_FLAGS) {
5205 if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5206 if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5207 MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5208 mlx5_ib_dbg(dev, "VLAN offloads are not "
5209 "supported\n");
5210 err = -EOPNOTSUPP;
5211 goto out;
5212 }
5213 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5214 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5215 MLX5_SET(rqc, rqc, vsd,
5216 (wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5217 }
b1383aa6
NO		 5218
5219 if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5220 mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5221 err = -EOPNOTSUPP;
5222 goto out;
5223 }
b1f74a84
NO		 5224 }
5225
23a6964e 5226 if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
3e1f000f
PP		 5227 u16 set_id;
5228
5229 set_id = mlx5_ib_get_counters_id(dev, 0);
23a6964e
MD		 5230 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5231 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5232 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
3e1f000f 5233 MLX5_SET(rqc, rqc, counter_set_id, set_id);
23a6964e 5234 } else
5a738b5d
JG		 5235 dev_info_once(
5236 &dev->ib_dev.dev,
5237 "Receive WQ counters are not supported on current FW\n");
23a6964e
MD		 5238 }
5239
e0b4b472 5240 err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in);
79b20a6c
YH		 5241 if (!err)
5242 rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5243
b1f74a84
NO		 5244out:
5245 kvfree(in);
79b20a6c
YH		 5246 return err;
5247}
d0e84c0a
YH		 5248
5249struct mlx5_ib_drain_cqe {
5250 struct ib_cqe cqe;
5251 struct completion done;
5252};
5253
5254static void mlx5_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
5255{
5256 struct mlx5_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
5257 struct mlx5_ib_drain_cqe,
5258 cqe);
5259
5260 complete(&cqe->done);
5261}
5262
5263/* This function returns only once the drained WR was completed */
5264static void handle_drain_completion(struct ib_cq *cq,
5265 struct mlx5_ib_drain_cqe *sdrain,
5266 struct mlx5_ib_dev *dev)
5267{
5268 struct mlx5_core_dev *mdev = dev->mdev;
5269
5270 if (cq->poll_ctx == IB_POLL_DIRECT) {
5271 while (wait_for_completion_timeout(&sdrain->done, HZ / 10) <= 0)
5272 ib_process_cq_direct(cq, -1);
5273 return;
5274 }
5275
5276 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5277 struct mlx5_ib_cq *mcq = to_mcq(cq);
5278 bool triggered = false;
5279 unsigned long flags;
5280
5281 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
5282 /* Make sure that the CQ handler won't run if wasn't run yet */
5283 if (!mcq->mcq.reset_notify_added)
5284 mcq->mcq.reset_notify_added = 1;
5285 else
5286 triggered = true;
5287 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
5288
5289 if (triggered) {
5290 /* Wait for any scheduled/running task to be ended */
5291 switch (cq->poll_ctx) {
5292 case IB_POLL_SOFTIRQ:
5293 irq_poll_disable(&cq->iop);
5294 irq_poll_enable(&cq->iop);
5295 break;
5296 case IB_POLL_WORKQUEUE:
5297 cancel_work_sync(&cq->work);
5298 break;
5299 default:
5300 WARN_ON_ONCE(1);
5301 }
5302 }
5303
5304 /* Run the CQ handler - this makes sure that the drain WR will
5305 * be processed if wasn't processed yet.
5306 */
4e0e2ea1 5307 mcq->mcq.comp(&mcq->mcq, NULL);
d0e84c0a
YH		 5308 }
5309
5310 wait_for_completion(&sdrain->done);
5311}
5312
5313void mlx5_ib_drain_sq(struct ib_qp *qp)
5314{
5315 struct ib_cq *cq = qp->send_cq;
5316 struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5317 struct mlx5_ib_drain_cqe sdrain;
d34ac5cd 5318 const struct ib_send_wr *bad_swr;
d0e84c0a
YH		 5319 struct ib_rdma_wr swr = {
5320 .wr = {
5321 .next = NULL,
5322 { .wr_cqe = &sdrain.cqe, },
5323 .opcode = IB_WR_RDMA_WRITE,
5324 },
5325 };
5326 int ret;
5327 struct mlx5_ib_dev *dev = to_mdev(qp->device);
5328 struct mlx5_core_dev *mdev = dev->mdev;
5329
5330 ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5331 if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5332 WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5333 return;
5334 }
5335
5336 sdrain.cqe.done = mlx5_ib_drain_qp_done;
5337 init_completion(&sdrain.done);
5338
029e88fd 5339 ret = mlx5_ib_post_send_drain(qp, &swr.wr, &bad_swr);
d0e84c0a
YH		 5340 if (ret) {
5341 WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5342 return;
5343 }
5344
5345 handle_drain_completion(cq, &sdrain, dev);
5346}
5347
5348void mlx5_ib_drain_rq(struct ib_qp *qp)
5349{
5350 struct ib_cq *cq = qp->recv_cq;
5351 struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5352 struct mlx5_ib_drain_cqe rdrain;
d34ac5cd
BVA		 5353 struct ib_recv_wr rwr = {};
5354 const struct ib_recv_wr *bad_rwr;
d0e84c0a
YH		 5355 int ret;
5356 struct mlx5_ib_dev *dev = to_mdev(qp->device);
5357 struct mlx5_core_dev *mdev = dev->mdev;
5358
5359 ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5360 if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5361 WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5362 return;
5363 }
5364
5365 rwr.wr_cqe = &rdrain.cqe;
5366 rdrain.cqe.done = mlx5_ib_drain_qp_done;
5367 init_completion(&rdrain.done);
5368
029e88fd 5369 ret = mlx5_ib_post_recv_drain(qp, &rwr, &bad_rwr);
d0e84c0a
YH		 5370 if (ret) {
5371 WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5372 return;
5373 }
5374
5375 handle_drain_completion(cq, &rdrain, dev);
5376}
d14133dd
MZ		 5377
5378/**
5379 * Bind a qp to a counter. If @counter is NULL then bind the qp to
5380 * the default counter
5381 */
5382int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)
5383{
10189e8e 5384 struct mlx5_ib_dev *dev = to_mdev(qp->device);
d14133dd
MZ		 5385 struct mlx5_ib_qp *mqp = to_mqp(qp);
5386 int err = 0;
5387
5388 mutex_lock(&mqp->mutex);
5389 if (mqp->state == IB_QPS_RESET) {
5390 qp->counter = counter;
5391 goto out;
5392 }
5393
10189e8e
MZ		 5394 if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) {
5395 err = -EOPNOTSUPP;
5396 goto out;
5397 }
5398
d14133dd
MZ		 5399 if (mqp->state == IB_QPS_RTS) {
5400 err = __mlx5_ib_qp_set_counter(qp, counter);
5401 if (!err)
5402 qp->counter = counter;
5403
5404 goto out;
5405 }
5406
5407 mqp->counter_pending = 1;
5408 qp->counter = counter;
5409
5410out:
5411 mutex_unlock(&mqp->mutex);
5412 return err;
5413}
Ubuntu Hirsute Linux kernel mirror