]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - drivers/infiniband/hw/mlx5/qp.c
projects / mirror_ubuntu-eoan-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sh-for-4.17-fixes' of git://git.libc.org/linux-sh
[mirror_ubuntu-eoan-kernel.git] / drivers / infiniband / hw / mlx5 / qp.c
CommitLineData
e126ba97 1/*
6cf0a15f 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/module.h>
34#include <rdma/ib_umem.h>
2811ba51 35#include <rdma/ib_cache.h>
cfb5e088 36#include <rdma/ib_user_verbs.h>
c2e53b2c 37#include <linux/mlx5/fs.h>
e126ba97 38#include "mlx5_ib.h"
b96c9dde 39#include "ib_rep.h"
e126ba97
EC		 40
41/* not supported currently */
42static int wq_signature;
43
44enum {
45 MLX5_IB_ACK_REQ_FREQ = 8,
46};
47
48enum {
49 MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83,
50 MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
51 MLX5_IB_LINK_TYPE_IB = 0,
52 MLX5_IB_LINK_TYPE_ETH = 1
53};
54
55enum {
56 MLX5_IB_SQ_STRIDE = 6,
e126ba97
EC		 57};
58
59static const u32 mlx5_ib_opcode[] = {
60 [IB_WR_SEND] = MLX5_OPCODE_SEND,
f0313965 61 [IB_WR_LSO] = MLX5_OPCODE_LSO,
e126ba97
EC		 62 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
63 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
64 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
65 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
66 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
67 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
68 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
69 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
8a187ee5 70 [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
e126ba97
EC		 71 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
72 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
73 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
74};
75
f0313965
ES		 76struct mlx5_wqe_eth_pad {
77 u8 rsvd0[16];
78};
e126ba97 79
eb49ab0c
AV		 80enum raw_qp_set_mask_map {
81 MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID = 1UL << 0,
7d29f349 82 MLX5_RAW_QP_RATE_LIMIT = 1UL << 1,
eb49ab0c
AV		 83};
84
0680efa2
AV		 85struct mlx5_modify_raw_qp_param {
86 u16 operation;
eb49ab0c
AV		 87
88 u32 set_mask; /* raw_qp_set_mask_map */
61147f39
BW		 89
90 struct mlx5_rate_limit rl;
91
eb49ab0c 92 u8 rq_q_ctr_id;
0680efa2
AV		 93};
94
89ea94a7
MG		 95static void get_cqs(enum ib_qp_type qp_type,
96 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
97 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
98
e126ba97
EC		 99static int is_qp0(enum ib_qp_type qp_type)
100{
101 return qp_type == IB_QPT_SMI;
102}
103
e126ba97
EC		 104static int is_sqp(enum ib_qp_type qp_type)
105{
106 return is_qp0(qp_type) || is_qp1(qp_type);
107}
108
109static void *get_wqe(struct mlx5_ib_qp *qp, int offset)
110{
111 return mlx5_buf_offset(&qp->buf, offset);
112}
113
114static void *get_recv_wqe(struct mlx5_ib_qp *qp, int n)
115{
116 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
117}
118
119void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n)
120{
121 return get_wqe(qp, qp->sq.offset + (n << MLX5_IB_SQ_STRIDE));
122}
123
c1395a2a
HE		 124/**
125 * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space.
126 *
127 * @qp: QP to copy from.
128 * @send: copy from the send queue when non-zero, use the receive queue
129 * otherwise.
130 * @wqe_index: index to start copying from. For send work queues, the
131 * wqe_index is in units of MLX5_SEND_WQE_BB.
132 * For receive work queue, it is the number of work queue
133 * element in the queue.
134 * @buffer: destination buffer.
135 * @length: maximum number of bytes to copy.
136 *
137 * Copies at least a single WQE, but may copy more data.
138 *
139 * Return: the number of bytes copied, or an error code.
140 */
141int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
19098df2 142 void *buffer, u32 length,
143 struct mlx5_ib_qp_base *base)
c1395a2a
HE		 144{
145 struct ib_device *ibdev = qp->ibqp.device;
146 struct mlx5_ib_dev *dev = to_mdev(ibdev);
147 struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq;
148 size_t offset;
149 size_t wq_end;
19098df2 150 struct ib_umem *umem = base->ubuffer.umem;
c1395a2a
HE		 151 u32 first_copy_length;
152 int wqe_length;
153 int ret;
154
155 if (wq->wqe_cnt == 0) {
156 mlx5_ib_dbg(dev, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n",
157 qp->ibqp.qp_type);
158 return -EINVAL;
159 }
160
161 offset = wq->offset + ((wqe_index % wq->wqe_cnt) << wq->wqe_shift);
162 wq_end = wq->offset + (wq->wqe_cnt << wq->wqe_shift);
163
164 if (send && length < sizeof(struct mlx5_wqe_ctrl_seg))
165 return -EINVAL;
166
167 if (offset > umem->length ||
168 (send && offset + sizeof(struct mlx5_wqe_ctrl_seg) > umem->length))
169 return -EINVAL;
170
171 first_copy_length = min_t(u32, offset + length, wq_end) - offset;
172 ret = ib_umem_copy_from(buffer, umem, offset, first_copy_length);
173 if (ret)
174 return ret;
175
176 if (send) {
177 struct mlx5_wqe_ctrl_seg *ctrl = buffer;
178 int ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
179
180 wqe_length = ds * MLX5_WQE_DS_UNITS;
181 } else {
182 wqe_length = 1 << wq->wqe_shift;
183 }
184
185 if (wqe_length <= first_copy_length)
186 return first_copy_length;
187
188 ret = ib_umem_copy_from(buffer + first_copy_length, umem, wq->offset,
189 wqe_length - first_copy_length);
190 if (ret)
191 return ret;
192
193 return wqe_length;
194}
195
e126ba97
EC		 196static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
197{
198 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
199 struct ib_event event;
200
19098df2 201 if (type == MLX5_EVENT_TYPE_PATH_MIG) {
202 /* This event is only valid for trans_qps */
203 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
204 }
e126ba97
EC		 205
206 if (ibqp->event_handler) {
207 event.device = ibqp->device;
208 event.element.qp = ibqp;
209 switch (type) {
210 case MLX5_EVENT_TYPE_PATH_MIG:
211 event.event = IB_EVENT_PATH_MIG;
212 break;
213 case MLX5_EVENT_TYPE_COMM_EST:
214 event.event = IB_EVENT_COMM_EST;
215 break;
216 case MLX5_EVENT_TYPE_SQ_DRAINED:
217 event.event = IB_EVENT_SQ_DRAINED;
218 break;
219 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
220 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
221 break;
222 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
223 event.event = IB_EVENT_QP_FATAL;
224 break;
225 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
226 event.event = IB_EVENT_PATH_MIG_ERR;
227 break;
228 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
229 event.event = IB_EVENT_QP_REQ_ERR;
230 break;
231 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
232 event.event = IB_EVENT_QP_ACCESS_ERR;
233 break;
234 default:
235 pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
236 return;
237 }
238
239 ibqp->event_handler(&event, ibqp->qp_context);
240 }
241}
242
243static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
244 int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
245{
246 int wqe_size;
247 int wq_size;
248
249 /* Sanity check RQ size before proceeding */
938fe83c 250 if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
e126ba97
EC		 251 return -EINVAL;
252
253 if (!has_rq) {
254 qp->rq.max_gs = 0;
255 qp->rq.wqe_cnt = 0;
256 qp->rq.wqe_shift = 0;
0540d814
NO		 257 cap->max_recv_wr = 0;
258 cap->max_recv_sge = 0;
e126ba97
EC		 259 } else {
260 if (ucmd) {
261 qp->rq.wqe_cnt = ucmd->rq_wqe_count;
002bf228
LR		 262 if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
263 return -EINVAL;
e126ba97 264 qp->rq.wqe_shift = ucmd->rq_wqe_shift;
002bf228
LR		 265 if ((1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) < qp->wq_sig)
266 return -EINVAL;
e126ba97
EC		 267 qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
268 qp->rq.max_post = qp->rq.wqe_cnt;
269 } else {
270 wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0;
271 wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
272 wqe_size = roundup_pow_of_two(wqe_size);
273 wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
274 wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
275 qp->rq.wqe_cnt = wq_size / wqe_size;
938fe83c 276 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
e126ba97
EC		 277 mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
278 wqe_size,
938fe83c
SM		 279 MLX5_CAP_GEN(dev->mdev,
280 max_wqe_sz_rq));
e126ba97
EC		 281 return -EINVAL;
282 }
283 qp->rq.wqe_shift = ilog2(wqe_size);
284 qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
285 qp->rq.max_post = qp->rq.wqe_cnt;
286 }
287 }
288
289 return 0;
290}
291
f0313965 292static int sq_overhead(struct ib_qp_init_attr *attr)
e126ba97 293{
618af384 294 int size = 0;
e126ba97 295
f0313965 296 switch (attr->qp_type) {
e126ba97 297 case IB_QPT_XRC_INI:
b125a54b 298 size += sizeof(struct mlx5_wqe_xrc_seg);
e126ba97
EC		 299 /* fall through */
300 case IB_QPT_RC:
301 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 302 max(sizeof(struct mlx5_wqe_atomic_seg) +
303 sizeof(struct mlx5_wqe_raddr_seg),
304 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
305 sizeof(struct mlx5_mkey_seg));
e126ba97
EC		 306 break;
307
b125a54b
EC		 308 case IB_QPT_XRC_TGT:
309 return 0;
310
e126ba97 311 case IB_QPT_UC:
b125a54b 312 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 313 max(sizeof(struct mlx5_wqe_raddr_seg),
314 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
315 sizeof(struct mlx5_mkey_seg));
e126ba97
EC		 316 break;
317
318 case IB_QPT_UD:
f0313965
ES		 319 if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
320 size += sizeof(struct mlx5_wqe_eth_pad) +
321 sizeof(struct mlx5_wqe_eth_seg);
322 /* fall through */
e126ba97 323 case IB_QPT_SMI:
d16e91da 324 case MLX5_IB_QPT_HW_GSI:
b125a54b 325 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 326 sizeof(struct mlx5_wqe_datagram_seg);
327 break;
328
329 case MLX5_IB_QPT_REG_UMR:
b125a54b 330 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 331 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
332 sizeof(struct mlx5_mkey_seg);
333 break;
334
335 default:
336 return -EINVAL;
337 }
338
339 return size;
340}
341
342static int calc_send_wqe(struct ib_qp_init_attr *attr)
343{
344 int inl_size = 0;
345 int size;
346
f0313965 347 size = sq_overhead(attr);
e126ba97
EC		 348 if (size < 0)
349 return size;
350
351 if (attr->cap.max_inline_data) {
352 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
353 attr->cap.max_inline_data;
354 }
355
356 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
e1e66cc2
SG		 357 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
358 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
359 return MLX5_SIG_WQE_SIZE;
360 else
361 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
e126ba97
EC		 362}
363
288c01b7
EC		 364static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
365{
366 int max_sge;
367
368 if (attr->qp_type == IB_QPT_RC)
369 max_sge = (min_t(int, wqe_size, 512) -
370 sizeof(struct mlx5_wqe_ctrl_seg) -
371 sizeof(struct mlx5_wqe_raddr_seg)) /
372 sizeof(struct mlx5_wqe_data_seg);
373 else if (attr->qp_type == IB_QPT_XRC_INI)
374 max_sge = (min_t(int, wqe_size, 512) -
375 sizeof(struct mlx5_wqe_ctrl_seg) -
376 sizeof(struct mlx5_wqe_xrc_seg) -
377 sizeof(struct mlx5_wqe_raddr_seg)) /
378 sizeof(struct mlx5_wqe_data_seg);
379 else
380 max_sge = (wqe_size - sq_overhead(attr)) /
381 sizeof(struct mlx5_wqe_data_seg);
382
383 return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
384 sizeof(struct mlx5_wqe_data_seg));
385}
386
e126ba97
EC		 387static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
388 struct mlx5_ib_qp *qp)
389{
390 int wqe_size;
391 int wq_size;
392
393 if (!attr->cap.max_send_wr)
394 return 0;
395
396 wqe_size = calc_send_wqe(attr);
397 mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
398 if (wqe_size < 0)
399 return wqe_size;
400
938fe83c 401 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
b125a54b 402 mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
938fe83c 403 wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 404 return -EINVAL;
405 }
406
f0313965
ES		 407 qp->max_inline_data = wqe_size - sq_overhead(attr) -
408 sizeof(struct mlx5_wqe_inline_seg);
e126ba97
EC		 409 attr->cap.max_inline_data = qp->max_inline_data;
410
e1e66cc2
SG		 411 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
412 qp->signature_en = true;
413
e126ba97
EC		 414 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
415 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
938fe83c 416 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
1974ab9d
BVA		 417 mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
418 attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
938fe83c
SM		 419 qp->sq.wqe_cnt,
420 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
b125a54b
EC		 421 return -ENOMEM;
422 }
e126ba97 423 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
288c01b7
EC		 424 qp->sq.max_gs = get_send_sge(attr, wqe_size);
425 if (qp->sq.max_gs < attr->cap.max_send_sge)
426 return -ENOMEM;
427
428 attr->cap.max_send_sge = qp->sq.max_gs;
b125a54b
EC		 429 qp->sq.max_post = wq_size / wqe_size;
430 attr->cap.max_send_wr = qp->sq.max_post;
e126ba97
EC		 431
432 return wq_size;
433}
434
435static int set_user_buf_size(struct mlx5_ib_dev *dev,
436 struct mlx5_ib_qp *qp,
19098df2 437 struct mlx5_ib_create_qp *ucmd,
0fb2ed66 438 struct mlx5_ib_qp_base *base,
439 struct ib_qp_init_attr *attr)
e126ba97
EC		 440{
441 int desc_sz = 1 << qp->sq.wqe_shift;
442
938fe83c 443 if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
e126ba97 444 mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
938fe83c 445 desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 446 return -EINVAL;
447 }
448
449 if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) {
450 mlx5_ib_warn(dev, "sq_wqe_count %d, sq_wqe_count %d\n",
451 ucmd->sq_wqe_count, ucmd->sq_wqe_count);
452 return -EINVAL;
453 }
454
455 qp->sq.wqe_cnt = ucmd->sq_wqe_count;
456
938fe83c 457 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
e126ba97 458 mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
938fe83c
SM		 459 qp->sq.wqe_cnt,
460 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
e126ba97
EC		 461 return -EINVAL;
462 }
463
c2e53b2c
YH		 464 if (attr->qp_type == IB_QPT_RAW_PACKET ||
465 qp->flags & MLX5_IB_QP_UNDERLAY) {
0fb2ed66 466 base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
467 qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
468 } else {
469 base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
470 (qp->sq.wqe_cnt << 6);
471 }
e126ba97
EC		 472
473 return 0;
474}
475
476static int qp_has_rq(struct ib_qp_init_attr *attr)
477{
478 if (attr->qp_type == IB_QPT_XRC_INI ||
479 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
480 attr->qp_type == MLX5_IB_QPT_REG_UMR ||
481 !attr->cap.max_recv_wr)
482 return 0;
483
484 return 1;
485}
486
2f5ff264 487static int first_med_bfreg(void)
c1be5232
EC		 488{
489 return 1;
490}
491
0b80c14f
EC		 492enum {
493 /* this is the first blue flame register in the array of bfregs assigned
494 * to a processes. Since we do not use it for blue flame but rather
495 * regular 64 bit doorbells, we do not need a lock for maintaiing
496 * "odd/even" order
497 */
498 NUM_NON_BLUE_FLAME_BFREGS = 1,
499};
500
b037c29a
EC		 501static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
502{
31a78a5a 503 return get_num_static_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a
EC		 504}
505
506static int num_med_bfreg(struct mlx5_ib_dev *dev,
507 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 508{
509 int n;
510
b037c29a
EC		 511 n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
512 NUM_NON_BLUE_FLAME_BFREGS;
c1be5232
EC		 513
514 return n >= 0 ? n : 0;
515}
516
b037c29a
EC		 517static int first_hi_bfreg(struct mlx5_ib_dev *dev,
518 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 519{
520 int med;
c1be5232 521
b037c29a
EC		 522 med = num_med_bfreg(dev, bfregi);
523 return ++med;
c1be5232
EC		 524}
525
b037c29a
EC		 526static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
527 struct mlx5_bfreg_info *bfregi)
e126ba97 528{
e126ba97
EC		 529 int i;
530
b037c29a
EC		 531 for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
532 if (!bfregi->count[i]) {
2f5ff264 533 bfregi->count[i]++;
e126ba97
EC		 534 return i;
535 }
536 }
537
538 return -ENOMEM;
539}
540
b037c29a
EC		 541static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
542 struct mlx5_bfreg_info *bfregi)
e126ba97 543{
2f5ff264 544 int minidx = first_med_bfreg();
e126ba97
EC		 545 int i;
546
b037c29a 547 for (i = first_med_bfreg(); i < first_hi_bfreg(dev, bfregi); i++) {
2f5ff264 548 if (bfregi->count[i] < bfregi->count[minidx])
e126ba97 549 minidx = i;
0b80c14f
EC		 550 if (!bfregi->count[minidx])
551 break;
e126ba97
EC		 552 }
553
2f5ff264 554 bfregi->count[minidx]++;
e126ba97
EC		 555 return minidx;
556}
557
b037c29a
EC		 558static int alloc_bfreg(struct mlx5_ib_dev *dev,
559 struct mlx5_bfreg_info *bfregi,
2f5ff264 560 enum mlx5_ib_latency_class lat)
e126ba97 561{
2f5ff264 562 int bfregn = -EINVAL;
e126ba97 563
2f5ff264 564 mutex_lock(&bfregi->lock);
e126ba97
EC		 565 switch (lat) {
566 case MLX5_IB_LATENCY_CLASS_LOW:
0b80c14f 567 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
2f5ff264
EC		 568 bfregn = 0;
569 bfregi->count[bfregn]++;
e126ba97
EC		 570 break;
571
572 case MLX5_IB_LATENCY_CLASS_MEDIUM:
2f5ff264
EC		 573 if (bfregi->ver < 2)
574 bfregn = -ENOMEM;
78c0f98c 575 else
b037c29a 576 bfregn = alloc_med_class_bfreg(dev, bfregi);
e126ba97
EC		 577 break;
578
579 case MLX5_IB_LATENCY_CLASS_HIGH:
2f5ff264
EC		 580 if (bfregi->ver < 2)
581 bfregn = -ENOMEM;
78c0f98c 582 else
b037c29a 583 bfregn = alloc_high_class_bfreg(dev, bfregi);
e126ba97
EC		 584 break;
585 }
2f5ff264 586 mutex_unlock(&bfregi->lock);
e126ba97 587
2f5ff264 588 return bfregn;
e126ba97
EC		 589}
590
4ed131d0 591void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
e126ba97 592{
2f5ff264 593 mutex_lock(&bfregi->lock);
b037c29a 594 bfregi->count[bfregn]--;
2f5ff264 595 mutex_unlock(&bfregi->lock);
e126ba97
EC		 596}
597
598static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
599{
600 switch (state) {
601 case IB_QPS_RESET: return MLX5_QP_STATE_RST;
602 case IB_QPS_INIT: return MLX5_QP_STATE_INIT;
603 case IB_QPS_RTR: return MLX5_QP_STATE_RTR;
604 case IB_QPS_RTS: return MLX5_QP_STATE_RTS;
605 case IB_QPS_SQD: return MLX5_QP_STATE_SQD;
606 case IB_QPS_SQE: return MLX5_QP_STATE_SQER;
607 case IB_QPS_ERR: return MLX5_QP_STATE_ERR;
608 default: return -1;
609 }
610}
611
612static int to_mlx5_st(enum ib_qp_type type)
613{
614 switch (type) {
615 case IB_QPT_RC: return MLX5_QP_ST_RC;
616 case IB_QPT_UC: return MLX5_QP_ST_UC;
617 case IB_QPT_UD: return MLX5_QP_ST_UD;
618 case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR;
619 case IB_QPT_XRC_INI:
620 case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC;
621 case IB_QPT_SMI: return MLX5_QP_ST_QP0;
d16e91da 622 case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1;
c32a4f29 623 case MLX5_IB_QPT_DCI: return MLX5_QP_ST_DCI;
e126ba97 624 case IB_QPT_RAW_IPV6: return MLX5_QP_ST_RAW_IPV6;
e126ba97 625 case IB_QPT_RAW_PACKET:
0fb2ed66 626 case IB_QPT_RAW_ETHERTYPE: return MLX5_QP_ST_RAW_ETHERTYPE;
e126ba97
EC		 627 case IB_QPT_MAX:
628 default: return -EINVAL;
629 }
630}
631
89ea94a7
MG		 632static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
633 struct mlx5_ib_cq *recv_cq);
634static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
635 struct mlx5_ib_cq *recv_cq);
636
b037c29a 637static int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
1ee47ab3
YH		 638 struct mlx5_bfreg_info *bfregi, int bfregn,
639 bool dyn_bfreg)
e126ba97 640{
b037c29a
EC		 641 int bfregs_per_sys_page;
642 int index_of_sys_page;
643 int offset;
644
645 bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
646 MLX5_NON_FP_BFREGS_PER_UAR;
647 index_of_sys_page = bfregn / bfregs_per_sys_page;
648
1ee47ab3
YH		 649 if (dyn_bfreg) {
650 index_of_sys_page += bfregi->num_static_sys_pages;
651 if (bfregn > bfregi->num_dyn_bfregs ||
652 bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
653 mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
654 return -EINVAL;
655 }
656 }
b037c29a 657
1ee47ab3 658 offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a 659 return bfregi->sys_pages[index_of_sys_page] + offset;
e126ba97
EC		 660}
661
19098df2 662static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
663 struct ib_pd *pd,
664 unsigned long addr, size_t size,
665 struct ib_umem **umem,
666 int *npages, int *page_shift, int *ncont,
667 u32 *offset)
668{
669 int err;
670
671 *umem = ib_umem_get(pd->uobject->context, addr, size, 0, 0);
672 if (IS_ERR(*umem)) {
673 mlx5_ib_dbg(dev, "umem_get failed\n");
674 return PTR_ERR(*umem);
675 }
676
762f899a 677 mlx5_ib_cont_pages(*umem, addr, 0, npages, page_shift, ncont, NULL);
19098df2 678
679 err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
680 if (err) {
681 mlx5_ib_warn(dev, "bad offset\n");
682 goto err_umem;
683 }
684
685 mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
686 addr, size, *npages, *page_shift, *ncont, *offset);
687
688 return 0;
689
690err_umem:
691 ib_umem_release(*umem);
692 *umem = NULL;
693
694 return err;
695}
696
fe248c3a
MG		 697static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
698 struct mlx5_ib_rwq *rwq)
79b20a6c
YH		 699{
700 struct mlx5_ib_ucontext *context;
701
fe248c3a
MG		 702 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
703 atomic_dec(&dev->delay_drop.rqs_cnt);
704
79b20a6c
YH		 705 context = to_mucontext(pd->uobject->context);
706 mlx5_ib_db_unmap_user(context, &rwq->db);
707 if (rwq->umem)
708 ib_umem_release(rwq->umem);
709}
710
711static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
712 struct mlx5_ib_rwq *rwq,
713 struct mlx5_ib_create_wq *ucmd)
714{
715 struct mlx5_ib_ucontext *context;
716 int page_shift = 0;
717 int npages;
718 u32 offset = 0;
719 int ncont = 0;
720 int err;
721
722 if (!ucmd->buf_addr)
723 return -EINVAL;
724
725 context = to_mucontext(pd->uobject->context);
726 rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,
727 rwq->buf_size, 0, 0);
728 if (IS_ERR(rwq->umem)) {
729 mlx5_ib_dbg(dev, "umem_get failed\n");
730 err = PTR_ERR(rwq->umem);
731 return err;
732 }
733
762f899a 734 mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,
79b20a6c
YH		 735 &ncont, NULL);
736 err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
737 &rwq->rq_page_offset);
738 if (err) {
739 mlx5_ib_warn(dev, "bad offset\n");
740 goto err_umem;
741 }
742
743 rwq->rq_num_pas = ncont;
744 rwq->page_shift = page_shift;
745 rwq->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
746 rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
747
748 mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
749 (unsigned long long)ucmd->buf_addr, rwq->buf_size,
750 npages, page_shift, ncont, offset);
751
752 err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);
753 if (err) {
754 mlx5_ib_dbg(dev, "map failed\n");
755 goto err_umem;
756 }
757
758 rwq->create_type = MLX5_WQ_USER;
759 return 0;
760
761err_umem:
762 ib_umem_release(rwq->umem);
763 return err;
764}
765
b037c29a
EC		 766static int adjust_bfregn(struct mlx5_ib_dev *dev,
767 struct mlx5_bfreg_info *bfregi, int bfregn)
768{
769 return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
770 bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
771}
772
e126ba97
EC		 773static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
774 struct mlx5_ib_qp *qp, struct ib_udata *udata,
0fb2ed66 775 struct ib_qp_init_attr *attr,
09a7d9ec 776 u32 **in,
19098df2 777 struct mlx5_ib_create_qp_resp *resp, int *inlen,
778 struct mlx5_ib_qp_base *base)
e126ba97
EC		 779{
780 struct mlx5_ib_ucontext *context;
781 struct mlx5_ib_create_qp ucmd;
19098df2 782 struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
9e9c47d0 783 int page_shift = 0;
1ee47ab3 784 int uar_index = 0;
e126ba97 785 int npages;
9e9c47d0 786 u32 offset = 0;
2f5ff264 787 int bfregn;
9e9c47d0 788 int ncont = 0;
09a7d9ec
SM		 789 __be64 *pas;
790 void *qpc;
e126ba97
EC		 791 int err;
792
793 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
794 if (err) {
795 mlx5_ib_dbg(dev, "copy failed\n");
796 return err;
797 }
798
799 context = to_mucontext(pd->uobject->context);
1ee47ab3
YH		 800 if (ucmd.flags & MLX5_QP_FLAG_BFREG_INDEX) {
801 uar_index = bfregn_to_uar_index(dev, &context->bfregi,
802 ucmd.bfreg_index, true);
803 if (uar_index < 0)
804 return uar_index;
805
806 bfregn = MLX5_IB_INVALID_BFREG;
807 } else if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) {
808 /*
809 * TBD: should come from the verbs when we have the API
810 */
051f2630 811 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
2f5ff264 812 bfregn = MLX5_CROSS_CHANNEL_BFREG;
1ee47ab3 813 }
051f2630 814 else {
b037c29a 815 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_HIGH);
2f5ff264
EC		 816 if (bfregn < 0) {
817 mlx5_ib_dbg(dev, "failed to allocate low latency BFREG\n");
051f2630 818 mlx5_ib_dbg(dev, "reverting to medium latency\n");
b037c29a 819 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_MEDIUM);
2f5ff264
EC		 820 if (bfregn < 0) {
821 mlx5_ib_dbg(dev, "failed to allocate medium latency BFREG\n");
051f2630 822 mlx5_ib_dbg(dev, "reverting to high latency\n");
b037c29a 823 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_LOW);
2f5ff264
EC		 824 if (bfregn < 0) {
825 mlx5_ib_warn(dev, "bfreg allocation failed\n");
826 return bfregn;
051f2630 827 }
c1be5232 828 }
e126ba97
EC		 829 }
830 }
831
2f5ff264 832 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
1ee47ab3
YH		 833 if (bfregn != MLX5_IB_INVALID_BFREG)
834 uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
835 false);
e126ba97 836
48fea837
HE		 837 qp->rq.offset = 0;
838 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
839 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
840
0fb2ed66 841 err = set_user_buf_size(dev, qp, &ucmd, base, attr);
e126ba97 842 if (err)
2f5ff264 843 goto err_bfreg;
e126ba97 844
19098df2 845 if (ucmd.buf_addr && ubuffer->buf_size) {
846 ubuffer->buf_addr = ucmd.buf_addr;
847 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr,
848 ubuffer->buf_size,
849 &ubuffer->umem, &npages, &page_shift,
850 &ncont, &offset);
851 if (err)
2f5ff264 852 goto err_bfreg;
9e9c47d0 853 } else {
19098df2 854 ubuffer->umem = NULL;
e126ba97 855 }
e126ba97 856
09a7d9ec
SM		 857 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
858 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1b9a07ee 859 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 860 if (!*in) {
861 err = -ENOMEM;
862 goto err_umem;
863 }
09a7d9ec
SM		 864
865 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
19098df2 866 if (ubuffer->umem)
09a7d9ec
SM		 867 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
868
869 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
870
871 MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
872 MLX5_SET(qpc, qpc, page_offset, offset);
e126ba97 873
09a7d9ec 874 MLX5_SET(qpc, qpc, uar_page, uar_index);
1ee47ab3
YH		 875 if (bfregn != MLX5_IB_INVALID_BFREG)
876 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
877 else
878 resp->bfreg_index = MLX5_IB_INVALID_BFREG;
2f5ff264 879 qp->bfregn = bfregn;
e126ba97
EC		 880
881 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
882 if (err) {
883 mlx5_ib_dbg(dev, "map failed\n");
884 goto err_free;
885 }
886
41d902cb 887 err = ib_copy_to_udata(udata, resp, min(udata->outlen, sizeof(*resp)));
e126ba97
EC		 888 if (err) {
889 mlx5_ib_dbg(dev, "copy failed\n");
890 goto err_unmap;
891 }
892 qp->create_type = MLX5_QP_USER;
893
894 return 0;
895
896err_unmap:
897 mlx5_ib_db_unmap_user(context, &qp->db);
898
899err_free:
479163f4 900 kvfree(*in);
e126ba97
EC		 901
902err_umem:
19098df2 903 if (ubuffer->umem)
904 ib_umem_release(ubuffer->umem);
e126ba97 905
2f5ff264 906err_bfreg:
1ee47ab3
YH		 907 if (bfregn != MLX5_IB_INVALID_BFREG)
908 mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
e126ba97
EC		 909 return err;
910}
911
b037c29a
EC		 912static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
913 struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base)
e126ba97
EC		 914{
915 struct mlx5_ib_ucontext *context;
916
917 context = to_mucontext(pd->uobject->context);
918 mlx5_ib_db_unmap_user(context, &qp->db);
19098df2 919 if (base->ubuffer.umem)
920 ib_umem_release(base->ubuffer.umem);
1ee47ab3
YH		 921
922 /*
923 * Free only the BFREGs which are handled by the kernel.
924 * BFREGs of UARs allocated dynamically are handled by user.
925 */
926 if (qp->bfregn != MLX5_IB_INVALID_BFREG)
927 mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
e126ba97
EC		 928}
929
930static int create_kernel_qp(struct mlx5_ib_dev *dev,
931 struct ib_qp_init_attr *init_attr,
932 struct mlx5_ib_qp *qp,
09a7d9ec 933 u32 **in, int *inlen,
19098df2 934 struct mlx5_ib_qp_base *base)
e126ba97 935{
e126ba97 936 int uar_index;
09a7d9ec 937 void *qpc;
e126ba97
EC		 938 int err;
939
f0313965
ES		 940 if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
941 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
b11a4f9c 942 IB_QP_CREATE_IPOIB_UD_LSO |
93d576af 943 IB_QP_CREATE_NETIF_QP |
b11a4f9c 944 mlx5_ib_create_qp_sqpn_qp1()))
1a4c3a3d 945 return -EINVAL;
e126ba97
EC		 946
947 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
5fe9dec0
EC		 948 qp->bf.bfreg = &dev->fp_bfreg;
949 else
950 qp->bf.bfreg = &dev->bfreg;
e126ba97 951
d8030b0d
EC		 952 /* We need to divide by two since each register is comprised of
953 * two buffers of identical size, namely odd and even
954 */
955 qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
5fe9dec0 956 uar_index = qp->bf.bfreg->index;
e126ba97
EC		 957
958 err = calc_sq_size(dev, init_attr, qp);
959 if (err < 0) {
960 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 961 return err;
e126ba97
EC		 962 }
963
964 qp->rq.offset = 0;
965 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
19098df2 966 base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
e126ba97 967
19098df2 968 err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
e126ba97
EC		 969 if (err) {
970 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 971 return err;
e126ba97
EC		 972 }
973
974 qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
09a7d9ec
SM		 975 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
976 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1b9a07ee 977 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 978 if (!*in) {
979 err = -ENOMEM;
980 goto err_buf;
981 }
09a7d9ec
SM		 982
983 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
984 MLX5_SET(qpc, qpc, uar_page, uar_index);
985 MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
986
e126ba97 987 /* Set "fast registration enabled" for all kernel QPs */
09a7d9ec
SM		 988 MLX5_SET(qpc, qpc, fre, 1);
989 MLX5_SET(qpc, qpc, rlky, 1);
e126ba97 990
b11a4f9c 991 if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) {
09a7d9ec 992 MLX5_SET(qpc, qpc, deth_sqpn, 1);
b11a4f9c
HE		 993 qp->flags |= MLX5_IB_QP_SQPN_QP1;
994 }
995
09a7d9ec
SM		 996 mlx5_fill_page_array(&qp->buf,
997 (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas));
e126ba97 998
9603b61d 999 err = mlx5_db_alloc(dev->mdev, &qp->db);
e126ba97
EC		 1000 if (err) {
1001 mlx5_ib_dbg(dev, "err %d\n", err);
1002 goto err_free;
1003 }
1004
b5883008
LD		 1005 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1006 sizeof(*qp->sq.wrid), GFP_KERNEL);
1007 qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1008 sizeof(*qp->sq.wr_data), GFP_KERNEL);
1009 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1010 sizeof(*qp->rq.wrid), GFP_KERNEL);
1011 qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1012 sizeof(*qp->sq.w_list), GFP_KERNEL);
1013 qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1014 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
e126ba97
EC		 1015
1016 if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1017 !qp->sq.w_list || !qp->sq.wqe_head) {
1018 err = -ENOMEM;
1019 goto err_wrid;
1020 }
1021 qp->create_type = MLX5_QP_KERNEL;
1022
1023 return 0;
1024
1025err_wrid:
b5883008
LD		 1026 kvfree(qp->sq.wqe_head);
1027 kvfree(qp->sq.w_list);
1028 kvfree(qp->sq.wrid);
1029 kvfree(qp->sq.wr_data);
1030 kvfree(qp->rq.wrid);
f4044dac 1031 mlx5_db_free(dev->mdev, &qp->db);
e126ba97
EC		 1032
1033err_free:
479163f4 1034 kvfree(*in);
e126ba97
EC		 1035
1036err_buf:
9603b61d 1037 mlx5_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1038 return err;
1039}
1040
1041static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1042{
b5883008
LD		 1043 kvfree(qp->sq.wqe_head);
1044 kvfree(qp->sq.w_list);
1045 kvfree(qp->sq.wrid);
1046 kvfree(qp->sq.wr_data);
1047 kvfree(qp->rq.wrid);
f4044dac 1048 mlx5_db_free(dev->mdev, &qp->db);
9603b61d 1049 mlx5_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1050}
1051
09a7d9ec 1052static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
e126ba97
EC		 1053{
1054 if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
c32a4f29 1055 (attr->qp_type == MLX5_IB_QPT_DCI) ||
e126ba97 1056 (attr->qp_type == IB_QPT_XRC_INI))
09a7d9ec 1057 return MLX5_SRQ_RQ;
e126ba97 1058 else if (!qp->has_rq)
09a7d9ec 1059 return MLX5_ZERO_LEN_RQ;
e126ba97 1060 else
09a7d9ec 1061 return MLX5_NON_ZERO_RQ;
e126ba97
EC		 1062}
1063
1064static int is_connected(enum ib_qp_type qp_type)
1065{
1066 if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC)
1067 return 1;
1068
1069 return 0;
1070}
1071
0fb2ed66 1072static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
c2e53b2c 1073 struct mlx5_ib_qp *qp,
0fb2ed66 1074 struct mlx5_ib_sq *sq, u32 tdn)
1075{
c4f287c4 1076 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
0fb2ed66 1077 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1078
0fb2ed66 1079 MLX5_SET(tisc, tisc, transport_domain, tdn);
c2e53b2c
YH		 1080 if (qp->flags & MLX5_IB_QP_UNDERLAY)
1081 MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1082
0fb2ed66 1083 return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
1084}
1085
1086static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1087 struct mlx5_ib_sq *sq)
1088{
1089 mlx5_core_destroy_tis(dev->mdev, sq->tisn);
1090}
1091
b96c9dde
MB		 1092static void destroy_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
1093 struct mlx5_ib_sq *sq)
1094{
1095 if (sq->flow_rule)
1096 mlx5_del_flow_rules(sq->flow_rule);
1097}
1098
0fb2ed66 1099static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1100 struct mlx5_ib_sq *sq, void *qpin,
1101 struct ib_pd *pd)
1102{
1103 struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1104 __be64 *pas;
1105 void *in;
1106 void *sqc;
1107 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1108 void *wq;
1109 int inlen;
1110 int err;
1111 int page_shift = 0;
1112 int npages;
1113 int ncont = 0;
1114 u32 offset = 0;
1115
1116 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, ubuffer->buf_size,
1117 &sq->ubuffer.umem, &npages, &page_shift,
1118 &ncont, &offset);
1119 if (err)
1120 return err;
1121
1122 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;
1b9a07ee 1123 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1124 if (!in) {
1125 err = -ENOMEM;
1126 goto err_umem;
1127 }
1128
1129 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1130 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
795b609c
BW		 1131 if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1132 MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
0fb2ed66 1133 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1134 MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1135 MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1136 MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1137 MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
96dc3fc5
NO		 1138 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1139 MLX5_CAP_ETH(dev->mdev, swp))
1140 MLX5_SET(sqc, sqc, allow_swp, 1);
0fb2ed66 1141
1142 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1143 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1144 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1145 MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1146 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1147 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1148 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1149 MLX5_SET(wq, wq, log_wq_pg_sz, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1150 MLX5_SET(wq, wq, page_offset, offset);
1151
1152 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1153 mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
1154
1155 err = mlx5_core_create_sq_tracked(dev->mdev, in, inlen, &sq->base.mqp);
1156
1157 kvfree(in);
1158
1159 if (err)
1160 goto err_umem;
1161
b96c9dde
MB		 1162 err = create_flow_rule_vport_sq(dev, sq);
1163 if (err)
1164 goto err_flow;
1165
0fb2ed66 1166 return 0;
1167
b96c9dde
MB		 1168err_flow:
1169 mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
1170
0fb2ed66 1171err_umem:
1172 ib_umem_release(sq->ubuffer.umem);
1173 sq->ubuffer.umem = NULL;
1174
1175 return err;
1176}
1177
1178static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1179 struct mlx5_ib_sq *sq)
1180{
b96c9dde 1181 destroy_flow_rule_vport_sq(dev, sq);
0fb2ed66 1182 mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
1183 ib_umem_release(sq->ubuffer.umem);
1184}
1185
2c292dbb 1186static size_t get_rq_pas_size(void *qpc)
0fb2ed66 1187{
1188 u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
1189 u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
1190 u32 log_rq_size = MLX5_GET(qpc, qpc, log_rq_size);
1191 u32 page_offset = MLX5_GET(qpc, qpc, page_offset);
1192 u32 po_quanta = 1 << (log_page_size - 6);
1193 u32 rq_sz = 1 << (log_rq_size + 4 + log_rq_stride);
1194 u32 page_size = 1 << log_page_size;
1195 u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
1196 u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size;
1197
1198 return rq_num_pas * sizeof(u64);
1199}
1200
1201static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2c292dbb
BP		 1202 struct mlx5_ib_rq *rq, void *qpin,
1203 size_t qpinlen)
0fb2ed66 1204{
358e42ea 1205 struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
0fb2ed66 1206 __be64 *pas;
1207 __be64 *qp_pas;
1208 void *in;
1209 void *rqc;
1210 void *wq;
1211 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
2c292dbb
BP		 1212 size_t rq_pas_size = get_rq_pas_size(qpc);
1213 size_t inlen;
0fb2ed66 1214 int err;
2c292dbb
BP		 1215
1216 if (qpinlen < rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas))
1217 return -EINVAL;
0fb2ed66 1218
1219 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
1b9a07ee 1220 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1221 if (!in)
1222 return -ENOMEM;
1223
1224 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
e4cc4fa7
NO		 1225 if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1226 MLX5_SET(rqc, rqc, vsd, 1);
0fb2ed66 1227 MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1228 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1229 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1230 MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1231 MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1232
358e42ea
MD		 1233 if (mqp->flags & MLX5_IB_QP_CAP_SCATTER_FCS)
1234 MLX5_SET(rqc, rqc, scatter_fcs, 1);
1235
0fb2ed66 1236 wq = MLX5_ADDR_OF(rqc, rqc, wq);
1237 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 1238 if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1239 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
0fb2ed66 1240 MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));
1241 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1242 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1243 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1244 MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));
1245 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1246
1247 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1248 qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
1249 memcpy(pas, qp_pas, rq_pas_size);
1250
1251 err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rq->base.mqp);
1252
1253 kvfree(in);
1254
1255 return err;
1256}
1257
1258static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1259 struct mlx5_ib_rq *rq)
1260{
1261 mlx5_core_destroy_rq_tracked(dev->mdev, &rq->base.mqp);
1262}
1263
f95ef6cb
MG		 1264static bool tunnel_offload_supported(struct mlx5_core_dev *dev)
1265{
1266 return (MLX5_CAP_ETH(dev, tunnel_stateless_vxlan) ||
1267 MLX5_CAP_ETH(dev, tunnel_stateless_gre) ||
1268 MLX5_CAP_ETH(dev, tunnel_stateless_geneve_rx));
1269}
1270
0fb2ed66 1271static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
f95ef6cb
MG		 1272 struct mlx5_ib_rq *rq, u32 tdn,
1273 bool tunnel_offload_en)
0fb2ed66 1274{
1275 u32 *in;
1276 void *tirc;
1277 int inlen;
1278 int err;
1279
1280 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 1281 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1282 if (!in)
1283 return -ENOMEM;
1284
1285 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1286 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1287 MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1288 MLX5_SET(tirc, tirc, transport_domain, tdn);
f95ef6cb
MG		 1289 if (tunnel_offload_en)
1290 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
0fb2ed66 1291
ec9c2fb8
MB		 1292 if (dev->rep)
1293 MLX5_SET(tirc, tirc, self_lb_block,
1294 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_);
1295
0fb2ed66 1296 err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn);
1297
1298 kvfree(in);
1299
1300 return err;
1301}
1302
1303static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1304 struct mlx5_ib_rq *rq)
1305{
1306 mlx5_core_destroy_tir(dev->mdev, rq->tirn);
1307}
1308
1309static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2c292dbb 1310 u32 *in, size_t inlen,
0fb2ed66 1311 struct ib_pd *pd)
1312{
1313 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1314 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1315 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1316 struct ib_uobject *uobj = pd->uobject;
1317 struct ib_ucontext *ucontext = uobj->context;
1318 struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1319 int err;
1320 u32 tdn = mucontext->tdn;
1321
1322 if (qp->sq.wqe_cnt) {
c2e53b2c 1323 err = create_raw_packet_qp_tis(dev, qp, sq, tdn);
0fb2ed66 1324 if (err)
1325 return err;
1326
1327 err = create_raw_packet_qp_sq(dev, sq, in, pd);
1328 if (err)
1329 goto err_destroy_tis;
1330
1331 sq->base.container_mibqp = qp;
1d31e9c0 1332 sq->base.mqp.event = mlx5_ib_qp_event;
0fb2ed66 1333 }
1334
1335 if (qp->rq.wqe_cnt) {
358e42ea
MD		 1336 rq->base.container_mibqp = qp;
1337
e4cc4fa7
NO		 1338 if (qp->flags & MLX5_IB_QP_CVLAN_STRIPPING)
1339 rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
b1383aa6
NO		 1340 if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING)
1341 rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
2c292dbb 1342 err = create_raw_packet_qp_rq(dev, rq, in, inlen);
0fb2ed66 1343 if (err)
1344 goto err_destroy_sq;
1345
0fb2ed66 1346
f95ef6cb
MG		 1347 err = create_raw_packet_qp_tir(dev, rq, tdn,
1348 qp->tunnel_offload_en);
0fb2ed66 1349 if (err)
1350 goto err_destroy_rq;
1351 }
1352
1353 qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1354 rq->base.mqp.qpn;
1355
1356 return 0;
1357
1358err_destroy_rq:
1359 destroy_raw_packet_qp_rq(dev, rq);
1360err_destroy_sq:
1361 if (!qp->sq.wqe_cnt)
1362 return err;
1363 destroy_raw_packet_qp_sq(dev, sq);
1364err_destroy_tis:
1365 destroy_raw_packet_qp_tis(dev, sq);
1366
1367 return err;
1368}
1369
1370static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1371 struct mlx5_ib_qp *qp)
1372{
1373 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1374 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1375 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1376
1377 if (qp->rq.wqe_cnt) {
1378 destroy_raw_packet_qp_tir(dev, rq);
1379 destroy_raw_packet_qp_rq(dev, rq);
1380 }
1381
1382 if (qp->sq.wqe_cnt) {
1383 destroy_raw_packet_qp_sq(dev, sq);
1384 destroy_raw_packet_qp_tis(dev, sq);
1385 }
1386}
1387
1388static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1389 struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1390{
1391 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1392 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1393
1394 sq->sq = &qp->sq;
1395 rq->rq = &qp->rq;
1396 sq->doorbell = &qp->db;
1397 rq->doorbell = &qp->db;
1398}
1399
28d61370
YH		 1400static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1401{
1402 mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn);
1403}
1404
1405static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1406 struct ib_pd *pd,
1407 struct ib_qp_init_attr *init_attr,
1408 struct ib_udata *udata)
1409{
1410 struct ib_uobject *uobj = pd->uobject;
1411 struct ib_ucontext *ucontext = uobj->context;
1412 struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1413 struct mlx5_ib_create_qp_resp resp = {};
1414 int inlen;
1415 int err;
1416 u32 *in;
1417 void *tirc;
1418 void *hfso;
1419 u32 selected_fields = 0;
2d93fc85 1420 u32 outer_l4;
28d61370
YH		 1421 size_t min_resp_len;
1422 u32 tdn = mucontext->tdn;
1423 struct mlx5_ib_create_qp_rss ucmd = {};
1424 size_t required_cmd_sz;
1425
1426 if (init_attr->qp_type != IB_QPT_RAW_PACKET)
1427 return -EOPNOTSUPP;
1428
1429 if (init_attr->create_flags || init_attr->send_cq)
1430 return -EINVAL;
1431
2f5ff264 1432 min_resp_len = offsetof(typeof(resp), bfreg_index) + sizeof(resp.bfreg_index);
28d61370
YH		 1433 if (udata->outlen < min_resp_len)
1434 return -EINVAL;
1435
f95ef6cb 1436 required_cmd_sz = offsetof(typeof(ucmd), flags) + sizeof(ucmd.flags);
28d61370
YH		 1437 if (udata->inlen < required_cmd_sz) {
1438 mlx5_ib_dbg(dev, "invalid inlen\n");
1439 return -EINVAL;
1440 }
1441
1442 if (udata->inlen > sizeof(ucmd) &&
1443 !ib_is_udata_cleared(udata, sizeof(ucmd),
1444 udata->inlen - sizeof(ucmd))) {
1445 mlx5_ib_dbg(dev, "inlen is not supported\n");
1446 return -EOPNOTSUPP;
1447 }
1448
1449 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
1450 mlx5_ib_dbg(dev, "copy failed\n");
1451 return -EFAULT;
1452 }
1453
1454 if (ucmd.comp_mask) {
1455 mlx5_ib_dbg(dev, "invalid comp mask\n");
1456 return -EOPNOTSUPP;
1457 }
1458
f95ef6cb
MG		 1459 if (ucmd.flags & ~MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
1460 mlx5_ib_dbg(dev, "invalid flags\n");
1461 return -EOPNOTSUPP;
1462 }
1463
1464 if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS &&
1465 !tunnel_offload_supported(dev->mdev)) {
1466 mlx5_ib_dbg(dev, "tunnel offloads isn't supported\n");
28d61370
YH		 1467 return -EOPNOTSUPP;
1468 }
1469
309fa347
MG		 1470 if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1471 !(ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
1472 mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1473 return -EOPNOTSUPP;
1474 }
1475
41d902cb 1476 err = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
28d61370
YH		 1477 if (err) {
1478 mlx5_ib_dbg(dev, "copy failed\n");
1479 return -EINVAL;
1480 }
1481
1482 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 1483 in = kvzalloc(inlen, GFP_KERNEL);
28d61370
YH		 1484 if (!in)
1485 return -ENOMEM;
1486
1487 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1488 MLX5_SET(tirc, tirc, disp_type,
1489 MLX5_TIRC_DISP_TYPE_INDIRECT);
1490 MLX5_SET(tirc, tirc, indirect_table,
1491 init_attr->rwq_ind_tbl->ind_tbl_num);
1492 MLX5_SET(tirc, tirc, transport_domain, tdn);
1493
1494 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
f95ef6cb
MG		 1495
1496 if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1497 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1498
309fa347
MG		 1499 if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER)
1500 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1501 else
1502 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1503
28d61370
YH		 1504 switch (ucmd.rx_hash_function) {
1505 case MLX5_RX_HASH_FUNC_TOEPLITZ:
1506 {
1507 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1508 size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1509
1510 if (len != ucmd.rx_key_len) {
1511 err = -EINVAL;
1512 goto err;
1513 }
1514
1515 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1516 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1517 memcpy(rss_key, ucmd.rx_hash_key, len);
1518 break;
1519 }
1520 default:
1521 err = -EOPNOTSUPP;
1522 goto err;
1523 }
1524
1525 if (!ucmd.rx_hash_fields_mask) {
1526 /* special case when this TIR serves as steering entry without hashing */
1527 if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1528 goto create_tir;
1529 err = -EINVAL;
1530 goto err;
1531 }
1532
1533 if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1534 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1535 ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1536 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1537 err = -EINVAL;
1538 goto err;
1539 }
1540
1541 /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1542 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1543 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1544 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1545 MLX5_L3_PROT_TYPE_IPV4);
1546 else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1547 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1548 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1549 MLX5_L3_PROT_TYPE_IPV6);
1550
2d93fc85
MB		 1551 outer_l4 = ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1552 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) << 0 |
1553 ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1554 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP)) << 1 |
1555 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
1556
1557 /* Check that only one l4 protocol is set */
1558 if (outer_l4 & (outer_l4 - 1)) {
28d61370
YH		 1559 err = -EINVAL;
1560 goto err;
1561 }
1562
1563 /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1564 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1565 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1566 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1567 MLX5_L4_PROT_TYPE_TCP);
1568 else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1569 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1570 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1571 MLX5_L4_PROT_TYPE_UDP);
1572
1573 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1574 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1575 selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1576
1577 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1578 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1579 selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1580
1581 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1582 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1583 selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1584
1585 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1586 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1587 selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1588
2d93fc85
MB		 1589 if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
1590 selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1591
28d61370
YH		 1592 MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1593
1594create_tir:
ec9c2fb8
MB		 1595 if (dev->rep)
1596 MLX5_SET(tirc, tirc, self_lb_block,
1597 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_);
1598
28d61370
YH		 1599 err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn);
1600
1601 if (err)
1602 goto err;
1603
1604 kvfree(in);
1605 /* qpn is reserved for that QP */
1606 qp->trans_qp.base.mqp.qpn = 0;
d9f88e5a 1607 qp->flags |= MLX5_IB_QP_RSS;
28d61370
YH		 1608 return 0;
1609
1610err:
1611 kvfree(in);
1612 return err;
1613}
1614
e126ba97
EC		 1615static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1616 struct ib_qp_init_attr *init_attr,
1617 struct ib_udata *udata, struct mlx5_ib_qp *qp)
1618{
1619 struct mlx5_ib_resources *devr = &dev->devr;
09a7d9ec 1620 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
938fe83c 1621 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97 1622 struct mlx5_ib_create_qp_resp resp;
89ea94a7
MG		 1623 struct mlx5_ib_cq *send_cq;
1624 struct mlx5_ib_cq *recv_cq;
1625 unsigned long flags;
cfb5e088 1626 u32 uidx = MLX5_IB_DEFAULT_UIDX;
09a7d9ec
SM		 1627 struct mlx5_ib_create_qp ucmd;
1628 struct mlx5_ib_qp_base *base;
e7b169f3 1629 int mlx5_st;
cfb5e088 1630 void *qpc;
09a7d9ec
SM		 1631 u32 *in;
1632 int err;
e126ba97
EC		 1633
1634 mutex_init(&qp->mutex);
1635 spin_lock_init(&qp->sq.lock);
1636 spin_lock_init(&qp->rq.lock);
1637
e7b169f3
NO		 1638 mlx5_st = to_mlx5_st(init_attr->qp_type);
1639 if (mlx5_st < 0)
1640 return -EINVAL;
1641
28d61370
YH		 1642 if (init_attr->rwq_ind_tbl) {
1643 if (!udata)
1644 return -ENOSYS;
1645
1646 err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata);
1647 return err;
1648 }
1649
f360d88a 1650 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
938fe83c 1651 if (!MLX5_CAP_GEN(mdev, block_lb_mc)) {
f360d88a
EC		 1652 mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");
1653 return -EINVAL;
1654 } else {
1655 qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1656 }
1657 }
1658
051f2630
LR		 1659 if (init_attr->create_flags &
1660 (IB_QP_CREATE_CROSS_CHANNEL |
1661 IB_QP_CREATE_MANAGED_SEND |
1662 IB_QP_CREATE_MANAGED_RECV)) {
1663 if (!MLX5_CAP_GEN(mdev, cd)) {
1664 mlx5_ib_dbg(dev, "cross-channel isn't supported\n");
1665 return -EINVAL;
1666 }
1667 if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)
1668 qp->flags |= MLX5_IB_QP_CROSS_CHANNEL;
1669 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)
1670 qp->flags |= MLX5_IB_QP_MANAGED_SEND;
1671 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
1672 qp->flags |= MLX5_IB_QP_MANAGED_RECV;
1673 }
f0313965
ES		 1674
1675 if (init_attr->qp_type == IB_QPT_UD &&
1676 (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO))
1677 if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1678 mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");
1679 return -EOPNOTSUPP;
1680 }
1681
358e42ea
MD		 1682 if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1683 if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1684 mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs");
1685 return -EOPNOTSUPP;
1686 }
1687 if (!MLX5_CAP_GEN(dev->mdev, eth_net_offloads) ||
1688 !MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
1689 mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n");
1690 return -EOPNOTSUPP;
1691 }
1692 qp->flags |= MLX5_IB_QP_CAP_SCATTER_FCS;
1693 }
1694
e126ba97
EC		 1695 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1696 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
1697
e4cc4fa7
NO		 1698 if (init_attr->create_flags & IB_QP_CREATE_CVLAN_STRIPPING) {
1699 if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1700 MLX5_CAP_ETH(dev->mdev, vlan_cap)) ||
1701 (init_attr->qp_type != IB_QPT_RAW_PACKET))
1702 return -EOPNOTSUPP;
1703 qp->flags |= MLX5_IB_QP_CVLAN_STRIPPING;
1704 }
1705
e126ba97
EC		 1706 if (pd && pd->uobject) {
1707 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
1708 mlx5_ib_dbg(dev, "copy failed\n");
1709 return -EFAULT;
1710 }
1711
cfb5e088
HA		 1712 err = get_qp_user_index(to_mucontext(pd->uobject->context),
1713 &ucmd, udata->inlen, &uidx);
1714 if (err)
1715 return err;
1716
e126ba97
EC		 1717 qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
1718 qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
f95ef6cb
MG		 1719 if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
1720 if (init_attr->qp_type != IB_QPT_RAW_PACKET ||
1721 !tunnel_offload_supported(mdev)) {
1722 mlx5_ib_dbg(dev, "Tunnel offload isn't supported\n");
1723 return -EOPNOTSUPP;
1724 }
1725 qp->tunnel_offload_en = true;
1726 }
c2e53b2c
YH		 1727
1728 if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) {
1729 if (init_attr->qp_type != IB_QPT_UD ||
1730 (MLX5_CAP_GEN(dev->mdev, port_type) !=
1731 MLX5_CAP_PORT_TYPE_IB) ||
1732 !mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS)) {
1733 mlx5_ib_dbg(dev, "Source QP option isn't supported\n");
1734 return -EOPNOTSUPP;
1735 }
1736
1737 qp->flags |= MLX5_IB_QP_UNDERLAY;
1738 qp->underlay_qpn = init_attr->source_qpn;
1739 }
e126ba97
EC		 1740 } else {
1741 qp->wq_sig = !!wq_signature;
1742 }
1743
c2e53b2c
YH		 1744 base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
1745 qp->flags & MLX5_IB_QP_UNDERLAY) ?
1746 &qp->raw_packet_qp.rq.base :
1747 &qp->trans_qp.base;
1748
e126ba97
EC		 1749 qp->has_rq = qp_has_rq(init_attr);
1750 err = set_rq_size(dev, &init_attr->cap, qp->has_rq,
1751 qp, (pd && pd->uobject) ? &ucmd : NULL);
1752 if (err) {
1753 mlx5_ib_dbg(dev, "err %d\n", err);
1754 return err;
1755 }
1756
1757 if (pd) {
1758 if (pd->uobject) {
938fe83c
SM		 1759 __u32 max_wqes =
1760 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
e126ba97
EC		 1761 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);
1762 if (ucmd.rq_wqe_shift != qp->rq.wqe_shift ||
1763 ucmd.rq_wqe_count != qp->rq.wqe_cnt) {
1764 mlx5_ib_dbg(dev, "invalid rq params\n");
1765 return -EINVAL;
1766 }
938fe83c 1767 if (ucmd.sq_wqe_count > max_wqes) {
e126ba97 1768 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n",
938fe83c 1769 ucmd.sq_wqe_count, max_wqes);
e126ba97
EC		 1770 return -EINVAL;
1771 }
b11a4f9c
HE		 1772 if (init_attr->create_flags &
1773 mlx5_ib_create_qp_sqpn_qp1()) {
1774 mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n");
1775 return -EINVAL;
1776 }
0fb2ed66 1777 err = create_user_qp(dev, pd, qp, udata, init_attr, &in,
1778 &resp, &inlen, base);
e126ba97
EC		 1779 if (err)
1780 mlx5_ib_dbg(dev, "err %d\n", err);
1781 } else {
19098df2 1782 err = create_kernel_qp(dev, init_attr, qp, &in, &inlen,
1783 base);
e126ba97
EC		 1784 if (err)
1785 mlx5_ib_dbg(dev, "err %d\n", err);
e126ba97
EC		 1786 }
1787
1788 if (err)
1789 return err;
1790 } else {
1b9a07ee 1791 in = kvzalloc(inlen, GFP_KERNEL);
e126ba97
EC		 1792 if (!in)
1793 return -ENOMEM;
1794
1795 qp->create_type = MLX5_QP_EMPTY;
1796 }
1797
1798 if (is_sqp(init_attr->qp_type))
1799 qp->port = init_attr->port_num;
1800
09a7d9ec
SM		 1801 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1802
e7b169f3 1803 MLX5_SET(qpc, qpc, st, mlx5_st);
09a7d9ec 1804 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
e126ba97
EC		 1805
1806 if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
09a7d9ec 1807 MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
e126ba97 1808 else
09a7d9ec
SM		 1809 MLX5_SET(qpc, qpc, latency_sensitive, 1);
1810
e126ba97
EC		 1811
1812 if (qp->wq_sig)
09a7d9ec 1813 MLX5_SET(qpc, qpc, wq_signature, 1);
e126ba97 1814
f360d88a 1815 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
09a7d9ec 1816 MLX5_SET(qpc, qpc, block_lb_mc, 1);
f360d88a 1817
051f2630 1818 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
09a7d9ec 1819 MLX5_SET(qpc, qpc, cd_master, 1);
051f2630 1820 if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
09a7d9ec 1821 MLX5_SET(qpc, qpc, cd_slave_send, 1);
051f2630 1822 if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
09a7d9ec 1823 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
051f2630 1824
e126ba97
EC		 1825 if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
1826 int rcqe_sz;
1827 int scqe_sz;
1828
1829 rcqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->recv_cq);
1830 scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq);
1831
1832 if (rcqe_sz == 128)
09a7d9ec 1833 MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
e126ba97 1834 else
09a7d9ec 1835 MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA32_CQE);
e126ba97
EC		 1836
1837 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) {
1838 if (scqe_sz == 128)
09a7d9ec 1839 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
e126ba97 1840 else
09a7d9ec 1841 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
e126ba97
EC		 1842 }
1843 }
1844
1845 if (qp->rq.wqe_cnt) {
09a7d9ec
SM		 1846 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
1847 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
e126ba97
EC		 1848 }
1849
09a7d9ec 1850 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
e126ba97 1851
3fd3307e 1852 if (qp->sq.wqe_cnt) {
09a7d9ec 1853 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
3fd3307e 1854 } else {
09a7d9ec 1855 MLX5_SET(qpc, qpc, no_sq, 1);
3fd3307e
AK		 1856 if (init_attr->srq &&
1857 init_attr->srq->srq_type == IB_SRQT_TM)
1858 MLX5_SET(qpc, qpc, offload_type,
1859 MLX5_QPC_OFFLOAD_TYPE_RNDV);
1860 }
e126ba97
EC		 1861
1862 /* Set default resources */
1863 switch (init_attr->qp_type) {
1864 case IB_QPT_XRC_TGT:
09a7d9ec
SM		 1865 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1866 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
1867 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1868 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn);
e126ba97
EC		 1869 break;
1870 case IB_QPT_XRC_INI:
09a7d9ec
SM		 1871 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1872 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
1873 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
e126ba97
EC		 1874 break;
1875 default:
1876 if (init_attr->srq) {
09a7d9ec
SM		 1877 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
1878 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
e126ba97 1879 } else {
09a7d9ec
SM		 1880 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
1881 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
e126ba97
EC		 1882 }
1883 }
1884
1885 if (init_attr->send_cq)
09a7d9ec 1886 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
e126ba97
EC		 1887
1888 if (init_attr->recv_cq)
09a7d9ec 1889 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
e126ba97 1890
09a7d9ec 1891 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
e126ba97 1892
09a7d9ec
SM		 1893 /* 0xffffff means we ask to work with cqe version 0 */
1894 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
cfb5e088 1895 MLX5_SET(qpc, qpc, user_index, uidx);
09a7d9ec 1896
f0313965
ES		 1897 /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
1898 if (init_attr->qp_type == IB_QPT_UD &&
1899 (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
f0313965
ES		 1900 MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
1901 qp->flags |= MLX5_IB_QP_LSO;
1902 }
cfb5e088 1903
b1383aa6
NO		 1904 if (init_attr->create_flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
1905 if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
1906 mlx5_ib_dbg(dev, "scatter end padding is not supported\n");
1907 err = -EOPNOTSUPP;
1908 goto err;
1909 } else if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1910 MLX5_SET(qpc, qpc, end_padding_mode,
1911 MLX5_WQ_END_PAD_MODE_ALIGN);
1912 } else {
1913 qp->flags |= MLX5_IB_QP_PCI_WRITE_END_PADDING;
1914 }
1915 }
1916
2c292dbb
BP		 1917 if (inlen < 0) {
1918 err = -EINVAL;
1919 goto err;
1920 }
1921
c2e53b2c
YH		 1922 if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
1923 qp->flags & MLX5_IB_QP_UNDERLAY) {
0fb2ed66 1924 qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
1925 raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
2c292dbb 1926 err = create_raw_packet_qp(dev, qp, in, inlen, pd);
0fb2ed66 1927 } else {
1928 err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen);
1929 }
1930
e126ba97
EC		 1931 if (err) {
1932 mlx5_ib_dbg(dev, "create qp failed\n");
1933 goto err_create;
1934 }
1935
479163f4 1936 kvfree(in);
e126ba97 1937
19098df2 1938 base->container_mibqp = qp;
1939 base->mqp.event = mlx5_ib_qp_event;
e126ba97 1940
89ea94a7
MG		 1941 get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq,
1942 &send_cq, &recv_cq);
1943 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1944 mlx5_ib_lock_cqs(send_cq, recv_cq);
1945 /* Maintain device to QPs access, needed for further handling via reset
1946 * flow
1947 */
1948 list_add_tail(&qp->qps_list, &dev->qp_list);
1949 /* Maintain CQ to QPs access, needed for further handling via reset flow
1950 */
1951 if (send_cq)
1952 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
1953 if (recv_cq)
1954 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
1955 mlx5_ib_unlock_cqs(send_cq, recv_cq);
1956 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1957
e126ba97
EC		 1958 return 0;
1959
1960err_create:
1961 if (qp->create_type == MLX5_QP_USER)
b037c29a 1962 destroy_qp_user(dev, pd, qp, base);
e126ba97
EC		 1963 else if (qp->create_type == MLX5_QP_KERNEL)
1964 destroy_qp_kernel(dev, qp);
1965
b1383aa6 1966err:
479163f4 1967 kvfree(in);
e126ba97
EC		 1968 return err;
1969}
1970
1971static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
1972 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1973{
1974 if (send_cq) {
1975 if (recv_cq) {
1976 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
89ea94a7 1977 spin_lock(&send_cq->lock);
e126ba97
EC		 1978 spin_lock_nested(&recv_cq->lock,
1979 SINGLE_DEPTH_NESTING);
1980 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
89ea94a7 1981 spin_lock(&send_cq->lock);
e126ba97
EC		 1982 __acquire(&recv_cq->lock);
1983 } else {
89ea94a7 1984 spin_lock(&recv_cq->lock);
e126ba97
EC		 1985 spin_lock_nested(&send_cq->lock,
1986 SINGLE_DEPTH_NESTING);
1987 }
1988 } else {
89ea94a7 1989 spin_lock(&send_cq->lock);
6a4f139a 1990 __acquire(&recv_cq->lock);
e126ba97
EC		 1991 }
1992 } else if (recv_cq) {
89ea94a7 1993 spin_lock(&recv_cq->lock);
6a4f139a
EC		 1994 __acquire(&send_cq->lock);
1995 } else {
1996 __acquire(&send_cq->lock);
1997 __acquire(&recv_cq->lock);
e126ba97
EC		 1998 }
1999}
2000
2001static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2002 __releases(&send_cq->lock) __releases(&recv_cq->lock)
2003{
2004 if (send_cq) {
2005 if (recv_cq) {
2006 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
2007 spin_unlock(&recv_cq->lock);
89ea94a7 2008 spin_unlock(&send_cq->lock);
e126ba97
EC		 2009 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2010 __release(&recv_cq->lock);
89ea94a7 2011 spin_unlock(&send_cq->lock);
e126ba97
EC		 2012 } else {
2013 spin_unlock(&send_cq->lock);
89ea94a7 2014 spin_unlock(&recv_cq->lock);
e126ba97
EC		 2015 }
2016 } else {
6a4f139a 2017 __release(&recv_cq->lock);
89ea94a7 2018 spin_unlock(&send_cq->lock);
e126ba97
EC		 2019 }
2020 } else if (recv_cq) {
6a4f139a 2021 __release(&send_cq->lock);
89ea94a7 2022 spin_unlock(&recv_cq->lock);
6a4f139a
EC		 2023 } else {
2024 __release(&recv_cq->lock);
2025 __release(&send_cq->lock);
e126ba97
EC		 2026 }
2027}
2028
2029static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp)
2030{
2031 return to_mpd(qp->ibqp.pd);
2032}
2033
89ea94a7
MG		 2034static void get_cqs(enum ib_qp_type qp_type,
2035 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
e126ba97
EC		 2036 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2037{
89ea94a7 2038 switch (qp_type) {
e126ba97
EC		 2039 case IB_QPT_XRC_TGT:
2040 *send_cq = NULL;
2041 *recv_cq = NULL;
2042 break;
2043 case MLX5_IB_QPT_REG_UMR:
2044 case IB_QPT_XRC_INI:
89ea94a7 2045 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
e126ba97
EC		 2046 *recv_cq = NULL;
2047 break;
2048
2049 case IB_QPT_SMI:
d16e91da 2050 case MLX5_IB_QPT_HW_GSI:
e126ba97
EC		 2051 case IB_QPT_RC:
2052 case IB_QPT_UC:
2053 case IB_QPT_UD:
2054 case IB_QPT_RAW_IPV6:
2055 case IB_QPT_RAW_ETHERTYPE:
0fb2ed66 2056 case IB_QPT_RAW_PACKET:
89ea94a7
MG		 2057 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2058 *recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
e126ba97
EC		 2059 break;
2060
e126ba97
EC		 2061 case IB_QPT_MAX:
2062 default:
2063 *send_cq = NULL;
2064 *recv_cq = NULL;
2065 break;
2066 }
2067}
2068
ad5f8e96 2069static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 2070 const struct mlx5_modify_raw_qp_param *raw_qp_param,
2071 u8 lag_tx_affinity);
ad5f8e96 2072
e126ba97
EC		 2073static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
2074{
2075 struct mlx5_ib_cq *send_cq, *recv_cq;
c2e53b2c 2076 struct mlx5_ib_qp_base *base;
89ea94a7 2077 unsigned long flags;
e126ba97
EC		 2078 int err;
2079
28d61370
YH		 2080 if (qp->ibqp.rwq_ind_tbl) {
2081 destroy_rss_raw_qp_tir(dev, qp);
2082 return;
2083 }
2084
c2e53b2c
YH		 2085 base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2086 qp->flags & MLX5_IB_QP_UNDERLAY) ?
0fb2ed66 2087 &qp->raw_packet_qp.rq.base :
2088 &qp->trans_qp.base;
2089
6aec21f6 2090 if (qp->state != IB_QPS_RESET) {
c2e53b2c
YH		 2091 if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET &&
2092 !(qp->flags & MLX5_IB_QP_UNDERLAY)) {
ad5f8e96 2093 err = mlx5_core_qp_modify(dev->mdev,
1a412fb1
SM		 2094 MLX5_CMD_OP_2RST_QP, 0,
2095 NULL, &base->mqp);
ad5f8e96 2096 } else {
0680efa2
AV		 2097 struct mlx5_modify_raw_qp_param raw_qp_param = {
2098 .operation = MLX5_CMD_OP_2RST_QP
2099 };
2100
13eab21f 2101 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
ad5f8e96 2102 }
2103 if (err)
427c1e7b 2104 mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
19098df2 2105 base->mqp.qpn);
6aec21f6 2106 }
e126ba97 2107
89ea94a7
MG		 2108 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
2109 &send_cq, &recv_cq);
2110
2111 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2112 mlx5_ib_lock_cqs(send_cq, recv_cq);
2113 /* del from lists under both locks above to protect reset flow paths */
2114 list_del(&qp->qps_list);
2115 if (send_cq)
2116 list_del(&qp->cq_send_list);
2117
2118 if (recv_cq)
2119 list_del(&qp->cq_recv_list);
e126ba97
EC		 2120
2121 if (qp->create_type == MLX5_QP_KERNEL) {
19098df2 2122 __mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 2123 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2124 if (send_cq != recv_cq)
19098df2 2125 __mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
2126 NULL);
e126ba97 2127 }
89ea94a7
MG		 2128 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2129 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
e126ba97 2130
c2e53b2c
YH		 2131 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2132 qp->flags & MLX5_IB_QP_UNDERLAY) {
0fb2ed66 2133 destroy_raw_packet_qp(dev, qp);
2134 } else {
2135 err = mlx5_core_destroy_qp(dev->mdev, &base->mqp);
2136 if (err)
2137 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2138 base->mqp.qpn);
2139 }
e126ba97 2140
e126ba97
EC		 2141 if (qp->create_type == MLX5_QP_KERNEL)
2142 destroy_qp_kernel(dev, qp);
2143 else if (qp->create_type == MLX5_QP_USER)
b037c29a 2144 destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
e126ba97
EC		 2145}
2146
2147static const char *ib_qp_type_str(enum ib_qp_type type)
2148{
2149 switch (type) {
2150 case IB_QPT_SMI:
2151 return "IB_QPT_SMI";
2152 case IB_QPT_GSI:
2153 return "IB_QPT_GSI";
2154 case IB_QPT_RC:
2155 return "IB_QPT_RC";
2156 case IB_QPT_UC:
2157 return "IB_QPT_UC";
2158 case IB_QPT_UD:
2159 return "IB_QPT_UD";
2160 case IB_QPT_RAW_IPV6:
2161 return "IB_QPT_RAW_IPV6";
2162 case IB_QPT_RAW_ETHERTYPE:
2163 return "IB_QPT_RAW_ETHERTYPE";
2164 case IB_QPT_XRC_INI:
2165 return "IB_QPT_XRC_INI";
2166 case IB_QPT_XRC_TGT:
2167 return "IB_QPT_XRC_TGT";
2168 case IB_QPT_RAW_PACKET:
2169 return "IB_QPT_RAW_PACKET";
2170 case MLX5_IB_QPT_REG_UMR:
2171 return "MLX5_IB_QPT_REG_UMR";
b4aaa1f0
MS		 2172 case IB_QPT_DRIVER:
2173 return "IB_QPT_DRIVER";
e126ba97
EC		 2174 case IB_QPT_MAX:
2175 default:
2176 return "Invalid QP type";
2177 }
2178}
2179
b4aaa1f0
MS		 2180static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd,
2181 struct ib_qp_init_attr *attr,
2182 struct mlx5_ib_create_qp *ucmd)
2183{
b4aaa1f0
MS		 2184 struct mlx5_ib_qp *qp;
2185 int err = 0;
2186 u32 uidx = MLX5_IB_DEFAULT_UIDX;
2187 void *dctc;
2188
2189 if (!attr->srq || !attr->recv_cq)
2190 return ERR_PTR(-EINVAL);
2191
b4aaa1f0
MS		 2192 err = get_qp_user_index(to_mucontext(pd->uobject->context),
2193 ucmd, sizeof(*ucmd), &uidx);
2194 if (err)
2195 return ERR_PTR(err);
2196
2197 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2198 if (!qp)
2199 return ERR_PTR(-ENOMEM);
2200
2201 qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
2202 if (!qp->dct.in) {
2203 err = -ENOMEM;
2204 goto err_free;
2205 }
2206
2207 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
776a3906 2208 qp->qp_sub_type = MLX5_IB_QPT_DCT;
b4aaa1f0
MS		 2209 MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2210 MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2211 MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2212 MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2213 MLX5_SET(dctc, dctc, user_index, uidx);
2214
2215 qp->state = IB_QPS_RESET;
2216
2217 return &qp->ibqp;
2218err_free:
2219 kfree(qp);
2220 return ERR_PTR(err);
2221}
2222
2223static int set_mlx_qp_type(struct mlx5_ib_dev *dev,
2224 struct ib_qp_init_attr *init_attr,
2225 struct mlx5_ib_create_qp *ucmd,
2226 struct ib_udata *udata)
2227{
2228 enum { MLX_QP_FLAGS = MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI };
2229 int err;
2230
2231 if (!udata)
2232 return -EINVAL;
2233
2234 if (udata->inlen < sizeof(*ucmd)) {
2235 mlx5_ib_dbg(dev, "create_qp user command is smaller than expected\n");
2236 return -EINVAL;
2237 }
2238 err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));
2239 if (err)
2240 return err;
2241
2242 if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {
2243 init_attr->qp_type = MLX5_IB_QPT_DCI;
2244 } else {
2245 if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {
2246 init_attr->qp_type = MLX5_IB_QPT_DCT;
2247 } else {
2248 mlx5_ib_dbg(dev, "Invalid QP flags\n");
2249 return -EINVAL;
2250 }
2251 }
2252
2253 if (!MLX5_CAP_GEN(dev->mdev, dct)) {
2254 mlx5_ib_dbg(dev, "DC transport is not supported\n");
2255 return -EOPNOTSUPP;
2256 }
2257
2258 return 0;
2259}
2260
e126ba97 2261struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
b4aaa1f0 2262 struct ib_qp_init_attr *verbs_init_attr,
e126ba97
EC		 2263 struct ib_udata *udata)
2264{
2265 struct mlx5_ib_dev *dev;
2266 struct mlx5_ib_qp *qp;
2267 u16 xrcdn = 0;
2268 int err;
b4aaa1f0
MS		 2269 struct ib_qp_init_attr mlx_init_attr;
2270 struct ib_qp_init_attr *init_attr = verbs_init_attr;
e126ba97
EC		 2271
2272 if (pd) {
2273 dev = to_mdev(pd->device);
0fb2ed66 2274
2275 if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
2276 if (!pd->uobject) {
2277 mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");
2278 return ERR_PTR(-EINVAL);
2279 } else if (!to_mucontext(pd->uobject->context)->cqe_version) {
2280 mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n");
2281 return ERR_PTR(-EINVAL);
2282 }
2283 }
09f16cf5
MD		 2284 } else {
2285 /* being cautious here */
2286 if (init_attr->qp_type != IB_QPT_XRC_TGT &&
2287 init_attr->qp_type != MLX5_IB_QPT_REG_UMR) {
2288 pr_warn("%s: no PD for transport %s\n", __func__,
2289 ib_qp_type_str(init_attr->qp_type));
2290 return ERR_PTR(-EINVAL);
2291 }
2292 dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
e126ba97
EC		 2293 }
2294
b4aaa1f0
MS		 2295 if (init_attr->qp_type == IB_QPT_DRIVER) {
2296 struct mlx5_ib_create_qp ucmd;
2297
2298 init_attr = &mlx_init_attr;
2299 memcpy(init_attr, verbs_init_attr, sizeof(*verbs_init_attr));
2300 err = set_mlx_qp_type(dev, init_attr, &ucmd, udata);
2301 if (err)
2302 return ERR_PTR(err);
c32a4f29
MS		 2303
2304 if (init_attr->qp_type == MLX5_IB_QPT_DCI) {
2305 if (init_attr->cap.max_recv_wr ||
2306 init_attr->cap.max_recv_sge) {
2307 mlx5_ib_dbg(dev, "DCI QP requires zero size receive queue\n");
2308 return ERR_PTR(-EINVAL);
2309 }
776a3906
MS		 2310 } else {
2311 return mlx5_ib_create_dct(pd, init_attr, &ucmd);
c32a4f29 2312 }
b4aaa1f0
MS		 2313 }
2314
e126ba97
EC		 2315 switch (init_attr->qp_type) {
2316 case IB_QPT_XRC_TGT:
2317 case IB_QPT_XRC_INI:
938fe83c 2318 if (!MLX5_CAP_GEN(dev->mdev, xrc)) {
e126ba97
EC		 2319 mlx5_ib_dbg(dev, "XRC not supported\n");
2320 return ERR_PTR(-ENOSYS);
2321 }
2322 init_attr->recv_cq = NULL;
2323 if (init_attr->qp_type == IB_QPT_XRC_TGT) {
2324 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
2325 init_attr->send_cq = NULL;
2326 }
2327
2328 /* fall through */
0fb2ed66 2329 case IB_QPT_RAW_PACKET:
e126ba97
EC		 2330 case IB_QPT_RC:
2331 case IB_QPT_UC:
2332 case IB_QPT_UD:
2333 case IB_QPT_SMI:
d16e91da 2334 case MLX5_IB_QPT_HW_GSI:
e126ba97 2335 case MLX5_IB_QPT_REG_UMR:
c32a4f29 2336 case MLX5_IB_QPT_DCI:
e126ba97
EC		 2337 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2338 if (!qp)
2339 return ERR_PTR(-ENOMEM);
2340
2341 err = create_qp_common(dev, pd, init_attr, udata, qp);
2342 if (err) {
2343 mlx5_ib_dbg(dev, "create_qp_common failed\n");
2344 kfree(qp);
2345 return ERR_PTR(err);
2346 }
2347
2348 if (is_qp0(init_attr->qp_type))
2349 qp->ibqp.qp_num = 0;
2350 else if (is_qp1(init_attr->qp_type))
2351 qp->ibqp.qp_num = 1;
2352 else
19098df2 2353 qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
e126ba97
EC		 2354
2355 mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
19098df2 2356 qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
a1ab8402
EC		 2357 init_attr->recv_cq ? to_mcq(init_attr->recv_cq)->mcq.cqn : -1,
2358 init_attr->send_cq ? to_mcq(init_attr->send_cq)->mcq.cqn : -1);
e126ba97 2359
19098df2 2360 qp->trans_qp.xrcdn = xrcdn;
e126ba97
EC		 2361
2362 break;
2363
d16e91da
HE		 2364 case IB_QPT_GSI:
2365 return mlx5_ib_gsi_create_qp(pd, init_attr);
2366
e126ba97
EC		 2367 case IB_QPT_RAW_IPV6:
2368 case IB_QPT_RAW_ETHERTYPE:
e126ba97
EC		 2369 case IB_QPT_MAX:
2370 default:
2371 mlx5_ib_dbg(dev, "unsupported qp type %d\n",
2372 init_attr->qp_type);
2373 /* Don't support raw QPs */
2374 return ERR_PTR(-EINVAL);
2375 }
2376
b4aaa1f0
MS		 2377 if (verbs_init_attr->qp_type == IB_QPT_DRIVER)
2378 qp->qp_sub_type = init_attr->qp_type;
2379
e126ba97
EC		 2380 return &qp->ibqp;
2381}
2382
776a3906
MS		 2383static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
2384{
2385 struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
2386
2387 if (mqp->state == IB_QPS_RTR) {
2388 int err;
2389
2390 err = mlx5_core_destroy_dct(dev->mdev, &mqp->dct.mdct);
2391 if (err) {
2392 mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
2393 return err;
2394 }
2395 }
2396
2397 kfree(mqp->dct.in);
2398 kfree(mqp);
2399 return 0;
2400}
2401
e126ba97
EC		 2402int mlx5_ib_destroy_qp(struct ib_qp *qp)
2403{
2404 struct mlx5_ib_dev *dev = to_mdev(qp->device);
2405 struct mlx5_ib_qp *mqp = to_mqp(qp);
2406
d16e91da
HE		 2407 if (unlikely(qp->qp_type == IB_QPT_GSI))
2408 return mlx5_ib_gsi_destroy_qp(qp);
2409
776a3906
MS		 2410 if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
2411 return mlx5_ib_destroy_dct(mqp);
2412
e126ba97
EC		 2413 destroy_qp_common(dev, mqp);
2414
2415 kfree(mqp);
2416
2417 return 0;
2418}
2419
2420static __be32 to_mlx5_access_flags(struct mlx5_ib_qp *qp, const struct ib_qp_attr *attr,
2421 int attr_mask)
2422{
2423 u32 hw_access_flags = 0;
2424 u8 dest_rd_atomic;
2425 u32 access_flags;
2426
2427 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2428 dest_rd_atomic = attr->max_dest_rd_atomic;
2429 else
19098df2 2430 dest_rd_atomic = qp->trans_qp.resp_depth;
e126ba97
EC		 2431
2432 if (attr_mask & IB_QP_ACCESS_FLAGS)
2433 access_flags = attr->qp_access_flags;
2434 else
19098df2 2435 access_flags = qp->trans_qp.atomic_rd_en;
e126ba97
EC		 2436
2437 if (!dest_rd_atomic)
2438 access_flags &= IB_ACCESS_REMOTE_WRITE;
2439
2440 if (access_flags & IB_ACCESS_REMOTE_READ)
2441 hw_access_flags |= MLX5_QP_BIT_RRE;
2442 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
2443 hw_access_flags |= (MLX5_QP_BIT_RAE | MLX5_ATOMIC_MODE_CX);
2444 if (access_flags & IB_ACCESS_REMOTE_WRITE)
2445 hw_access_flags |= MLX5_QP_BIT_RWE;
2446
2447 return cpu_to_be32(hw_access_flags);
2448}
2449
2450enum {
2451 MLX5_PATH_FLAG_FL = 1 << 0,
2452 MLX5_PATH_FLAG_FREE_AR = 1 << 1,
2453 MLX5_PATH_FLAG_COUNTER = 1 << 2,
2454};
2455
2456static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
2457{
4f32ac2e 2458 if (rate == IB_RATE_PORT_CURRENT)
e126ba97 2459 return 0;
4f32ac2e
DG		 2460
2461 if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS)
e126ba97 2462 return -EINVAL;
e126ba97 2463
4f32ac2e
DG		 2464 while (rate != IB_RATE_PORT_CURRENT &&
2465 !(1 << (rate + MLX5_STAT_RATE_OFFSET) &
2466 MLX5_CAP_GEN(dev->mdev, stat_rate_support)))
2467 --rate;
2468
2469 return rate ? rate + MLX5_STAT_RATE_OFFSET : rate;
e126ba97
EC		 2470}
2471
75850d0b 2472static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
2473 struct mlx5_ib_sq *sq, u8 sl)
2474{
2475 void *in;
2476 void *tisc;
2477 int inlen;
2478 int err;
2479
2480 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 2481 in = kvzalloc(inlen, GFP_KERNEL);
75850d0b 2482 if (!in)
2483 return -ENOMEM;
2484
2485 MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
2486
2487 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2488 MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
2489
2490 err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
2491
2492 kvfree(in);
2493
2494 return err;
2495}
2496
13eab21f
AH		 2497static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
2498 struct mlx5_ib_sq *sq, u8 tx_affinity)
2499{
2500 void *in;
2501 void *tisc;
2502 int inlen;
2503 int err;
2504
2505 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 2506 in = kvzalloc(inlen, GFP_KERNEL);
13eab21f
AH		 2507 if (!in)
2508 return -ENOMEM;
2509
2510 MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
2511
2512 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2513 MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
2514
2515 err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
2516
2517 kvfree(in);
2518
2519 return err;
2520}
2521
75850d0b 2522static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
90898850 2523 const struct rdma_ah_attr *ah,
e126ba97 2524 struct mlx5_qp_path *path, u8 port, int attr_mask,
f879ee8d
AS		 2525 u32 path_flags, const struct ib_qp_attr *attr,
2526 bool alt)
e126ba97 2527{
d8966fcd 2528 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
e126ba97 2529 int err;
ed88451e 2530 enum ib_gid_type gid_type;
d8966fcd
DC		 2531 u8 ah_flags = rdma_ah_get_ah_flags(ah);
2532 u8 sl = rdma_ah_get_sl(ah);
e126ba97 2533
e126ba97 2534 if (attr_mask & IB_QP_PKEY_INDEX)
f879ee8d
AS		 2535 path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index :
2536 attr->pkey_index);
e126ba97 2537
d8966fcd
DC		 2538 if (ah_flags & IB_AH_GRH) {
2539 if (grh->sgid_index >=
938fe83c 2540 dev->mdev->port_caps[port - 1].gid_table_len) {
f4f01b54 2541 pr_err("sgid_index (%u) too large. max is %d\n",
d8966fcd 2542 grh->sgid_index,
938fe83c 2543 dev->mdev->port_caps[port - 1].gid_table_len);
f83b4263
EC		 2544 return -EINVAL;
2545 }
2811ba51 2546 }
44c58487
DC		 2547
2548 if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
d8966fcd 2549 if (!(ah_flags & IB_AH_GRH))
2811ba51 2550 return -EINVAL;
d8966fcd 2551 err = mlx5_get_roce_gid_type(dev, port, grh->sgid_index,
ed88451e
MD		 2552 &gid_type);
2553 if (err)
2554 return err;
44c58487 2555 memcpy(path->rmac, ah->roce.dmac, sizeof(ah->roce.dmac));
2b621851
MD		 2556 if (qp->ibqp.qp_type == IB_QPT_RC ||
2557 qp->ibqp.qp_type == IB_QPT_UC ||
2558 qp->ibqp.qp_type == IB_QPT_XRC_INI ||
2559 qp->ibqp.qp_type == IB_QPT_XRC_TGT)
2560 path->udp_sport = mlx5_get_roce_udp_sport(dev, port,
2561 grh->sgid_index);
d8966fcd 2562 path->dci_cfi_prio_sl = (sl & 0x7) << 4;
ed88451e 2563 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
d8966fcd 2564 path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f;
2811ba51 2565 } else {
d3ae2bde
NO		 2566 path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
2567 path->fl_free_ar |=
2568 (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;
d8966fcd
DC		 2569 path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
2570 path->grh_mlid = rdma_ah_get_path_bits(ah) & 0x7f;
2571 if (ah_flags & IB_AH_GRH)
2811ba51 2572 path->grh_mlid |= 1 << 7;
d8966fcd 2573 path->dci_cfi_prio_sl = sl & 0xf;
2811ba51
AS		 2574 }
2575
d8966fcd
DC		 2576 if (ah_flags & IB_AH_GRH) {
2577 path->mgid_index = grh->sgid_index;
2578 path->hop_limit = grh->hop_limit;
e126ba97 2579 path->tclass_flowlabel =
d8966fcd
DC		 2580 cpu_to_be32((grh->traffic_class << 20) |
2581 (grh->flow_label));
2582 memcpy(path->rgid, grh->dgid.raw, 16);
e126ba97
EC		 2583 }
2584
d8966fcd 2585 err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
e126ba97
EC		 2586 if (err < 0)
2587 return err;
2588 path->static_rate = err;
2589 path->port = port;
2590
e126ba97 2591 if (attr_mask & IB_QP_TIMEOUT)
f879ee8d 2592 path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3;
e126ba97 2593
75850d0b 2594 if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
2595 return modify_raw_packet_eth_prio(dev->mdev,
2596 &qp->raw_packet_qp.sq,
d8966fcd 2597 sl & 0xf);
75850d0b 2598
e126ba97
EC		 2599 return 0;
2600}
2601
2602static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
2603 [MLX5_QP_STATE_INIT] = {
2604 [MLX5_QP_STATE_INIT] = {
2605 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
2606 MLX5_QP_OPTPAR_RAE |
2607 MLX5_QP_OPTPAR_RWE |
2608 MLX5_QP_OPTPAR_PKEY_INDEX |
2609 MLX5_QP_OPTPAR_PRI_PORT,
2610 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
2611 MLX5_QP_OPTPAR_PKEY_INDEX |
2612 MLX5_QP_OPTPAR_PRI_PORT,
2613 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
2614 MLX5_QP_OPTPAR_Q_KEY |
2615 MLX5_QP_OPTPAR_PRI_PORT,
2616 },
2617 [MLX5_QP_STATE_RTR] = {
2618 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2619 MLX5_QP_OPTPAR_RRE |
2620 MLX5_QP_OPTPAR_RAE |
2621 MLX5_QP_OPTPAR_RWE |
2622 MLX5_QP_OPTPAR_PKEY_INDEX,
2623 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2624 MLX5_QP_OPTPAR_RWE |
2625 MLX5_QP_OPTPAR_PKEY_INDEX,
2626 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
2627 MLX5_QP_OPTPAR_Q_KEY,
2628 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX |
2629 MLX5_QP_OPTPAR_Q_KEY,
a4774e90
EC		 2630 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2631 MLX5_QP_OPTPAR_RRE |
2632 MLX5_QP_OPTPAR_RAE |
2633 MLX5_QP_OPTPAR_RWE |
2634 MLX5_QP_OPTPAR_PKEY_INDEX,
e126ba97
EC		 2635 },
2636 },
2637 [MLX5_QP_STATE_RTR] = {
2638 [MLX5_QP_STATE_RTS] = {
2639 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2640 MLX5_QP_OPTPAR_RRE |
2641 MLX5_QP_OPTPAR_RAE |
2642 MLX5_QP_OPTPAR_RWE |
2643 MLX5_QP_OPTPAR_PM_STATE |
2644 MLX5_QP_OPTPAR_RNR_TIMEOUT,
2645 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2646 MLX5_QP_OPTPAR_RWE |
2647 MLX5_QP_OPTPAR_PM_STATE,
2648 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
2649 },
2650 },
2651 [MLX5_QP_STATE_RTS] = {
2652 [MLX5_QP_STATE_RTS] = {
2653 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
2654 MLX5_QP_OPTPAR_RAE |
2655 MLX5_QP_OPTPAR_RWE |
2656 MLX5_QP_OPTPAR_RNR_TIMEOUT |
c2a3431e
EC		 2657 MLX5_QP_OPTPAR_PM_STATE |
2658 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97 2659 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
c2a3431e
EC		 2660 MLX5_QP_OPTPAR_PM_STATE |
2661 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97
EC		 2662 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY |
2663 MLX5_QP_OPTPAR_SRQN |
2664 MLX5_QP_OPTPAR_CQN_RCV,
2665 },
2666 },
2667 [MLX5_QP_STATE_SQER] = {
2668 [MLX5_QP_STATE_RTS] = {
2669 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
2670 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
75959f56 2671 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
a4774e90
EC		 2672 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
2673 MLX5_QP_OPTPAR_RWE |
2674 MLX5_QP_OPTPAR_RAE |
2675 MLX5_QP_OPTPAR_RRE,
e126ba97
EC		 2676 },
2677 },
2678};
2679
2680static int ib_nr_to_mlx5_nr(int ib_mask)
2681{
2682 switch (ib_mask) {
2683 case IB_QP_STATE:
2684 return 0;
2685 case IB_QP_CUR_STATE:
2686 return 0;
2687 case IB_QP_EN_SQD_ASYNC_NOTIFY:
2688 return 0;
2689 case IB_QP_ACCESS_FLAGS:
2690 return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
2691 MLX5_QP_OPTPAR_RAE;
2692 case IB_QP_PKEY_INDEX:
2693 return MLX5_QP_OPTPAR_PKEY_INDEX;
2694 case IB_QP_PORT:
2695 return MLX5_QP_OPTPAR_PRI_PORT;
2696 case IB_QP_QKEY:
2697 return MLX5_QP_OPTPAR_Q_KEY;
2698 case IB_QP_AV:
2699 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
2700 MLX5_QP_OPTPAR_PRI_PORT;
2701 case IB_QP_PATH_MTU:
2702 return 0;
2703 case IB_QP_TIMEOUT:
2704 return MLX5_QP_OPTPAR_ACK_TIMEOUT;
2705 case IB_QP_RETRY_CNT:
2706 return MLX5_QP_OPTPAR_RETRY_COUNT;
2707 case IB_QP_RNR_RETRY:
2708 return MLX5_QP_OPTPAR_RNR_RETRY;
2709 case IB_QP_RQ_PSN:
2710 return 0;
2711 case IB_QP_MAX_QP_RD_ATOMIC:
2712 return MLX5_QP_OPTPAR_SRA_MAX;
2713 case IB_QP_ALT_PATH:
2714 return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
2715 case IB_QP_MIN_RNR_TIMER:
2716 return MLX5_QP_OPTPAR_RNR_TIMEOUT;
2717 case IB_QP_SQ_PSN:
2718 return 0;
2719 case IB_QP_MAX_DEST_RD_ATOMIC:
2720 return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
2721 MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
2722 case IB_QP_PATH_MIG_STATE:
2723 return MLX5_QP_OPTPAR_PM_STATE;
2724 case IB_QP_CAP:
2725 return 0;
2726 case IB_QP_DEST_QPN:
2727 return 0;
2728 }
2729 return 0;
2730}
2731
2732static int ib_mask_to_mlx5_opt(int ib_mask)
2733{
2734 int result = 0;
2735 int i;
2736
2737 for (i = 0; i < 8 * sizeof(int); i++) {
2738 if ((1 << i) & ib_mask)
2739 result |= ib_nr_to_mlx5_nr(1 << i);
2740 }
2741
2742 return result;
2743}
2744
eb49ab0c
AV		 2745static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2746 struct mlx5_ib_rq *rq, int new_state,
2747 const struct mlx5_modify_raw_qp_param *raw_qp_param)
ad5f8e96 2748{
2749 void *in;
2750 void *rqc;
2751 int inlen;
2752 int err;
2753
2754 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 2755 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 2756 if (!in)
2757 return -ENOMEM;
2758
2759 MLX5_SET(modify_rq_in, in, rq_state, rq->state);
2760
2761 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
2762 MLX5_SET(rqc, rqc, state, new_state);
2763
eb49ab0c
AV		 2764 if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
2765 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
2766 MLX5_SET64(modify_rq_in, in, modify_bitmask,
23a6964e 2767 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
eb49ab0c
AV		 2768 MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
2769 } else
2770 pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n",
2771 dev->ib_dev.name);
2772 }
2773
2774 err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in, inlen);
ad5f8e96 2775 if (err)
2776 goto out;
2777
2778 rq->state = new_state;
2779
2780out:
2781 kvfree(in);
2782 return err;
2783}
2784
2785static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
7d29f349
BW		 2786 struct mlx5_ib_sq *sq,
2787 int new_state,
2788 const struct mlx5_modify_raw_qp_param *raw_qp_param)
ad5f8e96 2789{
7d29f349 2790 struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
61147f39
BW		 2791 struct mlx5_rate_limit old_rl = ibqp->rl;
2792 struct mlx5_rate_limit new_rl = old_rl;
2793 bool new_rate_added = false;
7d29f349 2794 u16 rl_index = 0;
ad5f8e96 2795 void *in;
2796 void *sqc;
2797 int inlen;
2798 int err;
2799
2800 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 2801 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 2802 if (!in)
2803 return -ENOMEM;
2804
2805 MLX5_SET(modify_sq_in, in, sq_state, sq->state);
2806
2807 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
2808 MLX5_SET(sqc, sqc, state, new_state);
2809
7d29f349
BW		 2810 if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
2811 if (new_state != MLX5_SQC_STATE_RDY)
2812 pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
2813 __func__);
2814 else
61147f39 2815 new_rl = raw_qp_param->rl;
7d29f349
BW		 2816 }
2817
61147f39
BW		 2818 if (!mlx5_rl_are_equal(&old_rl, &new_rl)) {
2819 if (new_rl.rate) {
2820 err = mlx5_rl_add_rate(dev, &rl_index, &new_rl);
7d29f349 2821 if (err) {
61147f39
BW		 2822 pr_err("Failed configuring rate limit(err %d): \
2823 rate %u, max_burst_sz %u, typical_pkt_sz %u\n",
2824 err, new_rl.rate, new_rl.max_burst_sz,
2825 new_rl.typical_pkt_sz);
2826
7d29f349
BW		 2827 goto out;
2828 }
61147f39 2829 new_rate_added = true;
7d29f349
BW		 2830 }
2831
2832 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
61147f39 2833 /* index 0 means no limit */
7d29f349
BW		 2834 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
2835 }
2836
ad5f8e96 2837 err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen);
7d29f349
BW		 2838 if (err) {
2839 /* Remove new rate from table if failed */
61147f39
BW		 2840 if (new_rate_added)
2841 mlx5_rl_remove_rate(dev, &new_rl);
ad5f8e96 2842 goto out;
7d29f349
BW		 2843 }
2844
2845 /* Only remove the old rate after new rate was set */
61147f39
BW		 2846 if ((old_rl.rate &&
2847 !mlx5_rl_are_equal(&old_rl, &new_rl)) ||
7d29f349 2848 (new_state != MLX5_SQC_STATE_RDY))
61147f39 2849 mlx5_rl_remove_rate(dev, &old_rl);
ad5f8e96 2850
61147f39 2851 ibqp->rl = new_rl;
ad5f8e96 2852 sq->state = new_state;
2853
2854out:
2855 kvfree(in);
2856 return err;
2857}
2858
2859static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 2860 const struct mlx5_modify_raw_qp_param *raw_qp_param,
2861 u8 tx_affinity)
ad5f8e96 2862{
2863 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
2864 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
2865 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
7d29f349
BW		 2866 int modify_rq = !!qp->rq.wqe_cnt;
2867 int modify_sq = !!qp->sq.wqe_cnt;
ad5f8e96 2868 int rq_state;
2869 int sq_state;
2870 int err;
2871
0680efa2 2872 switch (raw_qp_param->operation) {
ad5f8e96 2873 case MLX5_CMD_OP_RST2INIT_QP:
2874 rq_state = MLX5_RQC_STATE_RDY;
2875 sq_state = MLX5_SQC_STATE_RDY;
2876 break;
2877 case MLX5_CMD_OP_2ERR_QP:
2878 rq_state = MLX5_RQC_STATE_ERR;
2879 sq_state = MLX5_SQC_STATE_ERR;
2880 break;
2881 case MLX5_CMD_OP_2RST_QP:
2882 rq_state = MLX5_RQC_STATE_RST;
2883 sq_state = MLX5_SQC_STATE_RST;
2884 break;
ad5f8e96 2885 case MLX5_CMD_OP_RTR2RTS_QP:
2886 case MLX5_CMD_OP_RTS2RTS_QP:
7d29f349
BW		 2887 if (raw_qp_param->set_mask ==
2888 MLX5_RAW_QP_RATE_LIMIT) {
2889 modify_rq = 0;
2890 sq_state = sq->state;
2891 } else {
2892 return raw_qp_param->set_mask ? -EINVAL : 0;
2893 }
2894 break;
2895 case MLX5_CMD_OP_INIT2INIT_QP:
2896 case MLX5_CMD_OP_INIT2RTR_QP:
eb49ab0c
AV		 2897 if (raw_qp_param->set_mask)
2898 return -EINVAL;
2899 else
2900 return 0;
ad5f8e96 2901 default:
2902 WARN_ON(1);
2903 return -EINVAL;
2904 }
2905
7d29f349
BW		 2906 if (modify_rq) {
2907 err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
ad5f8e96 2908 if (err)
2909 return err;
2910 }
2911
7d29f349 2912 if (modify_sq) {
13eab21f
AH		 2913 if (tx_affinity) {
2914 err = modify_raw_packet_tx_affinity(dev->mdev, sq,
2915 tx_affinity);
2916 if (err)
2917 return err;
2918 }
2919
7d29f349 2920 return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state, raw_qp_param);
13eab21f 2921 }
ad5f8e96 2922
2923 return 0;
2924}
2925
e126ba97
EC		 2926static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
2927 const struct ib_qp_attr *attr, int attr_mask,
61147f39
BW		 2928 enum ib_qp_state cur_state, enum ib_qp_state new_state,
2929 const struct mlx5_ib_modify_qp *ucmd)
e126ba97 2930{
427c1e7b 2931 static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
2932 [MLX5_QP_STATE_RST] = {
2933 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2934 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2935 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_RST2INIT_QP,
2936 },
2937 [MLX5_QP_STATE_INIT] = {
2938 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2939 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2940 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_INIT2INIT_QP,
2941 [MLX5_QP_STATE_RTR] = MLX5_CMD_OP_INIT2RTR_QP,
2942 },
2943 [MLX5_QP_STATE_RTR] = {
2944 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2945 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2946 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTR2RTS_QP,
2947 },
2948 [MLX5_QP_STATE_RTS] = {
2949 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2950 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2951 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTS2RTS_QP,
2952 },
2953 [MLX5_QP_STATE_SQD] = {
2954 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2955 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2956 },
2957 [MLX5_QP_STATE_SQER] = {
2958 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2959 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2960 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQERR2RTS_QP,
2961 },
2962 [MLX5_QP_STATE_ERR] = {
2963 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2964 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2965 }
2966 };
2967
e126ba97
EC		 2968 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2969 struct mlx5_ib_qp *qp = to_mqp(ibqp);
19098df2 2970 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
e126ba97
EC		 2971 struct mlx5_ib_cq *send_cq, *recv_cq;
2972 struct mlx5_qp_context *context;
e126ba97 2973 struct mlx5_ib_pd *pd;
eb49ab0c 2974 struct mlx5_ib_port *mibport = NULL;
e126ba97
EC		 2975 enum mlx5_qp_state mlx5_cur, mlx5_new;
2976 enum mlx5_qp_optpar optpar;
e126ba97
EC		 2977 int mlx5_st;
2978 int err;
427c1e7b 2979 u16 op;
13eab21f 2980 u8 tx_affinity = 0;
e126ba97 2981
55de9a77
LR		 2982 mlx5_st = to_mlx5_st(ibqp->qp_type == IB_QPT_DRIVER ?
2983 qp->qp_sub_type : ibqp->qp_type);
2984 if (mlx5_st < 0)
2985 return -EINVAL;
2986
1a412fb1
SM		 2987 context = kzalloc(sizeof(*context), GFP_KERNEL);
2988 if (!context)
e126ba97
EC		 2989 return -ENOMEM;
2990
55de9a77 2991 context->flags = cpu_to_be32(mlx5_st << 16);
e126ba97
EC		 2992
2993 if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
2994 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
2995 } else {
2996 switch (attr->path_mig_state) {
2997 case IB_MIG_MIGRATED:
2998 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
2999 break;
3000 case IB_MIG_REARM:
3001 context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11);
3002 break;
3003 case IB_MIG_ARMED:
3004 context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11);
3005 break;
3006 }
3007 }
3008
13eab21f
AH		 3009 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
3010 if ((ibqp->qp_type == IB_QPT_RC) ||
3011 (ibqp->qp_type == IB_QPT_UD &&
3012 !(qp->flags & MLX5_IB_QP_SQPN_QP1)) ||
3013 (ibqp->qp_type == IB_QPT_UC) ||
3014 (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
3015 (ibqp->qp_type == IB_QPT_XRC_INI) ||
3016 (ibqp->qp_type == IB_QPT_XRC_TGT)) {
3017 if (mlx5_lag_is_active(dev->mdev)) {
7fd8aefb 3018 u8 p = mlx5_core_native_port_num(dev->mdev);
13eab21f 3019 tx_affinity = (unsigned int)atomic_add_return(1,
7fd8aefb 3020 &dev->roce[p].next_port) %
13eab21f
AH		 3021 MLX5_MAX_PORTS + 1;
3022 context->flags |= cpu_to_be32(tx_affinity << 24);
3023 }
3024 }
3025 }
3026
d16e91da 3027 if (is_sqp(ibqp->qp_type)) {
e126ba97 3028 context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
c2e53b2c
YH		 3029 } else if ((ibqp->qp_type == IB_QPT_UD &&
3030 !(qp->flags & MLX5_IB_QP_UNDERLAY)) ||
e126ba97
EC		 3031 ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
3032 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
3033 } else if (attr_mask & IB_QP_PATH_MTU) {
3034 if (attr->path_mtu < IB_MTU_256 ||
3035 attr->path_mtu > IB_MTU_4096) {
3036 mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
3037 err = -EINVAL;
3038 goto out;
3039 }
938fe83c
SM		 3040 context->mtu_msgmax = (attr->path_mtu << 5) |
3041 (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg);
e126ba97
EC		 3042 }
3043
3044 if (attr_mask & IB_QP_DEST_QPN)
3045 context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num);
3046
3047 if (attr_mask & IB_QP_PKEY_INDEX)
d3ae2bde 3048 context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index);
e126ba97
EC		 3049
3050 /* todo implement counter_index functionality */
3051
3052 if (is_sqp(ibqp->qp_type))
3053 context->pri_path.port = qp->port;
3054
3055 if (attr_mask & IB_QP_PORT)
3056 context->pri_path.port = attr->port_num;
3057
3058 if (attr_mask & IB_QP_AV) {
75850d0b 3059 err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
e126ba97 3060 attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
f879ee8d 3061 attr_mask, 0, attr, false);
e126ba97
EC		 3062 if (err)
3063 goto out;
3064 }
3065
3066 if (attr_mask & IB_QP_TIMEOUT)
3067 context->pri_path.ackto_lt |= attr->timeout << 3;
3068
3069 if (attr_mask & IB_QP_ALT_PATH) {
75850d0b 3070 err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
3071 &context->alt_path,
f879ee8d
AS		 3072 attr->alt_port_num,
3073 attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT,
3074 0, attr, true);
e126ba97
EC		 3075 if (err)
3076 goto out;
3077 }
3078
3079 pd = get_pd(qp);
89ea94a7
MG		 3080 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
3081 &send_cq, &recv_cq);
e126ba97
EC		 3082
3083 context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
3084 context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0;
3085 context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0;
3086 context->params1 = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28);
3087
3088 if (attr_mask & IB_QP_RNR_RETRY)
3089 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
3090
3091 if (attr_mask & IB_QP_RETRY_CNT)
3092 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
3093
3094 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3095 if (attr->max_rd_atomic)
3096 context->params1 |=
3097 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
3098 }
3099
3100 if (attr_mask & IB_QP_SQ_PSN)
3101 context->next_send_psn = cpu_to_be32(attr->sq_psn);
3102
3103 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3104 if (attr->max_dest_rd_atomic)
3105 context->params2 |=
3106 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
3107 }
3108
3109 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
3110 context->params2 |= to_mlx5_access_flags(qp, attr, attr_mask);
3111
3112 if (attr_mask & IB_QP_MIN_RNR_TIMER)
3113 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
3114
3115 if (attr_mask & IB_QP_RQ_PSN)
3116 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
3117
3118 if (attr_mask & IB_QP_QKEY)
3119 context->qkey = cpu_to_be32(attr->qkey);
3120
3121 if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3122 context->db_rec_addr = cpu_to_be64(qp->db.dma);
3123
0837e86a
MB		 3124 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3125 u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
3126 qp->port) - 1;
c2e53b2c
YH		 3127
3128 /* Underlay port should be used - index 0 function per port */
3129 if (qp->flags & MLX5_IB_QP_UNDERLAY)
3130 port_num = 0;
3131
eb49ab0c 3132 mibport = &dev->port[port_num];
0837e86a 3133 context->qp_counter_set_usr_page |=
e1f24a79 3134 cpu_to_be32((u32)(mibport->cnts.set_id) << 24);
0837e86a
MB		 3135 }
3136
e126ba97
EC		 3137 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3138 context->sq_crq_size |= cpu_to_be16(1 << 4);
3139
b11a4f9c
HE		 3140 if (qp->flags & MLX5_IB_QP_SQPN_QP1)
3141 context->deth_sqpn = cpu_to_be32(1);
e126ba97
EC		 3142
3143 mlx5_cur = to_mlx5_state(cur_state);
3144 mlx5_new = to_mlx5_state(new_state);
e126ba97 3145
427c1e7b 3146 if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
5d414b17
DC		 3147 !optab[mlx5_cur][mlx5_new]) {
3148 err = -EINVAL;
427c1e7b 3149 goto out;
5d414b17 3150 }
427c1e7b 3151
3152 op = optab[mlx5_cur][mlx5_new];
e126ba97
EC		 3153 optpar = ib_mask_to_mlx5_opt(attr_mask);
3154 optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
ad5f8e96 3155
c2e53b2c
YH		 3156 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
3157 qp->flags & MLX5_IB_QP_UNDERLAY) {
0680efa2
AV		 3158 struct mlx5_modify_raw_qp_param raw_qp_param = {};
3159
3160 raw_qp_param.operation = op;
eb49ab0c 3161 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
e1f24a79 3162 raw_qp_param.rq_q_ctr_id = mibport->cnts.set_id;
eb49ab0c
AV		 3163 raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
3164 }
7d29f349
BW		 3165
3166 if (attr_mask & IB_QP_RATE_LIMIT) {
61147f39
BW		 3167 raw_qp_param.rl.rate = attr->rate_limit;
3168
3169 if (ucmd->burst_info.max_burst_sz) {
3170 if (attr->rate_limit &&
3171 MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) {
3172 raw_qp_param.rl.max_burst_sz =
3173 ucmd->burst_info.max_burst_sz;
3174 } else {
3175 err = -EINVAL;
3176 goto out;
3177 }
3178 }
3179
3180 if (ucmd->burst_info.typical_pkt_sz) {
3181 if (attr->rate_limit &&
3182 MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) {
3183 raw_qp_param.rl.typical_pkt_sz =
3184 ucmd->burst_info.typical_pkt_sz;
3185 } else {
3186 err = -EINVAL;
3187 goto out;
3188 }
3189 }
3190
7d29f349
BW		 3191 raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
3192 }
3193
13eab21f 3194 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
0680efa2 3195 } else {
1a412fb1 3196 err = mlx5_core_qp_modify(dev->mdev, op, optpar, context,
ad5f8e96 3197 &base->mqp);
0680efa2
AV		 3198 }
3199
e126ba97
EC		 3200 if (err)
3201 goto out;
3202
3203 qp->state = new_state;
3204
3205 if (attr_mask & IB_QP_ACCESS_FLAGS)
19098df2 3206 qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
e126ba97 3207 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
19098df2 3208 qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
e126ba97
EC		 3209 if (attr_mask & IB_QP_PORT)
3210 qp->port = attr->port_num;
3211 if (attr_mask & IB_QP_ALT_PATH)
19098df2 3212 qp->trans_qp.alt_port = attr->alt_port_num;
e126ba97
EC		 3213
3214 /*
3215 * If we moved a kernel QP to RESET, clean up all old CQ
3216 * entries and reinitialize the QP.
3217 */
75a45982
LR		 3218 if (new_state == IB_QPS_RESET &&
3219 !ibqp->uobject && ibqp->qp_type != IB_QPT_XRC_TGT) {
19098df2 3220 mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 3221 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
3222 if (send_cq != recv_cq)
19098df2 3223 mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
e126ba97
EC		 3224
3225 qp->rq.head = 0;
3226 qp->rq.tail = 0;
3227 qp->sq.head = 0;
3228 qp->sq.tail = 0;
3229 qp->sq.cur_post = 0;
3230 qp->sq.last_poll = 0;
3231 qp->db.db[MLX5_RCV_DBR] = 0;
3232 qp->db.db[MLX5_SND_DBR] = 0;
3233 }
3234
3235out:
1a412fb1 3236 kfree(context);
e126ba97
EC		 3237 return err;
3238}
3239
c32a4f29
MS		 3240static inline bool is_valid_mask(int mask, int req, int opt)
3241{
3242 if ((mask & req) != req)
3243 return false;
3244
3245 if (mask & ~(req | opt))
3246 return false;
3247
3248 return true;
3249}
3250
3251/* check valid transition for driver QP types
3252 * for now the only QP type that this function supports is DCI
3253 */
3254static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
3255 enum ib_qp_attr_mask attr_mask)
3256{
3257 int req = IB_QP_STATE;
3258 int opt = 0;
3259
3260 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3261 req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
3262 return is_valid_mask(attr_mask, req, opt);
3263 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3264 opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
3265 return is_valid_mask(attr_mask, req, opt);
3266 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3267 req |= IB_QP_PATH_MTU;
3268 opt = IB_QP_PKEY_INDEX;
3269 return is_valid_mask(attr_mask, req, opt);
3270 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3271 req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
3272 IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
3273 opt = IB_QP_MIN_RNR_TIMER;
3274 return is_valid_mask(attr_mask, req, opt);
3275 } else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
3276 opt = IB_QP_MIN_RNR_TIMER;
3277 return is_valid_mask(attr_mask, req, opt);
3278 } else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
3279 return is_valid_mask(attr_mask, req, opt);
3280 }
3281 return false;
3282}
3283
776a3906
MS		 3284/* mlx5_ib_modify_dct: modify a DCT QP
3285 * valid transitions are:
3286 * RESET to INIT: must set access_flags, pkey_index and port
3287 * INIT to RTR : must set min_rnr_timer, tclass, flow_label,
3288 * mtu, gid_index and hop_limit
3289 * Other transitions and attributes are illegal
3290 */
3291static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3292 int attr_mask, struct ib_udata *udata)
3293{
3294 struct mlx5_ib_qp *qp = to_mqp(ibqp);
3295 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3296 enum ib_qp_state cur_state, new_state;
3297 int err = 0;
3298 int required = IB_QP_STATE;
3299 void *dctc;
3300
3301 if (!(attr_mask & IB_QP_STATE))
3302 return -EINVAL;
3303
3304 cur_state = qp->state;
3305 new_state = attr->qp_state;
3306
3307 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
3308 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3309 required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
3310 if (!is_valid_mask(attr_mask, required, 0))
3311 return -EINVAL;
3312
3313 if (attr->port_num == 0 ||
3314 attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports)) {
3315 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
3316 attr->port_num, dev->num_ports);
3317 return -EINVAL;
3318 }
3319 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
3320 MLX5_SET(dctc, dctc, rre, 1);
3321 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
3322 MLX5_SET(dctc, dctc, rwe, 1);
3323 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
3324 if (!mlx5_ib_dc_atomic_is_supported(dev))
3325 return -EOPNOTSUPP;
3326 MLX5_SET(dctc, dctc, rae, 1);
3327 MLX5_SET(dctc, dctc, atomic_mode, MLX5_ATOMIC_MODE_DCT_CX);
3328 }
3329 MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
3330 MLX5_SET(dctc, dctc, port, attr->port_num);
3331 MLX5_SET(dctc, dctc, counter_set_id, dev->port[attr->port_num - 1].cnts.set_id);
3332
3333 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3334 struct mlx5_ib_modify_qp_resp resp = {};
3335 u32 min_resp_len = offsetof(typeof(resp), dctn) +
3336 sizeof(resp.dctn);
3337
3338 if (udata->outlen < min_resp_len)
3339 return -EINVAL;
3340 resp.response_length = min_resp_len;
3341
3342 required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
3343 if (!is_valid_mask(attr_mask, required, 0))
3344 return -EINVAL;
3345 MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
3346 MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
3347 MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
3348 MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
3349 MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
3350 MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
3351
3352 err = mlx5_core_create_dct(dev->mdev, &qp->dct.mdct, qp->dct.in,
3353 MLX5_ST_SZ_BYTES(create_dct_in));
3354 if (err)
3355 return err;
3356 resp.dctn = qp->dct.mdct.mqp.qpn;
3357 err = ib_copy_to_udata(udata, &resp, resp.response_length);
3358 if (err) {
3359 mlx5_core_destroy_dct(dev->mdev, &qp->dct.mdct);
3360 return err;
3361 }
3362 } else {
3363 mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
3364 return -EINVAL;
3365 }
3366 if (err)
3367 qp->state = IB_QPS_ERR;
3368 else
3369 qp->state = new_state;
3370 return err;
3371}
3372
e126ba97
EC		 3373int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3374 int attr_mask, struct ib_udata *udata)
3375{
3376 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3377 struct mlx5_ib_qp *qp = to_mqp(ibqp);
61147f39 3378 struct mlx5_ib_modify_qp ucmd = {};
d16e91da 3379 enum ib_qp_type qp_type;
e126ba97 3380 enum ib_qp_state cur_state, new_state;
61147f39 3381 size_t required_cmd_sz;
e126ba97
EC		 3382 int err = -EINVAL;
3383 int port;
2811ba51 3384 enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
e126ba97 3385
28d61370
YH		 3386 if (ibqp->rwq_ind_tbl)
3387 return -ENOSYS;
3388
61147f39
BW		 3389 if (udata && udata->inlen) {
3390 required_cmd_sz = offsetof(typeof(ucmd), reserved) +
3391 sizeof(ucmd.reserved);
3392 if (udata->inlen < required_cmd_sz)
3393 return -EINVAL;
3394
3395 if (udata->inlen > sizeof(ucmd) &&
3396 !ib_is_udata_cleared(udata, sizeof(ucmd),
3397 udata->inlen - sizeof(ucmd)))
3398 return -EOPNOTSUPP;
3399
3400 if (ib_copy_from_udata(&ucmd, udata,
3401 min(udata->inlen, sizeof(ucmd))))
3402 return -EFAULT;
3403
3404 if (ucmd.comp_mask ||
3405 memchr_inv(&ucmd.reserved, 0, sizeof(ucmd.reserved)) ||
3406 memchr_inv(&ucmd.burst_info.reserved, 0,
3407 sizeof(ucmd.burst_info.reserved)))
3408 return -EOPNOTSUPP;
3409 }
3410
d16e91da
HE		 3411 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
3412 return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
3413
c32a4f29
MS		 3414 if (ibqp->qp_type == IB_QPT_DRIVER)
3415 qp_type = qp->qp_sub_type;
3416 else
3417 qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
3418 IB_QPT_GSI : ibqp->qp_type;
3419
776a3906
MS		 3420 if (qp_type == MLX5_IB_QPT_DCT)
3421 return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);
d16e91da 3422
e126ba97
EC		 3423 mutex_lock(&qp->mutex);
3424
3425 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
3426 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
3427
2811ba51
AS		 3428 if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
3429 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
3430 ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port);
3431 }
3432
c2e53b2c
YH		 3433 if (qp->flags & MLX5_IB_QP_UNDERLAY) {
3434 if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
3435 mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
3436 attr_mask);
3437 goto out;
3438 }
3439 } else if (qp_type != MLX5_IB_QPT_REG_UMR &&
c32a4f29
MS		 3440 qp_type != MLX5_IB_QPT_DCI &&
3441 !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
158abf86
HE		 3442 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
3443 cur_state, new_state, ibqp->qp_type, attr_mask);
e126ba97 3444 goto out;
c32a4f29
MS		 3445 } else if (qp_type == MLX5_IB_QPT_DCI &&
3446 !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
3447 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
3448 cur_state, new_state, qp_type, attr_mask);
3449 goto out;
158abf86 3450 }
e126ba97
EC		 3451
3452 if ((attr_mask & IB_QP_PORT) &&
938fe83c 3453 (attr->port_num == 0 ||
508562d6 3454 attr->port_num > dev->num_ports)) {
158abf86
HE		 3455 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
3456 attr->port_num, dev->num_ports);
e126ba97 3457 goto out;
158abf86 3458 }
e126ba97
EC		 3459
3460 if (attr_mask & IB_QP_PKEY_INDEX) {
3461 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
938fe83c 3462 if (attr->pkey_index >=
158abf86
HE		 3463 dev->mdev->port_caps[port - 1].pkey_table_len) {
3464 mlx5_ib_dbg(dev, "invalid pkey index %d\n",
3465 attr->pkey_index);
e126ba97 3466 goto out;
158abf86 3467 }
e126ba97
EC		 3468 }
3469
3470 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
938fe83c 3471 attr->max_rd_atomic >
158abf86
HE		 3472 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) {
3473 mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
3474 attr->max_rd_atomic);
e126ba97 3475 goto out;
158abf86 3476 }
e126ba97
EC		 3477
3478 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
938fe83c 3479 attr->max_dest_rd_atomic >
158abf86
HE		 3480 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) {
3481 mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
3482 attr->max_dest_rd_atomic);
e126ba97 3483 goto out;
158abf86 3484 }
e126ba97
EC		 3485
3486 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
3487 err = 0;
3488 goto out;
3489 }
3490
61147f39
BW		 3491 err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
3492 new_state, &ucmd);
e126ba97
EC		 3493
3494out:
3495 mutex_unlock(&qp->mutex);
3496 return err;
3497}
3498
3499static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
3500{
3501 struct mlx5_ib_cq *cq;
3502 unsigned cur;
3503
3504 cur = wq->head - wq->tail;
3505 if (likely(cur + nreq < wq->max_post))
3506 return 0;
3507
3508 cq = to_mcq(ib_cq);
3509 spin_lock(&cq->lock);
3510 cur = wq->head - wq->tail;
3511 spin_unlock(&cq->lock);
3512
3513 return cur + nreq >= wq->max_post;
3514}
3515
3516static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
3517 u64 remote_addr, u32 rkey)
3518{
3519 rseg->raddr = cpu_to_be64(remote_addr);
3520 rseg->rkey = cpu_to_be32(rkey);
3521 rseg->reserved = 0;
3522}
3523
f0313965
ES		 3524static void *set_eth_seg(struct mlx5_wqe_eth_seg *eseg,
3525 struct ib_send_wr *wr, void *qend,
3526 struct mlx5_ib_qp *qp, int *size)
3527{
3528 void *seg = eseg;
3529
3530 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
3531
3532 if (wr->send_flags & IB_SEND_IP_CSUM)
3533 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
3534 MLX5_ETH_WQE_L4_CSUM;
3535
3536 seg += sizeof(struct mlx5_wqe_eth_seg);
3537 *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
3538
3539 if (wr->opcode == IB_WR_LSO) {
3540 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
2b31f7ae 3541 int size_of_inl_hdr_start = sizeof(eseg->inline_hdr.start);
f0313965
ES		 3542 u64 left, leftlen, copysz;
3543 void *pdata = ud_wr->header;
3544
3545 left = ud_wr->hlen;
3546 eseg->mss = cpu_to_be16(ud_wr->mss);
2b31f7ae 3547 eseg->inline_hdr.sz = cpu_to_be16(left);
f0313965
ES		 3548
3549 /*
3550 * check if there is space till the end of queue, if yes,
3551 * copy all in one shot, otherwise copy till the end of queue,
3552 * rollback and than the copy the left
3553 */
2b31f7ae 3554 leftlen = qend - (void *)eseg->inline_hdr.start;
f0313965
ES		 3555 copysz = min_t(u64, leftlen, left);
3556
3557 memcpy(seg - size_of_inl_hdr_start, pdata, copysz);
3558
3559 if (likely(copysz > size_of_inl_hdr_start)) {
3560 seg += ALIGN(copysz - size_of_inl_hdr_start, 16);
3561 *size += ALIGN(copysz - size_of_inl_hdr_start, 16) / 16;
3562 }
3563
3564 if (unlikely(copysz < left)) { /* the last wqe in the queue */
3565 seg = mlx5_get_send_wqe(qp, 0);
3566 left -= copysz;
3567 pdata += copysz;
3568 memcpy(seg, pdata, left);
3569 seg += ALIGN(left, 16);
3570 *size += ALIGN(left, 16) / 16;
3571 }
3572 }
3573
3574 return seg;
3575}
3576
e126ba97
EC		 3577static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
3578 struct ib_send_wr *wr)
3579{
e622f2f4
CH		 3580 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
3581 dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
3582 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
e126ba97
EC		 3583}
3584
3585static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
3586{
3587 dseg->byte_count = cpu_to_be32(sg->length);
3588 dseg->lkey = cpu_to_be32(sg->lkey);
3589 dseg->addr = cpu_to_be64(sg->addr);
3590}
3591
31616255 3592static u64 get_xlt_octo(u64 bytes)
e126ba97 3593{
31616255
AK		 3594 return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
3595 MLX5_IB_UMR_OCTOWORD;
e126ba97
EC		 3596}
3597
3598static __be64 frwr_mkey_mask(void)
3599{
3600 u64 result;
3601
3602 result = MLX5_MKEY_MASK_LEN |
3603 MLX5_MKEY_MASK_PAGE_SIZE |
3604 MLX5_MKEY_MASK_START_ADDR |
3605 MLX5_MKEY_MASK_EN_RINVAL |
3606 MLX5_MKEY_MASK_KEY |
3607 MLX5_MKEY_MASK_LR |
3608 MLX5_MKEY_MASK_LW |
3609 MLX5_MKEY_MASK_RR |
3610 MLX5_MKEY_MASK_RW |
3611 MLX5_MKEY_MASK_A |
3612 MLX5_MKEY_MASK_SMALL_FENCE |
3613 MLX5_MKEY_MASK_FREE;
3614
3615 return cpu_to_be64(result);
3616}
3617
e6631814
SG		 3618static __be64 sig_mkey_mask(void)
3619{
3620 u64 result;
3621
3622 result = MLX5_MKEY_MASK_LEN |
3623 MLX5_MKEY_MASK_PAGE_SIZE |
3624 MLX5_MKEY_MASK_START_ADDR |
d5436ba0 3625 MLX5_MKEY_MASK_EN_SIGERR |
e6631814
SG		 3626 MLX5_MKEY_MASK_EN_RINVAL |
3627 MLX5_MKEY_MASK_KEY |
3628 MLX5_MKEY_MASK_LR |
3629 MLX5_MKEY_MASK_LW |
3630 MLX5_MKEY_MASK_RR |
3631 MLX5_MKEY_MASK_RW |
3632 MLX5_MKEY_MASK_SMALL_FENCE |
3633 MLX5_MKEY_MASK_FREE |
3634 MLX5_MKEY_MASK_BSF_EN;
3635
3636 return cpu_to_be64(result);
3637}
3638
8a187ee5 3639static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
31616255 3640 struct mlx5_ib_mr *mr)
8a187ee5 3641{
31616255 3642 int size = mr->ndescs * mr->desc_size;
8a187ee5
SG		 3643
3644 memset(umr, 0, sizeof(*umr));
b005d316 3645
8a187ee5 3646 umr->flags = MLX5_UMR_CHECK_NOT_FREE;
31616255 3647 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
8a187ee5
SG		 3648 umr->mkey_mask = frwr_mkey_mask();
3649}
3650
dd01e66a 3651static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
e126ba97
EC		 3652{
3653 memset(umr, 0, sizeof(*umr));
dd01e66a 3654 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
2d221588 3655 umr->flags = MLX5_UMR_INLINE;
e126ba97
EC		 3656}
3657
31616255 3658static __be64 get_umr_enable_mr_mask(void)
968e78dd
HE		 3659{
3660 u64 result;
3661
31616255 3662 result = MLX5_MKEY_MASK_KEY |
968e78dd
HE		 3663 MLX5_MKEY_MASK_FREE;
3664
968e78dd
HE		 3665 return cpu_to_be64(result);
3666}
3667
31616255 3668static __be64 get_umr_disable_mr_mask(void)
968e78dd
HE		 3669{
3670 u64 result;
3671
3672 result = MLX5_MKEY_MASK_FREE;
3673
3674 return cpu_to_be64(result);
3675}
3676
56e11d62
NO		 3677static __be64 get_umr_update_translation_mask(void)
3678{
3679 u64 result;
3680
3681 result = MLX5_MKEY_MASK_LEN |
3682 MLX5_MKEY_MASK_PAGE_SIZE |
31616255 3683 MLX5_MKEY_MASK_START_ADDR;
56e11d62
NO		 3684
3685 return cpu_to_be64(result);
3686}
3687
31616255 3688static __be64 get_umr_update_access_mask(int atomic)
56e11d62
NO		 3689{
3690 u64 result;
3691
31616255
AK		 3692 result = MLX5_MKEY_MASK_LR |
3693 MLX5_MKEY_MASK_LW |
56e11d62 3694 MLX5_MKEY_MASK_RR |
31616255
AK		 3695 MLX5_MKEY_MASK_RW;
3696
3697 if (atomic)
3698 result |= MLX5_MKEY_MASK_A;
56e11d62
NO		 3699
3700 return cpu_to_be64(result);
3701}
3702
3703static __be64 get_umr_update_pd_mask(void)
3704{
3705 u64 result;
3706
31616255 3707 result = MLX5_MKEY_MASK_PD;
56e11d62
NO		 3708
3709 return cpu_to_be64(result);
3710}
3711
c8d75a98
MD		 3712static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
3713{
3714 if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
3715 MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
3716 (mask & MLX5_MKEY_MASK_A &&
3717 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
3718 return -EPERM;
3719 return 0;
3720}
3721
3722static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
3723 struct mlx5_wqe_umr_ctrl_seg *umr,
3724 struct ib_send_wr *wr, int atomic)
e126ba97 3725{
e622f2f4 3726 struct mlx5_umr_wr *umrwr = umr_wr(wr);
e126ba97
EC		 3727
3728 memset(umr, 0, sizeof(*umr));
3729
968e78dd
HE		 3730 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
3731 umr->flags = MLX5_UMR_CHECK_FREE; /* fail if free */
3732 else
3733 umr->flags = MLX5_UMR_CHECK_NOT_FREE; /* fail if not free */
3734
31616255
AK		 3735 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
3736 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
3737 u64 offset = get_xlt_octo(umrwr->offset);
3738
3739 umr->xlt_offset = cpu_to_be16(offset & 0xffff);
3740 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
3741 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
e126ba97 3742 }
31616255
AK		 3743 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
3744 umr->mkey_mask |= get_umr_update_translation_mask();
3745 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
3746 umr->mkey_mask |= get_umr_update_access_mask(atomic);
3747 umr->mkey_mask |= get_umr_update_pd_mask();
3748 }
3749 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
3750 umr->mkey_mask |= get_umr_enable_mr_mask();
3751 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
3752 umr->mkey_mask |= get_umr_disable_mr_mask();
e126ba97
EC		 3753
3754 if (!wr->num_sge)
968e78dd 3755 umr->flags |= MLX5_UMR_INLINE;
c8d75a98
MD		 3756
3757 return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
e126ba97
EC		 3758}
3759
3760static u8 get_umr_flags(int acc)
3761{
3762 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
3763 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
3764 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
3765 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
2ac45934 3766 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
e126ba97
EC		 3767}
3768
8a187ee5
SG		 3769static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
3770 struct mlx5_ib_mr *mr,
3771 u32 key, int access)
3772{
3773 int ndescs = ALIGN(mr->ndescs, 8) >> 1;
3774
3775 memset(seg, 0, sizeof(*seg));
b005d316 3776
ec22eb53 3777 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
b005d316 3778 seg->log2_page_size = ilog2(mr->ibmr.page_size);
ec22eb53 3779 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
b005d316
SG		 3780 /* KLMs take twice the size of MTTs */
3781 ndescs *= 2;
3782
3783 seg->flags = get_umr_flags(access) | mr->access_mode;
8a187ee5
SG		 3784 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
3785 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
3786 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
3787 seg->len = cpu_to_be64(mr->ibmr.length);
3788 seg->xlt_oct_size = cpu_to_be32(ndescs);
8a187ee5
SG		 3789}
3790
dd01e66a 3791static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
e126ba97
EC		 3792{
3793 memset(seg, 0, sizeof(*seg));
dd01e66a 3794 seg->status = MLX5_MKEY_STATUS_FREE;
e126ba97
EC		 3795}
3796
3797static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr)
3798{
e622f2f4 3799 struct mlx5_umr_wr *umrwr = umr_wr(wr);
968e78dd 3800
e126ba97 3801 memset(seg, 0, sizeof(*seg));
31616255 3802 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
968e78dd 3803 seg->status = MLX5_MKEY_STATUS_FREE;
e126ba97 3804
968e78dd 3805 seg->flags = convert_access(umrwr->access_flags);
31616255
AK		 3806 if (umrwr->pd)
3807 seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
3808 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
3809 !umrwr->length)
3810 seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
3811
3812 seg->start_addr = cpu_to_be64(umrwr->virt_addr);
968e78dd
HE		 3813 seg->len = cpu_to_be64(umrwr->length);
3814 seg->log2_page_size = umrwr->page_shift;
746b5583 3815 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
968e78dd 3816 mlx5_mkey_variant(umrwr->mkey));
e126ba97
EC		 3817}
3818
8a187ee5
SG		 3819static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
3820 struct mlx5_ib_mr *mr,
3821 struct mlx5_ib_pd *pd)
3822{
3823 int bcount = mr->desc_size * mr->ndescs;
3824
3825 dseg->addr = cpu_to_be64(mr->desc_map);
3826 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
3827 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
3828}
3829
e126ba97
EC		 3830static __be32 send_ieth(struct ib_send_wr *wr)
3831{
3832 switch (wr->opcode) {
3833 case IB_WR_SEND_WITH_IMM:
3834 case IB_WR_RDMA_WRITE_WITH_IMM:
3835 return wr->ex.imm_data;
3836
3837 case IB_WR_SEND_WITH_INV:
3838 return cpu_to_be32(wr->ex.invalidate_rkey);
3839
3840 default:
3841 return 0;
3842 }
3843}
3844
3845static u8 calc_sig(void *wqe, int size)
3846{
3847 u8 *p = wqe;
3848 u8 res = 0;
3849 int i;
3850
3851 for (i = 0; i < size; i++)
3852 res ^= p[i];
3853
3854 return ~res;
3855}
3856
3857static u8 wq_sig(void *wqe)
3858{
3859 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
3860}
3861
3862static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr,
3863 void *wqe, int *sz)
3864{
3865 struct mlx5_wqe_inline_seg *seg;
3866 void *qend = qp->sq.qend;
3867 void *addr;
3868 int inl = 0;
3869 int copy;
3870 int len;
3871 int i;
3872
3873 seg = wqe;
3874 wqe += sizeof(*seg);
3875 for (i = 0; i < wr->num_sge; i++) {
3876 addr = (void *)(unsigned long)(wr->sg_list[i].addr);
3877 len = wr->sg_list[i].length;
3878 inl += len;
3879
3880 if (unlikely(inl > qp->max_inline_data))
3881 return -ENOMEM;
3882
3883 if (unlikely(wqe + len > qend)) {
3884 copy = qend - wqe;
3885 memcpy(wqe, addr, copy);
3886 addr += copy;
3887 len -= copy;
3888 wqe = mlx5_get_send_wqe(qp, 0);
3889 }
3890 memcpy(wqe, addr, len);
3891 wqe += len;
3892 }
3893
3894 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
3895
3896 *sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
3897
3898 return 0;
3899}
3900
e6631814
SG		 3901static u16 prot_field_size(enum ib_signature_type type)
3902{
3903 switch (type) {
3904 case IB_SIG_TYPE_T10_DIF:
3905 return MLX5_DIF_SIZE;
3906 default:
3907 return 0;
3908 }
3909}
3910
3911static u8 bs_selector(int block_size)
3912{
3913 switch (block_size) {
3914 case 512: return 0x1;
3915 case 520: return 0x2;
3916 case 4096: return 0x3;
3917 case 4160: return 0x4;
3918 case 1073741824: return 0x5;
3919 default: return 0;
3920 }
3921}
3922
78eda2bb
SG		 3923static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
3924 struct mlx5_bsf_inl *inl)
e6631814 3925{
142537f4
SG		 3926 /* Valid inline section and allow BSF refresh */
3927 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
3928 MLX5_BSF_REFRESH_DIF);
3929 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
3930 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
78eda2bb
SG		 3931 /* repeating block */
3932 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
3933 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
3934 MLX5_DIF_CRC : MLX5_DIF_IPCS;
e6631814 3935
78eda2bb
SG		 3936 if (domain->sig.dif.ref_remap)
3937 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
e6631814 3938
78eda2bb
SG		 3939 if (domain->sig.dif.app_escape) {
3940 if (domain->sig.dif.ref_escape)
3941 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
3942 else
3943 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
e6631814
SG		 3944 }
3945
78eda2bb
SG		 3946 inl->dif_app_bitmask_check =
3947 cpu_to_be16(domain->sig.dif.apptag_check_mask);
e6631814
SG		 3948}
3949
3950static int mlx5_set_bsf(struct ib_mr *sig_mr,
3951 struct ib_sig_attrs *sig_attrs,
3952 struct mlx5_bsf *bsf, u32 data_size)
3953{
3954 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
3955 struct mlx5_bsf_basic *basic = &bsf->basic;
3956 struct ib_sig_domain *mem = &sig_attrs->mem;
3957 struct ib_sig_domain *wire = &sig_attrs->wire;
e6631814 3958
c7f44fbd 3959 memset(bsf, 0, sizeof(*bsf));
78eda2bb
SG		 3960
3961 /* Basic + Extended + Inline */
3962 basic->bsf_size_sbs = 1 << 7;
3963 /* Input domain check byte mask */
3964 basic->check_byte_mask = sig_attrs->check_mask;
3965 basic->raw_data_size = cpu_to_be32(data_size);
3966
3967 /* Memory domain */
e6631814 3968 switch (sig_attrs->mem.sig_type) {
78eda2bb
SG		 3969 case IB_SIG_TYPE_NONE:
3970 break;
e6631814 3971 case IB_SIG_TYPE_T10_DIF:
78eda2bb
SG		 3972 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
3973 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
3974 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
3975 break;
3976 default:
3977 return -EINVAL;
3978 }
e6631814 3979
78eda2bb
SG		 3980 /* Wire domain */
3981 switch (sig_attrs->wire.sig_type) {
3982 case IB_SIG_TYPE_NONE:
3983 break;
3984 case IB_SIG_TYPE_T10_DIF:
e6631814 3985 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
78eda2bb 3986 mem->sig_type == wire->sig_type) {
e6631814 3987 /* Same block structure */
142537f4 3988 basic->bsf_size_sbs |= 1 << 4;
e6631814 3989 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
fd22f78c 3990 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
c7f44fbd 3991 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
fd22f78c 3992 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
c7f44fbd 3993 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
fd22f78c 3994 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
e6631814
SG		 3995 } else
3996 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
3997
142537f4 3998 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
78eda2bb 3999 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
e6631814 4000 break;
e6631814
SG		 4001 default:
4002 return -EINVAL;
4003 }
4004
4005 return 0;
4006}
4007
e622f2f4
CH		 4008static int set_sig_data_segment(struct ib_sig_handover_wr *wr,
4009 struct mlx5_ib_qp *qp, void **seg, int *size)
e6631814 4010{
e622f2f4
CH		 4011 struct ib_sig_attrs *sig_attrs = wr->sig_attrs;
4012 struct ib_mr *sig_mr = wr->sig_mr;
e6631814 4013 struct mlx5_bsf *bsf;
e622f2f4
CH		 4014 u32 data_len = wr->wr.sg_list->length;
4015 u32 data_key = wr->wr.sg_list->lkey;
4016 u64 data_va = wr->wr.sg_list->addr;
e6631814
SG		 4017 int ret;
4018 int wqe_size;
4019
e622f2f4
CH		 4020 if (!wr->prot ||
4021 (data_key == wr->prot->lkey &&
4022 data_va == wr->prot->addr &&
4023 data_len == wr->prot->length)) {
e6631814
SG		 4024 /**
4025 * Source domain doesn't contain signature information
5c273b16 4026 * or data and protection are interleaved in memory.
e6631814
SG		 4027 * So need construct:
4028 * ------------------
4029 * | data_klm |
4030 * ------------------
4031 * | BSF |
4032 * ------------------
4033 **/
4034 struct mlx5_klm *data_klm = *seg;
4035
4036 data_klm->bcount = cpu_to_be32(data_len);
4037 data_klm->key = cpu_to_be32(data_key);
4038 data_klm->va = cpu_to_be64(data_va);
4039 wqe_size = ALIGN(sizeof(*data_klm), 64);
4040 } else {
4041 /**
4042 * Source domain contains signature information
4043 * So need construct a strided block format:
4044 * ---------------------------
4045 * | stride_block_ctrl |
4046 * ---------------------------
4047 * | data_klm |
4048 * ---------------------------
4049 * | prot_klm |
4050 * ---------------------------
4051 * | BSF |
4052 * ---------------------------
4053 **/
4054 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
4055 struct mlx5_stride_block_entry *data_sentry;
4056 struct mlx5_stride_block_entry *prot_sentry;
e622f2f4
CH		 4057 u32 prot_key = wr->prot->lkey;
4058 u64 prot_va = wr->prot->addr;
e6631814
SG		 4059 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
4060 int prot_size;
4061
4062 sblock_ctrl = *seg;
4063 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
4064 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
4065
4066 prot_size = prot_field_size(sig_attrs->mem.sig_type);
4067 if (!prot_size) {
4068 pr_err("Bad block size given: %u\n", block_size);
4069 return -EINVAL;
4070 }
4071 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
4072 prot_size);
4073 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
4074 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
4075 sblock_ctrl->num_entries = cpu_to_be16(2);
4076
4077 data_sentry->bcount = cpu_to_be16(block_size);
4078 data_sentry->key = cpu_to_be32(data_key);
4079 data_sentry->va = cpu_to_be64(data_va);
5c273b16
SG		 4080 data_sentry->stride = cpu_to_be16(block_size);
4081
e6631814
SG		 4082 prot_sentry->bcount = cpu_to_be16(prot_size);
4083 prot_sentry->key = cpu_to_be32(prot_key);
5c273b16
SG		 4084 prot_sentry->va = cpu_to_be64(prot_va);
4085 prot_sentry->stride = cpu_to_be16(prot_size);
e6631814 4086
e6631814
SG		 4087 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
4088 sizeof(*prot_sentry), 64);
4089 }
4090
4091 *seg += wqe_size;
4092 *size += wqe_size / 16;
4093 if (unlikely((*seg == qp->sq.qend)))
4094 *seg = mlx5_get_send_wqe(qp, 0);
4095
4096 bsf = *seg;
4097 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
4098 if (ret)
4099 return -EINVAL;
4100
4101 *seg += sizeof(*bsf);
4102 *size += sizeof(*bsf) / 16;
4103 if (unlikely((*seg == qp->sq.qend)))
4104 *seg = mlx5_get_send_wqe(qp, 0);
4105
4106 return 0;
4107}
4108
4109static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
31616255 4110 struct ib_sig_handover_wr *wr, u32 size,
e6631814
SG		 4111 u32 length, u32 pdn)
4112{
e622f2f4 4113 struct ib_mr *sig_mr = wr->sig_mr;
e6631814 4114 u32 sig_key = sig_mr->rkey;
d5436ba0 4115 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
e6631814
SG		 4116
4117 memset(seg, 0, sizeof(*seg));
4118
e622f2f4 4119 seg->flags = get_umr_flags(wr->access_flags) |
ec22eb53 4120 MLX5_MKC_ACCESS_MODE_KLMS;
e6631814 4121 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
d5436ba0 4122 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
e6631814
SG		 4123 MLX5_MKEY_BSF_EN | pdn);
4124 seg->len = cpu_to_be64(length);
31616255 4125 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
e6631814
SG		 4126 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
4127}
4128
4129static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
31616255 4130 u32 size)
e6631814
SG		 4131{
4132 memset(umr, 0, sizeof(*umr));
4133
4134 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
31616255 4135 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
e6631814
SG		 4136 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
4137 umr->mkey_mask = sig_mkey_mask();
4138}
4139
4140
e622f2f4 4141static int set_sig_umr_wr(struct ib_send_wr *send_wr, struct mlx5_ib_qp *qp,
e6631814
SG		 4142 void **seg, int *size)
4143{
e622f2f4
CH		 4144 struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);
4145 struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr);
e6631814 4146 u32 pdn = get_pd(qp)->pdn;
31616255 4147 u32 xlt_size;
e6631814
SG		 4148 int region_len, ret;
4149
e622f2f4
CH		 4150 if (unlikely(wr->wr.num_sge != 1) ||
4151 unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) ||
d5436ba0
SG		 4152 unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) ||
4153 unlikely(!sig_mr->sig->sig_status_checked))
e6631814
SG		 4154 return -EINVAL;
4155
4156 /* length of the protected region, data + protection */
e622f2f4
CH		 4157 region_len = wr->wr.sg_list->length;
4158 if (wr->prot &&
4159 (wr->prot->lkey != wr->wr.sg_list->lkey ||
4160 wr->prot->addr != wr->wr.sg_list->addr ||
4161 wr->prot->length != wr->wr.sg_list->length))
4162 region_len += wr->prot->length;
e6631814
SG		 4163
4164 /**
4165 * KLM octoword size - if protection was provided
4166 * then we use strided block format (3 octowords),
4167 * else we use single KLM (1 octoword)
4168 **/
31616255 4169 xlt_size = wr->prot ? 0x30 : sizeof(struct mlx5_klm);
e6631814 4170
31616255 4171 set_sig_umr_segment(*seg, xlt_size);
e6631814
SG		 4172 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4173 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4174 if (unlikely((*seg == qp->sq.qend)))
4175 *seg = mlx5_get_send_wqe(qp, 0);
4176
31616255 4177 set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn);
e6631814
SG		 4178 *seg += sizeof(struct mlx5_mkey_seg);
4179 *size += sizeof(struct mlx5_mkey_seg) / 16;
4180 if (unlikely((*seg == qp->sq.qend)))
4181 *seg = mlx5_get_send_wqe(qp, 0);
4182
4183 ret = set_sig_data_segment(wr, qp, seg, size);
4184 if (ret)
4185 return ret;
4186
d5436ba0 4187 sig_mr->sig->sig_status_checked = false;
e6631814
SG		 4188 return 0;
4189}
4190
4191static int set_psv_wr(struct ib_sig_domain *domain,
4192 u32 psv_idx, void **seg, int *size)
4193{
4194 struct mlx5_seg_set_psv *psv_seg = *seg;
4195
4196 memset(psv_seg, 0, sizeof(*psv_seg));
4197 psv_seg->psv_num = cpu_to_be32(psv_idx);
4198 switch (domain->sig_type) {
78eda2bb
SG		 4199 case IB_SIG_TYPE_NONE:
4200 break;
e6631814
SG		 4201 case IB_SIG_TYPE_T10_DIF:
4202 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
4203 domain->sig.dif.app_tag);
4204 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
e6631814 4205 break;
e6631814 4206 default:
12bbf1ea
LR		 4207 pr_err("Bad signature type (%d) is given.\n",
4208 domain->sig_type);
4209 return -EINVAL;
e6631814
SG		 4210 }
4211
78eda2bb
SG		 4212 *seg += sizeof(*psv_seg);
4213 *size += sizeof(*psv_seg) / 16;
4214
e6631814
SG		 4215 return 0;
4216}
4217
8a187ee5
SG		 4218static int set_reg_wr(struct mlx5_ib_qp *qp,
4219 struct ib_reg_wr *wr,
4220 void **seg, int *size)
4221{
4222 struct mlx5_ib_mr *mr = to_mmr(wr->mr);
4223 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
4224
4225 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
4226 mlx5_ib_warn(to_mdev(qp->ibqp.device),
4227 "Invalid IB_SEND_INLINE send flag\n");
4228 return -EINVAL;
4229 }
4230
4231 set_reg_umr_seg(*seg, mr);
4232 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4233 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4234 if (unlikely((*seg == qp->sq.qend)))
4235 *seg = mlx5_get_send_wqe(qp, 0);
4236
4237 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
4238 *seg += sizeof(struct mlx5_mkey_seg);
4239 *size += sizeof(struct mlx5_mkey_seg) / 16;
4240 if (unlikely((*seg == qp->sq.qend)))
4241 *seg = mlx5_get_send_wqe(qp, 0);
4242
4243 set_reg_data_seg(*seg, mr, pd);
4244 *seg += sizeof(struct mlx5_wqe_data_seg);
4245 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
4246
4247 return 0;
4248}
4249
dd01e66a 4250static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size)
e126ba97 4251{
dd01e66a 4252 set_linv_umr_seg(*seg);
e126ba97
EC		 4253 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4254 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4255 if (unlikely((*seg == qp->sq.qend)))
4256 *seg = mlx5_get_send_wqe(qp, 0);
dd01e66a 4257 set_linv_mkey_seg(*seg);
e126ba97
EC		 4258 *seg += sizeof(struct mlx5_mkey_seg);
4259 *size += sizeof(struct mlx5_mkey_seg) / 16;
4260 if (unlikely((*seg == qp->sq.qend)))
4261 *seg = mlx5_get_send_wqe(qp, 0);
e126ba97
EC		 4262}
4263
4264static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
4265{
4266 __be32 *p = NULL;
4267 int tidx = idx;
4268 int i, j;
4269
4270 pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp, tidx));
4271 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
4272 if ((i & 0xf) == 0) {
4273 void *buf = mlx5_get_send_wqe(qp, tidx);
4274 tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1);
4275 p = buf;
4276 j = 0;
4277 }
4278 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
4279 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
4280 be32_to_cpu(p[j + 3]));
4281 }
4282}
4283
6e5eadac
SG		 4284static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
4285 struct mlx5_wqe_ctrl_seg **ctrl,
6a4f139a 4286 struct ib_send_wr *wr, unsigned *idx,
6e5eadac
SG		 4287 int *size, int nreq)
4288{
b2a232d2
LR		 4289 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
4290 return -ENOMEM;
6e5eadac
SG		 4291
4292 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
4293 *seg = mlx5_get_send_wqe(qp, *idx);
4294 *ctrl = *seg;
4295 *(uint32_t *)(*seg + 8) = 0;
4296 (*ctrl)->imm = send_ieth(wr);
4297 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
4298 (wr->send_flags & IB_SEND_SIGNALED ?
4299 MLX5_WQE_CTRL_CQ_UPDATE : 0) |
4300 (wr->send_flags & IB_SEND_SOLICITED ?
4301 MLX5_WQE_CTRL_SOLICITED : 0);
4302
4303 *seg += sizeof(**ctrl);
4304 *size = sizeof(**ctrl) / 16;
4305
b2a232d2 4306 return 0;
6e5eadac
SG		 4307}
4308
4309static void finish_wqe(struct mlx5_ib_qp *qp,
4310 struct mlx5_wqe_ctrl_seg *ctrl,
4311 u8 size, unsigned idx, u64 wr_id,
6e8484c5 4312 int nreq, u8 fence, u32 mlx5_opcode)
6e5eadac
SG		 4313{
4314 u8 opmod = 0;
4315
4316 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
4317 mlx5_opcode | ((u32)opmod << 24));
19098df2 4318 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
6e5eadac 4319 ctrl->fm_ce_se |= fence;
6e5eadac
SG		 4320 if (unlikely(qp->wq_sig))
4321 ctrl->signature = wq_sig(ctrl);
4322
4323 qp->sq.wrid[idx] = wr_id;
4324 qp->sq.w_list[idx].opcode = mlx5_opcode;
4325 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
4326 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
4327 qp->sq.w_list[idx].next = qp->sq.cur_post;
4328}
4329
4330
e126ba97
EC		 4331int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
4332 struct ib_send_wr **bad_wr)
4333{
4334 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
4335 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
89ea94a7 4336 struct mlx5_core_dev *mdev = dev->mdev;
d16e91da 4337 struct mlx5_ib_qp *qp;
e6631814 4338 struct mlx5_ib_mr *mr;
e126ba97
EC		 4339 struct mlx5_wqe_data_seg *dpseg;
4340 struct mlx5_wqe_xrc_seg *xrc;
d16e91da 4341 struct mlx5_bf *bf;
e126ba97 4342 int uninitialized_var(size);
d16e91da 4343 void *qend;
e126ba97 4344 unsigned long flags;
e126ba97
EC		 4345 unsigned idx;
4346 int err = 0;
e126ba97
EC		 4347 int num_sge;
4348 void *seg;
4349 int nreq;
4350 int i;
4351 u8 next_fence = 0;
e126ba97
EC		 4352 u8 fence;
4353
d16e91da
HE		 4354 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4355 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
4356
4357 qp = to_mqp(ibqp);
5fe9dec0 4358 bf = &qp->bf;
d16e91da
HE		 4359 qend = qp->sq.qend;
4360
e126ba97
EC		 4361 spin_lock_irqsave(&qp->sq.lock, flags);
4362
89ea94a7
MG		 4363 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
4364 err = -EIO;
4365 *bad_wr = wr;
4366 nreq = 0;
4367 goto out;
4368 }
4369
e126ba97 4370 for (nreq = 0; wr; nreq++, wr = wr->next) {
a8f731eb 4371 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
e126ba97
EC		 4372 mlx5_ib_warn(dev, "\n");
4373 err = -EINVAL;
4374 *bad_wr = wr;
4375 goto out;
4376 }
4377
6e5eadac
SG		 4378 num_sge = wr->num_sge;
4379 if (unlikely(num_sge > qp->sq.max_gs)) {
e126ba97 4380 mlx5_ib_warn(dev, "\n");
24be409b 4381 err = -EINVAL;
e126ba97
EC		 4382 *bad_wr = wr;
4383 goto out;
4384 }
4385
6e5eadac
SG		 4386 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);
4387 if (err) {
e126ba97
EC		 4388 mlx5_ib_warn(dev, "\n");
4389 err = -ENOMEM;
4390 *bad_wr = wr;
4391 goto out;
4392 }
4393
6e8484c5
MG		 4394 if (wr->opcode == IB_WR_LOCAL_INV ||
4395 wr->opcode == IB_WR_REG_MR) {
4396 fence = dev->umr_fence;
4397 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
4398 } else if (wr->send_flags & IB_SEND_FENCE) {
4399 if (qp->next_fence)
4400 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
4401 else
4402 fence = MLX5_FENCE_MODE_FENCE;
4403 } else {
4404 fence = qp->next_fence;
4405 }
4406
e126ba97
EC		 4407 switch (ibqp->qp_type) {
4408 case IB_QPT_XRC_INI:
4409 xrc = seg;
e126ba97
EC		 4410 seg += sizeof(*xrc);
4411 size += sizeof(*xrc) / 16;
4412 /* fall through */
4413 case IB_QPT_RC:
4414 switch (wr->opcode) {
4415 case IB_WR_RDMA_READ:
4416 case IB_WR_RDMA_WRITE:
4417 case IB_WR_RDMA_WRITE_WITH_IMM:
e622f2f4
CH		 4418 set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
4419 rdma_wr(wr)->rkey);
f241e749 4420 seg += sizeof(struct mlx5_wqe_raddr_seg);
e126ba97
EC		 4421 size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
4422 break;
4423
4424 case IB_WR_ATOMIC_CMP_AND_SWP:
4425 case IB_WR_ATOMIC_FETCH_AND_ADD:
e126ba97 4426 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
81bea28f
EC		 4427 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
4428 err = -ENOSYS;
4429 *bad_wr = wr;
4430 goto out;
e126ba97
EC		 4431
4432 case IB_WR_LOCAL_INV:
e126ba97
EC		 4433 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
4434 ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
dd01e66a 4435 set_linv_wr(qp, &seg, &size);
e126ba97
EC		 4436 num_sge = 0;
4437 break;
4438
8a187ee5 4439 case IB_WR_REG_MR:
8a187ee5
SG		 4440 qp->sq.wr_data[idx] = IB_WR_REG_MR;
4441 ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
4442 err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
4443 if (err) {
4444 *bad_wr = wr;
4445 goto out;
4446 }
4447 num_sge = 0;
4448 break;
4449
e6631814
SG		 4450 case IB_WR_REG_SIG_MR:
4451 qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
e622f2f4 4452 mr = to_mmr(sig_handover_wr(wr)->sig_mr);
e6631814
SG		 4453
4454 ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
4455 err = set_sig_umr_wr(wr, qp, &seg, &size);
4456 if (err) {
4457 mlx5_ib_warn(dev, "\n");
4458 *bad_wr = wr;
4459 goto out;
4460 }
4461
6e8484c5
MG		 4462 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4463 fence, MLX5_OPCODE_UMR);
e6631814
SG		 4464 /*
4465 * SET_PSV WQEs are not signaled and solicited
4466 * on error
4467 */
4468 wr->send_flags &= ~IB_SEND_SIGNALED;
4469 wr->send_flags |= IB_SEND_SOLICITED;
4470 err = begin_wqe(qp, &seg, &ctrl, wr,
4471 &idx, &size, nreq);
4472 if (err) {
4473 mlx5_ib_warn(dev, "\n");
4474 err = -ENOMEM;
4475 *bad_wr = wr;
4476 goto out;
4477 }
4478
e622f2f4 4479 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem,
e6631814
SG		 4480 mr->sig->psv_memory.psv_idx, &seg,
4481 &size);
4482 if (err) {
4483 mlx5_ib_warn(dev, "\n");
4484 *bad_wr = wr;
4485 goto out;
4486 }
4487
6e8484c5
MG		 4488 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4489 fence, MLX5_OPCODE_SET_PSV);
e6631814
SG		 4490 err = begin_wqe(qp, &seg, &ctrl, wr,
4491 &idx, &size, nreq);
4492 if (err) {
4493 mlx5_ib_warn(dev, "\n");
4494 err = -ENOMEM;
4495 *bad_wr = wr;
4496 goto out;
4497 }
4498
e622f2f4 4499 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
e6631814
SG		 4500 mr->sig->psv_wire.psv_idx, &seg,
4501 &size);
4502 if (err) {
4503 mlx5_ib_warn(dev, "\n");
4504 *bad_wr = wr;
4505 goto out;
4506 }
4507
6e8484c5
MG		 4508 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4509 fence, MLX5_OPCODE_SET_PSV);
4510 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
e6631814
SG		 4511 num_sge = 0;
4512 goto skip_psv;
4513
e126ba97
EC		 4514 default:
4515 break;
4516 }
4517 break;
4518
4519 case IB_QPT_UC:
4520 switch (wr->opcode) {
4521 case IB_WR_RDMA_WRITE:
4522 case IB_WR_RDMA_WRITE_WITH_IMM:
e622f2f4
CH		 4523 set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
4524 rdma_wr(wr)->rkey);
e126ba97
EC		 4525 seg += sizeof(struct mlx5_wqe_raddr_seg);
4526 size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
4527 break;
4528
4529 default:
4530 break;
4531 }
4532 break;
4533
e126ba97 4534 case IB_QPT_SMI:
1e0e50b6
MG		 4535 if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
4536 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
4537 err = -EPERM;
4538 *bad_wr = wr;
4539 goto out;
4540 }
f6b1ee34 4541 /* fall through */
d16e91da 4542 case MLX5_IB_QPT_HW_GSI:
e126ba97 4543 set_datagram_seg(seg, wr);
f241e749 4544 seg += sizeof(struct mlx5_wqe_datagram_seg);
e126ba97
EC		 4545 size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
4546 if (unlikely((seg == qend)))
4547 seg = mlx5_get_send_wqe(qp, 0);
4548 break;
f0313965
ES		 4549 case IB_QPT_UD:
4550 set_datagram_seg(seg, wr);
4551 seg += sizeof(struct mlx5_wqe_datagram_seg);
4552 size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
4553
4554 if (unlikely((seg == qend)))
4555 seg = mlx5_get_send_wqe(qp, 0);
4556
4557 /* handle qp that supports ud offload */
4558 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
4559 struct mlx5_wqe_eth_pad *pad;
e126ba97 4560
f0313965
ES		 4561 pad = seg;
4562 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
4563 seg += sizeof(struct mlx5_wqe_eth_pad);
4564 size += sizeof(struct mlx5_wqe_eth_pad) / 16;
4565
4566 seg = set_eth_seg(seg, wr, qend, qp, &size);
4567
4568 if (unlikely((seg == qend)))
4569 seg = mlx5_get_send_wqe(qp, 0);
4570 }
4571 break;
e126ba97
EC		 4572 case MLX5_IB_QPT_REG_UMR:
4573 if (wr->opcode != MLX5_IB_WR_UMR) {
4574 err = -EINVAL;
4575 mlx5_ib_warn(dev, "bad opcode\n");
4576 goto out;
4577 }
4578 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
e622f2f4 4579 ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);
c8d75a98
MD		 4580 err = set_reg_umr_segment(dev, seg, wr, !!(MLX5_CAP_GEN(mdev, atomic)));
4581 if (unlikely(err))
4582 goto out;
e126ba97
EC		 4583 seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4584 size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4585 if (unlikely((seg == qend)))
4586 seg = mlx5_get_send_wqe(qp, 0);
4587 set_reg_mkey_segment(seg, wr);
4588 seg += sizeof(struct mlx5_mkey_seg);
4589 size += sizeof(struct mlx5_mkey_seg) / 16;
4590 if (unlikely((seg == qend)))
4591 seg = mlx5_get_send_wqe(qp, 0);
4592 break;
4593
4594 default:
4595 break;
4596 }
4597
4598 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
4599 int uninitialized_var(sz);
4600
4601 err = set_data_inl_seg(qp, wr, seg, &sz);
4602 if (unlikely(err)) {
4603 mlx5_ib_warn(dev, "\n");
4604 *bad_wr = wr;
4605 goto out;
4606 }
e126ba97
EC		 4607 size += sz;
4608 } else {
4609 dpseg = seg;
4610 for (i = 0; i < num_sge; i++) {
4611 if (unlikely(dpseg == qend)) {
4612 seg = mlx5_get_send_wqe(qp, 0);
4613 dpseg = seg;
4614 }
4615 if (likely(wr->sg_list[i].length)) {
4616 set_data_ptr_seg(dpseg, wr->sg_list + i);
4617 size += sizeof(struct mlx5_wqe_data_seg) / 16;
4618 dpseg++;
4619 }
4620 }
4621 }
4622
6e8484c5
MG		 4623 qp->next_fence = next_fence;
4624 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, fence,
6e5eadac 4625 mlx5_ib_opcode[wr->opcode]);
e6631814 4626skip_psv:
e126ba97
EC		 4627 if (0)
4628 dump_wqe(qp, idx, size);
4629 }
4630
4631out:
4632 if (likely(nreq)) {
4633 qp->sq.head += nreq;
4634
4635 /* Make sure that descriptors are written before
4636 * updating doorbell record and ringing the doorbell
4637 */
4638 wmb();
4639
4640 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
4641
ada388f7
EC		 4642 /* Make sure doorbell record is visible to the HCA before
4643 * we hit doorbell */
4644 wmb();
4645
5fe9dec0
EC		 4646 /* currently we support only regular doorbells */
4647 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset, NULL);
4648 /* Make sure doorbells don't leak out of SQ spinlock
4649 * and reach the HCA out of order.
4650 */
4651 mmiowb();
e126ba97 4652 bf->offset ^= bf->buf_size;
e126ba97
EC		 4653 }
4654
4655 spin_unlock_irqrestore(&qp->sq.lock, flags);
4656
4657 return err;
4658}
4659
4660static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size)
4661{
4662 sig->signature = calc_sig(sig, size);
4663}
4664
4665int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
4666 struct ib_recv_wr **bad_wr)
4667{
4668 struct mlx5_ib_qp *qp = to_mqp(ibqp);
4669 struct mlx5_wqe_data_seg *scat;
4670 struct mlx5_rwqe_sig *sig;
89ea94a7
MG		 4671 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4672 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97
EC		 4673 unsigned long flags;
4674 int err = 0;
4675 int nreq;
4676 int ind;
4677 int i;
4678
d16e91da
HE		 4679 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4680 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
4681
e126ba97
EC		 4682 spin_lock_irqsave(&qp->rq.lock, flags);
4683
89ea94a7
MG		 4684 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
4685 err = -EIO;
4686 *bad_wr = wr;
4687 nreq = 0;
4688 goto out;
4689 }
4690
e126ba97
EC		 4691 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
4692
4693 for (nreq = 0; wr; nreq++, wr = wr->next) {
4694 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
4695 err = -ENOMEM;
4696 *bad_wr = wr;
4697 goto out;
4698 }
4699
4700 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
4701 err = -EINVAL;
4702 *bad_wr = wr;
4703 goto out;
4704 }
4705
4706 scat = get_recv_wqe(qp, ind);
4707 if (qp->wq_sig)
4708 scat++;
4709
4710 for (i = 0; i < wr->num_sge; i++)
4711 set_data_ptr_seg(scat + i, wr->sg_list + i);
4712
4713 if (i < qp->rq.max_gs) {
4714 scat[i].byte_count = 0;
4715 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
4716 scat[i].addr = 0;
4717 }
4718
4719 if (qp->wq_sig) {
4720 sig = (struct mlx5_rwqe_sig *)scat;
4721 set_sig_seg(sig, (qp->rq.max_gs + 1) << 2);
4722 }
4723
4724 qp->rq.wrid[ind] = wr->wr_id;
4725
4726 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
4727 }
4728
4729out:
4730 if (likely(nreq)) {
4731 qp->rq.head += nreq;
4732
4733 /* Make sure that descriptors are written before
4734 * doorbell record.
4735 */
4736 wmb();
4737
4738 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
4739 }
4740
4741 spin_unlock_irqrestore(&qp->rq.lock, flags);
4742
4743 return err;
4744}
4745
4746static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4747{
4748 switch (mlx5_state) {
4749 case MLX5_QP_STATE_RST: return IB_QPS_RESET;
4750 case MLX5_QP_STATE_INIT: return IB_QPS_INIT;
4751 case MLX5_QP_STATE_RTR: return IB_QPS_RTR;
4752 case MLX5_QP_STATE_RTS: return IB_QPS_RTS;
4753 case MLX5_QP_STATE_SQ_DRAINING:
4754 case MLX5_QP_STATE_SQD: return IB_QPS_SQD;
4755 case MLX5_QP_STATE_SQER: return IB_QPS_SQE;
4756 case MLX5_QP_STATE_ERR: return IB_QPS_ERR;
4757 default: return -1;
4758 }
4759}
4760
4761static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4762{
4763 switch (mlx5_mig_state) {
4764 case MLX5_QP_PM_ARMED: return IB_MIG_ARMED;
4765 case MLX5_QP_PM_REARM: return IB_MIG_REARM;
4766 case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
4767 default: return -1;
4768 }
4769}
4770
4771static int to_ib_qp_access_flags(int mlx5_flags)
4772{
4773 int ib_flags = 0;
4774
4775 if (mlx5_flags & MLX5_QP_BIT_RRE)
4776 ib_flags |= IB_ACCESS_REMOTE_READ;
4777 if (mlx5_flags & MLX5_QP_BIT_RWE)
4778 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4779 if (mlx5_flags & MLX5_QP_BIT_RAE)
4780 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4781
4782 return ib_flags;
4783}
4784
38349389 4785static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
d8966fcd 4786 struct rdma_ah_attr *ah_attr,
38349389 4787 struct mlx5_qp_path *path)
e126ba97 4788{
e126ba97 4789
d8966fcd 4790 memset(ah_attr, 0, sizeof(*ah_attr));
e126ba97 4791
e7996a9a 4792 if (!path->port || path->port > ibdev->num_ports)
e126ba97
EC		 4793 return;
4794
ae59c3f0
LR		 4795 ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, path->port);
4796
d8966fcd
DC		 4797 rdma_ah_set_port_num(ah_attr, path->port);
4798 rdma_ah_set_sl(ah_attr, path->dci_cfi_prio_sl & 0xf);
4799
4800 rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4801 rdma_ah_set_path_bits(ah_attr, path->grh_mlid & 0x7f);
4802 rdma_ah_set_static_rate(ah_attr,
4803 path->static_rate ? path->static_rate - 5 : 0);
4804 if (path->grh_mlid & (1 << 7)) {
4805 u32 tc_fl = be32_to_cpu(path->tclass_flowlabel);
4806
4807 rdma_ah_set_grh(ah_attr, NULL,
4808 tc_fl & 0xfffff,
4809 path->mgid_index,
4810 path->hop_limit,
4811 (tc_fl >> 20) & 0xff);
4812 rdma_ah_set_dgid_raw(ah_attr, path->rgid);
e126ba97
EC		 4813 }
4814}
4815
6d2f89df 4816static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4817 struct mlx5_ib_sq *sq,
4818 u8 *sq_state)
4819{
6d2f89df 4820 int err;
4821
28160771 4822 err = mlx5_core_query_sq_state(dev->mdev, sq->base.mqp.qpn, sq_state);
6d2f89df 4823 if (err)
4824 goto out;
6d2f89df 4825 sq->state = *sq_state;
4826
4827out:
6d2f89df 4828 return err;
4829}
4830
4831static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4832 struct mlx5_ib_rq *rq,
4833 u8 *rq_state)
4834{
4835 void *out;
4836 void *rqc;
4837 int inlen;
4838 int err;
4839
4840 inlen = MLX5_ST_SZ_BYTES(query_rq_out);
1b9a07ee 4841 out = kvzalloc(inlen, GFP_KERNEL);
6d2f89df 4842 if (!out)
4843 return -ENOMEM;
4844
4845 err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4846 if (err)
4847 goto out;
4848
4849 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4850 *rq_state = MLX5_GET(rqc, rqc, state);
4851 rq->state = *rq_state;
4852
4853out:
4854 kvfree(out);
4855 return err;
4856}
4857
4858static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4859 struct mlx5_ib_qp *qp, u8 *qp_state)
4860{
4861 static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4862 [MLX5_RQC_STATE_RST] = {
4863 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4864 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4865 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE_BAD,
4866 [MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4867 },
4868 [MLX5_RQC_STATE_RDY] = {
4869 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4870 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4871 [MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4872 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
4873 },
4874 [MLX5_RQC_STATE_ERR] = {
4875 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4876 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4877 [MLX5_SQC_STATE_ERR] = IB_QPS_ERR,
4878 [MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4879 },
4880 [MLX5_RQ_STATE_NA] = {
4881 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4882 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4883 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4884 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
4885 },
4886 };
4887
4888 *qp_state = sqrq_trans[rq_state][sq_state];
4889
4890 if (*qp_state == MLX5_QP_STATE_BAD) {
4891 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4892 qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4893 qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4894 return -EINVAL;
4895 }
4896
4897 if (*qp_state == MLX5_QP_STATE)
4898 *qp_state = qp->state;
4899
4900 return 0;
4901}
4902
4903static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4904 struct mlx5_ib_qp *qp,
4905 u8 *raw_packet_qp_state)
4906{
4907 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4908 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4909 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4910 int err;
4911 u8 sq_state = MLX5_SQ_STATE_NA;
4912 u8 rq_state = MLX5_RQ_STATE_NA;
4913
4914 if (qp->sq.wqe_cnt) {
4915 err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4916 if (err)
4917 return err;
4918 }
4919
4920 if (qp->rq.wqe_cnt) {
4921 err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4922 if (err)
4923 return err;
4924 }
4925
4926 return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4927 raw_packet_qp_state);
4928}
4929
4930static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4931 struct ib_qp_attr *qp_attr)
e126ba97 4932{
09a7d9ec 4933 int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
e126ba97
EC		 4934 struct mlx5_qp_context *context;
4935 int mlx5_state;
09a7d9ec 4936 u32 *outb;
e126ba97
EC		 4937 int err = 0;
4938
09a7d9ec 4939 outb = kzalloc(outlen, GFP_KERNEL);
6d2f89df 4940 if (!outb)
4941 return -ENOMEM;
4942
19098df2 4943 err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb,
09a7d9ec 4944 outlen);
e126ba97 4945 if (err)
6d2f89df 4946 goto out;
e126ba97 4947
09a7d9ec
SM		 4948 /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
4949 context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
4950
e126ba97
EC		 4951 mlx5_state = be32_to_cpu(context->flags) >> 28;
4952
4953 qp->state = to_ib_qp_state(mlx5_state);
e126ba97
EC		 4954 qp_attr->path_mtu = context->mtu_msgmax >> 5;
4955 qp_attr->path_mig_state =
4956 to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
4957 qp_attr->qkey = be32_to_cpu(context->qkey);
4958 qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
4959 qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff;
4960 qp_attr->dest_qp_num = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff;
4961 qp_attr->qp_access_flags =
4962 to_ib_qp_access_flags(be32_to_cpu(context->params2));
4963
4964 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
38349389
DC		 4965 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
4966 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
d3ae2bde
NO		 4967 qp_attr->alt_pkey_index =
4968 be16_to_cpu(context->alt_path.pkey_index);
d8966fcd
DC		 4969 qp_attr->alt_port_num =
4970 rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
e126ba97
EC		 4971 }
4972
d3ae2bde 4973 qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index);
e126ba97
EC		 4974 qp_attr->port_num = context->pri_path.port;
4975
4976 /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4977 qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;
4978
4979 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
4980
4981 qp_attr->max_dest_rd_atomic =
4982 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
4983 qp_attr->min_rnr_timer =
4984 (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
4985 qp_attr->timeout = context->pri_path.ackto_lt >> 3;
4986 qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7;
4987 qp_attr->rnr_retry = (be32_to_cpu(context->params1) >> 13) & 0x7;
4988 qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3;
6d2f89df 4989
4990out:
4991 kfree(outb);
4992 return err;
4993}
4994
776a3906
MS		 4995static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
4996 struct ib_qp_attr *qp_attr, int qp_attr_mask,
4997 struct ib_qp_init_attr *qp_init_attr)
4998{
4999 struct mlx5_core_dct *dct = &mqp->dct.mdct;
5000 u32 *out;
5001 u32 access_flags = 0;
5002 int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
5003 void *dctc;
5004 int err;
5005 int supported_mask = IB_QP_STATE |
5006 IB_QP_ACCESS_FLAGS |
5007 IB_QP_PORT |
5008 IB_QP_MIN_RNR_TIMER |
5009 IB_QP_AV |
5010 IB_QP_PATH_MTU |
5011 IB_QP_PKEY_INDEX;
5012
5013 if (qp_attr_mask & ~supported_mask)
5014 return -EINVAL;
5015 if (mqp->state != IB_QPS_RTR)
5016 return -EINVAL;
5017
5018 out = kzalloc(outlen, GFP_KERNEL);
5019 if (!out)
5020 return -ENOMEM;
5021
5022 err = mlx5_core_dct_query(dev->mdev, dct, out, outlen);
5023 if (err)
5024 goto out;
5025
5026 dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
5027
5028 if (qp_attr_mask & IB_QP_STATE)
5029 qp_attr->qp_state = IB_QPS_RTR;
5030
5031 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
5032 if (MLX5_GET(dctc, dctc, rre))
5033 access_flags |= IB_ACCESS_REMOTE_READ;
5034 if (MLX5_GET(dctc, dctc, rwe))
5035 access_flags |= IB_ACCESS_REMOTE_WRITE;
5036 if (MLX5_GET(dctc, dctc, rae))
5037 access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5038 qp_attr->qp_access_flags = access_flags;
5039 }
5040
5041 if (qp_attr_mask & IB_QP_PORT)
5042 qp_attr->port_num = MLX5_GET(dctc, dctc, port);
5043 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
5044 qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
5045 if (qp_attr_mask & IB_QP_AV) {
5046 qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
5047 qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
5048 qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
5049 qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
5050 }
5051 if (qp_attr_mask & IB_QP_PATH_MTU)
5052 qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
5053 if (qp_attr_mask & IB_QP_PKEY_INDEX)
5054 qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
5055out:
5056 kfree(out);
5057 return err;
5058}
5059
6d2f89df 5060int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5061 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
5062{
5063 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
5064 struct mlx5_ib_qp *qp = to_mqp(ibqp);
5065 int err = 0;
5066 u8 raw_packet_qp_state;
5067
28d61370
YH		 5068 if (ibqp->rwq_ind_tbl)
5069 return -ENOSYS;
5070
d16e91da
HE		 5071 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
5072 return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
5073 qp_init_attr);
5074
c2e53b2c
YH		 5075 /* Not all of output fields are applicable, make sure to zero them */
5076 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5077 memset(qp_attr, 0, sizeof(*qp_attr));
5078
776a3906
MS		 5079 if (unlikely(qp->qp_sub_type == MLX5_IB_QPT_DCT))
5080 return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
5081 qp_attr_mask, qp_init_attr);
5082
6d2f89df 5083 mutex_lock(&qp->mutex);
5084
c2e53b2c
YH		 5085 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
5086 qp->flags & MLX5_IB_QP_UNDERLAY) {
6d2f89df 5087 err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
5088 if (err)
5089 goto out;
5090 qp->state = raw_packet_qp_state;
5091 qp_attr->port_num = 1;
5092 } else {
5093 err = query_qp_attr(dev, qp, qp_attr);
5094 if (err)
5095 goto out;
5096 }
5097
5098 qp_attr->qp_state = qp->state;
e126ba97
EC		 5099 qp_attr->cur_qp_state = qp_attr->qp_state;
5100 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
5101 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
5102
5103 if (!ibqp->uobject) {
0540d814 5104 qp_attr->cap.max_send_wr = qp->sq.max_post;
e126ba97 5105 qp_attr->cap.max_send_sge = qp->sq.max_gs;
0540d814 5106 qp_init_attr->qp_context = ibqp->qp_context;
e126ba97
EC		 5107 } else {
5108 qp_attr->cap.max_send_wr = 0;
5109 qp_attr->cap.max_send_sge = 0;
5110 }
5111
0540d814
NO		 5112 qp_init_attr->qp_type = ibqp->qp_type;
5113 qp_init_attr->recv_cq = ibqp->recv_cq;
5114 qp_init_attr->send_cq = ibqp->send_cq;
5115 qp_init_attr->srq = ibqp->srq;
5116 qp_attr->cap.max_inline_data = qp->max_inline_data;
e126ba97
EC		 5117
5118 qp_init_attr->cap = qp_attr->cap;
5119
5120 qp_init_attr->create_flags = 0;
5121 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
5122 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
5123
051f2630
LR		 5124 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
5125 qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL;
5126 if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
5127 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND;
5128 if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
5129 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV;
b11a4f9c
HE		 5130 if (qp->flags & MLX5_IB_QP_SQPN_QP1)
5131 qp_init_attr->create_flags |= mlx5_ib_create_qp_sqpn_qp1();
051f2630 5132
e126ba97
EC		 5133 qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5134 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
5135
e126ba97
EC		 5136out:
5137 mutex_unlock(&qp->mutex);
5138 return err;
5139}
5140
5141struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
5142 struct ib_ucontext *context,
5143 struct ib_udata *udata)
5144{
5145 struct mlx5_ib_dev *dev = to_mdev(ibdev);
5146 struct mlx5_ib_xrcd *xrcd;
5147 int err;
5148
938fe83c 5149 if (!MLX5_CAP_GEN(dev->mdev, xrc))
e126ba97
EC		 5150 return ERR_PTR(-ENOSYS);
5151
5152 xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
5153 if (!xrcd)
5154 return ERR_PTR(-ENOMEM);
5155
9603b61d 5156 err = mlx5_core_xrcd_alloc(dev->mdev, &xrcd->xrcdn);
e126ba97
EC		 5157 if (err) {
5158 kfree(xrcd);
5159 return ERR_PTR(-ENOMEM);
5160 }
5161
5162 return &xrcd->ibxrcd;
5163}
5164
5165int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
5166{
5167 struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
5168 u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
5169 int err;
5170
9603b61d 5171 err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn);
b081808a 5172 if (err)
e126ba97 5173 mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
e126ba97
EC		 5174
5175 kfree(xrcd);
e126ba97
EC		 5176 return 0;
5177}
79b20a6c 5178
350d0e4c
YH		 5179static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
5180{
5181 struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
5182 struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
5183 struct ib_event event;
5184
5185 if (rwq->ibwq.event_handler) {
5186 event.device = rwq->ibwq.device;
5187 event.element.wq = &rwq->ibwq;
5188 switch (type) {
5189 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
5190 event.event = IB_EVENT_WQ_FATAL;
5191 break;
5192 default:
5193 mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
5194 return;
5195 }
5196
5197 rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
5198 }
5199}
5200
03404e8a
MG		 5201static int set_delay_drop(struct mlx5_ib_dev *dev)
5202{
5203 int err = 0;
5204
5205 mutex_lock(&dev->delay_drop.lock);
5206 if (dev->delay_drop.activate)
5207 goto out;
5208
5209 err = mlx5_core_set_delay_drop(dev->mdev, dev->delay_drop.timeout);
5210 if (err)
5211 goto out;
5212
5213 dev->delay_drop.activate = true;
5214out:
5215 mutex_unlock(&dev->delay_drop.lock);
fe248c3a
MG		 5216
5217 if (!err)
5218 atomic_inc(&dev->delay_drop.rqs_cnt);
03404e8a
MG		 5219 return err;
5220}
5221
79b20a6c
YH		 5222static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
5223 struct ib_wq_init_attr *init_attr)
5224{
5225 struct mlx5_ib_dev *dev;
4be6da1e 5226 int has_net_offloads;
79b20a6c
YH		 5227 __be64 *rq_pas0;
5228 void *in;
5229 void *rqc;
5230 void *wq;
5231 int inlen;
5232 int err;
5233
5234 dev = to_mdev(pd->device);
5235
5236 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
1b9a07ee 5237 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5238 if (!in)
5239 return -ENOMEM;
5240
5241 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5242 MLX5_SET(rqc, rqc, mem_rq_type,
5243 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
5244 MLX5_SET(rqc, rqc, user_index, rwq->user_index);
5245 MLX5_SET(rqc, rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
5246 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
5247 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
5248 wq = MLX5_ADDR_OF(rqc, rqc, wq);
ccc87087
NO		 5249 MLX5_SET(wq, wq, wq_type,
5250 rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
5251 MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 5252 if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5253 if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
5254 mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
5255 err = -EOPNOTSUPP;
5256 goto out;
5257 } else {
5258 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
5259 }
5260 }
79b20a6c 5261 MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
ccc87087
NO		 5262 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
5263 MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5264 MLX5_SET(wq, wq, log_wqe_stride_size,
5265 rwq->single_stride_log_num_of_bytes -
5266 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
5267 MLX5_SET(wq, wq, log_wqe_num_of_strides, rwq->log_num_strides -
5268 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES);
5269 }
79b20a6c
YH		 5270 MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5271 MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
5272 MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
5273 MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
5274 MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
5275 MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
4be6da1e 5276 has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
b1f74a84 5277 if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
4be6da1e 5278 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
b1f74a84
NO		 5279 mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5280 err = -EOPNOTSUPP;
5281 goto out;
5282 }
5283 } else {
5284 MLX5_SET(rqc, rqc, vsd, 1);
5285 }
4be6da1e
NO		 5286 if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
5287 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
5288 mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
5289 err = -EOPNOTSUPP;
5290 goto out;
5291 }
5292 MLX5_SET(rqc, rqc, scatter_fcs, 1);
5293 }
03404e8a
MG		 5294 if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5295 if (!(dev->ib_dev.attrs.raw_packet_caps &
5296 IB_RAW_PACKET_CAP_DELAY_DROP)) {
5297 mlx5_ib_dbg(dev, "Delay drop is not supported\n");
5298 err = -EOPNOTSUPP;
5299 goto out;
5300 }
5301 MLX5_SET(rqc, rqc, delay_drop_en, 1);
5302 }
79b20a6c
YH		 5303 rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
5304 mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
350d0e4c 5305 err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);
03404e8a
MG		 5306 if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5307 err = set_delay_drop(dev);
5308 if (err) {
5309 mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5310 err);
5311 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
5312 } else {
5313 rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5314 }
5315 }
b1f74a84 5316out:
79b20a6c
YH		 5317 kvfree(in);
5318 return err;
5319}
5320
5321static int set_user_rq_size(struct mlx5_ib_dev *dev,
5322 struct ib_wq_init_attr *wq_init_attr,
5323 struct mlx5_ib_create_wq *ucmd,
5324 struct mlx5_ib_rwq *rwq)
5325{
5326 /* Sanity check RQ size before proceeding */
5327 if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
5328 return -EINVAL;
5329
5330 if (!ucmd->rq_wqe_count)
5331 return -EINVAL;
5332
5333 rwq->wqe_count = ucmd->rq_wqe_count;
5334 rwq->wqe_shift = ucmd->rq_wqe_shift;
5335 rwq->buf_size = (rwq->wqe_count << rwq->wqe_shift);
5336 rwq->log_rq_stride = rwq->wqe_shift;
5337 rwq->log_rq_size = ilog2(rwq->wqe_count);
5338 return 0;
5339}
5340
5341static int prepare_user_rq(struct ib_pd *pd,
5342 struct ib_wq_init_attr *init_attr,
5343 struct ib_udata *udata,
5344 struct mlx5_ib_rwq *rwq)
5345{
5346 struct mlx5_ib_dev *dev = to_mdev(pd->device);
5347 struct mlx5_ib_create_wq ucmd = {};
5348 int err;
5349 size_t required_cmd_sz;
5350
ccc87087
NO		 5351 required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes)
5352 + sizeof(ucmd.single_stride_log_num_of_bytes);
79b20a6c
YH		 5353 if (udata->inlen < required_cmd_sz) {
5354 mlx5_ib_dbg(dev, "invalid inlen\n");
5355 return -EINVAL;
5356 }
5357
5358 if (udata->inlen > sizeof(ucmd) &&
5359 !ib_is_udata_cleared(udata, sizeof(ucmd),
5360 udata->inlen - sizeof(ucmd))) {
5361 mlx5_ib_dbg(dev, "inlen is not supported\n");
5362 return -EOPNOTSUPP;
5363 }
5364
5365 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
5366 mlx5_ib_dbg(dev, "copy failed\n");
5367 return -EFAULT;
5368 }
5369
ccc87087 5370 if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
79b20a6c
YH		 5371 mlx5_ib_dbg(dev, "invalid comp mask\n");
5372 return -EOPNOTSUPP;
ccc87087
NO		 5373 } else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
5374 if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
5375 mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
5376 return -EOPNOTSUPP;
5377 }
5378 if ((ucmd.single_stride_log_num_of_bytes <
5379 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
5380 (ucmd.single_stride_log_num_of_bytes >
5381 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
5382 mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
5383 ucmd.single_stride_log_num_of_bytes,
5384 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
5385 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5386 return -EINVAL;
5387 }
5388 if ((ucmd.single_wqe_log_num_of_strides >
5389 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
5390 (ucmd.single_wqe_log_num_of_strides <
5391 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES)) {
5392 mlx5_ib_dbg(dev, "Invalid log num strides (%u. Range is %u - %u)\n",
5393 ucmd.single_wqe_log_num_of_strides,
5394 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5395 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
5396 return -EINVAL;
5397 }
5398 rwq->single_stride_log_num_of_bytes =
5399 ucmd.single_stride_log_num_of_bytes;
5400 rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
5401 rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
5402 rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
79b20a6c
YH		 5403 }
5404
5405 err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
5406 if (err) {
5407 mlx5_ib_dbg(dev, "err %d\n", err);
5408 return err;
5409 }
5410
5411 err = create_user_rq(dev, pd, rwq, &ucmd);
5412 if (err) {
5413 mlx5_ib_dbg(dev, "err %d\n", err);
5414 if (err)
5415 return err;
5416 }
5417
5418 rwq->user_index = ucmd.user_index;
5419 return 0;
5420}
5421
5422struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
5423 struct ib_wq_init_attr *init_attr,
5424 struct ib_udata *udata)
5425{
5426 struct mlx5_ib_dev *dev;
5427 struct mlx5_ib_rwq *rwq;
5428 struct mlx5_ib_create_wq_resp resp = {};
5429 size_t min_resp_len;
5430 int err;
5431
5432 if (!udata)
5433 return ERR_PTR(-ENOSYS);
5434
5435 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
5436 if (udata->outlen && udata->outlen < min_resp_len)
5437 return ERR_PTR(-EINVAL);
5438
5439 dev = to_mdev(pd->device);
5440 switch (init_attr->wq_type) {
5441 case IB_WQT_RQ:
5442 rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5443 if (!rwq)
5444 return ERR_PTR(-ENOMEM);
5445 err = prepare_user_rq(pd, init_attr, udata, rwq);
5446 if (err)
5447 goto err;
5448 err = create_rq(rwq, pd, init_attr);
5449 if (err)
5450 goto err_user_rq;
5451 break;
5452 default:
5453 mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5454 init_attr->wq_type);
5455 return ERR_PTR(-EINVAL);
5456 }
5457
350d0e4c 5458 rwq->ibwq.wq_num = rwq->core_qp.qpn;
79b20a6c
YH		 5459 rwq->ibwq.state = IB_WQS_RESET;
5460 if (udata->outlen) {
5461 resp.response_length = offsetof(typeof(resp), response_length) +
5462 sizeof(resp.response_length);
5463 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5464 if (err)
5465 goto err_copy;
5466 }
5467
350d0e4c
YH		 5468 rwq->core_qp.event = mlx5_ib_wq_event;
5469 rwq->ibwq.event_handler = init_attr->event_handler;
79b20a6c
YH		 5470 return &rwq->ibwq;
5471
5472err_copy:
350d0e4c 5473 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
79b20a6c 5474err_user_rq:
fe248c3a 5475 destroy_user_rq(dev, pd, rwq);
79b20a6c
YH		 5476err:
5477 kfree(rwq);
5478 return ERR_PTR(err);
5479}
5480
5481int mlx5_ib_destroy_wq(struct ib_wq *wq)
5482{
5483 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5484 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5485
350d0e4c 5486 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
fe248c3a 5487 destroy_user_rq(dev, wq->pd, rwq);
79b20a6c
YH		 5488 kfree(rwq);
5489
5490 return 0;
5491}
5492
c5f90929
YH		 5493struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
5494 struct ib_rwq_ind_table_init_attr *init_attr,
5495 struct ib_udata *udata)
5496{
5497 struct mlx5_ib_dev *dev = to_mdev(device);
5498 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl;
5499 int sz = 1 << init_attr->log_ind_tbl_size;
5500 struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5501 size_t min_resp_len;
5502 int inlen;
5503 int err;
5504 int i;
5505 u32 *in;
5506 void *rqtc;
5507
5508 if (udata->inlen > 0 &&
5509 !ib_is_udata_cleared(udata, 0,
5510 udata->inlen))
5511 return ERR_PTR(-EOPNOTSUPP);
5512
efd7f400
MG		 5513 if (init_attr->log_ind_tbl_size >
5514 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5515 mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5516 init_attr->log_ind_tbl_size,
5517 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5518 return ERR_PTR(-EINVAL);
5519 }
5520
c5f90929
YH		 5521 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
5522 if (udata->outlen && udata->outlen < min_resp_len)
5523 return ERR_PTR(-EINVAL);
5524
5525 rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL);
5526 if (!rwq_ind_tbl)
5527 return ERR_PTR(-ENOMEM);
5528
5529 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1b9a07ee 5530 in = kvzalloc(inlen, GFP_KERNEL);
c5f90929
YH		 5531 if (!in) {
5532 err = -ENOMEM;
5533 goto err;
5534 }
5535
5536 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5537
5538 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5539 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5540
5541 for (i = 0; i < sz; i++)
5542 MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5543
5544 err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5545 kvfree(in);
5546
5547 if (err)
5548 goto err;
5549
5550 rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5551 if (udata->outlen) {
5552 resp.response_length = offsetof(typeof(resp), response_length) +
5553 sizeof(resp.response_length);
5554 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5555 if (err)
5556 goto err_copy;
5557 }
5558
5559 return &rwq_ind_tbl->ib_rwq_ind_tbl;
5560
5561err_copy:
5562 mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
5563err:
5564 kfree(rwq_ind_tbl);
5565 return ERR_PTR(err);
5566}
5567
5568int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5569{
5570 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5571 struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5572
5573 mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
5574
5575 kfree(rwq_ind_tbl);
5576 return 0;
5577}
5578
79b20a6c
YH		 5579int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5580 u32 wq_attr_mask, struct ib_udata *udata)
5581{
5582 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5583 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5584 struct mlx5_ib_modify_wq ucmd = {};
5585 size_t required_cmd_sz;
5586 int curr_wq_state;
5587 int wq_state;
5588 int inlen;
5589 int err;
5590 void *rqc;
5591 void *in;
5592
5593 required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);
5594 if (udata->inlen < required_cmd_sz)
5595 return -EINVAL;
5596
5597 if (udata->inlen > sizeof(ucmd) &&
5598 !ib_is_udata_cleared(udata, sizeof(ucmd),
5599 udata->inlen - sizeof(ucmd)))
5600 return -EOPNOTSUPP;
5601
5602 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5603 return -EFAULT;
5604
5605 if (ucmd.comp_mask || ucmd.reserved)
5606 return -EOPNOTSUPP;
5607
5608 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 5609 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5610 if (!in)
5611 return -ENOMEM;
5612
5613 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5614
5615 curr_wq_state = (wq_attr_mask & IB_WQ_CUR_STATE) ?
5616 wq_attr->curr_wq_state : wq->state;
5617 wq_state = (wq_attr_mask & IB_WQ_STATE) ?
5618 wq_attr->wq_state : curr_wq_state;
5619 if (curr_wq_state == IB_WQS_ERR)
5620 curr_wq_state = MLX5_RQC_STATE_ERR;
5621 if (wq_state == IB_WQS_ERR)
5622 wq_state = MLX5_RQC_STATE_ERR;
5623 MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
5624 MLX5_SET(rqc, rqc, state, wq_state);
5625
b1f74a84
NO		 5626 if (wq_attr_mask & IB_WQ_FLAGS) {
5627 if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5628 if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5629 MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5630 mlx5_ib_dbg(dev, "VLAN offloads are not "
5631 "supported\n");
5632 err = -EOPNOTSUPP;
5633 goto out;
5634 }
5635 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5636 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5637 MLX5_SET(rqc, rqc, vsd,
5638 (wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5639 }
b1383aa6
NO		 5640
5641 if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5642 mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5643 err = -EOPNOTSUPP;
5644 goto out;
5645 }
b1f74a84
NO		 5646 }
5647
23a6964e
MD		 5648 if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
5649 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5650 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5651 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
e1f24a79
PP		 5652 MLX5_SET(rqc, rqc, counter_set_id,
5653 dev->port->cnts.set_id);
23a6964e
MD		 5654 } else
5655 pr_info_once("%s: Receive WQ counters are not supported on current FW\n",
5656 dev->ib_dev.name);
5657 }
5658
350d0e4c 5659 err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in, inlen);
79b20a6c
YH		 5660 if (!err)
5661 rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5662
b1f74a84
NO		 5663out:
5664 kvfree(in);
79b20a6c
YH		 5665 return err;
5666}
Ubuntu Eoan Linux kernel mirror