]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/infiniband/hw/mlx5/wr.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: Remove uninitialized_var() usage
[mirror_ubuntu-hirsute-kernel.git] / drivers / infiniband / hw / mlx5 / wr.c
CommitLineData
029e88fd
LR		 1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <linux/gfp.h>
7#include <linux/mlx5/qp.h>
8#include <linux/mlx5/driver.h>
9#include "wr.h"
10
11static const u32 mlx5_ib_opcode[] = {
12 [IB_WR_SEND] = MLX5_OPCODE_SEND,
13 [IB_WR_LSO] = MLX5_OPCODE_LSO,
14 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
15 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
16 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
17 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
18 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
19 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
20 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
21 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
22 [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
23 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
24 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
25 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
26};
27
28/* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
29 * next nearby edge and get new address translation for current WQE position.
30 * @sq - SQ buffer.
31 * @seg: Current WQE position (16B aligned).
32 * @wqe_sz: Total current WQE size [16B].
33 * @cur_edge: Updated current edge.
34 */
35static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
36 u32 wqe_sz, void **cur_edge)
37{
38 u32 idx;
39
40 if (likely(*seg != *cur_edge))
41 return;
42
43 idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
44 *cur_edge = get_sq_edge(sq, idx);
45
46 *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
47}
48
49/* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's
50 * pointers. At the end @seg is aligned to 16B regardless the copied size.
51 * @sq - SQ buffer.
52 * @cur_edge: Updated current edge.
53 * @seg: Current WQE position (16B aligned).
54 * @wqe_sz: Total current WQE size [16B].
55 * @src: Pointer to copy from.
56 * @n: Number of bytes to copy.
57 */
58static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
59 void **seg, u32 *wqe_sz, const void *src,
60 size_t n)
61{
62 while (likely(n)) {
63 size_t leftlen = *cur_edge - *seg;
64 size_t copysz = min_t(size_t, leftlen, n);
65 size_t stride;
66
67 memcpy(*seg, src, copysz);
68
69 n -= copysz;
70 src += copysz;
71 stride = !n ? ALIGN(copysz, 16) : copysz;
72 *seg += stride;
73 *wqe_sz += stride >> 4;
74 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
75 }
76}
77
78static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq,
79 struct ib_cq *ib_cq)
80{
81 struct mlx5_ib_cq *cq;
82 unsigned int cur;
83
84 cur = wq->head - wq->tail;
85 if (likely(cur + nreq < wq->max_post))
86 return 0;
87
88 cq = to_mcq(ib_cq);
89 spin_lock(&cq->lock);
90 cur = wq->head - wq->tail;
91 spin_unlock(&cq->lock);
92
93 return cur + nreq >= wq->max_post;
94}
95
96static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
97 u64 remote_addr, u32 rkey)
98{
99 rseg->raddr = cpu_to_be64(remote_addr);
100 rseg->rkey = cpu_to_be32(rkey);
101 rseg->reserved = 0;
102}
103
104static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
105 void **seg, int *size, void **cur_edge)
106{
107 struct mlx5_wqe_eth_seg *eseg = *seg;
108
109 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
110
111 if (wr->send_flags & IB_SEND_IP_CSUM)
112 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
113 MLX5_ETH_WQE_L4_CSUM;
114
115 if (wr->opcode == IB_WR_LSO) {
116 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
117 size_t left, copysz;
118 void *pdata = ud_wr->header;
119 size_t stride;
120
121 left = ud_wr->hlen;
122 eseg->mss = cpu_to_be16(ud_wr->mss);
123 eseg->inline_hdr.sz = cpu_to_be16(left);
124
125 /* memcpy_send_wqe should get a 16B align address. Hence, we
126 * first copy up to the current edge and then, if needed,
127 * continue to memcpy_send_wqe.
128 */
129 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
130 left);
131 memcpy(eseg->inline_hdr.start, pdata, copysz);
132 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
133 sizeof(eseg->inline_hdr.start) + copysz, 16);
134 *size += stride / 16;
135 *seg += stride;
136
137 if (copysz < left) {
138 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
139 left -= copysz;
140 pdata += copysz;
141 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
142 left);
143 }
144
145 return;
146 }
147
148 *seg += sizeof(struct mlx5_wqe_eth_seg);
149 *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
150}
151
152static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
153 const struct ib_send_wr *wr)
154{
155 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
156 dseg->av.dqp_dct =
157 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
158 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
159}
160
161static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
162{
163 dseg->byte_count = cpu_to_be32(sg->length);
164 dseg->lkey = cpu_to_be32(sg->lkey);
165 dseg->addr = cpu_to_be64(sg->addr);
166}
167
168static u64 get_xlt_octo(u64 bytes)
169{
170 return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
171 MLX5_IB_UMR_OCTOWORD;
172}
173
174static __be64 frwr_mkey_mask(bool atomic)
175{
176 u64 result;
177
178 result = MLX5_MKEY_MASK_LEN |
179 MLX5_MKEY_MASK_PAGE_SIZE |
180 MLX5_MKEY_MASK_START_ADDR |
181 MLX5_MKEY_MASK_EN_RINVAL |
182 MLX5_MKEY_MASK_KEY |
183 MLX5_MKEY_MASK_LR |
184 MLX5_MKEY_MASK_LW |
185 MLX5_MKEY_MASK_RR |
186 MLX5_MKEY_MASK_RW |
187 MLX5_MKEY_MASK_SMALL_FENCE |
188 MLX5_MKEY_MASK_FREE;
189
190 if (atomic)
191 result |= MLX5_MKEY_MASK_A;
192
193 return cpu_to_be64(result);
194}
195
196static __be64 sig_mkey_mask(void)
197{
198 u64 result;
199
200 result = MLX5_MKEY_MASK_LEN |
201 MLX5_MKEY_MASK_PAGE_SIZE |
202 MLX5_MKEY_MASK_START_ADDR |
203 MLX5_MKEY_MASK_EN_SIGERR |
204 MLX5_MKEY_MASK_EN_RINVAL |
205 MLX5_MKEY_MASK_KEY |
206 MLX5_MKEY_MASK_LR |
207 MLX5_MKEY_MASK_LW |
208 MLX5_MKEY_MASK_RR |
209 MLX5_MKEY_MASK_RW |
210 MLX5_MKEY_MASK_SMALL_FENCE |
211 MLX5_MKEY_MASK_FREE |
212 MLX5_MKEY_MASK_BSF_EN;
213
214 return cpu_to_be64(result);
215}
216
217static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
218 struct mlx5_ib_mr *mr, u8 flags, bool atomic)
219{
220 int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
221
222 memset(umr, 0, sizeof(*umr));
223
224 umr->flags = flags;
225 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
226 umr->mkey_mask = frwr_mkey_mask(atomic);
227}
228
229static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
230{
231 memset(umr, 0, sizeof(*umr));
232 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
233 umr->flags = MLX5_UMR_INLINE;
234}
235
236static __be64 get_umr_enable_mr_mask(void)
237{
238 u64 result;
239
240 result = MLX5_MKEY_MASK_KEY |
241 MLX5_MKEY_MASK_FREE;
242
243 return cpu_to_be64(result);
244}
245
246static __be64 get_umr_disable_mr_mask(void)
247{
248 u64 result;
249
250 result = MLX5_MKEY_MASK_FREE;
251
252 return cpu_to_be64(result);
253}
254
255static __be64 get_umr_update_translation_mask(void)
256{
257 u64 result;
258
259 result = MLX5_MKEY_MASK_LEN |
260 MLX5_MKEY_MASK_PAGE_SIZE |
261 MLX5_MKEY_MASK_START_ADDR;
262
263 return cpu_to_be64(result);
264}
265
266static __be64 get_umr_update_access_mask(int atomic)
267{
268 u64 result;
269
270 result = MLX5_MKEY_MASK_LR |
271 MLX5_MKEY_MASK_LW |
272 MLX5_MKEY_MASK_RR |
273 MLX5_MKEY_MASK_RW;
274
275 if (atomic)
276 result |= MLX5_MKEY_MASK_A;
277
278 return cpu_to_be64(result);
279}
280
281static __be64 get_umr_update_pd_mask(void)
282{
283 u64 result;
284
285 result = MLX5_MKEY_MASK_PD;
286
287 return cpu_to_be64(result);
288}
289
290static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
291{
292 if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
293 MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
294 (mask & MLX5_MKEY_MASK_A &&
295 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
296 return -EPERM;
297 return 0;
298}
299
300static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
301 struct mlx5_wqe_umr_ctrl_seg *umr,
302 const struct ib_send_wr *wr, int atomic)
303{
304 const struct mlx5_umr_wr *umrwr = umr_wr(wr);
305
306 memset(umr, 0, sizeof(*umr));
307
308 if (!umrwr->ignore_free_state) {
309 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
310 /* fail if free */
311 umr->flags = MLX5_UMR_CHECK_FREE;
312 else
313 /* fail if not free */
314 umr->flags = MLX5_UMR_CHECK_NOT_FREE;
315 }
316
317 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
318 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
319 u64 offset = get_xlt_octo(umrwr->offset);
320
321 umr->xlt_offset = cpu_to_be16(offset & 0xffff);
322 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
323 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
324 }
325 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
326 umr->mkey_mask |= get_umr_update_translation_mask();
327 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
328 umr->mkey_mask |= get_umr_update_access_mask(atomic);
329 umr->mkey_mask |= get_umr_update_pd_mask();
330 }
331 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
332 umr->mkey_mask |= get_umr_enable_mr_mask();
333 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
334 umr->mkey_mask |= get_umr_disable_mr_mask();
335
336 if (!wr->num_sge)
337 umr->flags |= MLX5_UMR_INLINE;
338
339 return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
340}
341
342static u8 get_umr_flags(int acc)
343{
344 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
345 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
346 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
347 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
348 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
349}
350
351static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
352 struct mlx5_ib_mr *mr,
353 u32 key, int access)
354{
355 int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
356
357 memset(seg, 0, sizeof(*seg));
358
359 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
360 seg->log2_page_size = ilog2(mr->ibmr.page_size);
361 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
362 /* KLMs take twice the size of MTTs */
363 ndescs *= 2;
364
365 seg->flags = get_umr_flags(access) | mr->access_mode;
366 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
367 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
368 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
369 seg->len = cpu_to_be64(mr->ibmr.length);
370 seg->xlt_oct_size = cpu_to_be32(ndescs);
371}
372
373static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
374{
375 memset(seg, 0, sizeof(*seg));
376 seg->status = MLX5_MKEY_STATUS_FREE;
377}
378
379static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,
380 const struct ib_send_wr *wr)
381{
382 const struct mlx5_umr_wr *umrwr = umr_wr(wr);
383
384 memset(seg, 0, sizeof(*seg));
385 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
386 seg->status = MLX5_MKEY_STATUS_FREE;
387
388 seg->flags = convert_access(umrwr->access_flags);
389 if (umrwr->pd)
390 seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
391 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
392 !umrwr->length)
393 seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
394
395 seg->start_addr = cpu_to_be64(umrwr->virt_addr);
396 seg->len = cpu_to_be64(umrwr->length);
397 seg->log2_page_size = umrwr->page_shift;
398 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
399 mlx5_mkey_variant(umrwr->mkey));
400}
401
402static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
403 struct mlx5_ib_mr *mr,
404 struct mlx5_ib_pd *pd)
405{
406 int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
407
408 dseg->addr = cpu_to_be64(mr->desc_map);
409 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
410 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
411}
412
413static __be32 send_ieth(const struct ib_send_wr *wr)
414{
415 switch (wr->opcode) {
416 case IB_WR_SEND_WITH_IMM:
417 case IB_WR_RDMA_WRITE_WITH_IMM:
418 return wr->ex.imm_data;
419
420 case IB_WR_SEND_WITH_INV:
421 return cpu_to_be32(wr->ex.invalidate_rkey);
422
423 default:
424 return 0;
425 }
426}
427
428static u8 calc_sig(void *wqe, int size)
429{
430 u8 *p = wqe;
431 u8 res = 0;
432 int i;
433
434 for (i = 0; i < size; i++)
435 res ^= p[i];
436
437 return ~res;
438}
439
440static u8 wq_sig(void *wqe)
441{
442 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
443}
444
445static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
446 void **wqe, int *wqe_sz, void **cur_edge)
447{
448 struct mlx5_wqe_inline_seg *seg;
449 size_t offset;
450 int inl = 0;
451 int i;
452
453 seg = *wqe;
454 *wqe += sizeof(*seg);
455 offset = sizeof(*seg);
456
457 for (i = 0; i < wr->num_sge; i++) {
458 size_t len = wr->sg_list[i].length;
459 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
460
461 inl += len;
462
463 if (unlikely(inl > qp->max_inline_data))
464 return -ENOMEM;
465
466 while (likely(len)) {
467 size_t leftlen;
468 size_t copysz;
469
470 handle_post_send_edge(&qp->sq, wqe,
471 *wqe_sz + (offset >> 4),
472 cur_edge);
473
474 leftlen = *cur_edge - *wqe;
475 copysz = min_t(size_t, leftlen, len);
476
477 memcpy(*wqe, addr, copysz);
478 len -= copysz;
479 addr += copysz;
480 *wqe += copysz;
481 offset += copysz;
482 }
483 }
484
485 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
486
487 *wqe_sz += ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
488
489 return 0;
490}
491
492static u16 prot_field_size(enum ib_signature_type type)
493{
494 switch (type) {
495 case IB_SIG_TYPE_T10_DIF:
496 return MLX5_DIF_SIZE;
497 default:
498 return 0;
499 }
500}
501
502static u8 bs_selector(int block_size)
503{
504 switch (block_size) {
505 case 512: return 0x1;
506 case 520: return 0x2;
507 case 4096: return 0x3;
508 case 4160: return 0x4;
509 case 1073741824: return 0x5;
510 default: return 0;
511 }
512}
513
514static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
515 struct mlx5_bsf_inl *inl)
516{
517 /* Valid inline section and allow BSF refresh */
518 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
519 MLX5_BSF_REFRESH_DIF);
520 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
521 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
522 /* repeating block */
523 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
524 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
525 MLX5_DIF_CRC : MLX5_DIF_IPCS;
526
527 if (domain->sig.dif.ref_remap)
528 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
529
530 if (domain->sig.dif.app_escape) {
531 if (domain->sig.dif.ref_escape)
532 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
533 else
534 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
535 }
536
537 inl->dif_app_bitmask_check =
538 cpu_to_be16(domain->sig.dif.apptag_check_mask);
539}
540
541static int mlx5_set_bsf(struct ib_mr *sig_mr,
542 struct ib_sig_attrs *sig_attrs,
543 struct mlx5_bsf *bsf, u32 data_size)
544{
545 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
546 struct mlx5_bsf_basic *basic = &bsf->basic;
547 struct ib_sig_domain *mem = &sig_attrs->mem;
548 struct ib_sig_domain *wire = &sig_attrs->wire;
549
550 memset(bsf, 0, sizeof(*bsf));
551
552 /* Basic + Extended + Inline */
553 basic->bsf_size_sbs = 1 << 7;
554 /* Input domain check byte mask */
555 basic->check_byte_mask = sig_attrs->check_mask;
556 basic->raw_data_size = cpu_to_be32(data_size);
557
558 /* Memory domain */
559 switch (sig_attrs->mem.sig_type) {
560 case IB_SIG_TYPE_NONE:
561 break;
562 case IB_SIG_TYPE_T10_DIF:
563 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
564 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
565 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
566 break;
567 default:
568 return -EINVAL;
569 }
570
571 /* Wire domain */
572 switch (sig_attrs->wire.sig_type) {
573 case IB_SIG_TYPE_NONE:
574 break;
575 case IB_SIG_TYPE_T10_DIF:
576 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
577 mem->sig_type == wire->sig_type) {
578 /* Same block structure */
579 basic->bsf_size_sbs |= 1 << 4;
580 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
581 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
582 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
583 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
584 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
585 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
586 } else
587 basic->wire.bs_selector =
588 bs_selector(wire->sig.dif.pi_interval);
589
590 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
591 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
592 break;
593 default:
594 return -EINVAL;
595 }
596
597 return 0;
598}
599
600
601static int set_sig_data_segment(const struct ib_send_wr *send_wr,
602 struct ib_mr *sig_mr,
603 struct ib_sig_attrs *sig_attrs,
604 struct mlx5_ib_qp *qp, void **seg, int *size,
605 void **cur_edge)
606{
607 struct mlx5_bsf *bsf;
608 u32 data_len;
609 u32 data_key;
610 u64 data_va;
611 u32 prot_len = 0;
612 u32 prot_key = 0;
613 u64 prot_va = 0;
614 bool prot = false;
615 int ret;
616 int wqe_size;
617 struct mlx5_ib_mr *mr = to_mmr(sig_mr);
618 struct mlx5_ib_mr *pi_mr = mr->pi_mr;
619
620 data_len = pi_mr->data_length;
621 data_key = pi_mr->ibmr.lkey;
622 data_va = pi_mr->data_iova;
623 if (pi_mr->meta_ndescs) {
624 prot_len = pi_mr->meta_length;
625 prot_key = pi_mr->ibmr.lkey;
626 prot_va = pi_mr->pi_iova;
627 prot = true;
628 }
629
630 if (!prot || (data_key == prot_key && data_va == prot_va &&
631 data_len == prot_len)) {
632 /**
633 * Source domain doesn't contain signature information
634 * or data and protection are interleaved in memory.
635 * So need construct:
636 * ------------------
637 * | data_klm |
638 * ------------------
639 * | BSF |
640 * ------------------
641 **/
642 struct mlx5_klm *data_klm = *seg;
643
644 data_klm->bcount = cpu_to_be32(data_len);
645 data_klm->key = cpu_to_be32(data_key);
646 data_klm->va = cpu_to_be64(data_va);
647 wqe_size = ALIGN(sizeof(*data_klm), 64);
648 } else {
649 /**
650 * Source domain contains signature information
651 * So need construct a strided block format:
652 * ---------------------------
653 * | stride_block_ctrl |
654 * ---------------------------
655 * | data_klm |
656 * ---------------------------
657 * | prot_klm |
658 * ---------------------------
659 * | BSF |
660 * ---------------------------
661 **/
662 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
663 struct mlx5_stride_block_entry *data_sentry;
664 struct mlx5_stride_block_entry *prot_sentry;
665 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
666 int prot_size;
667
668 sblock_ctrl = *seg;
669 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
670 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
671
672 prot_size = prot_field_size(sig_attrs->mem.sig_type);
673 if (!prot_size) {
674 pr_err("Bad block size given: %u\n", block_size);
675 return -EINVAL;
676 }
677 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
678 prot_size);
679 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
680 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
681 sblock_ctrl->num_entries = cpu_to_be16(2);
682
683 data_sentry->bcount = cpu_to_be16(block_size);
684 data_sentry->key = cpu_to_be32(data_key);
685 data_sentry->va = cpu_to_be64(data_va);
686 data_sentry->stride = cpu_to_be16(block_size);
687
688 prot_sentry->bcount = cpu_to_be16(prot_size);
689 prot_sentry->key = cpu_to_be32(prot_key);
690 prot_sentry->va = cpu_to_be64(prot_va);
691 prot_sentry->stride = cpu_to_be16(prot_size);
692
693 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
694 sizeof(*prot_sentry), 64);
695 }
696
697 *seg += wqe_size;
698 *size += wqe_size / 16;
699 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
700
701 bsf = *seg;
702 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
703 if (ret)
704 return -EINVAL;
705
706 *seg += sizeof(*bsf);
707 *size += sizeof(*bsf) / 16;
708 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
709
710 return 0;
711}
712
713static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
714 struct ib_mr *sig_mr, int access_flags,
715 u32 size, u32 length, u32 pdn)
716{
717 u32 sig_key = sig_mr->rkey;
718 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
719
720 memset(seg, 0, sizeof(*seg));
721
722 seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
723 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
724 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
725 MLX5_MKEY_BSF_EN | pdn);
726 seg->len = cpu_to_be64(length);
727 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
728 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
729}
730
731static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
732 u32 size)
733{
734 memset(umr, 0, sizeof(*umr));
735
736 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
737 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
738 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
739 umr->mkey_mask = sig_mkey_mask();
740}
741
742static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
743 struct mlx5_ib_qp *qp, void **seg, int *size,
744 void **cur_edge)
745{
746 const struct ib_reg_wr *wr = reg_wr(send_wr);
747 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
748 struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
749 struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
750 u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
751 u32 xlt_size;
752 int region_len, ret;
753
754 if (unlikely(send_wr->num_sge != 0) ||
755 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
756 unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
757 unlikely(!sig_mr->sig->sig_status_checked))
758 return -EINVAL;
759
760 /* length of the protected region, data + protection */
761 region_len = pi_mr->ibmr.length;
762
763 /**
764 * KLM octoword size - if protection was provided
765 * then we use strided block format (3 octowords),
766 * else we use single KLM (1 octoword)
767 **/
768 if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
769 xlt_size = 0x30;
770 else
771 xlt_size = sizeof(struct mlx5_klm);
772
773 set_sig_umr_segment(*seg, xlt_size);
774 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
775 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
776 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
777
778 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
779 pdn);
780 *seg += sizeof(struct mlx5_mkey_seg);
781 *size += sizeof(struct mlx5_mkey_seg) / 16;
782 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
783
784 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
785 cur_edge);
786 if (ret)
787 return ret;
788
789 sig_mr->sig->sig_status_checked = false;
790 return 0;
791}
792
793static int set_psv_wr(struct ib_sig_domain *domain,
794 u32 psv_idx, void **seg, int *size)
795{
796 struct mlx5_seg_set_psv *psv_seg = *seg;
797
798 memset(psv_seg, 0, sizeof(*psv_seg));
799 psv_seg->psv_num = cpu_to_be32(psv_idx);
800 switch (domain->sig_type) {
801 case IB_SIG_TYPE_NONE:
802 break;
803 case IB_SIG_TYPE_T10_DIF:
804 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
805 domain->sig.dif.app_tag);
806 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
807 break;
808 default:
809 pr_err("Bad signature type (%d) is given.\n",
810 domain->sig_type);
811 return -EINVAL;
812 }
813
814 *seg += sizeof(*psv_seg);
815 *size += sizeof(*psv_seg) / 16;
816
817 return 0;
818}
819
820static int set_reg_wr(struct mlx5_ib_qp *qp,
821 const struct ib_reg_wr *wr,
822 void **seg, int *size, void **cur_edge,
823 bool check_not_free)
824{
825 struct mlx5_ib_mr *mr = to_mmr(wr->mr);
826 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
827 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
828 int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
829 bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
830 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
831 u8 flags = 0;
832
833 if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) {
834 mlx5_ib_warn(to_mdev(qp->ibqp.device),
835 "Fast update of %s for MR is disabled\n",
836 (MLX5_CAP_GEN(dev->mdev,
837 umr_modify_entity_size_disabled)) ?
838 "entity size" :
839 "atomic access");
840 return -EINVAL;
841 }
842
843 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
844 mlx5_ib_warn(to_mdev(qp->ibqp.device),
845 "Invalid IB_SEND_INLINE send flag\n");
846 return -EINVAL;
847 }
848
849 if (check_not_free)
850 flags |= MLX5_UMR_CHECK_NOT_FREE;
851 if (umr_inline)
852 flags |= MLX5_UMR_INLINE;
853
854 set_reg_umr_seg(*seg, mr, flags, atomic);
855 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
856 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
857 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
858
859 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
860 *seg += sizeof(struct mlx5_mkey_seg);
861 *size += sizeof(struct mlx5_mkey_seg) / 16;
862 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
863
864 if (umr_inline) {
865 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
866 mr_list_size);
867 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
868 } else {
869 set_reg_data_seg(*seg, mr, pd);
870 *seg += sizeof(struct mlx5_wqe_data_seg);
871 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
872 }
873 return 0;
874}
875
876static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
877 void **cur_edge)
878{
879 set_linv_umr_seg(*seg);
880 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
881 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
882 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
883 set_linv_mkey_seg(*seg);
884 *seg += sizeof(struct mlx5_mkey_seg);
885 *size += sizeof(struct mlx5_mkey_seg) / 16;
886 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
887}
888
889static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
890{
891 __be32 *p = NULL;
892 int i, j;
893
894 pr_debug("dump WQE index %u:\n", idx);
895 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
896 if ((i & 0xf) == 0) {
897 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
898 pr_debug("WQBB at %p:\n", (void *)p);
899 j = 0;
900 idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
901 }
902 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
903 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
904 be32_to_cpu(p[j + 3]));
905 }
906}
907
908static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
909 struct mlx5_wqe_ctrl_seg **ctrl,
910 const struct ib_send_wr *wr, unsigned int *idx,
911 int *size, void **cur_edge, int nreq,
912 bool send_signaled, bool solicited)
913{
914 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
915 return -ENOMEM;
916
917 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
918 *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
919 *ctrl = *seg;
920 *(uint32_t *)(*seg + 8) = 0;
921 (*ctrl)->imm = send_ieth(wr);
922 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
923 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
924 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
925
926 *seg += sizeof(**ctrl);
927 *size = sizeof(**ctrl) / 16;
928 *cur_edge = qp->sq.cur_edge;
929
930 return 0;
931}
932
933static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
934 struct mlx5_wqe_ctrl_seg **ctrl,
935 const struct ib_send_wr *wr, unsigned int *idx, int *size,
936 void **cur_edge, int nreq)
937{
938 return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
939 wr->send_flags & IB_SEND_SIGNALED,
940 wr->send_flags & IB_SEND_SOLICITED);
941}
942
943static void finish_wqe(struct mlx5_ib_qp *qp,
944 struct mlx5_wqe_ctrl_seg *ctrl,
945 void *seg, u8 size, void *cur_edge,
946 unsigned int idx, u64 wr_id, int nreq, u8 fence,
947 u32 mlx5_opcode)
948{
949 u8 opmod = 0;
950
951 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
952 mlx5_opcode | ((u32)opmod << 24));
953 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
954 ctrl->fm_ce_se |= fence;
955 if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
956 ctrl->signature = wq_sig(ctrl);
957
958 qp->sq.wrid[idx] = wr_id;
959 qp->sq.w_list[idx].opcode = mlx5_opcode;
960 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
961 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
962 qp->sq.w_list[idx].next = qp->sq.cur_post;
963
964 /* We save the edge which was possibly updated during the WQE
965 * construction, into SQ's cache.
966 */
967 seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
968 qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
969 get_sq_edge(&qp->sq, qp->sq.cur_post &
970 (qp->sq.wqe_cnt - 1)) :
971 cur_edge;
972}
973
974static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
975{
976 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
977 *seg += sizeof(struct mlx5_wqe_raddr_seg);
978 *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
979}
980
981static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
982 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
983 int *size, void **cur_edge, unsigned int idx)
984{
985 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
986 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
987 set_linv_wr(qp, seg, size, cur_edge);
988}
989
990static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
991 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
992 void **cur_edge, unsigned int idx)
993{
994 qp->sq.wr_data[idx] = IB_WR_REG_MR;
995 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
996 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
997}
998
999static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1000 const struct ib_send_wr *wr,
1001 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1002 void **cur_edge, unsigned int *idx, int nreq,
1003 struct ib_sig_domain *domain, u32 psv_index,
1004 u8 next_fence)
1005{
1006 int err;
1007
1008 /*
1009 * SET_PSV WQEs are not signaled and solicited on error.
1010 */
1011 err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
1012 false, true);
1013 if (unlikely(err)) {
1014 mlx5_ib_warn(dev, "\n");
1015 err = -ENOMEM;
1016 goto out;
1017 }
1018 err = set_psv_wr(domain, psv_index, seg, size);
1019 if (unlikely(err)) {
1020 mlx5_ib_warn(dev, "\n");
1021 goto out;
1022 }
1023 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1024 next_fence, MLX5_OPCODE_SET_PSV);
1025
1026out:
1027 return err;
1028}
1029
1030static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
1031 struct mlx5_ib_qp *qp,
1032 const struct ib_send_wr *wr,
1033 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1034 int *size, void **cur_edge,
1035 unsigned int *idx, int nreq, u8 fence,
1036 u8 next_fence)
1037{
1038 struct mlx5_ib_mr *mr;
1039 struct mlx5_ib_mr *pi_mr;
1040 struct mlx5_ib_mr pa_pi_mr;
1041 struct ib_sig_attrs *sig_attrs;
1042 struct ib_reg_wr reg_pi_wr;
1043 int err;
1044
1045 qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
1046
1047 mr = to_mmr(reg_wr(wr)->mr);
1048 pi_mr = mr->pi_mr;
1049
1050 if (pi_mr) {
1051 memset(&reg_pi_wr, 0,
1052 sizeof(struct ib_reg_wr));
1053
1054 reg_pi_wr.mr = &pi_mr->ibmr;
1055 reg_pi_wr.access = reg_wr(wr)->access;
1056 reg_pi_wr.key = pi_mr->ibmr.rkey;
1057
1058 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
1059 /* UMR for data + prot registration */
1060 err = set_reg_wr(qp, &reg_pi_wr, seg, size, cur_edge, false);
1061 if (unlikely(err))
1062 goto out;
1063
1064 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
1065 nreq, fence, MLX5_OPCODE_UMR);
1066
1067 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
1068 if (unlikely(err)) {
1069 mlx5_ib_warn(dev, "\n");
1070 err = -ENOMEM;
1071 goto out;
1072 }
1073 } else {
1074 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
1075 /* No UMR, use local_dma_lkey */
1076 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
1077 pa_pi_mr.ndescs = mr->ndescs;
1078 pa_pi_mr.data_length = mr->data_length;
1079 pa_pi_mr.data_iova = mr->data_iova;
1080 if (mr->meta_ndescs) {
1081 pa_pi_mr.meta_ndescs = mr->meta_ndescs;
1082 pa_pi_mr.meta_length = mr->meta_length;
1083 pa_pi_mr.pi_iova = mr->pi_iova;
1084 }
1085
1086 pa_pi_mr.ibmr.length = mr->ibmr.length;
1087 mr->pi_mr = &pa_pi_mr;
1088 }
1089 (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
1090 /* UMR for sig MR */
1091 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
1092 if (unlikely(err)) {
1093 mlx5_ib_warn(dev, "\n");
1094 goto out;
1095 }
1096 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1097 fence, MLX5_OPCODE_UMR);
1098
1099 sig_attrs = mr->ibmr.sig_attrs;
1100 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1101 &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
1102 next_fence);
1103 if (unlikely(err))
1104 goto out;
1105
1106 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1107 &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
1108 next_fence);
1109 if (unlikely(err))
1110 goto out;
1111
1112 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1113
1114out:
1115 return err;
1116}
1117
1118static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1119 const struct ib_send_wr *wr,
1120 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1121 void **cur_edge, unsigned int *idx, int nreq, u8 fence,
1122 u8 next_fence, int *num_sge)
1123{
1124 int err = 0;
1125
1126 switch (wr->opcode) {
1127 case IB_WR_RDMA_READ:
1128 case IB_WR_RDMA_WRITE:
1129 case IB_WR_RDMA_WRITE_WITH_IMM:
1130 handle_rdma_op(wr, seg, size);
1131 break;
1132
1133 case IB_WR_ATOMIC_CMP_AND_SWP:
1134 case IB_WR_ATOMIC_FETCH_AND_ADD:
1135 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
1136 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
1137 err = -EOPNOTSUPP;
1138 goto out;
1139
1140 case IB_WR_LOCAL_INV:
1141 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
1142 *num_sge = 0;
1143 break;
1144
1145 case IB_WR_REG_MR:
1146 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
1147 if (unlikely(err))
1148 goto out;
1149 *num_sge = 0;
1150 break;
1151
1152 case IB_WR_REG_MR_INTEGRITY:
1153 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
1154 cur_edge, idx, nreq, fence,
1155 next_fence);
1156 if (unlikely(err))
1157 goto out;
1158 *num_sge = 0;
1159 break;
1160
1161 default:
1162 break;
1163 }
1164
1165out:
1166 return err;
1167}
1168
1169static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
1170{
1171 switch (wr->opcode) {
1172 case IB_WR_RDMA_WRITE:
1173 case IB_WR_RDMA_WRITE_WITH_IMM:
1174 handle_rdma_op(wr, seg, size);
1175 break;
1176 default:
1177 break;
1178 }
1179}
1180
1181static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
1182 const struct ib_send_wr *wr, void **seg,
1183 int *size, void **cur_edge)
1184{
1185 set_datagram_seg(*seg, wr);
1186 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1187 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1188 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1189}
1190
1191static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1192 void **seg, int *size, void **cur_edge)
1193{
1194 set_datagram_seg(*seg, wr);
1195 *seg += sizeof(struct mlx5_wqe_datagram_seg);
1196 *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1197 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1198
1199 /* handle qp that supports ud offload */
1200 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1201 struct mlx5_wqe_eth_pad *pad;
1202
1203 pad = *seg;
1204 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1205 *seg += sizeof(struct mlx5_wqe_eth_pad);
1206 *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1207 set_eth_seg(wr, qp, seg, size, cur_edge);
1208 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1209 }
1210}
1211
1212static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1213 const struct ib_send_wr *wr,
1214 struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1215 int *size, void **cur_edge, unsigned int idx)
1216{
1217 int err = 0;
1218
1219 if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) {
1220 err = -EINVAL;
1221 mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode);
1222 goto out;
1223 }
1224
1225 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
1226 (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
1227 err = set_reg_umr_segment(dev, *seg, wr,
1228 !!(MLX5_CAP_GEN(dev->mdev, atomic)));
1229 if (unlikely(err))
1230 goto out;
1231 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
1232 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
1233 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1234 set_reg_mkey_segment(*seg, wr);
1235 *seg += sizeof(struct mlx5_mkey_seg);
1236 *size += sizeof(struct mlx5_mkey_seg) / 16;
1237 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1238out:
1239 return err;
1240}
1241
1242int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1243 const struct ib_send_wr **bad_wr, bool drain)
1244{
1245 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
1246 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1247 struct mlx5_core_dev *mdev = dev->mdev;
1248 struct mlx5_ib_qp *qp;
1249 struct mlx5_wqe_xrc_seg *xrc;
1250 struct mlx5_bf *bf;
1251 void *cur_edge;
3f649ab7 1252 int size;
029e88fd
LR		 1253 unsigned long flags;
1254 unsigned int idx;
1255 int err = 0;
1256 int num_sge;
1257 void *seg;
1258 int nreq;
1259 int i;
1260 u8 next_fence = 0;
1261 u8 fence;
1262
1263 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1264 !drain)) {
1265 *bad_wr = wr;
1266 return -EIO;
1267 }
1268
1269 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1270 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1271
1272 qp = to_mqp(ibqp);
1273 bf = &qp->bf;
1274
1275 spin_lock_irqsave(&qp->sq.lock, flags);
1276
1277 for (nreq = 0; wr; nreq++, wr = wr->next) {
1278 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1279 mlx5_ib_warn(dev, "\n");
1280 err = -EINVAL;
1281 *bad_wr = wr;
1282 goto out;
1283 }
1284
1285 num_sge = wr->num_sge;
1286 if (unlikely(num_sge > qp->sq.max_gs)) {
1287 mlx5_ib_warn(dev, "\n");
1288 err = -EINVAL;
1289 *bad_wr = wr;
1290 goto out;
1291 }
1292
1293 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1294 nreq);
1295 if (err) {
1296 mlx5_ib_warn(dev, "\n");
1297 err = -ENOMEM;
1298 *bad_wr = wr;
1299 goto out;
1300 }
1301
1302 if (wr->opcode == IB_WR_REG_MR ||
1303 wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1304 fence = dev->umr_fence;
1305 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1306 } else {
1307 if (wr->send_flags & IB_SEND_FENCE) {
1308 if (qp->next_fence)
1309 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1310 else
1311 fence = MLX5_FENCE_MODE_FENCE;
1312 } else {
1313 fence = qp->next_fence;
1314 }
1315 }
1316
1317 switch (ibqp->qp_type) {
1318 case IB_QPT_XRC_INI:
1319 xrc = seg;
1320 seg += sizeof(*xrc);
1321 size += sizeof(*xrc) / 16;
1322 fallthrough;
1323 case IB_QPT_RC:
1324 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1325 &cur_edge, &idx, nreq, fence,
1326 next_fence, &num_sge);
1327 if (unlikely(err)) {
1328 *bad_wr = wr;
1329 goto out;
1330 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1331 goto skip_psv;
1332 }
1333 break;
1334
1335 case IB_QPT_UC:
1336 handle_qpt_uc(wr, &seg, &size);
1337 break;
1338 case IB_QPT_SMI:
1339 if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
1340 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1341 err = -EPERM;
1342 *bad_wr = wr;
1343 goto out;
1344 }
1345 fallthrough;
1346 case MLX5_IB_QPT_HW_GSI:
1347 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1348 break;
1349 case IB_QPT_UD:
1350 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1351 break;
1352 case MLX5_IB_QPT_REG_UMR:
1353 err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg,
1354 &size, &cur_edge, idx);
1355 if (unlikely(err))
1356 goto out;
1357 break;
1358
1359 default:
1360 break;
1361 }
1362
1363 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1364 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1365 if (unlikely(err)) {
1366 mlx5_ib_warn(dev, "\n");
1367 *bad_wr = wr;
1368 goto out;
1369 }
1370 } else {
1371 for (i = 0; i < num_sge; i++) {
1372 handle_post_send_edge(&qp->sq, &seg, size,
1373 &cur_edge);
1374 if (unlikely(!wr->sg_list[i].length))
1375 continue;
1376
1377 set_data_ptr_seg(
1378 (struct mlx5_wqe_data_seg *)seg,
1379 wr->sg_list + i);
1380 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1381 seg += sizeof(struct mlx5_wqe_data_seg);
1382 }
1383 }
1384
1385 qp->next_fence = next_fence;
1386 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,
1387 fence, mlx5_ib_opcode[wr->opcode]);
1388skip_psv:
1389 if (0)
1390 dump_wqe(qp, idx, size);
1391 }
1392
1393out:
1394 if (likely(nreq)) {
1395 qp->sq.head += nreq;
1396
1397 /* Make sure that descriptors are written before
1398 * updating doorbell record and ringing the doorbell
1399 */
1400 wmb();
1401
1402 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1403
1404 /* Make sure doorbell record is visible to the HCA before
1405 * we hit doorbell.
1406 */
1407 wmb();
1408
1409 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1410 /* Make sure doorbells don't leak out of SQ spinlock
1411 * and reach the HCA out of order.
1412 */
1413 bf->offset ^= bf->buf_size;
1414 }
1415
1416 spin_unlock_irqrestore(&qp->sq.lock, flags);
1417
1418 return err;
1419}
1420
1421static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1422{
1423 sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1424}
1425
1426int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1427 const struct ib_recv_wr **bad_wr, bool drain)
1428{
1429 struct mlx5_ib_qp *qp = to_mqp(ibqp);
1430 struct mlx5_wqe_data_seg *scat;
1431 struct mlx5_rwqe_sig *sig;
1432 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1433 struct mlx5_core_dev *mdev = dev->mdev;
1434 unsigned long flags;
1435 int err = 0;
1436 int nreq;
1437 int ind;
1438 int i;
1439
1440 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1441 !drain)) {
1442 *bad_wr = wr;
1443 return -EIO;
1444 }
1445
1446 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1447 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1448
1449 spin_lock_irqsave(&qp->rq.lock, flags);
1450
1451 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1452
1453 for (nreq = 0; wr; nreq++, wr = wr->next) {
1454 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1455 err = -ENOMEM;
1456 *bad_wr = wr;
1457 goto out;
1458 }
1459
1460 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1461 err = -EINVAL;
1462 *bad_wr = wr;
1463 goto out;
1464 }
1465
1466 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1467 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1468 scat++;
1469
1470 for (i = 0; i < wr->num_sge; i++)
1471 set_data_ptr_seg(scat + i, wr->sg_list + i);
1472
1473 if (i < qp->rq.max_gs) {
1474 scat[i].byte_count = 0;
1475 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
1476 scat[i].addr = 0;
1477 }
1478
1479 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1480 sig = (struct mlx5_rwqe_sig *)scat;
1481 set_sig_seg(sig, qp->rq.max_gs);
1482 }
1483
1484 qp->rq.wrid[ind] = wr->wr_id;
1485
1486 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1487 }
1488
1489out:
1490 if (likely(nreq)) {
1491 qp->rq.head += nreq;
1492
1493 /* Make sure that descriptors are written before
1494 * doorbell record.
1495 */
1496 wmb();
1497
1498 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1499 }
1500
1501 spin_unlock_irqrestore(&qp->rq.lock, flags);
1502
1503 return err;
1504}
Ubuntu Hirsute Linux kernel mirror