]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/infiniband/hw/qedr/verbs.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
qedr: remove pointless NULL check in qedr_post_send()
[mirror_ubuntu-jammy-kernel.git] / drivers / infiniband / hw / qedr / verbs.c
1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45
46 #include "qedr_hsi.h"
47 #include <linux/qed/qed_if.h>
48 #include "qedr.h"
49 #include "verbs.h"
50 #include <rdma/qedr-abi.h>
51 #include "qedr_cm.h"
52
53 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56 {
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62 }
63
64 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66 {
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83 }
84
85 int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88 {
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99 }
100
101 int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103 {
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114 }
115
116 int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118 {
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170 }
171
172 #define QEDR_SPEED_SDR (1)
173 #define QEDR_SPEED_DDR (2)
174 #define QEDR_SPEED_QDR (4)
175 #define QEDR_SPEED_FDR10 (8)
176 #define QEDR_SPEED_FDR (16)
177 #define QEDR_SPEED_EDR (32)
178
179 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181 {
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222 }
223
224 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225 {
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241 memset(attr, 0, sizeof(*attr));
242
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267 }
268
269 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271 {
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281 }
282
283 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285 {
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313 }
314
315 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317 {
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335 }
336
337 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339 {
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394 err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397 }
398
399 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400 {
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419 }
420
421 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422 {
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468 }
469
470 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472 {
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
474 struct qedr_ucontext *uctx = NULL;
475 struct qedr_alloc_pd_uresp uresp;
476 struct qedr_pd *pd;
477 u16 pd_id;
478 int rc;
479
480 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
481 (udata && context) ? "User Lib" : "Kernel");
482
483 if (!dev->rdma_ctx) {
484 DP_ERR(dev, "invlaid RDMA context\n");
485 return ERR_PTR(-EINVAL);
486 }
487
488 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
489 if (!pd)
490 return ERR_PTR(-ENOMEM);
491
492 dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
493
494 uresp.pd_id = pd_id;
495 pd->pd_id = pd_id;
496
497 if (udata && context) {
498 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
499 if (rc)
500 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
501 uctx = get_qedr_ucontext(context);
502 uctx->pd = pd;
503 pd->uctx = uctx;
504 }
505
506 return &pd->ibpd;
507 }
508
509 int qedr_dealloc_pd(struct ib_pd *ibpd)
510 {
511 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
512 struct qedr_pd *pd = get_qedr_pd(ibpd);
513
514 if (!pd) {
515 pr_err("Invalid PD received in dealloc_pd\n");
516 return -EINVAL;
517 }
518
519 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
520 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
521
522 kfree(pd);
523
524 return 0;
525 }
526
527 static void qedr_free_pbl(struct qedr_dev *dev,
528 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
529 {
530 struct pci_dev *pdev = dev->pdev;
531 int i;
532
533 for (i = 0; i < pbl_info->num_pbls; i++) {
534 if (!pbl[i].va)
535 continue;
536 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
537 pbl[i].va, pbl[i].pa);
538 }
539
540 kfree(pbl);
541 }
542
543 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
544 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
545
546 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
547 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
548 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
549
550 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
551 struct qedr_pbl_info *pbl_info,
552 gfp_t flags)
553 {
554 struct pci_dev *pdev = dev->pdev;
555 struct qedr_pbl *pbl_table;
556 dma_addr_t *pbl_main_tbl;
557 dma_addr_t pa;
558 void *va;
559 int i;
560
561 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
562 if (!pbl_table)
563 return ERR_PTR(-ENOMEM);
564
565 for (i = 0; i < pbl_info->num_pbls; i++) {
566 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
567 &pa, flags);
568 if (!va)
569 goto err;
570
571 memset(va, 0, pbl_info->pbl_size);
572 pbl_table[i].va = va;
573 pbl_table[i].pa = pa;
574 }
575
576 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
577 * the first one with physical pointers to all of the rest
578 */
579 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
580 for (i = 0; i < pbl_info->num_pbls - 1; i++)
581 pbl_main_tbl[i] = pbl_table[i + 1].pa;
582
583 return pbl_table;
584
585 err:
586 for (i--; i >= 0; i--)
587 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
588 pbl_table[i].va, pbl_table[i].pa);
589
590 qedr_free_pbl(dev, pbl_info, pbl_table);
591
592 return ERR_PTR(-ENOMEM);
593 }
594
595 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
596 struct qedr_pbl_info *pbl_info,
597 u32 num_pbes, int two_layer_capable)
598 {
599 u32 pbl_capacity;
600 u32 pbl_size;
601 u32 num_pbls;
602
603 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
604 if (num_pbes > MAX_PBES_TWO_LAYER) {
605 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
606 num_pbes);
607 return -EINVAL;
608 }
609
610 /* calculate required pbl page size */
611 pbl_size = MIN_FW_PBL_PAGE_SIZE;
612 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
613 NUM_PBES_ON_PAGE(pbl_size);
614
615 while (pbl_capacity < num_pbes) {
616 pbl_size *= 2;
617 pbl_capacity = pbl_size / sizeof(u64);
618 pbl_capacity = pbl_capacity * pbl_capacity;
619 }
620
621 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
622 num_pbls++; /* One for the layer0 ( points to the pbls) */
623 pbl_info->two_layered = true;
624 } else {
625 /* One layered PBL */
626 num_pbls = 1;
627 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
628 roundup_pow_of_two((num_pbes * sizeof(u64))));
629 pbl_info->two_layered = false;
630 }
631
632 pbl_info->num_pbls = num_pbls;
633 pbl_info->pbl_size = pbl_size;
634 pbl_info->num_pbes = num_pbes;
635
636 DP_DEBUG(dev, QEDR_MSG_MR,
637 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
638 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
639
640 return 0;
641 }
642
643 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
644 struct qedr_pbl *pbl,
645 struct qedr_pbl_info *pbl_info)
646 {
647 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
648 struct qedr_pbl *pbl_tbl;
649 struct scatterlist *sg;
650 struct regpair *pbe;
651 int entry;
652 u32 addr;
653
654 if (!pbl_info->num_pbes)
655 return;
656
657 /* If we have a two layered pbl, the first pbl points to the rest
658 * of the pbls and the first entry lays on the second pbl in the table
659 */
660 if (pbl_info->two_layered)
661 pbl_tbl = &pbl[1];
662 else
663 pbl_tbl = pbl;
664
665 pbe = (struct regpair *)pbl_tbl->va;
666 if (!pbe) {
667 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
668 return;
669 }
670
671 pbe_cnt = 0;
672
673 shift = ilog2(umem->page_size);
674
675 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
676 pages = sg_dma_len(sg) >> shift;
677 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
678 /* store the page address in pbe */
679 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
680 umem->page_size * pg_cnt);
681 addr = upper_32_bits(sg_dma_address(sg) +
682 umem->page_size * pg_cnt);
683 pbe->hi = cpu_to_le32(addr);
684 pbe_cnt++;
685 total_num_pbes++;
686 pbe++;
687
688 if (total_num_pbes == pbl_info->num_pbes)
689 return;
690
691 /* If the given pbl is full storing the pbes,
692 * move to next pbl.
693 */
694 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
695 pbl_tbl++;
696 pbe = (struct regpair *)pbl_tbl->va;
697 pbe_cnt = 0;
698 }
699 }
700 }
701 }
702
703 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
704 struct qedr_cq *cq, struct ib_udata *udata)
705 {
706 struct qedr_create_cq_uresp uresp;
707 int rc;
708
709 memset(&uresp, 0, sizeof(uresp));
710
711 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
712 uresp.icid = cq->icid;
713
714 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
715 if (rc)
716 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
717
718 return rc;
719 }
720
721 static void consume_cqe(struct qedr_cq *cq)
722 {
723 if (cq->latest_cqe == cq->toggle_cqe)
724 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
725
726 cq->latest_cqe = qed_chain_consume(&cq->pbl);
727 }
728
729 static inline int qedr_align_cq_entries(int entries)
730 {
731 u64 size, aligned_size;
732
733 /* We allocate an extra entry that we don't report to the FW. */
734 size = (entries + 1) * QEDR_CQE_SIZE;
735 aligned_size = ALIGN(size, PAGE_SIZE);
736
737 return aligned_size / QEDR_CQE_SIZE;
738 }
739
740 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
741 struct qedr_dev *dev,
742 struct qedr_userq *q,
743 u64 buf_addr, size_t buf_len,
744 int access, int dmasync)
745 {
746 int page_cnt;
747 int rc;
748
749 q->buf_addr = buf_addr;
750 q->buf_len = buf_len;
751 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
752 if (IS_ERR(q->umem)) {
753 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
754 PTR_ERR(q->umem));
755 return PTR_ERR(q->umem);
756 }
757
758 page_cnt = ib_umem_page_count(q->umem);
759 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
760 if (rc)
761 goto err0;
762
763 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
764 if (IS_ERR_OR_NULL(q->pbl_tbl))
765 goto err0;
766
767 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
768
769 return 0;
770
771 err0:
772 ib_umem_release(q->umem);
773
774 return rc;
775 }
776
777 static inline void qedr_init_cq_params(struct qedr_cq *cq,
778 struct qedr_ucontext *ctx,
779 struct qedr_dev *dev, int vector,
780 int chain_entries, int page_cnt,
781 u64 pbl_ptr,
782 struct qed_rdma_create_cq_in_params
783 *params)
784 {
785 memset(params, 0, sizeof(*params));
786 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
787 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
788 params->cnq_id = vector;
789 params->cq_size = chain_entries - 1;
790 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
791 params->pbl_num_pages = page_cnt;
792 params->pbl_ptr = pbl_ptr;
793 params->pbl_two_level = 0;
794 }
795
796 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
797 {
798 /* Flush data before signalling doorbell */
799 wmb();
800 cq->db.data.agg_flags = flags;
801 cq->db.data.value = cpu_to_le32(cons);
802 writeq(cq->db.raw, cq->db_addr);
803
804 /* Make sure write would stick */
805 mmiowb();
806 }
807
808 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
809 {
810 struct qedr_cq *cq = get_qedr_cq(ibcq);
811 unsigned long sflags;
812
813 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
814 return 0;
815
816 spin_lock_irqsave(&cq->cq_lock, sflags);
817
818 cq->arm_flags = 0;
819
820 if (flags & IB_CQ_SOLICITED)
821 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
822
823 if (flags & IB_CQ_NEXT_COMP)
824 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
825
826 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
827
828 spin_unlock_irqrestore(&cq->cq_lock, sflags);
829
830 return 0;
831 }
832
833 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
834 const struct ib_cq_init_attr *attr,
835 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
836 {
837 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
838 struct qed_rdma_destroy_cq_out_params destroy_oparams;
839 struct qed_rdma_destroy_cq_in_params destroy_iparams;
840 struct qedr_dev *dev = get_qedr_dev(ibdev);
841 struct qed_rdma_create_cq_in_params params;
842 struct qedr_create_cq_ureq ureq;
843 int vector = attr->comp_vector;
844 int entries = attr->cqe;
845 struct qedr_cq *cq;
846 int chain_entries;
847 int page_cnt;
848 u64 pbl_ptr;
849 u16 icid;
850 int rc;
851
852 DP_DEBUG(dev, QEDR_MSG_INIT,
853 "create_cq: called from %s. entries=%d, vector=%d\n",
854 udata ? "User Lib" : "Kernel", entries, vector);
855
856 if (entries > QEDR_MAX_CQES) {
857 DP_ERR(dev,
858 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
859 entries, QEDR_MAX_CQES);
860 return ERR_PTR(-EINVAL);
861 }
862
863 chain_entries = qedr_align_cq_entries(entries);
864 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
865
866 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
867 if (!cq)
868 return ERR_PTR(-ENOMEM);
869
870 if (udata) {
871 memset(&ureq, 0, sizeof(ureq));
872 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
873 DP_ERR(dev,
874 "create cq: problem copying data from user space\n");
875 goto err0;
876 }
877
878 if (!ureq.len) {
879 DP_ERR(dev,
880 "create cq: cannot create a cq with 0 entries\n");
881 goto err0;
882 }
883
884 cq->cq_type = QEDR_CQ_TYPE_USER;
885
886 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
887 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
888 if (rc)
889 goto err0;
890
891 pbl_ptr = cq->q.pbl_tbl->pa;
892 page_cnt = cq->q.pbl_info.num_pbes;
893 } else {
894 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
895
896 rc = dev->ops->common->chain_alloc(dev->cdev,
897 QED_CHAIN_USE_TO_CONSUME,
898 QED_CHAIN_MODE_PBL,
899 QED_CHAIN_CNT_TYPE_U32,
900 chain_entries,
901 sizeof(union rdma_cqe),
902 &cq->pbl);
903 if (rc)
904 goto err1;
905
906 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
907 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
908 }
909
910 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
911 pbl_ptr, &params);
912
913 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
914 if (rc)
915 goto err2;
916
917 cq->icid = icid;
918 cq->sig = QEDR_CQ_MAGIC_NUMBER;
919 spin_lock_init(&cq->cq_lock);
920
921 if (ib_ctx) {
922 rc = qedr_copy_cq_uresp(dev, cq, udata);
923 if (rc)
924 goto err3;
925 } else {
926 /* Generate doorbell address. */
927 cq->db_addr = dev->db_addr +
928 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
929 cq->db.data.icid = cq->icid;
930 cq->db.data.params = DB_AGG_CMD_SET <<
931 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
932
933 /* point to the very last element, passing it we will toggle */
934 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
935 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
936 cq->latest_cqe = NULL;
937 consume_cqe(cq);
938 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
939 }
940
941 DP_DEBUG(dev, QEDR_MSG_CQ,
942 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
943 cq->icid, cq, params.cq_size);
944
945 return &cq->ibcq;
946
947 err3:
948 destroy_iparams.icid = cq->icid;
949 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
950 &destroy_oparams);
951 err2:
952 if (udata)
953 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
954 else
955 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
956 err1:
957 if (udata)
958 ib_umem_release(cq->q.umem);
959 err0:
960 kfree(cq);
961 return ERR_PTR(-EINVAL);
962 }
963
964 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
965 {
966 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
967 struct qedr_cq *cq = get_qedr_cq(ibcq);
968
969 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
970
971 return 0;
972 }
973
974 int qedr_destroy_cq(struct ib_cq *ibcq)
975 {
976 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
977 struct qed_rdma_destroy_cq_out_params oparams;
978 struct qed_rdma_destroy_cq_in_params iparams;
979 struct qedr_cq *cq = get_qedr_cq(ibcq);
980
981 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
982
983 /* GSIs CQs are handled by driver, so they don't exist in the FW */
984 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
985 iparams.icid = cq->icid;
986 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
987 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
988 }
989
990 if (ibcq->uobject && ibcq->uobject->context) {
991 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
992 ib_umem_release(cq->q.umem);
993 }
994
995 kfree(cq);
996
997 return 0;
998 }
999
1000 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1001 struct ib_qp_attr *attr,
1002 int attr_mask,
1003 struct qed_rdma_modify_qp_in_params
1004 *qp_params)
1005 {
1006 enum rdma_network_type nw_type;
1007 struct ib_gid_attr gid_attr;
1008 union ib_gid gid;
1009 u32 ipv4_addr;
1010 int rc = 0;
1011 int i;
1012
1013 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1014 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1015 if (rc)
1016 return rc;
1017
1018 if (!memcmp(&gid, &zgid, sizeof(gid)))
1019 return -ENOENT;
1020
1021 if (gid_attr.ndev) {
1022 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1023
1024 dev_put(gid_attr.ndev);
1025 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1026 switch (nw_type) {
1027 case RDMA_NETWORK_IPV6:
1028 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1029 sizeof(qp_params->sgid));
1030 memcpy(&qp_params->dgid.bytes[0],
1031 &attr->ah_attr.grh.dgid,
1032 sizeof(qp_params->dgid));
1033 qp_params->roce_mode = ROCE_V2_IPV6;
1034 SET_FIELD(qp_params->modify_flags,
1035 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1036 break;
1037 case RDMA_NETWORK_IB:
1038 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1039 sizeof(qp_params->sgid));
1040 memcpy(&qp_params->dgid.bytes[0],
1041 &attr->ah_attr.grh.dgid,
1042 sizeof(qp_params->dgid));
1043 qp_params->roce_mode = ROCE_V1;
1044 break;
1045 case RDMA_NETWORK_IPV4:
1046 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1047 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1048 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1049 qp_params->sgid.ipv4_addr = ipv4_addr;
1050 ipv4_addr =
1051 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1052 qp_params->dgid.ipv4_addr = ipv4_addr;
1053 SET_FIELD(qp_params->modify_flags,
1054 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1055 qp_params->roce_mode = ROCE_V2_IPV4;
1056 break;
1057 }
1058 }
1059
1060 for (i = 0; i < 4; i++) {
1061 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1062 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1063 }
1064
1065 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1066 qp_params->vlan_id = 0;
1067
1068 return 0;
1069 }
1070
1071 static void qedr_cleanup_user_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1072 {
1073 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1074 ib_umem_release(qp->usq.umem);
1075 }
1076
1077 static void qedr_cleanup_user_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1078 {
1079 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1080 ib_umem_release(qp->urq.umem);
1081 }
1082
1083 static void qedr_cleanup_kernel_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1084 {
1085 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1086 kfree(qp->wqe_wr_id);
1087 }
1088
1089 static void qedr_cleanup_kernel_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1090 {
1091 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1092 kfree(qp->rqe_wr_id);
1093 }
1094
1095 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1096 struct ib_qp_init_attr *attrs)
1097 {
1098 struct qedr_device_attr *qattr = &dev->attr;
1099
1100 /* QP0... attrs->qp_type == IB_QPT_GSI */
1101 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1102 DP_DEBUG(dev, QEDR_MSG_QP,
1103 "create qp: unsupported qp type=0x%x requested\n",
1104 attrs->qp_type);
1105 return -EINVAL;
1106 }
1107
1108 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1109 DP_ERR(dev,
1110 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1111 attrs->cap.max_send_wr, qattr->max_sqe);
1112 return -EINVAL;
1113 }
1114
1115 if (attrs->cap.max_inline_data > qattr->max_inline) {
1116 DP_ERR(dev,
1117 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1118 attrs->cap.max_inline_data, qattr->max_inline);
1119 return -EINVAL;
1120 }
1121
1122 if (attrs->cap.max_send_sge > qattr->max_sge) {
1123 DP_ERR(dev,
1124 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1125 attrs->cap.max_send_sge, qattr->max_sge);
1126 return -EINVAL;
1127 }
1128
1129 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1130 DP_ERR(dev,
1131 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1132 attrs->cap.max_recv_sge, qattr->max_sge);
1133 return -EINVAL;
1134 }
1135
1136 /* Unprivileged user space cannot create special QP */
1137 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1138 DP_ERR(dev,
1139 "create qp: userspace can't create special QPs of type=0x%x\n",
1140 attrs->qp_type);
1141 return -EINVAL;
1142 }
1143
1144 return 0;
1145 }
1146
1147 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1148 struct qedr_qp *qp)
1149 {
1150 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1151 uresp->rq_icid = qp->icid;
1152 }
1153
1154 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1155 struct qedr_qp *qp)
1156 {
1157 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1158 uresp->sq_icid = qp->icid + 1;
1159 }
1160
1161 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1162 struct qedr_qp *qp, struct ib_udata *udata)
1163 {
1164 struct qedr_create_qp_uresp uresp;
1165 int rc;
1166
1167 memset(&uresp, 0, sizeof(uresp));
1168 qedr_copy_sq_uresp(&uresp, qp);
1169 qedr_copy_rq_uresp(&uresp, qp);
1170
1171 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1172 uresp.qp_id = qp->qp_id;
1173
1174 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1175 if (rc)
1176 DP_ERR(dev,
1177 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1178 qp->icid);
1179
1180 return rc;
1181 }
1182
1183 static void qedr_set_qp_init_params(struct qedr_dev *dev,
1184 struct qedr_qp *qp,
1185 struct qedr_pd *pd,
1186 struct ib_qp_init_attr *attrs)
1187 {
1188 qp->pd = pd;
1189
1190 spin_lock_init(&qp->q_lock);
1191
1192 qp->qp_type = attrs->qp_type;
1193 qp->max_inline_data = attrs->cap.max_inline_data;
1194 qp->sq.max_sges = attrs->cap.max_send_sge;
1195 qp->state = QED_ROCE_QP_STATE_RESET;
1196 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1197 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1198 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1199 qp->dev = dev;
1200
1201 DP_DEBUG(dev, QEDR_MSG_QP,
1202 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1203 pd->pd_id, qp->qp_type, qp->max_inline_data,
1204 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1205 DP_DEBUG(dev, QEDR_MSG_QP,
1206 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1207 qp->sq.max_sges, qp->sq_cq->icid);
1208 qp->rq.max_sges = attrs->cap.max_recv_sge;
1209 DP_DEBUG(dev, QEDR_MSG_QP,
1210 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1211 qp->rq.max_sges, qp->rq_cq->icid);
1212 }
1213
1214 static inline void
1215 qedr_init_qp_user_params(struct qed_rdma_create_qp_in_params *params,
1216 struct qedr_create_qp_ureq *ureq)
1217 {
1218 /* QP handle to be written in CQE */
1219 params->qp_handle_lo = ureq->qp_handle_lo;
1220 params->qp_handle_hi = ureq->qp_handle_hi;
1221 }
1222
1223 static inline void
1224 qedr_init_qp_kernel_doorbell_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1225 {
1226 qp->sq.db = dev->db_addr +
1227 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1228 qp->sq.db_data.data.icid = qp->icid + 1;
1229 }
1230
1231 static inline void
1232 qedr_init_qp_kernel_doorbell_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1233 {
1234 qp->rq.db = dev->db_addr +
1235 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1236 qp->rq.db_data.data.icid = qp->icid;
1237 }
1238
1239 static inline int
1240 qedr_init_qp_kernel_params_rq(struct qedr_dev *dev,
1241 struct qedr_qp *qp, struct ib_qp_init_attr *attrs)
1242 {
1243 /* Allocate driver internal RQ array */
1244 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1245 GFP_KERNEL);
1246 if (!qp->rqe_wr_id)
1247 return -ENOMEM;
1248
1249 DP_DEBUG(dev, QEDR_MSG_QP, "RQ max_wr set to %d.\n", qp->rq.max_wr);
1250
1251 return 0;
1252 }
1253
1254 static inline int
1255 qedr_init_qp_kernel_params_sq(struct qedr_dev *dev,
1256 struct qedr_qp *qp,
1257 struct ib_qp_init_attr *attrs,
1258 struct qed_rdma_create_qp_in_params *params)
1259 {
1260 u32 temp_max_wr;
1261
1262 /* Allocate driver internal SQ array */
1263 temp_max_wr = attrs->cap.max_send_wr * dev->wq_multiplier;
1264 temp_max_wr = min_t(u32, temp_max_wr, dev->attr.max_sqe);
1265
1266 /* temp_max_wr < attr->max_sqe < u16 so the casting is safe */
1267 qp->sq.max_wr = (u16)temp_max_wr;
1268 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1269 GFP_KERNEL);
1270 if (!qp->wqe_wr_id)
1271 return -ENOMEM;
1272
1273 DP_DEBUG(dev, QEDR_MSG_QP, "SQ max_wr set to %d.\n", qp->sq.max_wr);
1274
1275 /* QP handle to be written in CQE */
1276 params->qp_handle_lo = lower_32_bits((uintptr_t)qp);
1277 params->qp_handle_hi = upper_32_bits((uintptr_t)qp);
1278
1279 return 0;
1280 }
1281
1282 static inline int qedr_init_qp_kernel_sq(struct qedr_dev *dev,
1283 struct qedr_qp *qp,
1284 struct ib_qp_init_attr *attrs)
1285 {
1286 u32 n_sq_elems, n_sq_entries;
1287 int rc;
1288
1289 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1290 * the ring. The ring should allow at least a single WR, even if the
1291 * user requested none, due to allocation issues.
1292 */
1293 n_sq_entries = attrs->cap.max_send_wr;
1294 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1295 n_sq_entries = max_t(u32, n_sq_entries, 1);
1296 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1297 rc = dev->ops->common->chain_alloc(dev->cdev,
1298 QED_CHAIN_USE_TO_PRODUCE,
1299 QED_CHAIN_MODE_PBL,
1300 QED_CHAIN_CNT_TYPE_U32,
1301 n_sq_elems,
1302 QEDR_SQE_ELEMENT_SIZE,
1303 &qp->sq.pbl);
1304 if (rc) {
1305 DP_ERR(dev, "failed to allocate QP %p SQ\n", qp);
1306 return rc;
1307 }
1308
1309 DP_DEBUG(dev, QEDR_MSG_SQ,
1310 "SQ Pbl base addr = %llx max_send_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1311 qed_chain_get_pbl_phys(&qp->sq.pbl), attrs->cap.max_send_wr,
1312 n_sq_entries, qed_chain_get_capacity(&qp->sq.pbl), rc);
1313 return 0;
1314 }
1315
1316 static inline int qedr_init_qp_kernel_rq(struct qedr_dev *dev,
1317 struct qedr_qp *qp,
1318 struct ib_qp_init_attr *attrs)
1319 {
1320 u32 n_rq_elems, n_rq_entries;
1321 int rc;
1322
1323 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1324 * the ring. There ring should allow at least a single WR, even if the
1325 * user requested none, due to allocation issues.
1326 */
1327 n_rq_entries = max_t(u32, attrs->cap.max_recv_wr, 1);
1328 n_rq_elems = n_rq_entries * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1329 rc = dev->ops->common->chain_alloc(dev->cdev,
1330 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1331 QED_CHAIN_MODE_PBL,
1332 QED_CHAIN_CNT_TYPE_U32,
1333 n_rq_elems,
1334 QEDR_RQE_ELEMENT_SIZE,
1335 &qp->rq.pbl);
1336
1337 if (rc) {
1338 DP_ERR(dev, "failed to allocate memory for QP %p RQ\n", qp);
1339 return -ENOMEM;
1340 }
1341
1342 DP_DEBUG(dev, QEDR_MSG_RQ,
1343 "RQ Pbl base addr = %llx max_recv_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1344 qed_chain_get_pbl_phys(&qp->rq.pbl), attrs->cap.max_recv_wr,
1345 n_rq_entries, qed_chain_get_capacity(&qp->rq.pbl), rc);
1346
1347 /* n_rq_entries < u16 so the casting is safe */
1348 qp->rq.max_wr = (u16)n_rq_entries;
1349
1350 return 0;
1351 }
1352
1353 static inline void
1354 qedr_init_qp_in_params_sq(struct qedr_dev *dev,
1355 struct qedr_pd *pd,
1356 struct qedr_qp *qp,
1357 struct ib_qp_init_attr *attrs,
1358 struct ib_udata *udata,
1359 struct qed_rdma_create_qp_in_params *params)
1360 {
1361 /* QP handle to be written in an async event */
1362 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
1363 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
1364
1365 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1366 params->fmr_and_reserved_lkey = !udata;
1367 params->pd = pd->pd_id;
1368 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1369 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1370 params->max_sq_sges = 0;
1371 params->stats_queue = 0;
1372
1373 if (udata) {
1374 params->sq_num_pages = qp->usq.pbl_info.num_pbes;
1375 params->sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1376 } else {
1377 params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1378 params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1379 }
1380 }
1381
1382 static inline void
1383 qedr_init_qp_in_params_rq(struct qedr_qp *qp,
1384 struct ib_qp_init_attr *attrs,
1385 struct ib_udata *udata,
1386 struct qed_rdma_create_qp_in_params *params)
1387 {
1388 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1389 params->srq_id = 0;
1390 params->use_srq = false;
1391
1392 if (udata) {
1393 params->rq_num_pages = qp->urq.pbl_info.num_pbes;
1394 params->rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1395 } else {
1396 params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1397 params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1398 }
1399 }
1400
1401 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1402 {
1403 DP_DEBUG(dev, QEDR_MSG_QP,
1404 "create qp: successfully created user QP. qp=%p, sq_addr=0x%llx, sq_len=%zd, rq_addr=0x%llx, rq_len=%zd\n",
1405 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
1406 qp->urq.buf_len);
1407 }
1408
1409 static inline int qedr_init_user_qp(struct ib_ucontext *ib_ctx,
1410 struct qedr_dev *dev,
1411 struct qedr_qp *qp,
1412 struct qedr_create_qp_ureq *ureq)
1413 {
1414 int rc;
1415
1416 /* SQ - read access only (0), dma sync not required (0) */
1417 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq->sq_addr,
1418 ureq->sq_len, 0, 0);
1419 if (rc)
1420 return rc;
1421
1422 /* RQ - read access only (0), dma sync not required (0) */
1423 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq->rq_addr,
1424 ureq->rq_len, 0, 0);
1425
1426 if (rc)
1427 qedr_cleanup_user_sq(dev, qp);
1428 return rc;
1429 }
1430
1431 static inline int
1432 qedr_init_kernel_qp(struct qedr_dev *dev,
1433 struct qedr_qp *qp,
1434 struct ib_qp_init_attr *attrs,
1435 struct qed_rdma_create_qp_in_params *params)
1436 {
1437 int rc;
1438
1439 rc = qedr_init_qp_kernel_sq(dev, qp, attrs);
1440 if (rc) {
1441 DP_ERR(dev, "failed to init kernel QP %p SQ\n", qp);
1442 return rc;
1443 }
1444
1445 rc = qedr_init_qp_kernel_params_sq(dev, qp, attrs, params);
1446 if (rc) {
1447 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1448 DP_ERR(dev, "failed to init kernel QP %p SQ params\n", qp);
1449 return rc;
1450 }
1451
1452 rc = qedr_init_qp_kernel_rq(dev, qp, attrs);
1453 if (rc) {
1454 qedr_cleanup_kernel_sq(dev, qp);
1455 DP_ERR(dev, "failed to init kernel QP %p RQ\n", qp);
1456 return rc;
1457 }
1458
1459 rc = qedr_init_qp_kernel_params_rq(dev, qp, attrs);
1460 if (rc) {
1461 DP_ERR(dev, "failed to init kernel QP %p RQ params\n", qp);
1462 qedr_cleanup_kernel_sq(dev, qp);
1463 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1464 return rc;
1465 }
1466
1467 return rc;
1468 }
1469
1470 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1471 struct ib_qp_init_attr *attrs,
1472 struct ib_udata *udata)
1473 {
1474 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1475 struct qed_rdma_create_qp_out_params out_params;
1476 struct qed_rdma_create_qp_in_params in_params;
1477 struct qedr_pd *pd = get_qedr_pd(ibpd);
1478 struct ib_ucontext *ib_ctx = NULL;
1479 struct qedr_ucontext *ctx = NULL;
1480 struct qedr_create_qp_ureq ureq;
1481 struct qedr_qp *qp;
1482 struct ib_qp *ibqp;
1483 int rc = 0;
1484
1485 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1486 udata ? "user library" : "kernel", pd);
1487
1488 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1489 if (rc)
1490 return ERR_PTR(rc);
1491
1492 if (attrs->srq)
1493 return ERR_PTR(-EINVAL);
1494
1495 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1496 if (!qp)
1497 return ERR_PTR(-ENOMEM);
1498
1499 DP_DEBUG(dev, QEDR_MSG_QP,
1500 "create qp: sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1501 get_qedr_cq(attrs->send_cq),
1502 get_qedr_cq(attrs->send_cq)->icid,
1503 get_qedr_cq(attrs->recv_cq),
1504 get_qedr_cq(attrs->recv_cq)->icid);
1505
1506 qedr_set_qp_init_params(dev, qp, pd, attrs);
1507
1508 if (attrs->qp_type == IB_QPT_GSI) {
1509 if (udata) {
1510 DP_ERR(dev,
1511 "create qp: unexpected udata when creating GSI QP\n");
1512 goto err0;
1513 }
1514 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1515 if (IS_ERR(ibqp))
1516 kfree(qp);
1517 return ibqp;
1518 }
1519
1520 memset(&in_params, 0, sizeof(in_params));
1521
1522 if (udata) {
1523 if (!(udata && ibpd->uobject && ibpd->uobject->context))
1524 goto err0;
1525
1526 ib_ctx = ibpd->uobject->context;
1527 ctx = get_qedr_ucontext(ib_ctx);
1528
1529 memset(&ureq, 0, sizeof(ureq));
1530 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1531 DP_ERR(dev,
1532 "create qp: problem copying data from user space\n");
1533 goto err0;
1534 }
1535
1536 rc = qedr_init_user_qp(ib_ctx, dev, qp, &ureq);
1537 if (rc)
1538 goto err0;
1539
1540 qedr_init_qp_user_params(&in_params, &ureq);
1541 } else {
1542 rc = qedr_init_kernel_qp(dev, qp, attrs, &in_params);
1543 if (rc)
1544 goto err0;
1545 }
1546
1547 qedr_init_qp_in_params_sq(dev, pd, qp, attrs, udata, &in_params);
1548 qedr_init_qp_in_params_rq(qp, attrs, udata, &in_params);
1549
1550 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1551 &in_params, &out_params);
1552
1553 if (!qp->qed_qp)
1554 goto err1;
1555
1556 qp->qp_id = out_params.qp_id;
1557 qp->icid = out_params.icid;
1558 qp->ibqp.qp_num = qp->qp_id;
1559
1560 if (udata) {
1561 rc = qedr_copy_qp_uresp(dev, qp, udata);
1562 if (rc)
1563 goto err2;
1564
1565 qedr_qp_user_print(dev, qp);
1566 } else {
1567 qedr_init_qp_kernel_doorbell_sq(dev, qp);
1568 qedr_init_qp_kernel_doorbell_rq(dev, qp);
1569 }
1570
1571 DP_DEBUG(dev, QEDR_MSG_QP, "created %s space QP %p\n",
1572 udata ? "user" : "kernel", qp);
1573
1574 return &qp->ibqp;
1575
1576 err2:
1577 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1578 if (rc)
1579 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1580 err1:
1581 if (udata) {
1582 qedr_cleanup_user_sq(dev, qp);
1583 qedr_cleanup_user_rq(dev, qp);
1584 } else {
1585 qedr_cleanup_kernel_sq(dev, qp);
1586 qedr_cleanup_kernel_rq(dev, qp);
1587 }
1588
1589 err0:
1590 kfree(qp);
1591
1592 return ERR_PTR(-EFAULT);
1593 }
1594
1595 enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1596 {
1597 switch (qp_state) {
1598 case QED_ROCE_QP_STATE_RESET:
1599 return IB_QPS_RESET;
1600 case QED_ROCE_QP_STATE_INIT:
1601 return IB_QPS_INIT;
1602 case QED_ROCE_QP_STATE_RTR:
1603 return IB_QPS_RTR;
1604 case QED_ROCE_QP_STATE_RTS:
1605 return IB_QPS_RTS;
1606 case QED_ROCE_QP_STATE_SQD:
1607 return IB_QPS_SQD;
1608 case QED_ROCE_QP_STATE_ERR:
1609 return IB_QPS_ERR;
1610 case QED_ROCE_QP_STATE_SQE:
1611 return IB_QPS_SQE;
1612 }
1613 return IB_QPS_ERR;
1614 }
1615
1616 enum qed_roce_qp_state qedr_get_state_from_ibqp(enum ib_qp_state qp_state)
1617 {
1618 switch (qp_state) {
1619 case IB_QPS_RESET:
1620 return QED_ROCE_QP_STATE_RESET;
1621 case IB_QPS_INIT:
1622 return QED_ROCE_QP_STATE_INIT;
1623 case IB_QPS_RTR:
1624 return QED_ROCE_QP_STATE_RTR;
1625 case IB_QPS_RTS:
1626 return QED_ROCE_QP_STATE_RTS;
1627 case IB_QPS_SQD:
1628 return QED_ROCE_QP_STATE_SQD;
1629 case IB_QPS_ERR:
1630 return QED_ROCE_QP_STATE_ERR;
1631 default:
1632 return QED_ROCE_QP_STATE_ERR;
1633 }
1634 }
1635
1636 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1637 {
1638 qed_chain_reset(&qph->pbl);
1639 qph->prod = 0;
1640 qph->cons = 0;
1641 qph->wqe_cons = 0;
1642 qph->db_data.data.value = cpu_to_le16(0);
1643 }
1644
1645 static int qedr_update_qp_state(struct qedr_dev *dev,
1646 struct qedr_qp *qp,
1647 enum qed_roce_qp_state new_state)
1648 {
1649 int status = 0;
1650
1651 if (new_state == qp->state)
1652 return 1;
1653
1654 switch (qp->state) {
1655 case QED_ROCE_QP_STATE_RESET:
1656 switch (new_state) {
1657 case QED_ROCE_QP_STATE_INIT:
1658 qp->prev_wqe_size = 0;
1659 qedr_reset_qp_hwq_info(&qp->sq);
1660 qedr_reset_qp_hwq_info(&qp->rq);
1661 break;
1662 default:
1663 status = -EINVAL;
1664 break;
1665 };
1666 break;
1667 case QED_ROCE_QP_STATE_INIT:
1668 switch (new_state) {
1669 case QED_ROCE_QP_STATE_RTR:
1670 /* Update doorbell (in case post_recv was
1671 * done before move to RTR)
1672 */
1673 wmb();
1674 writel(qp->rq.db_data.raw, qp->rq.db);
1675 /* Make sure write takes effect */
1676 mmiowb();
1677 break;
1678 case QED_ROCE_QP_STATE_ERR:
1679 break;
1680 default:
1681 /* Invalid state change. */
1682 status = -EINVAL;
1683 break;
1684 };
1685 break;
1686 case QED_ROCE_QP_STATE_RTR:
1687 /* RTR->XXX */
1688 switch (new_state) {
1689 case QED_ROCE_QP_STATE_RTS:
1690 break;
1691 case QED_ROCE_QP_STATE_ERR:
1692 break;
1693 default:
1694 /* Invalid state change. */
1695 status = -EINVAL;
1696 break;
1697 };
1698 break;
1699 case QED_ROCE_QP_STATE_RTS:
1700 /* RTS->XXX */
1701 switch (new_state) {
1702 case QED_ROCE_QP_STATE_SQD:
1703 break;
1704 case QED_ROCE_QP_STATE_ERR:
1705 break;
1706 default:
1707 /* Invalid state change. */
1708 status = -EINVAL;
1709 break;
1710 };
1711 break;
1712 case QED_ROCE_QP_STATE_SQD:
1713 /* SQD->XXX */
1714 switch (new_state) {
1715 case QED_ROCE_QP_STATE_RTS:
1716 case QED_ROCE_QP_STATE_ERR:
1717 break;
1718 default:
1719 /* Invalid state change. */
1720 status = -EINVAL;
1721 break;
1722 };
1723 break;
1724 case QED_ROCE_QP_STATE_ERR:
1725 /* ERR->XXX */
1726 switch (new_state) {
1727 case QED_ROCE_QP_STATE_RESET:
1728 break;
1729 default:
1730 status = -EINVAL;
1731 break;
1732 };
1733 break;
1734 default:
1735 status = -EINVAL;
1736 break;
1737 };
1738
1739 return status;
1740 }
1741
1742 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1743 int attr_mask, struct ib_udata *udata)
1744 {
1745 struct qedr_qp *qp = get_qedr_qp(ibqp);
1746 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1747 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1748 enum ib_qp_state old_qp_state, new_qp_state;
1749 int rc = 0;
1750
1751 DP_DEBUG(dev, QEDR_MSG_QP,
1752 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1753 attr->qp_state);
1754
1755 old_qp_state = qedr_get_ibqp_state(qp->state);
1756 if (attr_mask & IB_QP_STATE)
1757 new_qp_state = attr->qp_state;
1758 else
1759 new_qp_state = old_qp_state;
1760
1761 if (!ib_modify_qp_is_ok
1762 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1763 IB_LINK_LAYER_ETHERNET)) {
1764 DP_ERR(dev,
1765 "modify qp: invalid attribute mask=0x%x specified for\n"
1766 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1767 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1768 new_qp_state);
1769 rc = -EINVAL;
1770 goto err;
1771 }
1772
1773 /* Translate the masks... */
1774 if (attr_mask & IB_QP_STATE) {
1775 SET_FIELD(qp_params.modify_flags,
1776 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1777 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1778 }
1779
1780 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1781 qp_params.sqd_async = true;
1782
1783 if (attr_mask & IB_QP_PKEY_INDEX) {
1784 SET_FIELD(qp_params.modify_flags,
1785 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1786 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1787 rc = -EINVAL;
1788 goto err;
1789 }
1790
1791 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1792 }
1793
1794 if (attr_mask & IB_QP_QKEY)
1795 qp->qkey = attr->qkey;
1796
1797 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1798 SET_FIELD(qp_params.modify_flags,
1799 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1800 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1801 IB_ACCESS_REMOTE_READ;
1802 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1803 IB_ACCESS_REMOTE_WRITE;
1804 qp_params.incoming_atomic_en = attr->qp_access_flags &
1805 IB_ACCESS_REMOTE_ATOMIC;
1806 }
1807
1808 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1809 if (attr_mask & IB_QP_PATH_MTU) {
1810 if (attr->path_mtu < IB_MTU_256 ||
1811 attr->path_mtu > IB_MTU_4096) {
1812 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1813 rc = -EINVAL;
1814 goto err;
1815 }
1816 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1817 ib_mtu_enum_to_int(iboe_get_mtu
1818 (dev->ndev->mtu)));
1819 }
1820
1821 if (!qp->mtu) {
1822 qp->mtu =
1823 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1824 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1825 }
1826
1827 SET_FIELD(qp_params.modify_flags,
1828 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1829
1830 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1831 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1832 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1833
1834 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1835
1836 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1837 if (rc) {
1838 DP_ERR(dev,
1839 "modify qp: problems with GID index %d (rc=%d)\n",
1840 attr->ah_attr.grh.sgid_index, rc);
1841 return rc;
1842 }
1843
1844 rc = qedr_get_dmac(dev, &attr->ah_attr,
1845 qp_params.remote_mac_addr);
1846 if (rc)
1847 return rc;
1848
1849 qp_params.use_local_mac = true;
1850 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1851
1852 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1853 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1854 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1855 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1856 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1857 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1858 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1859 qp_params.remote_mac_addr);
1860 ;
1861
1862 qp_params.mtu = qp->mtu;
1863 qp_params.lb_indication = false;
1864 }
1865
1866 if (!qp_params.mtu) {
1867 /* Stay with current MTU */
1868 if (qp->mtu)
1869 qp_params.mtu = qp->mtu;
1870 else
1871 qp_params.mtu =
1872 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1873 }
1874
1875 if (attr_mask & IB_QP_TIMEOUT) {
1876 SET_FIELD(qp_params.modify_flags,
1877 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1878
1879 qp_params.ack_timeout = attr->timeout;
1880 if (attr->timeout) {
1881 u32 temp;
1882
1883 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1884 /* FW requires [msec] */
1885 qp_params.ack_timeout = temp;
1886 } else {
1887 /* Infinite */
1888 qp_params.ack_timeout = 0;
1889 }
1890 }
1891 if (attr_mask & IB_QP_RETRY_CNT) {
1892 SET_FIELD(qp_params.modify_flags,
1893 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1894 qp_params.retry_cnt = attr->retry_cnt;
1895 }
1896
1897 if (attr_mask & IB_QP_RNR_RETRY) {
1898 SET_FIELD(qp_params.modify_flags,
1899 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1900 qp_params.rnr_retry_cnt = attr->rnr_retry;
1901 }
1902
1903 if (attr_mask & IB_QP_RQ_PSN) {
1904 SET_FIELD(qp_params.modify_flags,
1905 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1906 qp_params.rq_psn = attr->rq_psn;
1907 qp->rq_psn = attr->rq_psn;
1908 }
1909
1910 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1911 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1912 rc = -EINVAL;
1913 DP_ERR(dev,
1914 "unsupported max_rd_atomic=%d, supported=%d\n",
1915 attr->max_rd_atomic,
1916 dev->attr.max_qp_req_rd_atomic_resc);
1917 goto err;
1918 }
1919
1920 SET_FIELD(qp_params.modify_flags,
1921 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1922 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1923 }
1924
1925 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1926 SET_FIELD(qp_params.modify_flags,
1927 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1928 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1929 }
1930
1931 if (attr_mask & IB_QP_SQ_PSN) {
1932 SET_FIELD(qp_params.modify_flags,
1933 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1934 qp_params.sq_psn = attr->sq_psn;
1935 qp->sq_psn = attr->sq_psn;
1936 }
1937
1938 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1939 if (attr->max_dest_rd_atomic >
1940 dev->attr.max_qp_resp_rd_atomic_resc) {
1941 DP_ERR(dev,
1942 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1943 attr->max_dest_rd_atomic,
1944 dev->attr.max_qp_resp_rd_atomic_resc);
1945
1946 rc = -EINVAL;
1947 goto err;
1948 }
1949
1950 SET_FIELD(qp_params.modify_flags,
1951 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1952 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1953 }
1954
1955 if (attr_mask & IB_QP_DEST_QPN) {
1956 SET_FIELD(qp_params.modify_flags,
1957 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1958
1959 qp_params.dest_qp = attr->dest_qp_num;
1960 qp->dest_qp_num = attr->dest_qp_num;
1961 }
1962
1963 if (qp->qp_type != IB_QPT_GSI)
1964 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1965 qp->qed_qp, &qp_params);
1966
1967 if (attr_mask & IB_QP_STATE) {
1968 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1969 qedr_update_qp_state(dev, qp, qp_params.new_state);
1970 qp->state = qp_params.new_state;
1971 }
1972
1973 err:
1974 return rc;
1975 }
1976
1977 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1978 {
1979 int ib_qp_acc_flags = 0;
1980
1981 if (params->incoming_rdma_write_en)
1982 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1983 if (params->incoming_rdma_read_en)
1984 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
1985 if (params->incoming_atomic_en)
1986 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
1987 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1988 return ib_qp_acc_flags;
1989 }
1990
1991 int qedr_query_qp(struct ib_qp *ibqp,
1992 struct ib_qp_attr *qp_attr,
1993 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1994 {
1995 struct qed_rdma_query_qp_out_params params;
1996 struct qedr_qp *qp = get_qedr_qp(ibqp);
1997 struct qedr_dev *dev = qp->dev;
1998 int rc = 0;
1999
2000 memset(&params, 0, sizeof(params));
2001
2002 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2003 if (rc)
2004 goto err;
2005
2006 memset(qp_attr, 0, sizeof(*qp_attr));
2007 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2008
2009 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2010 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2011 qp_attr->path_mtu = iboe_get_mtu(params.mtu);
2012 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2013 qp_attr->rq_psn = params.rq_psn;
2014 qp_attr->sq_psn = params.sq_psn;
2015 qp_attr->dest_qp_num = params.dest_qp;
2016
2017 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2018
2019 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2020 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2021 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2022 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2023 qp_attr->cap.max_inline_data = qp->max_inline_data;
2024 qp_init_attr->cap = qp_attr->cap;
2025
2026 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
2027 sizeof(qp_attr->ah_attr.grh.dgid.raw));
2028
2029 qp_attr->ah_attr.grh.flow_label = params.flow_label;
2030 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
2031 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
2032 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
2033
2034 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
2035 qp_attr->ah_attr.port_num = 1;
2036 qp_attr->ah_attr.sl = 0;
2037 qp_attr->timeout = params.timeout;
2038 qp_attr->rnr_retry = params.rnr_retry;
2039 qp_attr->retry_cnt = params.retry_cnt;
2040 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2041 qp_attr->pkey_index = params.pkey_index;
2042 qp_attr->port_num = 1;
2043 qp_attr->ah_attr.src_path_bits = 0;
2044 qp_attr->ah_attr.static_rate = 0;
2045 qp_attr->alt_pkey_index = 0;
2046 qp_attr->alt_port_num = 0;
2047 qp_attr->alt_timeout = 0;
2048 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2049
2050 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2051 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2052 qp_attr->max_rd_atomic = params.max_rd_atomic;
2053 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2054
2055 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2056 qp_attr->cap.max_inline_data);
2057
2058 err:
2059 return rc;
2060 }
2061
2062 int qedr_destroy_qp(struct ib_qp *ibqp)
2063 {
2064 struct qedr_qp *qp = get_qedr_qp(ibqp);
2065 struct qedr_dev *dev = qp->dev;
2066 struct ib_qp_attr attr;
2067 int attr_mask = 0;
2068 int rc = 0;
2069
2070 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2071 qp, qp->qp_type);
2072
2073 if (qp->state != (QED_ROCE_QP_STATE_RESET | QED_ROCE_QP_STATE_ERR |
2074 QED_ROCE_QP_STATE_INIT)) {
2075 attr.qp_state = IB_QPS_ERR;
2076 attr_mask |= IB_QP_STATE;
2077
2078 /* Change the QP state to ERROR */
2079 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2080 }
2081
2082 if (qp->qp_type != IB_QPT_GSI) {
2083 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2084 if (rc)
2085 return rc;
2086 } else {
2087 qedr_destroy_gsi_qp(dev);
2088 }
2089
2090 if (ibqp->uobject && ibqp->uobject->context) {
2091 qedr_cleanup_user_sq(dev, qp);
2092 qedr_cleanup_user_rq(dev, qp);
2093 } else {
2094 qedr_cleanup_kernel_sq(dev, qp);
2095 qedr_cleanup_kernel_rq(dev, qp);
2096 }
2097
2098 kfree(qp);
2099
2100 return rc;
2101 }
2102
2103 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
2104 {
2105 struct qedr_ah *ah;
2106
2107 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2108 if (!ah)
2109 return ERR_PTR(-ENOMEM);
2110
2111 ah->attr = *attr;
2112
2113 return &ah->ibah;
2114 }
2115
2116 int qedr_destroy_ah(struct ib_ah *ibah)
2117 {
2118 struct qedr_ah *ah = get_qedr_ah(ibah);
2119
2120 kfree(ah);
2121 return 0;
2122 }
2123
2124 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2125 {
2126 struct qedr_pbl *pbl, *tmp;
2127
2128 if (info->pbl_table)
2129 list_add_tail(&info->pbl_table->list_entry,
2130 &info->free_pbl_list);
2131
2132 if (!list_empty(&info->inuse_pbl_list))
2133 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2134
2135 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2136 list_del(&pbl->list_entry);
2137 qedr_free_pbl(dev, &info->pbl_info, pbl);
2138 }
2139 }
2140
2141 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2142 size_t page_list_len, bool two_layered)
2143 {
2144 struct qedr_pbl *tmp;
2145 int rc;
2146
2147 INIT_LIST_HEAD(&info->free_pbl_list);
2148 INIT_LIST_HEAD(&info->inuse_pbl_list);
2149
2150 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2151 page_list_len, two_layered);
2152 if (rc)
2153 goto done;
2154
2155 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2156 if (!info->pbl_table) {
2157 rc = -ENOMEM;
2158 goto done;
2159 }
2160
2161 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2162 &info->pbl_table->pa);
2163
2164 /* in usual case we use 2 PBLs, so we add one to free
2165 * list and allocating another one
2166 */
2167 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2168 if (!tmp) {
2169 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2170 goto done;
2171 }
2172
2173 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2174
2175 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2176
2177 done:
2178 if (rc)
2179 free_mr_info(dev, info);
2180
2181 return rc;
2182 }
2183
2184 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2185 u64 usr_addr, int acc, struct ib_udata *udata)
2186 {
2187 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2188 struct qedr_mr *mr;
2189 struct qedr_pd *pd;
2190 int rc = -ENOMEM;
2191
2192 pd = get_qedr_pd(ibpd);
2193 DP_DEBUG(dev, QEDR_MSG_MR,
2194 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2195 pd->pd_id, start, len, usr_addr, acc);
2196
2197 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2198 return ERR_PTR(-EINVAL);
2199
2200 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2201 if (!mr)
2202 return ERR_PTR(rc);
2203
2204 mr->type = QEDR_MR_USER;
2205
2206 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2207 if (IS_ERR(mr->umem)) {
2208 rc = -EFAULT;
2209 goto err0;
2210 }
2211
2212 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2213 if (rc)
2214 goto err1;
2215
2216 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2217 &mr->info.pbl_info);
2218
2219 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2220 if (rc) {
2221 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2222 goto err1;
2223 }
2224
2225 /* Index only, 18 bit long, lkey = itid << 8 | key */
2226 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2227 mr->hw_mr.key = 0;
2228 mr->hw_mr.pd = pd->pd_id;
2229 mr->hw_mr.local_read = 1;
2230 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2231 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2232 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2233 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2234 mr->hw_mr.mw_bind = false;
2235 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2236 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2237 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2238 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size);
2239 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2240 mr->hw_mr.length = len;
2241 mr->hw_mr.vaddr = usr_addr;
2242 mr->hw_mr.zbva = false;
2243 mr->hw_mr.phy_mr = false;
2244 mr->hw_mr.dma_mr = false;
2245
2246 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2247 if (rc) {
2248 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2249 goto err2;
2250 }
2251
2252 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2253 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2254 mr->hw_mr.remote_atomic)
2255 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2256
2257 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2258 mr->ibmr.lkey);
2259 return &mr->ibmr;
2260
2261 err2:
2262 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2263 err1:
2264 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2265 err0:
2266 kfree(mr);
2267 return ERR_PTR(rc);
2268 }
2269
2270 int qedr_dereg_mr(struct ib_mr *ib_mr)
2271 {
2272 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2273 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2274 int rc = 0;
2275
2276 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2277 if (rc)
2278 return rc;
2279
2280 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2281
2282 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2283 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2284
2285 /* it could be user registered memory. */
2286 if (mr->umem)
2287 ib_umem_release(mr->umem);
2288
2289 kfree(mr);
2290
2291 return rc;
2292 }
2293
2294 struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
2295 {
2296 struct qedr_pd *pd = get_qedr_pd(ibpd);
2297 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2298 struct qedr_mr *mr;
2299 int rc = -ENOMEM;
2300
2301 DP_DEBUG(dev, QEDR_MSG_MR,
2302 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2303 max_page_list_len);
2304
2305 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2306 if (!mr)
2307 return ERR_PTR(rc);
2308
2309 mr->dev = dev;
2310 mr->type = QEDR_MR_FRMR;
2311
2312 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2313 if (rc)
2314 goto err0;
2315
2316 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2317 if (rc) {
2318 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2319 goto err0;
2320 }
2321
2322 /* Index only, 18 bit long, lkey = itid << 8 | key */
2323 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2324 mr->hw_mr.key = 0;
2325 mr->hw_mr.pd = pd->pd_id;
2326 mr->hw_mr.local_read = 1;
2327 mr->hw_mr.local_write = 0;
2328 mr->hw_mr.remote_read = 0;
2329 mr->hw_mr.remote_write = 0;
2330 mr->hw_mr.remote_atomic = 0;
2331 mr->hw_mr.mw_bind = false;
2332 mr->hw_mr.pbl_ptr = 0;
2333 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2334 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2335 mr->hw_mr.fbo = 0;
2336 mr->hw_mr.length = 0;
2337 mr->hw_mr.vaddr = 0;
2338 mr->hw_mr.zbva = false;
2339 mr->hw_mr.phy_mr = true;
2340 mr->hw_mr.dma_mr = false;
2341
2342 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2343 if (rc) {
2344 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2345 goto err1;
2346 }
2347
2348 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2349 mr->ibmr.rkey = mr->ibmr.lkey;
2350
2351 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2352 return mr;
2353
2354 err1:
2355 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2356 err0:
2357 kfree(mr);
2358 return ERR_PTR(rc);
2359 }
2360
2361 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2362 enum ib_mr_type mr_type, u32 max_num_sg)
2363 {
2364 struct qedr_dev *dev;
2365 struct qedr_mr *mr;
2366
2367 if (mr_type != IB_MR_TYPE_MEM_REG)
2368 return ERR_PTR(-EINVAL);
2369
2370 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2371
2372 if (IS_ERR(mr))
2373 return ERR_PTR(-EINVAL);
2374
2375 dev = mr->dev;
2376
2377 return &mr->ibmr;
2378 }
2379
2380 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2381 {
2382 struct qedr_mr *mr = get_qedr_mr(ibmr);
2383 struct qedr_pbl *pbl_table;
2384 struct regpair *pbe;
2385 u32 pbes_in_page;
2386
2387 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2388 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2389 return -ENOMEM;
2390 }
2391
2392 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2393 mr->npages, addr);
2394
2395 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2396 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2397 pbe = (struct regpair *)pbl_table->va;
2398 pbe += mr->npages % pbes_in_page;
2399 pbe->lo = cpu_to_le32((u32)addr);
2400 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2401
2402 mr->npages++;
2403
2404 return 0;
2405 }
2406
2407 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2408 {
2409 int work = info->completed - info->completed_handled - 1;
2410
2411 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2412 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2413 struct qedr_pbl *pbl;
2414
2415 /* Free all the page list that are possible to be freed
2416 * (all the ones that were invalidated), under the assumption
2417 * that if an FMR was completed successfully that means that
2418 * if there was an invalidate operation before it also ended
2419 */
2420 pbl = list_first_entry(&info->inuse_pbl_list,
2421 struct qedr_pbl, list_entry);
2422 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2423 info->completed_handled++;
2424 }
2425 }
2426
2427 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2428 int sg_nents, unsigned int *sg_offset)
2429 {
2430 struct qedr_mr *mr = get_qedr_mr(ibmr);
2431
2432 mr->npages = 0;
2433
2434 handle_completed_mrs(mr->dev, &mr->info);
2435 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2436 }
2437
2438 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2439 {
2440 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2441 struct qedr_pd *pd = get_qedr_pd(ibpd);
2442 struct qedr_mr *mr;
2443 int rc;
2444
2445 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2446 if (!mr)
2447 return ERR_PTR(-ENOMEM);
2448
2449 mr->type = QEDR_MR_DMA;
2450
2451 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2452 if (rc) {
2453 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2454 goto err1;
2455 }
2456
2457 /* index only, 18 bit long, lkey = itid << 8 | key */
2458 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2459 mr->hw_mr.pd = pd->pd_id;
2460 mr->hw_mr.local_read = 1;
2461 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2462 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2463 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2464 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2465 mr->hw_mr.dma_mr = true;
2466
2467 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2468 if (rc) {
2469 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2470 goto err2;
2471 }
2472
2473 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2474 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2475 mr->hw_mr.remote_atomic)
2476 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2477
2478 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2479 return &mr->ibmr;
2480
2481 err2:
2482 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2483 err1:
2484 kfree(mr);
2485 return ERR_PTR(rc);
2486 }
2487
2488 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2489 {
2490 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2491 }
2492
2493 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2494 {
2495 int i, len = 0;
2496
2497 for (i = 0; i < num_sge; i++)
2498 len += sg_list[i].length;
2499
2500 return len;
2501 }
2502
2503 static void swap_wqe_data64(u64 *p)
2504 {
2505 int i;
2506
2507 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2508 *p = cpu_to_be64(cpu_to_le64(*p));
2509 }
2510
2511 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2512 struct qedr_qp *qp, u8 *wqe_size,
2513 struct ib_send_wr *wr,
2514 struct ib_send_wr **bad_wr, u8 *bits,
2515 u8 bit)
2516 {
2517 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2518 char *seg_prt, *wqe;
2519 int i, seg_siz;
2520
2521 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2522 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2523 *bad_wr = wr;
2524 return 0;
2525 }
2526
2527 if (!data_size)
2528 return data_size;
2529
2530 *bits |= bit;
2531
2532 seg_prt = NULL;
2533 wqe = NULL;
2534 seg_siz = 0;
2535
2536 /* Copy data inline */
2537 for (i = 0; i < wr->num_sge; i++) {
2538 u32 len = wr->sg_list[i].length;
2539 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2540
2541 while (len > 0) {
2542 u32 cur;
2543
2544 /* New segment required */
2545 if (!seg_siz) {
2546 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2547 seg_prt = wqe;
2548 seg_siz = sizeof(struct rdma_sq_common_wqe);
2549 (*wqe_size)++;
2550 }
2551
2552 /* Calculate currently allowed length */
2553 cur = min_t(u32, len, seg_siz);
2554 memcpy(seg_prt, src, cur);
2555
2556 /* Update segment variables */
2557 seg_prt += cur;
2558 seg_siz -= cur;
2559
2560 /* Update sge variables */
2561 src += cur;
2562 len -= cur;
2563
2564 /* Swap fully-completed segments */
2565 if (!seg_siz)
2566 swap_wqe_data64((u64 *)wqe);
2567 }
2568 }
2569
2570 /* swap last not completed segment */
2571 if (seg_siz)
2572 swap_wqe_data64((u64 *)wqe);
2573
2574 return data_size;
2575 }
2576
2577 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2578 do { \
2579 DMA_REGPAIR_LE(sge->addr, vaddr); \
2580 (sge)->length = cpu_to_le32(vlength); \
2581 (sge)->flags = cpu_to_le32(vflags); \
2582 } while (0)
2583
2584 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2585 do { \
2586 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2587 (hdr)->num_sges = num_sge; \
2588 } while (0)
2589
2590 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2591 do { \
2592 DMA_REGPAIR_LE(sge->addr, vaddr); \
2593 (sge)->length = cpu_to_le32(vlength); \
2594 (sge)->l_key = cpu_to_le32(vlkey); \
2595 } while (0)
2596
2597 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2598 struct ib_send_wr *wr)
2599 {
2600 u32 data_size = 0;
2601 int i;
2602
2603 for (i = 0; i < wr->num_sge; i++) {
2604 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2605
2606 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2607 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2608 sge->length = cpu_to_le32(wr->sg_list[i].length);
2609 data_size += wr->sg_list[i].length;
2610 }
2611
2612 if (wqe_size)
2613 *wqe_size += wr->num_sge;
2614
2615 return data_size;
2616 }
2617
2618 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2619 struct qedr_qp *qp,
2620 struct rdma_sq_rdma_wqe_1st *rwqe,
2621 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2622 struct ib_send_wr *wr,
2623 struct ib_send_wr **bad_wr)
2624 {
2625 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2626 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2627
2628 if (wr->send_flags & IB_SEND_INLINE) {
2629 u8 flags = 0;
2630
2631 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2632 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2633 bad_wr, &rwqe->flags, flags);
2634 }
2635
2636 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2637 }
2638
2639 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2640 struct qedr_qp *qp,
2641 struct rdma_sq_send_wqe_1st *swqe,
2642 struct rdma_sq_send_wqe_2st *swqe2,
2643 struct ib_send_wr *wr,
2644 struct ib_send_wr **bad_wr)
2645 {
2646 memset(swqe2, 0, sizeof(*swqe2));
2647 if (wr->send_flags & IB_SEND_INLINE) {
2648 u8 flags = 0;
2649
2650 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2651 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2652 bad_wr, &swqe->flags, flags);
2653 }
2654
2655 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2656 }
2657
2658 static int qedr_prepare_reg(struct qedr_qp *qp,
2659 struct rdma_sq_fmr_wqe_1st *fwqe1,
2660 struct ib_reg_wr *wr)
2661 {
2662 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2663 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2664
2665 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2666 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2667 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2668 fwqe1->l_key = wr->key;
2669
2670 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2671 !!(wr->access & IB_ACCESS_REMOTE_READ));
2672 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2673 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2674 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2675 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2676 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2677 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2678 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2679 fwqe2->fmr_ctrl = 0;
2680
2681 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2682 ilog2(mr->ibmr.page_size) - 12);
2683
2684 fwqe2->length_hi = 0;
2685 fwqe2->length_lo = mr->ibmr.length;
2686 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2687 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2688
2689 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2690
2691 return 0;
2692 }
2693
2694 enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2695 {
2696 switch (opcode) {
2697 case IB_WR_RDMA_WRITE:
2698 case IB_WR_RDMA_WRITE_WITH_IMM:
2699 return IB_WC_RDMA_WRITE;
2700 case IB_WR_SEND_WITH_IMM:
2701 case IB_WR_SEND:
2702 case IB_WR_SEND_WITH_INV:
2703 return IB_WC_SEND;
2704 case IB_WR_RDMA_READ:
2705 return IB_WC_RDMA_READ;
2706 case IB_WR_ATOMIC_CMP_AND_SWP:
2707 return IB_WC_COMP_SWAP;
2708 case IB_WR_ATOMIC_FETCH_AND_ADD:
2709 return IB_WC_FETCH_ADD;
2710 case IB_WR_REG_MR:
2711 return IB_WC_REG_MR;
2712 case IB_WR_LOCAL_INV:
2713 return IB_WC_LOCAL_INV;
2714 default:
2715 return IB_WC_SEND;
2716 }
2717 }
2718
2719 inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2720 {
2721 int wq_is_full, err_wr, pbl_is_full;
2722 struct qedr_dev *dev = qp->dev;
2723
2724 /* prevent SQ overflow and/or processing of a bad WR */
2725 err_wr = wr->num_sge > qp->sq.max_sges;
2726 wq_is_full = qedr_wq_is_full(&qp->sq);
2727 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2728 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2729 if (wq_is_full || err_wr || pbl_is_full) {
2730 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2731 DP_ERR(dev,
2732 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2733 qp);
2734 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2735 }
2736
2737 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2738 DP_ERR(dev,
2739 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2740 qp);
2741 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2742 }
2743
2744 if (pbl_is_full &&
2745 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2746 DP_ERR(dev,
2747 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2748 qp);
2749 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2750 }
2751 return false;
2752 }
2753 return true;
2754 }
2755
2756 int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2757 struct ib_send_wr **bad_wr)
2758 {
2759 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2760 struct qedr_qp *qp = get_qedr_qp(ibqp);
2761 struct rdma_sq_atomic_wqe_1st *awqe1;
2762 struct rdma_sq_atomic_wqe_2nd *awqe2;
2763 struct rdma_sq_atomic_wqe_3rd *awqe3;
2764 struct rdma_sq_send_wqe_2st *swqe2;
2765 struct rdma_sq_local_inv_wqe *iwqe;
2766 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2767 struct rdma_sq_send_wqe_1st *swqe;
2768 struct rdma_sq_rdma_wqe_1st *rwqe;
2769 struct rdma_sq_fmr_wqe_1st *fwqe1;
2770 struct rdma_sq_common_wqe *wqe;
2771 u32 length;
2772 int rc = 0;
2773 bool comp;
2774
2775 if (!qedr_can_post_send(qp, wr)) {
2776 *bad_wr = wr;
2777 return -ENOMEM;
2778 }
2779
2780 wqe = qed_chain_produce(&qp->sq.pbl);
2781 qp->wqe_wr_id[qp->sq.prod].signaled =
2782 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2783
2784 wqe->flags = 0;
2785 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2786 !!(wr->send_flags & IB_SEND_SOLICITED));
2787 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2788 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2789 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2790 !!(wr->send_flags & IB_SEND_FENCE));
2791 wqe->prev_wqe_size = qp->prev_wqe_size;
2792
2793 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2794
2795 switch (wr->opcode) {
2796 case IB_WR_SEND_WITH_IMM:
2797 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2798 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2799 swqe->wqe_size = 2;
2800 swqe2 = qed_chain_produce(&qp->sq.pbl);
2801
2802 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2803 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2804 wr, bad_wr);
2805 swqe->length = cpu_to_le32(length);
2806 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2807 qp->prev_wqe_size = swqe->wqe_size;
2808 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2809 break;
2810 case IB_WR_SEND:
2811 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2812 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2813
2814 swqe->wqe_size = 2;
2815 swqe2 = qed_chain_produce(&qp->sq.pbl);
2816 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2817 wr, bad_wr);
2818 swqe->length = cpu_to_le32(length);
2819 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2820 qp->prev_wqe_size = swqe->wqe_size;
2821 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2822 break;
2823 case IB_WR_SEND_WITH_INV:
2824 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2825 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2826 swqe2 = qed_chain_produce(&qp->sq.pbl);
2827 swqe->wqe_size = 2;
2828 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2829 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2830 wr, bad_wr);
2831 swqe->length = cpu_to_le32(length);
2832 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2833 qp->prev_wqe_size = swqe->wqe_size;
2834 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2835 break;
2836
2837 case IB_WR_RDMA_WRITE_WITH_IMM:
2838 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2839 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2840
2841 rwqe->wqe_size = 2;
2842 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2843 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2844 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2845 wr, bad_wr);
2846 rwqe->length = cpu_to_le32(length);
2847 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2848 qp->prev_wqe_size = rwqe->wqe_size;
2849 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2850 break;
2851 case IB_WR_RDMA_WRITE:
2852 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2853 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2854
2855 rwqe->wqe_size = 2;
2856 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2857 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2858 wr, bad_wr);
2859 rwqe->length = cpu_to_le32(length);
2860 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2861 qp->prev_wqe_size = rwqe->wqe_size;
2862 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2863 break;
2864 case IB_WR_RDMA_READ_WITH_INV:
2865 DP_ERR(dev,
2866 "RDMA READ WITH INVALIDATE not supported\n");
2867 *bad_wr = wr;
2868 rc = -EINVAL;
2869 break;
2870
2871 case IB_WR_RDMA_READ:
2872 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2873 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2874
2875 rwqe->wqe_size = 2;
2876 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2877 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2878 wr, bad_wr);
2879 rwqe->length = cpu_to_le32(length);
2880 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2881 qp->prev_wqe_size = rwqe->wqe_size;
2882 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2883 break;
2884
2885 case IB_WR_ATOMIC_CMP_AND_SWP:
2886 case IB_WR_ATOMIC_FETCH_AND_ADD:
2887 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2888 awqe1->wqe_size = 4;
2889
2890 awqe2 = qed_chain_produce(&qp->sq.pbl);
2891 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2892 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2893
2894 awqe3 = qed_chain_produce(&qp->sq.pbl);
2895
2896 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2897 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2898 DMA_REGPAIR_LE(awqe3->swap_data,
2899 atomic_wr(wr)->compare_add);
2900 } else {
2901 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2902 DMA_REGPAIR_LE(awqe3->swap_data,
2903 atomic_wr(wr)->swap);
2904 DMA_REGPAIR_LE(awqe3->cmp_data,
2905 atomic_wr(wr)->compare_add);
2906 }
2907
2908 qedr_prepare_sq_sges(qp, NULL, wr);
2909
2910 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2911 qp->prev_wqe_size = awqe1->wqe_size;
2912 break;
2913
2914 case IB_WR_LOCAL_INV:
2915 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2916 iwqe->wqe_size = 1;
2917
2918 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2919 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2920 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2921 qp->prev_wqe_size = iwqe->wqe_size;
2922 break;
2923 case IB_WR_REG_MR:
2924 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2925 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2926 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2927 fwqe1->wqe_size = 2;
2928
2929 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2930 if (rc) {
2931 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2932 *bad_wr = wr;
2933 break;
2934 }
2935
2936 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2937 qp->prev_wqe_size = fwqe1->wqe_size;
2938 break;
2939 default:
2940 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2941 rc = -EINVAL;
2942 *bad_wr = wr;
2943 break;
2944 }
2945
2946 if (*bad_wr) {
2947 u16 value;
2948
2949 /* Restore prod to its position before
2950 * this WR was processed
2951 */
2952 value = le16_to_cpu(qp->sq.db_data.data.value);
2953 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2954
2955 /* Restore prev_wqe_size */
2956 qp->prev_wqe_size = wqe->prev_wqe_size;
2957 rc = -EINVAL;
2958 DP_ERR(dev, "POST SEND FAILED\n");
2959 }
2960
2961 return rc;
2962 }
2963
2964 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2965 struct ib_send_wr **bad_wr)
2966 {
2967 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2968 struct qedr_qp *qp = get_qedr_qp(ibqp);
2969 unsigned long flags;
2970 int rc = 0;
2971
2972 *bad_wr = NULL;
2973
2974 if (qp->qp_type == IB_QPT_GSI)
2975 return qedr_gsi_post_send(ibqp, wr, bad_wr);
2976
2977 spin_lock_irqsave(&qp->q_lock, flags);
2978
2979 if ((qp->state == QED_ROCE_QP_STATE_RESET) ||
2980 (qp->state == QED_ROCE_QP_STATE_ERR)) {
2981 spin_unlock_irqrestore(&qp->q_lock, flags);
2982 *bad_wr = wr;
2983 DP_DEBUG(dev, QEDR_MSG_CQ,
2984 "QP in wrong state! QP icid=0x%x state %d\n",
2985 qp->icid, qp->state);
2986 return -EINVAL;
2987 }
2988
2989 while (wr) {
2990 rc = __qedr_post_send(ibqp, wr, bad_wr);
2991 if (rc)
2992 break;
2993
2994 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
2995
2996 qedr_inc_sw_prod(&qp->sq);
2997
2998 qp->sq.db_data.data.value++;
2999
3000 wr = wr->next;
3001 }
3002
3003 /* Trigger doorbell
3004 * If there was a failure in the first WR then it will be triggered in
3005 * vane. However this is not harmful (as long as the producer value is
3006 * unchanged). For performance reasons we avoid checking for this
3007 * redundant doorbell.
3008 */
3009 wmb();
3010 writel(qp->sq.db_data.raw, qp->sq.db);
3011
3012 /* Make sure write sticks */
3013 mmiowb();
3014
3015 spin_unlock_irqrestore(&qp->q_lock, flags);
3016
3017 return rc;
3018 }
3019
3020 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3021 struct ib_recv_wr **bad_wr)
3022 {
3023 struct qedr_qp *qp = get_qedr_qp(ibqp);
3024 struct qedr_dev *dev = qp->dev;
3025 unsigned long flags;
3026 int status = 0;
3027
3028 if (qp->qp_type == IB_QPT_GSI)
3029 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3030
3031 spin_lock_irqsave(&qp->q_lock, flags);
3032
3033 if ((qp->state == QED_ROCE_QP_STATE_RESET) ||
3034 (qp->state == QED_ROCE_QP_STATE_ERR)) {
3035 spin_unlock_irqrestore(&qp->q_lock, flags);
3036 *bad_wr = wr;
3037 return -EINVAL;
3038 }
3039
3040 while (wr) {
3041 int i;
3042
3043 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3044 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3045 wr->num_sge > qp->rq.max_sges) {
3046 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3047 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3048 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3049 qp->rq.max_sges);
3050 status = -ENOMEM;
3051 *bad_wr = wr;
3052 break;
3053 }
3054 for (i = 0; i < wr->num_sge; i++) {
3055 u32 flags = 0;
3056 struct rdma_rq_sge *rqe =
3057 qed_chain_produce(&qp->rq.pbl);
3058
3059 /* First one must include the number
3060 * of SGE in the list
3061 */
3062 if (!i)
3063 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3064 wr->num_sge);
3065
3066 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3067 wr->sg_list[i].lkey);
3068
3069 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3070 wr->sg_list[i].length, flags);
3071 }
3072
3073 /* Special case of no sges. FW requires between 1-4 sges...
3074 * in this case we need to post 1 sge with length zero. this is
3075 * because rdma write with immediate consumes an RQ.
3076 */
3077 if (!wr->num_sge) {
3078 u32 flags = 0;
3079 struct rdma_rq_sge *rqe =
3080 qed_chain_produce(&qp->rq.pbl);
3081
3082 /* First one must include the number
3083 * of SGE in the list
3084 */
3085 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3086 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3087
3088 RQ_SGE_SET(rqe, 0, 0, flags);
3089 i = 1;
3090 }
3091
3092 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3093 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3094
3095 qedr_inc_sw_prod(&qp->rq);
3096
3097 /* Flush all the writes before signalling doorbell */
3098 wmb();
3099
3100 qp->rq.db_data.data.value++;
3101
3102 writel(qp->rq.db_data.raw, qp->rq.db);
3103
3104 /* Make sure write sticks */
3105 mmiowb();
3106
3107 wr = wr->next;
3108 }
3109
3110 spin_unlock_irqrestore(&qp->q_lock, flags);
3111
3112 return status;
3113 }
3114
3115 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3116 {
3117 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3118
3119 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3120 cq->pbl_toggle;
3121 }
3122
3123 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3124 {
3125 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3126 struct qedr_qp *qp;
3127
3128 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3129 resp_cqe->qp_handle.lo,
3130 u64);
3131 return qp;
3132 }
3133
3134 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3135 {
3136 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3137
3138 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3139 }
3140
3141 /* Return latest CQE (needs processing) */
3142 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3143 {
3144 return cq->latest_cqe;
3145 }
3146
3147 /* In fmr we need to increase the number of fmr completed counter for the fmr
3148 * algorithm determining whether we can free a pbl or not.
3149 * we need to perform this whether the work request was signaled or not. for
3150 * this purpose we call this function from the condition that checks if a wr
3151 * should be skipped, to make sure we don't miss it ( possibly this fmr
3152 * operation was not signalted)
3153 */
3154 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3155 {
3156 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3157 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3158 }
3159
3160 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3161 struct qedr_cq *cq, int num_entries,
3162 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3163 int force)
3164 {
3165 u16 cnt = 0;
3166
3167 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3168 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3169 qedr_chk_if_fmr(qp);
3170 /* skip WC */
3171 goto next_cqe;
3172 }
3173
3174 /* fill WC */
3175 wc->status = status;
3176 wc->wc_flags = 0;
3177 wc->src_qp = qp->id;
3178 wc->qp = &qp->ibqp;
3179
3180 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3181 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3182
3183 switch (wc->opcode) {
3184 case IB_WC_RDMA_WRITE:
3185 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3186 break;
3187 case IB_WC_COMP_SWAP:
3188 case IB_WC_FETCH_ADD:
3189 wc->byte_len = 8;
3190 break;
3191 case IB_WC_REG_MR:
3192 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3193 break;
3194 default:
3195 break;
3196 }
3197
3198 num_entries--;
3199 wc++;
3200 cnt++;
3201 next_cqe:
3202 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3203 qed_chain_consume(&qp->sq.pbl);
3204 qedr_inc_sw_cons(&qp->sq);
3205 }
3206
3207 return cnt;
3208 }
3209
3210 static int qedr_poll_cq_req(struct qedr_dev *dev,
3211 struct qedr_qp *qp, struct qedr_cq *cq,
3212 int num_entries, struct ib_wc *wc,
3213 struct rdma_cqe_requester *req)
3214 {
3215 int cnt = 0;
3216
3217 switch (req->status) {
3218 case RDMA_CQE_REQ_STS_OK:
3219 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3220 IB_WC_SUCCESS, 0);
3221 break;
3222 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3223 DP_ERR(dev,
3224 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3225 cq->icid, qp->icid);
3226 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3227 IB_WC_WR_FLUSH_ERR, 0);
3228 break;
3229 default:
3230 /* process all WQE before the cosumer */
3231 qp->state = QED_ROCE_QP_STATE_ERR;
3232 cnt = process_req(dev, qp, cq, num_entries, wc,
3233 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3234 wc += cnt;
3235 /* if we have extra WC fill it with actual error info */
3236 if (cnt < num_entries) {
3237 enum ib_wc_status wc_status;
3238
3239 switch (req->status) {
3240 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3241 DP_ERR(dev,
3242 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3243 cq->icid, qp->icid);
3244 wc_status = IB_WC_BAD_RESP_ERR;
3245 break;
3246 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3247 DP_ERR(dev,
3248 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3249 cq->icid, qp->icid);
3250 wc_status = IB_WC_LOC_LEN_ERR;
3251 break;
3252 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3253 DP_ERR(dev,
3254 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3255 cq->icid, qp->icid);
3256 wc_status = IB_WC_LOC_QP_OP_ERR;
3257 break;
3258 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3259 DP_ERR(dev,
3260 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3261 cq->icid, qp->icid);
3262 wc_status = IB_WC_LOC_PROT_ERR;
3263 break;
3264 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3265 DP_ERR(dev,
3266 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3267 cq->icid, qp->icid);
3268 wc_status = IB_WC_MW_BIND_ERR;
3269 break;
3270 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3271 DP_ERR(dev,
3272 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3273 cq->icid, qp->icid);
3274 wc_status = IB_WC_REM_INV_REQ_ERR;
3275 break;
3276 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3277 DP_ERR(dev,
3278 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3279 cq->icid, qp->icid);
3280 wc_status = IB_WC_REM_ACCESS_ERR;
3281 break;
3282 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3283 DP_ERR(dev,
3284 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3285 cq->icid, qp->icid);
3286 wc_status = IB_WC_REM_OP_ERR;
3287 break;
3288 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3289 DP_ERR(dev,
3290 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3291 cq->icid, qp->icid);
3292 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3293 break;
3294 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3295 DP_ERR(dev,
3296 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3297 cq->icid, qp->icid);
3298 wc_status = IB_WC_RETRY_EXC_ERR;
3299 break;
3300 default:
3301 DP_ERR(dev,
3302 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3303 cq->icid, qp->icid);
3304 wc_status = IB_WC_GENERAL_ERR;
3305 }
3306 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3307 wc_status, 1);
3308 }
3309 }
3310
3311 return cnt;
3312 }
3313
3314 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3315 struct qedr_cq *cq, struct ib_wc *wc,
3316 struct rdma_cqe_responder *resp, u64 wr_id)
3317 {
3318 enum ib_wc_status wc_status = IB_WC_SUCCESS;
3319 u8 flags;
3320
3321 wc->opcode = IB_WC_RECV;
3322 wc->wc_flags = 0;
3323
3324 switch (resp->status) {
3325 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3326 wc_status = IB_WC_LOC_ACCESS_ERR;
3327 break;
3328 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3329 wc_status = IB_WC_LOC_LEN_ERR;
3330 break;
3331 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3332 wc_status = IB_WC_LOC_QP_OP_ERR;
3333 break;
3334 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3335 wc_status = IB_WC_LOC_PROT_ERR;
3336 break;
3337 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3338 wc_status = IB_WC_MW_BIND_ERR;
3339 break;
3340 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3341 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
3342 break;
3343 case RDMA_CQE_RESP_STS_OK:
3344 wc_status = IB_WC_SUCCESS;
3345 wc->byte_len = le32_to_cpu(resp->length);
3346
3347 flags = resp->flags & QEDR_RESP_RDMA_IMM;
3348
3349 if (flags == QEDR_RESP_RDMA_IMM)
3350 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3351
3352 if (flags == QEDR_RESP_RDMA_IMM || flags == QEDR_RESP_IMM) {
3353 wc->ex.imm_data =
3354 le32_to_cpu(resp->imm_data_or_inv_r_Key);
3355 wc->wc_flags |= IB_WC_WITH_IMM;
3356 }
3357 break;
3358 default:
3359 wc->status = IB_WC_GENERAL_ERR;
3360 DP_ERR(dev, "Invalid CQE status detected\n");
3361 }
3362
3363 /* fill WC */
3364 wc->status = wc_status;
3365 wc->src_qp = qp->id;
3366 wc->qp = &qp->ibqp;
3367 wc->wr_id = wr_id;
3368 }
3369
3370 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3371 struct qedr_cq *cq, struct ib_wc *wc,
3372 struct rdma_cqe_responder *resp)
3373 {
3374 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3375
3376 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3377
3378 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3379 qed_chain_consume(&qp->rq.pbl);
3380 qedr_inc_sw_cons(&qp->rq);
3381
3382 return 1;
3383 }
3384
3385 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3386 int num_entries, struct ib_wc *wc, u16 hw_cons)
3387 {
3388 u16 cnt = 0;
3389
3390 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3391 /* fill WC */
3392 wc->status = IB_WC_WR_FLUSH_ERR;
3393 wc->wc_flags = 0;
3394 wc->src_qp = qp->id;
3395 wc->byte_len = 0;
3396 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3397 wc->qp = &qp->ibqp;
3398 num_entries--;
3399 wc++;
3400 cnt++;
3401 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3402 qed_chain_consume(&qp->rq.pbl);
3403 qedr_inc_sw_cons(&qp->rq);
3404 }
3405
3406 return cnt;
3407 }
3408
3409 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3410 struct rdma_cqe_responder *resp, int *update)
3411 {
3412 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3413 consume_cqe(cq);
3414 *update |= 1;
3415 }
3416 }
3417
3418 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3419 struct qedr_cq *cq, int num_entries,
3420 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3421 int *update)
3422 {
3423 int cnt;
3424
3425 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3426 cnt = process_resp_flush(qp, cq, num_entries, wc,
3427 resp->rq_cons);
3428 try_consume_resp_cqe(cq, qp, resp, update);
3429 } else {
3430 cnt = process_resp_one(dev, qp, cq, wc, resp);
3431 consume_cqe(cq);
3432 *update |= 1;
3433 }
3434
3435 return cnt;
3436 }
3437
3438 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3439 struct rdma_cqe_requester *req, int *update)
3440 {
3441 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3442 consume_cqe(cq);
3443 *update |= 1;
3444 }
3445 }
3446
3447 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3448 {
3449 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3450 struct qedr_cq *cq = get_qedr_cq(ibcq);
3451 union rdma_cqe *cqe = cq->latest_cqe;
3452 u32 old_cons, new_cons;
3453 unsigned long flags;
3454 int update = 0;
3455 int done = 0;
3456
3457 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3458 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3459
3460 spin_lock_irqsave(&cq->cq_lock, flags);
3461 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3462 while (num_entries && is_valid_cqe(cq, cqe)) {
3463 struct qedr_qp *qp;
3464 int cnt = 0;
3465
3466 /* prevent speculative reads of any field of CQE */
3467 rmb();
3468
3469 qp = cqe_get_qp(cqe);
3470 if (!qp) {
3471 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3472 break;
3473 }
3474
3475 wc->qp = &qp->ibqp;
3476
3477 switch (cqe_get_type(cqe)) {
3478 case RDMA_CQE_TYPE_REQUESTER:
3479 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3480 &cqe->req);
3481 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3482 break;
3483 case RDMA_CQE_TYPE_RESPONDER_RQ:
3484 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3485 &cqe->resp, &update);
3486 break;
3487 case RDMA_CQE_TYPE_INVALID:
3488 default:
3489 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3490 cqe_get_type(cqe));
3491 }
3492 num_entries -= cnt;
3493 wc += cnt;
3494 done += cnt;
3495
3496 cqe = get_cqe(cq);
3497 }
3498 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3499
3500 cq->cq_cons += new_cons - old_cons;
3501
3502 if (update)
3503 /* doorbell notifies abount latest VALID entry,
3504 * but chain already point to the next INVALID one
3505 */
3506 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3507
3508 spin_unlock_irqrestore(&cq->cq_lock, flags);
3509 return done;
3510 }
3511
3512 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3513 u8 port_num,
3514 const struct ib_wc *in_wc,
3515 const struct ib_grh *in_grh,
3516 const struct ib_mad_hdr *mad_hdr,
3517 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3518 size_t *out_mad_size, u16 *out_mad_pkey_index)
3519 {
3520 struct qedr_dev *dev = get_qedr_dev(ibdev);
3521
3522 DP_DEBUG(dev, QEDR_MSG_GSI,
3523 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3524 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3525 mad_hdr->class_specific, mad_hdr->class_version,
3526 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3527 return IB_MAD_RESULT_SUCCESS;
3528 }
3529
3530 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3531 struct ib_port_immutable *immutable)
3532 {
3533 struct ib_port_attr attr;
3534 int err;
3535
3536 err = qedr_query_port(ibdev, port_num, &attr);
3537 if (err)
3538 return err;
3539
3540 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3541 immutable->gid_tbl_len = attr.gid_tbl_len;
3542 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3543 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3544 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3545
3546 return 0;
3547 }
Ubuntu Jammy Linux kernel mirror