]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - drivers/infiniband/hw/qedr/verbs.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
qedr: return -EINVAL if pd is null and avoid null ptr dereference
[mirror_ubuntu-jammy-kernel.git] / drivers / infiniband / hw / qedr / verbs.c
1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45
46 #include "qedr_hsi.h"
47 #include <linux/qed/qed_if.h>
48 #include "qedr.h"
49 #include "verbs.h"
50 #include <rdma/qedr-abi.h>
51 #include "qedr_cm.h"
52
53 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56 {
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62 }
63
64 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66 {
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83 }
84
85 int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88 {
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99 }
100
101 int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103 {
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114 }
115
116 int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118 {
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170 }
171
172 #define QEDR_SPEED_SDR (1)
173 #define QEDR_SPEED_DDR (2)
174 #define QEDR_SPEED_QDR (4)
175 #define QEDR_SPEED_FDR10 (8)
176 #define QEDR_SPEED_FDR (16)
177 #define QEDR_SPEED_EDR (32)
178
179 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181 {
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222 }
223
224 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225 {
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241 memset(attr, 0, sizeof(*attr));
242
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267 }
268
269 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271 {
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281 }
282
283 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285 {
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313 }
314
315 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317 {
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335 }
336
337 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339 {
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394 err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397 }
398
399 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400 {
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419 }
420
421 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422 {
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468 }
469
470 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472 {
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
474 struct qedr_ucontext *uctx = NULL;
475 struct qedr_alloc_pd_uresp uresp;
476 struct qedr_pd *pd;
477 u16 pd_id;
478 int rc;
479
480 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
481 (udata && context) ? "User Lib" : "Kernel");
482
483 if (!dev->rdma_ctx) {
484 DP_ERR(dev, "invlaid RDMA context\n");
485 return ERR_PTR(-EINVAL);
486 }
487
488 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
489 if (!pd)
490 return ERR_PTR(-ENOMEM);
491
492 dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
493
494 uresp.pd_id = pd_id;
495 pd->pd_id = pd_id;
496
497 if (udata && context) {
498 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
499 if (rc)
500 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
501 uctx = get_qedr_ucontext(context);
502 uctx->pd = pd;
503 pd->uctx = uctx;
504 }
505
506 return &pd->ibpd;
507 }
508
509 int qedr_dealloc_pd(struct ib_pd *ibpd)
510 {
511 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
512 struct qedr_pd *pd = get_qedr_pd(ibpd);
513
514 if (!pd) {
515 pr_err("Invalid PD received in dealloc_pd\n");
516 return -EINVAL;
517 }
518
519 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
520 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
521
522 kfree(pd);
523
524 return 0;
525 }
526
527 static void qedr_free_pbl(struct qedr_dev *dev,
528 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
529 {
530 struct pci_dev *pdev = dev->pdev;
531 int i;
532
533 for (i = 0; i < pbl_info->num_pbls; i++) {
534 if (!pbl[i].va)
535 continue;
536 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
537 pbl[i].va, pbl[i].pa);
538 }
539
540 kfree(pbl);
541 }
542
543 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
544 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
545
546 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
547 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
548 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
549
550 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
551 struct qedr_pbl_info *pbl_info,
552 gfp_t flags)
553 {
554 struct pci_dev *pdev = dev->pdev;
555 struct qedr_pbl *pbl_table;
556 dma_addr_t *pbl_main_tbl;
557 dma_addr_t pa;
558 void *va;
559 int i;
560
561 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
562 if (!pbl_table)
563 return ERR_PTR(-ENOMEM);
564
565 for (i = 0; i < pbl_info->num_pbls; i++) {
566 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
567 &pa, flags);
568 if (!va)
569 goto err;
570
571 memset(va, 0, pbl_info->pbl_size);
572 pbl_table[i].va = va;
573 pbl_table[i].pa = pa;
574 }
575
576 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
577 * the first one with physical pointers to all of the rest
578 */
579 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
580 for (i = 0; i < pbl_info->num_pbls - 1; i++)
581 pbl_main_tbl[i] = pbl_table[i + 1].pa;
582
583 return pbl_table;
584
585 err:
586 for (i--; i >= 0; i--)
587 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
588 pbl_table[i].va, pbl_table[i].pa);
589
590 qedr_free_pbl(dev, pbl_info, pbl_table);
591
592 return ERR_PTR(-ENOMEM);
593 }
594
595 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
596 struct qedr_pbl_info *pbl_info,
597 u32 num_pbes, int two_layer_capable)
598 {
599 u32 pbl_capacity;
600 u32 pbl_size;
601 u32 num_pbls;
602
603 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
604 if (num_pbes > MAX_PBES_TWO_LAYER) {
605 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
606 num_pbes);
607 return -EINVAL;
608 }
609
610 /* calculate required pbl page size */
611 pbl_size = MIN_FW_PBL_PAGE_SIZE;
612 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
613 NUM_PBES_ON_PAGE(pbl_size);
614
615 while (pbl_capacity < num_pbes) {
616 pbl_size *= 2;
617 pbl_capacity = pbl_size / sizeof(u64);
618 pbl_capacity = pbl_capacity * pbl_capacity;
619 }
620
621 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
622 num_pbls++; /* One for the layer0 ( points to the pbls) */
623 pbl_info->two_layered = true;
624 } else {
625 /* One layered PBL */
626 num_pbls = 1;
627 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
628 roundup_pow_of_two((num_pbes * sizeof(u64))));
629 pbl_info->two_layered = false;
630 }
631
632 pbl_info->num_pbls = num_pbls;
633 pbl_info->pbl_size = pbl_size;
634 pbl_info->num_pbes = num_pbes;
635
636 DP_DEBUG(dev, QEDR_MSG_MR,
637 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
638 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
639
640 return 0;
641 }
642
643 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
644 struct qedr_pbl *pbl,
645 struct qedr_pbl_info *pbl_info)
646 {
647 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
648 struct qedr_pbl *pbl_tbl;
649 struct scatterlist *sg;
650 struct regpair *pbe;
651 int entry;
652 u32 addr;
653
654 if (!pbl_info->num_pbes)
655 return;
656
657 /* If we have a two layered pbl, the first pbl points to the rest
658 * of the pbls and the first entry lays on the second pbl in the table
659 */
660 if (pbl_info->two_layered)
661 pbl_tbl = &pbl[1];
662 else
663 pbl_tbl = pbl;
664
665 pbe = (struct regpair *)pbl_tbl->va;
666 if (!pbe) {
667 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
668 return;
669 }
670
671 pbe_cnt = 0;
672
673 shift = ilog2(umem->page_size);
674
675 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
676 pages = sg_dma_len(sg) >> shift;
677 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
678 /* store the page address in pbe */
679 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
680 umem->page_size * pg_cnt);
681 addr = upper_32_bits(sg_dma_address(sg) +
682 umem->page_size * pg_cnt);
683 pbe->hi = cpu_to_le32(addr);
684 pbe_cnt++;
685 total_num_pbes++;
686 pbe++;
687
688 if (total_num_pbes == pbl_info->num_pbes)
689 return;
690
691 /* If the given pbl is full storing the pbes,
692 * move to next pbl.
693 */
694 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
695 pbl_tbl++;
696 pbe = (struct regpair *)pbl_tbl->va;
697 pbe_cnt = 0;
698 }
699 }
700 }
701 }
702
703 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
704 struct qedr_cq *cq, struct ib_udata *udata)
705 {
706 struct qedr_create_cq_uresp uresp;
707 int rc;
708
709 memset(&uresp, 0, sizeof(uresp));
710
711 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
712 uresp.icid = cq->icid;
713
714 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
715 if (rc)
716 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
717
718 return rc;
719 }
720
721 static void consume_cqe(struct qedr_cq *cq)
722 {
723 if (cq->latest_cqe == cq->toggle_cqe)
724 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
725
726 cq->latest_cqe = qed_chain_consume(&cq->pbl);
727 }
728
729 static inline int qedr_align_cq_entries(int entries)
730 {
731 u64 size, aligned_size;
732
733 /* We allocate an extra entry that we don't report to the FW. */
734 size = (entries + 1) * QEDR_CQE_SIZE;
735 aligned_size = ALIGN(size, PAGE_SIZE);
736
737 return aligned_size / QEDR_CQE_SIZE;
738 }
739
740 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
741 struct qedr_dev *dev,
742 struct qedr_userq *q,
743 u64 buf_addr, size_t buf_len,
744 int access, int dmasync)
745 {
746 int page_cnt;
747 int rc;
748
749 q->buf_addr = buf_addr;
750 q->buf_len = buf_len;
751 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
752 if (IS_ERR(q->umem)) {
753 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
754 PTR_ERR(q->umem));
755 return PTR_ERR(q->umem);
756 }
757
758 page_cnt = ib_umem_page_count(q->umem);
759 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
760 if (rc)
761 goto err0;
762
763 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
764 if (IS_ERR_OR_NULL(q->pbl_tbl))
765 goto err0;
766
767 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
768
769 return 0;
770
771 err0:
772 ib_umem_release(q->umem);
773
774 return rc;
775 }
776
777 static inline void qedr_init_cq_params(struct qedr_cq *cq,
778 struct qedr_ucontext *ctx,
779 struct qedr_dev *dev, int vector,
780 int chain_entries, int page_cnt,
781 u64 pbl_ptr,
782 struct qed_rdma_create_cq_in_params
783 *params)
784 {
785 memset(params, 0, sizeof(*params));
786 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
787 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
788 params->cnq_id = vector;
789 params->cq_size = chain_entries - 1;
790 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
791 params->pbl_num_pages = page_cnt;
792 params->pbl_ptr = pbl_ptr;
793 params->pbl_two_level = 0;
794 }
795
796 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
797 {
798 /* Flush data before signalling doorbell */
799 wmb();
800 cq->db.data.agg_flags = flags;
801 cq->db.data.value = cpu_to_le32(cons);
802 writeq(cq->db.raw, cq->db_addr);
803
804 /* Make sure write would stick */
805 mmiowb();
806 }
807
808 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
809 {
810 struct qedr_cq *cq = get_qedr_cq(ibcq);
811 unsigned long sflags;
812
813 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
814 return 0;
815
816 spin_lock_irqsave(&cq->cq_lock, sflags);
817
818 cq->arm_flags = 0;
819
820 if (flags & IB_CQ_SOLICITED)
821 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
822
823 if (flags & IB_CQ_NEXT_COMP)
824 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
825
826 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
827
828 spin_unlock_irqrestore(&cq->cq_lock, sflags);
829
830 return 0;
831 }
832
833 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
834 const struct ib_cq_init_attr *attr,
835 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
836 {
837 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
838 struct qed_rdma_destroy_cq_out_params destroy_oparams;
839 struct qed_rdma_destroy_cq_in_params destroy_iparams;
840 struct qedr_dev *dev = get_qedr_dev(ibdev);
841 struct qed_rdma_create_cq_in_params params;
842 struct qedr_create_cq_ureq ureq;
843 int vector = attr->comp_vector;
844 int entries = attr->cqe;
845 struct qedr_cq *cq;
846 int chain_entries;
847 int page_cnt;
848 u64 pbl_ptr;
849 u16 icid;
850 int rc;
851
852 DP_DEBUG(dev, QEDR_MSG_INIT,
853 "create_cq: called from %s. entries=%d, vector=%d\n",
854 udata ? "User Lib" : "Kernel", entries, vector);
855
856 if (entries > QEDR_MAX_CQES) {
857 DP_ERR(dev,
858 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
859 entries, QEDR_MAX_CQES);
860 return ERR_PTR(-EINVAL);
861 }
862
863 chain_entries = qedr_align_cq_entries(entries);
864 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
865
866 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
867 if (!cq)
868 return ERR_PTR(-ENOMEM);
869
870 if (udata) {
871 memset(&ureq, 0, sizeof(ureq));
872 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
873 DP_ERR(dev,
874 "create cq: problem copying data from user space\n");
875 goto err0;
876 }
877
878 if (!ureq.len) {
879 DP_ERR(dev,
880 "create cq: cannot create a cq with 0 entries\n");
881 goto err0;
882 }
883
884 cq->cq_type = QEDR_CQ_TYPE_USER;
885
886 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
887 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
888 if (rc)
889 goto err0;
890
891 pbl_ptr = cq->q.pbl_tbl->pa;
892 page_cnt = cq->q.pbl_info.num_pbes;
893 } else {
894 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
895
896 rc = dev->ops->common->chain_alloc(dev->cdev,
897 QED_CHAIN_USE_TO_CONSUME,
898 QED_CHAIN_MODE_PBL,
899 QED_CHAIN_CNT_TYPE_U32,
900 chain_entries,
901 sizeof(union rdma_cqe),
902 &cq->pbl);
903 if (rc)
904 goto err1;
905
906 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
907 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
908 }
909
910 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
911 pbl_ptr, &params);
912
913 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
914 if (rc)
915 goto err2;
916
917 cq->icid = icid;
918 cq->sig = QEDR_CQ_MAGIC_NUMBER;
919 spin_lock_init(&cq->cq_lock);
920
921 if (ib_ctx) {
922 rc = qedr_copy_cq_uresp(dev, cq, udata);
923 if (rc)
924 goto err3;
925 } else {
926 /* Generate doorbell address. */
927 cq->db_addr = dev->db_addr +
928 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
929 cq->db.data.icid = cq->icid;
930 cq->db.data.params = DB_AGG_CMD_SET <<
931 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
932
933 /* point to the very last element, passing it we will toggle */
934 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
935 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
936 cq->latest_cqe = NULL;
937 consume_cqe(cq);
938 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
939 }
940
941 DP_DEBUG(dev, QEDR_MSG_CQ,
942 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
943 cq->icid, cq, params.cq_size);
944
945 return &cq->ibcq;
946
947 err3:
948 destroy_iparams.icid = cq->icid;
949 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
950 &destroy_oparams);
951 err2:
952 if (udata)
953 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
954 else
955 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
956 err1:
957 if (udata)
958 ib_umem_release(cq->q.umem);
959 err0:
960 kfree(cq);
961 return ERR_PTR(-EINVAL);
962 }
963
964 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
965 {
966 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
967 struct qedr_cq *cq = get_qedr_cq(ibcq);
968
969 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
970
971 return 0;
972 }
973
974 int qedr_destroy_cq(struct ib_cq *ibcq)
975 {
976 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
977 struct qed_rdma_destroy_cq_out_params oparams;
978 struct qed_rdma_destroy_cq_in_params iparams;
979 struct qedr_cq *cq = get_qedr_cq(ibcq);
980
981 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
982
983 /* GSIs CQs are handled by driver, so they don't exist in the FW */
984 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
985 iparams.icid = cq->icid;
986 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
987 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
988 }
989
990 if (ibcq->uobject && ibcq->uobject->context) {
991 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
992 ib_umem_release(cq->q.umem);
993 }
994
995 kfree(cq);
996
997 return 0;
998 }
999
1000 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1001 struct ib_qp_attr *attr,
1002 int attr_mask,
1003 struct qed_rdma_modify_qp_in_params
1004 *qp_params)
1005 {
1006 enum rdma_network_type nw_type;
1007 struct ib_gid_attr gid_attr;
1008 union ib_gid gid;
1009 u32 ipv4_addr;
1010 int rc = 0;
1011 int i;
1012
1013 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1014 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1015 if (rc)
1016 return rc;
1017
1018 if (!memcmp(&gid, &zgid, sizeof(gid)))
1019 return -ENOENT;
1020
1021 if (gid_attr.ndev) {
1022 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1023
1024 dev_put(gid_attr.ndev);
1025 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1026 switch (nw_type) {
1027 case RDMA_NETWORK_IPV6:
1028 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1029 sizeof(qp_params->sgid));
1030 memcpy(&qp_params->dgid.bytes[0],
1031 &attr->ah_attr.grh.dgid,
1032 sizeof(qp_params->dgid));
1033 qp_params->roce_mode = ROCE_V2_IPV6;
1034 SET_FIELD(qp_params->modify_flags,
1035 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1036 break;
1037 case RDMA_NETWORK_IB:
1038 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1039 sizeof(qp_params->sgid));
1040 memcpy(&qp_params->dgid.bytes[0],
1041 &attr->ah_attr.grh.dgid,
1042 sizeof(qp_params->dgid));
1043 qp_params->roce_mode = ROCE_V1;
1044 break;
1045 case RDMA_NETWORK_IPV4:
1046 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1047 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1048 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1049 qp_params->sgid.ipv4_addr = ipv4_addr;
1050 ipv4_addr =
1051 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1052 qp_params->dgid.ipv4_addr = ipv4_addr;
1053 SET_FIELD(qp_params->modify_flags,
1054 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1055 qp_params->roce_mode = ROCE_V2_IPV4;
1056 break;
1057 }
1058 }
1059
1060 for (i = 0; i < 4; i++) {
1061 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1062 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1063 }
1064
1065 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1066 qp_params->vlan_id = 0;
1067
1068 return 0;
1069 }
1070
1071 static void qedr_cleanup_user_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1072 {
1073 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1074 ib_umem_release(qp->usq.umem);
1075 }
1076
1077 static void qedr_cleanup_user_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1078 {
1079 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1080 ib_umem_release(qp->urq.umem);
1081 }
1082
1083 static void qedr_cleanup_kernel_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1084 {
1085 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1086 kfree(qp->wqe_wr_id);
1087 }
1088
1089 static void qedr_cleanup_kernel_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1090 {
1091 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1092 kfree(qp->rqe_wr_id);
1093 }
1094
1095 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1096 struct ib_qp_init_attr *attrs)
1097 {
1098 struct qedr_device_attr *qattr = &dev->attr;
1099
1100 /* QP0... attrs->qp_type == IB_QPT_GSI */
1101 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1102 DP_DEBUG(dev, QEDR_MSG_QP,
1103 "create qp: unsupported qp type=0x%x requested\n",
1104 attrs->qp_type);
1105 return -EINVAL;
1106 }
1107
1108 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1109 DP_ERR(dev,
1110 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1111 attrs->cap.max_send_wr, qattr->max_sqe);
1112 return -EINVAL;
1113 }
1114
1115 if (attrs->cap.max_inline_data > qattr->max_inline) {
1116 DP_ERR(dev,
1117 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1118 attrs->cap.max_inline_data, qattr->max_inline);
1119 return -EINVAL;
1120 }
1121
1122 if (attrs->cap.max_send_sge > qattr->max_sge) {
1123 DP_ERR(dev,
1124 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1125 attrs->cap.max_send_sge, qattr->max_sge);
1126 return -EINVAL;
1127 }
1128
1129 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1130 DP_ERR(dev,
1131 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1132 attrs->cap.max_recv_sge, qattr->max_sge);
1133 return -EINVAL;
1134 }
1135
1136 /* Unprivileged user space cannot create special QP */
1137 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1138 DP_ERR(dev,
1139 "create qp: userspace can't create special QPs of type=0x%x\n",
1140 attrs->qp_type);
1141 return -EINVAL;
1142 }
1143
1144 return 0;
1145 }
1146
1147 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1148 struct qedr_qp *qp)
1149 {
1150 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1151 uresp->rq_icid = qp->icid;
1152 }
1153
1154 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1155 struct qedr_qp *qp)
1156 {
1157 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1158 uresp->sq_icid = qp->icid + 1;
1159 }
1160
1161 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1162 struct qedr_qp *qp, struct ib_udata *udata)
1163 {
1164 struct qedr_create_qp_uresp uresp;
1165 int rc;
1166
1167 memset(&uresp, 0, sizeof(uresp));
1168 qedr_copy_sq_uresp(&uresp, qp);
1169 qedr_copy_rq_uresp(&uresp, qp);
1170
1171 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1172 uresp.qp_id = qp->qp_id;
1173
1174 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1175 if (rc)
1176 DP_ERR(dev,
1177 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1178 qp->icid);
1179
1180 return rc;
1181 }
1182
1183 static void qedr_set_qp_init_params(struct qedr_dev *dev,
1184 struct qedr_qp *qp,
1185 struct qedr_pd *pd,
1186 struct ib_qp_init_attr *attrs)
1187 {
1188 qp->pd = pd;
1189
1190 spin_lock_init(&qp->q_lock);
1191
1192 qp->qp_type = attrs->qp_type;
1193 qp->max_inline_data = attrs->cap.max_inline_data;
1194 qp->sq.max_sges = attrs->cap.max_send_sge;
1195 qp->state = QED_ROCE_QP_STATE_RESET;
1196 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1197 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1198 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1199 qp->dev = dev;
1200
1201 DP_DEBUG(dev, QEDR_MSG_QP,
1202 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1203 pd->pd_id, qp->qp_type, qp->max_inline_data,
1204 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1205 DP_DEBUG(dev, QEDR_MSG_QP,
1206 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1207 qp->sq.max_sges, qp->sq_cq->icid);
1208 qp->rq.max_sges = attrs->cap.max_recv_sge;
1209 DP_DEBUG(dev, QEDR_MSG_QP,
1210 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1211 qp->rq.max_sges, qp->rq_cq->icid);
1212 }
1213
1214 static inline void
1215 qedr_init_qp_user_params(struct qed_rdma_create_qp_in_params *params,
1216 struct qedr_create_qp_ureq *ureq)
1217 {
1218 /* QP handle to be written in CQE */
1219 params->qp_handle_lo = ureq->qp_handle_lo;
1220 params->qp_handle_hi = ureq->qp_handle_hi;
1221 }
1222
1223 static inline void
1224 qedr_init_qp_kernel_doorbell_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1225 {
1226 qp->sq.db = dev->db_addr +
1227 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1228 qp->sq.db_data.data.icid = qp->icid + 1;
1229 }
1230
1231 static inline void
1232 qedr_init_qp_kernel_doorbell_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1233 {
1234 qp->rq.db = dev->db_addr +
1235 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1236 qp->rq.db_data.data.icid = qp->icid;
1237 }
1238
1239 static inline int
1240 qedr_init_qp_kernel_params_rq(struct qedr_dev *dev,
1241 struct qedr_qp *qp, struct ib_qp_init_attr *attrs)
1242 {
1243 /* Allocate driver internal RQ array */
1244 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1245 GFP_KERNEL);
1246 if (!qp->rqe_wr_id)
1247 return -ENOMEM;
1248
1249 DP_DEBUG(dev, QEDR_MSG_QP, "RQ max_wr set to %d.\n", qp->rq.max_wr);
1250
1251 return 0;
1252 }
1253
1254 static inline int
1255 qedr_init_qp_kernel_params_sq(struct qedr_dev *dev,
1256 struct qedr_qp *qp,
1257 struct ib_qp_init_attr *attrs,
1258 struct qed_rdma_create_qp_in_params *params)
1259 {
1260 u32 temp_max_wr;
1261
1262 /* Allocate driver internal SQ array */
1263 temp_max_wr = attrs->cap.max_send_wr * dev->wq_multiplier;
1264 temp_max_wr = min_t(u32, temp_max_wr, dev->attr.max_sqe);
1265
1266 /* temp_max_wr < attr->max_sqe < u16 so the casting is safe */
1267 qp->sq.max_wr = (u16)temp_max_wr;
1268 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1269 GFP_KERNEL);
1270 if (!qp->wqe_wr_id)
1271 return -ENOMEM;
1272
1273 DP_DEBUG(dev, QEDR_MSG_QP, "SQ max_wr set to %d.\n", qp->sq.max_wr);
1274
1275 /* QP handle to be written in CQE */
1276 params->qp_handle_lo = lower_32_bits((uintptr_t)qp);
1277 params->qp_handle_hi = upper_32_bits((uintptr_t)qp);
1278
1279 return 0;
1280 }
1281
1282 static inline int qedr_init_qp_kernel_sq(struct qedr_dev *dev,
1283 struct qedr_qp *qp,
1284 struct ib_qp_init_attr *attrs)
1285 {
1286 u32 n_sq_elems, n_sq_entries;
1287 int rc;
1288
1289 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1290 * the ring. The ring should allow at least a single WR, even if the
1291 * user requested none, due to allocation issues.
1292 */
1293 n_sq_entries = attrs->cap.max_send_wr;
1294 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1295 n_sq_entries = max_t(u32, n_sq_entries, 1);
1296 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1297 rc = dev->ops->common->chain_alloc(dev->cdev,
1298 QED_CHAIN_USE_TO_PRODUCE,
1299 QED_CHAIN_MODE_PBL,
1300 QED_CHAIN_CNT_TYPE_U32,
1301 n_sq_elems,
1302 QEDR_SQE_ELEMENT_SIZE,
1303 &qp->sq.pbl);
1304 if (rc) {
1305 DP_ERR(dev, "failed to allocate QP %p SQ\n", qp);
1306 return rc;
1307 }
1308
1309 DP_DEBUG(dev, QEDR_MSG_SQ,
1310 "SQ Pbl base addr = %llx max_send_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1311 qed_chain_get_pbl_phys(&qp->sq.pbl), attrs->cap.max_send_wr,
1312 n_sq_entries, qed_chain_get_capacity(&qp->sq.pbl), rc);
1313 return 0;
1314 }
1315
1316 static inline int qedr_init_qp_kernel_rq(struct qedr_dev *dev,
1317 struct qedr_qp *qp,
1318 struct ib_qp_init_attr *attrs)
1319 {
1320 u32 n_rq_elems, n_rq_entries;
1321 int rc;
1322
1323 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1324 * the ring. There ring should allow at least a single WR, even if the
1325 * user requested none, due to allocation issues.
1326 */
1327 n_rq_entries = max_t(u32, attrs->cap.max_recv_wr, 1);
1328 n_rq_elems = n_rq_entries * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1329 rc = dev->ops->common->chain_alloc(dev->cdev,
1330 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1331 QED_CHAIN_MODE_PBL,
1332 QED_CHAIN_CNT_TYPE_U32,
1333 n_rq_elems,
1334 QEDR_RQE_ELEMENT_SIZE,
1335 &qp->rq.pbl);
1336
1337 if (rc) {
1338 DP_ERR(dev, "failed to allocate memory for QP %p RQ\n", qp);
1339 return -ENOMEM;
1340 }
1341
1342 DP_DEBUG(dev, QEDR_MSG_RQ,
1343 "RQ Pbl base addr = %llx max_recv_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1344 qed_chain_get_pbl_phys(&qp->rq.pbl), attrs->cap.max_recv_wr,
1345 n_rq_entries, qed_chain_get_capacity(&qp->rq.pbl), rc);
1346
1347 /* n_rq_entries < u16 so the casting is safe */
1348 qp->rq.max_wr = (u16)n_rq_entries;
1349
1350 return 0;
1351 }
1352
1353 static inline void
1354 qedr_init_qp_in_params_sq(struct qedr_dev *dev,
1355 struct qedr_pd *pd,
1356 struct qedr_qp *qp,
1357 struct ib_qp_init_attr *attrs,
1358 struct ib_udata *udata,
1359 struct qed_rdma_create_qp_in_params *params)
1360 {
1361 /* QP handle to be written in an async event */
1362 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
1363 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
1364
1365 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1366 params->fmr_and_reserved_lkey = !udata;
1367 params->pd = pd->pd_id;
1368 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1369 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1370 params->max_sq_sges = 0;
1371 params->stats_queue = 0;
1372
1373 if (udata) {
1374 params->sq_num_pages = qp->usq.pbl_info.num_pbes;
1375 params->sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1376 } else {
1377 params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1378 params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1379 }
1380 }
1381
1382 static inline void
1383 qedr_init_qp_in_params_rq(struct qedr_qp *qp,
1384 struct ib_qp_init_attr *attrs,
1385 struct ib_udata *udata,
1386 struct qed_rdma_create_qp_in_params *params)
1387 {
1388 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1389 params->srq_id = 0;
1390 params->use_srq = false;
1391
1392 if (udata) {
1393 params->rq_num_pages = qp->urq.pbl_info.num_pbes;
1394 params->rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1395 } else {
1396 params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1397 params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1398 }
1399 }
1400
1401 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1402 {
1403 DP_DEBUG(dev, QEDR_MSG_QP,
1404 "create qp: successfully created user QP. qp=%p, sq_addr=0x%llx, sq_len=%zd, rq_addr=0x%llx, rq_len=%zd\n",
1405 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
1406 qp->urq.buf_len);
1407 }
1408
1409 static inline int qedr_init_user_qp(struct ib_ucontext *ib_ctx,
1410 struct qedr_dev *dev,
1411 struct qedr_qp *qp,
1412 struct qedr_create_qp_ureq *ureq)
1413 {
1414 int rc;
1415
1416 /* SQ - read access only (0), dma sync not required (0) */
1417 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq->sq_addr,
1418 ureq->sq_len, 0, 0);
1419 if (rc)
1420 return rc;
1421
1422 /* RQ - read access only (0), dma sync not required (0) */
1423 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq->rq_addr,
1424 ureq->rq_len, 0, 0);
1425
1426 if (rc)
1427 qedr_cleanup_user_sq(dev, qp);
1428 return rc;
1429 }
1430
1431 static inline int
1432 qedr_init_kernel_qp(struct qedr_dev *dev,
1433 struct qedr_qp *qp,
1434 struct ib_qp_init_attr *attrs,
1435 struct qed_rdma_create_qp_in_params *params)
1436 {
1437 int rc;
1438
1439 rc = qedr_init_qp_kernel_sq(dev, qp, attrs);
1440 if (rc) {
1441 DP_ERR(dev, "failed to init kernel QP %p SQ\n", qp);
1442 return rc;
1443 }
1444
1445 rc = qedr_init_qp_kernel_params_sq(dev, qp, attrs, params);
1446 if (rc) {
1447 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1448 DP_ERR(dev, "failed to init kernel QP %p SQ params\n", qp);
1449 return rc;
1450 }
1451
1452 rc = qedr_init_qp_kernel_rq(dev, qp, attrs);
1453 if (rc) {
1454 qedr_cleanup_kernel_sq(dev, qp);
1455 DP_ERR(dev, "failed to init kernel QP %p RQ\n", qp);
1456 return rc;
1457 }
1458
1459 rc = qedr_init_qp_kernel_params_rq(dev, qp, attrs);
1460 if (rc) {
1461 DP_ERR(dev, "failed to init kernel QP %p RQ params\n", qp);
1462 qedr_cleanup_kernel_sq(dev, qp);
1463 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1464 return rc;
1465 }
1466
1467 return rc;
1468 }
1469
1470 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1471 struct ib_qp_init_attr *attrs,
1472 struct ib_udata *udata)
1473 {
1474 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1475 struct qed_rdma_create_qp_out_params out_params;
1476 struct qed_rdma_create_qp_in_params in_params;
1477 struct qedr_pd *pd = get_qedr_pd(ibpd);
1478 struct ib_ucontext *ib_ctx = NULL;
1479 struct qedr_ucontext *ctx = NULL;
1480 struct qedr_create_qp_ureq ureq;
1481 struct qedr_qp *qp;
1482 int rc = 0;
1483
1484 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1485 udata ? "user library" : "kernel", pd);
1486
1487 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1488 if (rc)
1489 return ERR_PTR(rc);
1490
1491 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1492 if (!qp)
1493 return ERR_PTR(-ENOMEM);
1494
1495 if (attrs->srq)
1496 return ERR_PTR(-EINVAL);
1497
1498 DP_DEBUG(dev, QEDR_MSG_QP,
1499 "create qp: sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1500 get_qedr_cq(attrs->send_cq),
1501 get_qedr_cq(attrs->send_cq)->icid,
1502 get_qedr_cq(attrs->recv_cq),
1503 get_qedr_cq(attrs->recv_cq)->icid);
1504
1505 qedr_set_qp_init_params(dev, qp, pd, attrs);
1506
1507 if (attrs->qp_type == IB_QPT_GSI) {
1508 if (udata) {
1509 DP_ERR(dev,
1510 "create qp: unexpected udata when creating GSI QP\n");
1511 goto err0;
1512 }
1513 return qedr_create_gsi_qp(dev, attrs, qp);
1514 }
1515
1516 memset(&in_params, 0, sizeof(in_params));
1517
1518 if (udata) {
1519 if (!(udata && ibpd->uobject && ibpd->uobject->context))
1520 goto err0;
1521
1522 ib_ctx = ibpd->uobject->context;
1523 ctx = get_qedr_ucontext(ib_ctx);
1524
1525 memset(&ureq, 0, sizeof(ureq));
1526 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1527 DP_ERR(dev,
1528 "create qp: problem copying data from user space\n");
1529 goto err0;
1530 }
1531
1532 rc = qedr_init_user_qp(ib_ctx, dev, qp, &ureq);
1533 if (rc)
1534 goto err0;
1535
1536 qedr_init_qp_user_params(&in_params, &ureq);
1537 } else {
1538 rc = qedr_init_kernel_qp(dev, qp, attrs, &in_params);
1539 if (rc)
1540 goto err0;
1541 }
1542
1543 qedr_init_qp_in_params_sq(dev, pd, qp, attrs, udata, &in_params);
1544 qedr_init_qp_in_params_rq(qp, attrs, udata, &in_params);
1545
1546 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1547 &in_params, &out_params);
1548
1549 if (!qp->qed_qp)
1550 goto err1;
1551
1552 qp->qp_id = out_params.qp_id;
1553 qp->icid = out_params.icid;
1554 qp->ibqp.qp_num = qp->qp_id;
1555
1556 if (udata) {
1557 rc = qedr_copy_qp_uresp(dev, qp, udata);
1558 if (rc)
1559 goto err2;
1560
1561 qedr_qp_user_print(dev, qp);
1562 } else {
1563 qedr_init_qp_kernel_doorbell_sq(dev, qp);
1564 qedr_init_qp_kernel_doorbell_rq(dev, qp);
1565 }
1566
1567 DP_DEBUG(dev, QEDR_MSG_QP, "created %s space QP %p\n",
1568 udata ? "user" : "kernel", qp);
1569
1570 return &qp->ibqp;
1571
1572 err2:
1573 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1574 if (rc)
1575 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1576 err1:
1577 if (udata) {
1578 qedr_cleanup_user_sq(dev, qp);
1579 qedr_cleanup_user_rq(dev, qp);
1580 } else {
1581 qedr_cleanup_kernel_sq(dev, qp);
1582 qedr_cleanup_kernel_rq(dev, qp);
1583 }
1584
1585 err0:
1586 kfree(qp);
1587
1588 return ERR_PTR(-EFAULT);
1589 }
1590
1591 enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1592 {
1593 switch (qp_state) {
1594 case QED_ROCE_QP_STATE_RESET:
1595 return IB_QPS_RESET;
1596 case QED_ROCE_QP_STATE_INIT:
1597 return IB_QPS_INIT;
1598 case QED_ROCE_QP_STATE_RTR:
1599 return IB_QPS_RTR;
1600 case QED_ROCE_QP_STATE_RTS:
1601 return IB_QPS_RTS;
1602 case QED_ROCE_QP_STATE_SQD:
1603 return IB_QPS_SQD;
1604 case QED_ROCE_QP_STATE_ERR:
1605 return IB_QPS_ERR;
1606 case QED_ROCE_QP_STATE_SQE:
1607 return IB_QPS_SQE;
1608 }
1609 return IB_QPS_ERR;
1610 }
1611
1612 enum qed_roce_qp_state qedr_get_state_from_ibqp(enum ib_qp_state qp_state)
1613 {
1614 switch (qp_state) {
1615 case IB_QPS_RESET:
1616 return QED_ROCE_QP_STATE_RESET;
1617 case IB_QPS_INIT:
1618 return QED_ROCE_QP_STATE_INIT;
1619 case IB_QPS_RTR:
1620 return QED_ROCE_QP_STATE_RTR;
1621 case IB_QPS_RTS:
1622 return QED_ROCE_QP_STATE_RTS;
1623 case IB_QPS_SQD:
1624 return QED_ROCE_QP_STATE_SQD;
1625 case IB_QPS_ERR:
1626 return QED_ROCE_QP_STATE_ERR;
1627 default:
1628 return QED_ROCE_QP_STATE_ERR;
1629 }
1630 }
1631
1632 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1633 {
1634 qed_chain_reset(&qph->pbl);
1635 qph->prod = 0;
1636 qph->cons = 0;
1637 qph->wqe_cons = 0;
1638 qph->db_data.data.value = cpu_to_le16(0);
1639 }
1640
1641 static int qedr_update_qp_state(struct qedr_dev *dev,
1642 struct qedr_qp *qp,
1643 enum qed_roce_qp_state new_state)
1644 {
1645 int status = 0;
1646
1647 if (new_state == qp->state)
1648 return 1;
1649
1650 switch (qp->state) {
1651 case QED_ROCE_QP_STATE_RESET:
1652 switch (new_state) {
1653 case QED_ROCE_QP_STATE_INIT:
1654 qp->prev_wqe_size = 0;
1655 qedr_reset_qp_hwq_info(&qp->sq);
1656 qedr_reset_qp_hwq_info(&qp->rq);
1657 break;
1658 default:
1659 status = -EINVAL;
1660 break;
1661 };
1662 break;
1663 case QED_ROCE_QP_STATE_INIT:
1664 switch (new_state) {
1665 case QED_ROCE_QP_STATE_RTR:
1666 /* Update doorbell (in case post_recv was
1667 * done before move to RTR)
1668 */
1669 wmb();
1670 writel(qp->rq.db_data.raw, qp->rq.db);
1671 /* Make sure write takes effect */
1672 mmiowb();
1673 break;
1674 case QED_ROCE_QP_STATE_ERR:
1675 break;
1676 default:
1677 /* Invalid state change. */
1678 status = -EINVAL;
1679 break;
1680 };
1681 break;
1682 case QED_ROCE_QP_STATE_RTR:
1683 /* RTR->XXX */
1684 switch (new_state) {
1685 case QED_ROCE_QP_STATE_RTS:
1686 break;
1687 case QED_ROCE_QP_STATE_ERR:
1688 break;
1689 default:
1690 /* Invalid state change. */
1691 status = -EINVAL;
1692 break;
1693 };
1694 break;
1695 case QED_ROCE_QP_STATE_RTS:
1696 /* RTS->XXX */
1697 switch (new_state) {
1698 case QED_ROCE_QP_STATE_SQD:
1699 break;
1700 case QED_ROCE_QP_STATE_ERR:
1701 break;
1702 default:
1703 /* Invalid state change. */
1704 status = -EINVAL;
1705 break;
1706 };
1707 break;
1708 case QED_ROCE_QP_STATE_SQD:
1709 /* SQD->XXX */
1710 switch (new_state) {
1711 case QED_ROCE_QP_STATE_RTS:
1712 case QED_ROCE_QP_STATE_ERR:
1713 break;
1714 default:
1715 /* Invalid state change. */
1716 status = -EINVAL;
1717 break;
1718 };
1719 break;
1720 case QED_ROCE_QP_STATE_ERR:
1721 /* ERR->XXX */
1722 switch (new_state) {
1723 case QED_ROCE_QP_STATE_RESET:
1724 break;
1725 default:
1726 status = -EINVAL;
1727 break;
1728 };
1729 break;
1730 default:
1731 status = -EINVAL;
1732 break;
1733 };
1734
1735 return status;
1736 }
1737
1738 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1739 int attr_mask, struct ib_udata *udata)
1740 {
1741 struct qedr_qp *qp = get_qedr_qp(ibqp);
1742 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1743 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1744 enum ib_qp_state old_qp_state, new_qp_state;
1745 int rc = 0;
1746
1747 DP_DEBUG(dev, QEDR_MSG_QP,
1748 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1749 attr->qp_state);
1750
1751 old_qp_state = qedr_get_ibqp_state(qp->state);
1752 if (attr_mask & IB_QP_STATE)
1753 new_qp_state = attr->qp_state;
1754 else
1755 new_qp_state = old_qp_state;
1756
1757 if (!ib_modify_qp_is_ok
1758 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1759 IB_LINK_LAYER_ETHERNET)) {
1760 DP_ERR(dev,
1761 "modify qp: invalid attribute mask=0x%x specified for\n"
1762 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1763 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1764 new_qp_state);
1765 rc = -EINVAL;
1766 goto err;
1767 }
1768
1769 /* Translate the masks... */
1770 if (attr_mask & IB_QP_STATE) {
1771 SET_FIELD(qp_params.modify_flags,
1772 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1773 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1774 }
1775
1776 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1777 qp_params.sqd_async = true;
1778
1779 if (attr_mask & IB_QP_PKEY_INDEX) {
1780 SET_FIELD(qp_params.modify_flags,
1781 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1782 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1783 rc = -EINVAL;
1784 goto err;
1785 }
1786
1787 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1788 }
1789
1790 if (attr_mask & IB_QP_QKEY)
1791 qp->qkey = attr->qkey;
1792
1793 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1794 SET_FIELD(qp_params.modify_flags,
1795 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1796 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1797 IB_ACCESS_REMOTE_READ;
1798 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1799 IB_ACCESS_REMOTE_WRITE;
1800 qp_params.incoming_atomic_en = attr->qp_access_flags &
1801 IB_ACCESS_REMOTE_ATOMIC;
1802 }
1803
1804 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1805 if (attr_mask & IB_QP_PATH_MTU) {
1806 if (attr->path_mtu < IB_MTU_256 ||
1807 attr->path_mtu > IB_MTU_4096) {
1808 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1809 rc = -EINVAL;
1810 goto err;
1811 }
1812 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1813 ib_mtu_enum_to_int(iboe_get_mtu
1814 (dev->ndev->mtu)));
1815 }
1816
1817 if (!qp->mtu) {
1818 qp->mtu =
1819 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1820 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1821 }
1822
1823 SET_FIELD(qp_params.modify_flags,
1824 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1825
1826 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1827 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1828 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1829
1830 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1831
1832 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1833 if (rc) {
1834 DP_ERR(dev,
1835 "modify qp: problems with GID index %d (rc=%d)\n",
1836 attr->ah_attr.grh.sgid_index, rc);
1837 return rc;
1838 }
1839
1840 rc = qedr_get_dmac(dev, &attr->ah_attr,
1841 qp_params.remote_mac_addr);
1842 if (rc)
1843 return rc;
1844
1845 qp_params.use_local_mac = true;
1846 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1847
1848 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1849 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1850 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1851 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1852 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1853 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1854 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1855 qp_params.remote_mac_addr);
1856 ;
1857
1858 qp_params.mtu = qp->mtu;
1859 qp_params.lb_indication = false;
1860 }
1861
1862 if (!qp_params.mtu) {
1863 /* Stay with current MTU */
1864 if (qp->mtu)
1865 qp_params.mtu = qp->mtu;
1866 else
1867 qp_params.mtu =
1868 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1869 }
1870
1871 if (attr_mask & IB_QP_TIMEOUT) {
1872 SET_FIELD(qp_params.modify_flags,
1873 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1874
1875 qp_params.ack_timeout = attr->timeout;
1876 if (attr->timeout) {
1877 u32 temp;
1878
1879 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1880 /* FW requires [msec] */
1881 qp_params.ack_timeout = temp;
1882 } else {
1883 /* Infinite */
1884 qp_params.ack_timeout = 0;
1885 }
1886 }
1887 if (attr_mask & IB_QP_RETRY_CNT) {
1888 SET_FIELD(qp_params.modify_flags,
1889 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1890 qp_params.retry_cnt = attr->retry_cnt;
1891 }
1892
1893 if (attr_mask & IB_QP_RNR_RETRY) {
1894 SET_FIELD(qp_params.modify_flags,
1895 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1896 qp_params.rnr_retry_cnt = attr->rnr_retry;
1897 }
1898
1899 if (attr_mask & IB_QP_RQ_PSN) {
1900 SET_FIELD(qp_params.modify_flags,
1901 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1902 qp_params.rq_psn = attr->rq_psn;
1903 qp->rq_psn = attr->rq_psn;
1904 }
1905
1906 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1907 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1908 rc = -EINVAL;
1909 DP_ERR(dev,
1910 "unsupported max_rd_atomic=%d, supported=%d\n",
1911 attr->max_rd_atomic,
1912 dev->attr.max_qp_req_rd_atomic_resc);
1913 goto err;
1914 }
1915
1916 SET_FIELD(qp_params.modify_flags,
1917 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1918 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1919 }
1920
1921 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1922 SET_FIELD(qp_params.modify_flags,
1923 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1924 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1925 }
1926
1927 if (attr_mask & IB_QP_SQ_PSN) {
1928 SET_FIELD(qp_params.modify_flags,
1929 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1930 qp_params.sq_psn = attr->sq_psn;
1931 qp->sq_psn = attr->sq_psn;
1932 }
1933
1934 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1935 if (attr->max_dest_rd_atomic >
1936 dev->attr.max_qp_resp_rd_atomic_resc) {
1937 DP_ERR(dev,
1938 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1939 attr->max_dest_rd_atomic,
1940 dev->attr.max_qp_resp_rd_atomic_resc);
1941
1942 rc = -EINVAL;
1943 goto err;
1944 }
1945
1946 SET_FIELD(qp_params.modify_flags,
1947 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1948 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1949 }
1950
1951 if (attr_mask & IB_QP_DEST_QPN) {
1952 SET_FIELD(qp_params.modify_flags,
1953 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1954
1955 qp_params.dest_qp = attr->dest_qp_num;
1956 qp->dest_qp_num = attr->dest_qp_num;
1957 }
1958
1959 if (qp->qp_type != IB_QPT_GSI)
1960 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1961 qp->qed_qp, &qp_params);
1962
1963 if (attr_mask & IB_QP_STATE) {
1964 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1965 qedr_update_qp_state(dev, qp, qp_params.new_state);
1966 qp->state = qp_params.new_state;
1967 }
1968
1969 err:
1970 return rc;
1971 }
1972
1973 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1974 {
1975 int ib_qp_acc_flags = 0;
1976
1977 if (params->incoming_rdma_write_en)
1978 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1979 if (params->incoming_rdma_read_en)
1980 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
1981 if (params->incoming_atomic_en)
1982 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
1983 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1984 return ib_qp_acc_flags;
1985 }
1986
1987 int qedr_query_qp(struct ib_qp *ibqp,
1988 struct ib_qp_attr *qp_attr,
1989 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1990 {
1991 struct qed_rdma_query_qp_out_params params;
1992 struct qedr_qp *qp = get_qedr_qp(ibqp);
1993 struct qedr_dev *dev = qp->dev;
1994 int rc = 0;
1995
1996 memset(&params, 0, sizeof(params));
1997
1998 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
1999 if (rc)
2000 goto err;
2001
2002 memset(qp_attr, 0, sizeof(*qp_attr));
2003 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2004
2005 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2006 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2007 qp_attr->path_mtu = iboe_get_mtu(params.mtu);
2008 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2009 qp_attr->rq_psn = params.rq_psn;
2010 qp_attr->sq_psn = params.sq_psn;
2011 qp_attr->dest_qp_num = params.dest_qp;
2012
2013 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2014
2015 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2016 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2017 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2018 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2019 qp_attr->cap.max_inline_data = qp->max_inline_data;
2020 qp_init_attr->cap = qp_attr->cap;
2021
2022 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
2023 sizeof(qp_attr->ah_attr.grh.dgid.raw));
2024
2025 qp_attr->ah_attr.grh.flow_label = params.flow_label;
2026 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
2027 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
2028 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
2029
2030 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
2031 qp_attr->ah_attr.port_num = 1;
2032 qp_attr->ah_attr.sl = 0;
2033 qp_attr->timeout = params.timeout;
2034 qp_attr->rnr_retry = params.rnr_retry;
2035 qp_attr->retry_cnt = params.retry_cnt;
2036 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2037 qp_attr->pkey_index = params.pkey_index;
2038 qp_attr->port_num = 1;
2039 qp_attr->ah_attr.src_path_bits = 0;
2040 qp_attr->ah_attr.static_rate = 0;
2041 qp_attr->alt_pkey_index = 0;
2042 qp_attr->alt_port_num = 0;
2043 qp_attr->alt_timeout = 0;
2044 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2045
2046 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2047 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2048 qp_attr->max_rd_atomic = params.max_rd_atomic;
2049 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2050
2051 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2052 qp_attr->cap.max_inline_data);
2053
2054 err:
2055 return rc;
2056 }
2057
2058 int qedr_destroy_qp(struct ib_qp *ibqp)
2059 {
2060 struct qedr_qp *qp = get_qedr_qp(ibqp);
2061 struct qedr_dev *dev = qp->dev;
2062 struct ib_qp_attr attr;
2063 int attr_mask = 0;
2064 int rc = 0;
2065
2066 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2067 qp, qp->qp_type);
2068
2069 if (qp->state != (QED_ROCE_QP_STATE_RESET | QED_ROCE_QP_STATE_ERR |
2070 QED_ROCE_QP_STATE_INIT)) {
2071 attr.qp_state = IB_QPS_ERR;
2072 attr_mask |= IB_QP_STATE;
2073
2074 /* Change the QP state to ERROR */
2075 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2076 }
2077
2078 if (qp->qp_type != IB_QPT_GSI) {
2079 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2080 if (rc)
2081 return rc;
2082 } else {
2083 qedr_destroy_gsi_qp(dev);
2084 }
2085
2086 if (ibqp->uobject && ibqp->uobject->context) {
2087 qedr_cleanup_user_sq(dev, qp);
2088 qedr_cleanup_user_rq(dev, qp);
2089 } else {
2090 qedr_cleanup_kernel_sq(dev, qp);
2091 qedr_cleanup_kernel_rq(dev, qp);
2092 }
2093
2094 kfree(qp);
2095
2096 return rc;
2097 }
2098
2099 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
2100 {
2101 struct qedr_ah *ah;
2102
2103 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2104 if (!ah)
2105 return ERR_PTR(-ENOMEM);
2106
2107 ah->attr = *attr;
2108
2109 return &ah->ibah;
2110 }
2111
2112 int qedr_destroy_ah(struct ib_ah *ibah)
2113 {
2114 struct qedr_ah *ah = get_qedr_ah(ibah);
2115
2116 kfree(ah);
2117 return 0;
2118 }
2119
2120 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2121 {
2122 struct qedr_pbl *pbl, *tmp;
2123
2124 if (info->pbl_table)
2125 list_add_tail(&info->pbl_table->list_entry,
2126 &info->free_pbl_list);
2127
2128 if (!list_empty(&info->inuse_pbl_list))
2129 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2130
2131 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2132 list_del(&pbl->list_entry);
2133 qedr_free_pbl(dev, &info->pbl_info, pbl);
2134 }
2135 }
2136
2137 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2138 size_t page_list_len, bool two_layered)
2139 {
2140 struct qedr_pbl *tmp;
2141 int rc;
2142
2143 INIT_LIST_HEAD(&info->free_pbl_list);
2144 INIT_LIST_HEAD(&info->inuse_pbl_list);
2145
2146 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2147 page_list_len, two_layered);
2148 if (rc)
2149 goto done;
2150
2151 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2152 if (!info->pbl_table) {
2153 rc = -ENOMEM;
2154 goto done;
2155 }
2156
2157 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2158 &info->pbl_table->pa);
2159
2160 /* in usual case we use 2 PBLs, so we add one to free
2161 * list and allocating another one
2162 */
2163 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2164 if (!tmp) {
2165 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2166 goto done;
2167 }
2168
2169 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2170
2171 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2172
2173 done:
2174 if (rc)
2175 free_mr_info(dev, info);
2176
2177 return rc;
2178 }
2179
2180 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2181 u64 usr_addr, int acc, struct ib_udata *udata)
2182 {
2183 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2184 struct qedr_mr *mr;
2185 struct qedr_pd *pd;
2186 int rc = -ENOMEM;
2187
2188 pd = get_qedr_pd(ibpd);
2189 DP_DEBUG(dev, QEDR_MSG_MR,
2190 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2191 pd->pd_id, start, len, usr_addr, acc);
2192
2193 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2194 return ERR_PTR(-EINVAL);
2195
2196 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2197 if (!mr)
2198 return ERR_PTR(rc);
2199
2200 mr->type = QEDR_MR_USER;
2201
2202 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2203 if (IS_ERR(mr->umem)) {
2204 rc = -EFAULT;
2205 goto err0;
2206 }
2207
2208 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2209 if (rc)
2210 goto err1;
2211
2212 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2213 &mr->info.pbl_info);
2214
2215 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2216 if (rc) {
2217 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2218 goto err1;
2219 }
2220
2221 /* Index only, 18 bit long, lkey = itid << 8 | key */
2222 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2223 mr->hw_mr.key = 0;
2224 mr->hw_mr.pd = pd->pd_id;
2225 mr->hw_mr.local_read = 1;
2226 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2227 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2228 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2229 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2230 mr->hw_mr.mw_bind = false;
2231 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2232 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2233 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2234 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size);
2235 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2236 mr->hw_mr.length = len;
2237 mr->hw_mr.vaddr = usr_addr;
2238 mr->hw_mr.zbva = false;
2239 mr->hw_mr.phy_mr = false;
2240 mr->hw_mr.dma_mr = false;
2241
2242 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2243 if (rc) {
2244 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2245 goto err2;
2246 }
2247
2248 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2249 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2250 mr->hw_mr.remote_atomic)
2251 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2252
2253 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2254 mr->ibmr.lkey);
2255 return &mr->ibmr;
2256
2257 err2:
2258 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2259 err1:
2260 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2261 err0:
2262 kfree(mr);
2263 return ERR_PTR(rc);
2264 }
2265
2266 int qedr_dereg_mr(struct ib_mr *ib_mr)
2267 {
2268 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2269 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2270 int rc = 0;
2271
2272 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2273 if (rc)
2274 return rc;
2275
2276 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2277
2278 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2279 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2280
2281 /* it could be user registered memory. */
2282 if (mr->umem)
2283 ib_umem_release(mr->umem);
2284
2285 kfree(mr);
2286
2287 return rc;
2288 }
2289
2290 struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
2291 {
2292 struct qedr_pd *pd = get_qedr_pd(ibpd);
2293 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2294 struct qedr_mr *mr;
2295 int rc = -ENOMEM;
2296
2297 DP_DEBUG(dev, QEDR_MSG_MR,
2298 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2299 max_page_list_len);
2300
2301 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2302 if (!mr)
2303 return ERR_PTR(rc);
2304
2305 mr->dev = dev;
2306 mr->type = QEDR_MR_FRMR;
2307
2308 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2309 if (rc)
2310 goto err0;
2311
2312 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2313 if (rc) {
2314 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2315 goto err0;
2316 }
2317
2318 /* Index only, 18 bit long, lkey = itid << 8 | key */
2319 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2320 mr->hw_mr.key = 0;
2321 mr->hw_mr.pd = pd->pd_id;
2322 mr->hw_mr.local_read = 1;
2323 mr->hw_mr.local_write = 0;
2324 mr->hw_mr.remote_read = 0;
2325 mr->hw_mr.remote_write = 0;
2326 mr->hw_mr.remote_atomic = 0;
2327 mr->hw_mr.mw_bind = false;
2328 mr->hw_mr.pbl_ptr = 0;
2329 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2330 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2331 mr->hw_mr.fbo = 0;
2332 mr->hw_mr.length = 0;
2333 mr->hw_mr.vaddr = 0;
2334 mr->hw_mr.zbva = false;
2335 mr->hw_mr.phy_mr = true;
2336 mr->hw_mr.dma_mr = false;
2337
2338 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2339 if (rc) {
2340 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2341 goto err1;
2342 }
2343
2344 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2345 mr->ibmr.rkey = mr->ibmr.lkey;
2346
2347 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2348 return mr;
2349
2350 err1:
2351 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2352 err0:
2353 kfree(mr);
2354 return ERR_PTR(rc);
2355 }
2356
2357 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2358 enum ib_mr_type mr_type, u32 max_num_sg)
2359 {
2360 struct qedr_dev *dev;
2361 struct qedr_mr *mr;
2362
2363 if (mr_type != IB_MR_TYPE_MEM_REG)
2364 return ERR_PTR(-EINVAL);
2365
2366 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2367
2368 if (IS_ERR(mr))
2369 return ERR_PTR(-EINVAL);
2370
2371 dev = mr->dev;
2372
2373 return &mr->ibmr;
2374 }
2375
2376 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2377 {
2378 struct qedr_mr *mr = get_qedr_mr(ibmr);
2379 struct qedr_pbl *pbl_table;
2380 struct regpair *pbe;
2381 u32 pbes_in_page;
2382
2383 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2384 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2385 return -ENOMEM;
2386 }
2387
2388 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2389 mr->npages, addr);
2390
2391 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2392 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2393 pbe = (struct regpair *)pbl_table->va;
2394 pbe += mr->npages % pbes_in_page;
2395 pbe->lo = cpu_to_le32((u32)addr);
2396 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2397
2398 mr->npages++;
2399
2400 return 0;
2401 }
2402
2403 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2404 {
2405 int work = info->completed - info->completed_handled - 1;
2406
2407 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2408 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2409 struct qedr_pbl *pbl;
2410
2411 /* Free all the page list that are possible to be freed
2412 * (all the ones that were invalidated), under the assumption
2413 * that if an FMR was completed successfully that means that
2414 * if there was an invalidate operation before it also ended
2415 */
2416 pbl = list_first_entry(&info->inuse_pbl_list,
2417 struct qedr_pbl, list_entry);
2418 list_del(&pbl->list_entry);
2419 list_add_tail(&pbl->list_entry, &info->free_pbl_list);
2420 info->completed_handled++;
2421 }
2422 }
2423
2424 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2425 int sg_nents, unsigned int *sg_offset)
2426 {
2427 struct qedr_mr *mr = get_qedr_mr(ibmr);
2428
2429 mr->npages = 0;
2430
2431 handle_completed_mrs(mr->dev, &mr->info);
2432 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2433 }
2434
2435 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2436 {
2437 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2438 struct qedr_pd *pd = get_qedr_pd(ibpd);
2439 struct qedr_mr *mr;
2440 int rc;
2441
2442 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2443 if (!mr)
2444 return ERR_PTR(-ENOMEM);
2445
2446 mr->type = QEDR_MR_DMA;
2447
2448 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2449 if (rc) {
2450 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2451 goto err1;
2452 }
2453
2454 /* index only, 18 bit long, lkey = itid << 8 | key */
2455 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2456 mr->hw_mr.pd = pd->pd_id;
2457 mr->hw_mr.local_read = 1;
2458 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2459 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2460 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2461 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2462 mr->hw_mr.dma_mr = true;
2463
2464 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2465 if (rc) {
2466 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2467 goto err2;
2468 }
2469
2470 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2471 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2472 mr->hw_mr.remote_atomic)
2473 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2474
2475 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2476 return &mr->ibmr;
2477
2478 err2:
2479 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2480 err1:
2481 kfree(mr);
2482 return ERR_PTR(rc);
2483 }
2484
2485 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2486 {
2487 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2488 }
2489
2490 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2491 {
2492 int i, len = 0;
2493
2494 for (i = 0; i < num_sge; i++)
2495 len += sg_list[i].length;
2496
2497 return len;
2498 }
2499
2500 static void swap_wqe_data64(u64 *p)
2501 {
2502 int i;
2503
2504 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2505 *p = cpu_to_be64(cpu_to_le64(*p));
2506 }
2507
2508 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2509 struct qedr_qp *qp, u8 *wqe_size,
2510 struct ib_send_wr *wr,
2511 struct ib_send_wr **bad_wr, u8 *bits,
2512 u8 bit)
2513 {
2514 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2515 char *seg_prt, *wqe;
2516 int i, seg_siz;
2517
2518 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2519 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2520 *bad_wr = wr;
2521 return 0;
2522 }
2523
2524 if (!data_size)
2525 return data_size;
2526
2527 *bits |= bit;
2528
2529 seg_prt = NULL;
2530 wqe = NULL;
2531 seg_siz = 0;
2532
2533 /* Copy data inline */
2534 for (i = 0; i < wr->num_sge; i++) {
2535 u32 len = wr->sg_list[i].length;
2536 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2537
2538 while (len > 0) {
2539 u32 cur;
2540
2541 /* New segment required */
2542 if (!seg_siz) {
2543 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2544 seg_prt = wqe;
2545 seg_siz = sizeof(struct rdma_sq_common_wqe);
2546 (*wqe_size)++;
2547 }
2548
2549 /* Calculate currently allowed length */
2550 cur = min_t(u32, len, seg_siz);
2551 memcpy(seg_prt, src, cur);
2552
2553 /* Update segment variables */
2554 seg_prt += cur;
2555 seg_siz -= cur;
2556
2557 /* Update sge variables */
2558 src += cur;
2559 len -= cur;
2560
2561 /* Swap fully-completed segments */
2562 if (!seg_siz)
2563 swap_wqe_data64((u64 *)wqe);
2564 }
2565 }
2566
2567 /* swap last not completed segment */
2568 if (seg_siz)
2569 swap_wqe_data64((u64 *)wqe);
2570
2571 return data_size;
2572 }
2573
2574 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2575 do { \
2576 DMA_REGPAIR_LE(sge->addr, vaddr); \
2577 (sge)->length = cpu_to_le32(vlength); \
2578 (sge)->flags = cpu_to_le32(vflags); \
2579 } while (0)
2580
2581 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2582 do { \
2583 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2584 (hdr)->num_sges = num_sge; \
2585 } while (0)
2586
2587 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2588 do { \
2589 DMA_REGPAIR_LE(sge->addr, vaddr); \
2590 (sge)->length = cpu_to_le32(vlength); \
2591 (sge)->l_key = cpu_to_le32(vlkey); \
2592 } while (0)
2593
2594 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2595 struct ib_send_wr *wr)
2596 {
2597 u32 data_size = 0;
2598 int i;
2599
2600 for (i = 0; i < wr->num_sge; i++) {
2601 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2602
2603 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2604 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2605 sge->length = cpu_to_le32(wr->sg_list[i].length);
2606 data_size += wr->sg_list[i].length;
2607 }
2608
2609 if (wqe_size)
2610 *wqe_size += wr->num_sge;
2611
2612 return data_size;
2613 }
2614
2615 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2616 struct qedr_qp *qp,
2617 struct rdma_sq_rdma_wqe_1st *rwqe,
2618 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2619 struct ib_send_wr *wr,
2620 struct ib_send_wr **bad_wr)
2621 {
2622 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2623 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2624
2625 if (wr->send_flags & IB_SEND_INLINE) {
2626 u8 flags = 0;
2627
2628 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2629 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2630 bad_wr, &rwqe->flags, flags);
2631 }
2632
2633 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2634 }
2635
2636 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2637 struct qedr_qp *qp,
2638 struct rdma_sq_send_wqe_1st *swqe,
2639 struct rdma_sq_send_wqe_2st *swqe2,
2640 struct ib_send_wr *wr,
2641 struct ib_send_wr **bad_wr)
2642 {
2643 memset(swqe2, 0, sizeof(*swqe2));
2644 if (wr->send_flags & IB_SEND_INLINE) {
2645 u8 flags = 0;
2646
2647 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2648 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2649 bad_wr, &swqe->flags, flags);
2650 }
2651
2652 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2653 }
2654
2655 static int qedr_prepare_reg(struct qedr_qp *qp,
2656 struct rdma_sq_fmr_wqe_1st *fwqe1,
2657 struct ib_reg_wr *wr)
2658 {
2659 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2660 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2661
2662 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2663 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2664 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2665 fwqe1->l_key = wr->key;
2666
2667 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2668 !!(wr->access & IB_ACCESS_REMOTE_READ));
2669 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2670 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2671 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2672 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2673 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2674 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2675 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2676 fwqe2->fmr_ctrl = 0;
2677
2678 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2679 ilog2(mr->ibmr.page_size) - 12);
2680
2681 fwqe2->length_hi = 0;
2682 fwqe2->length_lo = mr->ibmr.length;
2683 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2684 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2685
2686 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2687
2688 return 0;
2689 }
2690
2691 enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2692 {
2693 switch (opcode) {
2694 case IB_WR_RDMA_WRITE:
2695 case IB_WR_RDMA_WRITE_WITH_IMM:
2696 return IB_WC_RDMA_WRITE;
2697 case IB_WR_SEND_WITH_IMM:
2698 case IB_WR_SEND:
2699 case IB_WR_SEND_WITH_INV:
2700 return IB_WC_SEND;
2701 case IB_WR_RDMA_READ:
2702 return IB_WC_RDMA_READ;
2703 case IB_WR_ATOMIC_CMP_AND_SWP:
2704 return IB_WC_COMP_SWAP;
2705 case IB_WR_ATOMIC_FETCH_AND_ADD:
2706 return IB_WC_FETCH_ADD;
2707 case IB_WR_REG_MR:
2708 return IB_WC_REG_MR;
2709 case IB_WR_LOCAL_INV:
2710 return IB_WC_LOCAL_INV;
2711 default:
2712 return IB_WC_SEND;
2713 }
2714 }
2715
2716 inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2717 {
2718 int wq_is_full, err_wr, pbl_is_full;
2719 struct qedr_dev *dev = qp->dev;
2720
2721 /* prevent SQ overflow and/or processing of a bad WR */
2722 err_wr = wr->num_sge > qp->sq.max_sges;
2723 wq_is_full = qedr_wq_is_full(&qp->sq);
2724 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2725 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2726 if (wq_is_full || err_wr || pbl_is_full) {
2727 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2728 DP_ERR(dev,
2729 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2730 qp);
2731 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2732 }
2733
2734 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2735 DP_ERR(dev,
2736 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2737 qp);
2738 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2739 }
2740
2741 if (pbl_is_full &&
2742 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2743 DP_ERR(dev,
2744 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2745 qp);
2746 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2747 }
2748 return false;
2749 }
2750 return true;
2751 }
2752
2753 int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2754 struct ib_send_wr **bad_wr)
2755 {
2756 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2757 struct qedr_qp *qp = get_qedr_qp(ibqp);
2758 struct rdma_sq_atomic_wqe_1st *awqe1;
2759 struct rdma_sq_atomic_wqe_2nd *awqe2;
2760 struct rdma_sq_atomic_wqe_3rd *awqe3;
2761 struct rdma_sq_send_wqe_2st *swqe2;
2762 struct rdma_sq_local_inv_wqe *iwqe;
2763 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2764 struct rdma_sq_send_wqe_1st *swqe;
2765 struct rdma_sq_rdma_wqe_1st *rwqe;
2766 struct rdma_sq_fmr_wqe_1st *fwqe1;
2767 struct rdma_sq_common_wqe *wqe;
2768 u32 length;
2769 int rc = 0;
2770 bool comp;
2771
2772 if (!qedr_can_post_send(qp, wr)) {
2773 *bad_wr = wr;
2774 return -ENOMEM;
2775 }
2776
2777 wqe = qed_chain_produce(&qp->sq.pbl);
2778 qp->wqe_wr_id[qp->sq.prod].signaled =
2779 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2780
2781 wqe->flags = 0;
2782 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2783 !!(wr->send_flags & IB_SEND_SOLICITED));
2784 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2785 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2786 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2787 !!(wr->send_flags & IB_SEND_FENCE));
2788 wqe->prev_wqe_size = qp->prev_wqe_size;
2789
2790 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2791
2792 switch (wr->opcode) {
2793 case IB_WR_SEND_WITH_IMM:
2794 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2795 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2796 swqe->wqe_size = 2;
2797 swqe2 = qed_chain_produce(&qp->sq.pbl);
2798
2799 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2800 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2801 wr, bad_wr);
2802 swqe->length = cpu_to_le32(length);
2803 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2804 qp->prev_wqe_size = swqe->wqe_size;
2805 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2806 break;
2807 case IB_WR_SEND:
2808 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2809 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2810
2811 swqe->wqe_size = 2;
2812 swqe2 = qed_chain_produce(&qp->sq.pbl);
2813 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2814 wr, bad_wr);
2815 swqe->length = cpu_to_le32(length);
2816 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2817 qp->prev_wqe_size = swqe->wqe_size;
2818 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2819 break;
2820 case IB_WR_SEND_WITH_INV:
2821 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2822 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2823 swqe2 = qed_chain_produce(&qp->sq.pbl);
2824 swqe->wqe_size = 2;
2825 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2826 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2827 wr, bad_wr);
2828 swqe->length = cpu_to_le32(length);
2829 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2830 qp->prev_wqe_size = swqe->wqe_size;
2831 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2832 break;
2833
2834 case IB_WR_RDMA_WRITE_WITH_IMM:
2835 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2836 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2837
2838 rwqe->wqe_size = 2;
2839 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2840 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2841 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2842 wr, bad_wr);
2843 rwqe->length = cpu_to_le32(length);
2844 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2845 qp->prev_wqe_size = rwqe->wqe_size;
2846 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2847 break;
2848 case IB_WR_RDMA_WRITE:
2849 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2850 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2851
2852 rwqe->wqe_size = 2;
2853 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2854 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2855 wr, bad_wr);
2856 rwqe->length = cpu_to_le32(length);
2857 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2858 qp->prev_wqe_size = rwqe->wqe_size;
2859 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2860 break;
2861 case IB_WR_RDMA_READ_WITH_INV:
2862 DP_ERR(dev,
2863 "RDMA READ WITH INVALIDATE not supported\n");
2864 *bad_wr = wr;
2865 rc = -EINVAL;
2866 break;
2867
2868 case IB_WR_RDMA_READ:
2869 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2870 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2871
2872 rwqe->wqe_size = 2;
2873 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2874 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2875 wr, bad_wr);
2876 rwqe->length = cpu_to_le32(length);
2877 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2878 qp->prev_wqe_size = rwqe->wqe_size;
2879 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2880 break;
2881
2882 case IB_WR_ATOMIC_CMP_AND_SWP:
2883 case IB_WR_ATOMIC_FETCH_AND_ADD:
2884 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2885 awqe1->wqe_size = 4;
2886
2887 awqe2 = qed_chain_produce(&qp->sq.pbl);
2888 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2889 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2890
2891 awqe3 = qed_chain_produce(&qp->sq.pbl);
2892
2893 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2894 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2895 DMA_REGPAIR_LE(awqe3->swap_data,
2896 atomic_wr(wr)->compare_add);
2897 } else {
2898 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2899 DMA_REGPAIR_LE(awqe3->swap_data,
2900 atomic_wr(wr)->swap);
2901 DMA_REGPAIR_LE(awqe3->cmp_data,
2902 atomic_wr(wr)->compare_add);
2903 }
2904
2905 qedr_prepare_sq_sges(qp, NULL, wr);
2906
2907 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2908 qp->prev_wqe_size = awqe1->wqe_size;
2909 break;
2910
2911 case IB_WR_LOCAL_INV:
2912 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2913 iwqe->wqe_size = 1;
2914
2915 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2916 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2917 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2918 qp->prev_wqe_size = iwqe->wqe_size;
2919 break;
2920 case IB_WR_REG_MR:
2921 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2922 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2923 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2924 fwqe1->wqe_size = 2;
2925
2926 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2927 if (rc) {
2928 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2929 *bad_wr = wr;
2930 break;
2931 }
2932
2933 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2934 qp->prev_wqe_size = fwqe1->wqe_size;
2935 break;
2936 default:
2937 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2938 rc = -EINVAL;
2939 *bad_wr = wr;
2940 break;
2941 }
2942
2943 if (*bad_wr) {
2944 u16 value;
2945
2946 /* Restore prod to its position before
2947 * this WR was processed
2948 */
2949 value = le16_to_cpu(qp->sq.db_data.data.value);
2950 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2951
2952 /* Restore prev_wqe_size */
2953 qp->prev_wqe_size = wqe->prev_wqe_size;
2954 rc = -EINVAL;
2955 DP_ERR(dev, "POST SEND FAILED\n");
2956 }
2957
2958 return rc;
2959 }
2960
2961 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2962 struct ib_send_wr **bad_wr)
2963 {
2964 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2965 struct qedr_qp *qp = get_qedr_qp(ibqp);
2966 unsigned long flags;
2967 int rc = 0;
2968
2969 *bad_wr = NULL;
2970
2971 if (qp->qp_type == IB_QPT_GSI)
2972 return qedr_gsi_post_send(ibqp, wr, bad_wr);
2973
2974 spin_lock_irqsave(&qp->q_lock, flags);
2975
2976 if ((qp->state == QED_ROCE_QP_STATE_RESET) ||
2977 (qp->state == QED_ROCE_QP_STATE_ERR)) {
2978 spin_unlock_irqrestore(&qp->q_lock, flags);
2979 *bad_wr = wr;
2980 DP_DEBUG(dev, QEDR_MSG_CQ,
2981 "QP in wrong state! QP icid=0x%x state %d\n",
2982 qp->icid, qp->state);
2983 return -EINVAL;
2984 }
2985
2986 if (!wr) {
2987 DP_ERR(dev, "Got an empty post send.\n");
2988 return -EINVAL;
2989 }
2990
2991 while (wr) {
2992 rc = __qedr_post_send(ibqp, wr, bad_wr);
2993 if (rc)
2994 break;
2995
2996 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
2997
2998 qedr_inc_sw_prod(&qp->sq);
2999
3000 qp->sq.db_data.data.value++;
3001
3002 wr = wr->next;
3003 }
3004
3005 /* Trigger doorbell
3006 * If there was a failure in the first WR then it will be triggered in
3007 * vane. However this is not harmful (as long as the producer value is
3008 * unchanged). For performance reasons we avoid checking for this
3009 * redundant doorbell.
3010 */
3011 wmb();
3012 writel(qp->sq.db_data.raw, qp->sq.db);
3013
3014 /* Make sure write sticks */
3015 mmiowb();
3016
3017 spin_unlock_irqrestore(&qp->q_lock, flags);
3018
3019 return rc;
3020 }
3021
3022 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3023 struct ib_recv_wr **bad_wr)
3024 {
3025 struct qedr_qp *qp = get_qedr_qp(ibqp);
3026 struct qedr_dev *dev = qp->dev;
3027 unsigned long flags;
3028 int status = 0;
3029
3030 if (qp->qp_type == IB_QPT_GSI)
3031 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3032
3033 spin_lock_irqsave(&qp->q_lock, flags);
3034
3035 if ((qp->state == QED_ROCE_QP_STATE_RESET) ||
3036 (qp->state == QED_ROCE_QP_STATE_ERR)) {
3037 spin_unlock_irqrestore(&qp->q_lock, flags);
3038 *bad_wr = wr;
3039 return -EINVAL;
3040 }
3041
3042 while (wr) {
3043 int i;
3044
3045 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3046 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3047 wr->num_sge > qp->rq.max_sges) {
3048 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3049 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3050 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3051 qp->rq.max_sges);
3052 status = -ENOMEM;
3053 *bad_wr = wr;
3054 break;
3055 }
3056 for (i = 0; i < wr->num_sge; i++) {
3057 u32 flags = 0;
3058 struct rdma_rq_sge *rqe =
3059 qed_chain_produce(&qp->rq.pbl);
3060
3061 /* First one must include the number
3062 * of SGE in the list
3063 */
3064 if (!i)
3065 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3066 wr->num_sge);
3067
3068 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3069 wr->sg_list[i].lkey);
3070
3071 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3072 wr->sg_list[i].length, flags);
3073 }
3074
3075 /* Special case of no sges. FW requires between 1-4 sges...
3076 * in this case we need to post 1 sge with length zero. this is
3077 * because rdma write with immediate consumes an RQ.
3078 */
3079 if (!wr->num_sge) {
3080 u32 flags = 0;
3081 struct rdma_rq_sge *rqe =
3082 qed_chain_produce(&qp->rq.pbl);
3083
3084 /* First one must include the number
3085 * of SGE in the list
3086 */
3087 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3088 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3089
3090 RQ_SGE_SET(rqe, 0, 0, flags);
3091 i = 1;
3092 }
3093
3094 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3095 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3096
3097 qedr_inc_sw_prod(&qp->rq);
3098
3099 /* Flush all the writes before signalling doorbell */
3100 wmb();
3101
3102 qp->rq.db_data.data.value++;
3103
3104 writel(qp->rq.db_data.raw, qp->rq.db);
3105
3106 /* Make sure write sticks */
3107 mmiowb();
3108
3109 wr = wr->next;
3110 }
3111
3112 spin_unlock_irqrestore(&qp->q_lock, flags);
3113
3114 return status;
3115 }
3116
3117 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3118 {
3119 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3120
3121 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3122 cq->pbl_toggle;
3123 }
3124
3125 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3126 {
3127 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3128 struct qedr_qp *qp;
3129
3130 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3131 resp_cqe->qp_handle.lo,
3132 u64);
3133 return qp;
3134 }
3135
3136 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3137 {
3138 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3139
3140 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3141 }
3142
3143 /* Return latest CQE (needs processing) */
3144 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3145 {
3146 return cq->latest_cqe;
3147 }
3148
3149 /* In fmr we need to increase the number of fmr completed counter for the fmr
3150 * algorithm determining whether we can free a pbl or not.
3151 * we need to perform this whether the work request was signaled or not. for
3152 * this purpose we call this function from the condition that checks if a wr
3153 * should be skipped, to make sure we don't miss it ( possibly this fmr
3154 * operation was not signalted)
3155 */
3156 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3157 {
3158 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3159 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3160 }
3161
3162 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3163 struct qedr_cq *cq, int num_entries,
3164 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3165 int force)
3166 {
3167 u16 cnt = 0;
3168
3169 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3170 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3171 qedr_chk_if_fmr(qp);
3172 /* skip WC */
3173 goto next_cqe;
3174 }
3175
3176 /* fill WC */
3177 wc->status = status;
3178 wc->wc_flags = 0;
3179 wc->src_qp = qp->id;
3180 wc->qp = &qp->ibqp;
3181
3182 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3183 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3184
3185 switch (wc->opcode) {
3186 case IB_WC_RDMA_WRITE:
3187 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3188 break;
3189 case IB_WC_COMP_SWAP:
3190 case IB_WC_FETCH_ADD:
3191 wc->byte_len = 8;
3192 break;
3193 case IB_WC_REG_MR:
3194 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3195 break;
3196 default:
3197 break;
3198 }
3199
3200 num_entries--;
3201 wc++;
3202 cnt++;
3203 next_cqe:
3204 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3205 qed_chain_consume(&qp->sq.pbl);
3206 qedr_inc_sw_cons(&qp->sq);
3207 }
3208
3209 return cnt;
3210 }
3211
3212 static int qedr_poll_cq_req(struct qedr_dev *dev,
3213 struct qedr_qp *qp, struct qedr_cq *cq,
3214 int num_entries, struct ib_wc *wc,
3215 struct rdma_cqe_requester *req)
3216 {
3217 int cnt = 0;
3218
3219 switch (req->status) {
3220 case RDMA_CQE_REQ_STS_OK:
3221 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3222 IB_WC_SUCCESS, 0);
3223 break;
3224 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3225 DP_ERR(dev,
3226 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3227 cq->icid, qp->icid);
3228 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3229 IB_WC_WR_FLUSH_ERR, 0);
3230 break;
3231 default:
3232 /* process all WQE before the cosumer */
3233 qp->state = QED_ROCE_QP_STATE_ERR;
3234 cnt = process_req(dev, qp, cq, num_entries, wc,
3235 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3236 wc += cnt;
3237 /* if we have extra WC fill it with actual error info */
3238 if (cnt < num_entries) {
3239 enum ib_wc_status wc_status;
3240
3241 switch (req->status) {
3242 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3243 DP_ERR(dev,
3244 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3245 cq->icid, qp->icid);
3246 wc_status = IB_WC_BAD_RESP_ERR;
3247 break;
3248 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3249 DP_ERR(dev,
3250 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3251 cq->icid, qp->icid);
3252 wc_status = IB_WC_LOC_LEN_ERR;
3253 break;
3254 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3255 DP_ERR(dev,
3256 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3257 cq->icid, qp->icid);
3258 wc_status = IB_WC_LOC_QP_OP_ERR;
3259 break;
3260 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3261 DP_ERR(dev,
3262 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3263 cq->icid, qp->icid);
3264 wc_status = IB_WC_LOC_PROT_ERR;
3265 break;
3266 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3267 DP_ERR(dev,
3268 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3269 cq->icid, qp->icid);
3270 wc_status = IB_WC_MW_BIND_ERR;
3271 break;
3272 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3273 DP_ERR(dev,
3274 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3275 cq->icid, qp->icid);
3276 wc_status = IB_WC_REM_INV_REQ_ERR;
3277 break;
3278 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3279 DP_ERR(dev,
3280 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3281 cq->icid, qp->icid);
3282 wc_status = IB_WC_REM_ACCESS_ERR;
3283 break;
3284 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3285 DP_ERR(dev,
3286 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3287 cq->icid, qp->icid);
3288 wc_status = IB_WC_REM_OP_ERR;
3289 break;
3290 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3291 DP_ERR(dev,
3292 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3293 cq->icid, qp->icid);
3294 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3295 break;
3296 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3297 DP_ERR(dev,
3298 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3299 cq->icid, qp->icid);
3300 wc_status = IB_WC_RETRY_EXC_ERR;
3301 break;
3302 default:
3303 DP_ERR(dev,
3304 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3305 cq->icid, qp->icid);
3306 wc_status = IB_WC_GENERAL_ERR;
3307 }
3308 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3309 wc_status, 1);
3310 }
3311 }
3312
3313 return cnt;
3314 }
3315
3316 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3317 struct qedr_cq *cq, struct ib_wc *wc,
3318 struct rdma_cqe_responder *resp, u64 wr_id)
3319 {
3320 enum ib_wc_status wc_status = IB_WC_SUCCESS;
3321 u8 flags;
3322
3323 wc->opcode = IB_WC_RECV;
3324 wc->wc_flags = 0;
3325
3326 switch (resp->status) {
3327 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3328 wc_status = IB_WC_LOC_ACCESS_ERR;
3329 break;
3330 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3331 wc_status = IB_WC_LOC_LEN_ERR;
3332 break;
3333 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3334 wc_status = IB_WC_LOC_QP_OP_ERR;
3335 break;
3336 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3337 wc_status = IB_WC_LOC_PROT_ERR;
3338 break;
3339 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3340 wc_status = IB_WC_MW_BIND_ERR;
3341 break;
3342 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3343 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
3344 break;
3345 case RDMA_CQE_RESP_STS_OK:
3346 wc_status = IB_WC_SUCCESS;
3347 wc->byte_len = le32_to_cpu(resp->length);
3348
3349 flags = resp->flags & QEDR_RESP_RDMA_IMM;
3350
3351 if (flags == QEDR_RESP_RDMA_IMM)
3352 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3353
3354 if (flags == QEDR_RESP_RDMA_IMM || flags == QEDR_RESP_IMM) {
3355 wc->ex.imm_data =
3356 le32_to_cpu(resp->imm_data_or_inv_r_Key);
3357 wc->wc_flags |= IB_WC_WITH_IMM;
3358 }
3359 break;
3360 default:
3361 wc->status = IB_WC_GENERAL_ERR;
3362 DP_ERR(dev, "Invalid CQE status detected\n");
3363 }
3364
3365 /* fill WC */
3366 wc->status = wc_status;
3367 wc->src_qp = qp->id;
3368 wc->qp = &qp->ibqp;
3369 wc->wr_id = wr_id;
3370 }
3371
3372 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3373 struct qedr_cq *cq, struct ib_wc *wc,
3374 struct rdma_cqe_responder *resp)
3375 {
3376 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3377
3378 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3379
3380 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3381 qed_chain_consume(&qp->rq.pbl);
3382 qedr_inc_sw_cons(&qp->rq);
3383
3384 return 1;
3385 }
3386
3387 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3388 int num_entries, struct ib_wc *wc, u16 hw_cons)
3389 {
3390 u16 cnt = 0;
3391
3392 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3393 /* fill WC */
3394 wc->status = IB_WC_WR_FLUSH_ERR;
3395 wc->wc_flags = 0;
3396 wc->src_qp = qp->id;
3397 wc->byte_len = 0;
3398 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3399 wc->qp = &qp->ibqp;
3400 num_entries--;
3401 wc++;
3402 cnt++;
3403 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3404 qed_chain_consume(&qp->rq.pbl);
3405 qedr_inc_sw_cons(&qp->rq);
3406 }
3407
3408 return cnt;
3409 }
3410
3411 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3412 struct rdma_cqe_responder *resp, int *update)
3413 {
3414 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3415 consume_cqe(cq);
3416 *update |= 1;
3417 }
3418 }
3419
3420 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3421 struct qedr_cq *cq, int num_entries,
3422 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3423 int *update)
3424 {
3425 int cnt;
3426
3427 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3428 cnt = process_resp_flush(qp, cq, num_entries, wc,
3429 resp->rq_cons);
3430 try_consume_resp_cqe(cq, qp, resp, update);
3431 } else {
3432 cnt = process_resp_one(dev, qp, cq, wc, resp);
3433 consume_cqe(cq);
3434 *update |= 1;
3435 }
3436
3437 return cnt;
3438 }
3439
3440 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3441 struct rdma_cqe_requester *req, int *update)
3442 {
3443 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3444 consume_cqe(cq);
3445 *update |= 1;
3446 }
3447 }
3448
3449 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3450 {
3451 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3452 struct qedr_cq *cq = get_qedr_cq(ibcq);
3453 union rdma_cqe *cqe = cq->latest_cqe;
3454 u32 old_cons, new_cons;
3455 unsigned long flags;
3456 int update = 0;
3457 int done = 0;
3458
3459 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3460 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3461
3462 spin_lock_irqsave(&cq->cq_lock, flags);
3463 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3464 while (num_entries && is_valid_cqe(cq, cqe)) {
3465 struct qedr_qp *qp;
3466 int cnt = 0;
3467
3468 /* prevent speculative reads of any field of CQE */
3469 rmb();
3470
3471 qp = cqe_get_qp(cqe);
3472 if (!qp) {
3473 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3474 break;
3475 }
3476
3477 wc->qp = &qp->ibqp;
3478
3479 switch (cqe_get_type(cqe)) {
3480 case RDMA_CQE_TYPE_REQUESTER:
3481 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3482 &cqe->req);
3483 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3484 break;
3485 case RDMA_CQE_TYPE_RESPONDER_RQ:
3486 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3487 &cqe->resp, &update);
3488 break;
3489 case RDMA_CQE_TYPE_INVALID:
3490 default:
3491 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3492 cqe_get_type(cqe));
3493 }
3494 num_entries -= cnt;
3495 wc += cnt;
3496 done += cnt;
3497
3498 cqe = get_cqe(cq);
3499 }
3500 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3501
3502 cq->cq_cons += new_cons - old_cons;
3503
3504 if (update)
3505 /* doorbell notifies abount latest VALID entry,
3506 * but chain already point to the next INVALID one
3507 */
3508 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3509
3510 spin_unlock_irqrestore(&cq->cq_lock, flags);
3511 return done;
3512 }
3513
3514 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3515 u8 port_num,
3516 const struct ib_wc *in_wc,
3517 const struct ib_grh *in_grh,
3518 const struct ib_mad_hdr *mad_hdr,
3519 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3520 size_t *out_mad_size, u16 *out_mad_pkey_index)
3521 {
3522 struct qedr_dev *dev = get_qedr_dev(ibdev);
3523
3524 DP_DEBUG(dev, QEDR_MSG_GSI,
3525 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3526 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3527 mad_hdr->class_specific, mad_hdr->class_version,
3528 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3529 return IB_MAD_RESULT_SUCCESS;
3530 }
3531
3532 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3533 struct ib_port_immutable *immutable)
3534 {
3535 struct ib_port_attr attr;
3536 int err;
3537
3538 err = qedr_query_port(ibdev, port_num, &attr);
3539 if (err)
3540 return err;
3541
3542 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3543 immutable->gid_tbl_len = attr.gid_tbl_len;
3544 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3545 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3546 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3547
3548 return 0;
3549 }
Ubuntu Jammy Linux kernel mirror