]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/infiniband/hw/qedr/verbs.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma
[mirror_ubuntu-bionic-kernel.git] / drivers / infiniband / hw / qedr / verbs.c
1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45
46 #include "qedr_hsi.h"
47 #include <linux/qed/qed_if.h>
48 #include "qedr.h"
49 #include "verbs.h"
50 #include <rdma/qedr-abi.h>
51 #include "qedr_cm.h"
52
53 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56 {
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62 }
63
64 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66 {
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83 }
84
85 int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88 {
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99 }
100
101 int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103 {
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114 }
115
116 int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118 {
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170 }
171
172 #define QEDR_SPEED_SDR (1)
173 #define QEDR_SPEED_DDR (2)
174 #define QEDR_SPEED_QDR (4)
175 #define QEDR_SPEED_FDR10 (8)
176 #define QEDR_SPEED_FDR (16)
177 #define QEDR_SPEED_EDR (32)
178
179 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181 {
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222 }
223
224 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225 {
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241
242 /* *attr being zeroed by the caller, avoid zeroing it here */
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267 }
268
269 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271 {
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281 }
282
283 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285 {
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313 }
314
315 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317 {
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335 }
336
337 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339 {
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394 err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397 }
398
399 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400 {
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419 }
420
421 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422 {
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468 }
469
470 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472 {
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
474 struct qedr_pd *pd;
475 u16 pd_id;
476 int rc;
477
478 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
479 (udata && context) ? "User Lib" : "Kernel");
480
481 if (!dev->rdma_ctx) {
482 DP_ERR(dev, "invlaid RDMA context\n");
483 return ERR_PTR(-EINVAL);
484 }
485
486 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
487 if (!pd)
488 return ERR_PTR(-ENOMEM);
489
490 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
491 if (rc)
492 goto err;
493
494 pd->pd_id = pd_id;
495
496 if (udata && context) {
497 struct qedr_alloc_pd_uresp uresp;
498
499 uresp.pd_id = pd_id;
500
501 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
502 if (rc) {
503 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
504 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
505 goto err;
506 }
507
508 pd->uctx = get_qedr_ucontext(context);
509 pd->uctx->pd = pd;
510 }
511
512 return &pd->ibpd;
513
514 err:
515 kfree(pd);
516 return ERR_PTR(rc);
517 }
518
519 int qedr_dealloc_pd(struct ib_pd *ibpd)
520 {
521 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
522 struct qedr_pd *pd = get_qedr_pd(ibpd);
523
524 if (!pd) {
525 pr_err("Invalid PD received in dealloc_pd\n");
526 return -EINVAL;
527 }
528
529 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
530 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
531
532 kfree(pd);
533
534 return 0;
535 }
536
537 static void qedr_free_pbl(struct qedr_dev *dev,
538 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
539 {
540 struct pci_dev *pdev = dev->pdev;
541 int i;
542
543 for (i = 0; i < pbl_info->num_pbls; i++) {
544 if (!pbl[i].va)
545 continue;
546 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
547 pbl[i].va, pbl[i].pa);
548 }
549
550 kfree(pbl);
551 }
552
553 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
554 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
555
556 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
557 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
558 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
559
560 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
561 struct qedr_pbl_info *pbl_info,
562 gfp_t flags)
563 {
564 struct pci_dev *pdev = dev->pdev;
565 struct qedr_pbl *pbl_table;
566 dma_addr_t *pbl_main_tbl;
567 dma_addr_t pa;
568 void *va;
569 int i;
570
571 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
572 if (!pbl_table)
573 return ERR_PTR(-ENOMEM);
574
575 for (i = 0; i < pbl_info->num_pbls; i++) {
576 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
577 &pa, flags);
578 if (!va)
579 goto err;
580
581 memset(va, 0, pbl_info->pbl_size);
582 pbl_table[i].va = va;
583 pbl_table[i].pa = pa;
584 }
585
586 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
587 * the first one with physical pointers to all of the rest
588 */
589 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
590 for (i = 0; i < pbl_info->num_pbls - 1; i++)
591 pbl_main_tbl[i] = pbl_table[i + 1].pa;
592
593 return pbl_table;
594
595 err:
596 for (i--; i >= 0; i--)
597 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
598 pbl_table[i].va, pbl_table[i].pa);
599
600 qedr_free_pbl(dev, pbl_info, pbl_table);
601
602 return ERR_PTR(-ENOMEM);
603 }
604
605 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
606 struct qedr_pbl_info *pbl_info,
607 u32 num_pbes, int two_layer_capable)
608 {
609 u32 pbl_capacity;
610 u32 pbl_size;
611 u32 num_pbls;
612
613 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
614 if (num_pbes > MAX_PBES_TWO_LAYER) {
615 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
616 num_pbes);
617 return -EINVAL;
618 }
619
620 /* calculate required pbl page size */
621 pbl_size = MIN_FW_PBL_PAGE_SIZE;
622 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
623 NUM_PBES_ON_PAGE(pbl_size);
624
625 while (pbl_capacity < num_pbes) {
626 pbl_size *= 2;
627 pbl_capacity = pbl_size / sizeof(u64);
628 pbl_capacity = pbl_capacity * pbl_capacity;
629 }
630
631 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
632 num_pbls++; /* One for the layer0 ( points to the pbls) */
633 pbl_info->two_layered = true;
634 } else {
635 /* One layered PBL */
636 num_pbls = 1;
637 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
638 roundup_pow_of_two((num_pbes * sizeof(u64))));
639 pbl_info->two_layered = false;
640 }
641
642 pbl_info->num_pbls = num_pbls;
643 pbl_info->pbl_size = pbl_size;
644 pbl_info->num_pbes = num_pbes;
645
646 DP_DEBUG(dev, QEDR_MSG_MR,
647 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
648 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
649
650 return 0;
651 }
652
653 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
654 struct qedr_pbl *pbl,
655 struct qedr_pbl_info *pbl_info)
656 {
657 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
658 struct qedr_pbl *pbl_tbl;
659 struct scatterlist *sg;
660 struct regpair *pbe;
661 int entry;
662 u32 addr;
663
664 if (!pbl_info->num_pbes)
665 return;
666
667 /* If we have a two layered pbl, the first pbl points to the rest
668 * of the pbls and the first entry lays on the second pbl in the table
669 */
670 if (pbl_info->two_layered)
671 pbl_tbl = &pbl[1];
672 else
673 pbl_tbl = pbl;
674
675 pbe = (struct regpair *)pbl_tbl->va;
676 if (!pbe) {
677 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
678 return;
679 }
680
681 pbe_cnt = 0;
682
683 shift = ilog2(umem->page_size);
684
685 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
686 pages = sg_dma_len(sg) >> shift;
687 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
688 /* store the page address in pbe */
689 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
690 umem->page_size * pg_cnt);
691 addr = upper_32_bits(sg_dma_address(sg) +
692 umem->page_size * pg_cnt);
693 pbe->hi = cpu_to_le32(addr);
694 pbe_cnt++;
695 total_num_pbes++;
696 pbe++;
697
698 if (total_num_pbes == pbl_info->num_pbes)
699 return;
700
701 /* If the given pbl is full storing the pbes,
702 * move to next pbl.
703 */
704 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
705 pbl_tbl++;
706 pbe = (struct regpair *)pbl_tbl->va;
707 pbe_cnt = 0;
708 }
709 }
710 }
711 }
712
713 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
714 struct qedr_cq *cq, struct ib_udata *udata)
715 {
716 struct qedr_create_cq_uresp uresp;
717 int rc;
718
719 memset(&uresp, 0, sizeof(uresp));
720
721 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
722 uresp.icid = cq->icid;
723
724 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
725 if (rc)
726 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
727
728 return rc;
729 }
730
731 static void consume_cqe(struct qedr_cq *cq)
732 {
733 if (cq->latest_cqe == cq->toggle_cqe)
734 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
735
736 cq->latest_cqe = qed_chain_consume(&cq->pbl);
737 }
738
739 static inline int qedr_align_cq_entries(int entries)
740 {
741 u64 size, aligned_size;
742
743 /* We allocate an extra entry that we don't report to the FW. */
744 size = (entries + 1) * QEDR_CQE_SIZE;
745 aligned_size = ALIGN(size, PAGE_SIZE);
746
747 return aligned_size / QEDR_CQE_SIZE;
748 }
749
750 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
751 struct qedr_dev *dev,
752 struct qedr_userq *q,
753 u64 buf_addr, size_t buf_len,
754 int access, int dmasync)
755 {
756 int page_cnt;
757 int rc;
758
759 q->buf_addr = buf_addr;
760 q->buf_len = buf_len;
761 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
762 if (IS_ERR(q->umem)) {
763 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
764 PTR_ERR(q->umem));
765 return PTR_ERR(q->umem);
766 }
767
768 page_cnt = ib_umem_page_count(q->umem);
769 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
770 if (rc)
771 goto err0;
772
773 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
774 if (IS_ERR(q->pbl_tbl)) {
775 rc = PTR_ERR(q->pbl_tbl);
776 goto err0;
777 }
778
779 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
780
781 return 0;
782
783 err0:
784 ib_umem_release(q->umem);
785
786 return rc;
787 }
788
789 static inline void qedr_init_cq_params(struct qedr_cq *cq,
790 struct qedr_ucontext *ctx,
791 struct qedr_dev *dev, int vector,
792 int chain_entries, int page_cnt,
793 u64 pbl_ptr,
794 struct qed_rdma_create_cq_in_params
795 *params)
796 {
797 memset(params, 0, sizeof(*params));
798 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
799 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
800 params->cnq_id = vector;
801 params->cq_size = chain_entries - 1;
802 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
803 params->pbl_num_pages = page_cnt;
804 params->pbl_ptr = pbl_ptr;
805 params->pbl_two_level = 0;
806 }
807
808 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
809 {
810 /* Flush data before signalling doorbell */
811 wmb();
812 cq->db.data.agg_flags = flags;
813 cq->db.data.value = cpu_to_le32(cons);
814 writeq(cq->db.raw, cq->db_addr);
815
816 /* Make sure write would stick */
817 mmiowb();
818 }
819
820 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
821 {
822 struct qedr_cq *cq = get_qedr_cq(ibcq);
823 unsigned long sflags;
824
825 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
826 return 0;
827
828 spin_lock_irqsave(&cq->cq_lock, sflags);
829
830 cq->arm_flags = 0;
831
832 if (flags & IB_CQ_SOLICITED)
833 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
834
835 if (flags & IB_CQ_NEXT_COMP)
836 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
837
838 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
839
840 spin_unlock_irqrestore(&cq->cq_lock, sflags);
841
842 return 0;
843 }
844
845 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
846 const struct ib_cq_init_attr *attr,
847 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
848 {
849 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
850 struct qed_rdma_destroy_cq_out_params destroy_oparams;
851 struct qed_rdma_destroy_cq_in_params destroy_iparams;
852 struct qedr_dev *dev = get_qedr_dev(ibdev);
853 struct qed_rdma_create_cq_in_params params;
854 struct qedr_create_cq_ureq ureq;
855 int vector = attr->comp_vector;
856 int entries = attr->cqe;
857 struct qedr_cq *cq;
858 int chain_entries;
859 int page_cnt;
860 u64 pbl_ptr;
861 u16 icid;
862 int rc;
863
864 DP_DEBUG(dev, QEDR_MSG_INIT,
865 "create_cq: called from %s. entries=%d, vector=%d\n",
866 udata ? "User Lib" : "Kernel", entries, vector);
867
868 if (entries > QEDR_MAX_CQES) {
869 DP_ERR(dev,
870 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
871 entries, QEDR_MAX_CQES);
872 return ERR_PTR(-EINVAL);
873 }
874
875 chain_entries = qedr_align_cq_entries(entries);
876 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
877
878 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
879 if (!cq)
880 return ERR_PTR(-ENOMEM);
881
882 if (udata) {
883 memset(&ureq, 0, sizeof(ureq));
884 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
885 DP_ERR(dev,
886 "create cq: problem copying data from user space\n");
887 goto err0;
888 }
889
890 if (!ureq.len) {
891 DP_ERR(dev,
892 "create cq: cannot create a cq with 0 entries\n");
893 goto err0;
894 }
895
896 cq->cq_type = QEDR_CQ_TYPE_USER;
897
898 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
899 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
900 if (rc)
901 goto err0;
902
903 pbl_ptr = cq->q.pbl_tbl->pa;
904 page_cnt = cq->q.pbl_info.num_pbes;
905
906 cq->ibcq.cqe = chain_entries;
907 } else {
908 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
909
910 rc = dev->ops->common->chain_alloc(dev->cdev,
911 QED_CHAIN_USE_TO_CONSUME,
912 QED_CHAIN_MODE_PBL,
913 QED_CHAIN_CNT_TYPE_U32,
914 chain_entries,
915 sizeof(union rdma_cqe),
916 &cq->pbl);
917 if (rc)
918 goto err1;
919
920 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
921 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
922 cq->ibcq.cqe = cq->pbl.capacity;
923 }
924
925 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
926 pbl_ptr, &params);
927
928 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
929 if (rc)
930 goto err2;
931
932 cq->icid = icid;
933 cq->sig = QEDR_CQ_MAGIC_NUMBER;
934 spin_lock_init(&cq->cq_lock);
935
936 if (ib_ctx) {
937 rc = qedr_copy_cq_uresp(dev, cq, udata);
938 if (rc)
939 goto err3;
940 } else {
941 /* Generate doorbell address. */
942 cq->db_addr = dev->db_addr +
943 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
944 cq->db.data.icid = cq->icid;
945 cq->db.data.params = DB_AGG_CMD_SET <<
946 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
947
948 /* point to the very last element, passing it we will toggle */
949 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
950 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
951 cq->latest_cqe = NULL;
952 consume_cqe(cq);
953 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
954 }
955
956 DP_DEBUG(dev, QEDR_MSG_CQ,
957 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
958 cq->icid, cq, params.cq_size);
959
960 return &cq->ibcq;
961
962 err3:
963 destroy_iparams.icid = cq->icid;
964 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
965 &destroy_oparams);
966 err2:
967 if (udata)
968 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
969 else
970 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
971 err1:
972 if (udata)
973 ib_umem_release(cq->q.umem);
974 err0:
975 kfree(cq);
976 return ERR_PTR(-EINVAL);
977 }
978
979 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
980 {
981 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
982 struct qedr_cq *cq = get_qedr_cq(ibcq);
983
984 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
985
986 return 0;
987 }
988
989 int qedr_destroy_cq(struct ib_cq *ibcq)
990 {
991 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
992 struct qed_rdma_destroy_cq_out_params oparams;
993 struct qed_rdma_destroy_cq_in_params iparams;
994 struct qedr_cq *cq = get_qedr_cq(ibcq);
995
996 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
997
998 /* GSIs CQs are handled by driver, so they don't exist in the FW */
999 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
1000 int rc;
1001
1002 iparams.icid = cq->icid;
1003 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams,
1004 &oparams);
1005 if (rc)
1006 return rc;
1007 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1008 }
1009
1010 if (ibcq->uobject && ibcq->uobject->context) {
1011 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1012 ib_umem_release(cq->q.umem);
1013 }
1014
1015 kfree(cq);
1016
1017 return 0;
1018 }
1019
1020 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1021 struct ib_qp_attr *attr,
1022 int attr_mask,
1023 struct qed_rdma_modify_qp_in_params
1024 *qp_params)
1025 {
1026 enum rdma_network_type nw_type;
1027 struct ib_gid_attr gid_attr;
1028 union ib_gid gid;
1029 u32 ipv4_addr;
1030 int rc = 0;
1031 int i;
1032
1033 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1034 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1035 if (rc)
1036 return rc;
1037
1038 if (!memcmp(&gid, &zgid, sizeof(gid)))
1039 return -ENOENT;
1040
1041 if (gid_attr.ndev) {
1042 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1043
1044 dev_put(gid_attr.ndev);
1045 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1046 switch (nw_type) {
1047 case RDMA_NETWORK_IPV6:
1048 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1049 sizeof(qp_params->sgid));
1050 memcpy(&qp_params->dgid.bytes[0],
1051 &attr->ah_attr.grh.dgid,
1052 sizeof(qp_params->dgid));
1053 qp_params->roce_mode = ROCE_V2_IPV6;
1054 SET_FIELD(qp_params->modify_flags,
1055 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1056 break;
1057 case RDMA_NETWORK_IB:
1058 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1059 sizeof(qp_params->sgid));
1060 memcpy(&qp_params->dgid.bytes[0],
1061 &attr->ah_attr.grh.dgid,
1062 sizeof(qp_params->dgid));
1063 qp_params->roce_mode = ROCE_V1;
1064 break;
1065 case RDMA_NETWORK_IPV4:
1066 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1067 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1068 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1069 qp_params->sgid.ipv4_addr = ipv4_addr;
1070 ipv4_addr =
1071 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1072 qp_params->dgid.ipv4_addr = ipv4_addr;
1073 SET_FIELD(qp_params->modify_flags,
1074 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1075 qp_params->roce_mode = ROCE_V2_IPV4;
1076 break;
1077 }
1078 }
1079
1080 for (i = 0; i < 4; i++) {
1081 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1082 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1083 }
1084
1085 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1086 qp_params->vlan_id = 0;
1087
1088 return 0;
1089 }
1090
1091 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1092 struct ib_qp_init_attr *attrs)
1093 {
1094 struct qedr_device_attr *qattr = &dev->attr;
1095
1096 /* QP0... attrs->qp_type == IB_QPT_GSI */
1097 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1098 DP_DEBUG(dev, QEDR_MSG_QP,
1099 "create qp: unsupported qp type=0x%x requested\n",
1100 attrs->qp_type);
1101 return -EINVAL;
1102 }
1103
1104 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1105 DP_ERR(dev,
1106 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1107 attrs->cap.max_send_wr, qattr->max_sqe);
1108 return -EINVAL;
1109 }
1110
1111 if (attrs->cap.max_inline_data > qattr->max_inline) {
1112 DP_ERR(dev,
1113 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1114 attrs->cap.max_inline_data, qattr->max_inline);
1115 return -EINVAL;
1116 }
1117
1118 if (attrs->cap.max_send_sge > qattr->max_sge) {
1119 DP_ERR(dev,
1120 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1121 attrs->cap.max_send_sge, qattr->max_sge);
1122 return -EINVAL;
1123 }
1124
1125 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1126 DP_ERR(dev,
1127 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1128 attrs->cap.max_recv_sge, qattr->max_sge);
1129 return -EINVAL;
1130 }
1131
1132 /* Unprivileged user space cannot create special QP */
1133 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1134 DP_ERR(dev,
1135 "create qp: userspace can't create special QPs of type=0x%x\n",
1136 attrs->qp_type);
1137 return -EINVAL;
1138 }
1139
1140 return 0;
1141 }
1142
1143 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1144 struct qedr_qp *qp)
1145 {
1146 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1147 uresp->rq_icid = qp->icid;
1148 }
1149
1150 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1151 struct qedr_qp *qp)
1152 {
1153 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1154 uresp->sq_icid = qp->icid + 1;
1155 }
1156
1157 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1158 struct qedr_qp *qp, struct ib_udata *udata)
1159 {
1160 struct qedr_create_qp_uresp uresp;
1161 int rc;
1162
1163 memset(&uresp, 0, sizeof(uresp));
1164 qedr_copy_sq_uresp(&uresp, qp);
1165 qedr_copy_rq_uresp(&uresp, qp);
1166
1167 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1168 uresp.qp_id = qp->qp_id;
1169
1170 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1171 if (rc)
1172 DP_ERR(dev,
1173 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1174 qp->icid);
1175
1176 return rc;
1177 }
1178
1179 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1180 struct qedr_qp *qp,
1181 struct qedr_pd *pd,
1182 struct ib_qp_init_attr *attrs)
1183 {
1184 spin_lock_init(&qp->q_lock);
1185 qp->pd = pd;
1186 qp->qp_type = attrs->qp_type;
1187 qp->max_inline_data = attrs->cap.max_inline_data;
1188 qp->sq.max_sges = attrs->cap.max_send_sge;
1189 qp->state = QED_ROCE_QP_STATE_RESET;
1190 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1191 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1192 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1193 qp->dev = dev;
1194 qp->rq.max_sges = attrs->cap.max_recv_sge;
1195
1196 DP_DEBUG(dev, QEDR_MSG_QP,
1197 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1198 qp->rq.max_sges, qp->rq_cq->icid);
1199 DP_DEBUG(dev, QEDR_MSG_QP,
1200 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1201 pd->pd_id, qp->qp_type, qp->max_inline_data,
1202 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1203 DP_DEBUG(dev, QEDR_MSG_QP,
1204 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1205 qp->sq.max_sges, qp->sq_cq->icid);
1206 }
1207
1208 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1209 {
1210 qp->sq.db = dev->db_addr +
1211 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1212 qp->sq.db_data.data.icid = qp->icid + 1;
1213 qp->rq.db = dev->db_addr +
1214 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1215 qp->rq.db_data.data.icid = qp->icid;
1216 }
1217
1218 static inline void
1219 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1220 struct qedr_pd *pd,
1221 struct qedr_qp *qp,
1222 struct ib_qp_init_attr *attrs,
1223 bool fmr_and_reserved_lkey,
1224 struct qed_rdma_create_qp_in_params *params)
1225 {
1226 /* QP handle to be written in an async event */
1227 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1228 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1229
1230 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1231 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1232 params->pd = pd->pd_id;
1233 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1234 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1235 params->stats_queue = 0;
1236 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1237 params->srq_id = 0;
1238 params->use_srq = false;
1239 }
1240
1241 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1242 {
1243 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1244 "qp=%p. "
1245 "sq_addr=0x%llx, "
1246 "sq_len=%zd, "
1247 "rq_addr=0x%llx, "
1248 "rq_len=%zd"
1249 "\n",
1250 qp,
1251 qp->usq.buf_addr,
1252 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1253 }
1254
1255 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1256 {
1257 if (qp->usq.umem)
1258 ib_umem_release(qp->usq.umem);
1259 qp->usq.umem = NULL;
1260
1261 if (qp->urq.umem)
1262 ib_umem_release(qp->urq.umem);
1263 qp->urq.umem = NULL;
1264 }
1265
1266 static int qedr_create_user_qp(struct qedr_dev *dev,
1267 struct qedr_qp *qp,
1268 struct ib_pd *ibpd,
1269 struct ib_udata *udata,
1270 struct ib_qp_init_attr *attrs)
1271 {
1272 struct qed_rdma_create_qp_in_params in_params;
1273 struct qed_rdma_create_qp_out_params out_params;
1274 struct qedr_pd *pd = get_qedr_pd(ibpd);
1275 struct ib_ucontext *ib_ctx = NULL;
1276 struct qedr_ucontext *ctx = NULL;
1277 struct qedr_create_qp_ureq ureq;
1278 int rc = -EINVAL;
1279
1280 ib_ctx = ibpd->uobject->context;
1281 ctx = get_qedr_ucontext(ib_ctx);
1282
1283 memset(&ureq, 0, sizeof(ureq));
1284 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1285 if (rc) {
1286 DP_ERR(dev, "Problem copying data from user space\n");
1287 return rc;
1288 }
1289
1290 /* SQ - read access only (0), dma sync not required (0) */
1291 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1292 ureq.sq_len, 0, 0);
1293 if (rc)
1294 return rc;
1295
1296 /* RQ - read access only (0), dma sync not required (0) */
1297 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1298 ureq.rq_len, 0, 0);
1299
1300 if (rc)
1301 return rc;
1302
1303 memset(&in_params, 0, sizeof(in_params));
1304 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1305 in_params.qp_handle_lo = ureq.qp_handle_lo;
1306 in_params.qp_handle_hi = ureq.qp_handle_hi;
1307 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1308 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1309 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1310 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1311
1312 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1313 &in_params, &out_params);
1314
1315 if (!qp->qed_qp) {
1316 rc = -ENOMEM;
1317 goto err1;
1318 }
1319
1320 qp->qp_id = out_params.qp_id;
1321 qp->icid = out_params.icid;
1322
1323 rc = qedr_copy_qp_uresp(dev, qp, udata);
1324 if (rc)
1325 goto err;
1326
1327 qedr_qp_user_print(dev, qp);
1328
1329 return 0;
1330 err:
1331 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1332 if (rc)
1333 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1334
1335 err1:
1336 qedr_cleanup_user(dev, qp);
1337 return rc;
1338 }
1339
1340 static int
1341 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1342 struct qedr_qp *qp,
1343 struct qed_rdma_create_qp_in_params *in_params,
1344 u32 n_sq_elems, u32 n_rq_elems)
1345 {
1346 struct qed_rdma_create_qp_out_params out_params;
1347 int rc;
1348
1349 rc = dev->ops->common->chain_alloc(dev->cdev,
1350 QED_CHAIN_USE_TO_PRODUCE,
1351 QED_CHAIN_MODE_PBL,
1352 QED_CHAIN_CNT_TYPE_U32,
1353 n_sq_elems,
1354 QEDR_SQE_ELEMENT_SIZE,
1355 &qp->sq.pbl);
1356
1357 if (rc)
1358 return rc;
1359
1360 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1361 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1362
1363 rc = dev->ops->common->chain_alloc(dev->cdev,
1364 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1365 QED_CHAIN_MODE_PBL,
1366 QED_CHAIN_CNT_TYPE_U32,
1367 n_rq_elems,
1368 QEDR_RQE_ELEMENT_SIZE,
1369 &qp->rq.pbl);
1370 if (rc)
1371 return rc;
1372
1373 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1374 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1375
1376 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1377 in_params, &out_params);
1378
1379 if (!qp->qed_qp)
1380 return -EINVAL;
1381
1382 qp->qp_id = out_params.qp_id;
1383 qp->icid = out_params.icid;
1384
1385 qedr_set_roce_db_info(dev, qp);
1386
1387 return 0;
1388 }
1389
1390 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1391 {
1392 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1393 kfree(qp->wqe_wr_id);
1394
1395 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1396 kfree(qp->rqe_wr_id);
1397 }
1398
1399 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1400 struct qedr_qp *qp,
1401 struct ib_pd *ibpd,
1402 struct ib_qp_init_attr *attrs)
1403 {
1404 struct qed_rdma_create_qp_in_params in_params;
1405 struct qedr_pd *pd = get_qedr_pd(ibpd);
1406 int rc = -EINVAL;
1407 u32 n_rq_elems;
1408 u32 n_sq_elems;
1409 u32 n_sq_entries;
1410
1411 memset(&in_params, 0, sizeof(in_params));
1412
1413 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1414 * the ring. The ring should allow at least a single WR, even if the
1415 * user requested none, due to allocation issues.
1416 * We should add an extra WR since the prod and cons indices of
1417 * wqe_wr_id are managed in such a way that the WQ is considered full
1418 * when (prod+1)%max_wr==cons. We currently don't do that because we
1419 * double the number of entries due an iSER issue that pushes far more
1420 * WRs than indicated. If we decline its ib_post_send() then we get
1421 * error prints in the dmesg we'd like to avoid.
1422 */
1423 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1424 dev->attr.max_sqe);
1425
1426 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1427 GFP_KERNEL);
1428 if (!qp->wqe_wr_id) {
1429 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1430 return -ENOMEM;
1431 }
1432
1433 /* QP handle to be written in CQE */
1434 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1435 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1436
1437 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1438 * the ring. There ring should allow at least a single WR, even if the
1439 * user requested none, due to allocation issues.
1440 */
1441 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1442
1443 /* Allocate driver internal RQ array */
1444 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1445 GFP_KERNEL);
1446 if (!qp->rqe_wr_id) {
1447 DP_ERR(dev,
1448 "create qp: failed RQ shadow memory allocation\n");
1449 kfree(qp->wqe_wr_id);
1450 return -ENOMEM;
1451 }
1452
1453 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1454
1455 n_sq_entries = attrs->cap.max_send_wr;
1456 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1457 n_sq_entries = max_t(u32, n_sq_entries, 1);
1458 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1459
1460 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1461
1462 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1463 n_sq_elems, n_rq_elems);
1464 if (rc)
1465 qedr_cleanup_kernel(dev, qp);
1466
1467 return rc;
1468 }
1469
1470 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1471 struct ib_qp_init_attr *attrs,
1472 struct ib_udata *udata)
1473 {
1474 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1475 struct qedr_pd *pd = get_qedr_pd(ibpd);
1476 struct qedr_qp *qp;
1477 struct ib_qp *ibqp;
1478 int rc = 0;
1479
1480 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1481 udata ? "user library" : "kernel", pd);
1482
1483 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1484 if (rc)
1485 return ERR_PTR(rc);
1486
1487 if (attrs->srq)
1488 return ERR_PTR(-EINVAL);
1489
1490 DP_DEBUG(dev, QEDR_MSG_QP,
1491 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1492 udata ? "user library" : "kernel", attrs->event_handler, pd,
1493 get_qedr_cq(attrs->send_cq),
1494 get_qedr_cq(attrs->send_cq)->icid,
1495 get_qedr_cq(attrs->recv_cq),
1496 get_qedr_cq(attrs->recv_cq)->icid);
1497
1498 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1499 if (!qp) {
1500 DP_ERR(dev, "create qp: failed allocating memory\n");
1501 return ERR_PTR(-ENOMEM);
1502 }
1503
1504 qedr_set_common_qp_params(dev, qp, pd, attrs);
1505
1506 if (attrs->qp_type == IB_QPT_GSI) {
1507 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1508 if (IS_ERR(ibqp))
1509 kfree(qp);
1510 return ibqp;
1511 }
1512
1513 if (udata)
1514 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1515 else
1516 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1517
1518 if (rc)
1519 goto err;
1520
1521 qp->ibqp.qp_num = qp->qp_id;
1522
1523 return &qp->ibqp;
1524
1525 err:
1526 kfree(qp);
1527
1528 return ERR_PTR(-EFAULT);
1529 }
1530
1531 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1532 {
1533 switch (qp_state) {
1534 case QED_ROCE_QP_STATE_RESET:
1535 return IB_QPS_RESET;
1536 case QED_ROCE_QP_STATE_INIT:
1537 return IB_QPS_INIT;
1538 case QED_ROCE_QP_STATE_RTR:
1539 return IB_QPS_RTR;
1540 case QED_ROCE_QP_STATE_RTS:
1541 return IB_QPS_RTS;
1542 case QED_ROCE_QP_STATE_SQD:
1543 return IB_QPS_SQD;
1544 case QED_ROCE_QP_STATE_ERR:
1545 return IB_QPS_ERR;
1546 case QED_ROCE_QP_STATE_SQE:
1547 return IB_QPS_SQE;
1548 }
1549 return IB_QPS_ERR;
1550 }
1551
1552 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1553 enum ib_qp_state qp_state)
1554 {
1555 switch (qp_state) {
1556 case IB_QPS_RESET:
1557 return QED_ROCE_QP_STATE_RESET;
1558 case IB_QPS_INIT:
1559 return QED_ROCE_QP_STATE_INIT;
1560 case IB_QPS_RTR:
1561 return QED_ROCE_QP_STATE_RTR;
1562 case IB_QPS_RTS:
1563 return QED_ROCE_QP_STATE_RTS;
1564 case IB_QPS_SQD:
1565 return QED_ROCE_QP_STATE_SQD;
1566 case IB_QPS_ERR:
1567 return QED_ROCE_QP_STATE_ERR;
1568 default:
1569 return QED_ROCE_QP_STATE_ERR;
1570 }
1571 }
1572
1573 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1574 {
1575 qed_chain_reset(&qph->pbl);
1576 qph->prod = 0;
1577 qph->cons = 0;
1578 qph->wqe_cons = 0;
1579 qph->db_data.data.value = cpu_to_le16(0);
1580 }
1581
1582 static int qedr_update_qp_state(struct qedr_dev *dev,
1583 struct qedr_qp *qp,
1584 enum qed_roce_qp_state new_state)
1585 {
1586 int status = 0;
1587
1588 if (new_state == qp->state)
1589 return 0;
1590
1591 switch (qp->state) {
1592 case QED_ROCE_QP_STATE_RESET:
1593 switch (new_state) {
1594 case QED_ROCE_QP_STATE_INIT:
1595 qp->prev_wqe_size = 0;
1596 qedr_reset_qp_hwq_info(&qp->sq);
1597 qedr_reset_qp_hwq_info(&qp->rq);
1598 break;
1599 default:
1600 status = -EINVAL;
1601 break;
1602 };
1603 break;
1604 case QED_ROCE_QP_STATE_INIT:
1605 switch (new_state) {
1606 case QED_ROCE_QP_STATE_RTR:
1607 /* Update doorbell (in case post_recv was
1608 * done before move to RTR)
1609 */
1610 wmb();
1611 writel(qp->rq.db_data.raw, qp->rq.db);
1612 /* Make sure write takes effect */
1613 mmiowb();
1614 break;
1615 case QED_ROCE_QP_STATE_ERR:
1616 break;
1617 default:
1618 /* Invalid state change. */
1619 status = -EINVAL;
1620 break;
1621 };
1622 break;
1623 case QED_ROCE_QP_STATE_RTR:
1624 /* RTR->XXX */
1625 switch (new_state) {
1626 case QED_ROCE_QP_STATE_RTS:
1627 break;
1628 case QED_ROCE_QP_STATE_ERR:
1629 break;
1630 default:
1631 /* Invalid state change. */
1632 status = -EINVAL;
1633 break;
1634 };
1635 break;
1636 case QED_ROCE_QP_STATE_RTS:
1637 /* RTS->XXX */
1638 switch (new_state) {
1639 case QED_ROCE_QP_STATE_SQD:
1640 break;
1641 case QED_ROCE_QP_STATE_ERR:
1642 break;
1643 default:
1644 /* Invalid state change. */
1645 status = -EINVAL;
1646 break;
1647 };
1648 break;
1649 case QED_ROCE_QP_STATE_SQD:
1650 /* SQD->XXX */
1651 switch (new_state) {
1652 case QED_ROCE_QP_STATE_RTS:
1653 case QED_ROCE_QP_STATE_ERR:
1654 break;
1655 default:
1656 /* Invalid state change. */
1657 status = -EINVAL;
1658 break;
1659 };
1660 break;
1661 case QED_ROCE_QP_STATE_ERR:
1662 /* ERR->XXX */
1663 switch (new_state) {
1664 case QED_ROCE_QP_STATE_RESET:
1665 if ((qp->rq.prod != qp->rq.cons) ||
1666 (qp->sq.prod != qp->sq.cons)) {
1667 DP_NOTICE(dev,
1668 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1669 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1670 qp->sq.cons);
1671 status = -EINVAL;
1672 }
1673 break;
1674 default:
1675 status = -EINVAL;
1676 break;
1677 };
1678 break;
1679 default:
1680 status = -EINVAL;
1681 break;
1682 };
1683
1684 return status;
1685 }
1686
1687 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1688 int attr_mask, struct ib_udata *udata)
1689 {
1690 struct qedr_qp *qp = get_qedr_qp(ibqp);
1691 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1692 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1693 enum ib_qp_state old_qp_state, new_qp_state;
1694 int rc = 0;
1695
1696 DP_DEBUG(dev, QEDR_MSG_QP,
1697 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1698 attr->qp_state);
1699
1700 old_qp_state = qedr_get_ibqp_state(qp->state);
1701 if (attr_mask & IB_QP_STATE)
1702 new_qp_state = attr->qp_state;
1703 else
1704 new_qp_state = old_qp_state;
1705
1706 if (!ib_modify_qp_is_ok
1707 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1708 IB_LINK_LAYER_ETHERNET)) {
1709 DP_ERR(dev,
1710 "modify qp: invalid attribute mask=0x%x specified for\n"
1711 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1712 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1713 new_qp_state);
1714 rc = -EINVAL;
1715 goto err;
1716 }
1717
1718 /* Translate the masks... */
1719 if (attr_mask & IB_QP_STATE) {
1720 SET_FIELD(qp_params.modify_flags,
1721 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1722 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1723 }
1724
1725 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1726 qp_params.sqd_async = true;
1727
1728 if (attr_mask & IB_QP_PKEY_INDEX) {
1729 SET_FIELD(qp_params.modify_flags,
1730 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1731 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1732 rc = -EINVAL;
1733 goto err;
1734 }
1735
1736 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1737 }
1738
1739 if (attr_mask & IB_QP_QKEY)
1740 qp->qkey = attr->qkey;
1741
1742 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1743 SET_FIELD(qp_params.modify_flags,
1744 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1745 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1746 IB_ACCESS_REMOTE_READ;
1747 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1748 IB_ACCESS_REMOTE_WRITE;
1749 qp_params.incoming_atomic_en = attr->qp_access_flags &
1750 IB_ACCESS_REMOTE_ATOMIC;
1751 }
1752
1753 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1754 if (attr_mask & IB_QP_PATH_MTU) {
1755 if (attr->path_mtu < IB_MTU_256 ||
1756 attr->path_mtu > IB_MTU_4096) {
1757 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1758 rc = -EINVAL;
1759 goto err;
1760 }
1761 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1762 ib_mtu_enum_to_int(iboe_get_mtu
1763 (dev->ndev->mtu)));
1764 }
1765
1766 if (!qp->mtu) {
1767 qp->mtu =
1768 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1769 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1770 }
1771
1772 SET_FIELD(qp_params.modify_flags,
1773 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1774
1775 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1776 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1777 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1778
1779 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1780
1781 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1782 if (rc) {
1783 DP_ERR(dev,
1784 "modify qp: problems with GID index %d (rc=%d)\n",
1785 attr->ah_attr.grh.sgid_index, rc);
1786 return rc;
1787 }
1788
1789 rc = qedr_get_dmac(dev, &attr->ah_attr,
1790 qp_params.remote_mac_addr);
1791 if (rc)
1792 return rc;
1793
1794 qp_params.use_local_mac = true;
1795 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1796
1797 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1798 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1799 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1800 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1801 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1802 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1803 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1804 qp_params.remote_mac_addr);
1805
1806 qp_params.mtu = qp->mtu;
1807 qp_params.lb_indication = false;
1808 }
1809
1810 if (!qp_params.mtu) {
1811 /* Stay with current MTU */
1812 if (qp->mtu)
1813 qp_params.mtu = qp->mtu;
1814 else
1815 qp_params.mtu =
1816 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1817 }
1818
1819 if (attr_mask & IB_QP_TIMEOUT) {
1820 SET_FIELD(qp_params.modify_flags,
1821 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1822
1823 qp_params.ack_timeout = attr->timeout;
1824 if (attr->timeout) {
1825 u32 temp;
1826
1827 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1828 /* FW requires [msec] */
1829 qp_params.ack_timeout = temp;
1830 } else {
1831 /* Infinite */
1832 qp_params.ack_timeout = 0;
1833 }
1834 }
1835 if (attr_mask & IB_QP_RETRY_CNT) {
1836 SET_FIELD(qp_params.modify_flags,
1837 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1838 qp_params.retry_cnt = attr->retry_cnt;
1839 }
1840
1841 if (attr_mask & IB_QP_RNR_RETRY) {
1842 SET_FIELD(qp_params.modify_flags,
1843 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1844 qp_params.rnr_retry_cnt = attr->rnr_retry;
1845 }
1846
1847 if (attr_mask & IB_QP_RQ_PSN) {
1848 SET_FIELD(qp_params.modify_flags,
1849 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1850 qp_params.rq_psn = attr->rq_psn;
1851 qp->rq_psn = attr->rq_psn;
1852 }
1853
1854 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1855 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1856 rc = -EINVAL;
1857 DP_ERR(dev,
1858 "unsupported max_rd_atomic=%d, supported=%d\n",
1859 attr->max_rd_atomic,
1860 dev->attr.max_qp_req_rd_atomic_resc);
1861 goto err;
1862 }
1863
1864 SET_FIELD(qp_params.modify_flags,
1865 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1866 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1867 }
1868
1869 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1870 SET_FIELD(qp_params.modify_flags,
1871 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1872 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1873 }
1874
1875 if (attr_mask & IB_QP_SQ_PSN) {
1876 SET_FIELD(qp_params.modify_flags,
1877 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1878 qp_params.sq_psn = attr->sq_psn;
1879 qp->sq_psn = attr->sq_psn;
1880 }
1881
1882 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1883 if (attr->max_dest_rd_atomic >
1884 dev->attr.max_qp_resp_rd_atomic_resc) {
1885 DP_ERR(dev,
1886 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1887 attr->max_dest_rd_atomic,
1888 dev->attr.max_qp_resp_rd_atomic_resc);
1889
1890 rc = -EINVAL;
1891 goto err;
1892 }
1893
1894 SET_FIELD(qp_params.modify_flags,
1895 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1896 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1897 }
1898
1899 if (attr_mask & IB_QP_DEST_QPN) {
1900 SET_FIELD(qp_params.modify_flags,
1901 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1902
1903 qp_params.dest_qp = attr->dest_qp_num;
1904 qp->dest_qp_num = attr->dest_qp_num;
1905 }
1906
1907 if (qp->qp_type != IB_QPT_GSI)
1908 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1909 qp->qed_qp, &qp_params);
1910
1911 if (attr_mask & IB_QP_STATE) {
1912 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1913 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
1914 qp->state = qp_params.new_state;
1915 }
1916
1917 err:
1918 return rc;
1919 }
1920
1921 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1922 {
1923 int ib_qp_acc_flags = 0;
1924
1925 if (params->incoming_rdma_write_en)
1926 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1927 if (params->incoming_rdma_read_en)
1928 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
1929 if (params->incoming_atomic_en)
1930 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
1931 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1932 return ib_qp_acc_flags;
1933 }
1934
1935 int qedr_query_qp(struct ib_qp *ibqp,
1936 struct ib_qp_attr *qp_attr,
1937 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1938 {
1939 struct qed_rdma_query_qp_out_params params;
1940 struct qedr_qp *qp = get_qedr_qp(ibqp);
1941 struct qedr_dev *dev = qp->dev;
1942 int rc = 0;
1943
1944 memset(&params, 0, sizeof(params));
1945
1946 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
1947 if (rc)
1948 goto err;
1949
1950 memset(qp_attr, 0, sizeof(*qp_attr));
1951 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
1952
1953 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
1954 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
1955 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
1956 qp_attr->path_mig_state = IB_MIG_MIGRATED;
1957 qp_attr->rq_psn = params.rq_psn;
1958 qp_attr->sq_psn = params.sq_psn;
1959 qp_attr->dest_qp_num = params.dest_qp;
1960
1961 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
1962
1963 qp_attr->cap.max_send_wr = qp->sq.max_wr;
1964 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
1965 qp_attr->cap.max_send_sge = qp->sq.max_sges;
1966 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1967 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
1968 qp_init_attr->cap = qp_attr->cap;
1969
1970 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
1971 sizeof(qp_attr->ah_attr.grh.dgid.raw));
1972
1973 qp_attr->ah_attr.grh.flow_label = params.flow_label;
1974 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
1975 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
1976 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
1977
1978 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
1979 qp_attr->ah_attr.port_num = 1;
1980 qp_attr->ah_attr.sl = 0;
1981 qp_attr->timeout = params.timeout;
1982 qp_attr->rnr_retry = params.rnr_retry;
1983 qp_attr->retry_cnt = params.retry_cnt;
1984 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
1985 qp_attr->pkey_index = params.pkey_index;
1986 qp_attr->port_num = 1;
1987 qp_attr->ah_attr.src_path_bits = 0;
1988 qp_attr->ah_attr.static_rate = 0;
1989 qp_attr->alt_pkey_index = 0;
1990 qp_attr->alt_port_num = 0;
1991 qp_attr->alt_timeout = 0;
1992 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1993
1994 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
1995 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
1996 qp_attr->max_rd_atomic = params.max_rd_atomic;
1997 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
1998
1999 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2000 qp_attr->cap.max_inline_data);
2001
2002 err:
2003 return rc;
2004 }
2005
2006 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2007 {
2008 int rc = 0;
2009
2010 if (qp->qp_type != IB_QPT_GSI) {
2011 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2012 if (rc)
2013 return rc;
2014 }
2015
2016 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2017 qedr_cleanup_user(dev, qp);
2018 else
2019 qedr_cleanup_kernel(dev, qp);
2020
2021 return 0;
2022 }
2023
2024 int qedr_destroy_qp(struct ib_qp *ibqp)
2025 {
2026 struct qedr_qp *qp = get_qedr_qp(ibqp);
2027 struct qedr_dev *dev = qp->dev;
2028 struct ib_qp_attr attr;
2029 int attr_mask = 0;
2030 int rc = 0;
2031
2032 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2033 qp, qp->qp_type);
2034
2035 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2036 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2037 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2038
2039 attr.qp_state = IB_QPS_ERR;
2040 attr_mask |= IB_QP_STATE;
2041
2042 /* Change the QP state to ERROR */
2043 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2044 }
2045
2046 if (qp->qp_type == IB_QPT_GSI)
2047 qedr_destroy_gsi_qp(dev);
2048
2049 qedr_free_qp_resources(dev, qp);
2050
2051 kfree(qp);
2052
2053 return rc;
2054 }
2055
2056 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
2057 struct ib_udata *udata)
2058 {
2059 struct qedr_ah *ah;
2060
2061 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2062 if (!ah)
2063 return ERR_PTR(-ENOMEM);
2064
2065 ah->attr = *attr;
2066
2067 return &ah->ibah;
2068 }
2069
2070 int qedr_destroy_ah(struct ib_ah *ibah)
2071 {
2072 struct qedr_ah *ah = get_qedr_ah(ibah);
2073
2074 kfree(ah);
2075 return 0;
2076 }
2077
2078 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2079 {
2080 struct qedr_pbl *pbl, *tmp;
2081
2082 if (info->pbl_table)
2083 list_add_tail(&info->pbl_table->list_entry,
2084 &info->free_pbl_list);
2085
2086 if (!list_empty(&info->inuse_pbl_list))
2087 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2088
2089 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2090 list_del(&pbl->list_entry);
2091 qedr_free_pbl(dev, &info->pbl_info, pbl);
2092 }
2093 }
2094
2095 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2096 size_t page_list_len, bool two_layered)
2097 {
2098 struct qedr_pbl *tmp;
2099 int rc;
2100
2101 INIT_LIST_HEAD(&info->free_pbl_list);
2102 INIT_LIST_HEAD(&info->inuse_pbl_list);
2103
2104 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2105 page_list_len, two_layered);
2106 if (rc)
2107 goto done;
2108
2109 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2110 if (IS_ERR(info->pbl_table)) {
2111 rc = PTR_ERR(info->pbl_table);
2112 goto done;
2113 }
2114
2115 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2116 &info->pbl_table->pa);
2117
2118 /* in usual case we use 2 PBLs, so we add one to free
2119 * list and allocating another one
2120 */
2121 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2122 if (IS_ERR(tmp)) {
2123 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2124 goto done;
2125 }
2126
2127 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2128
2129 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2130
2131 done:
2132 if (rc)
2133 free_mr_info(dev, info);
2134
2135 return rc;
2136 }
2137
2138 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2139 u64 usr_addr, int acc, struct ib_udata *udata)
2140 {
2141 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2142 struct qedr_mr *mr;
2143 struct qedr_pd *pd;
2144 int rc = -ENOMEM;
2145
2146 pd = get_qedr_pd(ibpd);
2147 DP_DEBUG(dev, QEDR_MSG_MR,
2148 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2149 pd->pd_id, start, len, usr_addr, acc);
2150
2151 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2152 return ERR_PTR(-EINVAL);
2153
2154 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2155 if (!mr)
2156 return ERR_PTR(rc);
2157
2158 mr->type = QEDR_MR_USER;
2159
2160 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2161 if (IS_ERR(mr->umem)) {
2162 rc = -EFAULT;
2163 goto err0;
2164 }
2165
2166 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2167 if (rc)
2168 goto err1;
2169
2170 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2171 &mr->info.pbl_info);
2172
2173 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2174 if (rc) {
2175 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2176 goto err1;
2177 }
2178
2179 /* Index only, 18 bit long, lkey = itid << 8 | key */
2180 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2181 mr->hw_mr.key = 0;
2182 mr->hw_mr.pd = pd->pd_id;
2183 mr->hw_mr.local_read = 1;
2184 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2185 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2186 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2187 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2188 mr->hw_mr.mw_bind = false;
2189 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2190 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2191 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2192 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size);
2193 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2194 mr->hw_mr.length = len;
2195 mr->hw_mr.vaddr = usr_addr;
2196 mr->hw_mr.zbva = false;
2197 mr->hw_mr.phy_mr = false;
2198 mr->hw_mr.dma_mr = false;
2199
2200 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2201 if (rc) {
2202 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2203 goto err2;
2204 }
2205
2206 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2207 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2208 mr->hw_mr.remote_atomic)
2209 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2210
2211 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2212 mr->ibmr.lkey);
2213 return &mr->ibmr;
2214
2215 err2:
2216 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2217 err1:
2218 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2219 err0:
2220 kfree(mr);
2221 return ERR_PTR(rc);
2222 }
2223
2224 int qedr_dereg_mr(struct ib_mr *ib_mr)
2225 {
2226 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2227 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2228 int rc = 0;
2229
2230 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2231 if (rc)
2232 return rc;
2233
2234 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2235
2236 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2237 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2238
2239 /* it could be user registered memory. */
2240 if (mr->umem)
2241 ib_umem_release(mr->umem);
2242
2243 kfree(mr);
2244
2245 return rc;
2246 }
2247
2248 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2249 int max_page_list_len)
2250 {
2251 struct qedr_pd *pd = get_qedr_pd(ibpd);
2252 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2253 struct qedr_mr *mr;
2254 int rc = -ENOMEM;
2255
2256 DP_DEBUG(dev, QEDR_MSG_MR,
2257 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2258 max_page_list_len);
2259
2260 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2261 if (!mr)
2262 return ERR_PTR(rc);
2263
2264 mr->dev = dev;
2265 mr->type = QEDR_MR_FRMR;
2266
2267 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2268 if (rc)
2269 goto err0;
2270
2271 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2272 if (rc) {
2273 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2274 goto err0;
2275 }
2276
2277 /* Index only, 18 bit long, lkey = itid << 8 | key */
2278 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2279 mr->hw_mr.key = 0;
2280 mr->hw_mr.pd = pd->pd_id;
2281 mr->hw_mr.local_read = 1;
2282 mr->hw_mr.local_write = 0;
2283 mr->hw_mr.remote_read = 0;
2284 mr->hw_mr.remote_write = 0;
2285 mr->hw_mr.remote_atomic = 0;
2286 mr->hw_mr.mw_bind = false;
2287 mr->hw_mr.pbl_ptr = 0;
2288 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2289 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2290 mr->hw_mr.fbo = 0;
2291 mr->hw_mr.length = 0;
2292 mr->hw_mr.vaddr = 0;
2293 mr->hw_mr.zbva = false;
2294 mr->hw_mr.phy_mr = true;
2295 mr->hw_mr.dma_mr = false;
2296
2297 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2298 if (rc) {
2299 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2300 goto err1;
2301 }
2302
2303 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2304 mr->ibmr.rkey = mr->ibmr.lkey;
2305
2306 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2307 return mr;
2308
2309 err1:
2310 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2311 err0:
2312 kfree(mr);
2313 return ERR_PTR(rc);
2314 }
2315
2316 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2317 enum ib_mr_type mr_type, u32 max_num_sg)
2318 {
2319 struct qedr_dev *dev;
2320 struct qedr_mr *mr;
2321
2322 if (mr_type != IB_MR_TYPE_MEM_REG)
2323 return ERR_PTR(-EINVAL);
2324
2325 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2326
2327 if (IS_ERR(mr))
2328 return ERR_PTR(-EINVAL);
2329
2330 dev = mr->dev;
2331
2332 return &mr->ibmr;
2333 }
2334
2335 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2336 {
2337 struct qedr_mr *mr = get_qedr_mr(ibmr);
2338 struct qedr_pbl *pbl_table;
2339 struct regpair *pbe;
2340 u32 pbes_in_page;
2341
2342 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2343 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2344 return -ENOMEM;
2345 }
2346
2347 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2348 mr->npages, addr);
2349
2350 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2351 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2352 pbe = (struct regpair *)pbl_table->va;
2353 pbe += mr->npages % pbes_in_page;
2354 pbe->lo = cpu_to_le32((u32)addr);
2355 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2356
2357 mr->npages++;
2358
2359 return 0;
2360 }
2361
2362 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2363 {
2364 int work = info->completed - info->completed_handled - 1;
2365
2366 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2367 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2368 struct qedr_pbl *pbl;
2369
2370 /* Free all the page list that are possible to be freed
2371 * (all the ones that were invalidated), under the assumption
2372 * that if an FMR was completed successfully that means that
2373 * if there was an invalidate operation before it also ended
2374 */
2375 pbl = list_first_entry(&info->inuse_pbl_list,
2376 struct qedr_pbl, list_entry);
2377 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2378 info->completed_handled++;
2379 }
2380 }
2381
2382 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2383 int sg_nents, unsigned int *sg_offset)
2384 {
2385 struct qedr_mr *mr = get_qedr_mr(ibmr);
2386
2387 mr->npages = 0;
2388
2389 handle_completed_mrs(mr->dev, &mr->info);
2390 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2391 }
2392
2393 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2394 {
2395 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2396 struct qedr_pd *pd = get_qedr_pd(ibpd);
2397 struct qedr_mr *mr;
2398 int rc;
2399
2400 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2401 if (!mr)
2402 return ERR_PTR(-ENOMEM);
2403
2404 mr->type = QEDR_MR_DMA;
2405
2406 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2407 if (rc) {
2408 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2409 goto err1;
2410 }
2411
2412 /* index only, 18 bit long, lkey = itid << 8 | key */
2413 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2414 mr->hw_mr.pd = pd->pd_id;
2415 mr->hw_mr.local_read = 1;
2416 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2417 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2418 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2419 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2420 mr->hw_mr.dma_mr = true;
2421
2422 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2423 if (rc) {
2424 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2425 goto err2;
2426 }
2427
2428 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2429 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2430 mr->hw_mr.remote_atomic)
2431 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2432
2433 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2434 return &mr->ibmr;
2435
2436 err2:
2437 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2438 err1:
2439 kfree(mr);
2440 return ERR_PTR(rc);
2441 }
2442
2443 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2444 {
2445 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2446 }
2447
2448 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2449 {
2450 int i, len = 0;
2451
2452 for (i = 0; i < num_sge; i++)
2453 len += sg_list[i].length;
2454
2455 return len;
2456 }
2457
2458 static void swap_wqe_data64(u64 *p)
2459 {
2460 int i;
2461
2462 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2463 *p = cpu_to_be64(cpu_to_le64(*p));
2464 }
2465
2466 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2467 struct qedr_qp *qp, u8 *wqe_size,
2468 struct ib_send_wr *wr,
2469 struct ib_send_wr **bad_wr, u8 *bits,
2470 u8 bit)
2471 {
2472 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2473 char *seg_prt, *wqe;
2474 int i, seg_siz;
2475
2476 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2477 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2478 *bad_wr = wr;
2479 return 0;
2480 }
2481
2482 if (!data_size)
2483 return data_size;
2484
2485 *bits |= bit;
2486
2487 seg_prt = NULL;
2488 wqe = NULL;
2489 seg_siz = 0;
2490
2491 /* Copy data inline */
2492 for (i = 0; i < wr->num_sge; i++) {
2493 u32 len = wr->sg_list[i].length;
2494 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2495
2496 while (len > 0) {
2497 u32 cur;
2498
2499 /* New segment required */
2500 if (!seg_siz) {
2501 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2502 seg_prt = wqe;
2503 seg_siz = sizeof(struct rdma_sq_common_wqe);
2504 (*wqe_size)++;
2505 }
2506
2507 /* Calculate currently allowed length */
2508 cur = min_t(u32, len, seg_siz);
2509 memcpy(seg_prt, src, cur);
2510
2511 /* Update segment variables */
2512 seg_prt += cur;
2513 seg_siz -= cur;
2514
2515 /* Update sge variables */
2516 src += cur;
2517 len -= cur;
2518
2519 /* Swap fully-completed segments */
2520 if (!seg_siz)
2521 swap_wqe_data64((u64 *)wqe);
2522 }
2523 }
2524
2525 /* swap last not completed segment */
2526 if (seg_siz)
2527 swap_wqe_data64((u64 *)wqe);
2528
2529 return data_size;
2530 }
2531
2532 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2533 do { \
2534 DMA_REGPAIR_LE(sge->addr, vaddr); \
2535 (sge)->length = cpu_to_le32(vlength); \
2536 (sge)->flags = cpu_to_le32(vflags); \
2537 } while (0)
2538
2539 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2540 do { \
2541 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2542 (hdr)->num_sges = num_sge; \
2543 } while (0)
2544
2545 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2546 do { \
2547 DMA_REGPAIR_LE(sge->addr, vaddr); \
2548 (sge)->length = cpu_to_le32(vlength); \
2549 (sge)->l_key = cpu_to_le32(vlkey); \
2550 } while (0)
2551
2552 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2553 struct ib_send_wr *wr)
2554 {
2555 u32 data_size = 0;
2556 int i;
2557
2558 for (i = 0; i < wr->num_sge; i++) {
2559 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2560
2561 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2562 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2563 sge->length = cpu_to_le32(wr->sg_list[i].length);
2564 data_size += wr->sg_list[i].length;
2565 }
2566
2567 if (wqe_size)
2568 *wqe_size += wr->num_sge;
2569
2570 return data_size;
2571 }
2572
2573 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2574 struct qedr_qp *qp,
2575 struct rdma_sq_rdma_wqe_1st *rwqe,
2576 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2577 struct ib_send_wr *wr,
2578 struct ib_send_wr **bad_wr)
2579 {
2580 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2581 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2582
2583 if (wr->send_flags & IB_SEND_INLINE &&
2584 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2585 wr->opcode == IB_WR_RDMA_WRITE)) {
2586 u8 flags = 0;
2587
2588 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2589 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2590 bad_wr, &rwqe->flags, flags);
2591 }
2592
2593 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2594 }
2595
2596 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2597 struct qedr_qp *qp,
2598 struct rdma_sq_send_wqe_1st *swqe,
2599 struct rdma_sq_send_wqe_2st *swqe2,
2600 struct ib_send_wr *wr,
2601 struct ib_send_wr **bad_wr)
2602 {
2603 memset(swqe2, 0, sizeof(*swqe2));
2604 if (wr->send_flags & IB_SEND_INLINE) {
2605 u8 flags = 0;
2606
2607 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2608 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2609 bad_wr, &swqe->flags, flags);
2610 }
2611
2612 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2613 }
2614
2615 static int qedr_prepare_reg(struct qedr_qp *qp,
2616 struct rdma_sq_fmr_wqe_1st *fwqe1,
2617 struct ib_reg_wr *wr)
2618 {
2619 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2620 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2621
2622 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2623 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2624 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2625 fwqe1->l_key = wr->key;
2626
2627 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2628 !!(wr->access & IB_ACCESS_REMOTE_READ));
2629 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2630 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2631 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2632 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2633 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2634 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2635 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2636 fwqe2->fmr_ctrl = 0;
2637
2638 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2639 ilog2(mr->ibmr.page_size) - 12);
2640
2641 fwqe2->length_hi = 0;
2642 fwqe2->length_lo = mr->ibmr.length;
2643 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2644 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2645
2646 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2647
2648 return 0;
2649 }
2650
2651 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2652 {
2653 switch (opcode) {
2654 case IB_WR_RDMA_WRITE:
2655 case IB_WR_RDMA_WRITE_WITH_IMM:
2656 return IB_WC_RDMA_WRITE;
2657 case IB_WR_SEND_WITH_IMM:
2658 case IB_WR_SEND:
2659 case IB_WR_SEND_WITH_INV:
2660 return IB_WC_SEND;
2661 case IB_WR_RDMA_READ:
2662 return IB_WC_RDMA_READ;
2663 case IB_WR_ATOMIC_CMP_AND_SWP:
2664 return IB_WC_COMP_SWAP;
2665 case IB_WR_ATOMIC_FETCH_AND_ADD:
2666 return IB_WC_FETCH_ADD;
2667 case IB_WR_REG_MR:
2668 return IB_WC_REG_MR;
2669 case IB_WR_LOCAL_INV:
2670 return IB_WC_LOCAL_INV;
2671 default:
2672 return IB_WC_SEND;
2673 }
2674 }
2675
2676 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2677 {
2678 int wq_is_full, err_wr, pbl_is_full;
2679 struct qedr_dev *dev = qp->dev;
2680
2681 /* prevent SQ overflow and/or processing of a bad WR */
2682 err_wr = wr->num_sge > qp->sq.max_sges;
2683 wq_is_full = qedr_wq_is_full(&qp->sq);
2684 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2685 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2686 if (wq_is_full || err_wr || pbl_is_full) {
2687 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2688 DP_ERR(dev,
2689 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2690 qp);
2691 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2692 }
2693
2694 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2695 DP_ERR(dev,
2696 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2697 qp);
2698 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2699 }
2700
2701 if (pbl_is_full &&
2702 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2703 DP_ERR(dev,
2704 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2705 qp);
2706 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2707 }
2708 return false;
2709 }
2710 return true;
2711 }
2712
2713 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2714 struct ib_send_wr **bad_wr)
2715 {
2716 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2717 struct qedr_qp *qp = get_qedr_qp(ibqp);
2718 struct rdma_sq_atomic_wqe_1st *awqe1;
2719 struct rdma_sq_atomic_wqe_2nd *awqe2;
2720 struct rdma_sq_atomic_wqe_3rd *awqe3;
2721 struct rdma_sq_send_wqe_2st *swqe2;
2722 struct rdma_sq_local_inv_wqe *iwqe;
2723 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2724 struct rdma_sq_send_wqe_1st *swqe;
2725 struct rdma_sq_rdma_wqe_1st *rwqe;
2726 struct rdma_sq_fmr_wqe_1st *fwqe1;
2727 struct rdma_sq_common_wqe *wqe;
2728 u32 length;
2729 int rc = 0;
2730 bool comp;
2731
2732 if (!qedr_can_post_send(qp, wr)) {
2733 *bad_wr = wr;
2734 return -ENOMEM;
2735 }
2736
2737 wqe = qed_chain_produce(&qp->sq.pbl);
2738 qp->wqe_wr_id[qp->sq.prod].signaled =
2739 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2740
2741 wqe->flags = 0;
2742 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2743 !!(wr->send_flags & IB_SEND_SOLICITED));
2744 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2745 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2746 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2747 !!(wr->send_flags & IB_SEND_FENCE));
2748 wqe->prev_wqe_size = qp->prev_wqe_size;
2749
2750 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2751
2752 switch (wr->opcode) {
2753 case IB_WR_SEND_WITH_IMM:
2754 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2755 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2756 swqe->wqe_size = 2;
2757 swqe2 = qed_chain_produce(&qp->sq.pbl);
2758
2759 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2760 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2761 wr, bad_wr);
2762 swqe->length = cpu_to_le32(length);
2763 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2764 qp->prev_wqe_size = swqe->wqe_size;
2765 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2766 break;
2767 case IB_WR_SEND:
2768 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2769 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2770
2771 swqe->wqe_size = 2;
2772 swqe2 = qed_chain_produce(&qp->sq.pbl);
2773 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2774 wr, bad_wr);
2775 swqe->length = cpu_to_le32(length);
2776 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2777 qp->prev_wqe_size = swqe->wqe_size;
2778 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2779 break;
2780 case IB_WR_SEND_WITH_INV:
2781 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2782 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2783 swqe2 = qed_chain_produce(&qp->sq.pbl);
2784 swqe->wqe_size = 2;
2785 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2786 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2787 wr, bad_wr);
2788 swqe->length = cpu_to_le32(length);
2789 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2790 qp->prev_wqe_size = swqe->wqe_size;
2791 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2792 break;
2793
2794 case IB_WR_RDMA_WRITE_WITH_IMM:
2795 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2796 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2797
2798 rwqe->wqe_size = 2;
2799 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2800 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2801 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2802 wr, bad_wr);
2803 rwqe->length = cpu_to_le32(length);
2804 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2805 qp->prev_wqe_size = rwqe->wqe_size;
2806 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2807 break;
2808 case IB_WR_RDMA_WRITE:
2809 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2810 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2811
2812 rwqe->wqe_size = 2;
2813 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2814 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2815 wr, bad_wr);
2816 rwqe->length = cpu_to_le32(length);
2817 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2818 qp->prev_wqe_size = rwqe->wqe_size;
2819 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2820 break;
2821 case IB_WR_RDMA_READ_WITH_INV:
2822 DP_ERR(dev,
2823 "RDMA READ WITH INVALIDATE not supported\n");
2824 *bad_wr = wr;
2825 rc = -EINVAL;
2826 break;
2827
2828 case IB_WR_RDMA_READ:
2829 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2830 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2831
2832 rwqe->wqe_size = 2;
2833 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2834 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2835 wr, bad_wr);
2836 rwqe->length = cpu_to_le32(length);
2837 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2838 qp->prev_wqe_size = rwqe->wqe_size;
2839 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2840 break;
2841
2842 case IB_WR_ATOMIC_CMP_AND_SWP:
2843 case IB_WR_ATOMIC_FETCH_AND_ADD:
2844 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2845 awqe1->wqe_size = 4;
2846
2847 awqe2 = qed_chain_produce(&qp->sq.pbl);
2848 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2849 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2850
2851 awqe3 = qed_chain_produce(&qp->sq.pbl);
2852
2853 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2854 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2855 DMA_REGPAIR_LE(awqe3->swap_data,
2856 atomic_wr(wr)->compare_add);
2857 } else {
2858 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2859 DMA_REGPAIR_LE(awqe3->swap_data,
2860 atomic_wr(wr)->swap);
2861 DMA_REGPAIR_LE(awqe3->cmp_data,
2862 atomic_wr(wr)->compare_add);
2863 }
2864
2865 qedr_prepare_sq_sges(qp, NULL, wr);
2866
2867 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2868 qp->prev_wqe_size = awqe1->wqe_size;
2869 break;
2870
2871 case IB_WR_LOCAL_INV:
2872 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2873 iwqe->wqe_size = 1;
2874
2875 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2876 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2877 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2878 qp->prev_wqe_size = iwqe->wqe_size;
2879 break;
2880 case IB_WR_REG_MR:
2881 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2882 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2883 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2884 fwqe1->wqe_size = 2;
2885
2886 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2887 if (rc) {
2888 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2889 *bad_wr = wr;
2890 break;
2891 }
2892
2893 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2894 qp->prev_wqe_size = fwqe1->wqe_size;
2895 break;
2896 default:
2897 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2898 rc = -EINVAL;
2899 *bad_wr = wr;
2900 break;
2901 }
2902
2903 if (*bad_wr) {
2904 u16 value;
2905
2906 /* Restore prod to its position before
2907 * this WR was processed
2908 */
2909 value = le16_to_cpu(qp->sq.db_data.data.value);
2910 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2911
2912 /* Restore prev_wqe_size */
2913 qp->prev_wqe_size = wqe->prev_wqe_size;
2914 rc = -EINVAL;
2915 DP_ERR(dev, "POST SEND FAILED\n");
2916 }
2917
2918 return rc;
2919 }
2920
2921 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2922 struct ib_send_wr **bad_wr)
2923 {
2924 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2925 struct qedr_qp *qp = get_qedr_qp(ibqp);
2926 unsigned long flags;
2927 int rc = 0;
2928
2929 *bad_wr = NULL;
2930
2931 if (qp->qp_type == IB_QPT_GSI)
2932 return qedr_gsi_post_send(ibqp, wr, bad_wr);
2933
2934 spin_lock_irqsave(&qp->q_lock, flags);
2935
2936 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
2937 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2938 (qp->state != QED_ROCE_QP_STATE_SQD)) {
2939 spin_unlock_irqrestore(&qp->q_lock, flags);
2940 *bad_wr = wr;
2941 DP_DEBUG(dev, QEDR_MSG_CQ,
2942 "QP in wrong state! QP icid=0x%x state %d\n",
2943 qp->icid, qp->state);
2944 return -EINVAL;
2945 }
2946
2947 while (wr) {
2948 rc = __qedr_post_send(ibqp, wr, bad_wr);
2949 if (rc)
2950 break;
2951
2952 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
2953
2954 qedr_inc_sw_prod(&qp->sq);
2955
2956 qp->sq.db_data.data.value++;
2957
2958 wr = wr->next;
2959 }
2960
2961 /* Trigger doorbell
2962 * If there was a failure in the first WR then it will be triggered in
2963 * vane. However this is not harmful (as long as the producer value is
2964 * unchanged). For performance reasons we avoid checking for this
2965 * redundant doorbell.
2966 */
2967 wmb();
2968 writel(qp->sq.db_data.raw, qp->sq.db);
2969
2970 /* Make sure write sticks */
2971 mmiowb();
2972
2973 spin_unlock_irqrestore(&qp->q_lock, flags);
2974
2975 return rc;
2976 }
2977
2978 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
2979 struct ib_recv_wr **bad_wr)
2980 {
2981 struct qedr_qp *qp = get_qedr_qp(ibqp);
2982 struct qedr_dev *dev = qp->dev;
2983 unsigned long flags;
2984 int status = 0;
2985
2986 if (qp->qp_type == IB_QPT_GSI)
2987 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
2988
2989 spin_lock_irqsave(&qp->q_lock, flags);
2990
2991 if (qp->state == QED_ROCE_QP_STATE_RESET) {
2992 spin_unlock_irqrestore(&qp->q_lock, flags);
2993 *bad_wr = wr;
2994 return -EINVAL;
2995 }
2996
2997 while (wr) {
2998 int i;
2999
3000 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3001 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3002 wr->num_sge > qp->rq.max_sges) {
3003 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3004 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3005 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3006 qp->rq.max_sges);
3007 status = -ENOMEM;
3008 *bad_wr = wr;
3009 break;
3010 }
3011 for (i = 0; i < wr->num_sge; i++) {
3012 u32 flags = 0;
3013 struct rdma_rq_sge *rqe =
3014 qed_chain_produce(&qp->rq.pbl);
3015
3016 /* First one must include the number
3017 * of SGE in the list
3018 */
3019 if (!i)
3020 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3021 wr->num_sge);
3022
3023 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3024 wr->sg_list[i].lkey);
3025
3026 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3027 wr->sg_list[i].length, flags);
3028 }
3029
3030 /* Special case of no sges. FW requires between 1-4 sges...
3031 * in this case we need to post 1 sge with length zero. this is
3032 * because rdma write with immediate consumes an RQ.
3033 */
3034 if (!wr->num_sge) {
3035 u32 flags = 0;
3036 struct rdma_rq_sge *rqe =
3037 qed_chain_produce(&qp->rq.pbl);
3038
3039 /* First one must include the number
3040 * of SGE in the list
3041 */
3042 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3043 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3044
3045 RQ_SGE_SET(rqe, 0, 0, flags);
3046 i = 1;
3047 }
3048
3049 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3050 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3051
3052 qedr_inc_sw_prod(&qp->rq);
3053
3054 /* Flush all the writes before signalling doorbell */
3055 wmb();
3056
3057 qp->rq.db_data.data.value++;
3058
3059 writel(qp->rq.db_data.raw, qp->rq.db);
3060
3061 /* Make sure write sticks */
3062 mmiowb();
3063
3064 wr = wr->next;
3065 }
3066
3067 spin_unlock_irqrestore(&qp->q_lock, flags);
3068
3069 return status;
3070 }
3071
3072 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3073 {
3074 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3075
3076 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3077 cq->pbl_toggle;
3078 }
3079
3080 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3081 {
3082 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3083 struct qedr_qp *qp;
3084
3085 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3086 resp_cqe->qp_handle.lo,
3087 u64);
3088 return qp;
3089 }
3090
3091 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3092 {
3093 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3094
3095 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3096 }
3097
3098 /* Return latest CQE (needs processing) */
3099 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3100 {
3101 return cq->latest_cqe;
3102 }
3103
3104 /* In fmr we need to increase the number of fmr completed counter for the fmr
3105 * algorithm determining whether we can free a pbl or not.
3106 * we need to perform this whether the work request was signaled or not. for
3107 * this purpose we call this function from the condition that checks if a wr
3108 * should be skipped, to make sure we don't miss it ( possibly this fmr
3109 * operation was not signalted)
3110 */
3111 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3112 {
3113 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3114 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3115 }
3116
3117 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3118 struct qedr_cq *cq, int num_entries,
3119 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3120 int force)
3121 {
3122 u16 cnt = 0;
3123
3124 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3125 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3126 qedr_chk_if_fmr(qp);
3127 /* skip WC */
3128 goto next_cqe;
3129 }
3130
3131 /* fill WC */
3132 wc->status = status;
3133 wc->vendor_err = 0;
3134 wc->wc_flags = 0;
3135 wc->src_qp = qp->id;
3136 wc->qp = &qp->ibqp;
3137
3138 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3139 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3140
3141 switch (wc->opcode) {
3142 case IB_WC_RDMA_WRITE:
3143 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3144 break;
3145 case IB_WC_COMP_SWAP:
3146 case IB_WC_FETCH_ADD:
3147 wc->byte_len = 8;
3148 break;
3149 case IB_WC_REG_MR:
3150 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3151 break;
3152 default:
3153 break;
3154 }
3155
3156 num_entries--;
3157 wc++;
3158 cnt++;
3159 next_cqe:
3160 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3161 qed_chain_consume(&qp->sq.pbl);
3162 qedr_inc_sw_cons(&qp->sq);
3163 }
3164
3165 return cnt;
3166 }
3167
3168 static int qedr_poll_cq_req(struct qedr_dev *dev,
3169 struct qedr_qp *qp, struct qedr_cq *cq,
3170 int num_entries, struct ib_wc *wc,
3171 struct rdma_cqe_requester *req)
3172 {
3173 int cnt = 0;
3174
3175 switch (req->status) {
3176 case RDMA_CQE_REQ_STS_OK:
3177 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3178 IB_WC_SUCCESS, 0);
3179 break;
3180 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3181 if (qp->state != QED_ROCE_QP_STATE_ERR)
3182 DP_ERR(dev,
3183 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3184 cq->icid, qp->icid);
3185 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3186 IB_WC_WR_FLUSH_ERR, 1);
3187 break;
3188 default:
3189 /* process all WQE before the cosumer */
3190 qp->state = QED_ROCE_QP_STATE_ERR;
3191 cnt = process_req(dev, qp, cq, num_entries, wc,
3192 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3193 wc += cnt;
3194 /* if we have extra WC fill it with actual error info */
3195 if (cnt < num_entries) {
3196 enum ib_wc_status wc_status;
3197
3198 switch (req->status) {
3199 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3200 DP_ERR(dev,
3201 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3202 cq->icid, qp->icid);
3203 wc_status = IB_WC_BAD_RESP_ERR;
3204 break;
3205 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3206 DP_ERR(dev,
3207 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3208 cq->icid, qp->icid);
3209 wc_status = IB_WC_LOC_LEN_ERR;
3210 break;
3211 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3212 DP_ERR(dev,
3213 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3214 cq->icid, qp->icid);
3215 wc_status = IB_WC_LOC_QP_OP_ERR;
3216 break;
3217 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3218 DP_ERR(dev,
3219 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3220 cq->icid, qp->icid);
3221 wc_status = IB_WC_LOC_PROT_ERR;
3222 break;
3223 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3224 DP_ERR(dev,
3225 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3226 cq->icid, qp->icid);
3227 wc_status = IB_WC_MW_BIND_ERR;
3228 break;
3229 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3230 DP_ERR(dev,
3231 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3232 cq->icid, qp->icid);
3233 wc_status = IB_WC_REM_INV_REQ_ERR;
3234 break;
3235 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3236 DP_ERR(dev,
3237 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3238 cq->icid, qp->icid);
3239 wc_status = IB_WC_REM_ACCESS_ERR;
3240 break;
3241 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3242 DP_ERR(dev,
3243 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3244 cq->icid, qp->icid);
3245 wc_status = IB_WC_REM_OP_ERR;
3246 break;
3247 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3248 DP_ERR(dev,
3249 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3250 cq->icid, qp->icid);
3251 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3252 break;
3253 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3254 DP_ERR(dev,
3255 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3256 cq->icid, qp->icid);
3257 wc_status = IB_WC_RETRY_EXC_ERR;
3258 break;
3259 default:
3260 DP_ERR(dev,
3261 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3262 cq->icid, qp->icid);
3263 wc_status = IB_WC_GENERAL_ERR;
3264 }
3265 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3266 wc_status, 1);
3267 }
3268 }
3269
3270 return cnt;
3271 }
3272
3273 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3274 struct qedr_cq *cq, struct ib_wc *wc,
3275 struct rdma_cqe_responder *resp, u64 wr_id)
3276 {
3277 enum ib_wc_status wc_status = IB_WC_SUCCESS;
3278 u8 flags;
3279
3280 wc->opcode = IB_WC_RECV;
3281 wc->wc_flags = 0;
3282
3283 switch (resp->status) {
3284 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3285 wc_status = IB_WC_LOC_ACCESS_ERR;
3286 break;
3287 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3288 wc_status = IB_WC_LOC_LEN_ERR;
3289 break;
3290 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3291 wc_status = IB_WC_LOC_QP_OP_ERR;
3292 break;
3293 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3294 wc_status = IB_WC_LOC_PROT_ERR;
3295 break;
3296 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3297 wc_status = IB_WC_MW_BIND_ERR;
3298 break;
3299 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3300 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
3301 break;
3302 case RDMA_CQE_RESP_STS_OK:
3303 wc_status = IB_WC_SUCCESS;
3304 wc->byte_len = le32_to_cpu(resp->length);
3305
3306 flags = resp->flags & QEDR_RESP_RDMA_IMM;
3307
3308 if (flags == QEDR_RESP_RDMA_IMM)
3309 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3310
3311 if (flags == QEDR_RESP_RDMA_IMM || flags == QEDR_RESP_IMM) {
3312 wc->ex.imm_data =
3313 le32_to_cpu(resp->imm_data_or_inv_r_Key);
3314 wc->wc_flags |= IB_WC_WITH_IMM;
3315 }
3316 break;
3317 default:
3318 wc->status = IB_WC_GENERAL_ERR;
3319 DP_ERR(dev, "Invalid CQE status detected\n");
3320 }
3321
3322 /* fill WC */
3323 wc->status = wc_status;
3324 wc->vendor_err = 0;
3325 wc->src_qp = qp->id;
3326 wc->qp = &qp->ibqp;
3327 wc->wr_id = wr_id;
3328 }
3329
3330 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3331 struct qedr_cq *cq, struct ib_wc *wc,
3332 struct rdma_cqe_responder *resp)
3333 {
3334 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3335
3336 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3337
3338 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3339 qed_chain_consume(&qp->rq.pbl);
3340 qedr_inc_sw_cons(&qp->rq);
3341
3342 return 1;
3343 }
3344
3345 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3346 int num_entries, struct ib_wc *wc, u16 hw_cons)
3347 {
3348 u16 cnt = 0;
3349
3350 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3351 /* fill WC */
3352 wc->status = IB_WC_WR_FLUSH_ERR;
3353 wc->vendor_err = 0;
3354 wc->wc_flags = 0;
3355 wc->src_qp = qp->id;
3356 wc->byte_len = 0;
3357 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3358 wc->qp = &qp->ibqp;
3359 num_entries--;
3360 wc++;
3361 cnt++;
3362 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3363 qed_chain_consume(&qp->rq.pbl);
3364 qedr_inc_sw_cons(&qp->rq);
3365 }
3366
3367 return cnt;
3368 }
3369
3370 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3371 struct rdma_cqe_responder *resp, int *update)
3372 {
3373 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3374 consume_cqe(cq);
3375 *update |= 1;
3376 }
3377 }
3378
3379 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3380 struct qedr_cq *cq, int num_entries,
3381 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3382 int *update)
3383 {
3384 int cnt;
3385
3386 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3387 cnt = process_resp_flush(qp, cq, num_entries, wc,
3388 resp->rq_cons);
3389 try_consume_resp_cqe(cq, qp, resp, update);
3390 } else {
3391 cnt = process_resp_one(dev, qp, cq, wc, resp);
3392 consume_cqe(cq);
3393 *update |= 1;
3394 }
3395
3396 return cnt;
3397 }
3398
3399 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3400 struct rdma_cqe_requester *req, int *update)
3401 {
3402 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3403 consume_cqe(cq);
3404 *update |= 1;
3405 }
3406 }
3407
3408 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3409 {
3410 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3411 struct qedr_cq *cq = get_qedr_cq(ibcq);
3412 union rdma_cqe *cqe = cq->latest_cqe;
3413 u32 old_cons, new_cons;
3414 unsigned long flags;
3415 int update = 0;
3416 int done = 0;
3417
3418 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3419 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3420
3421 spin_lock_irqsave(&cq->cq_lock, flags);
3422 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3423 while (num_entries && is_valid_cqe(cq, cqe)) {
3424 struct qedr_qp *qp;
3425 int cnt = 0;
3426
3427 /* prevent speculative reads of any field of CQE */
3428 rmb();
3429
3430 qp = cqe_get_qp(cqe);
3431 if (!qp) {
3432 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3433 break;
3434 }
3435
3436 wc->qp = &qp->ibqp;
3437
3438 switch (cqe_get_type(cqe)) {
3439 case RDMA_CQE_TYPE_REQUESTER:
3440 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3441 &cqe->req);
3442 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3443 break;
3444 case RDMA_CQE_TYPE_RESPONDER_RQ:
3445 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3446 &cqe->resp, &update);
3447 break;
3448 case RDMA_CQE_TYPE_INVALID:
3449 default:
3450 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3451 cqe_get_type(cqe));
3452 }
3453 num_entries -= cnt;
3454 wc += cnt;
3455 done += cnt;
3456
3457 cqe = get_cqe(cq);
3458 }
3459 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3460
3461 cq->cq_cons += new_cons - old_cons;
3462
3463 if (update)
3464 /* doorbell notifies abount latest VALID entry,
3465 * but chain already point to the next INVALID one
3466 */
3467 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3468
3469 spin_unlock_irqrestore(&cq->cq_lock, flags);
3470 return done;
3471 }
3472
3473 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3474 u8 port_num,
3475 const struct ib_wc *in_wc,
3476 const struct ib_grh *in_grh,
3477 const struct ib_mad_hdr *mad_hdr,
3478 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3479 size_t *out_mad_size, u16 *out_mad_pkey_index)
3480 {
3481 struct qedr_dev *dev = get_qedr_dev(ibdev);
3482
3483 DP_DEBUG(dev, QEDR_MSG_GSI,
3484 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3485 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3486 mad_hdr->class_specific, mad_hdr->class_version,
3487 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3488 return IB_MAD_RESULT_SUCCESS;
3489 }
3490
3491 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3492 struct ib_port_immutable *immutable)
3493 {
3494 struct ib_port_attr attr;
3495 int err;
3496
3497 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3498 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3499
3500 err = ib_query_port(ibdev, port_num, &attr);
3501 if (err)
3502 return err;
3503
3504 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3505 immutable->gid_tbl_len = attr.gid_tbl_len;
3506 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3507
3508 return 0;
3509 }
ubuntu-bionic-kernel mirror