1 /* QLogic qed NIC Driver
2 * Copyright (c) 2015-2017 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/types.h>
33 #include <asm/byteorder.h>
34 #include <linux/bitops.h>
35 #include <linux/delay.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/errno.h>
38 #include <linux/etherdevice.h>
39 #include <linux/if_ether.h>
40 #include <linux/if_vlan.h>
43 #include <linux/ipv6.h>
44 #include <linux/kernel.h>
45 #include <linux/list.h>
46 #include <linux/module.h>
47 #include <linux/mutex.h>
48 #include <linux/pci.h>
49 #include <linux/slab.h>
50 #include <linux/spinlock.h>
51 #include <linux/string.h>
52 #include <linux/tcp.h>
53 #include <linux/bitops.h>
54 #include <linux/qed/qed_roce_if.h>
55 #include <linux/qed/qed_roce_if.h>
60 #include "qed_init_ops.h"
64 #include "qed_reg_addr.h"
69 static void qed_roce_free_real_icid(struct qed_hwfn
*p_hwfn
, u16 icid
);
71 void qed_roce_async_event(struct qed_hwfn
*p_hwfn
,
72 u8 fw_event_code
, union rdma_eqe_data
*rdma_data
)
74 if (fw_event_code
== ROCE_ASYNC_EVENT_DESTROY_QP_DONE
) {
76 (u16
)le32_to_cpu(rdma_data
->rdma_destroy_qp_data
.cid
);
78 /* icid release in this async event can occur only if the icid
79 * was offloaded to the FW. In case it wasn't offloaded this is
80 * handled in qed_roce_sp_destroy_qp.
82 qed_roce_free_real_icid(p_hwfn
, icid
);
84 struct qed_rdma_events
*events
= &p_hwfn
->p_rdma_info
->events
;
86 events
->affiliated_event(p_hwfn
->p_rdma_info
->events
.context
,
88 &rdma_data
->async_handle
);
92 static int qed_rdma_bmap_alloc(struct qed_hwfn
*p_hwfn
,
93 struct qed_bmap
*bmap
, u32 max_count
)
95 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "max_count = %08x\n", max_count
);
97 bmap
->max_count
= max_count
;
99 bmap
->bitmap
= kzalloc(BITS_TO_LONGS(max_count
) * sizeof(long),
103 "qed bmap alloc failed: cannot allocate memory (bitmap)\n");
107 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Allocated bitmap %p\n",
112 static int qed_rdma_bmap_alloc_id(struct qed_hwfn
*p_hwfn
,
113 struct qed_bmap
*bmap
, u32
*id_num
)
115 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "bmap = %p\n", bmap
);
117 *id_num
= find_first_zero_bit(bmap
->bitmap
, bmap
->max_count
);
119 if (*id_num
>= bmap
->max_count
) {
120 DP_NOTICE(p_hwfn
, "no id available max_count=%d\n",
125 __set_bit(*id_num
, bmap
->bitmap
);
130 static void qed_bmap_set_id(struct qed_hwfn
*p_hwfn
,
131 struct qed_bmap
*bmap
, u32 id_num
)
133 if (id_num
>= bmap
->max_count
)
136 __set_bit(id_num
, bmap
->bitmap
);
139 static void qed_bmap_release_id(struct qed_hwfn
*p_hwfn
,
140 struct qed_bmap
*bmap
, u32 id_num
)
144 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "id_num = %08x", id_num
);
145 if (id_num
>= bmap
->max_count
)
148 b_acquired
= test_and_clear_bit(id_num
, bmap
->bitmap
);
150 DP_NOTICE(p_hwfn
, "ID %d already released\n", id_num
);
155 static int qed_bmap_test_id(struct qed_hwfn
*p_hwfn
,
156 struct qed_bmap
*bmap
, u32 id_num
)
158 if (id_num
>= bmap
->max_count
)
161 return test_bit(id_num
, bmap
->bitmap
);
164 static u32
qed_rdma_get_sb_id(void *p_hwfn
, u32 rel_sb_id
)
166 /* First sb id for RoCE is after all the l2 sb */
167 return FEAT_NUM((struct qed_hwfn
*)p_hwfn
, QED_PF_L2_QUE
) + rel_sb_id
;
170 static int qed_rdma_alloc(struct qed_hwfn
*p_hwfn
,
171 struct qed_ptt
*p_ptt
,
172 struct qed_rdma_start_in_params
*params
)
174 struct qed_rdma_info
*p_rdma_info
;
175 u32 num_cons
, num_tasks
;
178 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Allocating RDMA\n");
180 /* Allocate a struct with current pf rdma info */
181 p_rdma_info
= kzalloc(sizeof(*p_rdma_info
), GFP_KERNEL
);
184 "qed rdma alloc failed: cannot allocate memory (rdma info). rc = %d\n",
189 p_hwfn
->p_rdma_info
= p_rdma_info
;
190 p_rdma_info
->proto
= PROTOCOLID_ROCE
;
192 num_cons
= qed_cxt_get_proto_cid_count(p_hwfn
, p_rdma_info
->proto
,
195 p_rdma_info
->num_qps
= num_cons
/ 2;
197 num_tasks
= qed_cxt_get_proto_tid_count(p_hwfn
, PROTOCOLID_ROCE
);
199 /* Each MR uses a single task */
200 p_rdma_info
->num_mrs
= num_tasks
;
202 /* Queue zone lines are shared between RoCE and L2 in such a way that
203 * they can be used by each without obstructing the other.
205 p_rdma_info
->queue_zone_base
= (u16
)RESC_START(p_hwfn
, QED_L2_QUEUE
);
206 p_rdma_info
->max_queue_zones
= (u16
)RESC_NUM(p_hwfn
, QED_L2_QUEUE
);
208 /* Allocate a struct with device params and fill it */
209 p_rdma_info
->dev
= kzalloc(sizeof(*p_rdma_info
->dev
), GFP_KERNEL
);
210 if (!p_rdma_info
->dev
) {
212 "qed rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n",
217 /* Allocate a struct with port params and fill it */
218 p_rdma_info
->port
= kzalloc(sizeof(*p_rdma_info
->port
), GFP_KERNEL
);
219 if (!p_rdma_info
->port
) {
221 "qed rdma alloc failed: cannot allocate memory (rdma info port). rc = %d\n",
226 /* Allocate bit map for pd's */
227 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->pd_map
, RDMA_MAX_PDS
);
229 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
230 "Failed to allocate pd_map, rc = %d\n",
235 /* Allocate DPI bitmap */
236 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->dpi_map
,
239 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
240 "Failed to allocate DPI bitmap, rc = %d\n", rc
);
244 /* Allocate bitmap for cq's. The maximum number of CQs is bounded to
245 * twice the number of QPs.
247 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->cq_map
,
248 p_rdma_info
->num_qps
* 2);
250 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
251 "Failed to allocate cq bitmap, rc = %d\n", rc
);
255 /* Allocate bitmap for toggle bit for cq icids
256 * We toggle the bit every time we create or resize cq for a given icid.
257 * The maximum number of CQs is bounded to twice the number of QPs.
259 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->toggle_bits
,
260 p_rdma_info
->num_qps
* 2);
262 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
263 "Failed to allocate toogle bits, rc = %d\n", rc
);
267 /* Allocate bitmap for itids */
268 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->tid_map
,
269 p_rdma_info
->num_mrs
);
271 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
272 "Failed to allocate itids bitmaps, rc = %d\n", rc
);
273 goto free_toggle_map
;
276 /* Allocate bitmap for cids used for qps. */
277 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->cid_map
, num_cons
);
279 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
280 "Failed to allocate cid bitmap, rc = %d\n", rc
);
284 /* Allocate bitmap for cids used for responders/requesters. */
285 rc
= qed_rdma_bmap_alloc(p_hwfn
, &p_rdma_info
->real_cid_map
, num_cons
);
287 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
288 "Failed to allocate real cid bitmap, rc = %d\n", rc
);
291 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Allocation successful\n");
295 kfree(p_rdma_info
->cid_map
.bitmap
);
297 kfree(p_rdma_info
->tid_map
.bitmap
);
299 kfree(p_rdma_info
->toggle_bits
.bitmap
);
301 kfree(p_rdma_info
->cq_map
.bitmap
);
303 kfree(p_rdma_info
->dpi_map
.bitmap
);
305 kfree(p_rdma_info
->pd_map
.bitmap
);
307 kfree(p_rdma_info
->port
);
309 kfree(p_rdma_info
->dev
);
316 static void qed_rdma_resc_free(struct qed_hwfn
*p_hwfn
)
318 struct qed_bmap
*rcid_map
= &p_hwfn
->p_rdma_info
->real_cid_map
;
319 struct qed_rdma_info
*p_rdma_info
= p_hwfn
->p_rdma_info
;
322 /* when destroying a_RoCE QP the control is returned to the user after
323 * the synchronous part. The asynchronous part may take a little longer.
324 * We delay for a short while if an async destroy QP is still expected.
325 * Beyond the added delay we clear the bitmap anyway.
327 while (bitmap_weight(rcid_map
->bitmap
, rcid_map
->max_count
)) {
329 if (wait_count
++ > 20) {
330 DP_NOTICE(p_hwfn
, "cid bitmap wait timed out\n");
335 kfree(p_rdma_info
->cid_map
.bitmap
);
336 kfree(p_rdma_info
->tid_map
.bitmap
);
337 kfree(p_rdma_info
->toggle_bits
.bitmap
);
338 kfree(p_rdma_info
->cq_map
.bitmap
);
339 kfree(p_rdma_info
->dpi_map
.bitmap
);
340 kfree(p_rdma_info
->pd_map
.bitmap
);
342 kfree(p_rdma_info
->port
);
343 kfree(p_rdma_info
->dev
);
348 static void qed_rdma_free(struct qed_hwfn
*p_hwfn
)
350 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Freeing RDMA\n");
352 qed_rdma_resc_free(p_hwfn
);
355 static void qed_rdma_get_guid(struct qed_hwfn
*p_hwfn
, u8
*guid
)
357 guid
[0] = p_hwfn
->hw_info
.hw_mac_addr
[0] ^ 2;
358 guid
[1] = p_hwfn
->hw_info
.hw_mac_addr
[1];
359 guid
[2] = p_hwfn
->hw_info
.hw_mac_addr
[2];
362 guid
[5] = p_hwfn
->hw_info
.hw_mac_addr
[3];
363 guid
[6] = p_hwfn
->hw_info
.hw_mac_addr
[4];
364 guid
[7] = p_hwfn
->hw_info
.hw_mac_addr
[5];
367 static void qed_rdma_init_events(struct qed_hwfn
*p_hwfn
,
368 struct qed_rdma_start_in_params
*params
)
370 struct qed_rdma_events
*events
;
372 events
= &p_hwfn
->p_rdma_info
->events
;
374 events
->unaffiliated_event
= params
->events
->unaffiliated_event
;
375 events
->affiliated_event
= params
->events
->affiliated_event
;
376 events
->context
= params
->events
->context
;
379 static void qed_rdma_init_devinfo(struct qed_hwfn
*p_hwfn
,
380 struct qed_rdma_start_in_params
*params
)
382 struct qed_rdma_device
*dev
= p_hwfn
->p_rdma_info
->dev
;
383 struct qed_dev
*cdev
= p_hwfn
->cdev
;
384 u32 pci_status_control
;
387 /* Vendor specific information */
388 dev
->vendor_id
= cdev
->vendor_id
;
389 dev
->vendor_part_id
= cdev
->device_id
;
391 dev
->fw_ver
= (FW_MAJOR_VERSION
<< 24) | (FW_MINOR_VERSION
<< 16) |
392 (FW_REVISION_VERSION
<< 8) | (FW_ENGINEERING_VERSION
);
394 qed_rdma_get_guid(p_hwfn
, (u8
*)&dev
->sys_image_guid
);
395 dev
->node_guid
= dev
->sys_image_guid
;
397 dev
->max_sge
= min_t(u32
, RDMA_MAX_SGE_PER_SQ_WQE
,
398 RDMA_MAX_SGE_PER_RQ_WQE
);
400 if (cdev
->rdma_max_sge
)
401 dev
->max_sge
= min_t(u32
, cdev
->rdma_max_sge
, dev
->max_sge
);
403 dev
->max_inline
= ROCE_REQ_MAX_INLINE_DATA_SIZE
;
405 dev
->max_inline
= (cdev
->rdma_max_inline
) ?
406 min_t(u32
, cdev
->rdma_max_inline
, dev
->max_inline
) :
409 dev
->max_wqe
= QED_RDMA_MAX_WQE
;
410 dev
->max_cnq
= (u8
)FEAT_NUM(p_hwfn
, QED_RDMA_CNQ
);
412 /* The number of QPs may be higher than QED_ROCE_MAX_QPS, because
413 * it is up-aligned to 16 and then to ILT page size within qed cxt.
414 * This is OK in terms of ILT but we don't want to configure the FW
415 * above its abilities
417 num_qps
= ROCE_MAX_QPS
;
418 num_qps
= min_t(u64
, num_qps
, p_hwfn
->p_rdma_info
->num_qps
);
419 dev
->max_qp
= num_qps
;
421 /* CQs uses the same icids that QPs use hence they are limited by the
422 * number of icids. There are two icids per QP.
424 dev
->max_cq
= num_qps
* 2;
426 /* The number of mrs is smaller by 1 since the first is reserved */
427 dev
->max_mr
= p_hwfn
->p_rdma_info
->num_mrs
- 1;
428 dev
->max_mr_size
= QED_RDMA_MAX_MR_SIZE
;
430 /* The maximum CQE capacity per CQ supported.
431 * max number of cqes will be in two layer pbl,
432 * 8 is the pointer size in bytes
433 * 32 is the size of cq element in bytes
435 if (params
->cq_mode
== QED_RDMA_CQ_MODE_32_BITS
)
436 dev
->max_cqe
= QED_RDMA_MAX_CQE_32_BIT
;
438 dev
->max_cqe
= QED_RDMA_MAX_CQE_16_BIT
;
441 dev
->max_fmr
= QED_RDMA_MAX_FMR
;
442 dev
->max_mr_mw_fmr_pbl
= (PAGE_SIZE
/ 8) * (PAGE_SIZE
/ 8);
443 dev
->max_mr_mw_fmr_size
= dev
->max_mr_mw_fmr_pbl
* PAGE_SIZE
;
444 dev
->max_pkey
= QED_RDMA_MAX_P_KEY
;
446 dev
->max_qp_resp_rd_atomic_resc
= RDMA_RING_PAGE_SIZE
/
447 (RDMA_RESP_RD_ATOMIC_ELM_SIZE
* 2);
448 dev
->max_qp_req_rd_atomic_resc
= RDMA_RING_PAGE_SIZE
/
449 RDMA_REQ_RD_ATOMIC_ELM_SIZE
;
450 dev
->max_dev_resp_rd_atomic_resc
= dev
->max_qp_resp_rd_atomic_resc
*
451 p_hwfn
->p_rdma_info
->num_qps
;
452 dev
->page_size_caps
= QED_RDMA_PAGE_SIZE_CAPS
;
453 dev
->dev_ack_delay
= QED_RDMA_ACK_DELAY
;
454 dev
->max_pd
= RDMA_MAX_PDS
;
455 dev
->max_ah
= p_hwfn
->p_rdma_info
->num_qps
;
456 dev
->max_stats_queues
= (u8
)RESC_NUM(p_hwfn
, QED_RDMA_STATS_QUEUE
);
458 /* Set capablities */
460 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_RNR_NAK
, 1);
461 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_PORT_ACTIVE_EVENT
, 1);
462 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_PORT_CHANGE_EVENT
, 1);
463 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_RESIZE_CQ
, 1);
464 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_BASE_MEMORY_EXT
, 1);
465 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_BASE_QUEUE_EXT
, 1);
466 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_ZBVA
, 1);
467 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_LOCAL_INV_FENCE
, 1);
469 /* Check atomic operations support in PCI configuration space. */
470 pci_read_config_dword(cdev
->pdev
,
471 cdev
->pdev
->pcie_cap
+ PCI_EXP_DEVCTL2
,
472 &pci_status_control
);
474 if (pci_status_control
& PCI_EXP_DEVCTL2_LTR_EN
)
475 SET_FIELD(dev
->dev_caps
, QED_RDMA_DEV_CAP_ATOMIC_OP
, 1);
478 static void qed_rdma_init_port(struct qed_hwfn
*p_hwfn
)
480 struct qed_rdma_port
*port
= p_hwfn
->p_rdma_info
->port
;
481 struct qed_rdma_device
*dev
= p_hwfn
->p_rdma_info
->dev
;
483 port
->port_state
= p_hwfn
->mcp_info
->link_output
.link_up
?
484 QED_RDMA_PORT_UP
: QED_RDMA_PORT_DOWN
;
486 port
->max_msg_size
= min_t(u64
,
487 (dev
->max_mr_mw_fmr_size
*
488 p_hwfn
->cdev
->rdma_max_sge
),
491 port
->pkey_bad_counter
= 0;
494 static int qed_rdma_init_hw(struct qed_hwfn
*p_hwfn
, struct qed_ptt
*p_ptt
)
496 u32 ll2_ethertype_en
;
498 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Initializing HW\n");
499 p_hwfn
->b_rdma_enabled_in_prs
= false;
501 qed_wr(p_hwfn
, p_ptt
, PRS_REG_ROCE_DEST_QP_MAX_PF
, 0);
503 p_hwfn
->rdma_prs_search_reg
= PRS_REG_SEARCH_ROCE
;
505 /* We delay writing to this reg until first cid is allocated. See
506 * qed_cxt_dynamic_ilt_alloc function for more details
508 ll2_ethertype_en
= qed_rd(p_hwfn
, p_ptt
, PRS_REG_LIGHT_L2_ETHERTYPE_EN
);
509 qed_wr(p_hwfn
, p_ptt
, PRS_REG_LIGHT_L2_ETHERTYPE_EN
,
510 (ll2_ethertype_en
| 0x01));
512 if (qed_cxt_get_proto_cid_start(p_hwfn
, PROTOCOLID_ROCE
) % 2) {
513 DP_NOTICE(p_hwfn
, "The first RoCE's cid should be even\n");
517 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Initializing HW - Done\n");
521 static int qed_rdma_start_fw(struct qed_hwfn
*p_hwfn
,
522 struct qed_rdma_start_in_params
*params
,
523 struct qed_ptt
*p_ptt
)
525 struct rdma_init_func_ramrod_data
*p_ramrod
;
526 struct qed_rdma_cnq_params
*p_cnq_pbl_list
;
527 struct rdma_init_func_hdr
*p_params_header
;
528 struct rdma_cnq_params
*p_cnq_params
;
529 struct qed_sp_init_data init_data
;
530 struct qed_spq_entry
*p_ent
;
534 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Starting FW\n");
536 /* Save the number of cnqs for the function close ramrod */
537 p_hwfn
->p_rdma_info
->num_cnqs
= params
->desired_cnq
;
540 memset(&init_data
, 0, sizeof(init_data
));
541 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
542 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
544 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, RDMA_RAMROD_FUNC_INIT
,
545 p_hwfn
->p_rdma_info
->proto
, &init_data
);
549 p_ramrod
= &p_ent
->ramrod
.roce_init_func
.rdma
;
551 p_params_header
= &p_ramrod
->params_header
;
552 p_params_header
->cnq_start_offset
= (u8
)RESC_START(p_hwfn
,
554 p_params_header
->num_cnqs
= params
->desired_cnq
;
556 if (params
->cq_mode
== QED_RDMA_CQ_MODE_16_BITS
)
557 p_params_header
->cq_ring_mode
= 1;
559 p_params_header
->cq_ring_mode
= 0;
561 for (cnq_id
= 0; cnq_id
< params
->desired_cnq
; cnq_id
++) {
562 sb_id
= qed_rdma_get_sb_id(p_hwfn
, cnq_id
);
563 p_cnq_params
= &p_ramrod
->cnq_params
[cnq_id
];
564 p_cnq_pbl_list
= ¶ms
->cnq_pbl_list
[cnq_id
];
565 p_cnq_params
->sb_num
=
566 cpu_to_le16(p_hwfn
->sbs_info
[sb_id
]->igu_sb_id
);
568 p_cnq_params
->sb_index
= p_hwfn
->pf_params
.rdma_pf_params
.gl_pi
;
569 p_cnq_params
->num_pbl_pages
= p_cnq_pbl_list
->num_pbl_pages
;
571 DMA_REGPAIR_LE(p_cnq_params
->pbl_base_addr
,
572 p_cnq_pbl_list
->pbl_ptr
);
574 /* we assume here that cnq_id and qz_offset are the same */
575 p_cnq_params
->queue_zone_num
=
576 cpu_to_le16(p_hwfn
->p_rdma_info
->queue_zone_base
+
580 return qed_spq_post(p_hwfn
, p_ent
, NULL
);
583 static int qed_rdma_alloc_tid(void *rdma_cxt
, u32
*itid
)
585 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
588 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Allocate TID\n");
590 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
591 rc
= qed_rdma_bmap_alloc_id(p_hwfn
,
592 &p_hwfn
->p_rdma_info
->tid_map
, itid
);
593 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
597 rc
= qed_cxt_dynamic_ilt_alloc(p_hwfn
, QED_ELEM_TASK
, *itid
);
599 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Allocate TID - done, rc = %d\n", rc
);
603 static int qed_rdma_reserve_lkey(struct qed_hwfn
*p_hwfn
)
605 struct qed_rdma_device
*dev
= p_hwfn
->p_rdma_info
->dev
;
607 /* The first DPI is reserved for the Kernel */
608 __set_bit(0, p_hwfn
->p_rdma_info
->dpi_map
.bitmap
);
610 /* Tid 0 will be used as the key for "reserved MR".
611 * The driver should allocate memory for it so it can be loaded but no
612 * ramrod should be passed on it.
614 qed_rdma_alloc_tid(p_hwfn
, &dev
->reserved_lkey
);
615 if (dev
->reserved_lkey
!= RDMA_RESERVED_LKEY
) {
617 "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n");
624 static int qed_rdma_setup(struct qed_hwfn
*p_hwfn
,
625 struct qed_ptt
*p_ptt
,
626 struct qed_rdma_start_in_params
*params
)
630 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "RDMA setup\n");
632 spin_lock_init(&p_hwfn
->p_rdma_info
->lock
);
634 qed_rdma_init_devinfo(p_hwfn
, params
);
635 qed_rdma_init_port(p_hwfn
);
636 qed_rdma_init_events(p_hwfn
, params
);
638 rc
= qed_rdma_reserve_lkey(p_hwfn
);
642 rc
= qed_rdma_init_hw(p_hwfn
, p_ptt
);
646 return qed_rdma_start_fw(p_hwfn
, params
, p_ptt
);
649 static int qed_rdma_stop(void *rdma_cxt
)
651 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
652 struct rdma_close_func_ramrod_data
*p_ramrod
;
653 struct qed_sp_init_data init_data
;
654 struct qed_spq_entry
*p_ent
;
655 struct qed_ptt
*p_ptt
;
656 u32 ll2_ethertype_en
;
659 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "RDMA stop\n");
661 p_ptt
= qed_ptt_acquire(p_hwfn
);
663 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Failed to acquire PTT\n");
667 /* Disable RoCE search */
668 qed_wr(p_hwfn
, p_ptt
, p_hwfn
->rdma_prs_search_reg
, 0);
669 p_hwfn
->b_rdma_enabled_in_prs
= false;
671 qed_wr(p_hwfn
, p_ptt
, PRS_REG_ROCE_DEST_QP_MAX_PF
, 0);
673 ll2_ethertype_en
= qed_rd(p_hwfn
, p_ptt
, PRS_REG_LIGHT_L2_ETHERTYPE_EN
);
675 qed_wr(p_hwfn
, p_ptt
, PRS_REG_LIGHT_L2_ETHERTYPE_EN
,
676 (ll2_ethertype_en
& 0xFFFE));
678 qed_ptt_release(p_hwfn
, p_ptt
);
681 memset(&init_data
, 0, sizeof(init_data
));
682 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
683 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
686 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, RDMA_RAMROD_FUNC_CLOSE
,
687 p_hwfn
->p_rdma_info
->proto
, &init_data
);
691 p_ramrod
= &p_ent
->ramrod
.rdma_close_func
;
693 p_ramrod
->num_cnqs
= p_hwfn
->p_rdma_info
->num_cnqs
;
694 p_ramrod
->cnq_start_offset
= (u8
)RESC_START(p_hwfn
, QED_RDMA_CNQ_RAM
);
696 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
699 qed_rdma_free(p_hwfn
);
701 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "RDMA stop done, rc = %d\n", rc
);
705 static int qed_rdma_add_user(void *rdma_cxt
,
706 struct qed_rdma_add_user_out_params
*out_params
)
708 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
709 u32 dpi_start_offset
;
713 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Adding User\n");
716 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
717 rc
= qed_rdma_bmap_alloc_id(p_hwfn
, &p_hwfn
->p_rdma_info
->dpi_map
,
719 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
721 out_params
->dpi
= (u16
)returned_id
;
723 /* Calculate the corresponding DPI address */
724 dpi_start_offset
= p_hwfn
->dpi_start_offset
;
726 out_params
->dpi_addr
= (u64
)((u8 __iomem
*)p_hwfn
->doorbells
+
728 ((out_params
->dpi
) * p_hwfn
->dpi_size
));
730 out_params
->dpi_phys_addr
= p_hwfn
->cdev
->db_phys_addr
+
732 ((out_params
->dpi
) * p_hwfn
->dpi_size
);
734 out_params
->dpi_size
= p_hwfn
->dpi_size
;
736 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Adding user - done, rc = %d\n", rc
);
740 static struct qed_rdma_port
*qed_rdma_query_port(void *rdma_cxt
)
742 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
743 struct qed_rdma_port
*p_port
= p_hwfn
->p_rdma_info
->port
;
745 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "RDMA Query port\n");
747 /* Link may have changed */
748 p_port
->port_state
= p_hwfn
->mcp_info
->link_output
.link_up
?
749 QED_RDMA_PORT_UP
: QED_RDMA_PORT_DOWN
;
751 p_port
->link_speed
= p_hwfn
->mcp_info
->link_output
.speed
;
753 p_port
->max_msg_size
= RDMA_MAX_DATA_SIZE_IN_WQE
;
758 static struct qed_rdma_device
*qed_rdma_query_device(void *rdma_cxt
)
760 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
762 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Query device\n");
764 /* Return struct with device parameters */
765 return p_hwfn
->p_rdma_info
->dev
;
768 static void qed_rdma_free_tid(void *rdma_cxt
, u32 itid
)
770 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
772 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "itid = %08x\n", itid
);
774 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
775 qed_bmap_release_id(p_hwfn
, &p_hwfn
->p_rdma_info
->tid_map
, itid
);
776 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
779 static void qed_rdma_cnq_prod_update(void *rdma_cxt
, u8 qz_offset
, u16 prod
)
781 struct qed_hwfn
*p_hwfn
;
785 p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
787 if (qz_offset
> p_hwfn
->p_rdma_info
->max_queue_zones
) {
789 "queue zone offset %d is too large (max is %d)\n",
790 qz_offset
, p_hwfn
->p_rdma_info
->max_queue_zones
);
794 qz_num
= p_hwfn
->p_rdma_info
->queue_zone_base
+ qz_offset
;
795 addr
= GTT_BAR0_MAP_REG_USDM_RAM
+
796 USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num
);
798 REG_WR16(p_hwfn
, addr
, prod
);
800 /* keep prod updates ordered */
804 static int qed_fill_rdma_dev_info(struct qed_dev
*cdev
,
805 struct qed_dev_rdma_info
*info
)
807 memset(info
, 0, sizeof(*info
));
809 info
->rdma_type
= QED_RDMA_TYPE_ROCE
;
811 qed_fill_dev_info(cdev
, &info
->common
);
816 static int qed_rdma_get_sb_start(struct qed_dev
*cdev
)
820 if (cdev
->num_hwfns
> 1)
821 feat_num
= FEAT_NUM(QED_LEADING_HWFN(cdev
), QED_PF_L2_QUE
);
823 feat_num
= FEAT_NUM(QED_LEADING_HWFN(cdev
), QED_PF_L2_QUE
) *
829 static int qed_rdma_get_min_cnq_msix(struct qed_dev
*cdev
)
831 int n_cnq
= FEAT_NUM(QED_LEADING_HWFN(cdev
), QED_RDMA_CNQ
);
832 int n_msix
= cdev
->int_params
.rdma_msix_cnt
;
834 return min_t(int, n_cnq
, n_msix
);
837 static int qed_rdma_set_int(struct qed_dev
*cdev
, u16 cnt
)
841 /* Mark the fastpath as free/used */
842 cdev
->int_params
.fp_initialized
= cnt
? true : false;
844 if (cdev
->int_params
.out
.int_mode
!= QED_INT_MODE_MSIX
) {
846 "qed roce supports only MSI-X interrupts (detected %d).\n",
847 cdev
->int_params
.out
.int_mode
);
849 } else if (cdev
->int_params
.fp_msix_cnt
) {
850 limit
= cdev
->int_params
.rdma_msix_cnt
;
856 return min_t(int, cnt
, limit
);
859 static int qed_rdma_get_int(struct qed_dev
*cdev
, struct qed_int_info
*info
)
861 memset(info
, 0, sizeof(*info
));
863 if (!cdev
->int_params
.fp_initialized
) {
865 "Protocol driver requested interrupt information, but its support is not yet configured\n");
869 if (cdev
->int_params
.out
.int_mode
== QED_INT_MODE_MSIX
) {
870 int msix_base
= cdev
->int_params
.rdma_msix_base
;
872 info
->msix_cnt
= cdev
->int_params
.rdma_msix_cnt
;
873 info
->msix
= &cdev
->int_params
.msix_table
[msix_base
];
875 DP_VERBOSE(cdev
, QED_MSG_RDMA
, "msix_cnt = %d msix_base=%d\n",
876 info
->msix_cnt
, msix_base
);
882 static int qed_rdma_alloc_pd(void *rdma_cxt
, u16
*pd
)
884 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
888 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Alloc PD\n");
890 /* Allocates an unused protection domain */
891 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
892 rc
= qed_rdma_bmap_alloc_id(p_hwfn
,
893 &p_hwfn
->p_rdma_info
->pd_map
, &returned_id
);
894 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
896 *pd
= (u16
)returned_id
;
898 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Alloc PD - done, rc = %d\n", rc
);
902 static void qed_rdma_free_pd(void *rdma_cxt
, u16 pd
)
904 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
906 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "pd = %08x\n", pd
);
908 /* Returns a previously allocated protection domain for reuse */
909 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
910 qed_bmap_release_id(p_hwfn
, &p_hwfn
->p_rdma_info
->pd_map
, pd
);
911 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
914 static enum qed_rdma_toggle_bit
915 qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn
*p_hwfn
, u16 icid
)
917 struct qed_rdma_info
*p_info
= p_hwfn
->p_rdma_info
;
918 enum qed_rdma_toggle_bit toggle_bit
;
921 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", icid
);
923 /* the function toggle the bit that is related to a given icid
924 * and returns the new toggle bit's value
926 bmap_id
= icid
- qed_cxt_get_proto_cid_start(p_hwfn
, p_info
->proto
);
928 spin_lock_bh(&p_info
->lock
);
929 toggle_bit
= !test_and_change_bit(bmap_id
,
930 p_info
->toggle_bits
.bitmap
);
931 spin_unlock_bh(&p_info
->lock
);
933 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "QED_RDMA_TOGGLE_BIT_= %d\n",
939 static int qed_rdma_create_cq(void *rdma_cxt
,
940 struct qed_rdma_create_cq_in_params
*params
,
943 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
944 struct qed_rdma_info
*p_info
= p_hwfn
->p_rdma_info
;
945 struct rdma_create_cq_ramrod_data
*p_ramrod
;
946 enum qed_rdma_toggle_bit toggle_bit
;
947 struct qed_sp_init_data init_data
;
948 struct qed_spq_entry
*p_ent
;
949 u32 returned_id
, start_cid
;
952 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "cq_handle = %08x%08x\n",
953 params
->cq_handle_hi
, params
->cq_handle_lo
);
956 spin_lock_bh(&p_info
->lock
);
957 rc
= qed_rdma_bmap_alloc_id(p_hwfn
,
958 &p_info
->cq_map
, &returned_id
);
959 spin_unlock_bh(&p_info
->lock
);
962 DP_NOTICE(p_hwfn
, "Can't create CQ, rc = %d\n", rc
);
966 start_cid
= qed_cxt_get_proto_cid_start(p_hwfn
,
968 *icid
= returned_id
+ start_cid
;
970 /* Check if icid requires a page allocation */
971 rc
= qed_cxt_dynamic_ilt_alloc(p_hwfn
, QED_ELEM_CXT
, *icid
);
976 memset(&init_data
, 0, sizeof(init_data
));
977 init_data
.cid
= *icid
;
978 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
979 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
981 /* Send create CQ ramrod */
982 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
983 RDMA_RAMROD_CREATE_CQ
,
984 p_info
->proto
, &init_data
);
988 p_ramrod
= &p_ent
->ramrod
.rdma_create_cq
;
990 p_ramrod
->cq_handle
.hi
= cpu_to_le32(params
->cq_handle_hi
);
991 p_ramrod
->cq_handle
.lo
= cpu_to_le32(params
->cq_handle_lo
);
992 p_ramrod
->dpi
= cpu_to_le16(params
->dpi
);
993 p_ramrod
->is_two_level_pbl
= params
->pbl_two_level
;
994 p_ramrod
->max_cqes
= cpu_to_le32(params
->cq_size
);
995 DMA_REGPAIR_LE(p_ramrod
->pbl_addr
, params
->pbl_ptr
);
996 p_ramrod
->pbl_num_pages
= cpu_to_le16(params
->pbl_num_pages
);
997 p_ramrod
->cnq_id
= (u8
)RESC_START(p_hwfn
, QED_RDMA_CNQ_RAM
) +
999 p_ramrod
->int_timeout
= params
->int_timeout
;
1001 /* toggle the bit for every resize or create cq for a given icid */
1002 toggle_bit
= qed_rdma_toggle_bit_create_resize_cq(p_hwfn
, *icid
);
1004 p_ramrod
->toggle_bit
= toggle_bit
;
1006 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1008 /* restore toggle bit */
1009 qed_rdma_toggle_bit_create_resize_cq(p_hwfn
, *icid
);
1013 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Created CQ, rc = %d\n", rc
);
1017 /* release allocated icid */
1018 spin_lock_bh(&p_info
->lock
);
1019 qed_bmap_release_id(p_hwfn
, &p_info
->cq_map
, returned_id
);
1020 spin_unlock_bh(&p_info
->lock
);
1021 DP_NOTICE(p_hwfn
, "Create CQ failed, rc = %d\n", rc
);
1027 qed_rdma_destroy_cq(void *rdma_cxt
,
1028 struct qed_rdma_destroy_cq_in_params
*in_params
,
1029 struct qed_rdma_destroy_cq_out_params
*out_params
)
1031 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
1032 struct rdma_destroy_cq_output_params
*p_ramrod_res
;
1033 struct rdma_destroy_cq_ramrod_data
*p_ramrod
;
1034 struct qed_sp_init_data init_data
;
1035 struct qed_spq_entry
*p_ent
;
1036 dma_addr_t ramrod_res_phys
;
1039 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", in_params
->icid
);
1042 (struct rdma_destroy_cq_output_params
*)
1043 dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1044 sizeof(struct rdma_destroy_cq_output_params
),
1045 &ramrod_res_phys
, GFP_KERNEL
);
1046 if (!p_ramrod_res
) {
1048 "qed destroy cq failed: cannot allocate memory (ramrod)\n");
1053 memset(&init_data
, 0, sizeof(init_data
));
1054 init_data
.cid
= in_params
->icid
;
1055 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1056 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1058 /* Send destroy CQ ramrod */
1059 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1060 RDMA_RAMROD_DESTROY_CQ
,
1061 p_hwfn
->p_rdma_info
->proto
, &init_data
);
1065 p_ramrod
= &p_ent
->ramrod
.rdma_destroy_cq
;
1066 DMA_REGPAIR_LE(p_ramrod
->output_params_addr
, ramrod_res_phys
);
1068 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1072 out_params
->num_cq_notif
= le16_to_cpu(p_ramrod_res
->cnq_num
);
1074 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1075 sizeof(struct rdma_destroy_cq_output_params
),
1076 p_ramrod_res
, ramrod_res_phys
);
1079 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
1081 qed_bmap_release_id(p_hwfn
,
1082 &p_hwfn
->p_rdma_info
->cq_map
,
1084 qed_cxt_get_proto_cid_start(p_hwfn
,
1086 p_rdma_info
->proto
)));
1088 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
1090 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Destroyed CQ, rc = %d\n", rc
);
1093 err
: dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1094 sizeof(struct rdma_destroy_cq_output_params
),
1095 p_ramrod_res
, ramrod_res_phys
);
1100 static void qed_rdma_set_fw_mac(u16
*p_fw_mac
, u8
*p_qed_mac
)
1102 p_fw_mac
[0] = cpu_to_le16((p_qed_mac
[0] << 8) + p_qed_mac
[1]);
1103 p_fw_mac
[1] = cpu_to_le16((p_qed_mac
[2] << 8) + p_qed_mac
[3]);
1104 p_fw_mac
[2] = cpu_to_le16((p_qed_mac
[4] << 8) + p_qed_mac
[5]);
1107 static void qed_rdma_copy_gids(struct qed_rdma_qp
*qp
, __le32
*src_gid
,
1112 if (qp
->roce_mode
== ROCE_V2_IPV4
) {
1113 /* The IPv4 addresses shall be aligned to the highest word.
1114 * The lower words must be zero.
1116 memset(src_gid
, 0, sizeof(union qed_gid
));
1117 memset(dst_gid
, 0, sizeof(union qed_gid
));
1118 src_gid
[3] = cpu_to_le32(qp
->sgid
.ipv4_addr
);
1119 dst_gid
[3] = cpu_to_le32(qp
->dgid
.ipv4_addr
);
1121 /* GIDs and IPv6 addresses coincide in location and size */
1122 for (i
= 0; i
< ARRAY_SIZE(qp
->sgid
.dwords
); i
++) {
1123 src_gid
[i
] = cpu_to_le32(qp
->sgid
.dwords
[i
]);
1124 dst_gid
[i
] = cpu_to_le32(qp
->dgid
.dwords
[i
]);
1129 static enum roce_flavor
qed_roce_mode_to_flavor(enum roce_mode roce_mode
)
1131 enum roce_flavor flavor
;
1133 switch (roce_mode
) {
1135 flavor
= PLAIN_ROCE
;
1138 flavor
= RROCE_IPV4
;
1141 flavor
= ROCE_V2_IPV6
;
1144 flavor
= MAX_ROCE_MODE
;
1150 void qed_roce_free_cid_pair(struct qed_hwfn
*p_hwfn
, u16 cid
)
1152 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
1153 qed_bmap_release_id(p_hwfn
, &p_hwfn
->p_rdma_info
->cid_map
, cid
);
1154 qed_bmap_release_id(p_hwfn
, &p_hwfn
->p_rdma_info
->cid_map
, cid
+ 1);
1155 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
1158 static int qed_roce_alloc_cid(struct qed_hwfn
*p_hwfn
, u16
*cid
)
1160 struct qed_rdma_info
*p_rdma_info
= p_hwfn
->p_rdma_info
;
1165 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
1166 rc
= qed_rdma_bmap_alloc_id(p_hwfn
, &p_rdma_info
->cid_map
,
1169 spin_unlock_bh(&p_rdma_info
->lock
);
1173 rc
= qed_rdma_bmap_alloc_id(p_hwfn
, &p_rdma_info
->cid_map
,
1176 spin_unlock_bh(&p_rdma_info
->lock
);
1180 /* the two icid's should be adjacent */
1181 if ((requester_icid
- responder_icid
) != 1) {
1182 DP_NOTICE(p_hwfn
, "Failed to allocate two adjacent qp's'\n");
1187 responder_icid
+= qed_cxt_get_proto_cid_start(p_hwfn
,
1188 p_rdma_info
->proto
);
1189 requester_icid
+= qed_cxt_get_proto_cid_start(p_hwfn
,
1190 p_rdma_info
->proto
);
1192 /* If these icids require a new ILT line allocate DMA-able context for
1195 rc
= qed_cxt_dynamic_ilt_alloc(p_hwfn
, QED_ELEM_CXT
, responder_icid
);
1199 rc
= qed_cxt_dynamic_ilt_alloc(p_hwfn
, QED_ELEM_CXT
, requester_icid
);
1203 *cid
= (u16
)responder_icid
;
1207 spin_lock_bh(&p_rdma_info
->lock
);
1208 qed_bmap_release_id(p_hwfn
, &p_rdma_info
->cid_map
, responder_icid
);
1209 qed_bmap_release_id(p_hwfn
, &p_rdma_info
->cid_map
, requester_icid
);
1211 spin_unlock_bh(&p_rdma_info
->lock
);
1212 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
1213 "Allocate CID - failed, rc = %d\n", rc
);
1217 static void qed_roce_set_real_cid(struct qed_hwfn
*p_hwfn
, u32 cid
)
1219 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
1220 qed_bmap_set_id(p_hwfn
, &p_hwfn
->p_rdma_info
->real_cid_map
, cid
);
1221 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
1224 static int qed_roce_sp_create_responder(struct qed_hwfn
*p_hwfn
,
1225 struct qed_rdma_qp
*qp
)
1227 struct roce_create_qp_resp_ramrod_data
*p_ramrod
;
1228 struct qed_sp_init_data init_data
;
1229 enum roce_flavor roce_flavor
;
1230 struct qed_spq_entry
*p_ent
;
1231 u16 regular_latency_queue
;
1232 enum protocol_type proto
;
1235 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
1237 /* Allocate DMA-able memory for IRQ */
1238 qp
->irq_num_pages
= 1;
1239 qp
->irq
= dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1240 RDMA_RING_PAGE_SIZE
,
1241 &qp
->irq_phys_addr
, GFP_KERNEL
);
1245 "qed create responder failed: cannot allocate memory (irq). rc = %d\n",
1251 memset(&init_data
, 0, sizeof(init_data
));
1252 init_data
.cid
= qp
->icid
;
1253 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1254 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1256 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, ROCE_RAMROD_CREATE_QP
,
1257 PROTOCOLID_ROCE
, &init_data
);
1261 p_ramrod
= &p_ent
->ramrod
.roce_create_qp_resp
;
1263 p_ramrod
->flags
= 0;
1265 roce_flavor
= qed_roce_mode_to_flavor(qp
->roce_mode
);
1266 SET_FIELD(p_ramrod
->flags
,
1267 ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR
, roce_flavor
);
1269 SET_FIELD(p_ramrod
->flags
,
1270 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN
,
1271 qp
->incoming_rdma_read_en
);
1273 SET_FIELD(p_ramrod
->flags
,
1274 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN
,
1275 qp
->incoming_rdma_write_en
);
1277 SET_FIELD(p_ramrod
->flags
,
1278 ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN
,
1279 qp
->incoming_atomic_en
);
1281 SET_FIELD(p_ramrod
->flags
,
1282 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN
,
1283 qp
->e2e_flow_control_en
);
1285 SET_FIELD(p_ramrod
->flags
,
1286 ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG
, qp
->use_srq
);
1288 SET_FIELD(p_ramrod
->flags
,
1289 ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN
,
1290 qp
->fmr_and_reserved_lkey
);
1292 SET_FIELD(p_ramrod
->flags
,
1293 ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER
,
1294 qp
->min_rnr_nak_timer
);
1296 p_ramrod
->max_ird
= qp
->max_rd_atomic_resp
;
1297 p_ramrod
->traffic_class
= qp
->traffic_class_tos
;
1298 p_ramrod
->hop_limit
= qp
->hop_limit_ttl
;
1299 p_ramrod
->irq_num_pages
= qp
->irq_num_pages
;
1300 p_ramrod
->p_key
= cpu_to_le16(qp
->pkey
);
1301 p_ramrod
->flow_label
= cpu_to_le32(qp
->flow_label
);
1302 p_ramrod
->dst_qp_id
= cpu_to_le32(qp
->dest_qp
);
1303 p_ramrod
->mtu
= cpu_to_le16(qp
->mtu
);
1304 p_ramrod
->initial_psn
= cpu_to_le32(qp
->rq_psn
);
1305 p_ramrod
->pd
= cpu_to_le16(qp
->pd
);
1306 p_ramrod
->rq_num_pages
= cpu_to_le16(qp
->rq_num_pages
);
1307 DMA_REGPAIR_LE(p_ramrod
->rq_pbl_addr
, qp
->rq_pbl_ptr
);
1308 DMA_REGPAIR_LE(p_ramrod
->irq_pbl_addr
, qp
->irq_phys_addr
);
1309 qed_rdma_copy_gids(qp
, p_ramrod
->src_gid
, p_ramrod
->dst_gid
);
1310 p_ramrod
->qp_handle_for_async
.hi
= cpu_to_le32(qp
->qp_handle_async
.hi
);
1311 p_ramrod
->qp_handle_for_async
.lo
= cpu_to_le32(qp
->qp_handle_async
.lo
);
1312 p_ramrod
->qp_handle_for_cqe
.hi
= cpu_to_le32(qp
->qp_handle
.hi
);
1313 p_ramrod
->qp_handle_for_cqe
.lo
= cpu_to_le32(qp
->qp_handle
.lo
);
1314 p_ramrod
->cq_cid
= cpu_to_le32((p_hwfn
->hw_info
.opaque_fid
<< 16) |
1317 regular_latency_queue
= qed_get_cm_pq_idx(p_hwfn
, PQ_FLAGS_OFLD
);
1319 p_ramrod
->regular_latency_phy_queue
=
1320 cpu_to_le16(regular_latency_queue
);
1321 p_ramrod
->low_latency_phy_queue
=
1322 cpu_to_le16(regular_latency_queue
);
1324 p_ramrod
->dpi
= cpu_to_le16(qp
->dpi
);
1326 qed_rdma_set_fw_mac(p_ramrod
->remote_mac_addr
, qp
->remote_mac_addr
);
1327 qed_rdma_set_fw_mac(p_ramrod
->local_mac_addr
, qp
->local_mac_addr
);
1329 p_ramrod
->udp_src_port
= qp
->udp_src_port
;
1330 p_ramrod
->vlan_id
= cpu_to_le16(qp
->vlan_id
);
1331 p_ramrod
->srq_id
.srq_idx
= cpu_to_le16(qp
->srq_id
);
1332 p_ramrod
->srq_id
.opaque_fid
= cpu_to_le16(p_hwfn
->hw_info
.opaque_fid
);
1334 p_ramrod
->stats_counter_id
= RESC_START(p_hwfn
, QED_RDMA_STATS_QUEUE
) +
1337 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1339 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
1340 "rc = %d regular physical queue = 0x%x\n", rc
,
1341 regular_latency_queue
);
1346 qp
->resp_offloaded
= true;
1349 proto
= p_hwfn
->p_rdma_info
->proto
;
1350 qed_roce_set_real_cid(p_hwfn
, qp
->icid
-
1351 qed_cxt_get_proto_cid_start(p_hwfn
, proto
));
1356 DP_NOTICE(p_hwfn
, "create responder - failed, rc = %d\n", rc
);
1357 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1358 qp
->irq_num_pages
* RDMA_RING_PAGE_SIZE
,
1359 qp
->irq
, qp
->irq_phys_addr
);
1364 static int qed_roce_sp_create_requester(struct qed_hwfn
*p_hwfn
,
1365 struct qed_rdma_qp
*qp
)
1367 struct roce_create_qp_req_ramrod_data
*p_ramrod
;
1368 struct qed_sp_init_data init_data
;
1369 enum roce_flavor roce_flavor
;
1370 struct qed_spq_entry
*p_ent
;
1371 u16 regular_latency_queue
;
1372 enum protocol_type proto
;
1375 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
1377 /* Allocate DMA-able memory for ORQ */
1378 qp
->orq_num_pages
= 1;
1379 qp
->orq
= dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1380 RDMA_RING_PAGE_SIZE
,
1381 &qp
->orq_phys_addr
, GFP_KERNEL
);
1385 "qed create requester failed: cannot allocate memory (orq). rc = %d\n",
1391 memset(&init_data
, 0, sizeof(init_data
));
1392 init_data
.cid
= qp
->icid
+ 1;
1393 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1394 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1396 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1397 ROCE_RAMROD_CREATE_QP
,
1398 PROTOCOLID_ROCE
, &init_data
);
1402 p_ramrod
= &p_ent
->ramrod
.roce_create_qp_req
;
1404 p_ramrod
->flags
= 0;
1406 roce_flavor
= qed_roce_mode_to_flavor(qp
->roce_mode
);
1407 SET_FIELD(p_ramrod
->flags
,
1408 ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR
, roce_flavor
);
1410 SET_FIELD(p_ramrod
->flags
,
1411 ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN
,
1412 qp
->fmr_and_reserved_lkey
);
1414 SET_FIELD(p_ramrod
->flags
,
1415 ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP
, qp
->signal_all
);
1417 SET_FIELD(p_ramrod
->flags
,
1418 ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT
, qp
->retry_cnt
);
1420 SET_FIELD(p_ramrod
->flags
,
1421 ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT
,
1424 p_ramrod
->max_ord
= qp
->max_rd_atomic_req
;
1425 p_ramrod
->traffic_class
= qp
->traffic_class_tos
;
1426 p_ramrod
->hop_limit
= qp
->hop_limit_ttl
;
1427 p_ramrod
->orq_num_pages
= qp
->orq_num_pages
;
1428 p_ramrod
->p_key
= cpu_to_le16(qp
->pkey
);
1429 p_ramrod
->flow_label
= cpu_to_le32(qp
->flow_label
);
1430 p_ramrod
->dst_qp_id
= cpu_to_le32(qp
->dest_qp
);
1431 p_ramrod
->ack_timeout_val
= cpu_to_le32(qp
->ack_timeout
);
1432 p_ramrod
->mtu
= cpu_to_le16(qp
->mtu
);
1433 p_ramrod
->initial_psn
= cpu_to_le32(qp
->sq_psn
);
1434 p_ramrod
->pd
= cpu_to_le16(qp
->pd
);
1435 p_ramrod
->sq_num_pages
= cpu_to_le16(qp
->sq_num_pages
);
1436 DMA_REGPAIR_LE(p_ramrod
->sq_pbl_addr
, qp
->sq_pbl_ptr
);
1437 DMA_REGPAIR_LE(p_ramrod
->orq_pbl_addr
, qp
->orq_phys_addr
);
1438 qed_rdma_copy_gids(qp
, p_ramrod
->src_gid
, p_ramrod
->dst_gid
);
1439 p_ramrod
->qp_handle_for_async
.hi
= cpu_to_le32(qp
->qp_handle_async
.hi
);
1440 p_ramrod
->qp_handle_for_async
.lo
= cpu_to_le32(qp
->qp_handle_async
.lo
);
1441 p_ramrod
->qp_handle_for_cqe
.hi
= cpu_to_le32(qp
->qp_handle
.hi
);
1442 p_ramrod
->qp_handle_for_cqe
.lo
= cpu_to_le32(qp
->qp_handle
.lo
);
1444 cpu_to_le32((p_hwfn
->hw_info
.opaque_fid
<< 16) | qp
->sq_cq_id
);
1446 regular_latency_queue
= qed_get_cm_pq_idx(p_hwfn
, PQ_FLAGS_OFLD
);
1448 p_ramrod
->regular_latency_phy_queue
=
1449 cpu_to_le16(regular_latency_queue
);
1450 p_ramrod
->low_latency_phy_queue
=
1451 cpu_to_le16(regular_latency_queue
);
1453 p_ramrod
->dpi
= cpu_to_le16(qp
->dpi
);
1455 qed_rdma_set_fw_mac(p_ramrod
->remote_mac_addr
, qp
->remote_mac_addr
);
1456 qed_rdma_set_fw_mac(p_ramrod
->local_mac_addr
, qp
->local_mac_addr
);
1458 p_ramrod
->udp_src_port
= qp
->udp_src_port
;
1459 p_ramrod
->vlan_id
= cpu_to_le16(qp
->vlan_id
);
1460 p_ramrod
->stats_counter_id
= RESC_START(p_hwfn
, QED_RDMA_STATS_QUEUE
) +
1463 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1465 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "rc = %d\n", rc
);
1470 qp
->req_offloaded
= true;
1471 proto
= p_hwfn
->p_rdma_info
->proto
;
1472 qed_roce_set_real_cid(p_hwfn
,
1474 qed_cxt_get_proto_cid_start(p_hwfn
, proto
));
1479 DP_NOTICE(p_hwfn
, "Create requested - failed, rc = %d\n", rc
);
1480 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1481 qp
->orq_num_pages
* RDMA_RING_PAGE_SIZE
,
1482 qp
->orq
, qp
->orq_phys_addr
);
1486 static int qed_roce_sp_modify_responder(struct qed_hwfn
*p_hwfn
,
1487 struct qed_rdma_qp
*qp
,
1488 bool move_to_err
, u32 modify_flags
)
1490 struct roce_modify_qp_resp_ramrod_data
*p_ramrod
;
1491 struct qed_sp_init_data init_data
;
1492 struct qed_spq_entry
*p_ent
;
1495 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
1497 if (move_to_err
&& !qp
->resp_offloaded
)
1501 memset(&init_data
, 0, sizeof(init_data
));
1502 init_data
.cid
= qp
->icid
;
1503 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1504 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1506 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1507 ROCE_EVENT_MODIFY_QP
,
1508 PROTOCOLID_ROCE
, &init_data
);
1510 DP_NOTICE(p_hwfn
, "rc = %d\n", rc
);
1514 p_ramrod
= &p_ent
->ramrod
.roce_modify_qp_resp
;
1516 p_ramrod
->flags
= 0;
1518 SET_FIELD(p_ramrod
->flags
,
1519 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG
, move_to_err
);
1521 SET_FIELD(p_ramrod
->flags
,
1522 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN
,
1523 qp
->incoming_rdma_read_en
);
1525 SET_FIELD(p_ramrod
->flags
,
1526 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN
,
1527 qp
->incoming_rdma_write_en
);
1529 SET_FIELD(p_ramrod
->flags
,
1530 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN
,
1531 qp
->incoming_atomic_en
);
1533 SET_FIELD(p_ramrod
->flags
,
1534 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN
,
1535 qp
->e2e_flow_control_en
);
1537 SET_FIELD(p_ramrod
->flags
,
1538 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG
,
1539 GET_FIELD(modify_flags
,
1540 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN
));
1542 SET_FIELD(p_ramrod
->flags
,
1543 ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG
,
1544 GET_FIELD(modify_flags
, QED_ROCE_MODIFY_QP_VALID_PKEY
));
1546 SET_FIELD(p_ramrod
->flags
,
1547 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG
,
1548 GET_FIELD(modify_flags
,
1549 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR
));
1551 SET_FIELD(p_ramrod
->flags
,
1552 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG
,
1553 GET_FIELD(modify_flags
,
1554 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP
));
1556 SET_FIELD(p_ramrod
->flags
,
1557 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG
,
1558 GET_FIELD(modify_flags
,
1559 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER
));
1561 p_ramrod
->fields
= 0;
1562 SET_FIELD(p_ramrod
->fields
,
1563 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER
,
1564 qp
->min_rnr_nak_timer
);
1566 p_ramrod
->max_ird
= qp
->max_rd_atomic_resp
;
1567 p_ramrod
->traffic_class
= qp
->traffic_class_tos
;
1568 p_ramrod
->hop_limit
= qp
->hop_limit_ttl
;
1569 p_ramrod
->p_key
= cpu_to_le16(qp
->pkey
);
1570 p_ramrod
->flow_label
= cpu_to_le32(qp
->flow_label
);
1571 p_ramrod
->mtu
= cpu_to_le16(qp
->mtu
);
1572 qed_rdma_copy_gids(qp
, p_ramrod
->src_gid
, p_ramrod
->dst_gid
);
1573 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1575 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Modify responder, rc = %d\n", rc
);
1579 static int qed_roce_sp_modify_requester(struct qed_hwfn
*p_hwfn
,
1580 struct qed_rdma_qp
*qp
,
1582 bool move_to_err
, u32 modify_flags
)
1584 struct roce_modify_qp_req_ramrod_data
*p_ramrod
;
1585 struct qed_sp_init_data init_data
;
1586 struct qed_spq_entry
*p_ent
;
1589 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
1591 if (move_to_err
&& !(qp
->req_offloaded
))
1595 memset(&init_data
, 0, sizeof(init_data
));
1596 init_data
.cid
= qp
->icid
+ 1;
1597 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1598 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1600 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1601 ROCE_EVENT_MODIFY_QP
,
1602 PROTOCOLID_ROCE
, &init_data
);
1604 DP_NOTICE(p_hwfn
, "rc = %d\n", rc
);
1608 p_ramrod
= &p_ent
->ramrod
.roce_modify_qp_req
;
1610 p_ramrod
->flags
= 0;
1612 SET_FIELD(p_ramrod
->flags
,
1613 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG
, move_to_err
);
1615 SET_FIELD(p_ramrod
->flags
,
1616 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG
, move_to_sqd
);
1618 SET_FIELD(p_ramrod
->flags
,
1619 ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY
,
1622 SET_FIELD(p_ramrod
->flags
,
1623 ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG
,
1624 GET_FIELD(modify_flags
, QED_ROCE_MODIFY_QP_VALID_PKEY
));
1626 SET_FIELD(p_ramrod
->flags
,
1627 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG
,
1628 GET_FIELD(modify_flags
,
1629 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR
));
1631 SET_FIELD(p_ramrod
->flags
,
1632 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG
,
1633 GET_FIELD(modify_flags
,
1634 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ
));
1636 SET_FIELD(p_ramrod
->flags
,
1637 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG
,
1638 GET_FIELD(modify_flags
,
1639 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT
));
1641 SET_FIELD(p_ramrod
->flags
,
1642 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG
,
1643 GET_FIELD(modify_flags
, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT
));
1645 SET_FIELD(p_ramrod
->flags
,
1646 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG
,
1647 GET_FIELD(modify_flags
,
1648 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT
));
1650 p_ramrod
->fields
= 0;
1651 SET_FIELD(p_ramrod
->fields
,
1652 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT
, qp
->retry_cnt
);
1654 SET_FIELD(p_ramrod
->fields
,
1655 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT
,
1658 p_ramrod
->max_ord
= qp
->max_rd_atomic_req
;
1659 p_ramrod
->traffic_class
= qp
->traffic_class_tos
;
1660 p_ramrod
->hop_limit
= qp
->hop_limit_ttl
;
1661 p_ramrod
->p_key
= cpu_to_le16(qp
->pkey
);
1662 p_ramrod
->flow_label
= cpu_to_le32(qp
->flow_label
);
1663 p_ramrod
->ack_timeout_val
= cpu_to_le32(qp
->ack_timeout
);
1664 p_ramrod
->mtu
= cpu_to_le16(qp
->mtu
);
1665 qed_rdma_copy_gids(qp
, p_ramrod
->src_gid
, p_ramrod
->dst_gid
);
1666 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1668 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Modify requester, rc = %d\n", rc
);
1672 static int qed_roce_sp_destroy_qp_responder(struct qed_hwfn
*p_hwfn
,
1673 struct qed_rdma_qp
*qp
,
1674 u32
*num_invalidated_mw
,
1677 struct roce_destroy_qp_resp_output_params
*p_ramrod_res
;
1678 struct roce_destroy_qp_resp_ramrod_data
*p_ramrod
;
1679 struct qed_sp_init_data init_data
;
1680 struct qed_spq_entry
*p_ent
;
1681 dma_addr_t ramrod_res_phys
;
1684 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
1686 *num_invalidated_mw
= 0;
1687 *cq_prod
= qp
->cq_prod
;
1689 if (!qp
->resp_offloaded
) {
1690 /* If a responder was never offload, we need to free the cids
1691 * allocated in create_qp as a FW async event will never arrive
1696 qed_cxt_get_proto_cid_start(p_hwfn
,
1697 p_hwfn
->p_rdma_info
->proto
);
1698 qed_roce_free_cid_pair(p_hwfn
, (u16
)cid
);
1704 memset(&init_data
, 0, sizeof(init_data
));
1705 init_data
.cid
= qp
->icid
;
1706 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1707 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1709 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1710 ROCE_RAMROD_DESTROY_QP
,
1711 PROTOCOLID_ROCE
, &init_data
);
1715 p_ramrod
= &p_ent
->ramrod
.roce_destroy_qp_resp
;
1717 p_ramrod_res
= (struct roce_destroy_qp_resp_output_params
*)
1718 dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
, sizeof(*p_ramrod_res
),
1719 &ramrod_res_phys
, GFP_KERNEL
);
1721 if (!p_ramrod_res
) {
1724 "qed destroy responder failed: cannot allocate memory (ramrod). rc = %d\n",
1729 DMA_REGPAIR_LE(p_ramrod
->output_params_addr
, ramrod_res_phys
);
1731 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1735 *num_invalidated_mw
= le32_to_cpu(p_ramrod_res
->num_invalidated_mw
);
1736 *cq_prod
= le32_to_cpu(p_ramrod_res
->cq_prod
);
1737 qp
->cq_prod
= *cq_prod
;
1739 /* Free IRQ - only if ramrod succeeded, in case FW is still using it */
1740 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1741 qp
->irq_num_pages
* RDMA_RING_PAGE_SIZE
,
1742 qp
->irq
, qp
->irq_phys_addr
);
1744 qp
->resp_offloaded
= false;
1746 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Destroy responder, rc = %d\n", rc
);
1749 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1750 sizeof(struct roce_destroy_qp_resp_output_params
),
1751 p_ramrod_res
, ramrod_res_phys
);
1756 static int qed_roce_sp_destroy_qp_requester(struct qed_hwfn
*p_hwfn
,
1757 struct qed_rdma_qp
*qp
,
1760 struct roce_destroy_qp_req_output_params
*p_ramrod_res
;
1761 struct roce_destroy_qp_req_ramrod_data
*p_ramrod
;
1762 struct qed_sp_init_data init_data
;
1763 struct qed_spq_entry
*p_ent
;
1764 dma_addr_t ramrod_res_phys
;
1767 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
1769 if (!qp
->req_offloaded
)
1772 p_ramrod_res
= (struct roce_destroy_qp_req_output_params
*)
1773 dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1774 sizeof(*p_ramrod_res
),
1775 &ramrod_res_phys
, GFP_KERNEL
);
1776 if (!p_ramrod_res
) {
1778 "qed destroy requester failed: cannot allocate memory (ramrod)\n");
1783 memset(&init_data
, 0, sizeof(init_data
));
1784 init_data
.cid
= qp
->icid
+ 1;
1785 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1786 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1788 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, ROCE_RAMROD_DESTROY_QP
,
1789 PROTOCOLID_ROCE
, &init_data
);
1793 p_ramrod
= &p_ent
->ramrod
.roce_destroy_qp_req
;
1794 DMA_REGPAIR_LE(p_ramrod
->output_params_addr
, ramrod_res_phys
);
1796 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1800 *num_bound_mw
= le32_to_cpu(p_ramrod_res
->num_bound_mw
);
1802 /* Free ORQ - only if ramrod succeeded, in case FW is still using it */
1803 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1804 qp
->orq_num_pages
* RDMA_RING_PAGE_SIZE
,
1805 qp
->orq
, qp
->orq_phys_addr
);
1807 qp
->req_offloaded
= false;
1809 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Destroy requester, rc = %d\n", rc
);
1812 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
, sizeof(*p_ramrod_res
),
1813 p_ramrod_res
, ramrod_res_phys
);
1818 static int qed_roce_query_qp(struct qed_hwfn
*p_hwfn
,
1819 struct qed_rdma_qp
*qp
,
1820 struct qed_rdma_query_qp_out_params
*out_params
)
1822 struct roce_query_qp_resp_output_params
*p_resp_ramrod_res
;
1823 struct roce_query_qp_req_output_params
*p_req_ramrod_res
;
1824 struct roce_query_qp_resp_ramrod_data
*p_resp_ramrod
;
1825 struct roce_query_qp_req_ramrod_data
*p_req_ramrod
;
1826 struct qed_sp_init_data init_data
;
1827 dma_addr_t resp_ramrod_res_phys
;
1828 dma_addr_t req_ramrod_res_phys
;
1829 struct qed_spq_entry
*p_ent
;
1835 if ((!(qp
->resp_offloaded
)) && (!(qp
->req_offloaded
))) {
1836 /* We can't send ramrod to the fw since this qp wasn't offloaded
1839 out_params
->draining
= false;
1840 out_params
->rq_psn
= qp
->rq_psn
;
1841 out_params
->sq_psn
= qp
->sq_psn
;
1842 out_params
->state
= qp
->cur_state
;
1844 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "No QPs as no offload\n");
1848 if (!(qp
->resp_offloaded
)) {
1850 "The responder's qp should be offloded before requester's\n");
1854 /* Send a query responder ramrod to FW to get RQ-PSN and state */
1855 p_resp_ramrod_res
= (struct roce_query_qp_resp_output_params
*)
1856 dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1857 sizeof(*p_resp_ramrod_res
),
1858 &resp_ramrod_res_phys
, GFP_KERNEL
);
1859 if (!p_resp_ramrod_res
) {
1861 "qed query qp failed: cannot allocate memory (ramrod)\n");
1866 memset(&init_data
, 0, sizeof(init_data
));
1867 init_data
.cid
= qp
->icid
;
1868 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1869 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1870 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, ROCE_RAMROD_QUERY_QP
,
1871 PROTOCOLID_ROCE
, &init_data
);
1875 p_resp_ramrod
= &p_ent
->ramrod
.roce_query_qp_resp
;
1876 DMA_REGPAIR_LE(p_resp_ramrod
->output_params_addr
, resp_ramrod_res_phys
);
1878 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1882 out_params
->rq_psn
= le32_to_cpu(p_resp_ramrod_res
->psn
);
1883 rq_err_state
= GET_FIELD(le32_to_cpu(p_resp_ramrod_res
->err_flag
),
1884 ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG
);
1886 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
, sizeof(*p_resp_ramrod_res
),
1887 p_resp_ramrod_res
, resp_ramrod_res_phys
);
1889 if (!(qp
->req_offloaded
)) {
1890 /* Don't send query qp for the requester */
1891 out_params
->sq_psn
= qp
->sq_psn
;
1892 out_params
->draining
= false;
1895 qp
->cur_state
= QED_ROCE_QP_STATE_ERR
;
1897 out_params
->state
= qp
->cur_state
;
1902 /* Send a query requester ramrod to FW to get SQ-PSN and state */
1903 p_req_ramrod_res
= (struct roce_query_qp_req_output_params
*)
1904 dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1905 sizeof(*p_req_ramrod_res
),
1906 &req_ramrod_res_phys
,
1908 if (!p_req_ramrod_res
) {
1911 "qed query qp failed: cannot allocate memory (ramrod)\n");
1916 init_data
.cid
= qp
->icid
+ 1;
1917 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, ROCE_RAMROD_QUERY_QP
,
1918 PROTOCOLID_ROCE
, &init_data
);
1922 p_req_ramrod
= &p_ent
->ramrod
.roce_query_qp_req
;
1923 DMA_REGPAIR_LE(p_req_ramrod
->output_params_addr
, req_ramrod_res_phys
);
1925 rc
= qed_spq_post(p_hwfn
, p_ent
, NULL
);
1929 out_params
->sq_psn
= le32_to_cpu(p_req_ramrod_res
->psn
);
1930 sq_err_state
= GET_FIELD(le32_to_cpu(p_req_ramrod_res
->flags
),
1931 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG
);
1933 GET_FIELD(le32_to_cpu(p_req_ramrod_res
->flags
),
1934 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG
);
1936 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
, sizeof(*p_req_ramrod_res
),
1937 p_req_ramrod_res
, req_ramrod_res_phys
);
1939 out_params
->draining
= false;
1941 if (rq_err_state
|| sq_err_state
)
1942 qp
->cur_state
= QED_ROCE_QP_STATE_ERR
;
1943 else if (sq_draining
)
1944 out_params
->draining
= true;
1945 out_params
->state
= qp
->cur_state
;
1950 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
, sizeof(*p_req_ramrod_res
),
1951 p_req_ramrod_res
, req_ramrod_res_phys
);
1954 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
, sizeof(*p_resp_ramrod_res
),
1955 p_resp_ramrod_res
, resp_ramrod_res_phys
);
1959 static int qed_roce_destroy_qp(struct qed_hwfn
*p_hwfn
, struct qed_rdma_qp
*qp
)
1961 u32 num_invalidated_mw
= 0;
1962 u32 num_bound_mw
= 0;
1966 /* Destroys the specified QP */
1967 if ((qp
->cur_state
!= QED_ROCE_QP_STATE_RESET
) &&
1968 (qp
->cur_state
!= QED_ROCE_QP_STATE_ERR
) &&
1969 (qp
->cur_state
!= QED_ROCE_QP_STATE_INIT
)) {
1971 "QP must be in error, reset or init state before destroying it\n");
1975 if (qp
->cur_state
!= QED_ROCE_QP_STATE_RESET
) {
1976 rc
= qed_roce_sp_destroy_qp_responder(p_hwfn
, qp
,
1977 &num_invalidated_mw
,
1982 /* Send destroy requester ramrod */
1983 rc
= qed_roce_sp_destroy_qp_requester(p_hwfn
, qp
,
1988 if (num_invalidated_mw
!= num_bound_mw
) {
1990 "number of invalidate memory windows is different from bounded ones\n");
1998 static int qed_rdma_query_qp(void *rdma_cxt
,
1999 struct qed_rdma_qp
*qp
,
2000 struct qed_rdma_query_qp_out_params
*out_params
)
2002 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2005 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
2007 /* The following fields are filled in from qp and not FW as they can't
2010 out_params
->mtu
= qp
->mtu
;
2011 out_params
->dest_qp
= qp
->dest_qp
;
2012 out_params
->incoming_atomic_en
= qp
->incoming_atomic_en
;
2013 out_params
->e2e_flow_control_en
= qp
->e2e_flow_control_en
;
2014 out_params
->incoming_rdma_read_en
= qp
->incoming_rdma_read_en
;
2015 out_params
->incoming_rdma_write_en
= qp
->incoming_rdma_write_en
;
2016 out_params
->dgid
= qp
->dgid
;
2017 out_params
->flow_label
= qp
->flow_label
;
2018 out_params
->hop_limit_ttl
= qp
->hop_limit_ttl
;
2019 out_params
->traffic_class_tos
= qp
->traffic_class_tos
;
2020 out_params
->timeout
= qp
->ack_timeout
;
2021 out_params
->rnr_retry
= qp
->rnr_retry_cnt
;
2022 out_params
->retry_cnt
= qp
->retry_cnt
;
2023 out_params
->min_rnr_nak_timer
= qp
->min_rnr_nak_timer
;
2024 out_params
->pkey_index
= 0;
2025 out_params
->max_rd_atomic
= qp
->max_rd_atomic_req
;
2026 out_params
->max_dest_rd_atomic
= qp
->max_rd_atomic_resp
;
2027 out_params
->sqd_async
= qp
->sqd_async
;
2029 rc
= qed_roce_query_qp(p_hwfn
, qp
, out_params
);
2031 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Query QP, rc = %d\n", rc
);
2035 static int qed_rdma_destroy_qp(void *rdma_cxt
, struct qed_rdma_qp
*qp
)
2037 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2040 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x\n", qp
->icid
);
2042 rc
= qed_roce_destroy_qp(p_hwfn
, qp
);
2044 /* free qp params struct */
2047 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "QP destroyed\n");
2051 static struct qed_rdma_qp
*
2052 qed_rdma_create_qp(void *rdma_cxt
,
2053 struct qed_rdma_create_qp_in_params
*in_params
,
2054 struct qed_rdma_create_qp_out_params
*out_params
)
2056 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2057 struct qed_rdma_qp
*qp
;
2058 u8 max_stats_queues
;
2061 if (!rdma_cxt
|| !in_params
|| !out_params
|| !p_hwfn
->p_rdma_info
) {
2062 DP_ERR(p_hwfn
->cdev
,
2063 "qed roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n",
2064 rdma_cxt
, in_params
, out_params
);
2068 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
2069 "qed rdma create qp called with qp_handle = %08x%08x\n",
2070 in_params
->qp_handle_hi
, in_params
->qp_handle_lo
);
2072 /* Some sanity checks... */
2073 max_stats_queues
= p_hwfn
->p_rdma_info
->dev
->max_stats_queues
;
2074 if (in_params
->stats_queue
>= max_stats_queues
) {
2075 DP_ERR(p_hwfn
->cdev
,
2076 "qed rdma create qp failed due to invalid statistics queue %d. maximum is %d\n",
2077 in_params
->stats_queue
, max_stats_queues
);
2081 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
2083 DP_NOTICE(p_hwfn
, "Failed to allocate qed_rdma_qp\n");
2087 rc
= qed_roce_alloc_cid(p_hwfn
, &qp
->icid
);
2088 qp
->qpid
= ((0xFF << 16) | qp
->icid
);
2090 DP_INFO(p_hwfn
, "ROCE qpid=%x\n", qp
->qpid
);
2097 qp
->cur_state
= QED_ROCE_QP_STATE_RESET
;
2098 qp
->qp_handle
.hi
= cpu_to_le32(in_params
->qp_handle_hi
);
2099 qp
->qp_handle
.lo
= cpu_to_le32(in_params
->qp_handle_lo
);
2100 qp
->qp_handle_async
.hi
= cpu_to_le32(in_params
->qp_handle_async_hi
);
2101 qp
->qp_handle_async
.lo
= cpu_to_le32(in_params
->qp_handle_async_lo
);
2102 qp
->use_srq
= in_params
->use_srq
;
2103 qp
->signal_all
= in_params
->signal_all
;
2104 qp
->fmr_and_reserved_lkey
= in_params
->fmr_and_reserved_lkey
;
2105 qp
->pd
= in_params
->pd
;
2106 qp
->dpi
= in_params
->dpi
;
2107 qp
->sq_cq_id
= in_params
->sq_cq_id
;
2108 qp
->sq_num_pages
= in_params
->sq_num_pages
;
2109 qp
->sq_pbl_ptr
= in_params
->sq_pbl_ptr
;
2110 qp
->rq_cq_id
= in_params
->rq_cq_id
;
2111 qp
->rq_num_pages
= in_params
->rq_num_pages
;
2112 qp
->rq_pbl_ptr
= in_params
->rq_pbl_ptr
;
2113 qp
->srq_id
= in_params
->srq_id
;
2114 qp
->req_offloaded
= false;
2115 qp
->resp_offloaded
= false;
2116 qp
->e2e_flow_control_en
= qp
->use_srq
? false : true;
2117 qp
->stats_queue
= in_params
->stats_queue
;
2119 out_params
->icid
= qp
->icid
;
2120 out_params
->qp_id
= qp
->qpid
;
2122 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Create QP, rc = %d\n", rc
);
2126 static int qed_roce_modify_qp(struct qed_hwfn
*p_hwfn
,
2127 struct qed_rdma_qp
*qp
,
2128 enum qed_roce_qp_state prev_state
,
2129 struct qed_rdma_modify_qp_in_params
*params
)
2131 u32 num_invalidated_mw
= 0, num_bound_mw
= 0;
2134 /* Perform additional operations according to the current state and the
2137 if (((prev_state
== QED_ROCE_QP_STATE_INIT
) ||
2138 (prev_state
== QED_ROCE_QP_STATE_RESET
)) &&
2139 (qp
->cur_state
== QED_ROCE_QP_STATE_RTR
)) {
2140 /* Init->RTR or Reset->RTR */
2141 rc
= qed_roce_sp_create_responder(p_hwfn
, qp
);
2143 } else if ((prev_state
== QED_ROCE_QP_STATE_RTR
) &&
2144 (qp
->cur_state
== QED_ROCE_QP_STATE_RTS
)) {
2146 rc
= qed_roce_sp_create_requester(p_hwfn
, qp
);
2150 /* Send modify responder ramrod */
2151 rc
= qed_roce_sp_modify_responder(p_hwfn
, qp
, false,
2152 params
->modify_flags
);
2154 } else if ((prev_state
== QED_ROCE_QP_STATE_RTS
) &&
2155 (qp
->cur_state
== QED_ROCE_QP_STATE_RTS
)) {
2157 rc
= qed_roce_sp_modify_responder(p_hwfn
, qp
, false,
2158 params
->modify_flags
);
2162 rc
= qed_roce_sp_modify_requester(p_hwfn
, qp
, false, false,
2163 params
->modify_flags
);
2165 } else if ((prev_state
== QED_ROCE_QP_STATE_RTS
) &&
2166 (qp
->cur_state
== QED_ROCE_QP_STATE_SQD
)) {
2168 rc
= qed_roce_sp_modify_requester(p_hwfn
, qp
, true, false,
2169 params
->modify_flags
);
2171 } else if ((prev_state
== QED_ROCE_QP_STATE_SQD
) &&
2172 (qp
->cur_state
== QED_ROCE_QP_STATE_SQD
)) {
2174 rc
= qed_roce_sp_modify_responder(p_hwfn
, qp
, false,
2175 params
->modify_flags
);
2179 rc
= qed_roce_sp_modify_requester(p_hwfn
, qp
, false, false,
2180 params
->modify_flags
);
2182 } else if ((prev_state
== QED_ROCE_QP_STATE_SQD
) &&
2183 (qp
->cur_state
== QED_ROCE_QP_STATE_RTS
)) {
2185 rc
= qed_roce_sp_modify_responder(p_hwfn
, qp
, false,
2186 params
->modify_flags
);
2190 rc
= qed_roce_sp_modify_requester(p_hwfn
, qp
, false, false,
2191 params
->modify_flags
);
2194 } else if (qp
->cur_state
== QED_ROCE_QP_STATE_ERR
) {
2196 rc
= qed_roce_sp_modify_responder(p_hwfn
, qp
, true,
2197 params
->modify_flags
);
2201 rc
= qed_roce_sp_modify_requester(p_hwfn
, qp
, false, true,
2202 params
->modify_flags
);
2204 } else if (qp
->cur_state
== QED_ROCE_QP_STATE_RESET
) {
2205 /* Any state -> RESET */
2208 /* Send destroy responder ramrod */
2209 rc
= qed_roce_sp_destroy_qp_responder(p_hwfn
,
2211 &num_invalidated_mw
,
2217 qp
->cq_prod
= cq_prod
;
2219 rc
= qed_roce_sp_destroy_qp_requester(p_hwfn
, qp
,
2222 if (num_invalidated_mw
!= num_bound_mw
) {
2224 "number of invalidate memory windows is different from bounded ones\n");
2228 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "0\n");
2234 static int qed_rdma_modify_qp(void *rdma_cxt
,
2235 struct qed_rdma_qp
*qp
,
2236 struct qed_rdma_modify_qp_in_params
*params
)
2238 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2239 enum qed_roce_qp_state prev_state
;
2242 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "icid = %08x params->new_state=%d\n",
2243 qp
->icid
, params
->new_state
);
2246 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "rc = %d\n", rc
);
2250 if (GET_FIELD(params
->modify_flags
,
2251 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN
)) {
2252 qp
->incoming_rdma_read_en
= params
->incoming_rdma_read_en
;
2253 qp
->incoming_rdma_write_en
= params
->incoming_rdma_write_en
;
2254 qp
->incoming_atomic_en
= params
->incoming_atomic_en
;
2257 /* Update QP structure with the updated values */
2258 if (GET_FIELD(params
->modify_flags
, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE
))
2259 qp
->roce_mode
= params
->roce_mode
;
2260 if (GET_FIELD(params
->modify_flags
, QED_ROCE_MODIFY_QP_VALID_PKEY
))
2261 qp
->pkey
= params
->pkey
;
2262 if (GET_FIELD(params
->modify_flags
,
2263 QED_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN
))
2264 qp
->e2e_flow_control_en
= params
->e2e_flow_control_en
;
2265 if (GET_FIELD(params
->modify_flags
, QED_ROCE_MODIFY_QP_VALID_DEST_QP
))
2266 qp
->dest_qp
= params
->dest_qp
;
2267 if (GET_FIELD(params
->modify_flags
,
2268 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR
)) {
2269 /* Indicates that the following parameters have changed:
2270 * Traffic class, flow label, hop limit, source GID,
2271 * destination GID, loopback indicator
2273 qp
->traffic_class_tos
= params
->traffic_class_tos
;
2274 qp
->flow_label
= params
->flow_label
;
2275 qp
->hop_limit_ttl
= params
->hop_limit_ttl
;
2277 qp
->sgid
= params
->sgid
;
2278 qp
->dgid
= params
->dgid
;
2279 qp
->udp_src_port
= 0;
2280 qp
->vlan_id
= params
->vlan_id
;
2281 qp
->mtu
= params
->mtu
;
2282 qp
->lb_indication
= params
->lb_indication
;
2283 memcpy((u8
*)&qp
->remote_mac_addr
[0],
2284 (u8
*)¶ms
->remote_mac_addr
[0], ETH_ALEN
);
2285 if (params
->use_local_mac
) {
2286 memcpy((u8
*)&qp
->local_mac_addr
[0],
2287 (u8
*)¶ms
->local_mac_addr
[0], ETH_ALEN
);
2289 memcpy((u8
*)&qp
->local_mac_addr
[0],
2290 (u8
*)&p_hwfn
->hw_info
.hw_mac_addr
, ETH_ALEN
);
2293 if (GET_FIELD(params
->modify_flags
, QED_ROCE_MODIFY_QP_VALID_RQ_PSN
))
2294 qp
->rq_psn
= params
->rq_psn
;
2295 if (GET_FIELD(params
->modify_flags
, QED_ROCE_MODIFY_QP_VALID_SQ_PSN
))
2296 qp
->sq_psn
= params
->sq_psn
;
2297 if (GET_FIELD(params
->modify_flags
,
2298 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ
))
2299 qp
->max_rd_atomic_req
= params
->max_rd_atomic_req
;
2300 if (GET_FIELD(params
->modify_flags
,
2301 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP
))
2302 qp
->max_rd_atomic_resp
= params
->max_rd_atomic_resp
;
2303 if (GET_FIELD(params
->modify_flags
,
2304 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT
))
2305 qp
->ack_timeout
= params
->ack_timeout
;
2306 if (GET_FIELD(params
->modify_flags
, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT
))
2307 qp
->retry_cnt
= params
->retry_cnt
;
2308 if (GET_FIELD(params
->modify_flags
,
2309 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT
))
2310 qp
->rnr_retry_cnt
= params
->rnr_retry_cnt
;
2311 if (GET_FIELD(params
->modify_flags
,
2312 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER
))
2313 qp
->min_rnr_nak_timer
= params
->min_rnr_nak_timer
;
2315 qp
->sqd_async
= params
->sqd_async
;
2317 prev_state
= qp
->cur_state
;
2318 if (GET_FIELD(params
->modify_flags
,
2319 QED_RDMA_MODIFY_QP_VALID_NEW_STATE
)) {
2320 qp
->cur_state
= params
->new_state
;
2321 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "qp->cur_state=%d\n",
2325 rc
= qed_roce_modify_qp(p_hwfn
, qp
, prev_state
, params
);
2327 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Modify QP, rc = %d\n", rc
);
2332 qed_rdma_register_tid(void *rdma_cxt
,
2333 struct qed_rdma_register_tid_in_params
*params
)
2335 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2336 struct rdma_register_tid_ramrod_data
*p_ramrod
;
2337 struct qed_sp_init_data init_data
;
2338 struct qed_spq_entry
*p_ent
;
2339 enum rdma_tid_type tid_type
;
2343 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "itid = %08x\n", params
->itid
);
2346 memset(&init_data
, 0, sizeof(init_data
));
2347 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
2348 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
2350 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, RDMA_RAMROD_REGISTER_MR
,
2351 p_hwfn
->p_rdma_info
->proto
, &init_data
);
2353 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "rc = %d\n", rc
);
2357 if (p_hwfn
->p_rdma_info
->last_tid
< params
->itid
)
2358 p_hwfn
->p_rdma_info
->last_tid
= params
->itid
;
2360 p_ramrod
= &p_ent
->ramrod
.rdma_register_tid
;
2362 p_ramrod
->flags
= 0;
2363 SET_FIELD(p_ramrod
->flags
,
2364 RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL
,
2365 params
->pbl_two_level
);
2367 SET_FIELD(p_ramrod
->flags
,
2368 RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED
, params
->zbva
);
2370 SET_FIELD(p_ramrod
->flags
,
2371 RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR
, params
->phy_mr
);
2373 /* Don't initialize D/C field, as it may override other bits. */
2374 if (!(params
->tid_type
== QED_RDMA_TID_FMR
) && !(params
->dma_mr
))
2375 SET_FIELD(p_ramrod
->flags
,
2376 RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG
,
2377 params
->page_size_log
- 12);
2379 SET_FIELD(p_ramrod
->flags
,
2380 RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID
,
2381 p_hwfn
->p_rdma_info
->last_tid
);
2383 SET_FIELD(p_ramrod
->flags
,
2384 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ
,
2385 params
->remote_read
);
2387 SET_FIELD(p_ramrod
->flags
,
2388 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE
,
2389 params
->remote_write
);
2391 SET_FIELD(p_ramrod
->flags
,
2392 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC
,
2393 params
->remote_atomic
);
2395 SET_FIELD(p_ramrod
->flags
,
2396 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE
,
2397 params
->local_write
);
2399 SET_FIELD(p_ramrod
->flags
,
2400 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ
, params
->local_read
);
2402 SET_FIELD(p_ramrod
->flags
,
2403 RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND
,
2406 SET_FIELD(p_ramrod
->flags1
,
2407 RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG
,
2408 params
->pbl_page_size_log
- 12);
2410 SET_FIELD(p_ramrod
->flags2
,
2411 RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR
, params
->dma_mr
);
2413 switch (params
->tid_type
) {
2414 case QED_RDMA_TID_REGISTERED_MR
:
2415 tid_type
= RDMA_TID_REGISTERED_MR
;
2417 case QED_RDMA_TID_FMR
:
2418 tid_type
= RDMA_TID_FMR
;
2420 case QED_RDMA_TID_MW_TYPE1
:
2421 tid_type
= RDMA_TID_MW_TYPE1
;
2423 case QED_RDMA_TID_MW_TYPE2A
:
2424 tid_type
= RDMA_TID_MW_TYPE2A
;
2428 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "rc = %d\n", rc
);
2431 SET_FIELD(p_ramrod
->flags1
,
2432 RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE
, tid_type
);
2434 p_ramrod
->itid
= cpu_to_le32(params
->itid
);
2435 p_ramrod
->key
= params
->key
;
2436 p_ramrod
->pd
= cpu_to_le16(params
->pd
);
2437 p_ramrod
->length_hi
= (u8
)(params
->length
>> 32);
2438 p_ramrod
->length_lo
= DMA_LO_LE(params
->length
);
2440 /* Lower 32 bits of the registered MR address.
2441 * In case of zero based MR, will hold FBO
2443 p_ramrod
->va
.hi
= 0;
2444 p_ramrod
->va
.lo
= cpu_to_le32(params
->fbo
);
2446 DMA_REGPAIR_LE(p_ramrod
->va
, params
->vaddr
);
2448 DMA_REGPAIR_LE(p_ramrod
->pbl_base
, params
->pbl_ptr
);
2451 if (params
->dif_enabled
) {
2452 SET_FIELD(p_ramrod
->flags2
,
2453 RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG
, 1);
2454 DMA_REGPAIR_LE(p_ramrod
->dif_error_addr
,
2455 params
->dif_error_addr
);
2456 DMA_REGPAIR_LE(p_ramrod
->dif_runt_addr
, params
->dif_runt_addr
);
2459 rc
= qed_spq_post(p_hwfn
, p_ent
, &fw_return_code
);
2463 if (fw_return_code
!= RDMA_RETURN_OK
) {
2464 DP_NOTICE(p_hwfn
, "fw_return_code = %d\n", fw_return_code
);
2468 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "Register TID, rc = %d\n", rc
);
2472 static int qed_rdma_deregister_tid(void *rdma_cxt
, u32 itid
)
2474 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2475 struct rdma_deregister_tid_ramrod_data
*p_ramrod
;
2476 struct qed_sp_init_data init_data
;
2477 struct qed_spq_entry
*p_ent
;
2478 struct qed_ptt
*p_ptt
;
2482 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "itid = %08x\n", itid
);
2485 memset(&init_data
, 0, sizeof(init_data
));
2486 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
2487 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
2489 rc
= qed_sp_init_request(p_hwfn
, &p_ent
, RDMA_RAMROD_DEREGISTER_MR
,
2490 p_hwfn
->p_rdma_info
->proto
, &init_data
);
2492 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "rc = %d\n", rc
);
2496 p_ramrod
= &p_ent
->ramrod
.rdma_deregister_tid
;
2497 p_ramrod
->itid
= cpu_to_le32(itid
);
2499 rc
= qed_spq_post(p_hwfn
, p_ent
, &fw_return_code
);
2501 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "rc = %d\n", rc
);
2505 if (fw_return_code
== RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR
) {
2506 DP_NOTICE(p_hwfn
, "fw_return_code = %d\n", fw_return_code
);
2508 } else if (fw_return_code
== RDMA_RETURN_NIG_DRAIN_REQ
) {
2509 /* Bit indicating that the TID is in use and a nig drain is
2510 * required before sending the ramrod again
2512 p_ptt
= qed_ptt_acquire(p_hwfn
);
2515 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
2516 "Failed to acquire PTT\n");
2520 rc
= qed_mcp_drain(p_hwfn
, p_ptt
);
2522 qed_ptt_release(p_hwfn
, p_ptt
);
2523 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
2528 qed_ptt_release(p_hwfn
, p_ptt
);
2530 /* Resend the ramrod */
2531 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
2532 RDMA_RAMROD_DEREGISTER_MR
,
2533 p_hwfn
->p_rdma_info
->proto
,
2536 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
2537 "Failed to init sp-element\n");
2541 rc
= qed_spq_post(p_hwfn
, p_ent
, &fw_return_code
);
2543 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
2548 if (fw_return_code
!= RDMA_RETURN_OK
) {
2549 DP_NOTICE(p_hwfn
, "fw_return_code = %d\n",
2555 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "De-registered TID, rc = %d\n", rc
);
2559 static void qed_roce_free_real_icid(struct qed_hwfn
*p_hwfn
, u16 icid
)
2561 struct qed_rdma_info
*p_rdma_info
= p_hwfn
->p_rdma_info
;
2562 u32 start_cid
, cid
, xcid
;
2564 /* an even icid belongs to a responder while an odd icid belongs to a
2565 * requester. The 'cid' received as an input can be either. We calculate
2566 * the "partner" icid and call it xcid. Only if both are free then the
2567 * "cid" map can be cleared.
2569 start_cid
= qed_cxt_get_proto_cid_start(p_hwfn
, p_rdma_info
->proto
);
2570 cid
= icid
- start_cid
;
2573 spin_lock_bh(&p_rdma_info
->lock
);
2575 qed_bmap_release_id(p_hwfn
, &p_rdma_info
->real_cid_map
, cid
);
2576 if (qed_bmap_test_id(p_hwfn
, &p_rdma_info
->real_cid_map
, xcid
) == 0) {
2577 qed_bmap_release_id(p_hwfn
, &p_rdma_info
->cid_map
, cid
);
2578 qed_bmap_release_id(p_hwfn
, &p_rdma_info
->cid_map
, xcid
);
2581 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
2584 static void *qed_rdma_get_rdma_ctx(struct qed_dev
*cdev
)
2586 return QED_LEADING_HWFN(cdev
);
2589 static void qed_rdma_dpm_conf(struct qed_hwfn
*p_hwfn
, struct qed_ptt
*p_ptt
)
2593 val
= (p_hwfn
->dcbx_no_edpm
|| p_hwfn
->db_bar_no_edpm
) ? 0 : 1;
2595 qed_wr(p_hwfn
, p_ptt
, DORQ_REG_PF_DPM_ENABLE
, val
);
2596 DP_VERBOSE(p_hwfn
, (QED_MSG_DCB
| QED_MSG_RDMA
),
2597 "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n",
2598 val
, p_hwfn
->dcbx_no_edpm
, p_hwfn
->db_bar_no_edpm
);
2601 void qed_rdma_dpm_bar(struct qed_hwfn
*p_hwfn
, struct qed_ptt
*p_ptt
)
2603 p_hwfn
->db_bar_no_edpm
= true;
2605 qed_rdma_dpm_conf(p_hwfn
, p_ptt
);
2608 static int qed_rdma_start(void *rdma_cxt
,
2609 struct qed_rdma_start_in_params
*params
)
2611 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2612 struct qed_ptt
*p_ptt
;
2615 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
,
2616 "desired_cnq = %08x\n", params
->desired_cnq
);
2618 p_ptt
= qed_ptt_acquire(p_hwfn
);
2622 rc
= qed_rdma_alloc(p_hwfn
, p_ptt
, params
);
2626 rc
= qed_rdma_setup(p_hwfn
, p_ptt
, params
);
2630 qed_ptt_release(p_hwfn
, p_ptt
);
2635 qed_rdma_free(p_hwfn
);
2637 qed_ptt_release(p_hwfn
, p_ptt
);
2639 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "RDMA start - error, rc = %d\n", rc
);
2643 static int qed_rdma_init(struct qed_dev
*cdev
,
2644 struct qed_rdma_start_in_params
*params
)
2646 return qed_rdma_start(QED_LEADING_HWFN(cdev
), params
);
2649 static void qed_rdma_remove_user(void *rdma_cxt
, u16 dpi
)
2651 struct qed_hwfn
*p_hwfn
= (struct qed_hwfn
*)rdma_cxt
;
2653 DP_VERBOSE(p_hwfn
, QED_MSG_RDMA
, "dpi = %08x\n", dpi
);
2655 spin_lock_bh(&p_hwfn
->p_rdma_info
->lock
);
2656 qed_bmap_release_id(p_hwfn
, &p_hwfn
->p_rdma_info
->dpi_map
, dpi
);
2657 spin_unlock_bh(&p_hwfn
->p_rdma_info
->lock
);
2660 void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn
*p_hwfn
,
2661 u8 connection_handle
,
2663 dma_addr_t first_frag_addr
,
2664 bool b_last_fragment
, bool b_last_packet
)
2666 struct qed_roce_ll2_packet
*packet
= cookie
;
2667 struct qed_roce_ll2_info
*roce_ll2
= p_hwfn
->ll2
;
2669 roce_ll2
->cbs
.tx_cb(roce_ll2
->cb_cookie
, packet
);
2672 void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn
*p_hwfn
,
2673 u8 connection_handle
,
2675 dma_addr_t first_frag_addr
,
2676 bool b_last_fragment
, bool b_last_packet
)
2678 qed_ll2b_complete_tx_gsi_packet(p_hwfn
, connection_handle
,
2679 cookie
, first_frag_addr
,
2680 b_last_fragment
, b_last_packet
);
2683 void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn
*p_hwfn
,
2684 u8 connection_handle
,
2686 dma_addr_t rx_buf_addr
,
2688 u8 data_length_error
,
2691 u32 src_mac_addr_hi
,
2692 u16 src_mac_addr_lo
, bool b_last_packet
)
2694 struct qed_roce_ll2_info
*roce_ll2
= p_hwfn
->ll2
;
2695 struct qed_roce_ll2_rx_params params
;
2696 struct qed_dev
*cdev
= p_hwfn
->cdev
;
2697 struct qed_roce_ll2_packet pkt
;
2701 "roce ll2 rx complete: bus_addr=%p, len=%d, data_len_err=%d\n",
2702 (void *)(uintptr_t)rx_buf_addr
,
2703 data_length
, data_length_error
);
2705 memset(&pkt
, 0, sizeof(pkt
));
2707 pkt
.payload
[0].baddr
= rx_buf_addr
;
2708 pkt
.payload
[0].len
= data_length
;
2710 memset(¶ms
, 0, sizeof(params
));
2711 params
.vlan_id
= vlan
;
2712 *((u32
*)¶ms
.smac
[0]) = ntohl(src_mac_addr_hi
);
2713 *((u16
*)¶ms
.smac
[4]) = ntohs(src_mac_addr_lo
);
2715 if (data_length_error
) {
2717 "roce ll2 rx complete: data length error %d, length=%d\n",
2718 data_length_error
, data_length
);
2719 params
.rc
= -EINVAL
;
2722 roce_ll2
->cbs
.rx_cb(roce_ll2
->cb_cookie
, &pkt
, ¶ms
);
2725 static int qed_roce_ll2_set_mac_filter(struct qed_dev
*cdev
,
2726 u8
*old_mac_address
,
2727 u8
*new_mac_address
)
2729 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
2730 struct qed_ptt
*p_ptt
;
2733 if (!hwfn
->ll2
|| hwfn
->ll2
->handle
== QED_LL2_UNUSED_HANDLE
) {
2735 "qed roce mac filter failed - roce_info/ll2 NULL\n");
2739 p_ptt
= qed_ptt_acquire(QED_LEADING_HWFN(cdev
));
2742 "qed roce ll2 mac filter set: failed to acquire PTT\n");
2746 mutex_lock(&hwfn
->ll2
->lock
);
2747 if (old_mac_address
)
2748 qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev
), p_ptt
,
2750 if (new_mac_address
)
2751 rc
= qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev
), p_ptt
,
2753 mutex_unlock(&hwfn
->ll2
->lock
);
2755 qed_ptt_release(QED_LEADING_HWFN(cdev
), p_ptt
);
2759 "qed roce ll2 mac filter set: failed to add mac filter\n");
2764 static int qed_roce_ll2_start(struct qed_dev
*cdev
,
2765 struct qed_roce_ll2_params
*params
)
2767 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
2768 struct qed_roce_ll2_info
*roce_ll2
;
2769 struct qed_ll2_conn ll2_params
;
2773 DP_ERR(cdev
, "qed roce ll2 start: failed due to NULL params\n");
2776 if (!params
->cbs
.tx_cb
|| !params
->cbs
.rx_cb
) {
2778 "qed roce ll2 start: failed due to NULL tx/rx. tx_cb=%p, rx_cb=%p\n",
2779 params
->cbs
.tx_cb
, params
->cbs
.rx_cb
);
2782 if (!is_valid_ether_addr(params
->mac_address
)) {
2784 "qed roce ll2 start: failed due to invalid Ethernet address %pM\n",
2785 params
->mac_address
);
2790 roce_ll2
= kzalloc(sizeof(*roce_ll2
), GFP_ATOMIC
);
2792 DP_ERR(cdev
, "qed roce ll2 start: failed memory allocation\n");
2795 roce_ll2
->handle
= QED_LL2_UNUSED_HANDLE
;
2796 roce_ll2
->cbs
= params
->cbs
;
2797 roce_ll2
->cb_cookie
= params
->cb_cookie
;
2798 mutex_init(&roce_ll2
->lock
);
2800 memset(&ll2_params
, 0, sizeof(ll2_params
));
2801 ll2_params
.conn_type
= QED_LL2_TYPE_ROCE
;
2802 ll2_params
.mtu
= params
->mtu
;
2803 ll2_params
.rx_drop_ttl0_flg
= true;
2804 ll2_params
.rx_vlan_removal_en
= false;
2805 ll2_params
.tx_dest
= CORE_TX_DEST_NW
;
2806 ll2_params
.ai_err_packet_too_big
= LL2_DROP_PACKET
;
2807 ll2_params
.ai_err_no_buf
= LL2_DROP_PACKET
;
2808 ll2_params
.gsi_enable
= true;
2810 rc
= qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev
), &ll2_params
,
2811 params
->max_rx_buffers
,
2812 params
->max_tx_buffers
,
2816 "qed roce ll2 start: failed to acquire LL2 connection (rc=%d)\n",
2821 rc
= qed_ll2_establish_connection(QED_LEADING_HWFN(cdev
),
2825 "qed roce ll2 start: failed to establish LL2 connection (rc=%d)\n",
2830 hwfn
->ll2
= roce_ll2
;
2832 rc
= qed_roce_ll2_set_mac_filter(cdev
, NULL
, params
->mac_address
);
2837 ether_addr_copy(roce_ll2
->mac_address
, params
->mac_address
);
2842 qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev
), roce_ll2
->handle
);
2844 qed_ll2_release_connection(QED_LEADING_HWFN(cdev
), roce_ll2
->handle
);
2850 static int qed_roce_ll2_stop(struct qed_dev
*cdev
)
2852 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
2853 struct qed_roce_ll2_info
*roce_ll2
= hwfn
->ll2
;
2856 if (roce_ll2
->handle
== QED_LL2_UNUSED_HANDLE
) {
2857 DP_ERR(cdev
, "qed roce ll2 stop: cannot stop an unused LL2\n");
2861 /* remove LL2 MAC address filter */
2862 rc
= qed_roce_ll2_set_mac_filter(cdev
, roce_ll2
->mac_address
, NULL
);
2863 eth_zero_addr(roce_ll2
->mac_address
);
2865 rc
= qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev
),
2869 "qed roce ll2 stop: failed to terminate LL2 connection (rc=%d)\n",
2872 qed_ll2_release_connection(QED_LEADING_HWFN(cdev
), roce_ll2
->handle
);
2874 roce_ll2
->handle
= QED_LL2_UNUSED_HANDLE
;
2881 static int qed_roce_ll2_tx(struct qed_dev
*cdev
,
2882 struct qed_roce_ll2_packet
*pkt
,
2883 struct qed_roce_ll2_tx_params
*params
)
2885 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
2886 struct qed_roce_ll2_info
*roce_ll2
= hwfn
->ll2
;
2887 enum qed_ll2_roce_flavor_type qed_roce_flavor
;
2892 if (!pkt
|| !params
) {
2894 "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n",
2899 qed_roce_flavor
= (pkt
->roce_mode
== ROCE_V1
) ? QED_LL2_ROCE
2902 if (pkt
->roce_mode
== ROCE_V2_IPV4
)
2903 flags
|= BIT(CORE_TX_BD_DATA_IP_CSUM_SHIFT
);
2906 rc
= qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev
), roce_ll2
->handle
,
2907 1 + pkt
->n_seg
, 0, flags
, 0,
2909 qed_roce_flavor
, pkt
->header
.baddr
,
2910 pkt
->header
.len
, pkt
, 1);
2912 DP_ERR(cdev
, "roce ll2 tx: header failed (rc=%d)\n", rc
);
2913 return QED_ROCE_TX_HEAD_FAILURE
;
2917 for (i
= 0; i
< pkt
->n_seg
; i
++) {
2918 rc
= qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev
),
2920 pkt
->payload
[i
].baddr
,
2921 pkt
->payload
[i
].len
);
2923 /* If failed not much to do here, partial packet has
2924 * been posted * we can't free memory, will need to wait
2928 "roce ll2 tx: payload failed (rc=%d)\n", rc
);
2929 return QED_ROCE_TX_FRAG_FAILURE
;
2936 static int qed_roce_ll2_post_rx_buffer(struct qed_dev
*cdev
,
2937 struct qed_roce_ll2_buffer
*buf
,
2938 u64 cookie
, u8 notify_fw
)
2940 return qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev
),
2941 QED_LEADING_HWFN(cdev
)->ll2
->handle
,
2942 buf
->baddr
, buf
->len
,
2943 (void *)(uintptr_t)cookie
, notify_fw
);
2946 static int qed_roce_ll2_stats(struct qed_dev
*cdev
, struct qed_ll2_stats
*stats
)
2948 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
2949 struct qed_roce_ll2_info
*roce_ll2
= hwfn
->ll2
;
2951 return qed_ll2_get_stats(QED_LEADING_HWFN(cdev
),
2952 roce_ll2
->handle
, stats
);
2955 static const struct qed_rdma_ops qed_rdma_ops_pass
= {
2956 .common
= &qed_common_ops_pass
,
2957 .fill_dev_info
= &qed_fill_rdma_dev_info
,
2958 .rdma_get_rdma_ctx
= &qed_rdma_get_rdma_ctx
,
2959 .rdma_init
= &qed_rdma_init
,
2960 .rdma_add_user
= &qed_rdma_add_user
,
2961 .rdma_remove_user
= &qed_rdma_remove_user
,
2962 .rdma_stop
= &qed_rdma_stop
,
2963 .rdma_query_port
= &qed_rdma_query_port
,
2964 .rdma_query_device
= &qed_rdma_query_device
,
2965 .rdma_get_start_sb
= &qed_rdma_get_sb_start
,
2966 .rdma_get_rdma_int
= &qed_rdma_get_int
,
2967 .rdma_set_rdma_int
= &qed_rdma_set_int
,
2968 .rdma_get_min_cnq_msix
= &qed_rdma_get_min_cnq_msix
,
2969 .rdma_cnq_prod_update
= &qed_rdma_cnq_prod_update
,
2970 .rdma_alloc_pd
= &qed_rdma_alloc_pd
,
2971 .rdma_dealloc_pd
= &qed_rdma_free_pd
,
2972 .rdma_create_cq
= &qed_rdma_create_cq
,
2973 .rdma_destroy_cq
= &qed_rdma_destroy_cq
,
2974 .rdma_create_qp
= &qed_rdma_create_qp
,
2975 .rdma_modify_qp
= &qed_rdma_modify_qp
,
2976 .rdma_query_qp
= &qed_rdma_query_qp
,
2977 .rdma_destroy_qp
= &qed_rdma_destroy_qp
,
2978 .rdma_alloc_tid
= &qed_rdma_alloc_tid
,
2979 .rdma_free_tid
= &qed_rdma_free_tid
,
2980 .rdma_register_tid
= &qed_rdma_register_tid
,
2981 .rdma_deregister_tid
= &qed_rdma_deregister_tid
,
2982 .roce_ll2_start
= &qed_roce_ll2_start
,
2983 .roce_ll2_stop
= &qed_roce_ll2_stop
,
2984 .roce_ll2_tx
= &qed_roce_ll2_tx
,
2985 .roce_ll2_post_rx_buffer
= &qed_roce_ll2_post_rx_buffer
,
2986 .roce_ll2_set_mac_filter
= &qed_roce_ll2_set_mac_filter
,
2987 .roce_ll2_stats
= &qed_roce_ll2_stats
,
2990 const struct qed_rdma_ops
*qed_get_rdma_ops(void)
2992 return &qed_rdma_ops_pass
;
2994 EXPORT_SYMBOL(qed_get_rdma_ops
);