1 /* QLogic qed NIC Driver
2 * Copyright (c) 2015-2017 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/types.h>
34 #include <asm/byteorder.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/if_vlan.h>
37 #include <linux/kernel.h>
38 #include <linux/pci.h>
39 #include <linux/slab.h>
40 #include <linux/stddef.h>
41 #include <linux/workqueue.h>
43 #include <linux/bitops.h>
44 #include <linux/delay.h>
45 #include <linux/errno.h>
46 #include <linux/etherdevice.h>
48 #include <linux/list.h>
49 #include <linux/mutex.h>
50 #include <linux/spinlock.h>
51 #include <linux/string.h>
52 #include <linux/qed/qed_ll2_if.h>
55 #include "qed_dev_api.h"
62 #include "qed_reg_addr.h"
66 #define QED_LL2_RX_REGISTERED(ll2) ((ll2)->rx_queue.b_cb_registred)
67 #define QED_LL2_TX_REGISTERED(ll2) ((ll2)->tx_queue.b_cb_registred)
69 #define QED_LL2_TX_SIZE (256)
70 #define QED_LL2_RX_SIZE (4096)
72 struct qed_cb_ll2_info
{
77 /* Lock protecting LL2 buffer lists in sleepless context */
79 struct list_head list
;
81 const struct qed_ll2_cb_ops
*cbs
;
85 struct qed_ll2_buffer
{
86 struct list_head list
;
91 static void qed_ll2b_complete_tx_packet(void *cxt
,
94 dma_addr_t first_frag_addr
,
98 struct qed_hwfn
*p_hwfn
= cxt
;
99 struct qed_dev
*cdev
= p_hwfn
->cdev
;
100 struct sk_buff
*skb
= cookie
;
102 /* All we need to do is release the mapping */
103 dma_unmap_single(&p_hwfn
->cdev
->pdev
->dev
, first_frag_addr
,
104 skb_headlen(skb
), DMA_TO_DEVICE
);
106 if (cdev
->ll2
->cbs
&& cdev
->ll2
->cbs
->tx_cb
)
107 cdev
->ll2
->cbs
->tx_cb(cdev
->ll2
->cb_cookie
, skb
,
110 dev_kfree_skb_any(skb
);
113 static int qed_ll2_alloc_buffer(struct qed_dev
*cdev
,
114 u8
**data
, dma_addr_t
*phys_addr
)
116 *data
= kmalloc(cdev
->ll2
->rx_size
, GFP_ATOMIC
);
118 DP_INFO(cdev
, "Failed to allocate LL2 buffer data\n");
122 *phys_addr
= dma_map_single(&cdev
->pdev
->dev
,
123 ((*data
) + NET_SKB_PAD
),
124 cdev
->ll2
->rx_size
, DMA_FROM_DEVICE
);
125 if (dma_mapping_error(&cdev
->pdev
->dev
, *phys_addr
)) {
126 DP_INFO(cdev
, "Failed to map LL2 buffer data\n");
134 static int qed_ll2_dealloc_buffer(struct qed_dev
*cdev
,
135 struct qed_ll2_buffer
*buffer
)
137 spin_lock_bh(&cdev
->ll2
->lock
);
139 dma_unmap_single(&cdev
->pdev
->dev
, buffer
->phys_addr
,
140 cdev
->ll2
->rx_size
, DMA_FROM_DEVICE
);
142 list_del(&buffer
->list
);
145 if (!cdev
->ll2
->rx_cnt
)
146 DP_INFO(cdev
, "All LL2 entries were removed\n");
148 spin_unlock_bh(&cdev
->ll2
->lock
);
153 static void qed_ll2_kill_buffers(struct qed_dev
*cdev
)
155 struct qed_ll2_buffer
*buffer
, *tmp_buffer
;
157 list_for_each_entry_safe(buffer
, tmp_buffer
, &cdev
->ll2
->list
, list
)
158 qed_ll2_dealloc_buffer(cdev
, buffer
);
161 void qed_ll2b_complete_rx_packet(void *cxt
, struct qed_ll2_comp_rx_data
*data
)
163 struct qed_hwfn
*p_hwfn
= cxt
;
164 struct qed_ll2_buffer
*buffer
= data
->cookie
;
165 struct qed_dev
*cdev
= p_hwfn
->cdev
;
166 dma_addr_t new_phys_addr
;
173 (NETIF_MSG_RX_STATUS
| QED_MSG_STORAGE
| NETIF_MSG_PKTDATA
),
174 "Got an LL2 Rx completion: [Buffer at phys 0x%llx, offset 0x%02x] Length 0x%04x Parse_flags 0x%04x vlan 0x%04x Opaque data [0x%08x:0x%08x]\n",
175 (u64
)data
->rx_buf_addr
,
176 data
->u
.placement_offset
,
177 data
->length
.packet_length
,
179 data
->vlan
, data
->opaque_data_0
, data
->opaque_data_1
);
181 if ((cdev
->dp_module
& NETIF_MSG_PKTDATA
) && buffer
->data
) {
182 print_hex_dump(KERN_INFO
, "",
183 DUMP_PREFIX_OFFSET
, 16, 1,
184 buffer
->data
, data
->length
.packet_length
, false);
187 /* Determine if data is valid */
188 if (data
->length
.packet_length
< ETH_HLEN
)
191 /* Allocate a replacement for buffer; Reuse upon failure */
193 rc
= qed_ll2_alloc_buffer(p_hwfn
->cdev
, &new_data
,
196 /* If need to reuse or there's no replacement buffer, repost this */
199 dma_unmap_single(&cdev
->pdev
->dev
, buffer
->phys_addr
,
200 cdev
->ll2
->rx_size
, DMA_FROM_DEVICE
);
202 skb
= build_skb(buffer
->data
, 0);
208 data
->u
.placement_offset
+= NET_SKB_PAD
;
209 skb_reserve(skb
, data
->u
.placement_offset
);
210 skb_put(skb
, data
->length
.packet_length
);
211 skb_checksum_none_assert(skb
);
213 /* Get parital ethernet information instead of eth_type_trans(),
214 * Since we don't have an associated net_device.
216 skb_reset_mac_header(skb
);
217 skb
->protocol
= eth_hdr(skb
)->h_proto
;
219 /* Pass SKB onward */
220 if (cdev
->ll2
->cbs
&& cdev
->ll2
->cbs
->rx_cb
) {
222 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
224 cdev
->ll2
->cbs
->rx_cb(cdev
->ll2
->cb_cookie
, skb
,
226 data
->opaque_data_1
);
229 /* Update Buffer information and update FW producer */
230 buffer
->data
= new_data
;
231 buffer
->phys_addr
= new_phys_addr
;
234 rc
= qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev
), cdev
->ll2
->handle
,
235 buffer
->phys_addr
, 0, buffer
, 1);
238 qed_ll2_dealloc_buffer(cdev
, buffer
);
241 static struct qed_ll2_info
*__qed_ll2_handle_sanity(struct qed_hwfn
*p_hwfn
,
242 u8 connection_handle
,
246 struct qed_ll2_info
*p_ll2_conn
, *p_ret
= NULL
;
248 if (connection_handle
>= QED_MAX_NUM_OF_LL2_CONNECTIONS
)
251 if (!p_hwfn
->p_ll2_info
)
254 p_ll2_conn
= &p_hwfn
->p_ll2_info
[connection_handle
];
258 mutex_lock(&p_ll2_conn
->mutex
);
259 if (p_ll2_conn
->b_active
)
262 mutex_unlock(&p_ll2_conn
->mutex
);
270 static struct qed_ll2_info
*qed_ll2_handle_sanity(struct qed_hwfn
*p_hwfn
,
271 u8 connection_handle
)
273 return __qed_ll2_handle_sanity(p_hwfn
, connection_handle
, false, true);
276 static struct qed_ll2_info
*qed_ll2_handle_sanity_lock(struct qed_hwfn
*p_hwfn
,
277 u8 connection_handle
)
279 return __qed_ll2_handle_sanity(p_hwfn
, connection_handle
, true, true);
282 static struct qed_ll2_info
*qed_ll2_handle_sanity_inactive(struct qed_hwfn
284 u8 connection_handle
)
286 return __qed_ll2_handle_sanity(p_hwfn
, connection_handle
, false, false);
289 static void qed_ll2_txq_flush(struct qed_hwfn
*p_hwfn
, u8 connection_handle
)
291 bool b_last_packet
= false, b_last_frag
= false;
292 struct qed_ll2_tx_packet
*p_pkt
= NULL
;
293 struct qed_ll2_info
*p_ll2_conn
;
294 struct qed_ll2_tx_queue
*p_tx
;
297 p_ll2_conn
= qed_ll2_handle_sanity_inactive(p_hwfn
, connection_handle
);
301 p_tx
= &p_ll2_conn
->tx_queue
;
303 while (!list_empty(&p_tx
->active_descq
)) {
304 p_pkt
= list_first_entry(&p_tx
->active_descq
,
305 struct qed_ll2_tx_packet
, list_entry
);
309 list_del(&p_pkt
->list_entry
);
310 b_last_packet
= list_empty(&p_tx
->active_descq
);
311 list_add_tail(&p_pkt
->list_entry
, &p_tx
->free_descq
);
312 if (p_ll2_conn
->input
.conn_type
== QED_LL2_TYPE_ISCSI_OOO
) {
313 struct qed_ooo_buffer
*p_buffer
;
315 p_buffer
= (struct qed_ooo_buffer
*)p_pkt
->cookie
;
316 qed_ooo_put_free_buffer(p_hwfn
, p_hwfn
->p_ooo_info
,
319 p_tx
->cur_completing_packet
= *p_pkt
;
320 p_tx
->cur_completing_bd_idx
= 1;
322 p_tx
->cur_completing_bd_idx
== p_pkt
->bd_used
;
323 tx_frag
= p_pkt
->bds_set
[0].tx_frag
;
324 p_ll2_conn
->cbs
.tx_release_cb(p_ll2_conn
->cbs
.cookie
,
334 static int qed_ll2_txq_completion(struct qed_hwfn
*p_hwfn
, void *p_cookie
)
336 struct qed_ll2_info
*p_ll2_conn
= p_cookie
;
337 struct qed_ll2_tx_queue
*p_tx
= &p_ll2_conn
->tx_queue
;
338 u16 new_idx
= 0, num_bds
= 0, num_bds_in_packet
= 0;
339 struct qed_ll2_tx_packet
*p_pkt
;
340 bool b_last_frag
= false;
344 spin_lock_irqsave(&p_tx
->lock
, flags
);
345 if (p_tx
->b_completing_packet
) {
350 new_idx
= le16_to_cpu(*p_tx
->p_fw_cons
);
351 num_bds
= ((s16
)new_idx
- (s16
)p_tx
->bds_idx
);
353 if (list_empty(&p_tx
->active_descq
))
356 p_pkt
= list_first_entry(&p_tx
->active_descq
,
357 struct qed_ll2_tx_packet
, list_entry
);
361 p_tx
->b_completing_packet
= true;
362 p_tx
->cur_completing_packet
= *p_pkt
;
363 num_bds_in_packet
= p_pkt
->bd_used
;
364 list_del(&p_pkt
->list_entry
);
366 if (num_bds
< num_bds_in_packet
) {
368 "Rest of BDs does not cover whole packet\n");
372 num_bds
-= num_bds_in_packet
;
373 p_tx
->bds_idx
+= num_bds_in_packet
;
374 while (num_bds_in_packet
--)
375 qed_chain_consume(&p_tx
->txq_chain
);
377 p_tx
->cur_completing_bd_idx
= 1;
378 b_last_frag
= p_tx
->cur_completing_bd_idx
== p_pkt
->bd_used
;
379 list_add_tail(&p_pkt
->list_entry
, &p_tx
->free_descq
);
381 spin_unlock_irqrestore(&p_tx
->lock
, flags
);
383 p_ll2_conn
->cbs
.tx_comp_cb(p_ll2_conn
->cbs
.cookie
,
386 p_pkt
->bds_set
[0].tx_frag
,
387 b_last_frag
, !num_bds
);
389 spin_lock_irqsave(&p_tx
->lock
, flags
);
392 p_tx
->b_completing_packet
= false;
395 spin_unlock_irqrestore(&p_tx
->lock
, flags
);
399 static void qed_ll2_rxq_parse_gsi(struct qed_hwfn
*p_hwfn
,
400 union core_rx_cqe_union
*p_cqe
,
401 struct qed_ll2_comp_rx_data
*data
)
403 data
->parse_flags
= le16_to_cpu(p_cqe
->rx_cqe_gsi
.parse_flags
.flags
);
404 data
->length
.data_length
= le16_to_cpu(p_cqe
->rx_cqe_gsi
.data_length
);
405 data
->vlan
= le16_to_cpu(p_cqe
->rx_cqe_gsi
.vlan
);
406 data
->opaque_data_0
= le32_to_cpu(p_cqe
->rx_cqe_gsi
.src_mac_addrhi
);
407 data
->opaque_data_1
= le16_to_cpu(p_cqe
->rx_cqe_gsi
.src_mac_addrlo
);
408 data
->u
.data_length_error
= p_cqe
->rx_cqe_gsi
.data_length_error
;
411 static void qed_ll2_rxq_parse_reg(struct qed_hwfn
*p_hwfn
,
412 union core_rx_cqe_union
*p_cqe
,
413 struct qed_ll2_comp_rx_data
*data
)
415 data
->parse_flags
= le16_to_cpu(p_cqe
->rx_cqe_fp
.parse_flags
.flags
);
416 data
->length
.packet_length
=
417 le16_to_cpu(p_cqe
->rx_cqe_fp
.packet_length
);
418 data
->vlan
= le16_to_cpu(p_cqe
->rx_cqe_fp
.vlan
);
419 data
->opaque_data_0
= le32_to_cpu(p_cqe
->rx_cqe_fp
.opaque_data
.data
[0]);
420 data
->opaque_data_1
= le32_to_cpu(p_cqe
->rx_cqe_fp
.opaque_data
.data
[1]);
421 data
->u
.placement_offset
= p_cqe
->rx_cqe_fp
.placement_offset
;
425 qed_ll2_rxq_handle_completion(struct qed_hwfn
*p_hwfn
,
426 struct qed_ll2_info
*p_ll2_conn
,
427 union core_rx_cqe_union
*p_cqe
,
428 unsigned long *p_lock_flags
, bool b_last_cqe
)
430 struct qed_ll2_rx_queue
*p_rx
= &p_ll2_conn
->rx_queue
;
431 struct qed_ll2_rx_packet
*p_pkt
= NULL
;
432 struct qed_ll2_comp_rx_data data
;
434 if (!list_empty(&p_rx
->active_descq
))
435 p_pkt
= list_first_entry(&p_rx
->active_descq
,
436 struct qed_ll2_rx_packet
, list_entry
);
439 "[%d] LL2 Rx completion but active_descq is empty\n",
440 p_ll2_conn
->input
.conn_type
);
444 list_del(&p_pkt
->list_entry
);
446 if (p_cqe
->rx_cqe_sp
.type
== CORE_RX_CQE_TYPE_REGULAR
)
447 qed_ll2_rxq_parse_reg(p_hwfn
, p_cqe
, &data
);
449 qed_ll2_rxq_parse_gsi(p_hwfn
, p_cqe
, &data
);
450 if (qed_chain_consume(&p_rx
->rxq_chain
) != p_pkt
->rxq_bd
)
452 "Mismatch between active_descq and the LL2 Rx chain\n");
454 list_add_tail(&p_pkt
->list_entry
, &p_rx
->free_descq
);
456 data
.connection_handle
= p_ll2_conn
->my_id
;
457 data
.cookie
= p_pkt
->cookie
;
458 data
.rx_buf_addr
= p_pkt
->rx_buf_addr
;
459 data
.b_last_packet
= b_last_cqe
;
461 spin_unlock_irqrestore(&p_rx
->lock
, *p_lock_flags
);
462 p_ll2_conn
->cbs
.rx_comp_cb(p_ll2_conn
->cbs
.cookie
, &data
);
464 spin_lock_irqsave(&p_rx
->lock
, *p_lock_flags
);
469 static int qed_ll2_rxq_completion(struct qed_hwfn
*p_hwfn
, void *cookie
)
471 struct qed_ll2_info
*p_ll2_conn
= (struct qed_ll2_info
*)cookie
;
472 struct qed_ll2_rx_queue
*p_rx
= &p_ll2_conn
->rx_queue
;
473 union core_rx_cqe_union
*cqe
= NULL
;
474 u16 cq_new_idx
= 0, cq_old_idx
= 0;
475 unsigned long flags
= 0;
478 spin_lock_irqsave(&p_rx
->lock
, flags
);
479 cq_new_idx
= le16_to_cpu(*p_rx
->p_fw_cons
);
480 cq_old_idx
= qed_chain_get_cons_idx(&p_rx
->rcq_chain
);
482 while (cq_new_idx
!= cq_old_idx
) {
483 bool b_last_cqe
= (cq_new_idx
== cq_old_idx
);
486 (union core_rx_cqe_union
*)
487 qed_chain_consume(&p_rx
->rcq_chain
);
488 cq_old_idx
= qed_chain_get_cons_idx(&p_rx
->rcq_chain
);
492 "LL2 [sw. cons %04x, fw. at %04x] - Got Packet of type %02x\n",
493 cq_old_idx
, cq_new_idx
, cqe
->rx_cqe_sp
.type
);
495 switch (cqe
->rx_cqe_sp
.type
) {
496 case CORE_RX_CQE_TYPE_SLOW_PATH
:
497 DP_NOTICE(p_hwfn
, "LL2 - unexpected Rx CQE slowpath\n");
500 case CORE_RX_CQE_TYPE_GSI_OFFLOAD
:
501 case CORE_RX_CQE_TYPE_REGULAR
:
502 rc
= qed_ll2_rxq_handle_completion(p_hwfn
, p_ll2_conn
,
511 spin_unlock_irqrestore(&p_rx
->lock
, flags
);
515 static void qed_ll2_rxq_flush(struct qed_hwfn
*p_hwfn
, u8 connection_handle
)
517 struct qed_ll2_info
*p_ll2_conn
= NULL
;
518 struct qed_ll2_rx_packet
*p_pkt
= NULL
;
519 struct qed_ll2_rx_queue
*p_rx
;
521 p_ll2_conn
= qed_ll2_handle_sanity_inactive(p_hwfn
, connection_handle
);
525 p_rx
= &p_ll2_conn
->rx_queue
;
527 while (!list_empty(&p_rx
->active_descq
)) {
528 p_pkt
= list_first_entry(&p_rx
->active_descq
,
529 struct qed_ll2_rx_packet
, list_entry
);
533 list_move_tail(&p_pkt
->list_entry
, &p_rx
->free_descq
);
535 if (p_ll2_conn
->input
.conn_type
== QED_LL2_TYPE_ISCSI_OOO
) {
536 struct qed_ooo_buffer
*p_buffer
;
538 p_buffer
= (struct qed_ooo_buffer
*)p_pkt
->cookie
;
539 qed_ooo_put_free_buffer(p_hwfn
, p_hwfn
->p_ooo_info
,
542 dma_addr_t rx_buf_addr
= p_pkt
->rx_buf_addr
;
543 void *cookie
= p_pkt
->cookie
;
546 b_last
= list_empty(&p_rx
->active_descq
);
547 p_ll2_conn
->cbs
.rx_release_cb(p_ll2_conn
->cbs
.cookie
,
550 rx_buf_addr
, b_last
);
555 static u8
qed_ll2_convert_rx_parse_to_tx_flags(u16 parse_flags
)
559 if (GET_FIELD(parse_flags
, PARSING_AND_ERR_FLAGS_TAG8021QEXIST
))
560 SET_FIELD(bd_flags
, CORE_TX_BD_DATA_VLAN_INSERTION
, 1);
565 static int qed_ll2_lb_rxq_handler(struct qed_hwfn
*p_hwfn
,
566 struct qed_ll2_info
*p_ll2_conn
)
568 struct qed_ll2_rx_queue
*p_rx
= &p_ll2_conn
->rx_queue
;
569 u16 packet_length
= 0, parse_flags
= 0, vlan
= 0;
570 struct qed_ll2_rx_packet
*p_pkt
= NULL
;
571 u32 num_ooo_add_to_peninsula
= 0, cid
;
572 union core_rx_cqe_union
*cqe
= NULL
;
573 u16 cq_new_idx
= 0, cq_old_idx
= 0;
574 struct qed_ooo_buffer
*p_buffer
;
575 struct ooo_opaque
*iscsi_ooo
;
576 u8 placement_offset
= 0;
579 cq_new_idx
= le16_to_cpu(*p_rx
->p_fw_cons
);
580 cq_old_idx
= qed_chain_get_cons_idx(&p_rx
->rcq_chain
);
581 if (cq_new_idx
== cq_old_idx
)
584 while (cq_new_idx
!= cq_old_idx
) {
585 struct core_rx_fast_path_cqe
*p_cqe_fp
;
587 cqe
= qed_chain_consume(&p_rx
->rcq_chain
);
588 cq_old_idx
= qed_chain_get_cons_idx(&p_rx
->rcq_chain
);
589 cqe_type
= cqe
->rx_cqe_sp
.type
;
591 if (cqe_type
!= CORE_RX_CQE_TYPE_REGULAR
) {
593 "Got a non-regular LB LL2 completion [type 0x%02x]\n",
597 p_cqe_fp
= &cqe
->rx_cqe_fp
;
599 placement_offset
= p_cqe_fp
->placement_offset
;
600 parse_flags
= le16_to_cpu(p_cqe_fp
->parse_flags
.flags
);
601 packet_length
= le16_to_cpu(p_cqe_fp
->packet_length
);
602 vlan
= le16_to_cpu(p_cqe_fp
->vlan
);
603 iscsi_ooo
= (struct ooo_opaque
*)&p_cqe_fp
->opaque_data
;
604 qed_ooo_save_history_entry(p_hwfn
, p_hwfn
->p_ooo_info
,
606 cid
= le32_to_cpu(iscsi_ooo
->cid
);
608 /* Process delete isle first */
609 if (iscsi_ooo
->drop_size
)
610 qed_ooo_delete_isles(p_hwfn
, p_hwfn
->p_ooo_info
, cid
,
611 iscsi_ooo
->drop_isle
,
612 iscsi_ooo
->drop_size
);
614 if (iscsi_ooo
->ooo_opcode
== TCP_EVENT_NOP
)
617 /* Now process create/add/join isles */
618 if (list_empty(&p_rx
->active_descq
)) {
620 "LL2 OOO RX chain has no submitted buffers\n"
625 p_pkt
= list_first_entry(&p_rx
->active_descq
,
626 struct qed_ll2_rx_packet
, list_entry
);
628 if ((iscsi_ooo
->ooo_opcode
== TCP_EVENT_ADD_NEW_ISLE
) ||
629 (iscsi_ooo
->ooo_opcode
== TCP_EVENT_ADD_ISLE_RIGHT
) ||
630 (iscsi_ooo
->ooo_opcode
== TCP_EVENT_ADD_ISLE_LEFT
) ||
631 (iscsi_ooo
->ooo_opcode
== TCP_EVENT_ADD_PEN
) ||
632 (iscsi_ooo
->ooo_opcode
== TCP_EVENT_JOIN
)) {
635 "LL2 OOO RX packet is not valid\n");
638 list_del(&p_pkt
->list_entry
);
639 p_buffer
= (struct qed_ooo_buffer
*)p_pkt
->cookie
;
640 p_buffer
->packet_length
= packet_length
;
641 p_buffer
->parse_flags
= parse_flags
;
642 p_buffer
->vlan
= vlan
;
643 p_buffer
->placement_offset
= placement_offset
;
644 qed_chain_consume(&p_rx
->rxq_chain
);
645 list_add_tail(&p_pkt
->list_entry
, &p_rx
->free_descq
);
647 switch (iscsi_ooo
->ooo_opcode
) {
648 case TCP_EVENT_ADD_NEW_ISLE
:
649 qed_ooo_add_new_isle(p_hwfn
,
655 case TCP_EVENT_ADD_ISLE_RIGHT
:
656 qed_ooo_add_new_buffer(p_hwfn
,
663 case TCP_EVENT_ADD_ISLE_LEFT
:
664 qed_ooo_add_new_buffer(p_hwfn
,
672 qed_ooo_add_new_buffer(p_hwfn
,
675 iscsi_ooo
->ooo_isle
+
679 qed_ooo_join_isles(p_hwfn
,
681 cid
, iscsi_ooo
->ooo_isle
);
683 case TCP_EVENT_ADD_PEN
:
684 num_ooo_add_to_peninsula
++;
685 qed_ooo_put_ready_buffer(p_hwfn
,
692 "Unexpected event (%d) TX OOO completion\n",
693 iscsi_ooo
->ooo_opcode
);
701 qed_ooo_submit_tx_buffers(struct qed_hwfn
*p_hwfn
,
702 struct qed_ll2_info
*p_ll2_conn
)
704 struct qed_ll2_tx_pkt_info tx_pkt
;
705 struct qed_ooo_buffer
*p_buffer
;
707 dma_addr_t first_frag
;
712 /* Submit Tx buffers here */
713 while ((p_buffer
= qed_ooo_get_ready_buffer(p_hwfn
,
714 p_hwfn
->p_ooo_info
))) {
718 first_frag
= p_buffer
->rx_buffer_phys_addr
+
719 p_buffer
->placement_offset
;
720 parse_flags
= p_buffer
->parse_flags
;
721 bd_flags
= qed_ll2_convert_rx_parse_to_tx_flags(parse_flags
);
722 SET_FIELD(bd_flags
, CORE_TX_BD_DATA_FORCE_VLAN_MODE
, 1);
723 SET_FIELD(bd_flags
, CORE_TX_BD_DATA_L4_PROTOCOL
, 1);
725 memset(&tx_pkt
, 0, sizeof(tx_pkt
));
726 tx_pkt
.num_of_bds
= 1;
727 tx_pkt
.vlan
= p_buffer
->vlan
;
728 tx_pkt
.bd_flags
= bd_flags
;
729 tx_pkt
.l4_hdr_offset_w
= l4_hdr_offset_w
;
730 tx_pkt
.tx_dest
= p_ll2_conn
->tx_dest
;
731 tx_pkt
.first_frag
= first_frag
;
732 tx_pkt
.first_frag_len
= p_buffer
->packet_length
;
733 tx_pkt
.cookie
= p_buffer
;
735 rc
= qed_ll2_prepare_tx_packet(p_hwfn
, p_ll2_conn
->my_id
,
738 qed_ooo_put_ready_buffer(p_hwfn
, p_hwfn
->p_ooo_info
,
746 qed_ooo_submit_rx_buffers(struct qed_hwfn
*p_hwfn
,
747 struct qed_ll2_info
*p_ll2_conn
)
749 struct qed_ooo_buffer
*p_buffer
;
752 while ((p_buffer
= qed_ooo_get_free_buffer(p_hwfn
,
753 p_hwfn
->p_ooo_info
))) {
754 rc
= qed_ll2_post_rx_buffer(p_hwfn
,
756 p_buffer
->rx_buffer_phys_addr
,
759 qed_ooo_put_free_buffer(p_hwfn
,
760 p_hwfn
->p_ooo_info
, p_buffer
);
766 static int qed_ll2_lb_rxq_completion(struct qed_hwfn
*p_hwfn
, void *p_cookie
)
768 struct qed_ll2_info
*p_ll2_conn
= (struct qed_ll2_info
*)p_cookie
;
771 rc
= qed_ll2_lb_rxq_handler(p_hwfn
, p_ll2_conn
);
775 qed_ooo_submit_rx_buffers(p_hwfn
, p_ll2_conn
);
776 qed_ooo_submit_tx_buffers(p_hwfn
, p_ll2_conn
);
781 static int qed_ll2_lb_txq_completion(struct qed_hwfn
*p_hwfn
, void *p_cookie
)
783 struct qed_ll2_info
*p_ll2_conn
= (struct qed_ll2_info
*)p_cookie
;
784 struct qed_ll2_tx_queue
*p_tx
= &p_ll2_conn
->tx_queue
;
785 struct qed_ll2_tx_packet
*p_pkt
= NULL
;
786 struct qed_ooo_buffer
*p_buffer
;
787 bool b_dont_submit_rx
= false;
788 u16 new_idx
= 0, num_bds
= 0;
791 new_idx
= le16_to_cpu(*p_tx
->p_fw_cons
);
792 num_bds
= ((s16
)new_idx
- (s16
)p_tx
->bds_idx
);
798 if (list_empty(&p_tx
->active_descq
))
801 p_pkt
= list_first_entry(&p_tx
->active_descq
,
802 struct qed_ll2_tx_packet
, list_entry
);
806 if (p_pkt
->bd_used
!= 1) {
808 "Unexpectedly many BDs(%d) in TX OOO completion\n",
813 list_del(&p_pkt
->list_entry
);
817 qed_chain_consume(&p_tx
->txq_chain
);
819 p_buffer
= (struct qed_ooo_buffer
*)p_pkt
->cookie
;
820 list_add_tail(&p_pkt
->list_entry
, &p_tx
->free_descq
);
822 if (b_dont_submit_rx
) {
823 qed_ooo_put_free_buffer(p_hwfn
, p_hwfn
->p_ooo_info
,
828 rc
= qed_ll2_post_rx_buffer(p_hwfn
, p_ll2_conn
->my_id
,
829 p_buffer
->rx_buffer_phys_addr
, 0,
832 qed_ooo_put_free_buffer(p_hwfn
,
833 p_hwfn
->p_ooo_info
, p_buffer
);
834 b_dont_submit_rx
= true;
838 qed_ooo_submit_tx_buffers(p_hwfn
, p_ll2_conn
);
843 static void qed_ll2_stop_ooo(struct qed_dev
*cdev
)
845 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
846 u8
*handle
= &hwfn
->pf_params
.iscsi_pf_params
.ll2_ooo_queue_id
;
848 DP_VERBOSE(cdev
, QED_MSG_STORAGE
, "Stopping LL2 OOO queue [%02x]\n",
851 qed_ll2_terminate_connection(hwfn
, *handle
);
852 qed_ll2_release_connection(hwfn
, *handle
);
853 *handle
= QED_LL2_UNUSED_HANDLE
;
856 static int qed_sp_ll2_rx_queue_start(struct qed_hwfn
*p_hwfn
,
857 struct qed_ll2_info
*p_ll2_conn
,
860 enum qed_ll2_conn_type conn_type
= p_ll2_conn
->input
.conn_type
;
861 struct qed_ll2_rx_queue
*p_rx
= &p_ll2_conn
->rx_queue
;
862 struct core_rx_start_ramrod_data
*p_ramrod
= NULL
;
863 struct qed_spq_entry
*p_ent
= NULL
;
864 struct qed_sp_init_data init_data
;
869 memset(&init_data
, 0, sizeof(init_data
));
870 init_data
.cid
= p_ll2_conn
->cid
;
871 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
872 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
874 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
875 CORE_RAMROD_RX_QUEUE_START
,
876 PROTOCOLID_CORE
, &init_data
);
880 p_ramrod
= &p_ent
->ramrod
.core_rx_queue_start
;
882 p_ramrod
->sb_id
= cpu_to_le16(qed_int_get_sp_sb_id(p_hwfn
));
883 p_ramrod
->sb_index
= p_rx
->rx_sb_index
;
884 p_ramrod
->complete_event_flg
= 1;
886 p_ramrod
->mtu
= cpu_to_le16(p_ll2_conn
->input
.mtu
);
887 DMA_REGPAIR_LE(p_ramrod
->bd_base
, p_rx
->rxq_chain
.p_phys_addr
);
888 cqe_pbl_size
= (u16
)qed_chain_get_page_cnt(&p_rx
->rcq_chain
);
889 p_ramrod
->num_of_pbl_pages
= cpu_to_le16(cqe_pbl_size
);
890 DMA_REGPAIR_LE(p_ramrod
->cqe_pbl_addr
,
891 qed_chain_get_pbl_phys(&p_rx
->rcq_chain
));
893 p_ramrod
->drop_ttl0_flg
= p_ll2_conn
->input
.rx_drop_ttl0_flg
;
894 p_ramrod
->inner_vlan_removal_en
= p_ll2_conn
->input
.rx_vlan_removal_en
;
895 p_ramrod
->queue_id
= p_ll2_conn
->queue_id
;
896 p_ramrod
->main_func_queue
= (conn_type
== QED_LL2_TYPE_ISCSI_OOO
) ? 0
899 if ((IS_MF_DEFAULT(p_hwfn
) || IS_MF_SI(p_hwfn
)) &&
900 p_ramrod
->main_func_queue
&& (conn_type
!= QED_LL2_TYPE_ROCE
)) {
901 p_ramrod
->mf_si_bcast_accept_all
= 1;
902 p_ramrod
->mf_si_mcast_accept_all
= 1;
904 p_ramrod
->mf_si_bcast_accept_all
= 0;
905 p_ramrod
->mf_si_mcast_accept_all
= 0;
908 p_ramrod
->action_on_error
.error_type
= action_on_error
;
909 p_ramrod
->gsi_offload_flag
= p_ll2_conn
->input
.gsi_enable
;
910 return qed_spq_post(p_hwfn
, p_ent
, NULL
);
913 static int qed_sp_ll2_tx_queue_start(struct qed_hwfn
*p_hwfn
,
914 struct qed_ll2_info
*p_ll2_conn
)
916 enum qed_ll2_conn_type conn_type
= p_ll2_conn
->input
.conn_type
;
917 struct qed_ll2_tx_queue
*p_tx
= &p_ll2_conn
->tx_queue
;
918 struct core_tx_start_ramrod_data
*p_ramrod
= NULL
;
919 struct qed_spq_entry
*p_ent
= NULL
;
920 struct qed_sp_init_data init_data
;
921 u16 pq_id
= 0, pbl_size
;
924 if (!QED_LL2_TX_REGISTERED(p_ll2_conn
))
927 if (p_ll2_conn
->input
.conn_type
== QED_LL2_TYPE_ISCSI_OOO
)
928 p_ll2_conn
->tx_stats_en
= 0;
930 p_ll2_conn
->tx_stats_en
= 1;
933 memset(&init_data
, 0, sizeof(init_data
));
934 init_data
.cid
= p_ll2_conn
->cid
;
935 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
936 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
938 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
939 CORE_RAMROD_TX_QUEUE_START
,
940 PROTOCOLID_CORE
, &init_data
);
944 p_ramrod
= &p_ent
->ramrod
.core_tx_queue_start
;
946 p_ramrod
->sb_id
= cpu_to_le16(qed_int_get_sp_sb_id(p_hwfn
));
947 p_ramrod
->sb_index
= p_tx
->tx_sb_index
;
948 p_ramrod
->mtu
= cpu_to_le16(p_ll2_conn
->input
.mtu
);
949 p_ramrod
->stats_en
= p_ll2_conn
->tx_stats_en
;
950 p_ramrod
->stats_id
= p_ll2_conn
->tx_stats_id
;
952 DMA_REGPAIR_LE(p_ramrod
->pbl_base_addr
,
953 qed_chain_get_pbl_phys(&p_tx
->txq_chain
));
954 pbl_size
= qed_chain_get_page_cnt(&p_tx
->txq_chain
);
955 p_ramrod
->pbl_size
= cpu_to_le16(pbl_size
);
957 switch (p_ll2_conn
->input
.tx_tc
) {
959 pq_id
= qed_get_cm_pq_idx(p_hwfn
, PQ_FLAGS_LB
);
962 pq_id
= qed_get_cm_pq_idx(p_hwfn
, PQ_FLAGS_OOO
);
965 pq_id
= qed_get_cm_pq_idx(p_hwfn
, PQ_FLAGS_OFLD
);
969 p_ramrod
->qm_pq_id
= cpu_to_le16(pq_id
);
972 case QED_LL2_TYPE_FCOE
:
973 p_ramrod
->conn_type
= PROTOCOLID_FCOE
;
975 case QED_LL2_TYPE_ISCSI
:
976 case QED_LL2_TYPE_ISCSI_OOO
:
977 p_ramrod
->conn_type
= PROTOCOLID_ISCSI
;
979 case QED_LL2_TYPE_ROCE
:
980 p_ramrod
->conn_type
= PROTOCOLID_ROCE
;
983 p_ramrod
->conn_type
= PROTOCOLID_ETH
;
984 DP_NOTICE(p_hwfn
, "Unknown connection type: %d\n", conn_type
);
987 p_ramrod
->gsi_offload_flag
= p_ll2_conn
->input
.gsi_enable
;
989 return qed_spq_post(p_hwfn
, p_ent
, NULL
);
992 static int qed_sp_ll2_rx_queue_stop(struct qed_hwfn
*p_hwfn
,
993 struct qed_ll2_info
*p_ll2_conn
)
995 struct core_rx_stop_ramrod_data
*p_ramrod
= NULL
;
996 struct qed_spq_entry
*p_ent
= NULL
;
997 struct qed_sp_init_data init_data
;
1001 memset(&init_data
, 0, sizeof(init_data
));
1002 init_data
.cid
= p_ll2_conn
->cid
;
1003 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1004 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1006 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1007 CORE_RAMROD_RX_QUEUE_STOP
,
1008 PROTOCOLID_CORE
, &init_data
);
1012 p_ramrod
= &p_ent
->ramrod
.core_rx_queue_stop
;
1014 p_ramrod
->complete_event_flg
= 1;
1015 p_ramrod
->queue_id
= p_ll2_conn
->queue_id
;
1017 return qed_spq_post(p_hwfn
, p_ent
, NULL
);
1020 static int qed_sp_ll2_tx_queue_stop(struct qed_hwfn
*p_hwfn
,
1021 struct qed_ll2_info
*p_ll2_conn
)
1023 struct qed_spq_entry
*p_ent
= NULL
;
1024 struct qed_sp_init_data init_data
;
1028 memset(&init_data
, 0, sizeof(init_data
));
1029 init_data
.cid
= p_ll2_conn
->cid
;
1030 init_data
.opaque_fid
= p_hwfn
->hw_info
.opaque_fid
;
1031 init_data
.comp_mode
= QED_SPQ_MODE_EBLOCK
;
1033 rc
= qed_sp_init_request(p_hwfn
, &p_ent
,
1034 CORE_RAMROD_TX_QUEUE_STOP
,
1035 PROTOCOLID_CORE
, &init_data
);
1039 return qed_spq_post(p_hwfn
, p_ent
, NULL
);
1043 qed_ll2_acquire_connection_rx(struct qed_hwfn
*p_hwfn
,
1044 struct qed_ll2_info
*p_ll2_info
)
1046 struct qed_ll2_rx_packet
*p_descq
;
1050 if (!p_ll2_info
->input
.rx_num_desc
)
1053 rc
= qed_chain_alloc(p_hwfn
->cdev
,
1054 QED_CHAIN_USE_TO_CONSUME_PRODUCE
,
1055 QED_CHAIN_MODE_NEXT_PTR
,
1056 QED_CHAIN_CNT_TYPE_U16
,
1057 p_ll2_info
->input
.rx_num_desc
,
1058 sizeof(struct core_rx_bd
),
1059 &p_ll2_info
->rx_queue
.rxq_chain
);
1061 DP_NOTICE(p_hwfn
, "Failed to allocate ll2 rxq chain\n");
1065 capacity
= qed_chain_get_capacity(&p_ll2_info
->rx_queue
.rxq_chain
);
1066 p_descq
= kcalloc(capacity
, sizeof(struct qed_ll2_rx_packet
),
1070 DP_NOTICE(p_hwfn
, "Failed to allocate ll2 Rx desc\n");
1073 p_ll2_info
->rx_queue
.descq_array
= p_descq
;
1075 rc
= qed_chain_alloc(p_hwfn
->cdev
,
1076 QED_CHAIN_USE_TO_CONSUME_PRODUCE
,
1078 QED_CHAIN_CNT_TYPE_U16
,
1079 p_ll2_info
->input
.rx_num_desc
,
1080 sizeof(struct core_rx_fast_path_cqe
),
1081 &p_ll2_info
->rx_queue
.rcq_chain
);
1083 DP_NOTICE(p_hwfn
, "Failed to allocate ll2 rcq chain\n");
1087 DP_VERBOSE(p_hwfn
, QED_MSG_LL2
,
1088 "Allocated LL2 Rxq [Type %08x] with 0x%08x buffers\n",
1089 p_ll2_info
->input
.conn_type
, p_ll2_info
->input
.rx_num_desc
);
1095 static int qed_ll2_acquire_connection_tx(struct qed_hwfn
*p_hwfn
,
1096 struct qed_ll2_info
*p_ll2_info
)
1098 struct qed_ll2_tx_packet
*p_descq
;
1102 if (!p_ll2_info
->input
.tx_num_desc
)
1105 rc
= qed_chain_alloc(p_hwfn
->cdev
,
1106 QED_CHAIN_USE_TO_CONSUME_PRODUCE
,
1108 QED_CHAIN_CNT_TYPE_U16
,
1109 p_ll2_info
->input
.tx_num_desc
,
1110 sizeof(struct core_tx_bd
),
1111 &p_ll2_info
->tx_queue
.txq_chain
);
1115 capacity
= qed_chain_get_capacity(&p_ll2_info
->tx_queue
.txq_chain
);
1116 p_descq
= kcalloc(capacity
, sizeof(struct qed_ll2_tx_packet
),
1122 p_ll2_info
->tx_queue
.descq_array
= p_descq
;
1124 DP_VERBOSE(p_hwfn
, QED_MSG_LL2
,
1125 "Allocated LL2 Txq [Type %08x] with 0x%08x buffers\n",
1126 p_ll2_info
->input
.conn_type
, p_ll2_info
->input
.tx_num_desc
);
1131 "Can't allocate memory for Tx LL2 with 0x%08x buffers\n",
1132 p_ll2_info
->input
.tx_num_desc
);
1137 qed_ll2_acquire_connection_ooo(struct qed_hwfn
*p_hwfn
,
1138 struct qed_ll2_info
*p_ll2_info
, u16 mtu
)
1140 struct qed_ooo_buffer
*p_buf
= NULL
;
1145 if (p_ll2_info
->input
.conn_type
!= QED_LL2_TYPE_ISCSI_OOO
)
1148 /* Correct number of requested OOO buffers if needed */
1149 if (!p_ll2_info
->input
.rx_num_ooo_buffers
) {
1150 u16 num_desc
= p_ll2_info
->input
.rx_num_desc
;
1154 p_ll2_info
->input
.rx_num_ooo_buffers
= num_desc
* 2;
1157 for (buf_idx
= 0; buf_idx
< p_ll2_info
->input
.rx_num_ooo_buffers
;
1159 p_buf
= kzalloc(sizeof(*p_buf
), GFP_KERNEL
);
1165 p_buf
->rx_buffer_size
= mtu
+ 26 + ETH_CACHE_LINE_SIZE
;
1166 p_buf
->rx_buffer_size
= (p_buf
->rx_buffer_size
+
1167 ETH_CACHE_LINE_SIZE
- 1) &
1168 ~(ETH_CACHE_LINE_SIZE
- 1);
1169 p_virt
= dma_alloc_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1170 p_buf
->rx_buffer_size
,
1171 &p_buf
->rx_buffer_phys_addr
,
1179 p_buf
->rx_buffer_virt_addr
= p_virt
;
1180 qed_ooo_put_free_buffer(p_hwfn
, p_hwfn
->p_ooo_info
, p_buf
);
1183 DP_VERBOSE(p_hwfn
, QED_MSG_LL2
,
1184 "Allocated [%04x] LL2 OOO buffers [each of size 0x%08x]\n",
1185 p_ll2_info
->input
.rx_num_ooo_buffers
, p_buf
->rx_buffer_size
);
1192 qed_ll2_set_cbs(struct qed_ll2_info
*p_ll2_info
, const struct qed_ll2_cbs
*cbs
)
1194 if (!cbs
|| (!cbs
->rx_comp_cb
||
1195 !cbs
->rx_release_cb
||
1196 !cbs
->tx_comp_cb
|| !cbs
->tx_release_cb
|| !cbs
->cookie
))
1199 p_ll2_info
->cbs
.rx_comp_cb
= cbs
->rx_comp_cb
;
1200 p_ll2_info
->cbs
.rx_release_cb
= cbs
->rx_release_cb
;
1201 p_ll2_info
->cbs
.tx_comp_cb
= cbs
->tx_comp_cb
;
1202 p_ll2_info
->cbs
.tx_release_cb
= cbs
->tx_release_cb
;
1203 p_ll2_info
->cbs
.cookie
= cbs
->cookie
;
1208 static enum core_error_handle
1209 qed_ll2_get_error_choice(enum qed_ll2_error_handle err
)
1212 case QED_LL2_DROP_PACKET
:
1213 return LL2_DROP_PACKET
;
1214 case QED_LL2_DO_NOTHING
:
1215 return LL2_DO_NOTHING
;
1216 case QED_LL2_ASSERT
:
1219 return LL2_DO_NOTHING
;
1223 int qed_ll2_acquire_connection(void *cxt
, struct qed_ll2_acquire_data
*data
)
1225 struct qed_hwfn
*p_hwfn
= cxt
;
1226 qed_int_comp_cb_t comp_rx_cb
, comp_tx_cb
;
1227 struct qed_ll2_info
*p_ll2_info
= NULL
;
1231 if (!data
->p_connection_handle
|| !p_hwfn
->p_ll2_info
)
1234 /* Find a free connection to be used */
1235 for (i
= 0; (i
< QED_MAX_NUM_OF_LL2_CONNECTIONS
); i
++) {
1236 mutex_lock(&p_hwfn
->p_ll2_info
[i
].mutex
);
1237 if (p_hwfn
->p_ll2_info
[i
].b_active
) {
1238 mutex_unlock(&p_hwfn
->p_ll2_info
[i
].mutex
);
1242 p_hwfn
->p_ll2_info
[i
].b_active
= true;
1243 p_ll2_info
= &p_hwfn
->p_ll2_info
[i
];
1244 mutex_unlock(&p_hwfn
->p_ll2_info
[i
].mutex
);
1250 memcpy(&p_ll2_info
->input
, &data
->input
, sizeof(p_ll2_info
->input
));
1252 p_ll2_info
->tx_dest
= (data
->input
.tx_dest
== QED_LL2_TX_DEST_NW
) ?
1253 CORE_TX_DEST_NW
: CORE_TX_DEST_LB
;
1255 /* Correct maximum number of Tx BDs */
1256 p_tx_max
= &p_ll2_info
->input
.tx_max_bds_per_packet
;
1258 *p_tx_max
= CORE_LL2_TX_MAX_BDS_PER_PACKET
;
1260 *p_tx_max
= min_t(u8
, *p_tx_max
,
1261 CORE_LL2_TX_MAX_BDS_PER_PACKET
);
1263 rc
= qed_ll2_set_cbs(p_ll2_info
, data
->cbs
);
1265 DP_NOTICE(p_hwfn
, "Invalid callback functions\n");
1266 goto q_allocate_fail
;
1269 rc
= qed_ll2_acquire_connection_rx(p_hwfn
, p_ll2_info
);
1271 goto q_allocate_fail
;
1273 rc
= qed_ll2_acquire_connection_tx(p_hwfn
, p_ll2_info
);
1275 goto q_allocate_fail
;
1277 rc
= qed_ll2_acquire_connection_ooo(p_hwfn
, p_ll2_info
,
1280 goto q_allocate_fail
;
1282 /* Register callbacks for the Rx/Tx queues */
1283 if (data
->input
.conn_type
== QED_LL2_TYPE_ISCSI_OOO
) {
1284 comp_rx_cb
= qed_ll2_lb_rxq_completion
;
1285 comp_tx_cb
= qed_ll2_lb_txq_completion
;
1287 comp_rx_cb
= qed_ll2_rxq_completion
;
1288 comp_tx_cb
= qed_ll2_txq_completion
;
1291 if (data
->input
.rx_num_desc
) {
1292 qed_int_register_cb(p_hwfn
, comp_rx_cb
,
1293 &p_hwfn
->p_ll2_info
[i
],
1294 &p_ll2_info
->rx_queue
.rx_sb_index
,
1295 &p_ll2_info
->rx_queue
.p_fw_cons
);
1296 p_ll2_info
->rx_queue
.b_cb_registred
= true;
1299 if (data
->input
.tx_num_desc
) {
1300 qed_int_register_cb(p_hwfn
,
1302 &p_hwfn
->p_ll2_info
[i
],
1303 &p_ll2_info
->tx_queue
.tx_sb_index
,
1304 &p_ll2_info
->tx_queue
.p_fw_cons
);
1305 p_ll2_info
->tx_queue
.b_cb_registred
= true;
1308 *data
->p_connection_handle
= i
;
1312 qed_ll2_release_connection(p_hwfn
, i
);
1316 static int qed_ll2_establish_connection_rx(struct qed_hwfn
*p_hwfn
,
1317 struct qed_ll2_info
*p_ll2_conn
)
1319 enum qed_ll2_error_handle error_input
;
1320 enum core_error_handle error_mode
;
1321 u8 action_on_error
= 0;
1323 if (!QED_LL2_RX_REGISTERED(p_ll2_conn
))
1326 DIRECT_REG_WR(p_ll2_conn
->rx_queue
.set_prod_addr
, 0x0);
1327 error_input
= p_ll2_conn
->input
.ai_err_packet_too_big
;
1328 error_mode
= qed_ll2_get_error_choice(error_input
);
1329 SET_FIELD(action_on_error
,
1330 CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG
, error_mode
);
1331 error_input
= p_ll2_conn
->input
.ai_err_no_buf
;
1332 error_mode
= qed_ll2_get_error_choice(error_input
);
1333 SET_FIELD(action_on_error
, CORE_RX_ACTION_ON_ERROR_NO_BUFF
, error_mode
);
1335 return qed_sp_ll2_rx_queue_start(p_hwfn
, p_ll2_conn
, action_on_error
);
1339 qed_ll2_establish_connection_ooo(struct qed_hwfn
*p_hwfn
,
1340 struct qed_ll2_info
*p_ll2_conn
)
1342 if (p_ll2_conn
->input
.conn_type
!= QED_LL2_TYPE_ISCSI_OOO
)
1345 qed_ooo_release_all_isles(p_hwfn
, p_hwfn
->p_ooo_info
);
1346 qed_ooo_submit_rx_buffers(p_hwfn
, p_ll2_conn
);
1349 int qed_ll2_establish_connection(void *cxt
, u8 connection_handle
)
1351 struct qed_hwfn
*p_hwfn
= cxt
;
1352 struct qed_ll2_info
*p_ll2_conn
;
1353 struct qed_ll2_rx_queue
*p_rx
;
1354 struct qed_ll2_tx_queue
*p_tx
;
1355 struct qed_ptt
*p_ptt
;
1360 p_ptt
= qed_ptt_acquire(p_hwfn
);
1364 p_ll2_conn
= qed_ll2_handle_sanity_lock(p_hwfn
, connection_handle
);
1370 p_rx
= &p_ll2_conn
->rx_queue
;
1371 p_tx
= &p_ll2_conn
->tx_queue
;
1373 qed_chain_reset(&p_rx
->rxq_chain
);
1374 qed_chain_reset(&p_rx
->rcq_chain
);
1375 INIT_LIST_HEAD(&p_rx
->active_descq
);
1376 INIT_LIST_HEAD(&p_rx
->free_descq
);
1377 INIT_LIST_HEAD(&p_rx
->posting_descq
);
1378 spin_lock_init(&p_rx
->lock
);
1379 capacity
= qed_chain_get_capacity(&p_rx
->rxq_chain
);
1380 for (i
= 0; i
< capacity
; i
++)
1381 list_add_tail(&p_rx
->descq_array
[i
].list_entry
,
1383 *p_rx
->p_fw_cons
= 0;
1385 qed_chain_reset(&p_tx
->txq_chain
);
1386 INIT_LIST_HEAD(&p_tx
->active_descq
);
1387 INIT_LIST_HEAD(&p_tx
->free_descq
);
1388 INIT_LIST_HEAD(&p_tx
->sending_descq
);
1389 spin_lock_init(&p_tx
->lock
);
1390 capacity
= qed_chain_get_capacity(&p_tx
->txq_chain
);
1391 for (i
= 0; i
< capacity
; i
++)
1392 list_add_tail(&p_tx
->descq_array
[i
].list_entry
,
1394 p_tx
->cur_completing_bd_idx
= 0;
1396 p_tx
->b_completing_packet
= false;
1397 p_tx
->cur_send_packet
= NULL
;
1398 p_tx
->cur_send_frag_num
= 0;
1399 p_tx
->cur_completing_frag_num
= 0;
1400 *p_tx
->p_fw_cons
= 0;
1402 rc
= qed_cxt_acquire_cid(p_hwfn
, PROTOCOLID_CORE
, &p_ll2_conn
->cid
);
1406 qid
= p_hwfn
->hw_info
.resc_start
[QED_LL2_QUEUE
] + connection_handle
;
1407 p_ll2_conn
->queue_id
= qid
;
1408 p_ll2_conn
->tx_stats_id
= qid
;
1409 p_rx
->set_prod_addr
= (u8 __iomem
*)p_hwfn
->regview
+
1410 GTT_BAR0_MAP_REG_TSDM_RAM
+
1411 TSTORM_LL2_RX_PRODS_OFFSET(qid
);
1412 p_tx
->doorbell_addr
= (u8 __iomem
*)p_hwfn
->doorbells
+
1413 qed_db_addr(p_ll2_conn
->cid
,
1416 rc
= qed_ll2_establish_connection_rx(p_hwfn
, p_ll2_conn
);
1420 rc
= qed_sp_ll2_tx_queue_start(p_hwfn
, p_ll2_conn
);
1424 if (p_hwfn
->hw_info
.personality
!= QED_PCI_ETH_ROCE
)
1425 qed_wr(p_hwfn
, p_ptt
, PRS_REG_USE_LIGHT_L2
, 1);
1427 qed_ll2_establish_connection_ooo(p_hwfn
, p_ll2_conn
);
1429 if (p_ll2_conn
->input
.conn_type
== QED_LL2_TYPE_FCOE
) {
1430 qed_llh_add_protocol_filter(p_hwfn
, p_ptt
,
1432 QED_LLH_FILTER_ETHERTYPE
);
1433 qed_llh_add_protocol_filter(p_hwfn
, p_ptt
,
1435 QED_LLH_FILTER_ETHERTYPE
);
1439 qed_ptt_release(p_hwfn
, p_ptt
);
1443 static void qed_ll2_post_rx_buffer_notify_fw(struct qed_hwfn
*p_hwfn
,
1444 struct qed_ll2_rx_queue
*p_rx
,
1445 struct qed_ll2_rx_packet
*p_curp
)
1447 struct qed_ll2_rx_packet
*p_posting_packet
= NULL
;
1448 struct core_ll2_rx_prod rx_prod
= { 0, 0, 0 };
1449 bool b_notify_fw
= false;
1450 u16 bd_prod
, cq_prod
;
1452 /* This handles the flushing of already posted buffers */
1453 while (!list_empty(&p_rx
->posting_descq
)) {
1454 p_posting_packet
= list_first_entry(&p_rx
->posting_descq
,
1455 struct qed_ll2_rx_packet
,
1457 list_move_tail(&p_posting_packet
->list_entry
,
1458 &p_rx
->active_descq
);
1462 /* This handles the supplied packet [if there is one] */
1464 list_add_tail(&p_curp
->list_entry
, &p_rx
->active_descq
);
1471 bd_prod
= qed_chain_get_prod_idx(&p_rx
->rxq_chain
);
1472 cq_prod
= qed_chain_get_prod_idx(&p_rx
->rcq_chain
);
1473 rx_prod
.bd_prod
= cpu_to_le16(bd_prod
);
1474 rx_prod
.cqe_prod
= cpu_to_le16(cq_prod
);
1475 DIRECT_REG_WR(p_rx
->set_prod_addr
, *((u32
*)&rx_prod
));
1478 int qed_ll2_post_rx_buffer(void *cxt
,
1479 u8 connection_handle
,
1481 u16 buf_len
, void *cookie
, u8 notify_fw
)
1483 struct qed_hwfn
*p_hwfn
= cxt
;
1484 struct core_rx_bd_with_buff_len
*p_curb
= NULL
;
1485 struct qed_ll2_rx_packet
*p_curp
= NULL
;
1486 struct qed_ll2_info
*p_ll2_conn
;
1487 struct qed_ll2_rx_queue
*p_rx
;
1488 unsigned long flags
;
1492 p_ll2_conn
= qed_ll2_handle_sanity(p_hwfn
, connection_handle
);
1495 p_rx
= &p_ll2_conn
->rx_queue
;
1497 spin_lock_irqsave(&p_rx
->lock
, flags
);
1498 if (!list_empty(&p_rx
->free_descq
))
1499 p_curp
= list_first_entry(&p_rx
->free_descq
,
1500 struct qed_ll2_rx_packet
, list_entry
);
1502 if (qed_chain_get_elem_left(&p_rx
->rxq_chain
) &&
1503 qed_chain_get_elem_left(&p_rx
->rcq_chain
)) {
1504 p_data
= qed_chain_produce(&p_rx
->rxq_chain
);
1505 p_curb
= (struct core_rx_bd_with_buff_len
*)p_data
;
1506 qed_chain_produce(&p_rx
->rcq_chain
);
1510 /* If we're lacking entires, let's try to flush buffers to FW */
1511 if (!p_curp
|| !p_curb
) {
1517 /* We have an Rx packet we can fill */
1518 DMA_REGPAIR_LE(p_curb
->addr
, addr
);
1519 p_curb
->buff_length
= cpu_to_le16(buf_len
);
1520 p_curp
->rx_buf_addr
= addr
;
1521 p_curp
->cookie
= cookie
;
1522 p_curp
->rxq_bd
= p_curb
;
1523 p_curp
->buf_length
= buf_len
;
1524 list_del(&p_curp
->list_entry
);
1526 /* Check if we only want to enqueue this packet without informing FW */
1528 list_add_tail(&p_curp
->list_entry
, &p_rx
->posting_descq
);
1533 qed_ll2_post_rx_buffer_notify_fw(p_hwfn
, p_rx
, p_curp
);
1535 spin_unlock_irqrestore(&p_rx
->lock
, flags
);
1539 static void qed_ll2_prepare_tx_packet_set(struct qed_hwfn
*p_hwfn
,
1540 struct qed_ll2_tx_queue
*p_tx
,
1541 struct qed_ll2_tx_packet
*p_curp
,
1542 struct qed_ll2_tx_pkt_info
*pkt
,
1545 list_del(&p_curp
->list_entry
);
1546 p_curp
->cookie
= pkt
->cookie
;
1547 p_curp
->bd_used
= pkt
->num_of_bds
;
1548 p_curp
->notify_fw
= notify_fw
;
1549 p_tx
->cur_send_packet
= p_curp
;
1550 p_tx
->cur_send_frag_num
= 0;
1552 p_curp
->bds_set
[p_tx
->cur_send_frag_num
].tx_frag
= pkt
->first_frag
;
1553 p_curp
->bds_set
[p_tx
->cur_send_frag_num
].frag_len
= pkt
->first_frag_len
;
1554 p_tx
->cur_send_frag_num
++;
1558 qed_ll2_prepare_tx_packet_set_bd(struct qed_hwfn
*p_hwfn
,
1559 struct qed_ll2_info
*p_ll2
,
1560 struct qed_ll2_tx_packet
*p_curp
,
1561 struct qed_ll2_tx_pkt_info
*pkt
)
1563 struct qed_chain
*p_tx_chain
= &p_ll2
->tx_queue
.txq_chain
;
1564 u16 prod_idx
= qed_chain_get_prod_idx(p_tx_chain
);
1565 struct core_tx_bd
*start_bd
= NULL
;
1566 enum core_roce_flavor_type roce_flavor
;
1567 enum core_tx_dest tx_dest
;
1568 u16 bd_data
= 0, frag_idx
;
1570 roce_flavor
= (pkt
->qed_roce_flavor
== QED_LL2_ROCE
) ? CORE_ROCE
1573 tx_dest
= (pkt
->tx_dest
== QED_LL2_TX_DEST_NW
) ? CORE_TX_DEST_NW
1576 start_bd
= (struct core_tx_bd
*)qed_chain_produce(p_tx_chain
);
1577 start_bd
->nw_vlan_or_lb_echo
= cpu_to_le16(pkt
->vlan
);
1578 SET_FIELD(start_bd
->bitfield1
, CORE_TX_BD_L4_HDR_OFFSET_W
,
1579 cpu_to_le16(pkt
->l4_hdr_offset_w
));
1580 SET_FIELD(start_bd
->bitfield1
, CORE_TX_BD_TX_DST
, tx_dest
);
1581 bd_data
|= pkt
->bd_flags
;
1582 SET_FIELD(bd_data
, CORE_TX_BD_DATA_START_BD
, 0x1);
1583 SET_FIELD(bd_data
, CORE_TX_BD_DATA_NBDS
, pkt
->num_of_bds
);
1584 SET_FIELD(bd_data
, CORE_TX_BD_DATA_ROCE_FLAV
, roce_flavor
);
1585 start_bd
->bd_data
.as_bitfield
= cpu_to_le16(bd_data
);
1586 DMA_REGPAIR_LE(start_bd
->addr
, pkt
->first_frag
);
1587 start_bd
->nbytes
= cpu_to_le16(pkt
->first_frag_len
);
1590 (NETIF_MSG_TX_QUEUED
| QED_MSG_LL2
),
1591 "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Tx Producer at [0x%04x] - set with a %04x bytes %02x BDs buffer at %08x:%08x\n",
1594 p_ll2
->input
.conn_type
,
1596 pkt
->first_frag_len
,
1598 le32_to_cpu(start_bd
->addr
.hi
),
1599 le32_to_cpu(start_bd
->addr
.lo
));
1601 if (p_ll2
->tx_queue
.cur_send_frag_num
== pkt
->num_of_bds
)
1604 /* Need to provide the packet with additional BDs for frags */
1605 for (frag_idx
= p_ll2
->tx_queue
.cur_send_frag_num
;
1606 frag_idx
< pkt
->num_of_bds
; frag_idx
++) {
1607 struct core_tx_bd
**p_bd
= &p_curp
->bds_set
[frag_idx
].txq_bd
;
1609 *p_bd
= (struct core_tx_bd
*)qed_chain_produce(p_tx_chain
);
1610 (*p_bd
)->bd_data
.as_bitfield
= 0;
1611 (*p_bd
)->bitfield1
= 0;
1612 p_curp
->bds_set
[frag_idx
].tx_frag
= 0;
1613 p_curp
->bds_set
[frag_idx
].frag_len
= 0;
1617 /* This should be called while the Txq spinlock is being held */
1618 static void qed_ll2_tx_packet_notify(struct qed_hwfn
*p_hwfn
,
1619 struct qed_ll2_info
*p_ll2_conn
)
1621 bool b_notify
= p_ll2_conn
->tx_queue
.cur_send_packet
->notify_fw
;
1622 struct qed_ll2_tx_queue
*p_tx
= &p_ll2_conn
->tx_queue
;
1623 struct qed_ll2_tx_packet
*p_pkt
= NULL
;
1624 struct core_db_data db_msg
= { 0, 0, 0 };
1627 /* If there are missing BDs, don't do anything now */
1628 if (p_ll2_conn
->tx_queue
.cur_send_frag_num
!=
1629 p_ll2_conn
->tx_queue
.cur_send_packet
->bd_used
)
1632 /* Push the current packet to the list and clean after it */
1633 list_add_tail(&p_ll2_conn
->tx_queue
.cur_send_packet
->list_entry
,
1634 &p_ll2_conn
->tx_queue
.sending_descq
);
1635 p_ll2_conn
->tx_queue
.cur_send_packet
= NULL
;
1636 p_ll2_conn
->tx_queue
.cur_send_frag_num
= 0;
1638 /* Notify FW of packet only if requested to */
1642 bd_prod
= qed_chain_get_prod_idx(&p_ll2_conn
->tx_queue
.txq_chain
);
1644 while (!list_empty(&p_tx
->sending_descq
)) {
1645 p_pkt
= list_first_entry(&p_tx
->sending_descq
,
1646 struct qed_ll2_tx_packet
, list_entry
);
1650 list_move_tail(&p_pkt
->list_entry
, &p_tx
->active_descq
);
1653 SET_FIELD(db_msg
.params
, CORE_DB_DATA_DEST
, DB_DEST_XCM
);
1654 SET_FIELD(db_msg
.params
, CORE_DB_DATA_AGG_CMD
, DB_AGG_CMD_SET
);
1655 SET_FIELD(db_msg
.params
, CORE_DB_DATA_AGG_VAL_SEL
,
1656 DQ_XCM_CORE_TX_BD_PROD_CMD
);
1657 db_msg
.agg_flags
= DQ_XCM_CORE_DQ_CF_CMD
;
1658 db_msg
.spq_prod
= cpu_to_le16(bd_prod
);
1660 /* Make sure the BDs data is updated before ringing the doorbell */
1663 DIRECT_REG_WR(p_tx
->doorbell_addr
, *((u32
*)&db_msg
));
1666 (NETIF_MSG_TX_QUEUED
| QED_MSG_LL2
),
1667 "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Doorbelled [producer 0x%04x]\n",
1668 p_ll2_conn
->queue_id
,
1670 p_ll2_conn
->input
.conn_type
, db_msg
.spq_prod
);
1673 int qed_ll2_prepare_tx_packet(void *cxt
,
1674 u8 connection_handle
,
1675 struct qed_ll2_tx_pkt_info
*pkt
,
1678 struct qed_hwfn
*p_hwfn
= cxt
;
1679 struct qed_ll2_tx_packet
*p_curp
= NULL
;
1680 struct qed_ll2_info
*p_ll2_conn
= NULL
;
1681 struct qed_ll2_tx_queue
*p_tx
;
1682 struct qed_chain
*p_tx_chain
;
1683 unsigned long flags
;
1686 p_ll2_conn
= qed_ll2_handle_sanity(p_hwfn
, connection_handle
);
1689 p_tx
= &p_ll2_conn
->tx_queue
;
1690 p_tx_chain
= &p_tx
->txq_chain
;
1692 if (pkt
->num_of_bds
> CORE_LL2_TX_MAX_BDS_PER_PACKET
)
1695 spin_lock_irqsave(&p_tx
->lock
, flags
);
1696 if (p_tx
->cur_send_packet
) {
1701 /* Get entry, but only if we have tx elements for it */
1702 if (!list_empty(&p_tx
->free_descq
))
1703 p_curp
= list_first_entry(&p_tx
->free_descq
,
1704 struct qed_ll2_tx_packet
, list_entry
);
1705 if (p_curp
&& qed_chain_get_elem_left(p_tx_chain
) < pkt
->num_of_bds
)
1713 /* Prepare packet and BD, and perhaps send a doorbell to FW */
1714 qed_ll2_prepare_tx_packet_set(p_hwfn
, p_tx
, p_curp
, pkt
, notify_fw
);
1716 qed_ll2_prepare_tx_packet_set_bd(p_hwfn
, p_ll2_conn
, p_curp
, pkt
);
1718 qed_ll2_tx_packet_notify(p_hwfn
, p_ll2_conn
);
1721 spin_unlock_irqrestore(&p_tx
->lock
, flags
);
1725 int qed_ll2_set_fragment_of_tx_packet(void *cxt
,
1726 u8 connection_handle
,
1727 dma_addr_t addr
, u16 nbytes
)
1729 struct qed_ll2_tx_packet
*p_cur_send_packet
= NULL
;
1730 struct qed_hwfn
*p_hwfn
= cxt
;
1731 struct qed_ll2_info
*p_ll2_conn
= NULL
;
1732 u16 cur_send_frag_num
= 0;
1733 struct core_tx_bd
*p_bd
;
1734 unsigned long flags
;
1736 p_ll2_conn
= qed_ll2_handle_sanity(p_hwfn
, connection_handle
);
1740 if (!p_ll2_conn
->tx_queue
.cur_send_packet
)
1743 p_cur_send_packet
= p_ll2_conn
->tx_queue
.cur_send_packet
;
1744 cur_send_frag_num
= p_ll2_conn
->tx_queue
.cur_send_frag_num
;
1746 if (cur_send_frag_num
>= p_cur_send_packet
->bd_used
)
1749 /* Fill the BD information, and possibly notify FW */
1750 p_bd
= p_cur_send_packet
->bds_set
[cur_send_frag_num
].txq_bd
;
1751 DMA_REGPAIR_LE(p_bd
->addr
, addr
);
1752 p_bd
->nbytes
= cpu_to_le16(nbytes
);
1753 p_cur_send_packet
->bds_set
[cur_send_frag_num
].tx_frag
= addr
;
1754 p_cur_send_packet
->bds_set
[cur_send_frag_num
].frag_len
= nbytes
;
1756 p_ll2_conn
->tx_queue
.cur_send_frag_num
++;
1758 spin_lock_irqsave(&p_ll2_conn
->tx_queue
.lock
, flags
);
1759 qed_ll2_tx_packet_notify(p_hwfn
, p_ll2_conn
);
1760 spin_unlock_irqrestore(&p_ll2_conn
->tx_queue
.lock
, flags
);
1765 int qed_ll2_terminate_connection(void *cxt
, u8 connection_handle
)
1767 struct qed_hwfn
*p_hwfn
= cxt
;
1768 struct qed_ll2_info
*p_ll2_conn
= NULL
;
1770 struct qed_ptt
*p_ptt
;
1772 p_ptt
= qed_ptt_acquire(p_hwfn
);
1776 p_ll2_conn
= qed_ll2_handle_sanity_lock(p_hwfn
, connection_handle
);
1782 /* Stop Tx & Rx of connection, if needed */
1783 if (QED_LL2_TX_REGISTERED(p_ll2_conn
)) {
1784 rc
= qed_sp_ll2_tx_queue_stop(p_hwfn
, p_ll2_conn
);
1787 qed_ll2_txq_flush(p_hwfn
, connection_handle
);
1790 if (QED_LL2_RX_REGISTERED(p_ll2_conn
)) {
1791 rc
= qed_sp_ll2_rx_queue_stop(p_hwfn
, p_ll2_conn
);
1794 qed_ll2_rxq_flush(p_hwfn
, connection_handle
);
1797 if (p_ll2_conn
->input
.conn_type
== QED_LL2_TYPE_ISCSI_OOO
)
1798 qed_ooo_release_all_isles(p_hwfn
, p_hwfn
->p_ooo_info
);
1800 if (p_ll2_conn
->input
.conn_type
== QED_LL2_TYPE_FCOE
) {
1801 qed_llh_remove_protocol_filter(p_hwfn
, p_ptt
,
1803 QED_LLH_FILTER_ETHERTYPE
);
1804 qed_llh_remove_protocol_filter(p_hwfn
, p_ptt
,
1806 QED_LLH_FILTER_ETHERTYPE
);
1810 qed_ptt_release(p_hwfn
, p_ptt
);
1814 static void qed_ll2_release_connection_ooo(struct qed_hwfn
*p_hwfn
,
1815 struct qed_ll2_info
*p_ll2_conn
)
1817 struct qed_ooo_buffer
*p_buffer
;
1819 if (p_ll2_conn
->input
.conn_type
!= QED_LL2_TYPE_ISCSI_OOO
)
1822 qed_ooo_release_all_isles(p_hwfn
, p_hwfn
->p_ooo_info
);
1823 while ((p_buffer
= qed_ooo_get_free_buffer(p_hwfn
,
1824 p_hwfn
->p_ooo_info
))) {
1825 dma_free_coherent(&p_hwfn
->cdev
->pdev
->dev
,
1826 p_buffer
->rx_buffer_size
,
1827 p_buffer
->rx_buffer_virt_addr
,
1828 p_buffer
->rx_buffer_phys_addr
);
1833 void qed_ll2_release_connection(void *cxt
, u8 connection_handle
)
1835 struct qed_hwfn
*p_hwfn
= cxt
;
1836 struct qed_ll2_info
*p_ll2_conn
= NULL
;
1838 p_ll2_conn
= qed_ll2_handle_sanity(p_hwfn
, connection_handle
);
1842 if (QED_LL2_RX_REGISTERED(p_ll2_conn
)) {
1843 p_ll2_conn
->rx_queue
.b_cb_registred
= false;
1844 qed_int_unregister_cb(p_hwfn
, p_ll2_conn
->rx_queue
.rx_sb_index
);
1847 if (QED_LL2_TX_REGISTERED(p_ll2_conn
)) {
1848 p_ll2_conn
->tx_queue
.b_cb_registred
= false;
1849 qed_int_unregister_cb(p_hwfn
, p_ll2_conn
->tx_queue
.tx_sb_index
);
1852 kfree(p_ll2_conn
->tx_queue
.descq_array
);
1853 qed_chain_free(p_hwfn
->cdev
, &p_ll2_conn
->tx_queue
.txq_chain
);
1855 kfree(p_ll2_conn
->rx_queue
.descq_array
);
1856 qed_chain_free(p_hwfn
->cdev
, &p_ll2_conn
->rx_queue
.rxq_chain
);
1857 qed_chain_free(p_hwfn
->cdev
, &p_ll2_conn
->rx_queue
.rcq_chain
);
1859 qed_cxt_release_cid(p_hwfn
, p_ll2_conn
->cid
);
1861 qed_ll2_release_connection_ooo(p_hwfn
, p_ll2_conn
);
1863 mutex_lock(&p_ll2_conn
->mutex
);
1864 p_ll2_conn
->b_active
= false;
1865 mutex_unlock(&p_ll2_conn
->mutex
);
1868 int qed_ll2_alloc(struct qed_hwfn
*p_hwfn
)
1870 struct qed_ll2_info
*p_ll2_connections
;
1873 /* Allocate LL2's set struct */
1874 p_ll2_connections
= kcalloc(QED_MAX_NUM_OF_LL2_CONNECTIONS
,
1875 sizeof(struct qed_ll2_info
), GFP_KERNEL
);
1876 if (!p_ll2_connections
) {
1877 DP_NOTICE(p_hwfn
, "Failed to allocate `struct qed_ll2'\n");
1881 for (i
= 0; i
< QED_MAX_NUM_OF_LL2_CONNECTIONS
; i
++)
1882 p_ll2_connections
[i
].my_id
= i
;
1884 p_hwfn
->p_ll2_info
= p_ll2_connections
;
1888 void qed_ll2_setup(struct qed_hwfn
*p_hwfn
)
1892 for (i
= 0; i
< QED_MAX_NUM_OF_LL2_CONNECTIONS
; i
++)
1893 mutex_init(&p_hwfn
->p_ll2_info
[i
].mutex
);
1896 void qed_ll2_free(struct qed_hwfn
*p_hwfn
)
1898 if (!p_hwfn
->p_ll2_info
)
1901 kfree(p_hwfn
->p_ll2_info
);
1902 p_hwfn
->p_ll2_info
= NULL
;
1905 static void _qed_ll2_get_tstats(struct qed_hwfn
*p_hwfn
,
1906 struct qed_ptt
*p_ptt
,
1907 struct qed_ll2_info
*p_ll2_conn
,
1908 struct qed_ll2_stats
*p_stats
)
1910 struct core_ll2_tstorm_per_queue_stat tstats
;
1911 u8 qid
= p_ll2_conn
->queue_id
;
1914 memset(&tstats
, 0, sizeof(tstats
));
1915 tstats_addr
= BAR0_MAP_REG_TSDM_RAM
+
1916 CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(qid
);
1917 qed_memcpy_from(p_hwfn
, p_ptt
, &tstats
, tstats_addr
, sizeof(tstats
));
1919 p_stats
->packet_too_big_discard
=
1920 HILO_64_REGPAIR(tstats
.packet_too_big_discard
);
1921 p_stats
->no_buff_discard
= HILO_64_REGPAIR(tstats
.no_buff_discard
);
1924 static void _qed_ll2_get_ustats(struct qed_hwfn
*p_hwfn
,
1925 struct qed_ptt
*p_ptt
,
1926 struct qed_ll2_info
*p_ll2_conn
,
1927 struct qed_ll2_stats
*p_stats
)
1929 struct core_ll2_ustorm_per_queue_stat ustats
;
1930 u8 qid
= p_ll2_conn
->queue_id
;
1933 memset(&ustats
, 0, sizeof(ustats
));
1934 ustats_addr
= BAR0_MAP_REG_USDM_RAM
+
1935 CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(qid
);
1936 qed_memcpy_from(p_hwfn
, p_ptt
, &ustats
, ustats_addr
, sizeof(ustats
));
1938 p_stats
->rcv_ucast_bytes
= HILO_64_REGPAIR(ustats
.rcv_ucast_bytes
);
1939 p_stats
->rcv_mcast_bytes
= HILO_64_REGPAIR(ustats
.rcv_mcast_bytes
);
1940 p_stats
->rcv_bcast_bytes
= HILO_64_REGPAIR(ustats
.rcv_bcast_bytes
);
1941 p_stats
->rcv_ucast_pkts
= HILO_64_REGPAIR(ustats
.rcv_ucast_pkts
);
1942 p_stats
->rcv_mcast_pkts
= HILO_64_REGPAIR(ustats
.rcv_mcast_pkts
);
1943 p_stats
->rcv_bcast_pkts
= HILO_64_REGPAIR(ustats
.rcv_bcast_pkts
);
1946 static void _qed_ll2_get_pstats(struct qed_hwfn
*p_hwfn
,
1947 struct qed_ptt
*p_ptt
,
1948 struct qed_ll2_info
*p_ll2_conn
,
1949 struct qed_ll2_stats
*p_stats
)
1951 struct core_ll2_pstorm_per_queue_stat pstats
;
1952 u8 stats_id
= p_ll2_conn
->tx_stats_id
;
1955 memset(&pstats
, 0, sizeof(pstats
));
1956 pstats_addr
= BAR0_MAP_REG_PSDM_RAM
+
1957 CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(stats_id
);
1958 qed_memcpy_from(p_hwfn
, p_ptt
, &pstats
, pstats_addr
, sizeof(pstats
));
1960 p_stats
->sent_ucast_bytes
= HILO_64_REGPAIR(pstats
.sent_ucast_bytes
);
1961 p_stats
->sent_mcast_bytes
= HILO_64_REGPAIR(pstats
.sent_mcast_bytes
);
1962 p_stats
->sent_bcast_bytes
= HILO_64_REGPAIR(pstats
.sent_bcast_bytes
);
1963 p_stats
->sent_ucast_pkts
= HILO_64_REGPAIR(pstats
.sent_ucast_pkts
);
1964 p_stats
->sent_mcast_pkts
= HILO_64_REGPAIR(pstats
.sent_mcast_pkts
);
1965 p_stats
->sent_bcast_pkts
= HILO_64_REGPAIR(pstats
.sent_bcast_pkts
);
1968 int qed_ll2_get_stats(void *cxt
,
1969 u8 connection_handle
, struct qed_ll2_stats
*p_stats
)
1971 struct qed_hwfn
*p_hwfn
= cxt
;
1972 struct qed_ll2_info
*p_ll2_conn
= NULL
;
1973 struct qed_ptt
*p_ptt
;
1975 memset(p_stats
, 0, sizeof(*p_stats
));
1977 if ((connection_handle
>= QED_MAX_NUM_OF_LL2_CONNECTIONS
) ||
1978 !p_hwfn
->p_ll2_info
)
1981 p_ll2_conn
= &p_hwfn
->p_ll2_info
[connection_handle
];
1983 p_ptt
= qed_ptt_acquire(p_hwfn
);
1985 DP_ERR(p_hwfn
, "Failed to acquire ptt\n");
1989 _qed_ll2_get_tstats(p_hwfn
, p_ptt
, p_ll2_conn
, p_stats
);
1990 _qed_ll2_get_ustats(p_hwfn
, p_ptt
, p_ll2_conn
, p_stats
);
1991 if (p_ll2_conn
->tx_stats_en
)
1992 _qed_ll2_get_pstats(p_hwfn
, p_ptt
, p_ll2_conn
, p_stats
);
1994 qed_ptt_release(p_hwfn
, p_ptt
);
1998 static void qed_ll2b_release_rx_packet(void *cxt
,
1999 u8 connection_handle
,
2001 dma_addr_t rx_buf_addr
,
2004 struct qed_hwfn
*p_hwfn
= cxt
;
2006 qed_ll2_dealloc_buffer(p_hwfn
->cdev
, cookie
);
2009 static void qed_ll2_register_cb_ops(struct qed_dev
*cdev
,
2010 const struct qed_ll2_cb_ops
*ops
,
2013 cdev
->ll2
->cbs
= ops
;
2014 cdev
->ll2
->cb_cookie
= cookie
;
2017 struct qed_ll2_cbs ll2_cbs
= {
2018 .rx_comp_cb
= &qed_ll2b_complete_rx_packet
,
2019 .rx_release_cb
= &qed_ll2b_release_rx_packet
,
2020 .tx_comp_cb
= &qed_ll2b_complete_tx_packet
,
2021 .tx_release_cb
= &qed_ll2b_complete_tx_packet
,
2024 static void qed_ll2_set_conn_data(struct qed_dev
*cdev
,
2025 struct qed_ll2_acquire_data
*data
,
2026 struct qed_ll2_params
*params
,
2027 enum qed_ll2_conn_type conn_type
,
2028 u8
*handle
, bool lb
)
2030 memset(data
, 0, sizeof(*data
));
2032 data
->input
.conn_type
= conn_type
;
2033 data
->input
.mtu
= params
->mtu
;
2034 data
->input
.rx_num_desc
= QED_LL2_RX_SIZE
;
2035 data
->input
.rx_drop_ttl0_flg
= params
->drop_ttl0_packets
;
2036 data
->input
.rx_vlan_removal_en
= params
->rx_vlan_stripping
;
2037 data
->input
.tx_num_desc
= QED_LL2_TX_SIZE
;
2038 data
->p_connection_handle
= handle
;
2039 data
->cbs
= &ll2_cbs
;
2040 ll2_cbs
.cookie
= QED_LEADING_HWFN(cdev
);
2043 data
->input
.tx_tc
= OOO_LB_TC
;
2044 data
->input
.tx_dest
= QED_LL2_TX_DEST_LB
;
2046 data
->input
.tx_tc
= 0;
2047 data
->input
.tx_dest
= QED_LL2_TX_DEST_NW
;
2051 static int qed_ll2_start_ooo(struct qed_dev
*cdev
,
2052 struct qed_ll2_params
*params
)
2054 struct qed_hwfn
*hwfn
= QED_LEADING_HWFN(cdev
);
2055 u8
*handle
= &hwfn
->pf_params
.iscsi_pf_params
.ll2_ooo_queue_id
;
2056 struct qed_ll2_acquire_data data
;
2059 qed_ll2_set_conn_data(cdev
, &data
, params
,
2060 QED_LL2_TYPE_ISCSI_OOO
, handle
, true);
2062 rc
= qed_ll2_acquire_connection(hwfn
, &data
);
2064 DP_INFO(cdev
, "Failed to acquire LL2 OOO connection\n");
2068 rc
= qed_ll2_establish_connection(hwfn
, *handle
);
2070 DP_INFO(cdev
, "Failed to establist LL2 OOO connection\n");
2077 qed_ll2_release_connection(hwfn
, *handle
);
2079 *handle
= QED_LL2_UNUSED_HANDLE
;
2083 static int qed_ll2_start(struct qed_dev
*cdev
, struct qed_ll2_params
*params
)
2085 struct qed_ll2_buffer
*buffer
, *tmp_buffer
;
2086 enum qed_ll2_conn_type conn_type
;
2087 struct qed_ll2_acquire_data data
;
2088 struct qed_ptt
*p_ptt
;
2092 /* Initialize LL2 locks & lists */
2093 INIT_LIST_HEAD(&cdev
->ll2
->list
);
2094 spin_lock_init(&cdev
->ll2
->lock
);
2095 cdev
->ll2
->rx_size
= NET_SKB_PAD
+ ETH_HLEN
+
2096 L1_CACHE_BYTES
+ params
->mtu
;
2098 /*Allocate memory for LL2 */
2099 DP_INFO(cdev
, "Allocating LL2 buffers of size %08x bytes\n",
2100 cdev
->ll2
->rx_size
);
2101 for (i
= 0; i
< QED_LL2_RX_SIZE
; i
++) {
2102 buffer
= kzalloc(sizeof(*buffer
), GFP_KERNEL
);
2104 DP_INFO(cdev
, "Failed to allocate LL2 buffers\n");
2108 rc
= qed_ll2_alloc_buffer(cdev
, (u8
**)&buffer
->data
,
2109 &buffer
->phys_addr
);
2115 list_add_tail(&buffer
->list
, &cdev
->ll2
->list
);
2118 switch (QED_LEADING_HWFN(cdev
)->hw_info
.personality
) {
2120 conn_type
= QED_LL2_TYPE_FCOE
;
2123 conn_type
= QED_LL2_TYPE_ISCSI
;
2125 case QED_PCI_ETH_ROCE
:
2126 conn_type
= QED_LL2_TYPE_ROCE
;
2129 conn_type
= QED_LL2_TYPE_TEST
;
2132 qed_ll2_set_conn_data(cdev
, &data
, params
, conn_type
,
2133 &cdev
->ll2
->handle
, false);
2135 rc
= qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev
), &data
);
2137 DP_INFO(cdev
, "Failed to acquire LL2 connection\n");
2141 rc
= qed_ll2_establish_connection(QED_LEADING_HWFN(cdev
),
2144 DP_INFO(cdev
, "Failed to establish LL2 connection\n");
2148 /* Post all Rx buffers to FW */
2149 spin_lock_bh(&cdev
->ll2
->lock
);
2150 list_for_each_entry_safe(buffer
, tmp_buffer
, &cdev
->ll2
->list
, list
) {
2151 rc
= qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev
),
2153 buffer
->phys_addr
, 0, buffer
, 1);
2156 "Failed to post an Rx buffer; Deleting it\n");
2157 dma_unmap_single(&cdev
->pdev
->dev
, buffer
->phys_addr
,
2158 cdev
->ll2
->rx_size
, DMA_FROM_DEVICE
);
2159 kfree(buffer
->data
);
2160 list_del(&buffer
->list
);
2163 cdev
->ll2
->rx_cnt
++;
2166 spin_unlock_bh(&cdev
->ll2
->lock
);
2168 if (!cdev
->ll2
->rx_cnt
) {
2169 DP_INFO(cdev
, "Failed passing even a single Rx buffer\n");
2170 goto release_terminate
;
2173 if (!is_valid_ether_addr(params
->ll2_mac_address
)) {
2174 DP_INFO(cdev
, "Invalid Ethernet address\n");
2175 goto release_terminate
;
2178 if (cdev
->hwfns
[0].hw_info
.personality
== QED_PCI_ISCSI
&&
2179 cdev
->hwfns
[0].pf_params
.iscsi_pf_params
.ooo_enable
) {
2180 DP_VERBOSE(cdev
, QED_MSG_STORAGE
, "Starting OOO LL2 queue\n");
2181 rc
= qed_ll2_start_ooo(cdev
, params
);
2184 "Failed to initialize the OOO LL2 queue\n");
2185 goto release_terminate
;
2189 p_ptt
= qed_ptt_acquire(QED_LEADING_HWFN(cdev
));
2191 DP_INFO(cdev
, "Failed to acquire PTT\n");
2192 goto release_terminate
;
2195 rc
= qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev
), p_ptt
,
2196 params
->ll2_mac_address
);
2197 qed_ptt_release(QED_LEADING_HWFN(cdev
), p_ptt
);
2199 DP_ERR(cdev
, "Failed to allocate LLH filter\n");
2200 goto release_terminate_all
;
2203 ether_addr_copy(cdev
->ll2_mac_address
, params
->ll2_mac_address
);
2206 release_terminate_all
:
2209 qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev
), cdev
->ll2
->handle
);
2211 qed_ll2_release_connection(QED_LEADING_HWFN(cdev
), cdev
->ll2
->handle
);
2213 qed_ll2_kill_buffers(cdev
);
2214 cdev
->ll2
->handle
= QED_LL2_UNUSED_HANDLE
;
2218 static int qed_ll2_stop(struct qed_dev
*cdev
)
2220 struct qed_ptt
*p_ptt
;
2223 if (cdev
->ll2
->handle
== QED_LL2_UNUSED_HANDLE
)
2226 p_ptt
= qed_ptt_acquire(QED_LEADING_HWFN(cdev
));
2228 DP_INFO(cdev
, "Failed to acquire PTT\n");
2232 qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev
), p_ptt
,
2233 cdev
->ll2_mac_address
);
2234 qed_ptt_release(QED_LEADING_HWFN(cdev
), p_ptt
);
2235 eth_zero_addr(cdev
->ll2_mac_address
);
2237 if (cdev
->hwfns
[0].hw_info
.personality
== QED_PCI_ISCSI
&&
2238 cdev
->hwfns
[0].pf_params
.iscsi_pf_params
.ooo_enable
)
2239 qed_ll2_stop_ooo(cdev
);
2241 rc
= qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev
),
2244 DP_INFO(cdev
, "Failed to terminate LL2 connection\n");
2246 qed_ll2_kill_buffers(cdev
);
2248 qed_ll2_release_connection(QED_LEADING_HWFN(cdev
), cdev
->ll2
->handle
);
2249 cdev
->ll2
->handle
= QED_LL2_UNUSED_HANDLE
;
2256 static int qed_ll2_start_xmit(struct qed_dev
*cdev
, struct sk_buff
*skb
)
2258 struct qed_ll2_tx_pkt_info pkt
;
2259 const skb_frag_t
*frag
;
2260 int rc
= -EINVAL
, i
;
2265 if (unlikely(skb
->ip_summed
!= CHECKSUM_NONE
)) {
2266 DP_INFO(cdev
, "Cannot transmit a checksumed packet\n");
2270 if (1 + skb_shinfo(skb
)->nr_frags
> CORE_LL2_TX_MAX_BDS_PER_PACKET
) {
2271 DP_ERR(cdev
, "Cannot transmit a packet with %d fragments\n",
2272 1 + skb_shinfo(skb
)->nr_frags
);
2276 mapping
= dma_map_single(&cdev
->pdev
->dev
, skb
->data
,
2277 skb
->len
, DMA_TO_DEVICE
);
2278 if (unlikely(dma_mapping_error(&cdev
->pdev
->dev
, mapping
))) {
2279 DP_NOTICE(cdev
, "SKB mapping failed\n");
2283 /* Request HW to calculate IP csum */
2284 if (!((vlan_get_protocol(skb
) == htons(ETH_P_IPV6
)) &&
2285 ipv6_hdr(skb
)->nexthdr
== NEXTHDR_IPV6
))
2286 flags
|= BIT(CORE_TX_BD_DATA_IP_CSUM_SHIFT
);
2288 if (skb_vlan_tag_present(skb
)) {
2289 vlan
= skb_vlan_tag_get(skb
);
2290 flags
|= BIT(CORE_TX_BD_DATA_VLAN_INSERTION_SHIFT
);
2293 memset(&pkt
, 0, sizeof(pkt
));
2294 pkt
.num_of_bds
= 1 + skb_shinfo(skb
)->nr_frags
;
2296 pkt
.bd_flags
= flags
;
2297 pkt
.tx_dest
= QED_LL2_TX_DEST_NW
;
2298 pkt
.first_frag
= mapping
;
2299 pkt
.first_frag_len
= skb
->len
;
2302 rc
= qed_ll2_prepare_tx_packet(&cdev
->hwfns
[0], cdev
->ll2
->handle
,
2307 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
2308 frag
= &skb_shinfo(skb
)->frags
[i
];
2310 mapping
= skb_frag_dma_map(&cdev
->pdev
->dev
, frag
, 0,
2311 skb_frag_size(frag
), DMA_TO_DEVICE
);
2313 if (unlikely(dma_mapping_error(&cdev
->pdev
->dev
, mapping
))) {
2315 "Unable to map frag - dropping packet\n");
2319 rc
= qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev
),
2322 skb_frag_size(frag
));
2324 /* if failed not much to do here, partial packet has been posted
2325 * we can't free memory, will need to wait for completion.
2334 dma_unmap_single(&cdev
->pdev
->dev
, mapping
, skb
->len
, DMA_TO_DEVICE
);
2340 static int qed_ll2_stats(struct qed_dev
*cdev
, struct qed_ll2_stats
*stats
)
2345 return qed_ll2_get_stats(QED_LEADING_HWFN(cdev
),
2346 cdev
->ll2
->handle
, stats
);
2349 const struct qed_ll2_ops qed_ll2_ops_pass
= {
2350 .start
= &qed_ll2_start
,
2351 .stop
= &qed_ll2_stop
,
2352 .start_xmit
= &qed_ll2_start_xmit
,
2353 .register_cb_ops
= &qed_ll2_register_cb_ops
,
2354 .get_stats
= &qed_ll2_stats
,
2357 int qed_ll2_alloc_if(struct qed_dev
*cdev
)
2359 cdev
->ll2
= kzalloc(sizeof(*cdev
->ll2
), GFP_KERNEL
);
2360 return cdev
->ll2
? 0 : -ENOMEM
;
2363 void qed_ll2_dealloc_if(struct qed_dev
*cdev
)