1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2014-2018 Chelsio Communications.
14 #include <netinet/in.h>
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_cycles.h>
19 #include <rte_interrupts.h>
21 #include <rte_debug.h>
23 #include <rte_atomic.h>
24 #include <rte_branch_prediction.h>
25 #include <rte_memory.h>
26 #include <rte_tailq.h>
28 #include <rte_alarm.h>
29 #include <rte_ether.h>
30 #include <rte_ethdev_driver.h>
31 #include <rte_ethdev_pci.h>
32 #include <rte_random.h>
34 #include <rte_kvargs.h>
36 #include "base/common.h"
37 #include "base/t4_regs.h"
38 #include "base/t4_msg.h"
45 * Allocate a chunk of memory. The allocated memory is cleared.
47 void *t4_alloc_mem(size_t size
)
49 return rte_zmalloc(NULL
, size
, 0);
53 * Free memory allocated through t4_alloc_mem().
55 void t4_free_mem(void *addr
)
61 * Response queue handler for the FW event queue.
63 static int fwevtq_handler(struct sge_rspq
*q
, const __be64
*rsp
,
64 __rte_unused
const struct pkt_gl
*gl
)
66 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
68 rsp
++; /* skip RSS header */
71 * FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
73 if (unlikely(opcode
== CPL_FW4_MSG
&&
74 ((const struct cpl_fw4_msg
*)rsp
)->type
==
77 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
79 if (opcode
!= CPL_SGE_EGR_UPDATE
) {
80 dev_err(q
->adapter
, "unexpected FW4/CPL %#x on FW event queue\n",
86 if (likely(opcode
== CPL_SGE_EGR_UPDATE
)) {
88 } else if (opcode
== CPL_FW6_MSG
|| opcode
== CPL_FW4_MSG
) {
89 const struct cpl_fw6_msg
*msg
= (const void *)rsp
;
91 t4_handle_fw_rpl(q
->adapter
, msg
->data
);
92 } else if (opcode
== CPL_ABORT_RPL_RSS
) {
93 const struct cpl_abort_rpl_rss
*p
= (const void *)rsp
;
95 hash_del_filter_rpl(q
->adapter
, p
);
96 } else if (opcode
== CPL_SET_TCB_RPL
) {
97 const struct cpl_set_tcb_rpl
*p
= (const void *)rsp
;
99 filter_rpl(q
->adapter
, p
);
100 } else if (opcode
== CPL_ACT_OPEN_RPL
) {
101 const struct cpl_act_open_rpl
*p
= (const void *)rsp
;
103 hash_filter_rpl(q
->adapter
, p
);
104 } else if (opcode
== CPL_L2T_WRITE_RPL
) {
105 const struct cpl_l2t_write_rpl
*p
= (const void *)rsp
;
107 do_l2t_write_rpl(q
->adapter
, p
);
109 dev_err(adapter
, "unexpected CPL %#x on FW event queue\n",
117 * Setup sge control queues to pass control information.
119 int cxgbe_setup_sge_ctrl_txq(struct adapter
*adapter
)
121 struct sge
*s
= &adapter
->sge
;
124 for_each_port(adapter
, i
) {
125 struct port_info
*pi
= adap2pinfo(adapter
, i
);
126 char name
[RTE_ETH_NAME_MAX_LEN
];
127 struct sge_ctrl_txq
*q
= &s
->ctrlq
[i
];
130 err
= t4_sge_alloc_ctrl_txq(adapter
, q
,
135 dev_err(adapter
, "Failed to alloc ctrl txq. Err: %d",
139 snprintf(name
, sizeof(name
), "%s_ctrl_pool_%d",
140 pi
->eth_dev
->device
->driver
->name
,
141 pi
->eth_dev
->data
->port_id
);
142 q
->mb_pool
= rte_pktmbuf_pool_create(name
, s
->ctrlq
[i
].q
.size
,
145 RTE_MBUF_DEFAULT_BUF_SIZE
,
150 "Can't create ctrl pool for port %d. Err: %d\n",
151 pi
->eth_dev
->data
->port_id
, err
);
157 t4_free_sge_resources(adapter
);
162 * cxgbe_poll_for_completion: Poll rxq for completion
164 * @ms: milliseconds to delay
165 * @cnt: number of times to poll
166 * @c: completion to check for 'done' status
168 * Polls the rxq for reples until completion is done or the count
171 int cxgbe_poll_for_completion(struct sge_rspq
*q
, unsigned int ms
,
172 unsigned int cnt
, struct t4_completion
*c
)
175 unsigned int work_done
, budget
= 32;
180 for (i
= 0; i
< cnt
; i
++) {
181 cxgbe_poll(q
, NULL
, budget
, &work_done
);
182 t4_os_lock(&c
->lock
);
184 t4_os_unlock(&c
->lock
);
187 t4_os_unlock(&c
->lock
);
193 int cxgbe_setup_sge_fwevtq(struct adapter
*adapter
)
195 struct sge
*s
= &adapter
->sge
;
199 err
= t4_sge_alloc_rxq(adapter
, &s
->fw_evtq
, true, adapter
->eth_dev
,
200 msi_idx
, NULL
, fwevtq_handler
, -1, NULL
, 0,
205 static int closest_timer(const struct sge
*s
, int time
)
207 unsigned int i
, match
= 0;
208 int delta
, min_delta
= INT_MAX
;
210 for (i
= 0; i
< ARRAY_SIZE(s
->timer_val
); i
++) {
211 delta
= time
- s
->timer_val
[i
];
214 if (delta
< min_delta
) {
222 static int closest_thres(const struct sge
*s
, int thres
)
224 unsigned int i
, match
= 0;
225 int delta
, min_delta
= INT_MAX
;
227 for (i
= 0; i
< ARRAY_SIZE(s
->counter_val
); i
++) {
228 delta
= thres
- s
->counter_val
[i
];
231 if (delta
< min_delta
) {
240 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters
242 * @us: the hold-off time in us, or 0 to disable timer
243 * @cnt: the hold-off packet count, or 0 to disable counter
245 * Sets an Rx queue's interrupt hold-off time and packet count. At least
246 * one of the two needs to be enabled for the queue to generate interrupts.
248 int cxgb4_set_rspq_intr_params(struct sge_rspq
*q
, unsigned int us
,
251 struct adapter
*adap
= q
->adapter
;
252 unsigned int timer_val
;
258 new_idx
= closest_thres(&adap
->sge
, cnt
);
259 if (q
->desc
&& q
->pktcnt_idx
!= new_idx
) {
260 /* the queue has already been created, update it */
261 v
= V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ
) |
263 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH
) |
264 V_FW_PARAMS_PARAM_YZ(q
->cntxt_id
);
265 err
= t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1,
270 q
->pktcnt_idx
= new_idx
;
273 timer_val
= (us
== 0) ? X_TIMERREG_RESTART_COUNTER
:
274 closest_timer(&adap
->sge
, us
);
277 q
->intr_params
= V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX
);
279 q
->intr_params
= V_QINTR_TIMER_IDX(timer_val
) |
280 V_QINTR_CNT_EN(cnt
> 0);
285 * Allocate an active-open TID and set it to the supplied value.
287 int cxgbe_alloc_atid(struct tid_info
*t
, void *data
)
291 t4_os_lock(&t
->atid_lock
);
293 union aopen_entry
*p
= t
->afree
;
295 atid
= p
- t
->atid_tab
;
300 t4_os_unlock(&t
->atid_lock
);
305 * Release an active-open TID.
307 void cxgbe_free_atid(struct tid_info
*t
, unsigned int atid
)
309 union aopen_entry
*p
= &t
->atid_tab
[atid
];
311 t4_os_lock(&t
->atid_lock
);
315 t4_os_unlock(&t
->atid_lock
);
319 * Populate a TID_RELEASE WR. Caller must properly size the skb.
321 static void mk_tid_release(struct rte_mbuf
*mbuf
, unsigned int tid
)
323 struct cpl_tid_release
*req
;
325 req
= rte_pktmbuf_mtod(mbuf
, struct cpl_tid_release
*);
326 INIT_TP_WR_MIT_CPL(req
, CPL_TID_RELEASE
, tid
);
330 * Release a TID and inform HW. If we are unable to allocate the release
331 * message we defer to a work queue.
333 void cxgbe_remove_tid(struct tid_info
*t
, unsigned int chan
, unsigned int tid
,
334 unsigned short family
)
336 struct rte_mbuf
*mbuf
;
337 struct adapter
*adap
= container_of(t
, struct adapter
, tids
);
339 WARN_ON(tid
>= t
->ntids
);
341 if (t
->tid_tab
[tid
]) {
342 t
->tid_tab
[tid
] = NULL
;
343 rte_atomic32_dec(&t
->conns_in_use
);
344 if (t
->hash_base
&& tid
>= t
->hash_base
) {
345 if (family
== FILTER_TYPE_IPV4
)
346 rte_atomic32_dec(&t
->hash_tids_in_use
);
348 if (family
== FILTER_TYPE_IPV4
)
349 rte_atomic32_dec(&t
->tids_in_use
);
353 mbuf
= rte_pktmbuf_alloc((&adap
->sge
.ctrlq
[chan
])->mb_pool
);
355 mbuf
->data_len
= sizeof(struct cpl_tid_release
);
356 mbuf
->pkt_len
= mbuf
->data_len
;
357 mk_tid_release(mbuf
, tid
);
358 t4_mgmt_tx(&adap
->sge
.ctrlq
[chan
], mbuf
);
365 void cxgbe_insert_tid(struct tid_info
*t
, void *data
, unsigned int tid
,
366 unsigned short family
)
368 t
->tid_tab
[tid
] = data
;
369 if (t
->hash_base
&& tid
>= t
->hash_base
) {
370 if (family
== FILTER_TYPE_IPV4
)
371 rte_atomic32_inc(&t
->hash_tids_in_use
);
373 if (family
== FILTER_TYPE_IPV4
)
374 rte_atomic32_inc(&t
->tids_in_use
);
377 rte_atomic32_inc(&t
->conns_in_use
);
383 static void tid_free(struct tid_info
*t
)
387 rte_bitmap_free(t
->ftid_bmap
);
389 if (t
->ftid_bmap_array
)
390 t4_os_free(t
->ftid_bmap_array
);
392 t4_os_free(t
->tid_tab
);
395 memset(t
, 0, sizeof(struct tid_info
));
399 * Allocate and initialize the TID tables. Returns 0 on success.
401 static int tid_init(struct tid_info
*t
)
404 unsigned int ftid_bmap_size
;
405 unsigned int natids
= t
->natids
;
406 unsigned int max_ftids
= t
->nftids
;
408 ftid_bmap_size
= rte_bitmap_get_memory_footprint(t
->nftids
);
409 size
= t
->ntids
* sizeof(*t
->tid_tab
) +
410 max_ftids
* sizeof(*t
->ftid_tab
) +
411 natids
* sizeof(*t
->atid_tab
);
413 t
->tid_tab
= t4_os_alloc(size
);
417 t
->atid_tab
= (union aopen_entry
*)&t
->tid_tab
[t
->ntids
];
418 t
->ftid_tab
= (struct filter_entry
*)&t
->atid_tab
[t
->natids
];
419 t
->ftid_bmap_array
= t4_os_alloc(ftid_bmap_size
);
420 if (!t
->ftid_bmap_array
) {
425 t4_os_lock_init(&t
->atid_lock
);
426 t4_os_lock_init(&t
->ftid_lock
);
430 rte_atomic32_init(&t
->tids_in_use
);
431 rte_atomic32_set(&t
->tids_in_use
, 0);
432 rte_atomic32_init(&t
->conns_in_use
);
433 rte_atomic32_set(&t
->conns_in_use
, 0);
435 /* Setup the free list for atid_tab and clear the stid bitmap. */
438 t
->atid_tab
[natids
- 1].next
= &t
->atid_tab
[natids
];
439 t
->afree
= t
->atid_tab
;
442 t
->ftid_bmap
= rte_bitmap_init(t
->nftids
, t
->ftid_bmap_array
,
452 static inline bool is_x_1g_port(const struct link_config
*lc
)
454 return (lc
->pcaps
& FW_PORT_CAP32_SPEED_1G
) != 0;
457 static inline bool is_x_10g_port(const struct link_config
*lc
)
459 unsigned int speeds
, high_speeds
;
461 speeds
= V_FW_PORT_CAP32_SPEED(G_FW_PORT_CAP32_SPEED(lc
->pcaps
));
462 high_speeds
= speeds
&
463 ~(FW_PORT_CAP32_SPEED_100M
| FW_PORT_CAP32_SPEED_1G
);
465 return high_speeds
!= 0;
468 static inline void init_rspq(struct adapter
*adap
, struct sge_rspq
*q
,
469 unsigned int us
, unsigned int cnt
,
470 unsigned int size
, unsigned int iqe_size
)
473 cxgb4_set_rspq_intr_params(q
, us
, cnt
);
474 q
->iqe_len
= iqe_size
;
478 int cxgbe_cfg_queue_count(struct rte_eth_dev
*eth_dev
)
480 struct port_info
*pi
= (struct port_info
*)(eth_dev
->data
->dev_private
);
481 struct adapter
*adap
= pi
->adapter
;
482 struct sge
*s
= &adap
->sge
;
483 unsigned int max_queues
= s
->max_ethqsets
/ adap
->params
.nports
;
485 if ((eth_dev
->data
->nb_rx_queues
< 1) ||
486 (eth_dev
->data
->nb_tx_queues
< 1))
489 if ((eth_dev
->data
->nb_rx_queues
> max_queues
) ||
490 (eth_dev
->data
->nb_tx_queues
> max_queues
))
493 if (eth_dev
->data
->nb_rx_queues
> pi
->rss_size
)
496 /* We must configure RSS, since config has changed*/
497 pi
->flags
&= ~PORT_RSS_DONE
;
499 pi
->n_rx_qsets
= eth_dev
->data
->nb_rx_queues
;
500 pi
->n_tx_qsets
= eth_dev
->data
->nb_tx_queues
;
505 void cxgbe_cfg_queues(struct rte_eth_dev
*eth_dev
)
507 struct rte_config
*config
= rte_eal_get_configuration();
508 struct port_info
*pi
= (struct port_info
*)(eth_dev
->data
->dev_private
);
509 struct adapter
*adap
= pi
->adapter
;
510 struct sge
*s
= &adap
->sge
;
511 unsigned int i
, nb_ports
= 0, qidx
= 0;
512 unsigned int q_per_port
= 0;
514 if (!(adap
->flags
& CFG_QUEUES
)) {
515 for_each_port(adap
, i
) {
516 struct port_info
*tpi
= adap2pinfo(adap
, i
);
518 nb_ports
+= (is_x_10g_port(&tpi
->link_cfg
)) ||
519 is_x_1g_port(&tpi
->link_cfg
) ? 1 : 0;
523 * We default up to # of cores queues per 1G/10G port.
526 q_per_port
= (s
->max_ethqsets
-
527 (adap
->params
.nports
- nb_ports
)) /
530 if (q_per_port
> config
->lcore_count
)
531 q_per_port
= config
->lcore_count
;
533 for_each_port(adap
, i
) {
534 struct port_info
*pi
= adap2pinfo(adap
, i
);
536 pi
->first_qset
= qidx
;
538 /* Initially n_rx_qsets == n_tx_qsets */
539 pi
->n_rx_qsets
= (is_x_10g_port(&pi
->link_cfg
) ||
540 is_x_1g_port(&pi
->link_cfg
)) ?
542 pi
->n_tx_qsets
= pi
->n_rx_qsets
;
544 if (pi
->n_rx_qsets
> pi
->rss_size
)
545 pi
->n_rx_qsets
= pi
->rss_size
;
547 qidx
+= pi
->n_rx_qsets
;
550 for (i
= 0; i
< ARRAY_SIZE(s
->ethrxq
); i
++) {
551 struct sge_eth_rxq
*r
= &s
->ethrxq
[i
];
553 init_rspq(adap
, &r
->rspq
, 5, 32, 1024, 64);
555 r
->fl
.size
= (r
->usembufs
? 1024 : 72);
558 for (i
= 0; i
< ARRAY_SIZE(s
->ethtxq
); i
++)
559 s
->ethtxq
[i
].q
.size
= 1024;
561 init_rspq(adap
, &adap
->sge
.fw_evtq
, 0, 0, 1024, 64);
562 adap
->flags
|= CFG_QUEUES
;
566 void cxgbe_stats_get(struct port_info
*pi
, struct port_stats
*stats
)
568 t4_get_port_stats_offset(pi
->adapter
, pi
->tx_chan
, stats
,
572 void cxgbe_stats_reset(struct port_info
*pi
)
574 t4_clr_port_stats(pi
->adapter
, pi
->tx_chan
);
577 static void setup_memwin(struct adapter
*adap
)
581 /* For T5, only relative offset inside the PCIe BAR is passed */
582 mem_win0_base
= MEMWIN0_BASE
;
585 * Set up memory window for accessing adapter memory ranges. (Read
586 * back MA register to ensure that changes propagate before we attempt
587 * to use the new values.)
590 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN
,
592 mem_win0_base
| V_BIR(0) |
593 V_WINDOW(ilog2(MEMWIN0_APERTURE
) - X_WINDOW_SHIFT
));
595 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN
,
599 int cxgbe_init_rss(struct adapter
*adap
)
606 err
= t4_init_rss_mode(adap
, adap
->mbox
);
611 for_each_port(adap
, i
) {
612 struct port_info
*pi
= adap2pinfo(adap
, i
);
614 pi
->rss
= rte_zmalloc(NULL
, pi
->rss_size
* sizeof(u16
), 0);
618 pi
->rss_hf
= CXGBE_RSS_HF_ALL
;
624 * Dump basic information about the adapter.
626 void cxgbe_print_adapter_info(struct adapter
*adap
)
629 * Hardware/Firmware/etc. Version/Revision IDs.
631 t4_dump_version_info(adap
);
634 void cxgbe_print_port_info(struct adapter
*adap
)
638 struct rte_pci_addr
*loc
= &adap
->pdev
->addr
;
640 for_each_port(adap
, i
) {
641 const struct port_info
*pi
= adap2pinfo(adap
, i
);
644 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_100M
)
645 bufp
+= sprintf(bufp
, "100M/");
646 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_1G
)
647 bufp
+= sprintf(bufp
, "1G/");
648 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_10G
)
649 bufp
+= sprintf(bufp
, "10G/");
650 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_25G
)
651 bufp
+= sprintf(bufp
, "25G/");
652 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_40G
)
653 bufp
+= sprintf(bufp
, "40G/");
654 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_50G
)
655 bufp
+= sprintf(bufp
, "50G/");
656 if (pi
->link_cfg
.pcaps
& FW_PORT_CAP32_SPEED_100G
)
657 bufp
+= sprintf(bufp
, "100G/");
660 sprintf(bufp
, "BASE-%s",
661 t4_get_port_type_description(
662 (enum fw_port_type
)pi
->port_type
));
665 " " PCI_PRI_FMT
" Chelsio rev %d %s %s\n",
666 loc
->domain
, loc
->bus
, loc
->devid
, loc
->function
,
667 CHELSIO_CHIP_RELEASE(adap
->params
.chip
), buf
,
668 (adap
->flags
& USING_MSIX
) ? " MSI-X" :
669 (adap
->flags
& USING_MSI
) ? " MSI" : "");
674 check_devargs_handler(__rte_unused
const char *key
, const char *value
,
675 __rte_unused
void *opaque
)
677 if (strcmp(value
, "1"))
683 int cxgbe_get_devargs(struct rte_devargs
*devargs
, const char *key
)
685 struct rte_kvargs
*kvlist
;
690 kvlist
= rte_kvargs_parse(devargs
->args
, NULL
);
694 if (!rte_kvargs_count(kvlist
, key
)) {
695 rte_kvargs_free(kvlist
);
699 if (rte_kvargs_process(kvlist
, key
,
700 check_devargs_handler
, NULL
) < 0) {
701 rte_kvargs_free(kvlist
);
704 rte_kvargs_free(kvlist
);
709 static void configure_vlan_types(struct adapter
*adapter
)
711 struct rte_pci_device
*pdev
= adapter
->pdev
;
714 for_each_port(adapter
, i
) {
715 /* OVLAN Type 0x88a8 */
716 t4_set_reg_field(adapter
, MPS_PORT_RX_OVLAN_REG(i
, A_RX_OVLAN0
),
717 V_OVLAN_MASK(M_OVLAN_MASK
) |
718 V_OVLAN_ETYPE(M_OVLAN_ETYPE
),
719 V_OVLAN_MASK(M_OVLAN_MASK
) |
720 V_OVLAN_ETYPE(0x88a8));
721 /* OVLAN Type 0x9100 */
722 t4_set_reg_field(adapter
, MPS_PORT_RX_OVLAN_REG(i
, A_RX_OVLAN1
),
723 V_OVLAN_MASK(M_OVLAN_MASK
) |
724 V_OVLAN_ETYPE(M_OVLAN_ETYPE
),
725 V_OVLAN_MASK(M_OVLAN_MASK
) |
726 V_OVLAN_ETYPE(0x9100));
727 /* OVLAN Type 0x8100 */
728 t4_set_reg_field(adapter
, MPS_PORT_RX_OVLAN_REG(i
, A_RX_OVLAN2
),
729 V_OVLAN_MASK(M_OVLAN_MASK
) |
730 V_OVLAN_ETYPE(M_OVLAN_ETYPE
),
731 V_OVLAN_MASK(M_OVLAN_MASK
) |
732 V_OVLAN_ETYPE(0x8100));
735 t4_set_reg_field(adapter
, MPS_PORT_RX_IVLAN(i
),
736 V_IVLAN_ETYPE(M_IVLAN_ETYPE
),
737 V_IVLAN_ETYPE(0x8100));
739 t4_set_reg_field(adapter
, MPS_PORT_RX_CTL(i
),
740 F_OVLAN_EN0
| F_OVLAN_EN1
|
741 F_OVLAN_EN2
| F_IVLAN_EN
,
742 F_OVLAN_EN0
| F_OVLAN_EN1
|
743 F_OVLAN_EN2
| F_IVLAN_EN
);
746 if (cxgbe_get_devargs(pdev
->device
.devargs
, CXGBE_DEVARG_KEEP_OVLAN
))
747 t4_tp_wr_bits_indirect(adapter
, A_TP_INGRESS_CONFIG
,
748 V_RM_OVLAN(1), V_RM_OVLAN(0));
751 static void configure_pcie_ext_tag(struct adapter
*adapter
)
754 int pos
= t4_os_find_pci_capability(adapter
, PCI_CAP_ID_EXP
);
760 t4_os_pci_read_cfg2(adapter
, pos
+ PCI_EXP_DEVCTL
, &v
);
761 v
|= PCI_EXP_DEVCTL_EXT_TAG
;
762 t4_os_pci_write_cfg2(adapter
, pos
+ PCI_EXP_DEVCTL
, v
);
763 if (is_t6(adapter
->params
.chip
)) {
764 t4_set_reg_field(adapter
, A_PCIE_CFG2
,
765 V_T6_TOTMAXTAG(M_T6_TOTMAXTAG
),
767 t4_set_reg_field(adapter
, A_PCIE_CMD_CFG
,
768 V_T6_MINTAG(M_T6_MINTAG
),
771 t4_set_reg_field(adapter
, A_PCIE_CFG2
,
772 V_TOTMAXTAG(M_TOTMAXTAG
),
774 t4_set_reg_field(adapter
, A_PCIE_CMD_CFG
,
781 /* Figure out how many Queue Sets we can support */
782 void cxgbe_configure_max_ethqsets(struct adapter
*adapter
)
784 unsigned int ethqsets
;
787 * We need to reserve an Ingress Queue for the Asynchronous Firmware
790 * For each Queue Set, we'll need the ability to allocate two Egress
791 * Contexts -- one for the Ingress Queue Free List and one for the TX
794 if (is_pf4(adapter
)) {
795 struct pf_resources
*pfres
= &adapter
->params
.pfres
;
797 ethqsets
= pfres
->niqflint
- 1;
798 if (pfres
->neq
< ethqsets
* 2)
799 ethqsets
= pfres
->neq
/ 2;
801 struct vf_resources
*vfres
= &adapter
->params
.vfres
;
803 ethqsets
= vfres
->niqflint
- 1;
804 if (vfres
->nethctrl
!= ethqsets
)
805 ethqsets
= min(vfres
->nethctrl
, ethqsets
);
806 if (vfres
->neq
< ethqsets
* 2)
807 ethqsets
= vfres
->neq
/ 2;
810 if (ethqsets
> MAX_ETH_QSETS
)
811 ethqsets
= MAX_ETH_QSETS
;
812 adapter
->sge
.max_ethqsets
= ethqsets
;
816 * Tweak configuration based on system architecture, etc. Most of these have
817 * defaults assigned to them by Firmware Configuration Files (if we're using
818 * them) but need to be explicitly set if we're using hard-coded
819 * initialization. So these are essentially common tweaks/settings for
820 * Configuration Files and hard-coded initialization ...
822 static int adap_init0_tweaks(struct adapter
*adapter
)
827 * Fix up various Host-Dependent Parameters like Page Size, Cache
828 * Line Size, etc. The firmware default is for a 4KB Page Size and
829 * 64B Cache Line Size ...
831 t4_fixup_host_params_compat(adapter
, CXGBE_PAGE_SIZE
, L1_CACHE_BYTES
,
835 * Keep the chip default offset to deliver Ingress packets into our
836 * DMA buffers to zero
839 t4_set_reg_field(adapter
, A_SGE_CONTROL
, V_PKTSHIFT(M_PKTSHIFT
),
840 V_PKTSHIFT(rx_dma_offset
));
842 t4_set_reg_field(adapter
, A_SGE_FLM_CFG
,
843 V_CREDITCNT(M_CREDITCNT
) | M_CREDITCNTPACKING
,
844 V_CREDITCNT(3) | V_CREDITCNTPACKING(1));
846 t4_set_reg_field(adapter
, A_SGE_INGRESS_RX_THRESHOLD
,
847 V_THRESHOLD_3(M_THRESHOLD_3
), V_THRESHOLD_3(32U));
849 t4_set_reg_field(adapter
, A_SGE_CONTROL2
, V_IDMAARBROUNDROBIN(1U),
850 V_IDMAARBROUNDROBIN(1U));
853 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
854 * adds the pseudo header itself.
856 t4_tp_wr_bits_indirect(adapter
, A_TP_INGRESS_CONFIG
,
857 F_CSUM_HAS_PSEUDO_HDR
, 0);
863 * Attempt to initialize the adapter via a Firmware Configuration File.
865 static int adap_init0_config(struct adapter
*adapter
, int reset
)
867 struct fw_caps_config_cmd caps_cmd
;
868 unsigned long mtype
= 0, maddr
= 0;
869 u32 finiver
, finicsum
, cfcsum
;
871 int config_issued
= 0;
873 char config_name
[20];
876 * Reset device if necessary.
879 ret
= t4_fw_reset(adapter
, adapter
->mbox
,
880 F_PIORSTMODE
| F_PIORST
);
882 dev_warn(adapter
, "Firmware reset failed, error %d\n",
888 cfg_addr
= t4_flash_cfg_addr(adapter
);
891 dev_warn(adapter
, "Finding address for firmware config file in flash failed, error %d\n",
896 strcpy(config_name
, "On Flash");
897 mtype
= FW_MEMTYPE_CF_FLASH
;
901 * Issue a Capability Configuration command to the firmware to get it
902 * to parse the Configuration File. We don't use t4_fw_config_file()
903 * because we want the ability to modify various features after we've
904 * processed the configuration file ...
906 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
907 caps_cmd
.op_to_write
= cpu_to_be32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
908 F_FW_CMD_REQUEST
| F_FW_CMD_READ
);
909 caps_cmd
.cfvalid_to_len16
=
910 cpu_to_be32(F_FW_CAPS_CONFIG_CMD_CFVALID
|
911 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype
) |
912 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr
>> 16) |
914 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
917 * If the CAPS_CONFIG failed with an ENOENT (for a Firmware
918 * Configuration File in FLASH), our last gasp effort is to use the
919 * Firmware Configuration File which is embedded in the firmware. A
920 * very few early versions of the firmware didn't have one embedded
921 * but we can ignore those.
923 if (ret
== -ENOENT
) {
924 dev_info(adapter
, "%s: Going for embedded config in firmware..\n",
927 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
928 caps_cmd
.op_to_write
=
929 cpu_to_be32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
930 F_FW_CMD_REQUEST
| F_FW_CMD_READ
);
931 caps_cmd
.cfvalid_to_len16
= cpu_to_be32(FW_LEN16(caps_cmd
));
932 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
,
933 sizeof(caps_cmd
), &caps_cmd
);
934 strcpy(config_name
, "Firmware Default");
941 finiver
= be32_to_cpu(caps_cmd
.finiver
);
942 finicsum
= be32_to_cpu(caps_cmd
.finicsum
);
943 cfcsum
= be32_to_cpu(caps_cmd
.cfcsum
);
944 if (finicsum
!= cfcsum
)
945 dev_warn(adapter
, "Configuration File checksum mismatch: [fini] csum=%#x, computed csum=%#x\n",
949 * If we're a pure NIC driver then disable all offloading facilities.
950 * This will allow the firmware to optimize aspects of the hardware
951 * configuration which will result in improved performance.
953 caps_cmd
.niccaps
&= cpu_to_be16(~FW_CAPS_CONFIG_NIC_ETHOFLD
);
954 caps_cmd
.toecaps
= 0;
955 caps_cmd
.iscsicaps
= 0;
956 caps_cmd
.rdmacaps
= 0;
957 caps_cmd
.fcoecaps
= 0;
960 * And now tell the firmware to use the configuration we just loaded.
962 caps_cmd
.op_to_write
= cpu_to_be32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
963 F_FW_CMD_REQUEST
| F_FW_CMD_WRITE
);
964 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
965 ret
= t4_wr_mbox(adapter
, adapter
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
968 dev_warn(adapter
, "Unable to finalize Firmware Capabilities %d\n",
974 * Tweak configuration based on system architecture, etc.
976 ret
= adap_init0_tweaks(adapter
);
978 dev_warn(adapter
, "Unable to do init0-tweaks %d\n", -ret
);
983 * And finally tell the firmware to initialize itself using the
984 * parameters from the Configuration File.
986 ret
= t4_fw_initialize(adapter
, adapter
->mbox
);
988 dev_warn(adapter
, "Initializing Firmware failed, error %d\n",
994 * Return successfully and note that we're operating with parameters
995 * not supplied by the driver, rather than from hard-wired
996 * initialization constants buried in the driver.
999 "Successfully configured using Firmware Configuration File \"%s\", version %#x, computed checksum %#x\n",
1000 config_name
, finiver
, cfcsum
);
1005 * Something bad happened. Return the error ... (If the "error"
1006 * is that there's no Configuration File on the adapter we don't
1007 * want to issue a warning since this is fairly common.)
1010 if (config_issued
&& ret
!= -ENOENT
)
1011 dev_warn(adapter
, "\"%s\" configuration file error %d\n",
1014 dev_debug(adapter
, "%s: returning ret = %d ..\n", __func__
, ret
);
1018 static int adap_init0(struct adapter
*adap
)
1020 struct fw_caps_config_cmd caps_cmd
;
1023 enum dev_state state
;
1024 u32 params
[7], val
[7];
1026 int mbox
= adap
->mbox
;
1029 * Contact FW, advertising Master capability.
1031 ret
= t4_fw_hello(adap
, adap
->mbox
, adap
->mbox
, MASTER_MAY
, &state
);
1033 dev_err(adap
, "%s: could not connect to FW, error %d\n",
1038 CXGBE_DEBUG_MBOX(adap
, "%s: adap->mbox = %d; ret = %d\n", __func__
,
1042 adap
->flags
|= MASTER_PF
;
1044 if (state
== DEV_STATE_INIT
) {
1046 * Force halt and reset FW because a previous instance may have
1047 * exited abnormally without properly shutting down
1049 ret
= t4_fw_halt(adap
, adap
->mbox
, reset
);
1051 dev_err(adap
, "Failed to halt. Exit.\n");
1055 ret
= t4_fw_restart(adap
, adap
->mbox
, reset
);
1057 dev_err(adap
, "Failed to restart. Exit.\n");
1060 state
= (enum dev_state
)((unsigned)state
& ~DEV_STATE_INIT
);
1063 t4_get_version_info(adap
);
1065 ret
= t4_get_core_clock(adap
, &adap
->params
.vpd
);
1067 dev_err(adap
, "%s: could not get core clock, error %d\n",
1073 * If the firmware is initialized already (and we're not forcing a
1074 * master initialization), note that we're living with existing
1075 * adapter parameters. Otherwise, it's time to try initializing the
1078 if (state
== DEV_STATE_INIT
) {
1079 dev_info(adap
, "Coming up as %s: Adapter already initialized\n",
1080 adap
->flags
& MASTER_PF
? "MASTER" : "SLAVE");
1082 dev_info(adap
, "Coming up as MASTER: Initializing adapter\n");
1084 ret
= adap_init0_config(adap
, reset
);
1085 if (ret
== -ENOENT
) {
1087 "No Configuration File present on adapter. Using hard-wired configuration parameters.\n");
1092 dev_err(adap
, "could not initialize adapter, error %d\n", -ret
);
1096 /* Now that we've successfully configured and initialized the adapter
1097 * (or found it already initialized), we can ask the Firmware what
1098 * resources it has provisioned for us.
1100 ret
= t4_get_pfres(adap
);
1102 dev_err(adap
->pdev_dev
,
1103 "Unable to retrieve resource provisioning info\n");
1107 /* Find out what ports are available to us. */
1108 v
= V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV
) |
1109 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC
);
1110 ret
= t4_query_params(adap
, adap
->mbox
, adap
->pf
, 0, 1, &v
, &port_vec
);
1112 dev_err(adap
, "%s: failure in t4_query_params; error = %d\n",
1117 adap
->params
.nports
= hweight32(port_vec
);
1118 adap
->params
.portvec
= port_vec
;
1120 dev_debug(adap
, "%s: adap->params.nports = %u\n", __func__
,
1121 adap
->params
.nports
);
1124 * Give the SGE code a chance to pull in anything that it needs ...
1125 * Note that this must be called after we retrieve our VPD parameters
1126 * in order to know how to convert core ticks to seconds, etc.
1128 ret
= t4_sge_init(adap
);
1130 dev_err(adap
, "t4_sge_init failed with error %d\n",
1136 * Grab some of our basic fundamental operating parameters.
1138 #define FW_PARAM_DEV(param) \
1139 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1140 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1142 #define FW_PARAM_PFVF(param) \
1143 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1144 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
1145 V_FW_PARAMS_PARAM_Y(0) | \
1146 V_FW_PARAMS_PARAM_Z(0))
1148 params
[0] = FW_PARAM_PFVF(L2T_START
);
1149 params
[1] = FW_PARAM_PFVF(L2T_END
);
1150 params
[2] = FW_PARAM_PFVF(FILTER_START
);
1151 params
[3] = FW_PARAM_PFVF(FILTER_END
);
1152 ret
= t4_query_params(adap
, adap
->mbox
, adap
->pf
, 0, 4, params
, val
);
1155 adap
->l2t_start
= val
[0];
1156 adap
->l2t_end
= val
[1];
1157 adap
->tids
.ftid_base
= val
[2];
1158 adap
->tids
.nftids
= val
[3] - val
[2] + 1;
1160 params
[0] = FW_PARAM_PFVF(CLIP_START
);
1161 params
[1] = FW_PARAM_PFVF(CLIP_END
);
1162 ret
= t4_query_params(adap
, adap
->mbox
, adap
->pf
, 0, 2, params
, val
);
1165 adap
->clipt_start
= val
[0];
1166 adap
->clipt_end
= val
[1];
1169 * Get device capabilities so we can determine what resources we need
1172 memset(&caps_cmd
, 0, sizeof(caps_cmd
));
1173 caps_cmd
.op_to_write
= htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD
) |
1174 F_FW_CMD_REQUEST
| F_FW_CMD_READ
);
1175 caps_cmd
.cfvalid_to_len16
= htonl(FW_LEN16(caps_cmd
));
1176 ret
= t4_wr_mbox(adap
, adap
->mbox
, &caps_cmd
, sizeof(caps_cmd
),
1181 if ((caps_cmd
.niccaps
& cpu_to_be16(FW_CAPS_CONFIG_NIC_HASHFILTER
)) &&
1182 is_t6(adap
->params
.chip
)) {
1183 if (init_hash_filter(adap
) < 0)
1187 /* See if FW supports FW_FILTER2 work request */
1188 if (is_t4(adap
->params
.chip
)) {
1189 adap
->params
.filter2_wr_support
= 0;
1191 params
[0] = FW_PARAM_DEV(FILTER2_WR
);
1192 ret
= t4_query_params(adap
, adap
->mbox
, adap
->pf
, 0,
1194 adap
->params
.filter2_wr_support
= (ret
== 0 && val
[0] != 0);
1197 /* query tid-related parameters */
1198 params
[0] = FW_PARAM_DEV(NTID
);
1199 ret
= t4_query_params(adap
, adap
->mbox
, adap
->pf
, 0, 1,
1203 adap
->tids
.ntids
= val
[0];
1204 adap
->tids
.natids
= min(adap
->tids
.ntids
/ 2, MAX_ATIDS
);
1206 /* If we're running on newer firmware, let it know that we're
1207 * prepared to deal with encapsulated CPL messages. Older
1208 * firmware won't understand this and we'll just get
1209 * unencapsulated messages ...
1211 params
[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP
);
1213 (void)t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1, params
, val
);
1216 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
1217 * capability. Earlier versions of the firmware didn't have the
1218 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
1219 * permission to use ULPTX MEMWRITE DSGL.
1221 if (is_t4(adap
->params
.chip
)) {
1222 adap
->params
.ulptx_memwrite_dsgl
= false;
1224 params
[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL
);
1225 ret
= t4_query_params(adap
, adap
->mbox
, adap
->pf
, 0,
1227 adap
->params
.ulptx_memwrite_dsgl
= (ret
== 0 && val
[0] != 0);
1231 * The MTU/MSS Table is initialized by now, so load their values. If
1232 * we're initializing the adapter, then we'll make any modifications
1233 * we want to the MTU/MSS Table and also initialize the congestion
1236 t4_read_mtu_tbl(adap
, adap
->params
.mtus
, NULL
);
1237 if (state
!= DEV_STATE_INIT
) {
1241 * The default MTU Table contains values 1492 and 1500.
1242 * However, for TCP, it's better to have two values which are
1243 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
1244 * This allows us to have a TCP Data Payload which is a
1245 * multiple of 8 regardless of what combination of TCP Options
1246 * are in use (always a multiple of 4 bytes) which is
1247 * important for performance reasons. For instance, if no
1248 * options are in use, then we have a 20-byte IP header and a
1249 * 20-byte TCP header. In this case, a 1500-byte MSS would
1250 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
1251 * which is not a multiple of 8. So using an MSS of 1488 in
1252 * this case results in a TCP Data Payload of 1448 bytes which
1253 * is a multiple of 8. On the other hand, if 12-byte TCP Time
1254 * Stamps have been negotiated, then an MTU of 1500 bytes
1255 * results in a TCP Data Payload of 1448 bytes which, as
1256 * above, is a multiple of 8 bytes ...
1258 for (i
= 0; i
< NMTUS
; i
++)
1259 if (adap
->params
.mtus
[i
] == 1492) {
1260 adap
->params
.mtus
[i
] = 1488;
1264 t4_load_mtus(adap
, adap
->params
.mtus
, adap
->params
.a_wnd
,
1265 adap
->params
.b_wnd
);
1267 t4_init_sge_params(adap
);
1268 t4_init_tp_params(adap
);
1269 configure_pcie_ext_tag(adap
);
1270 configure_vlan_types(adap
);
1271 cxgbe_configure_max_ethqsets(adap
);
1273 adap
->params
.drv_memwin
= MEMWIN_NIC
;
1274 adap
->flags
|= FW_OK
;
1275 dev_debug(adap
, "%s: returning zero..\n", __func__
);
1279 * Something bad happened. If a command timed out or failed with EIO
1280 * FW does not operate within its spec or something catastrophic
1281 * happened to HW/FW, stop issuing commands.
1284 if (ret
!= -ETIMEDOUT
&& ret
!= -EIO
)
1285 t4_fw_bye(adap
, adap
->mbox
);
1290 * t4_os_portmod_changed - handle port module changes
1291 * @adap: the adapter associated with the module change
1292 * @port_id: the port index whose module status has changed
1294 * This is the OS-dependent handler for port module changes. It is
1295 * invoked when a port module is removed or inserted for any OS-specific
1298 void t4_os_portmod_changed(const struct adapter
*adap
, int port_id
)
1300 static const char * const mod_str
[] = {
1301 NULL
, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
1304 const struct port_info
*pi
= adap2pinfo(adap
, port_id
);
1306 if (pi
->mod_type
== FW_PORT_MOD_TYPE_NONE
)
1307 dev_info(adap
, "Port%d: port module unplugged\n", pi
->port_id
);
1308 else if (pi
->mod_type
< ARRAY_SIZE(mod_str
))
1309 dev_info(adap
, "Port%d: %s port module inserted\n", pi
->port_id
,
1310 mod_str
[pi
->mod_type
]);
1311 else if (pi
->mod_type
== FW_PORT_MOD_TYPE_NOTSUPPORTED
)
1312 dev_info(adap
, "Port%d: unsupported port module inserted\n",
1314 else if (pi
->mod_type
== FW_PORT_MOD_TYPE_UNKNOWN
)
1315 dev_info(adap
, "Port%d: unknown port module inserted\n",
1317 else if (pi
->mod_type
== FW_PORT_MOD_TYPE_ERROR
)
1318 dev_info(adap
, "Port%d: transceiver module error\n",
1321 dev_info(adap
, "Port%d: unknown module type %d inserted\n",
1322 pi
->port_id
, pi
->mod_type
);
1325 bool cxgbe_force_linkup(struct adapter
*adap
)
1327 struct rte_pci_device
*pdev
= adap
->pdev
;
1330 return false; /* force_linkup not required for pf driver*/
1331 if (!cxgbe_get_devargs(pdev
->device
.devargs
,
1332 CXGBE_DEVARG_FORCE_LINK_UP
))
1338 * link_start - enable a port
1339 * @dev: the port to enable
1341 * Performs the MAC and PHY actions needed to enable a port.
1343 int cxgbe_link_start(struct port_info
*pi
)
1345 struct adapter
*adapter
= pi
->adapter
;
1350 mtu
= pi
->eth_dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
-
1351 (ETHER_HDR_LEN
+ ETHER_CRC_LEN
);
1353 conf_offloads
= pi
->eth_dev
->data
->dev_conf
.rxmode
.offloads
;
1356 * We do not set address filters and promiscuity here, the stack does
1357 * that step explicitly.
1359 ret
= t4_set_rxmode(adapter
, adapter
->mbox
, pi
->viid
, mtu
, -1, -1, -1,
1360 !!(conf_offloads
& DEV_RX_OFFLOAD_VLAN_STRIP
),
1363 ret
= cxgbe_mpstcam_modify(pi
, (int)pi
->xact_addr_filt
,
1364 (u8
*)&pi
->eth_dev
->data
->mac_addrs
[0]);
1366 pi
->xact_addr_filt
= ret
;
1370 if (ret
== 0 && is_pf4(adapter
))
1371 ret
= t4_link_l1cfg(adapter
, adapter
->mbox
, pi
->tx_chan
,
1375 * Enabling a Virtual Interface can result in an interrupt
1376 * during the processing of the VI Enable command and, in some
1377 * paths, result in an attempt to issue another command in the
1378 * interrupt context. Thus, we disable interrupts during the
1379 * course of the VI Enable command ...
1381 ret
= t4_enable_vi_params(adapter
, adapter
->mbox
, pi
->viid
,
1385 if (ret
== 0 && cxgbe_force_linkup(adapter
))
1386 pi
->eth_dev
->data
->dev_link
.link_status
= ETH_LINK_UP
;
1391 * cxgbe_write_rss_conf - flash the RSS configuration for a given port
1393 * @rss_hf: Hash configuration to apply
1395 int cxgbe_write_rss_conf(const struct port_info
*pi
, uint64_t rss_hf
)
1397 struct adapter
*adapter
= pi
->adapter
;
1398 const struct sge_eth_rxq
*rxq
;
1403 /* Should never be called before setting up sge eth rx queues */
1404 if (!(adapter
->flags
& FULL_INIT_DONE
)) {
1405 dev_err(adap
, "%s No RXQs available on port %d\n",
1406 __func__
, pi
->port_id
);
1410 /* Don't allow unsupported hash functions */
1411 if (rss_hf
& ~CXGBE_RSS_HF_ALL
)
1414 if (rss_hf
& CXGBE_RSS_HF_IPV4_MASK
)
1415 flags
|= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN
;
1417 if (rss_hf
& ETH_RSS_NONFRAG_IPV4_TCP
)
1418 flags
|= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
;
1420 if (rss_hf
& ETH_RSS_NONFRAG_IPV4_UDP
)
1421 flags
|= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN
|
1422 F_FW_RSS_VI_CONFIG_CMD_UDPEN
;
1424 if (rss_hf
& CXGBE_RSS_HF_IPV6_MASK
)
1425 flags
|= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
;
1427 if (rss_hf
& CXGBE_RSS_HF_TCP_IPV6_MASK
)
1428 flags
|= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
|
1429 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
;
1431 if (rss_hf
& CXGBE_RSS_HF_UDP_IPV6_MASK
)
1432 flags
|= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN
|
1433 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN
|
1434 F_FW_RSS_VI_CONFIG_CMD_UDPEN
;
1436 rxq
= &adapter
->sge
.ethrxq
[pi
->first_qset
];
1437 rss
= rxq
[0].rspq
.abs_id
;
1439 /* If Tunnel All Lookup isn't specified in the global RSS
1440 * Configuration, then we need to specify a default Ingress
1441 * Queue for any ingress packets which aren't hashed. We'll
1442 * use our first ingress queue ...
1444 err
= t4_config_vi_rss(adapter
, adapter
->mbox
, pi
->viid
,
1450 * cxgbe_write_rss - write the RSS table for a given port
1452 * @queues: array of queue indices for RSS
1454 * Sets up the portion of the HW RSS table for the port's VI to distribute
1455 * packets to the Rx queues in @queues.
1457 int cxgbe_write_rss(const struct port_info
*pi
, const u16
*queues
)
1461 struct adapter
*adapter
= pi
->adapter
;
1462 const struct sge_eth_rxq
*rxq
;
1464 /* Should never be called before setting up sge eth rx queues */
1465 BUG_ON(!(adapter
->flags
& FULL_INIT_DONE
));
1467 rxq
= &adapter
->sge
.ethrxq
[pi
->first_qset
];
1468 rss
= rte_zmalloc(NULL
, pi
->rss_size
* sizeof(u16
), 0);
1472 /* map the queue indices to queue ids */
1473 for (i
= 0; i
< pi
->rss_size
; i
++, queues
++)
1474 rss
[i
] = rxq
[*queues
].rspq
.abs_id
;
1476 err
= t4_config_rss_range(adapter
, adapter
->pf
, pi
->viid
, 0,
1477 pi
->rss_size
, rss
, pi
->rss_size
);
1483 * setup_rss - configure RSS
1484 * @adapter: the adapter
1486 * Sets up RSS to distribute packets to multiple receive queues. We
1487 * configure the RSS CPU lookup table to distribute to the number of HW
1488 * receive queues, and the response queue lookup table to narrow that
1489 * down to the response queues actually configured for each port.
1490 * We always configure the RSS mapping for all ports since the mapping
1491 * table has plenty of entries.
1493 int cxgbe_setup_rss(struct port_info
*pi
)
1496 struct adapter
*adapter
= pi
->adapter
;
1498 dev_debug(adapter
, "%s: pi->rss_size = %u; pi->n_rx_qsets = %u\n",
1499 __func__
, pi
->rss_size
, pi
->n_rx_qsets
);
1501 if (!(pi
->flags
& PORT_RSS_DONE
)) {
1502 if (adapter
->flags
& FULL_INIT_DONE
) {
1503 /* Fill default values with equal distribution */
1504 for (j
= 0; j
< pi
->rss_size
; j
++)
1505 pi
->rss
[j
] = j
% pi
->n_rx_qsets
;
1507 err
= cxgbe_write_rss(pi
, pi
->rss
);
1511 err
= cxgbe_write_rss_conf(pi
, pi
->rss_hf
);
1514 pi
->flags
|= PORT_RSS_DONE
;
1521 * Enable NAPI scheduling and interrupt generation for all Rx queues.
1523 static void enable_rx(struct adapter
*adap
, struct sge_rspq
*q
)
1525 /* 0-increment GTS to start the timer and enable interrupts */
1526 t4_write_reg(adap
, is_pf4(adap
) ? MYPF_REG(A_SGE_PF_GTS
) :
1527 T4VF_SGE_BASE_ADDR
+ A_SGE_VF_GTS
,
1528 V_SEINTARM(q
->intr_params
) |
1529 V_INGRESSQID(q
->cntxt_id
));
1532 void cxgbe_enable_rx_queues(struct port_info
*pi
)
1534 struct adapter
*adap
= pi
->adapter
;
1535 struct sge
*s
= &adap
->sge
;
1538 for (i
= 0; i
< pi
->n_rx_qsets
; i
++)
1539 enable_rx(adap
, &s
->ethrxq
[pi
->first_qset
+ i
].rspq
);
1543 * fw_caps_to_speed_caps - translate Firmware Port Caps to Speed Caps.
1544 * @port_type: Firmware Port Type
1545 * @fw_caps: Firmware Port Capabilities
1546 * @speed_caps: Device Info Speed Capabilities
1548 * Translate a Firmware Port Capabilities specification to Device Info
1549 * Speed Capabilities.
1551 static void fw_caps_to_speed_caps(enum fw_port_type port_type
,
1552 unsigned int fw_caps
,
1555 #define SET_SPEED(__speed_name) \
1557 *speed_caps |= ETH_LINK_ ## __speed_name; \
1560 #define FW_CAPS_TO_SPEED(__fw_name) \
1562 if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \
1563 SET_SPEED(__fw_name); \
1566 switch (port_type
) {
1567 case FW_PORT_TYPE_BT_SGMII
:
1568 case FW_PORT_TYPE_BT_XFI
:
1569 case FW_PORT_TYPE_BT_XAUI
:
1570 FW_CAPS_TO_SPEED(SPEED_100M
);
1571 FW_CAPS_TO_SPEED(SPEED_1G
);
1572 FW_CAPS_TO_SPEED(SPEED_10G
);
1575 case FW_PORT_TYPE_KX4
:
1576 case FW_PORT_TYPE_KX
:
1577 case FW_PORT_TYPE_FIBER_XFI
:
1578 case FW_PORT_TYPE_FIBER_XAUI
:
1579 case FW_PORT_TYPE_SFP
:
1580 case FW_PORT_TYPE_QSFP_10G
:
1581 case FW_PORT_TYPE_QSA
:
1582 FW_CAPS_TO_SPEED(SPEED_1G
);
1583 FW_CAPS_TO_SPEED(SPEED_10G
);
1586 case FW_PORT_TYPE_KR
:
1587 SET_SPEED(SPEED_10G
);
1590 case FW_PORT_TYPE_BP_AP
:
1591 case FW_PORT_TYPE_BP4_AP
:
1592 SET_SPEED(SPEED_1G
);
1593 SET_SPEED(SPEED_10G
);
1596 case FW_PORT_TYPE_BP40_BA
:
1597 case FW_PORT_TYPE_QSFP
:
1598 SET_SPEED(SPEED_40G
);
1601 case FW_PORT_TYPE_CR_QSFP
:
1602 case FW_PORT_TYPE_SFP28
:
1603 case FW_PORT_TYPE_KR_SFP28
:
1604 FW_CAPS_TO_SPEED(SPEED_1G
);
1605 FW_CAPS_TO_SPEED(SPEED_10G
);
1606 FW_CAPS_TO_SPEED(SPEED_25G
);
1609 case FW_PORT_TYPE_CR2_QSFP
:
1610 SET_SPEED(SPEED_50G
);
1613 case FW_PORT_TYPE_KR4_100G
:
1614 case FW_PORT_TYPE_CR4_QSFP
:
1615 FW_CAPS_TO_SPEED(SPEED_25G
);
1616 FW_CAPS_TO_SPEED(SPEED_40G
);
1617 FW_CAPS_TO_SPEED(SPEED_50G
);
1618 FW_CAPS_TO_SPEED(SPEED_100G
);
1625 #undef FW_CAPS_TO_SPEED
1630 * cxgbe_get_speed_caps - Fetch supported speed capabilities
1631 * @pi: Underlying port's info
1632 * @speed_caps: Device Info speed capabilities
1634 * Fetch supported speed capabilities of the underlying port.
1636 void cxgbe_get_speed_caps(struct port_info
*pi
, u32
*speed_caps
)
1640 fw_caps_to_speed_caps(pi
->port_type
, pi
->link_cfg
.pcaps
,
1643 if (!(pi
->link_cfg
.pcaps
& FW_PORT_CAP32_ANEG
))
1644 *speed_caps
|= ETH_LINK_SPEED_FIXED
;
1648 * cxgbe_set_link_status - Set device link up or down.
1649 * @pi: Underlying port's info
1650 * @status: 0 - down, 1 - up
1652 * Set the device link up or down.
1654 int cxgbe_set_link_status(struct port_info
*pi
, bool status
)
1656 struct adapter
*adapter
= pi
->adapter
;
1659 err
= t4_enable_vi(adapter
, adapter
->mbox
, pi
->viid
, status
, status
);
1661 dev_err(adapter
, "%s: disable_vi failed: %d\n", __func__
, err
);
1666 t4_reset_link_config(adapter
, pi
->pidx
);
1672 * cxgb_up - enable the adapter
1673 * @adap: adapter being enabled
1675 * Called when the first port is enabled, this function performs the
1676 * actions necessary to make an adapter operational, such as completing
1677 * the initialization of HW modules, and enabling interrupts.
1679 int cxgbe_up(struct adapter
*adap
)
1681 enable_rx(adap
, &adap
->sge
.fw_evtq
);
1682 t4_sge_tx_monitor_start(adap
);
1684 t4_intr_enable(adap
);
1685 adap
->flags
|= FULL_INIT_DONE
;
1687 /* TODO: deadman watchdog ?? */
1694 int cxgbe_down(struct port_info
*pi
)
1696 return cxgbe_set_link_status(pi
, false);
1700 * Release resources when all the ports have been stopped.
1702 void cxgbe_close(struct adapter
*adapter
)
1704 struct port_info
*pi
;
1707 if (adapter
->flags
& FULL_INIT_DONE
) {
1708 tid_free(&adapter
->tids
);
1709 t4_cleanup_mpstcam(adapter
);
1710 t4_cleanup_clip_tbl(adapter
);
1711 t4_cleanup_l2t(adapter
);
1712 if (is_pf4(adapter
))
1713 t4_intr_disable(adapter
);
1714 t4_sge_tx_monitor_stop(adapter
);
1715 t4_free_sge_resources(adapter
);
1716 for_each_port(adapter
, i
) {
1717 pi
= adap2pinfo(adapter
, i
);
1719 t4_free_vi(adapter
, adapter
->mbox
,
1720 adapter
->pf
, 0, pi
->viid
);
1721 rte_eth_dev_release_port(pi
->eth_dev
);
1723 adapter
->flags
&= ~FULL_INIT_DONE
;
1726 if (is_pf4(adapter
) && (adapter
->flags
& FW_OK
))
1727 t4_fw_bye(adapter
, adapter
->mbox
);
1730 int cxgbe_probe(struct adapter
*adapter
)
1732 struct port_info
*pi
;
1738 whoami
= t4_read_reg(adapter
, A_PL_WHOAMI
);
1739 chip
= t4_get_chip_type(adapter
,
1740 CHELSIO_PCI_ID_VER(adapter
->pdev
->id
.device_id
));
1744 func
= CHELSIO_CHIP_VERSION(chip
) <= CHELSIO_T5
?
1745 G_SOURCEPF(whoami
) : G_T6_SOURCEPF(whoami
);
1747 adapter
->mbox
= func
;
1750 t4_os_lock_init(&adapter
->mbox_lock
);
1751 TAILQ_INIT(&adapter
->mbox_list
);
1752 t4_os_lock_init(&adapter
->win0_lock
);
1754 err
= t4_prep_adapter(adapter
);
1758 setup_memwin(adapter
);
1759 err
= adap_init0(adapter
);
1761 dev_err(adapter
, "%s: Adapter initialization failed, error %d\n",
1766 if (!is_t4(adapter
->params
.chip
)) {
1768 * The userspace doorbell BAR is split evenly into doorbell
1769 * regions, each associated with an egress queue. If this
1770 * per-queue region is large enough (at least UDBS_SEG_SIZE)
1771 * then it can be used to submit a tx work request with an
1772 * implied doorbell. Enable write combining on the BAR if
1773 * there is room for such work requests.
1775 int s_qpp
, qpp
, num_seg
;
1777 s_qpp
= (S_QUEUESPERPAGEPF0
+
1778 (S_QUEUESPERPAGEPF1
- S_QUEUESPERPAGEPF0
) *
1780 qpp
= 1 << ((t4_read_reg(adapter
,
1781 A_SGE_EGRESS_QUEUES_PER_PAGE_PF
) >> s_qpp
)
1782 & M_QUEUESPERPAGEPF0
);
1783 num_seg
= CXGBE_PAGE_SIZE
/ UDBS_SEG_SIZE
;
1785 dev_warn(adapter
, "Incorrect SGE EGRESS QUEUES_PER_PAGE configuration, continuing in debug mode\n");
1787 adapter
->bar2
= (void *)adapter
->pdev
->mem_resource
[2].addr
;
1788 if (!adapter
->bar2
) {
1789 dev_err(adapter
, "cannot map device bar2 region\n");
1793 t4_write_reg(adapter
, A_SGE_STAT_CFG
, V_STATSOURCE_T5(7) |
1797 for_each_port(adapter
, i
) {
1798 const unsigned int numa_node
= rte_socket_id();
1799 char name
[RTE_ETH_NAME_MAX_LEN
];
1800 struct rte_eth_dev
*eth_dev
;
1802 snprintf(name
, sizeof(name
), "%s_%d",
1803 adapter
->pdev
->device
.name
, i
);
1806 /* First port is already allocated by DPDK */
1807 eth_dev
= adapter
->eth_dev
;
1812 * now do all data allocation - for eth_dev structure,
1813 * and internal (private) data for the remaining ports
1816 /* reserve an ethdev entry */
1817 eth_dev
= rte_eth_dev_allocate(name
);
1821 eth_dev
->data
->dev_private
=
1822 rte_zmalloc_socket(name
, sizeof(struct port_info
),
1823 RTE_CACHE_LINE_SIZE
, numa_node
);
1824 if (!eth_dev
->data
->dev_private
)
1828 pi
= (struct port_info
*)eth_dev
->data
->dev_private
;
1829 adapter
->port
[i
] = pi
;
1830 pi
->eth_dev
= eth_dev
;
1831 pi
->adapter
= adapter
;
1832 pi
->xact_addr_filt
= -1;
1836 pi
->eth_dev
->device
= &adapter
->pdev
->device
;
1837 pi
->eth_dev
->dev_ops
= adapter
->eth_dev
->dev_ops
;
1838 pi
->eth_dev
->tx_pkt_burst
= adapter
->eth_dev
->tx_pkt_burst
;
1839 pi
->eth_dev
->rx_pkt_burst
= adapter
->eth_dev
->rx_pkt_burst
;
1841 rte_eth_copy_pci_info(pi
->eth_dev
, adapter
->pdev
);
1843 pi
->eth_dev
->data
->mac_addrs
= rte_zmalloc(name
,
1845 if (!pi
->eth_dev
->data
->mac_addrs
) {
1846 dev_err(adapter
, "%s: Mem allocation failed for storing mac addr, aborting\n",
1853 /* First port will be notified by upper layer */
1854 rte_eth_dev_probing_finish(eth_dev
);
1858 if (adapter
->flags
& FW_OK
) {
1859 err
= t4_port_init(adapter
, adapter
->mbox
, adapter
->pf
, 0);
1861 dev_err(adapter
, "%s: t4_port_init failed with err %d\n",
1867 cxgbe_cfg_queues(adapter
->eth_dev
);
1869 cxgbe_print_adapter_info(adapter
);
1870 cxgbe_print_port_info(adapter
);
1872 adapter
->clipt
= t4_init_clip_tbl(adapter
->clipt_start
,
1873 adapter
->clipt_end
);
1874 if (!adapter
->clipt
) {
1875 /* We tolerate a lack of clip_table, giving up some
1878 dev_warn(adapter
, "could not allocate CLIP. Continuing\n");
1881 adapter
->l2t
= t4_init_l2t(adapter
->l2t_start
, adapter
->l2t_end
);
1882 if (!adapter
->l2t
) {
1883 /* We tolerate a lack of L2T, giving up some functionality */
1884 dev_warn(adapter
, "could not allocate L2T. Continuing\n");
1887 if (tid_init(&adapter
->tids
) < 0) {
1888 /* Disable filtering support */
1889 dev_warn(adapter
, "could not allocate TID table, "
1890 "filter support disabled. Continuing\n");
1893 adapter
->mpstcam
= t4_init_mpstcam(adapter
);
1894 if (!adapter
->mpstcam
)
1895 dev_warn(adapter
, "could not allocate mps tcam table."
1898 if (is_hashfilter(adapter
)) {
1899 if (t4_read_reg(adapter
, A_LE_DB_CONFIG
) & F_HASHEN
) {
1900 u32 hash_base
, hash_reg
;
1902 hash_reg
= A_LE_DB_TID_HASHBASE
;
1903 hash_base
= t4_read_reg(adapter
, hash_reg
);
1904 adapter
->tids
.hash_base
= hash_base
/ 4;
1907 /* Disable hash filtering support */
1909 "Maskless filter support disabled. Continuing\n");
1912 err
= cxgbe_init_rss(adapter
);
1919 for_each_port(adapter
, i
) {
1920 pi
= adap2pinfo(adapter
, i
);
1922 t4_free_vi(adapter
, adapter
->mbox
, adapter
->pf
,
1924 rte_eth_dev_release_port(pi
->eth_dev
);
1927 if (adapter
->flags
& FW_OK
)
1928 t4_fw_bye(adapter
, adapter
->mbox
);