1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
7 #include <linux/of_mdio.h>
8 #include <linux/vmalloc.h>
10 /* ENETC overhead: optional extension BD + 1 BD gap */
11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
12 /* max # of chained Tx BDs is 15, including head and extension BD */
13 #define ENETC_MAX_SKB_FRAGS 13
14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
16 static int enetc_map_tx_buffs(struct enetc_bdr
*tx_ring
, struct sk_buff
*skb
,
19 netdev_tx_t
enetc_xmit(struct sk_buff
*skb
, struct net_device
*ndev
)
21 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
22 struct enetc_bdr
*tx_ring
;
25 tx_ring
= priv
->tx_ring
[skb
->queue_mapping
];
27 if (unlikely(skb_shinfo(skb
)->nr_frags
> ENETC_MAX_SKB_FRAGS
))
28 if (unlikely(skb_linearize(skb
)))
31 count
= skb_shinfo(skb
)->nr_frags
+ 1; /* fragments + head */
32 if (enetc_bd_unused(tx_ring
) < ENETC_TXBDS_NEEDED(count
)) {
33 netif_stop_subqueue(ndev
, tx_ring
->index
);
34 return NETDEV_TX_BUSY
;
37 count
= enetc_map_tx_buffs(tx_ring
, skb
, priv
->active_offloads
);
41 if (enetc_bd_unused(tx_ring
) < ENETC_TXBDS_MAX_NEEDED
)
42 netif_stop_subqueue(ndev
, tx_ring
->index
);
47 dev_kfree_skb_any(skb
);
51 static bool enetc_tx_csum(struct sk_buff
*skb
, union enetc_tx_bd
*txbd
)
53 int l3_start
, l3_hsize
;
54 u16 l3_flags
, l4_flags
;
56 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
59 switch (skb
->csum_offset
) {
60 case offsetof(struct tcphdr
, check
):
61 l4_flags
= ENETC_TXBD_L4_TCP
;
63 case offsetof(struct udphdr
, check
):
64 l4_flags
= ENETC_TXBD_L4_UDP
;
67 skb_checksum_help(skb
);
71 l3_start
= skb_network_offset(skb
);
72 l3_hsize
= skb_network_header_len(skb
);
75 if (skb
->protocol
== htons(ETH_P_IPV6
))
76 l3_flags
= ENETC_TXBD_L3_IPV6
;
79 txbd
->l3_csoff
= enetc_txbd_l3_csoff(l3_start
, l3_hsize
, l3_flags
);
80 txbd
->l4_csoff
= l4_flags
;
85 static void enetc_unmap_tx_buff(struct enetc_bdr
*tx_ring
,
86 struct enetc_tx_swbd
*tx_swbd
)
88 if (tx_swbd
->is_dma_page
)
89 dma_unmap_page(tx_ring
->dev
, tx_swbd
->dma
,
90 tx_swbd
->len
, DMA_TO_DEVICE
);
92 dma_unmap_single(tx_ring
->dev
, tx_swbd
->dma
,
93 tx_swbd
->len
, DMA_TO_DEVICE
);
97 static void enetc_free_tx_skb(struct enetc_bdr
*tx_ring
,
98 struct enetc_tx_swbd
*tx_swbd
)
101 enetc_unmap_tx_buff(tx_ring
, tx_swbd
);
104 dev_kfree_skb_any(tx_swbd
->skb
);
109 static int enetc_map_tx_buffs(struct enetc_bdr
*tx_ring
, struct sk_buff
*skb
,
112 struct enetc_tx_swbd
*tx_swbd
;
114 int len
= skb_headlen(skb
);
115 union enetc_tx_bd temp_bd
;
116 union enetc_tx_bd
*txbd
;
117 bool do_vlan
, do_tstamp
;
123 i
= tx_ring
->next_to_use
;
124 txbd
= ENETC_TXBD(*tx_ring
, i
);
127 dma
= dma_map_single(tx_ring
->dev
, skb
->data
, len
, DMA_TO_DEVICE
);
128 if (unlikely(dma_mapping_error(tx_ring
->dev
, dma
)))
131 temp_bd
.addr
= cpu_to_le64(dma
);
132 temp_bd
.buf_len
= cpu_to_le16(len
);
135 tx_swbd
= &tx_ring
->tx_swbd
[i
];
138 tx_swbd
->is_dma_page
= 0;
141 do_vlan
= skb_vlan_tag_present(skb
);
142 do_tstamp
= (active_offloads
& ENETC_F_TX_TSTAMP
) &&
143 (skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
);
144 tx_swbd
->do_tstamp
= do_tstamp
;
145 tx_swbd
->check_wb
= tx_swbd
->do_tstamp
;
147 if (do_vlan
|| do_tstamp
)
148 flags
|= ENETC_TXBD_FLAGS_EX
;
150 if (enetc_tx_csum(skb
, &temp_bd
))
151 flags
|= ENETC_TXBD_FLAGS_CSUM
| ENETC_TXBD_FLAGS_L4CS
;
153 /* first BD needs frm_len and offload flags set */
154 temp_bd
.frm_len
= cpu_to_le16(skb
->len
);
155 temp_bd
.flags
= flags
;
157 if (flags
& ENETC_TXBD_FLAGS_EX
) {
160 enetc_clear_tx_bd(&temp_bd
);
162 /* add extension BD for VLAN and/or timestamping */
167 if (unlikely(i
== tx_ring
->bd_count
)) {
169 tx_swbd
= tx_ring
->tx_swbd
;
170 txbd
= ENETC_TXBD(*tx_ring
, 0);
175 temp_bd
.ext
.vid
= cpu_to_le16(skb_vlan_tag_get(skb
));
176 temp_bd
.ext
.tpid
= 0; /* < C-TAG */
177 e_flags
|= ENETC_TXBD_E_FLAGS_VLAN_INS
;
181 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
182 e_flags
|= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP
;
185 temp_bd
.ext
.e_flags
= e_flags
;
189 frag
= &skb_shinfo(skb
)->frags
[0];
190 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++, frag
++) {
191 len
= skb_frag_size(frag
);
192 dma
= skb_frag_dma_map(tx_ring
->dev
, frag
, 0, len
,
194 if (dma_mapping_error(tx_ring
->dev
, dma
))
198 enetc_clear_tx_bd(&temp_bd
);
204 if (unlikely(i
== tx_ring
->bd_count
)) {
206 tx_swbd
= tx_ring
->tx_swbd
;
207 txbd
= ENETC_TXBD(*tx_ring
, 0);
211 temp_bd
.addr
= cpu_to_le64(dma
);
212 temp_bd
.buf_len
= cpu_to_le16(len
);
216 tx_swbd
->is_dma_page
= 1;
220 /* last BD needs 'F' bit set */
221 flags
|= ENETC_TXBD_FLAGS_F
;
222 temp_bd
.flags
= flags
;
225 tx_ring
->tx_swbd
[i
].skb
= skb
;
227 enetc_bdr_idx_inc(tx_ring
, &i
);
228 tx_ring
->next_to_use
= i
;
230 /* let H/W know BD ring has been updated */
231 enetc_wr_reg(tx_ring
->tpir
, i
); /* includes wmb() */
236 dev_err(tx_ring
->dev
, "DMA map error");
239 tx_swbd
= &tx_ring
->tx_swbd
[i
];
240 enetc_free_tx_skb(tx_ring
, tx_swbd
);
242 i
= tx_ring
->bd_count
;
249 static irqreturn_t
enetc_msix(int irq
, void *data
)
251 struct enetc_int_vector
*v
= data
;
254 /* disable interrupts */
255 enetc_wr_reg(v
->rbier
, 0);
257 for_each_set_bit(i
, &v
->tx_rings_map
, v
->count_tx_rings
)
258 enetc_wr_reg(v
->tbier_base
+ ENETC_BDR_OFF(i
), 0);
260 napi_schedule_irqoff(&v
->napi
);
265 static bool enetc_clean_tx_ring(struct enetc_bdr
*tx_ring
, int napi_budget
);
266 static int enetc_clean_rx_ring(struct enetc_bdr
*rx_ring
,
267 struct napi_struct
*napi
, int work_limit
);
269 static int enetc_poll(struct napi_struct
*napi
, int budget
)
271 struct enetc_int_vector
272 *v
= container_of(napi
, struct enetc_int_vector
, napi
);
273 bool complete
= true;
277 for (i
= 0; i
< v
->count_tx_rings
; i
++)
278 if (!enetc_clean_tx_ring(&v
->tx_ring
[i
], budget
))
281 work_done
= enetc_clean_rx_ring(&v
->rx_ring
, napi
, budget
);
282 if (work_done
== budget
)
288 napi_complete_done(napi
, work_done
);
290 /* enable interrupts */
291 enetc_wr_reg(v
->rbier
, ENETC_RBIER_RXTIE
);
293 for_each_set_bit(i
, &v
->tx_rings_map
, v
->count_tx_rings
)
294 enetc_wr_reg(v
->tbier_base
+ ENETC_BDR_OFF(i
),
300 static int enetc_bd_ready_count(struct enetc_bdr
*tx_ring
, int ci
)
302 int pi
= enetc_rd_reg(tx_ring
->tcir
) & ENETC_TBCIR_IDX_MASK
;
304 return pi
>= ci
? pi
- ci
: tx_ring
->bd_count
- ci
+ pi
;
307 static void enetc_get_tx_tstamp(struct enetc_hw
*hw
, union enetc_tx_bd
*txbd
,
310 u32 lo
, hi
, tstamp_lo
;
312 lo
= enetc_rd(hw
, ENETC_SICTR0
);
313 hi
= enetc_rd(hw
, ENETC_SICTR1
);
314 tstamp_lo
= le32_to_cpu(txbd
->wb
.tstamp
);
317 *tstamp
= (u64
)hi
<< 32 | tstamp_lo
;
320 static void enetc_tstamp_tx(struct sk_buff
*skb
, u64 tstamp
)
322 struct skb_shared_hwtstamps shhwtstamps
;
324 if (skb_shinfo(skb
)->tx_flags
& SKBTX_IN_PROGRESS
) {
325 memset(&shhwtstamps
, 0, sizeof(shhwtstamps
));
326 shhwtstamps
.hwtstamp
= ns_to_ktime(tstamp
);
327 skb_tstamp_tx(skb
, &shhwtstamps
);
331 static bool enetc_clean_tx_ring(struct enetc_bdr
*tx_ring
, int napi_budget
)
333 struct net_device
*ndev
= tx_ring
->ndev
;
334 int tx_frm_cnt
= 0, tx_byte_cnt
= 0;
335 struct enetc_tx_swbd
*tx_swbd
;
340 i
= tx_ring
->next_to_clean
;
341 tx_swbd
= &tx_ring
->tx_swbd
[i
];
342 bds_to_clean
= enetc_bd_ready_count(tx_ring
, i
);
346 while (bds_to_clean
&& tx_frm_cnt
< ENETC_DEFAULT_TX_WORK
) {
347 bool is_eof
= !!tx_swbd
->skb
;
349 if (unlikely(tx_swbd
->check_wb
)) {
350 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
351 union enetc_tx_bd
*txbd
;
353 txbd
= ENETC_TXBD(*tx_ring
, i
);
355 if (txbd
->flags
& ENETC_TXBD_FLAGS_W
&&
356 tx_swbd
->do_tstamp
) {
357 enetc_get_tx_tstamp(&priv
->si
->hw
, txbd
,
363 if (likely(tx_swbd
->dma
))
364 enetc_unmap_tx_buff(tx_ring
, tx_swbd
);
367 if (unlikely(do_tstamp
)) {
368 enetc_tstamp_tx(tx_swbd
->skb
, tstamp
);
371 napi_consume_skb(tx_swbd
->skb
, napi_budget
);
375 tx_byte_cnt
+= tx_swbd
->len
;
380 if (unlikely(i
== tx_ring
->bd_count
)) {
382 tx_swbd
= tx_ring
->tx_swbd
;
385 /* BD iteration loop end */
388 /* re-arm interrupt source */
389 enetc_wr_reg(tx_ring
->idr
, BIT(tx_ring
->index
) |
390 BIT(16 + tx_ring
->index
));
393 if (unlikely(!bds_to_clean
))
394 bds_to_clean
= enetc_bd_ready_count(tx_ring
, i
);
397 tx_ring
->next_to_clean
= i
;
398 tx_ring
->stats
.packets
+= tx_frm_cnt
;
399 tx_ring
->stats
.bytes
+= tx_byte_cnt
;
401 if (unlikely(tx_frm_cnt
&& netif_carrier_ok(ndev
) &&
402 __netif_subqueue_stopped(ndev
, tx_ring
->index
) &&
403 (enetc_bd_unused(tx_ring
) >= ENETC_TXBDS_MAX_NEEDED
))) {
404 netif_wake_subqueue(ndev
, tx_ring
->index
);
407 return tx_frm_cnt
!= ENETC_DEFAULT_TX_WORK
;
410 static bool enetc_new_page(struct enetc_bdr
*rx_ring
,
411 struct enetc_rx_swbd
*rx_swbd
)
416 page
= dev_alloc_page();
420 addr
= dma_map_page(rx_ring
->dev
, page
, 0, PAGE_SIZE
, DMA_FROM_DEVICE
);
421 if (unlikely(dma_mapping_error(rx_ring
->dev
, addr
))) {
428 rx_swbd
->page
= page
;
429 rx_swbd
->page_offset
= ENETC_RXB_PAD
;
434 static int enetc_refill_rx_ring(struct enetc_bdr
*rx_ring
, const int buff_cnt
)
436 struct enetc_rx_swbd
*rx_swbd
;
437 union enetc_rx_bd
*rxbd
;
440 i
= rx_ring
->next_to_use
;
441 rx_swbd
= &rx_ring
->rx_swbd
[i
];
442 rxbd
= ENETC_RXBD(*rx_ring
, i
);
444 for (j
= 0; j
< buff_cnt
; j
++) {
446 if (unlikely(!rx_swbd
->page
)) {
447 if (unlikely(!enetc_new_page(rx_ring
, rx_swbd
))) {
448 rx_ring
->stats
.rx_alloc_errs
++;
454 rxbd
->w
.addr
= cpu_to_le64(rx_swbd
->dma
+
455 rx_swbd
->page_offset
);
456 /* clear 'R" as well */
462 if (unlikely(i
== rx_ring
->bd_count
)) {
464 rx_swbd
= rx_ring
->rx_swbd
;
465 rxbd
= ENETC_RXBD(*rx_ring
, 0);
470 rx_ring
->next_to_alloc
= i
; /* keep track from page reuse */
471 rx_ring
->next_to_use
= i
;
472 /* update ENETC's consumer index */
473 enetc_wr_reg(rx_ring
->rcir
, i
);
479 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
480 static void enetc_get_rx_tstamp(struct net_device
*ndev
,
481 union enetc_rx_bd
*rxbd
,
484 struct skb_shared_hwtstamps
*shhwtstamps
= skb_hwtstamps(skb
);
485 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
486 struct enetc_hw
*hw
= &priv
->si
->hw
;
487 u32 lo
, hi
, tstamp_lo
;
490 if (le16_to_cpu(rxbd
->r
.flags
) & ENETC_RXBD_FLAG_TSTMP
) {
491 lo
= enetc_rd(hw
, ENETC_SICTR0
);
492 hi
= enetc_rd(hw
, ENETC_SICTR1
);
493 tstamp_lo
= le32_to_cpu(rxbd
->r
.tstamp
);
497 tstamp
= (u64
)hi
<< 32 | tstamp_lo
;
498 memset(shhwtstamps
, 0, sizeof(*shhwtstamps
));
499 shhwtstamps
->hwtstamp
= ns_to_ktime(tstamp
);
504 static void enetc_get_offloads(struct enetc_bdr
*rx_ring
,
505 union enetc_rx_bd
*rxbd
, struct sk_buff
*skb
)
507 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
508 struct enetc_ndev_priv
*priv
= netdev_priv(rx_ring
->ndev
);
511 if (rx_ring
->ndev
->features
& NETIF_F_RXCSUM
) {
512 u16 inet_csum
= le16_to_cpu(rxbd
->r
.inet_csum
);
514 skb
->csum
= csum_unfold((__force __sum16
)~htons(inet_csum
));
515 skb
->ip_summed
= CHECKSUM_COMPLETE
;
518 /* copy VLAN to skb, if one is extracted, for now we assume it's a
519 * standard TPID, but HW also supports custom values
521 if (le16_to_cpu(rxbd
->r
.flags
) & ENETC_RXBD_FLAG_VLAN
)
522 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
523 le16_to_cpu(rxbd
->r
.vlan_opt
));
524 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
525 if (priv
->active_offloads
& ENETC_F_RX_TSTAMP
)
526 enetc_get_rx_tstamp(rx_ring
->ndev
, rxbd
, skb
);
530 static void enetc_process_skb(struct enetc_bdr
*rx_ring
,
533 skb_record_rx_queue(skb
, rx_ring
->index
);
534 skb
->protocol
= eth_type_trans(skb
, rx_ring
->ndev
);
537 static bool enetc_page_reusable(struct page
*page
)
539 return (!page_is_pfmemalloc(page
) && page_ref_count(page
) == 1);
542 static void enetc_reuse_page(struct enetc_bdr
*rx_ring
,
543 struct enetc_rx_swbd
*old
)
545 struct enetc_rx_swbd
*new;
547 new = &rx_ring
->rx_swbd
[rx_ring
->next_to_alloc
];
549 /* next buf that may reuse a page */
550 enetc_bdr_idx_inc(rx_ring
, &rx_ring
->next_to_alloc
);
552 /* copy page reference */
556 static struct enetc_rx_swbd
*enetc_get_rx_buff(struct enetc_bdr
*rx_ring
,
559 struct enetc_rx_swbd
*rx_swbd
= &rx_ring
->rx_swbd
[i
];
561 dma_sync_single_range_for_cpu(rx_ring
->dev
, rx_swbd
->dma
,
562 rx_swbd
->page_offset
,
563 size
, DMA_FROM_DEVICE
);
567 static void enetc_put_rx_buff(struct enetc_bdr
*rx_ring
,
568 struct enetc_rx_swbd
*rx_swbd
)
570 if (likely(enetc_page_reusable(rx_swbd
->page
))) {
571 rx_swbd
->page_offset
^= ENETC_RXB_TRUESIZE
;
572 page_ref_inc(rx_swbd
->page
);
574 enetc_reuse_page(rx_ring
, rx_swbd
);
576 /* sync for use by the device */
577 dma_sync_single_range_for_device(rx_ring
->dev
, rx_swbd
->dma
,
578 rx_swbd
->page_offset
,
582 dma_unmap_page(rx_ring
->dev
, rx_swbd
->dma
,
583 PAGE_SIZE
, DMA_FROM_DEVICE
);
586 rx_swbd
->page
= NULL
;
589 static struct sk_buff
*enetc_map_rx_buff_to_skb(struct enetc_bdr
*rx_ring
,
592 struct enetc_rx_swbd
*rx_swbd
= enetc_get_rx_buff(rx_ring
, i
, size
);
596 ba
= page_address(rx_swbd
->page
) + rx_swbd
->page_offset
;
597 skb
= build_skb(ba
- ENETC_RXB_PAD
, ENETC_RXB_TRUESIZE
);
598 if (unlikely(!skb
)) {
599 rx_ring
->stats
.rx_alloc_errs
++;
603 skb_reserve(skb
, ENETC_RXB_PAD
);
604 __skb_put(skb
, size
);
606 enetc_put_rx_buff(rx_ring
, rx_swbd
);
611 static void enetc_add_rx_buff_to_skb(struct enetc_bdr
*rx_ring
, int i
,
612 u16 size
, struct sk_buff
*skb
)
614 struct enetc_rx_swbd
*rx_swbd
= enetc_get_rx_buff(rx_ring
, i
, size
);
616 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_swbd
->page
,
617 rx_swbd
->page_offset
, size
, ENETC_RXB_TRUESIZE
);
619 enetc_put_rx_buff(rx_ring
, rx_swbd
);
622 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
624 static int enetc_clean_rx_ring(struct enetc_bdr
*rx_ring
,
625 struct napi_struct
*napi
, int work_limit
)
627 int rx_frm_cnt
= 0, rx_byte_cnt
= 0;
630 cleaned_cnt
= enetc_bd_unused(rx_ring
);
631 /* next descriptor to process */
632 i
= rx_ring
->next_to_clean
;
634 while (likely(rx_frm_cnt
< work_limit
)) {
635 union enetc_rx_bd
*rxbd
;
640 if (cleaned_cnt
>= ENETC_RXBD_BUNDLE
) {
641 int count
= enetc_refill_rx_ring(rx_ring
, cleaned_cnt
);
643 cleaned_cnt
-= count
;
646 rxbd
= ENETC_RXBD(*rx_ring
, i
);
647 bd_status
= le32_to_cpu(rxbd
->r
.lstatus
);
651 enetc_wr_reg(rx_ring
->idr
, BIT(rx_ring
->index
));
652 dma_rmb(); /* for reading other rxbd fields */
653 size
= le16_to_cpu(rxbd
->r
.buf_len
);
654 skb
= enetc_map_rx_buff_to_skb(rx_ring
, i
, size
);
658 enetc_get_offloads(rx_ring
, rxbd
, skb
);
663 if (unlikely(i
== rx_ring
->bd_count
)) {
665 rxbd
= ENETC_RXBD(*rx_ring
, 0);
668 if (unlikely(bd_status
&
669 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK
))) {
671 while (!(bd_status
& ENETC_RXBD_LSTATUS_F
)) {
673 bd_status
= le32_to_cpu(rxbd
->r
.lstatus
);
676 if (unlikely(i
== rx_ring
->bd_count
)) {
678 rxbd
= ENETC_RXBD(*rx_ring
, 0);
682 rx_ring
->ndev
->stats
.rx_dropped
++;
683 rx_ring
->ndev
->stats
.rx_errors
++;
688 /* not last BD in frame? */
689 while (!(bd_status
& ENETC_RXBD_LSTATUS_F
)) {
690 bd_status
= le32_to_cpu(rxbd
->r
.lstatus
);
691 size
= ENETC_RXB_DMA_SIZE
;
693 if (bd_status
& ENETC_RXBD_LSTATUS_F
) {
695 size
= le16_to_cpu(rxbd
->r
.buf_len
);
698 enetc_add_rx_buff_to_skb(rx_ring
, i
, size
, skb
);
703 if (unlikely(i
== rx_ring
->bd_count
)) {
705 rxbd
= ENETC_RXBD(*rx_ring
, 0);
709 rx_byte_cnt
+= skb
->len
;
711 enetc_process_skb(rx_ring
, skb
);
713 napi_gro_receive(napi
, skb
);
718 rx_ring
->next_to_clean
= i
;
720 rx_ring
->stats
.packets
+= rx_frm_cnt
;
721 rx_ring
->stats
.bytes
+= rx_byte_cnt
;
726 /* Probing and Init */
727 #define ENETC_MAX_RFS_SIZE 64
728 void enetc_get_si_caps(struct enetc_si
*si
)
730 struct enetc_hw
*hw
= &si
->hw
;
733 /* find out how many of various resources we have to work with */
734 val
= enetc_rd(hw
, ENETC_SICAPR0
);
735 si
->num_rx_rings
= (val
>> 16) & 0xff;
736 si
->num_tx_rings
= val
& 0xff;
738 val
= enetc_rd(hw
, ENETC_SIRFSCAPR
);
739 si
->num_fs_entries
= ENETC_SIRFSCAPR_GET_NUM_RFS(val
);
740 si
->num_fs_entries
= min(si
->num_fs_entries
, ENETC_MAX_RFS_SIZE
);
743 val
= enetc_rd(hw
, ENETC_SIPCAPR0
);
744 if (val
& ENETC_SIPCAPR0_RSS
) {
745 val
= enetc_rd(hw
, ENETC_SIRSSCAPR
);
746 si
->num_rss
= ENETC_SIRSSCAPR_GET_NUM_RSS(val
);
750 static int enetc_dma_alloc_bdr(struct enetc_bdr
*r
, size_t bd_size
)
752 r
->bd_base
= dma_alloc_coherent(r
->dev
, r
->bd_count
* bd_size
,
753 &r
->bd_dma_base
, GFP_KERNEL
);
757 /* h/w requires 128B alignment */
758 if (!IS_ALIGNED(r
->bd_dma_base
, 128)) {
759 dma_free_coherent(r
->dev
, r
->bd_count
* bd_size
, r
->bd_base
,
767 static int enetc_alloc_txbdr(struct enetc_bdr
*txr
)
771 txr
->tx_swbd
= vzalloc(txr
->bd_count
* sizeof(struct enetc_tx_swbd
));
775 err
= enetc_dma_alloc_bdr(txr
, sizeof(union enetc_tx_bd
));
781 txr
->next_to_clean
= 0;
782 txr
->next_to_use
= 0;
787 static void enetc_free_txbdr(struct enetc_bdr
*txr
)
791 for (i
= 0; i
< txr
->bd_count
; i
++)
792 enetc_free_tx_skb(txr
, &txr
->tx_swbd
[i
]);
794 size
= txr
->bd_count
* sizeof(union enetc_tx_bd
);
796 dma_free_coherent(txr
->dev
, size
, txr
->bd_base
, txr
->bd_dma_base
);
803 static int enetc_alloc_tx_resources(struct enetc_ndev_priv
*priv
)
807 for (i
= 0; i
< priv
->num_tx_rings
; i
++) {
808 err
= enetc_alloc_txbdr(priv
->tx_ring
[i
]);
818 enetc_free_txbdr(priv
->tx_ring
[i
]);
823 static void enetc_free_tx_resources(struct enetc_ndev_priv
*priv
)
827 for (i
= 0; i
< priv
->num_tx_rings
; i
++)
828 enetc_free_txbdr(priv
->tx_ring
[i
]);
831 static int enetc_alloc_rxbdr(struct enetc_bdr
*rxr
)
835 rxr
->rx_swbd
= vzalloc(rxr
->bd_count
* sizeof(struct enetc_rx_swbd
));
839 err
= enetc_dma_alloc_bdr(rxr
, sizeof(union enetc_rx_bd
));
845 rxr
->next_to_clean
= 0;
846 rxr
->next_to_use
= 0;
847 rxr
->next_to_alloc
= 0;
852 static void enetc_free_rxbdr(struct enetc_bdr
*rxr
)
856 size
= rxr
->bd_count
* sizeof(union enetc_rx_bd
);
858 dma_free_coherent(rxr
->dev
, size
, rxr
->bd_base
, rxr
->bd_dma_base
);
865 static int enetc_alloc_rx_resources(struct enetc_ndev_priv
*priv
)
869 for (i
= 0; i
< priv
->num_rx_rings
; i
++) {
870 err
= enetc_alloc_rxbdr(priv
->rx_ring
[i
]);
880 enetc_free_rxbdr(priv
->rx_ring
[i
]);
885 static void enetc_free_rx_resources(struct enetc_ndev_priv
*priv
)
889 for (i
= 0; i
< priv
->num_rx_rings
; i
++)
890 enetc_free_rxbdr(priv
->rx_ring
[i
]);
893 static void enetc_free_tx_ring(struct enetc_bdr
*tx_ring
)
897 if (!tx_ring
->tx_swbd
)
900 for (i
= 0; i
< tx_ring
->bd_count
; i
++) {
901 struct enetc_tx_swbd
*tx_swbd
= &tx_ring
->tx_swbd
[i
];
903 enetc_free_tx_skb(tx_ring
, tx_swbd
);
906 tx_ring
->next_to_clean
= 0;
907 tx_ring
->next_to_use
= 0;
910 static void enetc_free_rx_ring(struct enetc_bdr
*rx_ring
)
914 if (!rx_ring
->rx_swbd
)
917 for (i
= 0; i
< rx_ring
->bd_count
; i
++) {
918 struct enetc_rx_swbd
*rx_swbd
= &rx_ring
->rx_swbd
[i
];
923 dma_unmap_page(rx_ring
->dev
, rx_swbd
->dma
,
924 PAGE_SIZE
, DMA_FROM_DEVICE
);
925 __free_page(rx_swbd
->page
);
926 rx_swbd
->page
= NULL
;
929 rx_ring
->next_to_clean
= 0;
930 rx_ring
->next_to_use
= 0;
931 rx_ring
->next_to_alloc
= 0;
934 static void enetc_free_rxtx_rings(struct enetc_ndev_priv
*priv
)
938 for (i
= 0; i
< priv
->num_rx_rings
; i
++)
939 enetc_free_rx_ring(priv
->rx_ring
[i
]);
941 for (i
= 0; i
< priv
->num_tx_rings
; i
++)
942 enetc_free_tx_ring(priv
->tx_ring
[i
]);
945 static int enetc_alloc_cbdr(struct device
*dev
, struct enetc_cbdr
*cbdr
)
947 int size
= cbdr
->bd_count
* sizeof(struct enetc_cbd
);
949 cbdr
->bd_base
= dma_alloc_coherent(dev
, size
, &cbdr
->bd_dma_base
,
954 /* h/w requires 128B alignment */
955 if (!IS_ALIGNED(cbdr
->bd_dma_base
, 128)) {
956 dma_free_coherent(dev
, size
, cbdr
->bd_base
, cbdr
->bd_dma_base
);
960 cbdr
->next_to_clean
= 0;
961 cbdr
->next_to_use
= 0;
966 static void enetc_free_cbdr(struct device
*dev
, struct enetc_cbdr
*cbdr
)
968 int size
= cbdr
->bd_count
* sizeof(struct enetc_cbd
);
970 dma_free_coherent(dev
, size
, cbdr
->bd_base
, cbdr
->bd_dma_base
);
971 cbdr
->bd_base
= NULL
;
974 static void enetc_setup_cbdr(struct enetc_hw
*hw
, struct enetc_cbdr
*cbdr
)
976 /* set CBDR cache attributes */
977 enetc_wr(hw
, ENETC_SICAR2
,
978 ENETC_SICAR_RD_COHERENT
| ENETC_SICAR_WR_COHERENT
);
980 enetc_wr(hw
, ENETC_SICBDRBAR0
, lower_32_bits(cbdr
->bd_dma_base
));
981 enetc_wr(hw
, ENETC_SICBDRBAR1
, upper_32_bits(cbdr
->bd_dma_base
));
982 enetc_wr(hw
, ENETC_SICBDRLENR
, ENETC_RTBLENR_LEN(cbdr
->bd_count
));
984 enetc_wr(hw
, ENETC_SICBDRPIR
, 0);
985 enetc_wr(hw
, ENETC_SICBDRCIR
, 0);
988 enetc_wr(hw
, ENETC_SICBDRMR
, BIT(31));
990 cbdr
->pir
= hw
->reg
+ ENETC_SICBDRPIR
;
991 cbdr
->cir
= hw
->reg
+ ENETC_SICBDRCIR
;
994 static void enetc_clear_cbdr(struct enetc_hw
*hw
)
996 enetc_wr(hw
, ENETC_SICBDRMR
, 0);
999 static int enetc_setup_default_rss_table(struct enetc_si
*si
, int num_groups
)
1004 rss_table
= kmalloc_array(si
->num_rss
, sizeof(*rss_table
), GFP_KERNEL
);
1008 /* Set up RSS table defaults */
1009 for (i
= 0; i
< si
->num_rss
; i
++)
1010 rss_table
[i
] = i
% num_groups
;
1012 enetc_set_rss_table(si
, rss_table
, si
->num_rss
);
1019 static int enetc_configure_si(struct enetc_ndev_priv
*priv
)
1021 struct enetc_si
*si
= priv
->si
;
1022 struct enetc_hw
*hw
= &si
->hw
;
1025 enetc_setup_cbdr(hw
, &si
->cbd_ring
);
1026 /* set SI cache attributes */
1027 enetc_wr(hw
, ENETC_SICAR0
,
1028 ENETC_SICAR_RD_COHERENT
| ENETC_SICAR_WR_COHERENT
);
1029 enetc_wr(hw
, ENETC_SICAR1
, ENETC_SICAR_MSI
);
1031 enetc_wr(hw
, ENETC_SIMR
, ENETC_SIMR_EN
);
1034 err
= enetc_setup_default_rss_table(si
, priv
->num_rx_rings
);
1042 void enetc_init_si_rings_params(struct enetc_ndev_priv
*priv
)
1044 struct enetc_si
*si
= priv
->si
;
1045 int cpus
= num_online_cpus();
1047 priv
->tx_bd_count
= ENETC_BDR_DEFAULT_SIZE
;
1048 priv
->rx_bd_count
= ENETC_BDR_DEFAULT_SIZE
;
1050 /* Enable all available TX rings in order to configure as many
1051 * priorities as possible, when needed.
1052 * TODO: Make # of TX rings run-time configurable
1054 priv
->num_rx_rings
= min_t(int, cpus
, si
->num_rx_rings
);
1055 priv
->num_tx_rings
= si
->num_tx_rings
;
1056 priv
->bdr_int_num
= cpus
;
1059 si
->cbd_ring
.bd_count
= ENETC_CBDR_DEFAULT_SIZE
;
1062 int enetc_alloc_si_resources(struct enetc_ndev_priv
*priv
)
1064 struct enetc_si
*si
= priv
->si
;
1067 err
= enetc_alloc_cbdr(priv
->dev
, &si
->cbd_ring
);
1071 priv
->cls_rules
= kcalloc(si
->num_fs_entries
, sizeof(*priv
->cls_rules
),
1073 if (!priv
->cls_rules
) {
1078 err
= enetc_configure_si(priv
);
1085 kfree(priv
->cls_rules
);
1087 enetc_clear_cbdr(&si
->hw
);
1088 enetc_free_cbdr(priv
->dev
, &si
->cbd_ring
);
1093 void enetc_free_si_resources(struct enetc_ndev_priv
*priv
)
1095 struct enetc_si
*si
= priv
->si
;
1097 enetc_clear_cbdr(&si
->hw
);
1098 enetc_free_cbdr(priv
->dev
, &si
->cbd_ring
);
1100 kfree(priv
->cls_rules
);
1103 static void enetc_setup_txbdr(struct enetc_hw
*hw
, struct enetc_bdr
*tx_ring
)
1105 int idx
= tx_ring
->index
;
1108 enetc_txbdr_wr(hw
, idx
, ENETC_TBBAR0
,
1109 lower_32_bits(tx_ring
->bd_dma_base
));
1111 enetc_txbdr_wr(hw
, idx
, ENETC_TBBAR1
,
1112 upper_32_bits(tx_ring
->bd_dma_base
));
1114 WARN_ON(!IS_ALIGNED(tx_ring
->bd_count
, 64)); /* multiple of 64 */
1115 enetc_txbdr_wr(hw
, idx
, ENETC_TBLENR
,
1116 ENETC_RTBLENR_LEN(tx_ring
->bd_count
));
1118 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1119 tx_ring
->next_to_use
= enetc_txbdr_rd(hw
, idx
, ENETC_TBPIR
);
1120 tx_ring
->next_to_clean
= enetc_txbdr_rd(hw
, idx
, ENETC_TBCIR
);
1122 /* enable Tx ints by setting pkt thr to 1 */
1123 enetc_txbdr_wr(hw
, idx
, ENETC_TBICIR0
, ENETC_TBICIR0_ICEN
| 0x1);
1125 tbmr
= ENETC_TBMR_EN
;
1126 if (tx_ring
->ndev
->features
& NETIF_F_HW_VLAN_CTAG_TX
)
1127 tbmr
|= ENETC_TBMR_VIH
;
1130 enetc_txbdr_wr(hw
, idx
, ENETC_TBMR
, tbmr
);
1132 tx_ring
->tpir
= hw
->reg
+ ENETC_BDR(TX
, idx
, ENETC_TBPIR
);
1133 tx_ring
->tcir
= hw
->reg
+ ENETC_BDR(TX
, idx
, ENETC_TBCIR
);
1134 tx_ring
->idr
= hw
->reg
+ ENETC_SITXIDR
;
1137 static void enetc_setup_rxbdr(struct enetc_hw
*hw
, struct enetc_bdr
*rx_ring
)
1139 int idx
= rx_ring
->index
;
1142 enetc_rxbdr_wr(hw
, idx
, ENETC_RBBAR0
,
1143 lower_32_bits(rx_ring
->bd_dma_base
));
1145 enetc_rxbdr_wr(hw
, idx
, ENETC_RBBAR1
,
1146 upper_32_bits(rx_ring
->bd_dma_base
));
1148 WARN_ON(!IS_ALIGNED(rx_ring
->bd_count
, 64)); /* multiple of 64 */
1149 enetc_rxbdr_wr(hw
, idx
, ENETC_RBLENR
,
1150 ENETC_RTBLENR_LEN(rx_ring
->bd_count
));
1152 enetc_rxbdr_wr(hw
, idx
, ENETC_RBBSR
, ENETC_RXB_DMA_SIZE
);
1154 enetc_rxbdr_wr(hw
, idx
, ENETC_RBPIR
, 0);
1156 /* enable Rx ints by setting pkt thr to 1 */
1157 enetc_rxbdr_wr(hw
, idx
, ENETC_RBICIR0
, ENETC_RBICIR0_ICEN
| 0x1);
1159 rbmr
= ENETC_RBMR_EN
;
1160 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1161 rbmr
|= ENETC_RBMR_BDS
;
1163 if (rx_ring
->ndev
->features
& NETIF_F_HW_VLAN_CTAG_RX
)
1164 rbmr
|= ENETC_RBMR_VTE
;
1166 rx_ring
->rcir
= hw
->reg
+ ENETC_BDR(RX
, idx
, ENETC_RBCIR
);
1167 rx_ring
->idr
= hw
->reg
+ ENETC_SIRXIDR
;
1169 enetc_refill_rx_ring(rx_ring
, enetc_bd_unused(rx_ring
));
1172 enetc_rxbdr_wr(hw
, idx
, ENETC_RBMR
, rbmr
);
1175 static void enetc_setup_bdrs(struct enetc_ndev_priv
*priv
)
1179 for (i
= 0; i
< priv
->num_tx_rings
; i
++)
1180 enetc_setup_txbdr(&priv
->si
->hw
, priv
->tx_ring
[i
]);
1182 for (i
= 0; i
< priv
->num_rx_rings
; i
++)
1183 enetc_setup_rxbdr(&priv
->si
->hw
, priv
->rx_ring
[i
]);
1186 static void enetc_clear_rxbdr(struct enetc_hw
*hw
, struct enetc_bdr
*rx_ring
)
1188 int idx
= rx_ring
->index
;
1190 /* disable EN bit on ring */
1191 enetc_rxbdr_wr(hw
, idx
, ENETC_RBMR
, 0);
1194 static void enetc_clear_txbdr(struct enetc_hw
*hw
, struct enetc_bdr
*tx_ring
)
1196 int delay
= 8, timeout
= 100;
1197 int idx
= tx_ring
->index
;
1199 /* disable EN bit on ring */
1200 enetc_txbdr_wr(hw
, idx
, ENETC_TBMR
, 0);
1202 /* wait for busy to clear */
1203 while (delay
< timeout
&&
1204 enetc_txbdr_rd(hw
, idx
, ENETC_TBSR
) & ENETC_TBSR_BUSY
) {
1209 if (delay
>= timeout
)
1210 netdev_warn(tx_ring
->ndev
, "timeout for tx ring #%d clear\n",
1214 static void enetc_clear_bdrs(struct enetc_ndev_priv
*priv
)
1218 for (i
= 0; i
< priv
->num_tx_rings
; i
++)
1219 enetc_clear_txbdr(&priv
->si
->hw
, priv
->tx_ring
[i
]);
1221 for (i
= 0; i
< priv
->num_rx_rings
; i
++)
1222 enetc_clear_rxbdr(&priv
->si
->hw
, priv
->rx_ring
[i
]);
1227 static int enetc_setup_irqs(struct enetc_ndev_priv
*priv
)
1229 struct pci_dev
*pdev
= priv
->si
->pdev
;
1233 for (i
= 0; i
< priv
->bdr_int_num
; i
++) {
1234 int irq
= pci_irq_vector(pdev
, ENETC_BDR_INT_BASE_IDX
+ i
);
1235 struct enetc_int_vector
*v
= priv
->int_vector
[i
];
1236 int entry
= ENETC_BDR_INT_BASE_IDX
+ i
;
1237 struct enetc_hw
*hw
= &priv
->si
->hw
;
1239 snprintf(v
->name
, sizeof(v
->name
), "%s-rxtx%d",
1240 priv
->ndev
->name
, i
);
1241 err
= request_irq(irq
, enetc_msix
, 0, v
->name
, v
);
1243 dev_err(priv
->dev
, "request_irq() failed!\n");
1247 v
->tbier_base
= hw
->reg
+ ENETC_BDR(TX
, 0, ENETC_TBIER
);
1248 v
->rbier
= hw
->reg
+ ENETC_BDR(RX
, i
, ENETC_RBIER
);
1250 enetc_wr(hw
, ENETC_SIMSIRRV(i
), entry
);
1252 for (j
= 0; j
< v
->count_tx_rings
; j
++) {
1253 int idx
= v
->tx_ring
[j
].index
;
1255 enetc_wr(hw
, ENETC_SIMSITRV(idx
), entry
);
1257 cpumask_clear(&cpu_mask
);
1258 cpumask_set_cpu(i
% num_online_cpus(), &cpu_mask
);
1259 irq_set_affinity_hint(irq
, &cpu_mask
);
1266 int irq
= pci_irq_vector(pdev
, ENETC_BDR_INT_BASE_IDX
+ i
);
1268 irq_set_affinity_hint(irq
, NULL
);
1269 free_irq(irq
, priv
->int_vector
[i
]);
1275 static void enetc_free_irqs(struct enetc_ndev_priv
*priv
)
1277 struct pci_dev
*pdev
= priv
->si
->pdev
;
1280 for (i
= 0; i
< priv
->bdr_int_num
; i
++) {
1281 int irq
= pci_irq_vector(pdev
, ENETC_BDR_INT_BASE_IDX
+ i
);
1283 irq_set_affinity_hint(irq
, NULL
);
1284 free_irq(irq
, priv
->int_vector
[i
]);
1288 static void enetc_enable_interrupts(struct enetc_ndev_priv
*priv
)
1292 /* enable Tx & Rx event indication */
1293 for (i
= 0; i
< priv
->num_rx_rings
; i
++) {
1294 enetc_rxbdr_wr(&priv
->si
->hw
, i
,
1295 ENETC_RBIER
, ENETC_RBIER_RXTIE
);
1298 for (i
= 0; i
< priv
->num_tx_rings
; i
++) {
1299 enetc_txbdr_wr(&priv
->si
->hw
, i
,
1300 ENETC_TBIER
, ENETC_TBIER_TXTIE
);
1304 static void enetc_disable_interrupts(struct enetc_ndev_priv
*priv
)
1308 for (i
= 0; i
< priv
->num_tx_rings
; i
++)
1309 enetc_txbdr_wr(&priv
->si
->hw
, i
, ENETC_TBIER
, 0);
1311 for (i
= 0; i
< priv
->num_rx_rings
; i
++)
1312 enetc_rxbdr_wr(&priv
->si
->hw
, i
, ENETC_RBIER
, 0);
1315 static void adjust_link(struct net_device
*ndev
)
1317 struct phy_device
*phydev
= ndev
->phydev
;
1319 phy_print_status(phydev
);
1322 static int enetc_phy_connect(struct net_device
*ndev
)
1324 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1325 struct phy_device
*phydev
;
1327 if (!priv
->phy_node
)
1328 return 0; /* phy-less mode */
1330 phydev
= of_phy_connect(ndev
, priv
->phy_node
, &adjust_link
,
1333 dev_err(&ndev
->dev
, "could not attach to PHY\n");
1337 phy_attached_info(phydev
);
1342 int enetc_open(struct net_device
*ndev
)
1344 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1347 err
= enetc_setup_irqs(priv
);
1351 err
= enetc_phy_connect(ndev
);
1353 goto err_phy_connect
;
1355 err
= enetc_alloc_tx_resources(priv
);
1359 err
= enetc_alloc_rx_resources(priv
);
1363 enetc_setup_bdrs(priv
);
1365 err
= netif_set_real_num_tx_queues(ndev
, priv
->num_tx_rings
);
1367 goto err_set_queues
;
1369 err
= netif_set_real_num_rx_queues(ndev
, priv
->num_rx_rings
);
1371 goto err_set_queues
;
1373 for (i
= 0; i
< priv
->bdr_int_num
; i
++)
1374 napi_enable(&priv
->int_vector
[i
]->napi
);
1376 enetc_enable_interrupts(priv
);
1379 phy_start(ndev
->phydev
);
1381 netif_carrier_on(ndev
);
1383 netif_tx_start_all_queues(ndev
);
1388 enetc_free_rx_resources(priv
);
1390 enetc_free_tx_resources(priv
);
1393 phy_disconnect(ndev
->phydev
);
1395 enetc_free_irqs(priv
);
1400 int enetc_close(struct net_device
*ndev
)
1402 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1405 netif_tx_stop_all_queues(ndev
);
1408 phy_stop(ndev
->phydev
);
1409 phy_disconnect(ndev
->phydev
);
1411 netif_carrier_off(ndev
);
1414 for (i
= 0; i
< priv
->bdr_int_num
; i
++) {
1415 napi_synchronize(&priv
->int_vector
[i
]->napi
);
1416 napi_disable(&priv
->int_vector
[i
]->napi
);
1419 enetc_disable_interrupts(priv
);
1420 enetc_clear_bdrs(priv
);
1422 enetc_free_rxtx_rings(priv
);
1423 enetc_free_rx_resources(priv
);
1424 enetc_free_tx_resources(priv
);
1425 enetc_free_irqs(priv
);
1430 int enetc_setup_tc(struct net_device
*ndev
, enum tc_setup_type type
,
1433 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1434 struct tc_mqprio_qopt
*mqprio
= type_data
;
1435 struct enetc_bdr
*tx_ring
;
1439 if (type
!= TC_SETUP_QDISC_MQPRIO
)
1442 mqprio
->hw
= TC_MQPRIO_HW_OFFLOAD_TCS
;
1443 num_tc
= mqprio
->num_tc
;
1446 netdev_reset_tc(ndev
);
1447 netif_set_real_num_tx_queues(ndev
, priv
->num_tx_rings
);
1449 /* Reset all ring priorities to 0 */
1450 for (i
= 0; i
< priv
->num_tx_rings
; i
++) {
1451 tx_ring
= priv
->tx_ring
[i
];
1452 enetc_set_bdr_prio(&priv
->si
->hw
, tx_ring
->index
, 0);
1458 /* Check if we have enough BD rings available to accommodate all TCs */
1459 if (num_tc
> priv
->num_tx_rings
) {
1460 netdev_err(ndev
, "Max %d traffic classes supported\n",
1461 priv
->num_tx_rings
);
1465 /* For the moment, we use only one BD ring per TC.
1467 * Configure num_tc BD rings with increasing priorities.
1469 for (i
= 0; i
< num_tc
; i
++) {
1470 tx_ring
= priv
->tx_ring
[i
];
1471 enetc_set_bdr_prio(&priv
->si
->hw
, tx_ring
->index
, i
);
1474 /* Reset the number of netdev queues based on the TC count */
1475 netif_set_real_num_tx_queues(ndev
, num_tc
);
1477 netdev_set_num_tc(ndev
, num_tc
);
1479 /* Each TC is associated with one netdev queue */
1480 for (i
= 0; i
< num_tc
; i
++)
1481 netdev_set_tc_queue(ndev
, i
, 1, i
);
1486 struct net_device_stats
*enetc_get_stats(struct net_device
*ndev
)
1488 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1489 struct net_device_stats
*stats
= &ndev
->stats
;
1490 unsigned long packets
= 0, bytes
= 0;
1493 for (i
= 0; i
< priv
->num_rx_rings
; i
++) {
1494 packets
+= priv
->rx_ring
[i
]->stats
.packets
;
1495 bytes
+= priv
->rx_ring
[i
]->stats
.bytes
;
1498 stats
->rx_packets
= packets
;
1499 stats
->rx_bytes
= bytes
;
1503 for (i
= 0; i
< priv
->num_tx_rings
; i
++) {
1504 packets
+= priv
->tx_ring
[i
]->stats
.packets
;
1505 bytes
+= priv
->tx_ring
[i
]->stats
.bytes
;
1508 stats
->tx_packets
= packets
;
1509 stats
->tx_bytes
= bytes
;
1514 static int enetc_set_rss(struct net_device
*ndev
, int en
)
1516 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1517 struct enetc_hw
*hw
= &priv
->si
->hw
;
1520 enetc_wr(hw
, ENETC_SIRBGCR
, priv
->num_rx_rings
);
1522 reg
= enetc_rd(hw
, ENETC_SIMR
);
1523 reg
&= ~ENETC_SIMR_RSSE
;
1524 reg
|= (en
) ? ENETC_SIMR_RSSE
: 0;
1525 enetc_wr(hw
, ENETC_SIMR
, reg
);
1530 int enetc_set_features(struct net_device
*ndev
,
1531 netdev_features_t features
)
1533 netdev_features_t changed
= ndev
->features
^ features
;
1535 if (changed
& NETIF_F_RXHASH
)
1536 enetc_set_rss(ndev
, !!(features
& NETIF_F_RXHASH
));
1541 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1542 static int enetc_hwtstamp_set(struct net_device
*ndev
, struct ifreq
*ifr
)
1544 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1545 struct hwtstamp_config config
;
1547 if (copy_from_user(&config
, ifr
->ifr_data
, sizeof(config
)))
1550 switch (config
.tx_type
) {
1551 case HWTSTAMP_TX_OFF
:
1552 priv
->active_offloads
&= ~ENETC_F_TX_TSTAMP
;
1554 case HWTSTAMP_TX_ON
:
1555 priv
->active_offloads
|= ENETC_F_TX_TSTAMP
;
1561 switch (config
.rx_filter
) {
1562 case HWTSTAMP_FILTER_NONE
:
1563 priv
->active_offloads
&= ~ENETC_F_RX_TSTAMP
;
1566 priv
->active_offloads
|= ENETC_F_RX_TSTAMP
;
1567 config
.rx_filter
= HWTSTAMP_FILTER_ALL
;
1570 return copy_to_user(ifr
->ifr_data
, &config
, sizeof(config
)) ?
1574 static int enetc_hwtstamp_get(struct net_device
*ndev
, struct ifreq
*ifr
)
1576 struct enetc_ndev_priv
*priv
= netdev_priv(ndev
);
1577 struct hwtstamp_config config
;
1581 if (priv
->active_offloads
& ENETC_F_TX_TSTAMP
)
1582 config
.tx_type
= HWTSTAMP_TX_ON
;
1584 config
.tx_type
= HWTSTAMP_TX_OFF
;
1586 config
.rx_filter
= (priv
->active_offloads
& ENETC_F_RX_TSTAMP
) ?
1587 HWTSTAMP_FILTER_ALL
: HWTSTAMP_FILTER_NONE
;
1589 return copy_to_user(ifr
->ifr_data
, &config
, sizeof(config
)) ?
1594 int enetc_ioctl(struct net_device
*ndev
, struct ifreq
*rq
, int cmd
)
1596 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1597 if (cmd
== SIOCSHWTSTAMP
)
1598 return enetc_hwtstamp_set(ndev
, rq
);
1599 if (cmd
== SIOCGHWTSTAMP
)
1600 return enetc_hwtstamp_get(ndev
, rq
);
1605 int enetc_alloc_msix(struct enetc_ndev_priv
*priv
)
1607 struct pci_dev
*pdev
= priv
->si
->pdev
;
1608 int size
, v_tx_rings
;
1609 int i
, n
, err
, nvec
;
1611 nvec
= ENETC_BDR_INT_BASE_IDX
+ priv
->bdr_int_num
;
1612 /* allocate MSIX for both messaging and Rx/Tx interrupts */
1613 n
= pci_alloc_irq_vectors(pdev
, nvec
, nvec
, PCI_IRQ_MSIX
);
1621 /* # of tx rings per int vector */
1622 v_tx_rings
= priv
->num_tx_rings
/ priv
->bdr_int_num
;
1623 size
= sizeof(struct enetc_int_vector
) +
1624 sizeof(struct enetc_bdr
) * v_tx_rings
;
1626 for (i
= 0; i
< priv
->bdr_int_num
; i
++) {
1627 struct enetc_int_vector
*v
;
1628 struct enetc_bdr
*bdr
;
1631 v
= kzalloc(size
, GFP_KERNEL
);
1637 priv
->int_vector
[i
] = v
;
1639 netif_napi_add(priv
->ndev
, &v
->napi
, enetc_poll
,
1641 v
->count_tx_rings
= v_tx_rings
;
1643 for (j
= 0; j
< v_tx_rings
; j
++) {
1646 /* default tx ring mapping policy */
1647 if (priv
->bdr_int_num
== ENETC_MAX_BDR_INT
)
1648 idx
= 2 * j
+ i
; /* 2 CPUs */
1650 idx
= j
+ i
* v_tx_rings
; /* default */
1652 __set_bit(idx
, &v
->tx_rings_map
);
1653 bdr
= &v
->tx_ring
[j
];
1655 bdr
->ndev
= priv
->ndev
;
1656 bdr
->dev
= priv
->dev
;
1657 bdr
->bd_count
= priv
->tx_bd_count
;
1658 priv
->tx_ring
[idx
] = bdr
;
1663 bdr
->ndev
= priv
->ndev
;
1664 bdr
->dev
= priv
->dev
;
1665 bdr
->bd_count
= priv
->rx_bd_count
;
1666 priv
->rx_ring
[i
] = bdr
;
1673 netif_napi_del(&priv
->int_vector
[i
]->napi
);
1674 kfree(priv
->int_vector
[i
]);
1677 pci_free_irq_vectors(pdev
);
1682 void enetc_free_msix(struct enetc_ndev_priv
*priv
)
1686 for (i
= 0; i
< priv
->bdr_int_num
; i
++) {
1687 struct enetc_int_vector
*v
= priv
->int_vector
[i
];
1689 netif_napi_del(&v
->napi
);
1692 for (i
= 0; i
< priv
->num_rx_rings
; i
++)
1693 priv
->rx_ring
[i
] = NULL
;
1695 for (i
= 0; i
< priv
->num_tx_rings
; i
++)
1696 priv
->tx_ring
[i
] = NULL
;
1698 for (i
= 0; i
< priv
->bdr_int_num
; i
++) {
1699 kfree(priv
->int_vector
[i
]);
1700 priv
->int_vector
[i
] = NULL
;
1703 /* disable all MSIX for this device */
1704 pci_free_irq_vectors(priv
->si
->pdev
);
1707 static void enetc_kfree_si(struct enetc_si
*si
)
1709 char *p
= (char *)si
- si
->pad
;
1714 static void enetc_detect_errata(struct enetc_si
*si
)
1716 if (si
->pdev
->revision
== ENETC_REV1
)
1717 si
->errata
= ENETC_ERR_TXCSUM
| ENETC_ERR_VLAN_ISOL
|
1721 int enetc_pci_probe(struct pci_dev
*pdev
, const char *name
, int sizeof_priv
)
1723 struct enetc_si
*si
, *p
;
1724 struct enetc_hw
*hw
;
1729 err
= pci_enable_device_mem(pdev
);
1731 dev_err(&pdev
->dev
, "device enable failed\n");
1735 /* set up for high or low dma */
1736 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
1738 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(32));
1741 "DMA configuration failed: 0x%x\n", err
);
1746 err
= pci_request_mem_regions(pdev
, name
);
1748 dev_err(&pdev
->dev
, "pci_request_regions failed err=%d\n", err
);
1749 goto err_pci_mem_reg
;
1752 pci_set_master(pdev
);
1754 alloc_size
= sizeof(struct enetc_si
);
1756 /* align priv to 32B */
1757 alloc_size
= ALIGN(alloc_size
, ENETC_SI_ALIGN
);
1758 alloc_size
+= sizeof_priv
;
1760 /* force 32B alignment for enetc_si */
1761 alloc_size
+= ENETC_SI_ALIGN
- 1;
1763 p
= kzalloc(alloc_size
, GFP_KERNEL
);
1769 si
= PTR_ALIGN(p
, ENETC_SI_ALIGN
);
1770 si
->pad
= (char *)si
- (char *)p
;
1772 pci_set_drvdata(pdev
, si
);
1776 len
= pci_resource_len(pdev
, ENETC_BAR_REGS
);
1777 hw
->reg
= ioremap(pci_resource_start(pdev
, ENETC_BAR_REGS
), len
);
1780 dev_err(&pdev
->dev
, "ioremap() failed\n");
1783 if (len
> ENETC_PORT_BASE
)
1784 hw
->port
= hw
->reg
+ ENETC_PORT_BASE
;
1785 if (len
> ENETC_GLOBAL_BASE
)
1786 hw
->global
= hw
->reg
+ ENETC_GLOBAL_BASE
;
1788 enetc_detect_errata(si
);
1795 pci_release_mem_regions(pdev
);
1798 pci_disable_device(pdev
);
1803 void enetc_pci_remove(struct pci_dev
*pdev
)
1805 struct enetc_si
*si
= pci_get_drvdata(pdev
);
1806 struct enetc_hw
*hw
= &si
->hw
;
1810 pci_release_mem_regions(pdev
);
1811 pci_disable_device(pdev
);