1 /* Applied Micro X-Gene SoC Ethernet Driver
3 * Copyright (c) 2014, Applied Micro Circuits Corporation
4 * Authors: Iyappan Subramanian <isubramanian@apm.com>
5 * Ravi Patel <rapatel@apm.com>
6 * Keyur Chudgar <kchudgar@apm.com>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the
10 * Free Software Foundation; either version 2 of the License, or (at your
11 * option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
22 #include <linux/gpio.h>
23 #include "xgene_enet_main.h"
24 #include "xgene_enet_hw.h"
25 #include "xgene_enet_sgmac.h"
26 #include "xgene_enet_xgmac.h"
28 #define RES_ENET_CSR 0
29 #define RES_RING_CSR 1
30 #define RES_RING_CMD 2
32 static const struct of_device_id xgene_enet_of_match
[];
33 static const struct acpi_device_id xgene_enet_acpi_match
[];
35 static void xgene_enet_init_bufpool(struct xgene_enet_desc_ring
*buf_pool
)
37 struct xgene_enet_raw_desc16
*raw_desc
;
43 for (i
= 0; i
< buf_pool
->slots
; i
++) {
44 raw_desc
= &buf_pool
->raw_desc16
[i
];
46 /* Hardware expects descriptor in little endian format */
47 raw_desc
->m0
= cpu_to_le64(i
|
48 SET_VAL(FPQNUM
, buf_pool
->dst_ring_num
) |
53 static u16
xgene_enet_get_data_len(u64 bufdatalen
)
57 hw_len
= GET_VAL(BUFDATALEN
, bufdatalen
);
59 if (unlikely(hw_len
== 0x7800)) {
61 } else if (!(hw_len
& BIT(14))) {
62 mask
= GENMASK(13, 0);
63 return (hw_len
& mask
) ? (hw_len
& mask
) : SIZE_16K
;
64 } else if (!(hw_len
& GENMASK(13, 12))) {
65 mask
= GENMASK(11, 0);
66 return (hw_len
& mask
) ? (hw_len
& mask
) : SIZE_4K
;
68 mask
= GENMASK(11, 0);
69 return (hw_len
& mask
) ? (hw_len
& mask
) : SIZE_2K
;
73 static u16
xgene_enet_set_data_len(u32 size
)
77 hw_len
= (size
== SIZE_4K
) ? BIT(14) : 0;
82 static int xgene_enet_refill_pagepool(struct xgene_enet_desc_ring
*buf_pool
,
85 struct xgene_enet_raw_desc16
*raw_desc
;
86 struct xgene_enet_pdata
*pdata
;
87 struct net_device
*ndev
;
95 if (unlikely(!buf_pool
))
98 ndev
= buf_pool
->ndev
;
99 pdata
= netdev_priv(ndev
);
100 dev
= ndev_to_dev(ndev
);
101 slots
= buf_pool
->slots
- 1;
102 tail
= buf_pool
->tail
;
104 for (i
= 0; i
< nbuf
; i
++) {
105 raw_desc
= &buf_pool
->raw_desc16
[tail
];
107 page
= dev_alloc_page();
111 dma_addr
= dma_map_page(dev
, page
, 0,
112 PAGE_SIZE
, DMA_FROM_DEVICE
);
113 if (unlikely(dma_mapping_error(dev
, dma_addr
))) {
118 hw_len
= xgene_enet_set_data_len(PAGE_SIZE
);
119 raw_desc
->m1
= cpu_to_le64(SET_VAL(DATAADDR
, dma_addr
) |
120 SET_VAL(BUFDATALEN
, hw_len
) |
123 buf_pool
->frag_page
[tail
] = page
;
124 tail
= (tail
+ 1) & slots
;
127 pdata
->ring_ops
->wr_cmd(buf_pool
, nbuf
);
128 buf_pool
->tail
= tail
;
133 static int xgene_enet_refill_bufpool(struct xgene_enet_desc_ring
*buf_pool
,
137 struct xgene_enet_raw_desc16
*raw_desc
;
138 struct xgene_enet_pdata
*pdata
;
139 struct net_device
*ndev
;
142 u32 tail
= buf_pool
->tail
;
143 u32 slots
= buf_pool
->slots
- 1;
147 ndev
= buf_pool
->ndev
;
148 dev
= ndev_to_dev(buf_pool
->ndev
);
149 pdata
= netdev_priv(ndev
);
151 bufdatalen
= BUF_LEN_CODE_2K
| (SKB_BUFFER_SIZE
& GENMASK(11, 0));
152 len
= XGENE_ENET_STD_MTU
;
154 for (i
= 0; i
< nbuf
; i
++) {
155 raw_desc
= &buf_pool
->raw_desc16
[tail
];
157 skb
= netdev_alloc_skb_ip_align(ndev
, len
);
161 dma_addr
= dma_map_single(dev
, skb
->data
, len
, DMA_FROM_DEVICE
);
162 if (dma_mapping_error(dev
, dma_addr
)) {
163 netdev_err(ndev
, "DMA mapping error\n");
164 dev_kfree_skb_any(skb
);
168 buf_pool
->rx_skb
[tail
] = skb
;
170 raw_desc
->m1
= cpu_to_le64(SET_VAL(DATAADDR
, dma_addr
) |
171 SET_VAL(BUFDATALEN
, bufdatalen
) |
173 tail
= (tail
+ 1) & slots
;
176 pdata
->ring_ops
->wr_cmd(buf_pool
, nbuf
);
177 buf_pool
->tail
= tail
;
182 static u8
xgene_enet_hdr_len(const void *data
)
184 const struct ethhdr
*eth
= data
;
186 return (eth
->h_proto
== htons(ETH_P_8021Q
)) ? VLAN_ETH_HLEN
: ETH_HLEN
;
189 static void xgene_enet_delete_bufpool(struct xgene_enet_desc_ring
*buf_pool
)
191 struct device
*dev
= ndev_to_dev(buf_pool
->ndev
);
192 struct xgene_enet_raw_desc16
*raw_desc
;
196 /* Free up the buffers held by hardware */
197 for (i
= 0; i
< buf_pool
->slots
; i
++) {
198 if (buf_pool
->rx_skb
[i
]) {
199 dev_kfree_skb_any(buf_pool
->rx_skb
[i
]);
201 raw_desc
= &buf_pool
->raw_desc16
[i
];
202 dma_addr
= GET_VAL(DATAADDR
, le64_to_cpu(raw_desc
->m1
));
203 dma_unmap_single(dev
, dma_addr
, XGENE_ENET_MAX_MTU
,
209 static void xgene_enet_delete_pagepool(struct xgene_enet_desc_ring
*buf_pool
)
211 struct device
*dev
= ndev_to_dev(buf_pool
->ndev
);
216 /* Free up the buffers held by hardware */
217 for (i
= 0; i
< buf_pool
->slots
; i
++) {
218 page
= buf_pool
->frag_page
[i
];
220 dma_addr
= buf_pool
->frag_dma_addr
[i
];
221 dma_unmap_page(dev
, dma_addr
, PAGE_SIZE
,
228 static irqreturn_t
xgene_enet_rx_irq(const int irq
, void *data
)
230 struct xgene_enet_desc_ring
*rx_ring
= data
;
232 if (napi_schedule_prep(&rx_ring
->napi
)) {
233 disable_irq_nosync(irq
);
234 __napi_schedule(&rx_ring
->napi
);
240 static int xgene_enet_tx_completion(struct xgene_enet_desc_ring
*cp_ring
,
241 struct xgene_enet_raw_desc
*raw_desc
)
243 struct xgene_enet_pdata
*pdata
= netdev_priv(cp_ring
->ndev
);
247 dma_addr_t
*frag_dma_addr
;
253 skb_index
= GET_VAL(USERINFO
, le64_to_cpu(raw_desc
->m0
));
254 skb
= cp_ring
->cp_skb
[skb_index
];
255 frag_dma_addr
= &cp_ring
->frag_dma_addr
[skb_index
* MAX_SKB_FRAGS
];
257 dev
= ndev_to_dev(cp_ring
->ndev
);
258 dma_unmap_single(dev
, GET_VAL(DATAADDR
, le64_to_cpu(raw_desc
->m1
)),
262 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
263 frag
= &skb_shinfo(skb
)->frags
[i
];
264 dma_unmap_page(dev
, frag_dma_addr
[i
], skb_frag_size(frag
),
268 if (GET_BIT(ET
, le64_to_cpu(raw_desc
->m3
))) {
269 mss_index
= GET_VAL(MSS
, le64_to_cpu(raw_desc
->m3
));
270 spin_lock(&pdata
->mss_lock
);
271 pdata
->mss_refcnt
[mss_index
]--;
272 spin_unlock(&pdata
->mss_lock
);
275 /* Checking for error */
276 status
= GET_VAL(LERR
, le64_to_cpu(raw_desc
->m0
));
277 if (unlikely(status
> 2)) {
278 cp_ring
->tx_dropped
++;
279 cp_ring
->tx_errors
++;
283 dev_kfree_skb_any(skb
);
285 netdev_err(cp_ring
->ndev
, "completion skb is NULL\n");
291 static int xgene_enet_setup_mss(struct net_device
*ndev
, u32 mss
)
293 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
294 int mss_index
= -EBUSY
;
297 spin_lock(&pdata
->mss_lock
);
299 /* Reuse the slot if MSS matches */
300 for (i
= 0; mss_index
< 0 && i
< NUM_MSS_REG
; i
++) {
301 if (pdata
->mss
[i
] == mss
) {
302 pdata
->mss_refcnt
[i
]++;
307 /* Overwrite the slot with ref_count = 0 */
308 for (i
= 0; mss_index
< 0 && i
< NUM_MSS_REG
; i
++) {
309 if (!pdata
->mss_refcnt
[i
]) {
310 pdata
->mss_refcnt
[i
]++;
311 pdata
->mac_ops
->set_mss(pdata
, mss
, i
);
317 spin_unlock(&pdata
->mss_lock
);
322 static int xgene_enet_work_msg(struct sk_buff
*skb
, u64
*hopinfo
)
324 struct net_device
*ndev
= skb
->dev
;
326 u8 l3hlen
= 0, l4hlen
= 0;
327 u8 ethhdr
, proto
= 0, csum_enable
= 0;
328 u32 hdr_len
, mss
= 0;
329 u32 i
, len
, nr_frags
;
332 ethhdr
= xgene_enet_hdr_len(skb
->data
);
334 if (unlikely(skb
->protocol
!= htons(ETH_P_IP
)) &&
335 unlikely(skb
->protocol
!= htons(ETH_P_8021Q
)))
338 if (unlikely(!(skb
->dev
->features
& NETIF_F_IP_CSUM
)))
342 if (unlikely(ip_is_fragment(iph
)))
345 if (likely(iph
->protocol
== IPPROTO_TCP
)) {
346 l4hlen
= tcp_hdrlen(skb
) >> 2;
348 proto
= TSO_IPPROTO_TCP
;
349 if (ndev
->features
& NETIF_F_TSO
) {
350 hdr_len
= ethhdr
+ ip_hdrlen(skb
) + tcp_hdrlen(skb
);
351 mss
= skb_shinfo(skb
)->gso_size
;
353 if (skb_is_nonlinear(skb
)) {
354 len
= skb_headlen(skb
);
355 nr_frags
= skb_shinfo(skb
)->nr_frags
;
357 for (i
= 0; i
< 2 && i
< nr_frags
; i
++)
358 len
+= skb_shinfo(skb
)->frags
[i
].size
;
360 /* HW requires header must reside in 3 buffer */
361 if (unlikely(hdr_len
> len
)) {
362 if (skb_linearize(skb
))
367 if (!mss
|| ((skb
->len
- hdr_len
) <= mss
))
370 mss_index
= xgene_enet_setup_mss(ndev
, mss
);
371 if (unlikely(mss_index
< 0))
374 *hopinfo
|= SET_BIT(ET
) | SET_VAL(MSS
, mss_index
);
376 } else if (iph
->protocol
== IPPROTO_UDP
) {
377 l4hlen
= UDP_HDR_SIZE
;
381 l3hlen
= ip_hdrlen(skb
) >> 2;
382 *hopinfo
|= SET_VAL(TCPHDR
, l4hlen
) |
383 SET_VAL(IPHDR
, l3hlen
) |
384 SET_VAL(ETHHDR
, ethhdr
) |
385 SET_VAL(EC
, csum_enable
) |
388 SET_BIT(TYPE_ETH_WORK_MESSAGE
);
393 static u16
xgene_enet_encode_len(u16 len
)
395 return (len
== BUFLEN_16K
) ? 0 : len
;
398 static void xgene_set_addr_len(__le64
*desc
, u32 idx
, dma_addr_t addr
, u32 len
)
400 desc
[idx
^ 1] = cpu_to_le64(SET_VAL(DATAADDR
, addr
) |
401 SET_VAL(BUFDATALEN
, len
));
404 static __le64
*xgene_enet_get_exp_bufs(struct xgene_enet_desc_ring
*ring
)
408 exp_bufs
= &ring
->exp_bufs
[ring
->exp_buf_tail
* MAX_EXP_BUFFS
];
409 memset(exp_bufs
, 0, sizeof(__le64
) * MAX_EXP_BUFFS
);
410 ring
->exp_buf_tail
= (ring
->exp_buf_tail
+ 1) & ((ring
->slots
/ 2) - 1);
415 static dma_addr_t
*xgene_get_frag_dma_array(struct xgene_enet_desc_ring
*ring
)
417 return &ring
->cp_ring
->frag_dma_addr
[ring
->tail
* MAX_SKB_FRAGS
];
420 static int xgene_enet_setup_tx_desc(struct xgene_enet_desc_ring
*tx_ring
,
423 struct device
*dev
= ndev_to_dev(tx_ring
->ndev
);
424 struct xgene_enet_pdata
*pdata
= netdev_priv(tx_ring
->ndev
);
425 struct xgene_enet_raw_desc
*raw_desc
;
426 __le64
*exp_desc
= NULL
, *exp_bufs
= NULL
;
427 dma_addr_t dma_addr
, pbuf_addr
, *frag_dma_addr
;
429 u16 tail
= tx_ring
->tail
;
432 u8 ll
= 0, nv
= 0, idx
= 0;
434 u32 size
, offset
, ell_bytes
= 0;
435 u32 i
, fidx
, nr_frags
, count
= 1;
438 raw_desc
= &tx_ring
->raw_desc
[tail
];
439 tail
= (tail
+ 1) & (tx_ring
->slots
- 1);
440 memset(raw_desc
, 0, sizeof(struct xgene_enet_raw_desc
));
442 ret
= xgene_enet_work_msg(skb
, &hopinfo
);
446 raw_desc
->m3
= cpu_to_le64(SET_VAL(HENQNUM
, tx_ring
->dst_ring_num
) |
449 len
= skb_headlen(skb
);
450 hw_len
= xgene_enet_encode_len(len
);
452 dma_addr
= dma_map_single(dev
, skb
->data
, len
, DMA_TO_DEVICE
);
453 if (dma_mapping_error(dev
, dma_addr
)) {
454 netdev_err(tx_ring
->ndev
, "DMA mapping error\n");
458 /* Hardware expects descriptor in little endian format */
459 raw_desc
->m1
= cpu_to_le64(SET_VAL(DATAADDR
, dma_addr
) |
460 SET_VAL(BUFDATALEN
, hw_len
) |
463 if (!skb_is_nonlinear(skb
))
468 exp_desc
= (void *)&tx_ring
->raw_desc
[tail
];
469 tail
= (tail
+ 1) & (tx_ring
->slots
- 1);
470 memset(exp_desc
, 0, sizeof(struct xgene_enet_raw_desc
));
472 nr_frags
= skb_shinfo(skb
)->nr_frags
;
473 for (i
= nr_frags
; i
< 4 ; i
++)
474 exp_desc
[i
^ 1] = cpu_to_le64(LAST_BUFFER
);
476 frag_dma_addr
= xgene_get_frag_dma_array(tx_ring
);
478 for (i
= 0, fidx
= 0; split
|| (fidx
< nr_frags
); i
++) {
480 frag
= &skb_shinfo(skb
)->frags
[fidx
];
481 size
= skb_frag_size(frag
);
484 pbuf_addr
= skb_frag_dma_map(dev
, frag
, 0, size
,
486 if (dma_mapping_error(dev
, pbuf_addr
))
489 frag_dma_addr
[fidx
] = pbuf_addr
;
492 if (size
> BUFLEN_16K
)
496 if (size
> BUFLEN_16K
) {
504 dma_addr
= pbuf_addr
+ offset
;
505 hw_len
= xgene_enet_encode_len(len
);
511 xgene_set_addr_len(exp_desc
, i
, dma_addr
, hw_len
);
514 if (split
|| (fidx
!= nr_frags
)) {
515 exp_bufs
= xgene_enet_get_exp_bufs(tx_ring
);
516 xgene_set_addr_len(exp_bufs
, idx
, dma_addr
,
521 xgene_set_addr_len(exp_desc
, i
, dma_addr
,
526 xgene_set_addr_len(exp_bufs
, idx
, dma_addr
, hw_len
);
533 offset
+= BUFLEN_16K
;
539 dma_addr
= dma_map_single(dev
, exp_bufs
,
540 sizeof(u64
) * MAX_EXP_BUFFS
,
542 if (dma_mapping_error(dev
, dma_addr
)) {
543 dev_kfree_skb_any(skb
);
546 i
= ell_bytes
>> LL_BYTES_LSB_LEN
;
547 exp_desc
[2] = cpu_to_le64(SET_VAL(DATAADDR
, dma_addr
) |
548 SET_VAL(LL_BYTES_MSB
, i
) |
549 SET_VAL(LL_LEN
, idx
));
550 raw_desc
->m2
= cpu_to_le64(SET_VAL(LL_BYTES_LSB
, ell_bytes
));
554 raw_desc
->m0
= cpu_to_le64(SET_VAL(LL
, ll
) | SET_VAL(NV
, nv
) |
555 SET_VAL(USERINFO
, tx_ring
->tail
));
556 tx_ring
->cp_ring
->cp_skb
[tx_ring
->tail
] = skb
;
557 pdata
->tx_level
[tx_ring
->cp_ring
->index
] += count
;
558 tx_ring
->tail
= tail
;
563 static netdev_tx_t
xgene_enet_start_xmit(struct sk_buff
*skb
,
564 struct net_device
*ndev
)
566 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
567 struct xgene_enet_desc_ring
*tx_ring
;
568 int index
= skb
->queue_mapping
;
569 u32 tx_level
= pdata
->tx_level
[index
];
572 tx_ring
= pdata
->tx_ring
[index
];
573 if (tx_level
< pdata
->txc_level
[index
])
574 tx_level
+= ((typeof(pdata
->tx_level
[index
]))~0U);
576 if ((tx_level
- pdata
->txc_level
[index
]) > pdata
->tx_qcnt_hi
) {
577 netif_stop_subqueue(ndev
, index
);
578 return NETDEV_TX_BUSY
;
581 if (skb_padto(skb
, XGENE_MIN_ENET_FRAME_SIZE
))
584 count
= xgene_enet_setup_tx_desc(tx_ring
, skb
);
586 return NETDEV_TX_BUSY
;
589 dev_kfree_skb_any(skb
);
593 skb_tx_timestamp(skb
);
595 tx_ring
->tx_packets
++;
596 tx_ring
->tx_bytes
+= skb
->len
;
598 pdata
->ring_ops
->wr_cmd(tx_ring
, count
);
602 static void xgene_enet_rx_csum(struct sk_buff
*skb
)
604 struct net_device
*ndev
= skb
->dev
;
605 struct iphdr
*iph
= ip_hdr(skb
);
607 if (!(ndev
->features
& NETIF_F_RXCSUM
))
610 if (skb
->protocol
!= htons(ETH_P_IP
))
613 if (ip_is_fragment(iph
))
616 if (iph
->protocol
!= IPPROTO_TCP
&& iph
->protocol
!= IPPROTO_UDP
)
619 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
622 static void xgene_enet_free_pagepool(struct xgene_enet_desc_ring
*buf_pool
,
623 struct xgene_enet_raw_desc
*raw_desc
,
624 struct xgene_enet_raw_desc
*exp_desc
)
626 __le64
*desc
= (void *)exp_desc
;
634 if (!buf_pool
|| !raw_desc
|| !exp_desc
||
635 (!GET_VAL(NV
, le64_to_cpu(raw_desc
->m0
))))
638 dev
= ndev_to_dev(buf_pool
->ndev
);
639 slots
= buf_pool
->slots
- 1;
640 head
= buf_pool
->head
;
642 for (i
= 0; i
< 4; i
++) {
643 frag_size
= xgene_enet_get_data_len(le64_to_cpu(desc
[i
^ 1]));
647 dma_addr
= GET_VAL(DATAADDR
, le64_to_cpu(desc
[i
^ 1]));
648 dma_unmap_page(dev
, dma_addr
, PAGE_SIZE
, DMA_FROM_DEVICE
);
650 page
= buf_pool
->frag_page
[head
];
653 buf_pool
->frag_page
[head
] = NULL
;
654 head
= (head
+ 1) & slots
;
656 buf_pool
->head
= head
;
659 /* Errata 10GE_8 and ENET_11 - allow packet with length <=64B */
660 static bool xgene_enet_errata_10GE_8(struct sk_buff
*skb
, u32 len
, u8 status
)
662 if (status
== INGRESS_PKT_LEN
&& len
== ETHER_MIN_PACKET
) {
663 if (ntohs(eth_hdr(skb
)->h_proto
) < 46)
670 static int xgene_enet_rx_frame(struct xgene_enet_desc_ring
*rx_ring
,
671 struct xgene_enet_raw_desc
*raw_desc
,
672 struct xgene_enet_raw_desc
*exp_desc
)
674 struct xgene_enet_desc_ring
*buf_pool
, *page_pool
;
675 u32 datalen
, frag_size
, skb_index
;
676 struct xgene_enet_pdata
*pdata
;
677 struct net_device
*ndev
;
688 ndev
= rx_ring
->ndev
;
689 pdata
= netdev_priv(ndev
);
690 dev
= ndev_to_dev(rx_ring
->ndev
);
691 buf_pool
= rx_ring
->buf_pool
;
692 page_pool
= rx_ring
->page_pool
;
694 dma_unmap_single(dev
, GET_VAL(DATAADDR
, le64_to_cpu(raw_desc
->m1
)),
695 XGENE_ENET_STD_MTU
, DMA_FROM_DEVICE
);
696 skb_index
= GET_VAL(USERINFO
, le64_to_cpu(raw_desc
->m0
));
697 skb
= buf_pool
->rx_skb
[skb_index
];
698 buf_pool
->rx_skb
[skb_index
] = NULL
;
700 datalen
= xgene_enet_get_data_len(le64_to_cpu(raw_desc
->m1
));
701 skb_put(skb
, datalen
);
702 prefetch(skb
->data
- NET_IP_ALIGN
);
703 skb
->protocol
= eth_type_trans(skb
, ndev
);
705 /* checking for error */
706 status
= (GET_VAL(ELERR
, le64_to_cpu(raw_desc
->m0
)) << LERR_LEN
) |
707 GET_VAL(LERR
, le64_to_cpu(raw_desc
->m0
));
708 if (unlikely(status
)) {
709 if (!xgene_enet_errata_10GE_8(skb
, datalen
, status
)) {
710 dev_kfree_skb_any(skb
);
711 xgene_enet_free_pagepool(page_pool
, raw_desc
, exp_desc
);
712 xgene_enet_parse_error(rx_ring
, status
);
713 rx_ring
->rx_dropped
++;
720 nv
= GET_VAL(NV
, le64_to_cpu(raw_desc
->m0
));
722 /* strip off CRC as HW isn't doing this */
727 slots
= page_pool
->slots
- 1;
728 head
= page_pool
->head
;
729 desc
= (void *)exp_desc
;
731 for (i
= 0; i
< 4; i
++) {
732 frag_size
= xgene_enet_get_data_len(le64_to_cpu(desc
[i
^ 1]));
736 dma_addr
= GET_VAL(DATAADDR
, le64_to_cpu(desc
[i
^ 1]));
737 dma_unmap_page(dev
, dma_addr
, PAGE_SIZE
, DMA_FROM_DEVICE
);
739 page
= page_pool
->frag_page
[head
];
740 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, page
, 0,
741 frag_size
, PAGE_SIZE
);
743 datalen
+= frag_size
;
745 page_pool
->frag_page
[head
] = NULL
;
746 head
= (head
+ 1) & slots
;
749 page_pool
->head
= head
;
750 rx_ring
->npagepool
-= skb_shinfo(skb
)->nr_frags
;
753 skb_checksum_none_assert(skb
);
754 xgene_enet_rx_csum(skb
);
756 rx_ring
->rx_packets
++;
757 rx_ring
->rx_bytes
+= datalen
;
758 napi_gro_receive(&rx_ring
->napi
, skb
);
761 if (rx_ring
->npagepool
<= 0) {
762 ret
= xgene_enet_refill_pagepool(page_pool
, NUM_NXTBUFPOOL
);
763 rx_ring
->npagepool
= NUM_NXTBUFPOOL
;
768 if (--rx_ring
->nbufpool
== 0) {
769 ret
= xgene_enet_refill_bufpool(buf_pool
, NUM_BUFPOOL
);
770 rx_ring
->nbufpool
= NUM_BUFPOOL
;
776 static bool is_rx_desc(struct xgene_enet_raw_desc
*raw_desc
)
778 return GET_VAL(FPQNUM
, le64_to_cpu(raw_desc
->m0
)) ? true : false;
781 static int xgene_enet_process_ring(struct xgene_enet_desc_ring
*ring
,
784 struct net_device
*ndev
= ring
->ndev
;
785 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
786 struct xgene_enet_raw_desc
*raw_desc
, *exp_desc
;
787 u16 head
= ring
->head
;
788 u16 slots
= ring
->slots
- 1;
789 int ret
, desc_count
, count
= 0, processed
= 0;
793 raw_desc
= &ring
->raw_desc
[head
];
795 is_completion
= false;
797 if (unlikely(xgene_enet_is_desc_slot_empty(raw_desc
)))
800 /* read fpqnum field after dataaddr field */
802 if (GET_BIT(NV
, le64_to_cpu(raw_desc
->m0
))) {
803 head
= (head
+ 1) & slots
;
804 exp_desc
= &ring
->raw_desc
[head
];
806 if (unlikely(xgene_enet_is_desc_slot_empty(exp_desc
))) {
807 head
= (head
- 1) & slots
;
814 if (is_rx_desc(raw_desc
)) {
815 ret
= xgene_enet_rx_frame(ring
, raw_desc
, exp_desc
);
817 ret
= xgene_enet_tx_completion(ring
, raw_desc
);
818 is_completion
= true;
820 xgene_enet_mark_desc_slot_empty(raw_desc
);
822 xgene_enet_mark_desc_slot_empty(exp_desc
);
824 head
= (head
+ 1) & slots
;
829 pdata
->txc_level
[ring
->index
] += desc_count
;
836 pdata
->ring_ops
->wr_cmd(ring
, -count
);
839 if (__netif_subqueue_stopped(ndev
, ring
->index
))
840 netif_start_subqueue(ndev
, ring
->index
);
846 static int xgene_enet_napi(struct napi_struct
*napi
, const int budget
)
848 struct xgene_enet_desc_ring
*ring
;
851 ring
= container_of(napi
, struct xgene_enet_desc_ring
, napi
);
852 processed
= xgene_enet_process_ring(ring
, budget
);
854 if (processed
!= budget
) {
855 napi_complete_done(napi
, processed
);
856 enable_irq(ring
->irq
);
862 static void xgene_enet_timeout(struct net_device
*ndev
)
864 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
865 struct netdev_queue
*txq
;
868 pdata
->mac_ops
->reset(pdata
);
870 for (i
= 0; i
< pdata
->txq_cnt
; i
++) {
871 txq
= netdev_get_tx_queue(ndev
, i
);
872 txq
->trans_start
= jiffies
;
873 netif_tx_start_queue(txq
);
877 static void xgene_enet_set_irq_name(struct net_device
*ndev
)
879 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
880 struct xgene_enet_desc_ring
*ring
;
883 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
884 ring
= pdata
->rx_ring
[i
];
885 if (!pdata
->cq_cnt
) {
886 snprintf(ring
->irq_name
, IRQ_ID_SIZE
, "%s-rx-txc",
889 snprintf(ring
->irq_name
, IRQ_ID_SIZE
, "%s-rx-%d",
894 for (i
= 0; i
< pdata
->cq_cnt
; i
++) {
895 ring
= pdata
->tx_ring
[i
]->cp_ring
;
896 snprintf(ring
->irq_name
, IRQ_ID_SIZE
, "%s-txc-%d",
901 static int xgene_enet_register_irq(struct net_device
*ndev
)
903 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
904 struct device
*dev
= ndev_to_dev(ndev
);
905 struct xgene_enet_desc_ring
*ring
;
908 xgene_enet_set_irq_name(ndev
);
909 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
910 ring
= pdata
->rx_ring
[i
];
911 irq_set_status_flags(ring
->irq
, IRQ_DISABLE_UNLAZY
);
912 ret
= devm_request_irq(dev
, ring
->irq
, xgene_enet_rx_irq
,
913 0, ring
->irq_name
, ring
);
915 netdev_err(ndev
, "Failed to request irq %s\n",
920 for (i
= 0; i
< pdata
->cq_cnt
; i
++) {
921 ring
= pdata
->tx_ring
[i
]->cp_ring
;
922 irq_set_status_flags(ring
->irq
, IRQ_DISABLE_UNLAZY
);
923 ret
= devm_request_irq(dev
, ring
->irq
, xgene_enet_rx_irq
,
924 0, ring
->irq_name
, ring
);
926 netdev_err(ndev
, "Failed to request irq %s\n",
934 static void xgene_enet_free_irq(struct net_device
*ndev
)
936 struct xgene_enet_pdata
*pdata
;
937 struct xgene_enet_desc_ring
*ring
;
941 pdata
= netdev_priv(ndev
);
942 dev
= ndev_to_dev(ndev
);
944 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
945 ring
= pdata
->rx_ring
[i
];
946 irq_clear_status_flags(ring
->irq
, IRQ_DISABLE_UNLAZY
);
947 devm_free_irq(dev
, ring
->irq
, ring
);
950 for (i
= 0; i
< pdata
->cq_cnt
; i
++) {
951 ring
= pdata
->tx_ring
[i
]->cp_ring
;
952 irq_clear_status_flags(ring
->irq
, IRQ_DISABLE_UNLAZY
);
953 devm_free_irq(dev
, ring
->irq
, ring
);
957 static void xgene_enet_napi_enable(struct xgene_enet_pdata
*pdata
)
959 struct napi_struct
*napi
;
962 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
963 napi
= &pdata
->rx_ring
[i
]->napi
;
967 for (i
= 0; i
< pdata
->cq_cnt
; i
++) {
968 napi
= &pdata
->tx_ring
[i
]->cp_ring
->napi
;
973 static void xgene_enet_napi_disable(struct xgene_enet_pdata
*pdata
)
975 struct napi_struct
*napi
;
978 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
979 napi
= &pdata
->rx_ring
[i
]->napi
;
983 for (i
= 0; i
< pdata
->cq_cnt
; i
++) {
984 napi
= &pdata
->tx_ring
[i
]->cp_ring
->napi
;
989 static int xgene_enet_open(struct net_device
*ndev
)
991 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
992 const struct xgene_mac_ops
*mac_ops
= pdata
->mac_ops
;
995 ret
= netif_set_real_num_tx_queues(ndev
, pdata
->txq_cnt
);
999 ret
= netif_set_real_num_rx_queues(ndev
, pdata
->rxq_cnt
);
1003 xgene_enet_napi_enable(pdata
);
1004 ret
= xgene_enet_register_irq(ndev
);
1009 phy_start(ndev
->phydev
);
1011 schedule_delayed_work(&pdata
->link_work
, PHY_POLL_LINK_OFF
);
1012 netif_carrier_off(ndev
);
1015 mac_ops
->tx_enable(pdata
);
1016 mac_ops
->rx_enable(pdata
);
1017 netif_tx_start_all_queues(ndev
);
1022 static int xgene_enet_close(struct net_device
*ndev
)
1024 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
1025 const struct xgene_mac_ops
*mac_ops
= pdata
->mac_ops
;
1028 netif_tx_stop_all_queues(ndev
);
1029 mac_ops
->tx_disable(pdata
);
1030 mac_ops
->rx_disable(pdata
);
1033 phy_stop(ndev
->phydev
);
1035 cancel_delayed_work_sync(&pdata
->link_work
);
1037 xgene_enet_free_irq(ndev
);
1038 xgene_enet_napi_disable(pdata
);
1039 for (i
= 0; i
< pdata
->rxq_cnt
; i
++)
1040 xgene_enet_process_ring(pdata
->rx_ring
[i
], -1);
1044 static void xgene_enet_delete_ring(struct xgene_enet_desc_ring
*ring
)
1046 struct xgene_enet_pdata
*pdata
;
1049 pdata
= netdev_priv(ring
->ndev
);
1050 dev
= ndev_to_dev(ring
->ndev
);
1052 pdata
->ring_ops
->clear(ring
);
1053 dmam_free_coherent(dev
, ring
->size
, ring
->desc_addr
, ring
->dma
);
1056 static void xgene_enet_delete_desc_rings(struct xgene_enet_pdata
*pdata
)
1058 struct xgene_enet_desc_ring
*buf_pool
, *page_pool
;
1059 struct xgene_enet_desc_ring
*ring
;
1062 for (i
= 0; i
< pdata
->txq_cnt
; i
++) {
1063 ring
= pdata
->tx_ring
[i
];
1065 xgene_enet_delete_ring(ring
);
1066 pdata
->port_ops
->clear(pdata
, ring
);
1068 xgene_enet_delete_ring(ring
->cp_ring
);
1069 pdata
->tx_ring
[i
] = NULL
;
1074 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
1075 ring
= pdata
->rx_ring
[i
];
1077 page_pool
= ring
->page_pool
;
1079 xgene_enet_delete_pagepool(page_pool
);
1080 xgene_enet_delete_ring(page_pool
);
1081 pdata
->port_ops
->clear(pdata
, page_pool
);
1084 buf_pool
= ring
->buf_pool
;
1085 xgene_enet_delete_bufpool(buf_pool
);
1086 xgene_enet_delete_ring(buf_pool
);
1087 pdata
->port_ops
->clear(pdata
, buf_pool
);
1089 xgene_enet_delete_ring(ring
);
1090 pdata
->rx_ring
[i
] = NULL
;
1096 static int xgene_enet_get_ring_size(struct device
*dev
,
1097 enum xgene_enet_ring_cfgsize cfgsize
)
1102 case RING_CFGSIZE_512B
:
1105 case RING_CFGSIZE_2KB
:
1108 case RING_CFGSIZE_16KB
:
1111 case RING_CFGSIZE_64KB
:
1114 case RING_CFGSIZE_512KB
:
1118 dev_err(dev
, "Unsupported cfg ring size %d\n", cfgsize
);
1125 static void xgene_enet_free_desc_ring(struct xgene_enet_desc_ring
*ring
)
1127 struct xgene_enet_pdata
*pdata
;
1133 dev
= ndev_to_dev(ring
->ndev
);
1134 pdata
= netdev_priv(ring
->ndev
);
1136 if (ring
->desc_addr
) {
1137 pdata
->ring_ops
->clear(ring
);
1138 dmam_free_coherent(dev
, ring
->size
, ring
->desc_addr
, ring
->dma
);
1140 devm_kfree(dev
, ring
);
1143 static void xgene_enet_free_desc_rings(struct xgene_enet_pdata
*pdata
)
1145 struct xgene_enet_desc_ring
*page_pool
;
1146 struct device
*dev
= &pdata
->pdev
->dev
;
1147 struct xgene_enet_desc_ring
*ring
;
1151 for (i
= 0; i
< pdata
->txq_cnt
; i
++) {
1152 ring
= pdata
->tx_ring
[i
];
1154 if (ring
->cp_ring
&& ring
->cp_ring
->cp_skb
)
1155 devm_kfree(dev
, ring
->cp_ring
->cp_skb
);
1157 if (ring
->cp_ring
&& pdata
->cq_cnt
)
1158 xgene_enet_free_desc_ring(ring
->cp_ring
);
1160 xgene_enet_free_desc_ring(ring
);
1165 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
1166 ring
= pdata
->rx_ring
[i
];
1168 if (ring
->buf_pool
) {
1169 if (ring
->buf_pool
->rx_skb
)
1170 devm_kfree(dev
, ring
->buf_pool
->rx_skb
);
1172 xgene_enet_free_desc_ring(ring
->buf_pool
);
1175 page_pool
= ring
->page_pool
;
1177 p
= page_pool
->frag_page
;
1181 p
= page_pool
->frag_dma_addr
;
1186 xgene_enet_free_desc_ring(ring
);
1191 static bool is_irq_mbox_required(struct xgene_enet_pdata
*pdata
,
1192 struct xgene_enet_desc_ring
*ring
)
1194 if ((pdata
->enet_id
== XGENE_ENET2
) &&
1195 (xgene_enet_ring_owner(ring
->id
) == RING_OWNER_CPU
)) {
1202 static void __iomem
*xgene_enet_ring_cmd_base(struct xgene_enet_pdata
*pdata
,
1203 struct xgene_enet_desc_ring
*ring
)
1205 u8 num_ring_id_shift
= pdata
->ring_ops
->num_ring_id_shift
;
1207 return pdata
->ring_cmd_addr
+ (ring
->num
<< num_ring_id_shift
);
1210 static struct xgene_enet_desc_ring
*xgene_enet_create_desc_ring(
1211 struct net_device
*ndev
, u32 ring_num
,
1212 enum xgene_enet_ring_cfgsize cfgsize
, u32 ring_id
)
1214 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
1215 struct device
*dev
= ndev_to_dev(ndev
);
1216 struct xgene_enet_desc_ring
*ring
;
1217 void *irq_mbox_addr
;
1220 size
= xgene_enet_get_ring_size(dev
, cfgsize
);
1224 ring
= devm_kzalloc(dev
, sizeof(struct xgene_enet_desc_ring
),
1230 ring
->num
= ring_num
;
1231 ring
->cfgsize
= cfgsize
;
1234 ring
->desc_addr
= dmam_alloc_coherent(dev
, size
, &ring
->dma
,
1235 GFP_KERNEL
| __GFP_ZERO
);
1236 if (!ring
->desc_addr
) {
1237 devm_kfree(dev
, ring
);
1242 if (is_irq_mbox_required(pdata
, ring
)) {
1243 irq_mbox_addr
= dmam_alloc_coherent(dev
, INTR_MBOX_SIZE
,
1244 &ring
->irq_mbox_dma
,
1245 GFP_KERNEL
| __GFP_ZERO
);
1246 if (!irq_mbox_addr
) {
1247 dmam_free_coherent(dev
, size
, ring
->desc_addr
,
1249 devm_kfree(dev
, ring
);
1252 ring
->irq_mbox_addr
= irq_mbox_addr
;
1255 ring
->cmd_base
= xgene_enet_ring_cmd_base(pdata
, ring
);
1256 ring
->cmd
= ring
->cmd_base
+ INC_DEC_CMD_ADDR
;
1257 ring
= pdata
->ring_ops
->setup(ring
);
1258 netdev_dbg(ndev
, "ring info: num=%d size=%d id=%d slots=%d\n",
1259 ring
->num
, ring
->size
, ring
->id
, ring
->slots
);
1264 static u16
xgene_enet_get_ring_id(enum xgene_ring_owner owner
, u8 bufnum
)
1266 return (owner
<< 6) | (bufnum
& GENMASK(5, 0));
1269 static enum xgene_ring_owner
xgene_derive_ring_owner(struct xgene_enet_pdata
*p
)
1271 enum xgene_ring_owner owner
;
1273 if (p
->enet_id
== XGENE_ENET1
) {
1274 switch (p
->phy_mode
) {
1275 case PHY_INTERFACE_MODE_SGMII
:
1276 owner
= RING_OWNER_ETH0
;
1279 owner
= (!p
->port_id
) ? RING_OWNER_ETH0
:
1284 owner
= (!p
->port_id
) ? RING_OWNER_ETH0
: RING_OWNER_ETH1
;
1290 static u8
xgene_start_cpu_bufnum(struct xgene_enet_pdata
*pdata
)
1292 struct device
*dev
= &pdata
->pdev
->dev
;
1296 ret
= device_property_read_u32(dev
, "channel", &cpu_bufnum
);
1298 return (!ret
) ? cpu_bufnum
: pdata
->cpu_bufnum
;
1301 static int xgene_enet_create_desc_rings(struct net_device
*ndev
)
1303 struct xgene_enet_desc_ring
*rx_ring
, *tx_ring
, *cp_ring
;
1304 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
1305 struct xgene_enet_desc_ring
*page_pool
= NULL
;
1306 struct xgene_enet_desc_ring
*buf_pool
= NULL
;
1307 struct device
*dev
= ndev_to_dev(ndev
);
1308 u8 eth_bufnum
= pdata
->eth_bufnum
;
1309 u8 bp_bufnum
= pdata
->bp_bufnum
;
1310 u16 ring_num
= pdata
->ring_num
;
1311 enum xgene_ring_owner owner
;
1312 dma_addr_t dma_exp_bufs
;
1318 cpu_bufnum
= xgene_start_cpu_bufnum(pdata
);
1320 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
1321 /* allocate rx descriptor ring */
1322 owner
= xgene_derive_ring_owner(pdata
);
1323 ring_id
= xgene_enet_get_ring_id(RING_OWNER_CPU
, cpu_bufnum
++);
1324 rx_ring
= xgene_enet_create_desc_ring(ndev
, ring_num
++,
1332 /* allocate buffer pool for receiving packets */
1333 owner
= xgene_derive_ring_owner(pdata
);
1334 ring_id
= xgene_enet_get_ring_id(owner
, bp_bufnum
++);
1335 buf_pool
= xgene_enet_create_desc_ring(ndev
, ring_num
++,
1343 rx_ring
->nbufpool
= NUM_BUFPOOL
;
1344 rx_ring
->npagepool
= NUM_NXTBUFPOOL
;
1345 rx_ring
->irq
= pdata
->irqs
[i
];
1346 buf_pool
->rx_skb
= devm_kcalloc(dev
, buf_pool
->slots
,
1347 sizeof(struct sk_buff
*),
1349 if (!buf_pool
->rx_skb
) {
1354 buf_pool
->dst_ring_num
= xgene_enet_dst_ring_num(buf_pool
);
1355 rx_ring
->buf_pool
= buf_pool
;
1356 pdata
->rx_ring
[i
] = rx_ring
;
1358 if ((pdata
->enet_id
== XGENE_ENET1
&& pdata
->rxq_cnt
> 4) ||
1359 (pdata
->enet_id
== XGENE_ENET2
&& pdata
->rxq_cnt
> 16)) {
1363 /* allocate next buffer pool for jumbo packets */
1364 owner
= xgene_derive_ring_owner(pdata
);
1365 ring_id
= xgene_enet_get_ring_id(owner
, bp_bufnum
++);
1366 page_pool
= xgene_enet_create_desc_ring(ndev
, ring_num
++,
1374 slots
= page_pool
->slots
;
1375 page_pool
->frag_page
= devm_kcalloc(dev
, slots
,
1376 sizeof(struct page
*),
1378 if (!page_pool
->frag_page
) {
1383 page_pool
->frag_dma_addr
= devm_kcalloc(dev
, slots
,
1386 if (!page_pool
->frag_dma_addr
) {
1391 page_pool
->dst_ring_num
= xgene_enet_dst_ring_num(page_pool
);
1392 rx_ring
->page_pool
= page_pool
;
1395 for (i
= 0; i
< pdata
->txq_cnt
; i
++) {
1396 /* allocate tx descriptor ring */
1397 owner
= xgene_derive_ring_owner(pdata
);
1398 ring_id
= xgene_enet_get_ring_id(owner
, eth_bufnum
++);
1399 tx_ring
= xgene_enet_create_desc_ring(ndev
, ring_num
++,
1407 size
= (tx_ring
->slots
/ 2) * sizeof(__le64
) * MAX_EXP_BUFFS
;
1408 exp_bufs
= dmam_alloc_coherent(dev
, size
, &dma_exp_bufs
,
1409 GFP_KERNEL
| __GFP_ZERO
);
1414 tx_ring
->exp_bufs
= exp_bufs
;
1416 pdata
->tx_ring
[i
] = tx_ring
;
1418 if (!pdata
->cq_cnt
) {
1419 cp_ring
= pdata
->rx_ring
[i
];
1421 /* allocate tx completion descriptor ring */
1422 ring_id
= xgene_enet_get_ring_id(RING_OWNER_CPU
,
1424 cp_ring
= xgene_enet_create_desc_ring(ndev
, ring_num
++,
1432 cp_ring
->irq
= pdata
->irqs
[pdata
->rxq_cnt
+ i
];
1436 cp_ring
->cp_skb
= devm_kcalloc(dev
, tx_ring
->slots
,
1437 sizeof(struct sk_buff
*),
1439 if (!cp_ring
->cp_skb
) {
1444 size
= sizeof(dma_addr_t
) * MAX_SKB_FRAGS
;
1445 cp_ring
->frag_dma_addr
= devm_kcalloc(dev
, tx_ring
->slots
,
1447 if (!cp_ring
->frag_dma_addr
) {
1448 devm_kfree(dev
, cp_ring
->cp_skb
);
1453 tx_ring
->cp_ring
= cp_ring
;
1454 tx_ring
->dst_ring_num
= xgene_enet_dst_ring_num(cp_ring
);
1457 if (pdata
->ring_ops
->coalesce
)
1458 pdata
->ring_ops
->coalesce(pdata
->tx_ring
[0]);
1459 pdata
->tx_qcnt_hi
= pdata
->tx_ring
[0]->slots
- 128;
1464 xgene_enet_free_desc_rings(pdata
);
1468 static void xgene_enet_get_stats64(
1469 struct net_device
*ndev
,
1470 struct rtnl_link_stats64
*stats
)
1472 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
1473 struct xgene_enet_desc_ring
*ring
;
1476 for (i
= 0; i
< pdata
->txq_cnt
; i
++) {
1477 ring
= pdata
->tx_ring
[i
];
1479 stats
->tx_packets
+= ring
->tx_packets
;
1480 stats
->tx_bytes
+= ring
->tx_bytes
;
1481 stats
->tx_dropped
+= ring
->tx_dropped
;
1482 stats
->tx_errors
+= ring
->tx_errors
;
1486 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
1487 ring
= pdata
->rx_ring
[i
];
1489 stats
->rx_packets
+= ring
->rx_packets
;
1490 stats
->rx_bytes
+= ring
->rx_bytes
;
1491 stats
->rx_dropped
+= ring
->rx_dropped
;
1492 stats
->rx_errors
+= ring
->rx_errors
+
1493 ring
->rx_length_errors
+
1494 ring
->rx_crc_errors
+
1495 ring
->rx_frame_errors
+
1496 ring
->rx_fifo_errors
;
1497 stats
->rx_length_errors
+= ring
->rx_length_errors
;
1498 stats
->rx_crc_errors
+= ring
->rx_crc_errors
;
1499 stats
->rx_frame_errors
+= ring
->rx_frame_errors
;
1500 stats
->rx_fifo_errors
+= ring
->rx_fifo_errors
;
1505 static int xgene_enet_set_mac_address(struct net_device
*ndev
, void *addr
)
1507 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
1510 ret
= eth_mac_addr(ndev
, addr
);
1513 pdata
->mac_ops
->set_mac_addr(pdata
);
1518 static int xgene_change_mtu(struct net_device
*ndev
, int new_mtu
)
1520 struct xgene_enet_pdata
*pdata
= netdev_priv(ndev
);
1523 if (!netif_running(ndev
))
1526 frame_size
= (new_mtu
> ETH_DATA_LEN
) ? (new_mtu
+ 18) : 0x600;
1528 xgene_enet_close(ndev
);
1529 ndev
->mtu
= new_mtu
;
1530 pdata
->mac_ops
->set_framesize(pdata
, frame_size
);
1531 xgene_enet_open(ndev
);
1536 static const struct net_device_ops xgene_ndev_ops
= {
1537 .ndo_open
= xgene_enet_open
,
1538 .ndo_stop
= xgene_enet_close
,
1539 .ndo_start_xmit
= xgene_enet_start_xmit
,
1540 .ndo_tx_timeout
= xgene_enet_timeout
,
1541 .ndo_get_stats64
= xgene_enet_get_stats64
,
1542 .ndo_change_mtu
= xgene_change_mtu
,
1543 .ndo_set_mac_address
= xgene_enet_set_mac_address
,
1547 static void xgene_get_port_id_acpi(struct device
*dev
,
1548 struct xgene_enet_pdata
*pdata
)
1553 status
= acpi_evaluate_integer(ACPI_HANDLE(dev
), "_SUN", NULL
, &temp
);
1554 if (ACPI_FAILURE(status
)) {
1557 pdata
->port_id
= temp
;
1564 static void xgene_get_port_id_dt(struct device
*dev
, struct xgene_enet_pdata
*pdata
)
1568 of_property_read_u32(dev
->of_node
, "port-id", &id
);
1570 pdata
->port_id
= id
& BIT(0);
1575 static int xgene_get_tx_delay(struct xgene_enet_pdata
*pdata
)
1577 struct device
*dev
= &pdata
->pdev
->dev
;
1580 ret
= of_property_read_u32(dev
->of_node
, "tx-delay", &delay
);
1582 pdata
->tx_delay
= 4;
1586 if (delay
< 0 || delay
> 7) {
1587 dev_err(dev
, "Invalid tx-delay specified\n");
1591 pdata
->tx_delay
= delay
;
1596 static int xgene_get_rx_delay(struct xgene_enet_pdata
*pdata
)
1598 struct device
*dev
= &pdata
->pdev
->dev
;
1601 ret
= of_property_read_u32(dev
->of_node
, "rx-delay", &delay
);
1603 pdata
->rx_delay
= 2;
1607 if (delay
< 0 || delay
> 7) {
1608 dev_err(dev
, "Invalid rx-delay specified\n");
1612 pdata
->rx_delay
= delay
;
1617 static int xgene_enet_get_irqs(struct xgene_enet_pdata
*pdata
)
1619 struct platform_device
*pdev
= pdata
->pdev
;
1620 struct device
*dev
= &pdev
->dev
;
1621 int i
, ret
, max_irqs
;
1623 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_RGMII
)
1625 else if (pdata
->phy_mode
== PHY_INTERFACE_MODE_SGMII
)
1628 max_irqs
= XGENE_MAX_ENET_IRQ
;
1630 for (i
= 0; i
< max_irqs
; i
++) {
1631 ret
= platform_get_irq(pdev
, i
);
1633 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
) {
1635 pdata
->rxq_cnt
= max_irqs
/ 2;
1636 pdata
->txq_cnt
= max_irqs
/ 2;
1637 pdata
->cq_cnt
= max_irqs
/ 2;
1640 dev_err(dev
, "Unable to get ENET IRQ\n");
1641 ret
= ret
? : -ENXIO
;
1644 pdata
->irqs
[i
] = ret
;
1650 static int xgene_enet_check_phy_handle(struct xgene_enet_pdata
*pdata
)
1654 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
)
1657 if (!IS_ENABLED(CONFIG_MDIO_XGENE
))
1660 ret
= xgene_enet_phy_connect(pdata
->ndev
);
1662 pdata
->mdio_driver
= true;
1667 static void xgene_enet_gpiod_get(struct xgene_enet_pdata
*pdata
)
1669 struct device
*dev
= &pdata
->pdev
->dev
;
1671 pdata
->sfp_gpio_en
= false;
1672 if (pdata
->phy_mode
!= PHY_INTERFACE_MODE_XGMII
||
1673 (!device_property_present(dev
, "sfp-gpios") &&
1674 !device_property_present(dev
, "rxlos-gpios")))
1677 pdata
->sfp_gpio_en
= true;
1678 pdata
->sfp_rdy
= gpiod_get(dev
, "rxlos", GPIOD_IN
);
1679 if (IS_ERR(pdata
->sfp_rdy
))
1680 pdata
->sfp_rdy
= gpiod_get(dev
, "sfp", GPIOD_IN
);
1683 static int xgene_enet_get_resources(struct xgene_enet_pdata
*pdata
)
1685 struct platform_device
*pdev
;
1686 struct net_device
*ndev
;
1688 struct resource
*res
;
1689 void __iomem
*base_addr
;
1697 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, RES_ENET_CSR
);
1699 dev_err(dev
, "Resource enet_csr not defined\n");
1702 pdata
->base_addr
= devm_ioremap(dev
, res
->start
, resource_size(res
));
1703 if (!pdata
->base_addr
) {
1704 dev_err(dev
, "Unable to retrieve ENET Port CSR region\n");
1708 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, RES_RING_CSR
);
1710 dev_err(dev
, "Resource ring_csr not defined\n");
1713 pdata
->ring_csr_addr
= devm_ioremap(dev
, res
->start
,
1714 resource_size(res
));
1715 if (!pdata
->ring_csr_addr
) {
1716 dev_err(dev
, "Unable to retrieve ENET Ring CSR region\n");
1720 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, RES_RING_CMD
);
1722 dev_err(dev
, "Resource ring_cmd not defined\n");
1725 pdata
->ring_cmd_addr
= devm_ioremap(dev
, res
->start
,
1726 resource_size(res
));
1727 if (!pdata
->ring_cmd_addr
) {
1728 dev_err(dev
, "Unable to retrieve ENET Ring command region\n");
1733 xgene_get_port_id_dt(dev
, pdata
);
1736 xgene_get_port_id_acpi(dev
, pdata
);
1739 if (!device_get_mac_address(dev
, ndev
->dev_addr
, ETH_ALEN
))
1740 eth_hw_addr_random(ndev
);
1742 memcpy(ndev
->perm_addr
, ndev
->dev_addr
, ndev
->addr_len
);
1744 pdata
->phy_mode
= device_get_phy_mode(dev
);
1745 if (pdata
->phy_mode
< 0) {
1746 dev_err(dev
, "Unable to get phy-connection-type\n");
1747 return pdata
->phy_mode
;
1749 if (pdata
->phy_mode
!= PHY_INTERFACE_MODE_RGMII
&&
1750 pdata
->phy_mode
!= PHY_INTERFACE_MODE_SGMII
&&
1751 pdata
->phy_mode
!= PHY_INTERFACE_MODE_XGMII
) {
1752 dev_err(dev
, "Incorrect phy-connection-type specified\n");
1756 ret
= xgene_get_tx_delay(pdata
);
1760 ret
= xgene_get_rx_delay(pdata
);
1764 ret
= xgene_enet_get_irqs(pdata
);
1768 ret
= xgene_enet_check_phy_handle(pdata
);
1772 xgene_enet_gpiod_get(pdata
);
1774 pdata
->clk
= devm_clk_get(&pdev
->dev
, NULL
);
1775 if (IS_ERR(pdata
->clk
)) {
1776 /* Abort if the clock is defined but couldn't be retrived.
1777 * Always abort if the clock is missing on DT system as
1778 * the driver can't cope with this case.
1780 if (PTR_ERR(pdata
->clk
) != -ENOENT
|| dev
->of_node
)
1781 return PTR_ERR(pdata
->clk
);
1782 /* Firmware may have set up the clock already. */
1783 dev_info(dev
, "clocks have been setup already\n");
1786 if (pdata
->phy_mode
!= PHY_INTERFACE_MODE_XGMII
)
1787 base_addr
= pdata
->base_addr
- (pdata
->port_id
* MAC_OFFSET
);
1789 base_addr
= pdata
->base_addr
;
1790 pdata
->eth_csr_addr
= base_addr
+ BLOCK_ETH_CSR_OFFSET
;
1791 pdata
->cle
.base
= base_addr
+ BLOCK_ETH_CLE_CSR_OFFSET
;
1792 pdata
->eth_ring_if_addr
= base_addr
+ BLOCK_ETH_RING_IF_OFFSET
;
1793 pdata
->eth_diag_csr_addr
= base_addr
+ BLOCK_ETH_DIAG_CSR_OFFSET
;
1794 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_RGMII
||
1795 pdata
->phy_mode
== PHY_INTERFACE_MODE_SGMII
) {
1796 pdata
->mcx_mac_addr
= pdata
->base_addr
+ BLOCK_ETH_MAC_OFFSET
;
1797 pdata
->mcx_stats_addr
=
1798 pdata
->base_addr
+ BLOCK_ETH_STATS_OFFSET
;
1799 offset
= (pdata
->enet_id
== XGENE_ENET1
) ?
1800 BLOCK_ETH_MAC_CSR_OFFSET
:
1801 X2_BLOCK_ETH_MAC_CSR_OFFSET
;
1802 pdata
->mcx_mac_csr_addr
= base_addr
+ offset
;
1804 pdata
->mcx_mac_addr
= base_addr
+ BLOCK_AXG_MAC_OFFSET
;
1805 pdata
->mcx_stats_addr
= base_addr
+ BLOCK_AXG_STATS_OFFSET
;
1806 pdata
->mcx_mac_csr_addr
= base_addr
+ BLOCK_AXG_MAC_CSR_OFFSET
;
1807 pdata
->pcs_addr
= base_addr
+ BLOCK_PCS_OFFSET
;
1809 pdata
->rx_buff_cnt
= NUM_PKT_BUF
;
1814 static int xgene_enet_init_hw(struct xgene_enet_pdata
*pdata
)
1816 struct xgene_enet_cle
*enet_cle
= &pdata
->cle
;
1817 struct xgene_enet_desc_ring
*page_pool
;
1818 struct net_device
*ndev
= pdata
->ndev
;
1819 struct xgene_enet_desc_ring
*buf_pool
;
1820 u16 dst_ring_num
, ring_id
;
1824 ret
= pdata
->port_ops
->reset(pdata
);
1828 ret
= xgene_enet_create_desc_rings(ndev
);
1830 netdev_err(ndev
, "Error in ring configuration\n");
1834 /* setup buffer pool */
1835 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
1836 buf_pool
= pdata
->rx_ring
[i
]->buf_pool
;
1837 xgene_enet_init_bufpool(buf_pool
);
1838 page_pool
= pdata
->rx_ring
[i
]->page_pool
;
1839 xgene_enet_init_bufpool(page_pool
);
1841 count
= pdata
->rx_buff_cnt
;
1842 ret
= xgene_enet_refill_bufpool(buf_pool
, count
);
1846 ret
= xgene_enet_refill_pagepool(page_pool
, count
);
1852 dst_ring_num
= xgene_enet_dst_ring_num(pdata
->rx_ring
[0]);
1853 buf_pool
= pdata
->rx_ring
[0]->buf_pool
;
1854 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
) {
1855 /* Initialize and Enable PreClassifier Tree */
1856 enet_cle
->max_nodes
= 512;
1857 enet_cle
->max_dbptrs
= 1024;
1858 enet_cle
->parsers
= 3;
1859 enet_cle
->active_parser
= PARSER_ALL
;
1860 enet_cle
->ptree
.start_node
= 0;
1861 enet_cle
->ptree
.start_dbptr
= 0;
1862 enet_cle
->jump_bytes
= 8;
1863 ret
= pdata
->cle_ops
->cle_init(pdata
);
1865 netdev_err(ndev
, "Preclass Tree init error\n");
1870 dst_ring_num
= xgene_enet_dst_ring_num(pdata
->rx_ring
[0]);
1871 buf_pool
= pdata
->rx_ring
[0]->buf_pool
;
1872 page_pool
= pdata
->rx_ring
[0]->page_pool
;
1873 ring_id
= (page_pool
) ? page_pool
->id
: 0;
1874 pdata
->port_ops
->cle_bypass(pdata
, dst_ring_num
,
1875 buf_pool
->id
, ring_id
);
1878 ndev
->max_mtu
= XGENE_ENET_MAX_MTU
;
1879 pdata
->phy_speed
= SPEED_UNKNOWN
;
1880 pdata
->mac_ops
->init(pdata
);
1885 xgene_enet_delete_desc_rings(pdata
);
1889 static void xgene_enet_setup_ops(struct xgene_enet_pdata
*pdata
)
1891 switch (pdata
->phy_mode
) {
1892 case PHY_INTERFACE_MODE_RGMII
:
1893 pdata
->mac_ops
= &xgene_gmac_ops
;
1894 pdata
->port_ops
= &xgene_gport_ops
;
1900 case PHY_INTERFACE_MODE_SGMII
:
1901 pdata
->mac_ops
= &xgene_sgmac_ops
;
1902 pdata
->port_ops
= &xgene_sgport_ops
;
1909 pdata
->mac_ops
= &xgene_xgmac_ops
;
1910 pdata
->port_ops
= &xgene_xgport_ops
;
1911 pdata
->cle_ops
= &xgene_cle3in_ops
;
1913 if (!pdata
->rxq_cnt
) {
1914 pdata
->rxq_cnt
= XGENE_NUM_RX_RING
;
1915 pdata
->txq_cnt
= XGENE_NUM_TX_RING
;
1916 pdata
->cq_cnt
= XGENE_NUM_TXC_RING
;
1921 if (pdata
->enet_id
== XGENE_ENET1
) {
1922 switch (pdata
->port_id
) {
1924 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
) {
1925 pdata
->cpu_bufnum
= X2_START_CPU_BUFNUM_0
;
1926 pdata
->eth_bufnum
= X2_START_ETH_BUFNUM_0
;
1927 pdata
->bp_bufnum
= X2_START_BP_BUFNUM_0
;
1928 pdata
->ring_num
= START_RING_NUM_0
;
1930 pdata
->cpu_bufnum
= START_CPU_BUFNUM_0
;
1931 pdata
->eth_bufnum
= START_ETH_BUFNUM_0
;
1932 pdata
->bp_bufnum
= START_BP_BUFNUM_0
;
1933 pdata
->ring_num
= START_RING_NUM_0
;
1937 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
) {
1938 pdata
->cpu_bufnum
= XG_START_CPU_BUFNUM_1
;
1939 pdata
->eth_bufnum
= XG_START_ETH_BUFNUM_1
;
1940 pdata
->bp_bufnum
= XG_START_BP_BUFNUM_1
;
1941 pdata
->ring_num
= XG_START_RING_NUM_1
;
1943 pdata
->cpu_bufnum
= START_CPU_BUFNUM_1
;
1944 pdata
->eth_bufnum
= START_ETH_BUFNUM_1
;
1945 pdata
->bp_bufnum
= START_BP_BUFNUM_1
;
1946 pdata
->ring_num
= START_RING_NUM_1
;
1952 pdata
->ring_ops
= &xgene_ring1_ops
;
1954 switch (pdata
->port_id
) {
1956 pdata
->cpu_bufnum
= X2_START_CPU_BUFNUM_0
;
1957 pdata
->eth_bufnum
= X2_START_ETH_BUFNUM_0
;
1958 pdata
->bp_bufnum
= X2_START_BP_BUFNUM_0
;
1959 pdata
->ring_num
= X2_START_RING_NUM_0
;
1962 pdata
->cpu_bufnum
= X2_START_CPU_BUFNUM_1
;
1963 pdata
->eth_bufnum
= X2_START_ETH_BUFNUM_1
;
1964 pdata
->bp_bufnum
= X2_START_BP_BUFNUM_1
;
1965 pdata
->ring_num
= X2_START_RING_NUM_1
;
1971 pdata
->ring_ops
= &xgene_ring2_ops
;
1975 static void xgene_enet_napi_add(struct xgene_enet_pdata
*pdata
)
1977 struct napi_struct
*napi
;
1980 for (i
= 0; i
< pdata
->rxq_cnt
; i
++) {
1981 napi
= &pdata
->rx_ring
[i
]->napi
;
1982 netif_napi_add(pdata
->ndev
, napi
, xgene_enet_napi
,
1986 for (i
= 0; i
< pdata
->cq_cnt
; i
++) {
1987 napi
= &pdata
->tx_ring
[i
]->cp_ring
->napi
;
1988 netif_napi_add(pdata
->ndev
, napi
, xgene_enet_napi
,
1994 static const struct acpi_device_id xgene_enet_acpi_match
[] = {
1995 { "APMC0D05", XGENE_ENET1
},
1996 { "APMC0D30", XGENE_ENET1
},
1997 { "APMC0D31", XGENE_ENET1
},
1998 { "APMC0D3F", XGENE_ENET1
},
1999 { "APMC0D26", XGENE_ENET2
},
2000 { "APMC0D25", XGENE_ENET2
},
2003 MODULE_DEVICE_TABLE(acpi
, xgene_enet_acpi_match
);
2006 static const struct of_device_id xgene_enet_of_match
[] = {
2007 {.compatible
= "apm,xgene-enet", .data
= (void *)XGENE_ENET1
},
2008 {.compatible
= "apm,xgene1-sgenet", .data
= (void *)XGENE_ENET1
},
2009 {.compatible
= "apm,xgene1-xgenet", .data
= (void *)XGENE_ENET1
},
2010 {.compatible
= "apm,xgene2-sgenet", .data
= (void *)XGENE_ENET2
},
2011 {.compatible
= "apm,xgene2-xgenet", .data
= (void *)XGENE_ENET2
},
2015 MODULE_DEVICE_TABLE(of
, xgene_enet_of_match
);
2017 static int xgene_enet_probe(struct platform_device
*pdev
)
2019 struct net_device
*ndev
;
2020 struct xgene_enet_pdata
*pdata
;
2021 struct device
*dev
= &pdev
->dev
;
2022 void (*link_state
)(struct work_struct
*);
2023 const struct of_device_id
*of_id
;
2026 ndev
= alloc_etherdev_mqs(sizeof(struct xgene_enet_pdata
),
2027 XGENE_NUM_RX_RING
, XGENE_NUM_TX_RING
);
2031 pdata
= netdev_priv(ndev
);
2035 SET_NETDEV_DEV(ndev
, dev
);
2036 platform_set_drvdata(pdev
, pdata
);
2037 ndev
->netdev_ops
= &xgene_ndev_ops
;
2038 xgene_enet_set_ethtool_ops(ndev
);
2039 ndev
->features
|= NETIF_F_IP_CSUM
|
2044 of_id
= of_match_device(xgene_enet_of_match
, &pdev
->dev
);
2046 pdata
->enet_id
= (enum xgene_enet_id
)of_id
->data
;
2050 const struct acpi_device_id
*acpi_id
;
2052 acpi_id
= acpi_match_device(xgene_enet_acpi_match
, &pdev
->dev
);
2054 pdata
->enet_id
= (enum xgene_enet_id
) acpi_id
->driver_data
;
2057 if (!pdata
->enet_id
) {
2062 ret
= xgene_enet_get_resources(pdata
);
2066 xgene_enet_setup_ops(pdata
);
2067 spin_lock_init(&pdata
->mac_lock
);
2069 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
) {
2070 ndev
->features
|= NETIF_F_TSO
| NETIF_F_RXCSUM
;
2071 spin_lock_init(&pdata
->mss_lock
);
2073 ndev
->hw_features
= ndev
->features
;
2075 ret
= dma_coerce_mask_and_coherent(dev
, DMA_BIT_MASK(64));
2077 netdev_err(ndev
, "No usable DMA configuration\n");
2081 ret
= xgene_enet_init_hw(pdata
);
2085 link_state
= pdata
->mac_ops
->link_state
;
2086 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_XGMII
) {
2087 INIT_DELAYED_WORK(&pdata
->link_work
, link_state
);
2088 } else if (!pdata
->mdio_driver
) {
2089 if (pdata
->phy_mode
== PHY_INTERFACE_MODE_RGMII
)
2090 ret
= xgene_enet_mdio_config(pdata
);
2092 INIT_DELAYED_WORK(&pdata
->link_work
, link_state
);
2098 spin_lock_init(&pdata
->stats_lock
);
2099 ret
= xgene_extd_stats_init(pdata
);
2103 xgene_enet_napi_add(pdata
);
2104 ret
= register_netdev(ndev
);
2106 netdev_err(ndev
, "Failed to register netdev\n");
2114 * If necessary, free_netdev() will call netif_napi_del() and undo
2115 * the effects of xgene_enet_napi_add()'s calls to netif_napi_add().
2118 if (pdata
->mdio_driver
)
2119 xgene_enet_phy_disconnect(pdata
);
2120 else if (pdata
->phy_mode
== PHY_INTERFACE_MODE_RGMII
)
2121 xgene_enet_mdio_remove(pdata
);
2123 xgene_enet_delete_desc_rings(pdata
);
2129 static int xgene_enet_remove(struct platform_device
*pdev
)
2131 struct xgene_enet_pdata
*pdata
;
2132 struct net_device
*ndev
;
2134 pdata
= platform_get_drvdata(pdev
);
2138 if (netif_running(ndev
))
2142 if (pdata
->mdio_driver
)
2143 xgene_enet_phy_disconnect(pdata
);
2144 else if (pdata
->phy_mode
== PHY_INTERFACE_MODE_RGMII
)
2145 xgene_enet_mdio_remove(pdata
);
2147 unregister_netdev(ndev
);
2148 pdata
->port_ops
->shutdown(pdata
);
2149 xgene_enet_delete_desc_rings(pdata
);
2155 static void xgene_enet_shutdown(struct platform_device
*pdev
)
2157 struct xgene_enet_pdata
*pdata
;
2159 pdata
= platform_get_drvdata(pdev
);
2166 xgene_enet_remove(pdev
);
2169 static struct platform_driver xgene_enet_driver
= {
2171 .name
= "xgene-enet",
2172 .of_match_table
= of_match_ptr(xgene_enet_of_match
),
2173 .acpi_match_table
= ACPI_PTR(xgene_enet_acpi_match
),
2175 .probe
= xgene_enet_probe
,
2176 .remove
= xgene_enet_remove
,
2177 .shutdown
= xgene_enet_shutdown
,
2180 module_platform_driver(xgene_enet_driver
);
2182 MODULE_DESCRIPTION("APM X-Gene SoC Ethernet driver");
2183 MODULE_VERSION(XGENE_DRV_VERSION
);
2184 MODULE_AUTHOR("Iyappan Subramanian <isubramanian@apm.com>");
2185 MODULE_AUTHOR("Keyur Chudgar <kchudgar@apm.com>");
2186 MODULE_LICENSE("GPL");