1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (c) 2014-2015 Hisilicon Limited.
7 #include <linux/cpumask.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/interrupt.h>
13 #include <linux/ipv6.h>
14 #include <linux/irq.h>
15 #include <linux/module.h>
16 #include <linux/phy.h>
17 #include <linux/platform_device.h>
18 #include <linux/skbuff.h>
22 #include "hns_dsaf_mac.h"
24 #define NIC_MAX_Q_PER_VF 16
25 #define HNS_NIC_TX_TIMEOUT (5 * HZ)
27 #define SERVICE_TIMER_HZ (1 * HZ)
29 #define RCB_IRQ_NOT_INITED 0
30 #define RCB_IRQ_INITED 1
31 #define HNS_BUFFER_SIZE_2048 2048
33 #define BD_MAX_SEND_SIZE 8191
34 #define SKB_TMP_LEN(SKB) \
35 (((SKB)->transport_header - (SKB)->mac_header) + tcp_hdrlen(SKB))
37 static void fill_v2_desc_hw(struct hnae_ring
*ring
, void *priv
, int size
,
38 int send_sz
, dma_addr_t dma
, int frag_end
,
39 int buf_num
, enum hns_desc_type type
, int mtu
)
41 struct hnae_desc
*desc
= &ring
->desc
[ring
->next_to_use
];
42 struct hnae_desc_cb
*desc_cb
= &ring
->desc_cb
[ring
->next_to_use
];
44 struct ipv6hdr
*ipv6hdr
;
56 desc_cb
->length
= size
;
60 desc
->addr
= cpu_to_le64(dma
);
61 desc
->tx
.send_size
= cpu_to_le16((u16
)send_sz
);
63 /* config bd buffer end */
64 hnae_set_bit(rrcfv
, HNSV2_TXD_VLD_B
, 1);
65 hnae_set_field(bn_pid
, HNSV2_TXD_BUFNUM_M
, 0, buf_num
- 1);
67 /* fill port_id in the tx bd for sending management pkts */
68 hnae_set_field(bn_pid
, HNSV2_TXD_PORTID_M
,
69 HNSV2_TXD_PORTID_S
, ring
->q
->handle
->dport_id
);
71 if (type
== DESC_TYPE_SKB
) {
72 skb
= (struct sk_buff
*)priv
;
74 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
75 skb_reset_mac_len(skb
);
76 protocol
= skb
->protocol
;
79 if (protocol
== htons(ETH_P_8021Q
)) {
80 ip_offset
+= VLAN_HLEN
;
81 protocol
= vlan_get_protocol(skb
);
82 skb
->protocol
= protocol
;
85 if (skb
->protocol
== htons(ETH_P_IP
)) {
87 hnae_set_bit(rrcfv
, HNSV2_TXD_L3CS_B
, 1);
88 hnae_set_bit(rrcfv
, HNSV2_TXD_L4CS_B
, 1);
90 /* check for tcp/udp header */
91 if (iphdr
->protocol
== IPPROTO_TCP
&&
95 l4_len
= tcp_hdrlen(skb
);
96 mss
= skb_shinfo(skb
)->gso_size
;
97 paylen
= skb
->len
- SKB_TMP_LEN(skb
);
99 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
100 hnae_set_bit(tvsvsn
, HNSV2_TXD_IPV6_B
, 1);
101 ipv6hdr
= ipv6_hdr(skb
);
102 hnae_set_bit(rrcfv
, HNSV2_TXD_L4CS_B
, 1);
104 /* check for tcp/udp header */
105 if (ipv6hdr
->nexthdr
== IPPROTO_TCP
&&
106 skb_is_gso(skb
) && skb_is_gso_v6(skb
)) {
109 l4_len
= tcp_hdrlen(skb
);
110 mss
= skb_shinfo(skb
)->gso_size
;
111 paylen
= skb
->len
- SKB_TMP_LEN(skb
);
114 desc
->tx
.ip_offset
= ip_offset
;
115 desc
->tx
.tse_vlan_snap_v6_sctp_nth
= tvsvsn
;
116 desc
->tx
.mss
= cpu_to_le16(mss
);
117 desc
->tx
.l4_len
= l4_len
;
118 desc
->tx
.paylen
= cpu_to_le16(paylen
);
122 hnae_set_bit(rrcfv
, HNSV2_TXD_FE_B
, frag_end
);
124 desc
->tx
.bn_pid
= bn_pid
;
125 desc
->tx
.ra_ri_cs_fe_vld
= rrcfv
;
127 ring_ptr_move_fw(ring
, next_to_use
);
130 static void fill_v2_desc(struct hnae_ring
*ring
, void *priv
,
131 int size
, dma_addr_t dma
, int frag_end
,
132 int buf_num
, enum hns_desc_type type
, int mtu
)
134 fill_v2_desc_hw(ring
, priv
, size
, size
, dma
, frag_end
,
138 static const struct acpi_device_id hns_enet_acpi_match
[] = {
143 MODULE_DEVICE_TABLE(acpi
, hns_enet_acpi_match
);
145 static void fill_desc(struct hnae_ring
*ring
, void *priv
,
146 int size
, dma_addr_t dma
, int frag_end
,
147 int buf_num
, enum hns_desc_type type
, int mtu
)
149 struct hnae_desc
*desc
= &ring
->desc
[ring
->next_to_use
];
150 struct hnae_desc_cb
*desc_cb
= &ring
->desc_cb
[ring
->next_to_use
];
154 u32 asid_bufnum_pid
= 0;
155 u32 flag_ipoffset
= 0;
157 desc_cb
->priv
= priv
;
158 desc_cb
->length
= size
;
160 desc_cb
->type
= type
;
162 desc
->addr
= cpu_to_le64(dma
);
163 desc
->tx
.send_size
= cpu_to_le16((u16
)size
);
165 /*config bd buffer end */
166 flag_ipoffset
|= 1 << HNS_TXD_VLD_B
;
168 asid_bufnum_pid
|= buf_num
<< HNS_TXD_BUFNUM_S
;
170 if (type
== DESC_TYPE_SKB
) {
171 skb
= (struct sk_buff
*)priv
;
173 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
174 protocol
= skb
->protocol
;
175 ip_offset
= ETH_HLEN
;
177 /*if it is a SW VLAN check the next protocol*/
178 if (protocol
== htons(ETH_P_8021Q
)) {
179 ip_offset
+= VLAN_HLEN
;
180 protocol
= vlan_get_protocol(skb
);
181 skb
->protocol
= protocol
;
184 if (skb
->protocol
== htons(ETH_P_IP
)) {
185 flag_ipoffset
|= 1 << HNS_TXD_L3CS_B
;
186 /* check for tcp/udp header */
187 flag_ipoffset
|= 1 << HNS_TXD_L4CS_B
;
189 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
190 /* ipv6 has not l3 cs, check for L4 header */
191 flag_ipoffset
|= 1 << HNS_TXD_L4CS_B
;
194 flag_ipoffset
|= ip_offset
<< HNS_TXD_IPOFFSET_S
;
198 flag_ipoffset
|= frag_end
<< HNS_TXD_FE_B
;
200 desc
->tx
.asid_bufnum_pid
= cpu_to_le16(asid_bufnum_pid
);
201 desc
->tx
.flag_ipoffset
= cpu_to_le32(flag_ipoffset
);
203 ring_ptr_move_fw(ring
, next_to_use
);
206 static void unfill_desc(struct hnae_ring
*ring
)
208 ring_ptr_move_bw(ring
, next_to_use
);
211 static int hns_nic_maybe_stop_tx(
212 struct sk_buff
**out_skb
, int *bnum
, struct hnae_ring
*ring
)
214 struct sk_buff
*skb
= *out_skb
;
215 struct sk_buff
*new_skb
= NULL
;
218 /* no. of segments (plus a header) */
219 buf_num
= skb_shinfo(skb
)->nr_frags
+ 1;
221 if (unlikely(buf_num
> ring
->max_desc_num_per_pkt
)) {
222 if (ring_space(ring
) < 1)
225 new_skb
= skb_copy(skb
, GFP_ATOMIC
);
229 dev_kfree_skb_any(skb
);
232 } else if (buf_num
> ring_space(ring
)) {
240 static int hns_nic_maybe_stop_tso(
241 struct sk_buff
**out_skb
, int *bnum
, struct hnae_ring
*ring
)
247 struct sk_buff
*skb
= *out_skb
;
248 struct sk_buff
*new_skb
= NULL
;
251 size
= skb_headlen(skb
);
252 buf_num
= (size
+ BD_MAX_SEND_SIZE
- 1) / BD_MAX_SEND_SIZE
;
254 frag_num
= skb_shinfo(skb
)->nr_frags
;
255 for (i
= 0; i
< frag_num
; i
++) {
256 frag
= &skb_shinfo(skb
)->frags
[i
];
257 size
= skb_frag_size(frag
);
258 buf_num
+= (size
+ BD_MAX_SEND_SIZE
- 1) / BD_MAX_SEND_SIZE
;
261 if (unlikely(buf_num
> ring
->max_desc_num_per_pkt
)) {
262 buf_num
= (skb
->len
+ BD_MAX_SEND_SIZE
- 1) / BD_MAX_SEND_SIZE
;
263 if (ring_space(ring
) < buf_num
)
265 /* manual split the send packet */
266 new_skb
= skb_copy(skb
, GFP_ATOMIC
);
269 dev_kfree_skb_any(skb
);
272 } else if (ring_space(ring
) < buf_num
) {
280 static void fill_tso_desc(struct hnae_ring
*ring
, void *priv
,
281 int size
, dma_addr_t dma
, int frag_end
,
282 int buf_num
, enum hns_desc_type type
, int mtu
)
288 frag_buf_num
= (size
+ BD_MAX_SEND_SIZE
- 1) / BD_MAX_SEND_SIZE
;
289 sizeoflast
= size
% BD_MAX_SEND_SIZE
;
290 sizeoflast
= sizeoflast
? sizeoflast
: BD_MAX_SEND_SIZE
;
292 /* when the frag size is bigger than hardware, split this frag */
293 for (k
= 0; k
< frag_buf_num
; k
++)
294 fill_v2_desc_hw(ring
, priv
, k
== 0 ? size
: 0,
295 (k
== frag_buf_num
- 1) ?
296 sizeoflast
: BD_MAX_SEND_SIZE
,
297 dma
+ BD_MAX_SEND_SIZE
* k
,
298 frag_end
&& (k
== frag_buf_num
- 1) ? 1 : 0,
300 (type
== DESC_TYPE_SKB
&& !k
) ?
301 DESC_TYPE_SKB
: DESC_TYPE_PAGE
,
305 netdev_tx_t
hns_nic_net_xmit_hw(struct net_device
*ndev
,
307 struct hns_nic_ring_data
*ring_data
)
309 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
310 struct hnae_ring
*ring
= ring_data
->ring
;
311 struct device
*dev
= ring_to_dev(ring
);
312 struct netdev_queue
*dev_queue
;
317 int size
, next_to_use
;
320 switch (priv
->ops
.maybe_stop_tx(&skb
, &buf_num
, ring
)) {
322 ring
->stats
.tx_busy
++;
323 goto out_net_tx_busy
;
325 ring
->stats
.sw_err_cnt
++;
326 netdev_err(ndev
, "no memory to xmit!\n");
332 /* no. of segments (plus a header) */
333 seg_num
= skb_shinfo(skb
)->nr_frags
+ 1;
334 next_to_use
= ring
->next_to_use
;
336 /* fill the first part */
337 size
= skb_headlen(skb
);
338 dma
= dma_map_single(dev
, skb
->data
, size
, DMA_TO_DEVICE
);
339 if (dma_mapping_error(dev
, dma
)) {
340 netdev_err(ndev
, "TX head DMA map failed\n");
341 ring
->stats
.sw_err_cnt
++;
344 priv
->ops
.fill_desc(ring
, skb
, size
, dma
, seg_num
== 1 ? 1 : 0,
345 buf_num
, DESC_TYPE_SKB
, ndev
->mtu
);
347 /* fill the fragments */
348 for (i
= 1; i
< seg_num
; i
++) {
349 frag
= &skb_shinfo(skb
)->frags
[i
- 1];
350 size
= skb_frag_size(frag
);
351 dma
= skb_frag_dma_map(dev
, frag
, 0, size
, DMA_TO_DEVICE
);
352 if (dma_mapping_error(dev
, dma
)) {
353 netdev_err(ndev
, "TX frag(%d) DMA map failed\n", i
);
354 ring
->stats
.sw_err_cnt
++;
355 goto out_map_frag_fail
;
357 priv
->ops
.fill_desc(ring
, skb_frag_page(frag
), size
, dma
,
358 seg_num
- 1 == i
? 1 : 0, buf_num
,
359 DESC_TYPE_PAGE
, ndev
->mtu
);
362 /*complete translate all packets*/
363 dev_queue
= netdev_get_tx_queue(ndev
, skb
->queue_mapping
);
364 netdev_tx_sent_queue(dev_queue
, skb
->len
);
366 netif_trans_update(ndev
);
367 ndev
->stats
.tx_bytes
+= skb
->len
;
368 ndev
->stats
.tx_packets
++;
370 wmb(); /* commit all data before submit */
371 assert(skb
->queue_mapping
< priv
->ae_handle
->q_num
);
372 hnae_queue_xmit(priv
->ae_handle
->qs
[skb
->queue_mapping
], buf_num
);
378 while (ring
->next_to_use
!= next_to_use
) {
380 if (ring
->next_to_use
!= next_to_use
)
382 ring
->desc_cb
[ring
->next_to_use
].dma
,
383 ring
->desc_cb
[ring
->next_to_use
].length
,
386 dma_unmap_single(dev
,
387 ring
->desc_cb
[next_to_use
].dma
,
388 ring
->desc_cb
[next_to_use
].length
,
394 dev_kfree_skb_any(skb
);
399 netif_stop_subqueue(ndev
, skb
->queue_mapping
);
401 /* Herbert's original patch had:
402 * smp_mb__after_netif_stop_queue();
403 * but since that doesn't exist yet, just open code it.
406 return NETDEV_TX_BUSY
;
409 static void hns_nic_reuse_page(struct sk_buff
*skb
, int i
,
410 struct hnae_ring
*ring
, int pull_len
,
411 struct hnae_desc_cb
*desc_cb
)
413 struct hnae_desc
*desc
;
419 twobufs
= ((PAGE_SIZE
< 8192) &&
420 hnae_buf_size(ring
) == HNS_BUFFER_SIZE_2048
);
422 desc
= &ring
->desc
[ring
->next_to_clean
];
423 size
= le16_to_cpu(desc
->rx
.size
);
426 truesize
= hnae_buf_size(ring
);
428 truesize
= ALIGN(size
, L1_CACHE_BYTES
);
429 last_offset
= hnae_page_size(ring
) - hnae_buf_size(ring
);
432 skb_add_rx_frag(skb
, i
, desc_cb
->priv
, desc_cb
->page_offset
+ pull_len
,
433 size
- pull_len
, truesize
);
435 /* avoid re-using remote pages,flag default unreuse */
436 if (unlikely(page_to_nid(desc_cb
->priv
) != numa_node_id()))
440 /* if we are only owner of page we can reuse it */
441 if (likely(page_count(desc_cb
->priv
) == 1)) {
442 /* flip page offset to other buffer */
443 desc_cb
->page_offset
^= truesize
;
445 desc_cb
->reuse_flag
= 1;
446 /* bump ref count on page before it is given*/
447 get_page(desc_cb
->priv
);
452 /* move offset up to the next cache line */
453 desc_cb
->page_offset
+= truesize
;
455 if (desc_cb
->page_offset
<= last_offset
) {
456 desc_cb
->reuse_flag
= 1;
457 /* bump ref count on page before it is given*/
458 get_page(desc_cb
->priv
);
462 static void get_v2rx_desc_bnum(u32 bnum_flag
, int *out_bnum
)
464 *out_bnum
= hnae_get_field(bnum_flag
,
465 HNS_RXD_BUFNUM_M
, HNS_RXD_BUFNUM_S
) + 1;
468 static void get_rx_desc_bnum(u32 bnum_flag
, int *out_bnum
)
470 *out_bnum
= hnae_get_field(bnum_flag
,
471 HNS_RXD_BUFNUM_M
, HNS_RXD_BUFNUM_S
);
474 static void hns_nic_rx_checksum(struct hns_nic_ring_data
*ring_data
,
475 struct sk_buff
*skb
, u32 flag
)
477 struct net_device
*netdev
= ring_data
->napi
.dev
;
481 /* check if RX checksum offload is enabled */
482 if (unlikely(!(netdev
->features
& NETIF_F_RXCSUM
)))
485 /* In hardware, we only support checksum for the following protocols:
487 * 2) TCP(over IPv4 or IPv6),
488 * 3) UDP(over IPv4 or IPv6),
489 * 4) SCTP(over IPv4 or IPv6)
490 * but we support many L3(IPv4, IPv6, MPLS, PPPoE etc) and L4(TCP,
491 * UDP, GRE, SCTP, IGMP, ICMP etc.) protocols.
493 * Hardware limitation:
494 * Our present hardware RX Descriptor lacks L3/L4 checksum "Status &
495 * Error" bit (which usually can be used to indicate whether checksum
496 * was calculated by the hardware and if there was any error encountered
497 * during checksum calculation).
499 * Software workaround:
500 * We do get info within the RX descriptor about the kind of L3/L4
501 * protocol coming in the packet and the error status. These errors
502 * might not just be checksum errors but could be related to version,
503 * length of IPv4, UDP, TCP etc.
504 * Because there is no-way of knowing if it is a L3/L4 error due to bad
505 * checksum or any other L3/L4 error, we will not (cannot) convey
506 * checksum status for such cases to upper stack and will not maintain
507 * the RX L3/L4 checksum counters as well.
510 l3id
= hnae_get_field(flag
, HNS_RXD_L3ID_M
, HNS_RXD_L3ID_S
);
511 l4id
= hnae_get_field(flag
, HNS_RXD_L4ID_M
, HNS_RXD_L4ID_S
);
513 /* check L3 protocol for which checksum is supported */
514 if ((l3id
!= HNS_RX_FLAG_L3ID_IPV4
) && (l3id
!= HNS_RX_FLAG_L3ID_IPV6
))
517 /* check for any(not just checksum)flagged L3 protocol errors */
518 if (unlikely(hnae_get_bit(flag
, HNS_RXD_L3E_B
)))
521 /* we do not support checksum of fragmented packets */
522 if (unlikely(hnae_get_bit(flag
, HNS_RXD_FRAG_B
)))
525 /* check L4 protocol for which checksum is supported */
526 if ((l4id
!= HNS_RX_FLAG_L4ID_TCP
) &&
527 (l4id
!= HNS_RX_FLAG_L4ID_UDP
) &&
528 (l4id
!= HNS_RX_FLAG_L4ID_SCTP
))
531 /* check for any(not just checksum)flagged L4 protocol errors */
532 if (unlikely(hnae_get_bit(flag
, HNS_RXD_L4E_B
)))
535 /* now, this has to be a packet with valid RX checksum */
536 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
539 static int hns_nic_poll_rx_skb(struct hns_nic_ring_data
*ring_data
,
540 struct sk_buff
**out_skb
, int *out_bnum
)
542 struct hnae_ring
*ring
= ring_data
->ring
;
543 struct net_device
*ndev
= ring_data
->napi
.dev
;
544 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
546 struct hnae_desc
*desc
;
547 struct hnae_desc_cb
*desc_cb
;
553 desc
= &ring
->desc
[ring
->next_to_clean
];
554 desc_cb
= &ring
->desc_cb
[ring
->next_to_clean
];
558 va
= (unsigned char *)desc_cb
->buf
+ desc_cb
->page_offset
;
560 /* prefetch first cache line of first page */
563 skb
= *out_skb
= napi_alloc_skb(&ring_data
->napi
,
565 if (unlikely(!skb
)) {
566 ring
->stats
.sw_err_cnt
++;
570 prefetchw(skb
->data
);
571 length
= le16_to_cpu(desc
->rx
.pkt_len
);
572 bnum_flag
= le32_to_cpu(desc
->rx
.ipoff_bnum_pid_flag
);
573 priv
->ops
.get_rxd_bnum(bnum_flag
, &bnum
);
576 if (length
<= HNS_RX_HEAD_SIZE
) {
577 memcpy(__skb_put(skb
, length
), va
, ALIGN(length
, sizeof(long)));
579 /* we can reuse buffer as-is, just make sure it is local */
580 if (likely(page_to_nid(desc_cb
->priv
) == numa_node_id()))
581 desc_cb
->reuse_flag
= 1;
582 else /* this page cannot be reused so discard it */
583 put_page(desc_cb
->priv
);
585 ring_ptr_move_fw(ring
, next_to_clean
);
587 if (unlikely(bnum
!= 1)) { /* check err*/
592 ring
->stats
.seg_pkt_cnt
++;
594 pull_len
= eth_get_headlen(ndev
, va
, HNS_RX_HEAD_SIZE
);
595 memcpy(__skb_put(skb
, pull_len
), va
,
596 ALIGN(pull_len
, sizeof(long)));
598 hns_nic_reuse_page(skb
, 0, ring
, pull_len
, desc_cb
);
599 ring_ptr_move_fw(ring
, next_to_clean
);
601 if (unlikely(bnum
>= (int)MAX_SKB_FRAGS
)) { /* check err*/
605 for (i
= 1; i
< bnum
; i
++) {
606 desc
= &ring
->desc
[ring
->next_to_clean
];
607 desc_cb
= &ring
->desc_cb
[ring
->next_to_clean
];
609 hns_nic_reuse_page(skb
, i
, ring
, 0, desc_cb
);
610 ring_ptr_move_fw(ring
, next_to_clean
);
614 /* check except process, free skb and jump the desc */
615 if (unlikely((!bnum
) || (bnum
> ring
->max_desc_num_per_pkt
))) {
617 *out_bnum
= *out_bnum
? *out_bnum
: 1; /* ntc moved,cannot 0*/
618 netdev_err(ndev
, "invalid bnum(%d,%d,%d,%d),%016llx,%016llx\n",
619 bnum
, ring
->max_desc_num_per_pkt
,
620 length
, (int)MAX_SKB_FRAGS
,
621 ((u64
*)desc
)[0], ((u64
*)desc
)[1]);
622 ring
->stats
.err_bd_num
++;
623 dev_kfree_skb_any(skb
);
627 bnum_flag
= le32_to_cpu(desc
->rx
.ipoff_bnum_pid_flag
);
629 if (unlikely(!hnae_get_bit(bnum_flag
, HNS_RXD_VLD_B
))) {
630 netdev_err(ndev
, "no valid bd,%016llx,%016llx\n",
631 ((u64
*)desc
)[0], ((u64
*)desc
)[1]);
632 ring
->stats
.non_vld_descs
++;
633 dev_kfree_skb_any(skb
);
637 if (unlikely((!desc
->rx
.pkt_len
) ||
638 hnae_get_bit(bnum_flag
, HNS_RXD_DROP_B
))) {
639 ring
->stats
.err_pkt_len
++;
640 dev_kfree_skb_any(skb
);
644 if (unlikely(hnae_get_bit(bnum_flag
, HNS_RXD_L2E_B
))) {
645 ring
->stats
.l2_err
++;
646 dev_kfree_skb_any(skb
);
650 ring
->stats
.rx_pkts
++;
651 ring
->stats
.rx_bytes
+= skb
->len
;
653 /* indicate to upper stack if our hardware has already calculated
656 hns_nic_rx_checksum(ring_data
, skb
, bnum_flag
);
662 hns_nic_alloc_rx_buffers(struct hns_nic_ring_data
*ring_data
, int cleand_count
)
665 struct hnae_desc_cb res_cbs
;
666 struct hnae_desc_cb
*desc_cb
;
667 struct hnae_ring
*ring
= ring_data
->ring
;
668 struct net_device
*ndev
= ring_data
->napi
.dev
;
670 for (i
= 0; i
< cleand_count
; i
++) {
671 desc_cb
= &ring
->desc_cb
[ring
->next_to_use
];
672 if (desc_cb
->reuse_flag
) {
673 ring
->stats
.reuse_pg_cnt
++;
674 hnae_reuse_buffer(ring
, ring
->next_to_use
);
676 ret
= hnae_reserve_buffer_map(ring
, &res_cbs
);
678 ring
->stats
.sw_err_cnt
++;
679 netdev_err(ndev
, "hnae reserve buffer map failed.\n");
682 hnae_replace_buffer(ring
, ring
->next_to_use
, &res_cbs
);
685 ring_ptr_move_fw(ring
, next_to_use
);
688 wmb(); /* make all data has been write before submit */
689 writel_relaxed(i
, ring
->io_base
+ RCB_REG_HEAD
);
692 /* return error number for error or number of desc left to take
694 static void hns_nic_rx_up_pro(struct hns_nic_ring_data
*ring_data
,
697 struct net_device
*ndev
= ring_data
->napi
.dev
;
699 skb
->protocol
= eth_type_trans(skb
, ndev
);
700 napi_gro_receive(&ring_data
->napi
, skb
);
703 static int hns_desc_unused(struct hnae_ring
*ring
)
705 int ntc
= ring
->next_to_clean
;
706 int ntu
= ring
->next_to_use
;
708 return ((ntc
>= ntu
) ? 0 : ring
->desc_num
) + ntc
- ntu
;
711 #define HNS_LOWEST_LATENCY_RATE 27 /* 27 MB/s */
712 #define HNS_LOW_LATENCY_RATE 80 /* 80 MB/s */
714 #define HNS_COAL_BDNUM 3
716 static u32
hns_coal_rx_bdnum(struct hnae_ring
*ring
)
718 bool coal_enable
= ring
->q
->handle
->coal_adapt_en
;
721 ring
->coal_last_rx_bytes
> HNS_LOWEST_LATENCY_RATE
)
722 return HNS_COAL_BDNUM
;
727 static void hns_update_rx_rate(struct hnae_ring
*ring
)
729 bool coal_enable
= ring
->q
->handle
->coal_adapt_en
;
734 time_before(jiffies
, ring
->coal_last_jiffies
+ (HZ
>> 4)))
737 /* ring->stats.rx_bytes overflowed */
738 if (ring
->coal_last_rx_bytes
> ring
->stats
.rx_bytes
) {
739 ring
->coal_last_rx_bytes
= ring
->stats
.rx_bytes
;
740 ring
->coal_last_jiffies
= jiffies
;
744 total_bytes
= ring
->stats
.rx_bytes
- ring
->coal_last_rx_bytes
;
745 time_passed_ms
= jiffies_to_msecs(jiffies
- ring
->coal_last_jiffies
);
746 do_div(total_bytes
, time_passed_ms
);
747 ring
->coal_rx_rate
= total_bytes
>> 10;
749 ring
->coal_last_rx_bytes
= ring
->stats
.rx_bytes
;
750 ring
->coal_last_jiffies
= jiffies
;
754 * smooth_alg - smoothing algrithm for adjusting coalesce parameter
755 * @new_param: new value
756 * @old_param: old value
758 static u32
smooth_alg(u32 new_param
, u32 old_param
)
760 u32 gap
= (new_param
> old_param
) ? new_param
- old_param
761 : old_param
- new_param
;
766 if (new_param
> old_param
)
767 return old_param
+ gap
;
769 return old_param
- gap
;
773 * hns_nic_adpt_coalesce - self adapte coalesce according to rx rate
774 * @ring_data: pointer to hns_nic_ring_data
776 static void hns_nic_adpt_coalesce(struct hns_nic_ring_data
*ring_data
)
778 struct hnae_ring
*ring
= ring_data
->ring
;
779 struct hnae_handle
*handle
= ring
->q
->handle
;
780 u32 new_coal_param
, old_coal_param
= ring
->coal_param
;
782 if (ring
->coal_rx_rate
< HNS_LOWEST_LATENCY_RATE
)
783 new_coal_param
= HNAE_LOWEST_LATENCY_COAL_PARAM
;
784 else if (ring
->coal_rx_rate
< HNS_LOW_LATENCY_RATE
)
785 new_coal_param
= HNAE_LOW_LATENCY_COAL_PARAM
;
787 new_coal_param
= HNAE_BULK_LATENCY_COAL_PARAM
;
789 if (new_coal_param
== old_coal_param
&&
790 new_coal_param
== handle
->coal_param
)
793 new_coal_param
= smooth_alg(new_coal_param
, old_coal_param
);
794 ring
->coal_param
= new_coal_param
;
797 * Because all ring in one port has one coalesce param, when one ring
798 * calculate its own coalesce param, it cannot write to hardware at
799 * once. There are three conditions as follows:
800 * 1. current ring's coalesce param is larger than the hardware.
801 * 2. or ring which adapt last time can change again.
804 if (new_coal_param
== handle
->coal_param
) {
805 handle
->coal_last_jiffies
= jiffies
;
806 handle
->coal_ring_idx
= ring_data
->queue_index
;
807 } else if (new_coal_param
> handle
->coal_param
||
808 handle
->coal_ring_idx
== ring_data
->queue_index
||
809 time_after(jiffies
, handle
->coal_last_jiffies
+ (HZ
>> 4))) {
810 handle
->dev
->ops
->set_coalesce_usecs(handle
,
812 handle
->dev
->ops
->set_coalesce_frames(handle
,
814 handle
->coal_param
= new_coal_param
;
815 handle
->coal_ring_idx
= ring_data
->queue_index
;
816 handle
->coal_last_jiffies
= jiffies
;
820 static int hns_nic_rx_poll_one(struct hns_nic_ring_data
*ring_data
,
823 struct hnae_ring
*ring
= ring_data
->ring
;
826 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
827 int recv_pkts
, recv_bds
, clean_count
, err
;
828 int unused_count
= hns_desc_unused(ring
);
830 num
= readl_relaxed(ring
->io_base
+ RCB_REG_FBDNUM
);
831 rmb(); /* make sure num taken effect before the other data is touched */
833 recv_pkts
= 0, recv_bds
= 0, clean_count
= 0;
836 while (recv_pkts
< budget
&& recv_bds
< num
) {
837 /* reuse or realloc buffers */
838 if (clean_count
+ unused_count
>= RCB_NOF_ALLOC_RX_BUFF_ONCE
) {
839 hns_nic_alloc_rx_buffers(ring_data
,
840 clean_count
+ unused_count
);
842 unused_count
= hns_desc_unused(ring
);
846 err
= hns_nic_poll_rx_skb(ring_data
, &skb
, &bnum
);
847 if (unlikely(!skb
)) /* this fault cannot be repaired */
852 if (unlikely(err
)) { /* do jump the err */
857 /* do update ip stack process*/
858 ((void (*)(struct hns_nic_ring_data
*, struct sk_buff
*))v
)(
864 /* make all data has been write before submit */
865 if (clean_count
+ unused_count
> 0)
866 hns_nic_alloc_rx_buffers(ring_data
,
867 clean_count
+ unused_count
);
872 static bool hns_nic_rx_fini_pro(struct hns_nic_ring_data
*ring_data
)
874 struct hnae_ring
*ring
= ring_data
->ring
;
878 hns_update_rx_rate(ring
);
880 /* for hardware bug fixed */
881 ring_data
->ring
->q
->handle
->dev
->ops
->toggle_ring_irq(ring
, 0);
882 num
= readl_relaxed(ring
->io_base
+ RCB_REG_FBDNUM
);
884 if (num
<= hns_coal_rx_bdnum(ring
)) {
885 if (ring
->q
->handle
->coal_adapt_en
)
886 hns_nic_adpt_coalesce(ring_data
);
890 ring_data
->ring
->q
->handle
->dev
->ops
->toggle_ring_irq(
899 static bool hns_nic_rx_fini_pro_v2(struct hns_nic_ring_data
*ring_data
)
901 struct hnae_ring
*ring
= ring_data
->ring
;
904 hns_update_rx_rate(ring
);
905 num
= readl_relaxed(ring
->io_base
+ RCB_REG_FBDNUM
);
907 if (num
<= hns_coal_rx_bdnum(ring
)) {
908 if (ring
->q
->handle
->coal_adapt_en
)
909 hns_nic_adpt_coalesce(ring_data
);
917 static inline void hns_nic_reclaim_one_desc(struct hnae_ring
*ring
,
918 int *bytes
, int *pkts
)
920 struct hnae_desc_cb
*desc_cb
= &ring
->desc_cb
[ring
->next_to_clean
];
922 (*pkts
) += (desc_cb
->type
== DESC_TYPE_SKB
);
923 (*bytes
) += desc_cb
->length
;
924 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
925 hnae_free_buffer_detach(ring
, ring
->next_to_clean
);
927 ring_ptr_move_fw(ring
, next_to_clean
);
930 static int is_valid_clean_head(struct hnae_ring
*ring
, int h
)
932 int u
= ring
->next_to_use
;
933 int c
= ring
->next_to_clean
;
935 if (unlikely(h
> ring
->desc_num
))
938 assert(u
> 0 && u
< ring
->desc_num
);
939 assert(c
> 0 && c
< ring
->desc_num
);
940 assert(u
!= c
&& h
!= c
); /* must be checked before call this func */
942 return u
> c
? (h
> c
&& h
<= u
) : (h
> c
|| h
<= u
);
945 /* reclaim all desc in one budget
946 * return error or number of desc left
948 static int hns_nic_tx_poll_one(struct hns_nic_ring_data
*ring_data
,
951 struct hnae_ring
*ring
= ring_data
->ring
;
952 struct net_device
*ndev
= ring_data
->napi
.dev
;
953 struct netdev_queue
*dev_queue
;
954 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
958 head
= readl_relaxed(ring
->io_base
+ RCB_REG_HEAD
);
959 rmb(); /* make sure head is ready before touch any data */
961 if (is_ring_empty(ring
) || head
== ring
->next_to_clean
)
962 return 0; /* no data to poll */
964 if (!is_valid_clean_head(ring
, head
)) {
965 netdev_err(ndev
, "wrong head (%d, %d-%d)\n", head
,
966 ring
->next_to_use
, ring
->next_to_clean
);
967 ring
->stats
.io_err_cnt
++;
973 while (head
!= ring
->next_to_clean
) {
974 hns_nic_reclaim_one_desc(ring
, &bytes
, &pkts
);
975 /* issue prefetch for next Tx descriptor */
976 prefetch(&ring
->desc_cb
[ring
->next_to_clean
]);
978 /* update tx ring statistics. */
979 ring
->stats
.tx_pkts
+= pkts
;
980 ring
->stats
.tx_bytes
+= bytes
;
982 dev_queue
= netdev_get_tx_queue(ndev
, ring_data
->queue_index
);
983 netdev_tx_completed_queue(dev_queue
, pkts
, bytes
);
985 if (unlikely(priv
->link
&& !netif_carrier_ok(ndev
)))
986 netif_carrier_on(ndev
);
988 if (unlikely(pkts
&& netif_carrier_ok(ndev
) &&
989 (ring_space(ring
) >= ring
->max_desc_num_per_pkt
* 2))) {
990 /* Make sure that anybody stopping the queue after this
991 * sees the new next_to_clean.
994 if (netif_tx_queue_stopped(dev_queue
) &&
995 !test_bit(NIC_STATE_DOWN
, &priv
->state
)) {
996 netif_tx_wake_queue(dev_queue
);
997 ring
->stats
.restart_queue
++;
1003 static bool hns_nic_tx_fini_pro(struct hns_nic_ring_data
*ring_data
)
1005 struct hnae_ring
*ring
= ring_data
->ring
;
1008 ring_data
->ring
->q
->handle
->dev
->ops
->toggle_ring_irq(ring
, 0);
1010 head
= readl_relaxed(ring
->io_base
+ RCB_REG_HEAD
);
1012 if (head
!= ring
->next_to_clean
) {
1013 ring_data
->ring
->q
->handle
->dev
->ops
->toggle_ring_irq(
1014 ring_data
->ring
, 1);
1022 static bool hns_nic_tx_fini_pro_v2(struct hns_nic_ring_data
*ring_data
)
1024 struct hnae_ring
*ring
= ring_data
->ring
;
1025 int head
= readl_relaxed(ring
->io_base
+ RCB_REG_HEAD
);
1027 if (head
== ring
->next_to_clean
)
1033 static void hns_nic_tx_clr_all_bufs(struct hns_nic_ring_data
*ring_data
)
1035 struct hnae_ring
*ring
= ring_data
->ring
;
1036 struct net_device
*ndev
= ring_data
->napi
.dev
;
1037 struct netdev_queue
*dev_queue
;
1041 head
= ring
->next_to_use
; /* ntu :soft setted ring position*/
1044 while (head
!= ring
->next_to_clean
)
1045 hns_nic_reclaim_one_desc(ring
, &bytes
, &pkts
);
1047 dev_queue
= netdev_get_tx_queue(ndev
, ring_data
->queue_index
);
1048 netdev_tx_reset_queue(dev_queue
);
1051 static int hns_nic_common_poll(struct napi_struct
*napi
, int budget
)
1053 int clean_complete
= 0;
1054 struct hns_nic_ring_data
*ring_data
=
1055 container_of(napi
, struct hns_nic_ring_data
, napi
);
1056 struct hnae_ring
*ring
= ring_data
->ring
;
1058 clean_complete
+= ring_data
->poll_one(
1059 ring_data
, budget
- clean_complete
,
1060 ring_data
->ex_process
);
1062 if (clean_complete
< budget
) {
1063 if (ring_data
->fini_process(ring_data
)) {
1064 napi_complete(napi
);
1065 ring
->q
->handle
->dev
->ops
->toggle_ring_irq(ring
, 0);
1071 return clean_complete
;
1074 static irqreturn_t
hns_irq_handle(int irq
, void *dev
)
1076 struct hns_nic_ring_data
*ring_data
= (struct hns_nic_ring_data
*)dev
;
1078 ring_data
->ring
->q
->handle
->dev
->ops
->toggle_ring_irq(
1079 ring_data
->ring
, 1);
1080 napi_schedule(&ring_data
->napi
);
1086 *hns_nic_adjust_link - adjust net work mode by the phy stat or new param
1089 static void hns_nic_adjust_link(struct net_device
*ndev
)
1091 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1092 struct hnae_handle
*h
= priv
->ae_handle
;
1095 /* If there is no phy, do not need adjust link */
1097 /* When phy link down, do nothing */
1098 if (ndev
->phydev
->link
== 0)
1101 if (h
->dev
->ops
->need_adjust_link(h
, ndev
->phydev
->speed
,
1102 ndev
->phydev
->duplex
)) {
1103 /* because Hi161X chip don't support to change gmac
1104 * speed and duplex with traffic. Delay 200ms to
1105 * make sure there is no more data in chip FIFO.
1107 netif_carrier_off(ndev
);
1109 h
->dev
->ops
->adjust_link(h
, ndev
->phydev
->speed
,
1110 ndev
->phydev
->duplex
);
1111 netif_carrier_on(ndev
);
1115 state
= state
&& h
->dev
->ops
->get_status(h
);
1117 if (state
!= priv
->link
) {
1119 netif_carrier_on(ndev
);
1120 netif_tx_wake_all_queues(ndev
);
1121 netdev_info(ndev
, "link up\n");
1123 netif_carrier_off(ndev
);
1124 netdev_info(ndev
, "link down\n");
1131 *hns_nic_init_phy - init phy
1134 * Return 0 on success, negative on failure
1136 int hns_nic_init_phy(struct net_device
*ndev
, struct hnae_handle
*h
)
1138 __ETHTOOL_DECLARE_LINK_MODE_MASK(supported
) = { 0, };
1139 struct phy_device
*phy_dev
= h
->phy_dev
;
1145 ethtool_convert_legacy_u32_to_link_mode(supported
, h
->if_support
);
1146 linkmode_and(phy_dev
->supported
, phy_dev
->supported
, supported
);
1147 linkmode_copy(phy_dev
->advertising
, phy_dev
->supported
);
1149 if (h
->phy_if
== PHY_INTERFACE_MODE_XGMII
)
1150 phy_dev
->autoneg
= false;
1152 if (h
->phy_if
!= PHY_INTERFACE_MODE_XGMII
) {
1153 phy_dev
->dev_flags
= 0;
1155 ret
= phy_connect_direct(ndev
, phy_dev
, hns_nic_adjust_link
,
1158 ret
= phy_attach_direct(ndev
, phy_dev
, 0, h
->phy_if
);
1163 phy_attached_info(phy_dev
);
1168 static int hns_nic_ring_open(struct net_device
*netdev
, int idx
)
1170 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1171 struct hnae_handle
*h
= priv
->ae_handle
;
1173 napi_enable(&priv
->ring_data
[idx
].napi
);
1175 enable_irq(priv
->ring_data
[idx
].ring
->irq
);
1176 h
->dev
->ops
->toggle_ring_irq(priv
->ring_data
[idx
].ring
, 0);
1181 static int hns_nic_net_set_mac_address(struct net_device
*ndev
, void *p
)
1183 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1184 struct hnae_handle
*h
= priv
->ae_handle
;
1185 struct sockaddr
*mac_addr
= p
;
1188 if (!mac_addr
|| !is_valid_ether_addr((const u8
*)mac_addr
->sa_data
))
1189 return -EADDRNOTAVAIL
;
1191 ret
= h
->dev
->ops
->set_mac_addr(h
, mac_addr
->sa_data
);
1193 netdev_err(ndev
, "set_mac_address fail, ret=%d!\n", ret
);
1197 memcpy(ndev
->dev_addr
, mac_addr
->sa_data
, ndev
->addr_len
);
1202 static void hns_nic_update_stats(struct net_device
*netdev
)
1204 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1205 struct hnae_handle
*h
= priv
->ae_handle
;
1207 h
->dev
->ops
->update_stats(h
, &netdev
->stats
);
1210 /* set mac addr if it is configed. or leave it to the AE driver */
1211 static void hns_init_mac_addr(struct net_device
*ndev
)
1213 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1215 if (!device_get_mac_address(priv
->dev
, ndev
->dev_addr
, ETH_ALEN
)) {
1216 eth_hw_addr_random(ndev
);
1217 dev_warn(priv
->dev
, "No valid mac, use random mac %pM",
1222 static void hns_nic_ring_close(struct net_device
*netdev
, int idx
)
1224 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1225 struct hnae_handle
*h
= priv
->ae_handle
;
1227 h
->dev
->ops
->toggle_ring_irq(priv
->ring_data
[idx
].ring
, 1);
1228 disable_irq(priv
->ring_data
[idx
].ring
->irq
);
1230 napi_disable(&priv
->ring_data
[idx
].napi
);
1233 static int hns_nic_init_affinity_mask(int q_num
, int ring_idx
,
1234 struct hnae_ring
*ring
, cpumask_t
*mask
)
1238 /* Different irq balance between 16core and 32core.
1239 * The cpu mask set by ring index according to the ring flag
1240 * which indicate the ring is tx or rx.
1242 if (q_num
== num_possible_cpus()) {
1243 if (is_tx_ring(ring
))
1246 cpu
= ring_idx
- q_num
;
1248 if (is_tx_ring(ring
))
1251 cpu
= (ring_idx
- q_num
) * 2 + 1;
1254 cpumask_clear(mask
);
1255 cpumask_set_cpu(cpu
, mask
);
1260 static void hns_nic_free_irq(int q_num
, struct hns_nic_priv
*priv
)
1264 for (i
= 0; i
< q_num
* 2; i
++) {
1265 if (priv
->ring_data
[i
].ring
->irq_init_flag
== RCB_IRQ_INITED
) {
1266 irq_set_affinity_hint(priv
->ring_data
[i
].ring
->irq
,
1268 free_irq(priv
->ring_data
[i
].ring
->irq
,
1269 &priv
->ring_data
[i
]);
1270 priv
->ring_data
[i
].ring
->irq_init_flag
=
1276 static int hns_nic_init_irq(struct hns_nic_priv
*priv
)
1278 struct hnae_handle
*h
= priv
->ae_handle
;
1279 struct hns_nic_ring_data
*rd
;
1284 for (i
= 0; i
< h
->q_num
* 2; i
++) {
1285 rd
= &priv
->ring_data
[i
];
1287 if (rd
->ring
->irq_init_flag
== RCB_IRQ_INITED
)
1290 snprintf(rd
->ring
->ring_name
, RCB_RING_NAME_LEN
,
1291 "%s-%s%d", priv
->netdev
->name
,
1292 (is_tx_ring(rd
->ring
) ? "tx" : "rx"), rd
->queue_index
);
1294 rd
->ring
->ring_name
[RCB_RING_NAME_LEN
- 1] = '\0';
1296 irq_set_status_flags(rd
->ring
->irq
, IRQ_NOAUTOEN
);
1297 ret
= request_irq(rd
->ring
->irq
,
1298 hns_irq_handle
, 0, rd
->ring
->ring_name
, rd
);
1300 netdev_err(priv
->netdev
, "request irq(%d) fail\n",
1305 cpu
= hns_nic_init_affinity_mask(h
->q_num
, i
,
1306 rd
->ring
, &rd
->mask
);
1308 if (cpu_online(cpu
))
1309 irq_set_affinity_hint(rd
->ring
->irq
,
1312 rd
->ring
->irq_init_flag
= RCB_IRQ_INITED
;
1318 hns_nic_free_irq(h
->q_num
, priv
);
1322 static int hns_nic_net_up(struct net_device
*ndev
)
1324 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1325 struct hnae_handle
*h
= priv
->ae_handle
;
1329 if (!test_bit(NIC_STATE_DOWN
, &priv
->state
))
1332 ret
= hns_nic_init_irq(priv
);
1334 netdev_err(ndev
, "hns init irq failed! ret=%d\n", ret
);
1338 for (i
= 0; i
< h
->q_num
* 2; i
++) {
1339 ret
= hns_nic_ring_open(ndev
, i
);
1341 goto out_has_some_queues
;
1344 ret
= h
->dev
->ops
->set_mac_addr(h
, ndev
->dev_addr
);
1346 goto out_set_mac_addr_err
;
1348 ret
= h
->dev
->ops
->start
? h
->dev
->ops
->start(h
) : 0;
1353 phy_start(ndev
->phydev
);
1355 clear_bit(NIC_STATE_DOWN
, &priv
->state
);
1356 (void)mod_timer(&priv
->service_timer
, jiffies
+ SERVICE_TIMER_HZ
);
1361 netif_stop_queue(ndev
);
1362 out_set_mac_addr_err
:
1363 out_has_some_queues
:
1364 for (j
= i
- 1; j
>= 0; j
--)
1365 hns_nic_ring_close(ndev
, j
);
1367 hns_nic_free_irq(h
->q_num
, priv
);
1368 set_bit(NIC_STATE_DOWN
, &priv
->state
);
1373 static void hns_nic_net_down(struct net_device
*ndev
)
1376 struct hnae_ae_ops
*ops
;
1377 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1379 if (test_and_set_bit(NIC_STATE_DOWN
, &priv
->state
))
1382 (void)del_timer_sync(&priv
->service_timer
);
1383 netif_tx_stop_all_queues(ndev
);
1384 netif_carrier_off(ndev
);
1385 netif_tx_disable(ndev
);
1389 phy_stop(ndev
->phydev
);
1391 ops
= priv
->ae_handle
->dev
->ops
;
1394 ops
->stop(priv
->ae_handle
);
1396 netif_tx_stop_all_queues(ndev
);
1398 for (i
= priv
->ae_handle
->q_num
- 1; i
>= 0; i
--) {
1399 hns_nic_ring_close(ndev
, i
);
1400 hns_nic_ring_close(ndev
, i
+ priv
->ae_handle
->q_num
);
1402 /* clean tx buffers*/
1403 hns_nic_tx_clr_all_bufs(priv
->ring_data
+ i
);
1407 void hns_nic_net_reset(struct net_device
*ndev
)
1409 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1410 struct hnae_handle
*handle
= priv
->ae_handle
;
1412 while (test_and_set_bit(NIC_STATE_RESETTING
, &priv
->state
))
1413 usleep_range(1000, 2000);
1415 (void)hnae_reinit_handle(handle
);
1417 clear_bit(NIC_STATE_RESETTING
, &priv
->state
);
1420 void hns_nic_net_reinit(struct net_device
*netdev
)
1422 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1423 enum hnae_port_type type
= priv
->ae_handle
->port_type
;
1425 netif_trans_update(priv
->netdev
);
1426 while (test_and_set_bit(NIC_STATE_REINITING
, &priv
->state
))
1427 usleep_range(1000, 2000);
1429 hns_nic_net_down(netdev
);
1431 /* Only do hns_nic_net_reset in debug mode
1432 * because of hardware limitation.
1434 if (type
== HNAE_PORT_DEBUG
)
1435 hns_nic_net_reset(netdev
);
1437 (void)hns_nic_net_up(netdev
);
1438 clear_bit(NIC_STATE_REINITING
, &priv
->state
);
1441 static int hns_nic_net_open(struct net_device
*ndev
)
1443 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1444 struct hnae_handle
*h
= priv
->ae_handle
;
1447 if (test_bit(NIC_STATE_TESTING
, &priv
->state
))
1451 netif_carrier_off(ndev
);
1453 ret
= netif_set_real_num_tx_queues(ndev
, h
->q_num
);
1455 netdev_err(ndev
, "netif_set_real_num_tx_queues fail, ret=%d!\n",
1460 ret
= netif_set_real_num_rx_queues(ndev
, h
->q_num
);
1463 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret
);
1467 ret
= hns_nic_net_up(ndev
);
1470 "hns net up fail, ret=%d!\n", ret
);
1477 static int hns_nic_net_stop(struct net_device
*ndev
)
1479 hns_nic_net_down(ndev
);
1484 static void hns_tx_timeout_reset(struct hns_nic_priv
*priv
);
1485 #define HNS_TX_TIMEO_LIMIT (40 * HZ)
1486 static void hns_nic_net_timeout(struct net_device
*ndev
, unsigned int txqueue
)
1488 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1490 if (ndev
->watchdog_timeo
< HNS_TX_TIMEO_LIMIT
) {
1491 ndev
->watchdog_timeo
*= 2;
1492 netdev_info(ndev
, "watchdog_timo changed to %d.\n",
1493 ndev
->watchdog_timeo
);
1495 ndev
->watchdog_timeo
= HNS_NIC_TX_TIMEOUT
;
1496 hns_tx_timeout_reset(priv
);
1500 static netdev_tx_t
hns_nic_net_xmit(struct sk_buff
*skb
,
1501 struct net_device
*ndev
)
1503 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1505 assert(skb
->queue_mapping
< priv
->ae_handle
->q_num
);
1507 return hns_nic_net_xmit_hw(ndev
, skb
,
1508 &tx_ring_data(priv
, skb
->queue_mapping
));
1511 static void hns_nic_drop_rx_fetch(struct hns_nic_ring_data
*ring_data
,
1512 struct sk_buff
*skb
)
1514 dev_kfree_skb_any(skb
);
1517 #define HNS_LB_TX_RING 0
1518 static struct sk_buff
*hns_assemble_skb(struct net_device
*ndev
)
1520 struct sk_buff
*skb
;
1521 struct ethhdr
*ethhdr
;
1524 /* allocate test skb */
1525 skb
= alloc_skb(64, GFP_KERNEL
);
1531 memset(skb
->data
, 0xFF, skb
->len
);
1533 /* must be tcp/ip package */
1534 ethhdr
= (struct ethhdr
*)skb
->data
;
1535 ethhdr
->h_proto
= htons(ETH_P_IP
);
1537 frame_len
= skb
->len
& (~1ul);
1538 memset(&skb
->data
[frame_len
/ 2], 0xAA,
1541 skb
->queue_mapping
= HNS_LB_TX_RING
;
1546 static int hns_enable_serdes_lb(struct net_device
*ndev
)
1548 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1549 struct hnae_handle
*h
= priv
->ae_handle
;
1550 struct hnae_ae_ops
*ops
= h
->dev
->ops
;
1554 ret
= ops
->set_loopback(h
, MAC_INTERNALLOOP_SERDES
, 1);
1558 ret
= ops
->start
? ops
->start(h
) : 0;
1562 /* link adjust duplex*/
1563 if (h
->phy_if
!= PHY_INTERFACE_MODE_XGMII
)
1569 ops
->adjust_link(h
, speed
, duplex
);
1571 /* wait h/w ready */
1577 static void hns_disable_serdes_lb(struct net_device
*ndev
)
1579 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1580 struct hnae_handle
*h
= priv
->ae_handle
;
1581 struct hnae_ae_ops
*ops
= h
->dev
->ops
;
1584 ops
->set_loopback(h
, MAC_INTERNALLOOP_SERDES
, 0);
1588 *hns_nic_clear_all_rx_fetch - clear the chip fetched descriptions. The
1589 *function as follows:
1590 * 1. if one rx ring has found the page_offset is not equal 0 between head
1591 * and tail, it means that the chip fetched the wrong descs for the ring
1592 * which buffer size is 4096.
1593 * 2. we set the chip serdes loopback and set rss indirection to the ring.
1594 * 3. construct 64-bytes ip broadcast packages, wait the associated rx ring
1595 * receiving all packages and it will fetch new descriptions.
1596 * 4. recover to the original state.
1600 static int hns_nic_clear_all_rx_fetch(struct net_device
*ndev
)
1602 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1603 struct hnae_handle
*h
= priv
->ae_handle
;
1604 struct hnae_ae_ops
*ops
= h
->dev
->ops
;
1605 struct hns_nic_ring_data
*rd
;
1606 struct hnae_ring
*ring
;
1607 struct sk_buff
*skb
;
1618 /* alloc indir memory */
1619 indir_size
= ops
->get_rss_indir_size(h
) * sizeof(*org_indir
);
1620 org_indir
= kzalloc(indir_size
, GFP_KERNEL
);
1624 /* store the original indirection */
1625 ops
->get_rss(h
, org_indir
, NULL
, NULL
);
1627 cur_indir
= kzalloc(indir_size
, GFP_KERNEL
);
1630 goto cur_indir_alloc_err
;
1634 if (hns_enable_serdes_lb(ndev
)) {
1636 goto enable_serdes_lb_err
;
1639 /* foreach every rx ring to clear fetch desc */
1640 for (i
= 0; i
< h
->q_num
; i
++) {
1641 ring
= &h
->qs
[i
]->rx_ring
;
1642 head
= readl_relaxed(ring
->io_base
+ RCB_REG_HEAD
);
1643 tail
= readl_relaxed(ring
->io_base
+ RCB_REG_TAIL
);
1645 fetch_num
= ring_dist(ring
, head
, tail
);
1647 while (head
!= tail
) {
1648 if (ring
->desc_cb
[head
].page_offset
!= 0) {
1654 if (head
== ring
->desc_num
)
1659 for (j
= 0; j
< indir_size
/ sizeof(*org_indir
); j
++)
1661 ops
->set_rss(h
, cur_indir
, NULL
, 0);
1663 for (j
= 0; j
< fetch_num
; j
++) {
1664 /* alloc one skb and init */
1665 skb
= hns_assemble_skb(ndev
);
1670 rd
= &tx_ring_data(priv
, skb
->queue_mapping
);
1671 hns_nic_net_xmit_hw(ndev
, skb
, rd
);
1674 while (retry_times
++ < 10) {
1677 rd
= &rx_ring_data(priv
, i
);
1678 if (rd
->poll_one(rd
, fetch_num
,
1679 hns_nic_drop_rx_fetch
))
1684 while (retry_times
++ < 10) {
1686 /* clean tx ring 0 send package */
1687 rd
= &tx_ring_data(priv
,
1689 if (rd
->poll_one(rd
, fetch_num
, NULL
))
1697 /* restore everything */
1698 ops
->set_rss(h
, org_indir
, NULL
, 0);
1699 hns_disable_serdes_lb(ndev
);
1700 enable_serdes_lb_err
:
1702 cur_indir_alloc_err
:
1708 static int hns_nic_change_mtu(struct net_device
*ndev
, int new_mtu
)
1710 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1711 struct hnae_handle
*h
= priv
->ae_handle
;
1712 bool if_running
= netif_running(ndev
);
1715 /* MTU < 68 is an error and causes problems on some kernels */
1720 if (new_mtu
== ndev
->mtu
)
1723 if (!h
->dev
->ops
->set_mtu
)
1727 (void)hns_nic_net_stop(ndev
);
1731 if (priv
->enet_ver
!= AE_VERSION_1
&&
1732 ndev
->mtu
<= BD_SIZE_2048_MAX_MTU
&&
1733 new_mtu
> BD_SIZE_2048_MAX_MTU
) {
1735 hnae_reinit_all_ring_desc(h
);
1737 /* clear the package which the chip has fetched */
1738 ret
= hns_nic_clear_all_rx_fetch(ndev
);
1740 /* the page offset must be consist with desc */
1741 hnae_reinit_all_ring_page_off(h
);
1744 netdev_err(ndev
, "clear the fetched desc fail\n");
1749 ret
= h
->dev
->ops
->set_mtu(h
, new_mtu
);
1751 netdev_err(ndev
, "set mtu fail, return value %d\n",
1756 /* finally, set new mtu to netdevice */
1757 ndev
->mtu
= new_mtu
;
1761 if (hns_nic_net_open(ndev
)) {
1762 netdev_err(ndev
, "hns net open fail\n");
1770 static int hns_nic_set_features(struct net_device
*netdev
,
1771 netdev_features_t features
)
1773 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1775 switch (priv
->enet_ver
) {
1777 if (features
& (NETIF_F_TSO
| NETIF_F_TSO6
))
1778 netdev_info(netdev
, "enet v1 do not support tso!\n");
1781 if (features
& (NETIF_F_TSO
| NETIF_F_TSO6
)) {
1782 priv
->ops
.fill_desc
= fill_tso_desc
;
1783 priv
->ops
.maybe_stop_tx
= hns_nic_maybe_stop_tso
;
1784 /* The chip only support 7*4096 */
1785 netif_set_gso_max_size(netdev
, 7 * 4096);
1787 priv
->ops
.fill_desc
= fill_v2_desc
;
1788 priv
->ops
.maybe_stop_tx
= hns_nic_maybe_stop_tx
;
1792 netdev
->features
= features
;
1796 static netdev_features_t
hns_nic_fix_features(
1797 struct net_device
*netdev
, netdev_features_t features
)
1799 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1801 switch (priv
->enet_ver
) {
1803 features
&= ~(NETIF_F_TSO
| NETIF_F_TSO6
|
1804 NETIF_F_HW_VLAN_CTAG_FILTER
);
1812 static int hns_nic_uc_sync(struct net_device
*netdev
, const unsigned char *addr
)
1814 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1815 struct hnae_handle
*h
= priv
->ae_handle
;
1817 if (h
->dev
->ops
->add_uc_addr
)
1818 return h
->dev
->ops
->add_uc_addr(h
, addr
);
1823 static int hns_nic_uc_unsync(struct net_device
*netdev
,
1824 const unsigned char *addr
)
1826 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1827 struct hnae_handle
*h
= priv
->ae_handle
;
1829 if (h
->dev
->ops
->rm_uc_addr
)
1830 return h
->dev
->ops
->rm_uc_addr(h
, addr
);
1836 * hns_set_multicast_list - set mutl mac address
1841 static void hns_set_multicast_list(struct net_device
*ndev
)
1843 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1844 struct hnae_handle
*h
= priv
->ae_handle
;
1845 struct netdev_hw_addr
*ha
= NULL
;
1848 netdev_err(ndev
, "hnae handle is null\n");
1852 if (h
->dev
->ops
->clr_mc_addr
)
1853 if (h
->dev
->ops
->clr_mc_addr(h
))
1854 netdev_err(ndev
, "clear multicast address fail\n");
1856 if (h
->dev
->ops
->set_mc_addr
) {
1857 netdev_for_each_mc_addr(ha
, ndev
)
1858 if (h
->dev
->ops
->set_mc_addr(h
, ha
->addr
))
1859 netdev_err(ndev
, "set multicast fail\n");
1863 static void hns_nic_set_rx_mode(struct net_device
*ndev
)
1865 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1866 struct hnae_handle
*h
= priv
->ae_handle
;
1868 if (h
->dev
->ops
->set_promisc_mode
) {
1869 if (ndev
->flags
& IFF_PROMISC
)
1870 h
->dev
->ops
->set_promisc_mode(h
, 1);
1872 h
->dev
->ops
->set_promisc_mode(h
, 0);
1875 hns_set_multicast_list(ndev
);
1877 if (__dev_uc_sync(ndev
, hns_nic_uc_sync
, hns_nic_uc_unsync
))
1878 netdev_err(ndev
, "sync uc address fail\n");
1881 static void hns_nic_get_stats64(struct net_device
*ndev
,
1882 struct rtnl_link_stats64
*stats
)
1889 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1890 struct hnae_handle
*h
= priv
->ae_handle
;
1892 for (idx
= 0; idx
< h
->q_num
; idx
++) {
1893 tx_bytes
+= h
->qs
[idx
]->tx_ring
.stats
.tx_bytes
;
1894 tx_pkts
+= h
->qs
[idx
]->tx_ring
.stats
.tx_pkts
;
1895 rx_bytes
+= h
->qs
[idx
]->rx_ring
.stats
.rx_bytes
;
1896 rx_pkts
+= h
->qs
[idx
]->rx_ring
.stats
.rx_pkts
;
1899 stats
->tx_bytes
= tx_bytes
;
1900 stats
->tx_packets
= tx_pkts
;
1901 stats
->rx_bytes
= rx_bytes
;
1902 stats
->rx_packets
= rx_pkts
;
1904 stats
->rx_errors
= ndev
->stats
.rx_errors
;
1905 stats
->multicast
= ndev
->stats
.multicast
;
1906 stats
->rx_length_errors
= ndev
->stats
.rx_length_errors
;
1907 stats
->rx_crc_errors
= ndev
->stats
.rx_crc_errors
;
1908 stats
->rx_missed_errors
= ndev
->stats
.rx_missed_errors
;
1910 stats
->tx_errors
= ndev
->stats
.tx_errors
;
1911 stats
->rx_dropped
= ndev
->stats
.rx_dropped
;
1912 stats
->tx_dropped
= ndev
->stats
.tx_dropped
;
1913 stats
->collisions
= ndev
->stats
.collisions
;
1914 stats
->rx_over_errors
= ndev
->stats
.rx_over_errors
;
1915 stats
->rx_frame_errors
= ndev
->stats
.rx_frame_errors
;
1916 stats
->rx_fifo_errors
= ndev
->stats
.rx_fifo_errors
;
1917 stats
->tx_aborted_errors
= ndev
->stats
.tx_aborted_errors
;
1918 stats
->tx_carrier_errors
= ndev
->stats
.tx_carrier_errors
;
1919 stats
->tx_fifo_errors
= ndev
->stats
.tx_fifo_errors
;
1920 stats
->tx_heartbeat_errors
= ndev
->stats
.tx_heartbeat_errors
;
1921 stats
->tx_window_errors
= ndev
->stats
.tx_window_errors
;
1922 stats
->rx_compressed
= ndev
->stats
.rx_compressed
;
1923 stats
->tx_compressed
= ndev
->stats
.tx_compressed
;
1927 hns_nic_select_queue(struct net_device
*ndev
, struct sk_buff
*skb
,
1928 struct net_device
*sb_dev
)
1930 struct ethhdr
*eth_hdr
= (struct ethhdr
*)skb
->data
;
1931 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
1933 /* fix hardware broadcast/multicast packets queue loopback */
1934 if (!AE_IS_VER1(priv
->enet_ver
) &&
1935 is_multicast_ether_addr(eth_hdr
->h_dest
))
1938 return netdev_pick_tx(ndev
, skb
, NULL
);
1941 static const struct net_device_ops hns_nic_netdev_ops
= {
1942 .ndo_open
= hns_nic_net_open
,
1943 .ndo_stop
= hns_nic_net_stop
,
1944 .ndo_start_xmit
= hns_nic_net_xmit
,
1945 .ndo_tx_timeout
= hns_nic_net_timeout
,
1946 .ndo_set_mac_address
= hns_nic_net_set_mac_address
,
1947 .ndo_change_mtu
= hns_nic_change_mtu
,
1948 .ndo_eth_ioctl
= phy_do_ioctl_running
,
1949 .ndo_set_features
= hns_nic_set_features
,
1950 .ndo_fix_features
= hns_nic_fix_features
,
1951 .ndo_get_stats64
= hns_nic_get_stats64
,
1952 .ndo_set_rx_mode
= hns_nic_set_rx_mode
,
1953 .ndo_select_queue
= hns_nic_select_queue
,
1956 static void hns_nic_update_link_status(struct net_device
*netdev
)
1958 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
1960 struct hnae_handle
*h
= priv
->ae_handle
;
1963 if (h
->phy_if
!= PHY_INTERFACE_MODE_XGMII
)
1966 (void)genphy_read_status(h
->phy_dev
);
1968 hns_nic_adjust_link(netdev
);
1971 /* for dumping key regs*/
1972 static void hns_nic_dump(struct hns_nic_priv
*priv
)
1974 struct hnae_handle
*h
= priv
->ae_handle
;
1975 struct hnae_ae_ops
*ops
= h
->dev
->ops
;
1976 u32
*data
, reg_num
, i
;
1978 if (ops
->get_regs_len
&& ops
->get_regs
) {
1979 reg_num
= ops
->get_regs_len(priv
->ae_handle
);
1980 reg_num
= (reg_num
+ 3ul) & ~3ul;
1981 data
= kcalloc(reg_num
, sizeof(u32
), GFP_KERNEL
);
1983 ops
->get_regs(priv
->ae_handle
, data
);
1984 for (i
= 0; i
< reg_num
; i
+= 4)
1985 pr_info("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1986 i
, data
[i
], data
[i
+ 1],
1987 data
[i
+ 2], data
[i
+ 3]);
1992 for (i
= 0; i
< h
->q_num
; i
++) {
1993 pr_info("tx_queue%d_next_to_clean:%d\n",
1994 i
, h
->qs
[i
]->tx_ring
.next_to_clean
);
1995 pr_info("tx_queue%d_next_to_use:%d\n",
1996 i
, h
->qs
[i
]->tx_ring
.next_to_use
);
1997 pr_info("rx_queue%d_next_to_clean:%d\n",
1998 i
, h
->qs
[i
]->rx_ring
.next_to_clean
);
1999 pr_info("rx_queue%d_next_to_use:%d\n",
2000 i
, h
->qs
[i
]->rx_ring
.next_to_use
);
2004 /* for resetting subtask */
2005 static void hns_nic_reset_subtask(struct hns_nic_priv
*priv
)
2007 enum hnae_port_type type
= priv
->ae_handle
->port_type
;
2009 if (!test_bit(NIC_STATE2_RESET_REQUESTED
, &priv
->state
))
2011 clear_bit(NIC_STATE2_RESET_REQUESTED
, &priv
->state
);
2013 /* If we're already down, removing or resetting, just bail */
2014 if (test_bit(NIC_STATE_DOWN
, &priv
->state
) ||
2015 test_bit(NIC_STATE_REMOVING
, &priv
->state
) ||
2016 test_bit(NIC_STATE_RESETTING
, &priv
->state
))
2020 netdev_info(priv
->netdev
, "try to reset %s port!\n",
2021 (type
== HNAE_PORT_DEBUG
? "debug" : "service"));
2024 /* put off any impending NetWatchDogTimeout */
2025 netif_trans_update(priv
->netdev
);
2026 hns_nic_net_reinit(priv
->netdev
);
2031 /* for doing service complete*/
2032 static void hns_nic_service_event_complete(struct hns_nic_priv
*priv
)
2034 WARN_ON(!test_bit(NIC_STATE_SERVICE_SCHED
, &priv
->state
));
2035 /* make sure to commit the things */
2036 smp_mb__before_atomic();
2037 clear_bit(NIC_STATE_SERVICE_SCHED
, &priv
->state
);
2040 static void hns_nic_service_task(struct work_struct
*work
)
2042 struct hns_nic_priv
*priv
2043 = container_of(work
, struct hns_nic_priv
, service_task
);
2044 struct hnae_handle
*h
= priv
->ae_handle
;
2046 hns_nic_reset_subtask(priv
);
2047 hns_nic_update_link_status(priv
->netdev
);
2048 h
->dev
->ops
->update_led_status(h
);
2049 hns_nic_update_stats(priv
->netdev
);
2051 hns_nic_service_event_complete(priv
);
2054 static void hns_nic_task_schedule(struct hns_nic_priv
*priv
)
2056 if (!test_bit(NIC_STATE_DOWN
, &priv
->state
) &&
2057 !test_bit(NIC_STATE_REMOVING
, &priv
->state
) &&
2058 !test_and_set_bit(NIC_STATE_SERVICE_SCHED
, &priv
->state
))
2059 (void)schedule_work(&priv
->service_task
);
2062 static void hns_nic_service_timer(struct timer_list
*t
)
2064 struct hns_nic_priv
*priv
= from_timer(priv
, t
, service_timer
);
2066 (void)mod_timer(&priv
->service_timer
, jiffies
+ SERVICE_TIMER_HZ
);
2068 hns_nic_task_schedule(priv
);
2072 * hns_tx_timeout_reset - initiate reset due to Tx timeout
2073 * @priv: driver private struct
2075 static void hns_tx_timeout_reset(struct hns_nic_priv
*priv
)
2077 /* Do the reset outside of interrupt context */
2078 if (!test_bit(NIC_STATE_DOWN
, &priv
->state
)) {
2079 set_bit(NIC_STATE2_RESET_REQUESTED
, &priv
->state
);
2080 netdev_warn(priv
->netdev
,
2081 "initiating reset due to tx timeout(%llu,0x%lx)\n",
2082 priv
->tx_timeout_count
, priv
->state
);
2083 priv
->tx_timeout_count
++;
2084 hns_nic_task_schedule(priv
);
2088 static int hns_nic_init_ring_data(struct hns_nic_priv
*priv
)
2090 struct hnae_handle
*h
= priv
->ae_handle
;
2091 struct hns_nic_ring_data
*rd
;
2092 bool is_ver1
= AE_IS_VER1(priv
->enet_ver
);
2095 if (h
->q_num
> NIC_MAX_Q_PER_VF
) {
2096 netdev_err(priv
->netdev
, "too much queue (%d)\n", h
->q_num
);
2100 priv
->ring_data
= kzalloc(array3_size(h
->q_num
,
2101 sizeof(*priv
->ring_data
), 2),
2103 if (!priv
->ring_data
)
2106 for (i
= 0; i
< h
->q_num
; i
++) {
2107 rd
= &priv
->ring_data
[i
];
2108 rd
->queue_index
= i
;
2109 rd
->ring
= &h
->qs
[i
]->tx_ring
;
2110 rd
->poll_one
= hns_nic_tx_poll_one
;
2111 rd
->fini_process
= is_ver1
? hns_nic_tx_fini_pro
:
2112 hns_nic_tx_fini_pro_v2
;
2114 netif_napi_add(priv
->netdev
, &rd
->napi
,
2115 hns_nic_common_poll
, NAPI_POLL_WEIGHT
);
2116 rd
->ring
->irq_init_flag
= RCB_IRQ_NOT_INITED
;
2118 for (i
= h
->q_num
; i
< h
->q_num
* 2; i
++) {
2119 rd
= &priv
->ring_data
[i
];
2120 rd
->queue_index
= i
- h
->q_num
;
2121 rd
->ring
= &h
->qs
[i
- h
->q_num
]->rx_ring
;
2122 rd
->poll_one
= hns_nic_rx_poll_one
;
2123 rd
->ex_process
= hns_nic_rx_up_pro
;
2124 rd
->fini_process
= is_ver1
? hns_nic_rx_fini_pro
:
2125 hns_nic_rx_fini_pro_v2
;
2127 netif_napi_add(priv
->netdev
, &rd
->napi
,
2128 hns_nic_common_poll
, NAPI_POLL_WEIGHT
);
2129 rd
->ring
->irq_init_flag
= RCB_IRQ_NOT_INITED
;
2135 static void hns_nic_uninit_ring_data(struct hns_nic_priv
*priv
)
2137 struct hnae_handle
*h
= priv
->ae_handle
;
2140 for (i
= 0; i
< h
->q_num
* 2; i
++) {
2141 netif_napi_del(&priv
->ring_data
[i
].napi
);
2142 if (priv
->ring_data
[i
].ring
->irq_init_flag
== RCB_IRQ_INITED
) {
2143 (void)irq_set_affinity_hint(
2144 priv
->ring_data
[i
].ring
->irq
,
2146 free_irq(priv
->ring_data
[i
].ring
->irq
,
2147 &priv
->ring_data
[i
]);
2150 priv
->ring_data
[i
].ring
->irq_init_flag
= RCB_IRQ_NOT_INITED
;
2152 kfree(priv
->ring_data
);
2155 static void hns_nic_set_priv_ops(struct net_device
*netdev
)
2157 struct hns_nic_priv
*priv
= netdev_priv(netdev
);
2158 struct hnae_handle
*h
= priv
->ae_handle
;
2160 if (AE_IS_VER1(priv
->enet_ver
)) {
2161 priv
->ops
.fill_desc
= fill_desc
;
2162 priv
->ops
.get_rxd_bnum
= get_rx_desc_bnum
;
2163 priv
->ops
.maybe_stop_tx
= hns_nic_maybe_stop_tx
;
2165 priv
->ops
.get_rxd_bnum
= get_v2rx_desc_bnum
;
2166 if ((netdev
->features
& NETIF_F_TSO
) ||
2167 (netdev
->features
& NETIF_F_TSO6
)) {
2168 priv
->ops
.fill_desc
= fill_tso_desc
;
2169 priv
->ops
.maybe_stop_tx
= hns_nic_maybe_stop_tso
;
2170 /* This chip only support 7*4096 */
2171 netif_set_gso_max_size(netdev
, 7 * 4096);
2173 priv
->ops
.fill_desc
= fill_v2_desc
;
2174 priv
->ops
.maybe_stop_tx
= hns_nic_maybe_stop_tx
;
2176 /* enable tso when init
2177 * control tso on/off through TSE bit in bd
2179 h
->dev
->ops
->set_tso_stats(h
, 1);
2183 static int hns_nic_try_get_ae(struct net_device
*ndev
)
2185 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
2186 struct hnae_handle
*h
;
2189 h
= hnae_get_handle(&priv
->netdev
->dev
,
2190 priv
->fwnode
, priv
->port_id
, NULL
);
2191 if (IS_ERR_OR_NULL(h
)) {
2193 dev_dbg(priv
->dev
, "has not handle, register notifier!\n");
2196 priv
->ae_handle
= h
;
2198 ret
= hns_nic_init_phy(ndev
, h
);
2200 dev_err(priv
->dev
, "probe phy device fail!\n");
2204 ret
= hns_nic_init_ring_data(priv
);
2207 goto out_init_ring_data
;
2210 hns_nic_set_priv_ops(ndev
);
2212 ret
= register_netdev(ndev
);
2214 dev_err(priv
->dev
, "probe register netdev fail!\n");
2215 goto out_reg_ndev_fail
;
2220 hns_nic_uninit_ring_data(priv
);
2221 priv
->ring_data
= NULL
;
2224 hnae_put_handle(priv
->ae_handle
);
2225 priv
->ae_handle
= NULL
;
2230 static int hns_nic_notifier_action(struct notifier_block
*nb
,
2231 unsigned long action
, void *data
)
2233 struct hns_nic_priv
*priv
=
2234 container_of(nb
, struct hns_nic_priv
, notifier_block
);
2236 assert(action
== HNAE_AE_REGISTER
);
2238 if (!hns_nic_try_get_ae(priv
->netdev
)) {
2239 hnae_unregister_notifier(&priv
->notifier_block
);
2240 priv
->notifier_block
.notifier_call
= NULL
;
2245 static int hns_nic_dev_probe(struct platform_device
*pdev
)
2247 struct device
*dev
= &pdev
->dev
;
2248 struct net_device
*ndev
;
2249 struct hns_nic_priv
*priv
;
2253 ndev
= alloc_etherdev_mq(sizeof(struct hns_nic_priv
), NIC_MAX_Q_PER_VF
);
2257 platform_set_drvdata(pdev
, ndev
);
2259 priv
= netdev_priv(ndev
);
2261 priv
->netdev
= ndev
;
2263 if (dev_of_node(dev
)) {
2264 struct device_node
*ae_node
;
2266 if (of_device_is_compatible(dev
->of_node
,
2267 "hisilicon,hns-nic-v1"))
2268 priv
->enet_ver
= AE_VERSION_1
;
2270 priv
->enet_ver
= AE_VERSION_2
;
2272 ae_node
= of_parse_phandle(dev
->of_node
, "ae-handle", 0);
2275 dev_err(dev
, "not find ae-handle\n");
2276 goto out_read_prop_fail
;
2278 priv
->fwnode
= &ae_node
->fwnode
;
2279 } else if (is_acpi_node(dev
->fwnode
)) {
2280 struct fwnode_reference_args args
;
2282 if (acpi_dev_found(hns_enet_acpi_match
[0].id
))
2283 priv
->enet_ver
= AE_VERSION_1
;
2284 else if (acpi_dev_found(hns_enet_acpi_match
[1].id
))
2285 priv
->enet_ver
= AE_VERSION_2
;
2288 goto out_read_prop_fail
;
2291 /* try to find port-idx-in-ae first */
2292 ret
= acpi_node_get_property_reference(dev
->fwnode
,
2293 "ae-handle", 0, &args
);
2295 dev_err(dev
, "not find ae-handle\n");
2296 goto out_read_prop_fail
;
2298 if (!is_acpi_device_node(args
.fwnode
)) {
2300 goto out_read_prop_fail
;
2302 priv
->fwnode
= args
.fwnode
;
2304 dev_err(dev
, "cannot read cfg data from OF or acpi\n");
2306 goto out_read_prop_fail
;
2309 ret
= device_property_read_u32(dev
, "port-idx-in-ae", &port_id
);
2311 /* only for old code compatible */
2312 ret
= device_property_read_u32(dev
, "port-id", &port_id
);
2314 goto out_read_prop_fail
;
2315 /* for old dts, we need to caculate the port offset */
2316 port_id
= port_id
< HNS_SRV_OFFSET
? port_id
+ HNS_DEBUG_OFFSET
2317 : port_id
- HNS_SRV_OFFSET
;
2319 priv
->port_id
= port_id
;
2321 hns_init_mac_addr(ndev
);
2323 ndev
->watchdog_timeo
= HNS_NIC_TX_TIMEOUT
;
2324 ndev
->priv_flags
|= IFF_UNICAST_FLT
;
2325 ndev
->netdev_ops
= &hns_nic_netdev_ops
;
2326 hns_ethtool_set_ops(ndev
);
2328 ndev
->features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2329 NETIF_F_RXCSUM
| NETIF_F_SG
| NETIF_F_GSO
|
2331 ndev
->vlan_features
|=
2332 NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
| NETIF_F_RXCSUM
;
2333 ndev
->vlan_features
|= NETIF_F_SG
| NETIF_F_GSO
| NETIF_F_GRO
;
2335 /* MTU range: 68 - 9578 (v1) or 9706 (v2) */
2336 ndev
->min_mtu
= MAC_MIN_MTU
;
2337 switch (priv
->enet_ver
) {
2339 ndev
->features
|= NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_NTUPLE
;
2340 ndev
->hw_features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2341 NETIF_F_RXCSUM
| NETIF_F_SG
| NETIF_F_GSO
|
2342 NETIF_F_GRO
| NETIF_F_TSO
| NETIF_F_TSO6
;
2343 ndev
->vlan_features
|= NETIF_F_TSO
| NETIF_F_TSO6
;
2344 ndev
->max_mtu
= MAC_MAX_MTU_V2
-
2345 (ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
);
2348 ndev
->max_mtu
= MAC_MAX_MTU
-
2349 (ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
);
2353 SET_NETDEV_DEV(ndev
, dev
);
2355 if (!dma_set_mask_and_coherent(dev
, DMA_BIT_MASK(64)))
2356 dev_dbg(dev
, "set mask to 64bit\n");
2358 dev_err(dev
, "set mask to 64bit fail!\n");
2360 /* carrier off reporting is important to ethtool even BEFORE open */
2361 netif_carrier_off(ndev
);
2363 timer_setup(&priv
->service_timer
, hns_nic_service_timer
, 0);
2364 INIT_WORK(&priv
->service_task
, hns_nic_service_task
);
2366 set_bit(NIC_STATE_SERVICE_INITED
, &priv
->state
);
2367 clear_bit(NIC_STATE_SERVICE_SCHED
, &priv
->state
);
2368 set_bit(NIC_STATE_DOWN
, &priv
->state
);
2370 if (hns_nic_try_get_ae(priv
->netdev
)) {
2371 priv
->notifier_block
.notifier_call
= hns_nic_notifier_action
;
2372 ret
= hnae_register_notifier(&priv
->notifier_block
);
2374 dev_err(dev
, "register notifier fail!\n");
2375 goto out_notify_fail
;
2377 dev_dbg(dev
, "has not handle, register notifier!\n");
2383 (void)cancel_work_sync(&priv
->service_task
);
2385 /* safe for ACPI FW */
2386 of_node_put(to_of_node(priv
->fwnode
));
2391 static int hns_nic_dev_remove(struct platform_device
*pdev
)
2393 struct net_device
*ndev
= platform_get_drvdata(pdev
);
2394 struct hns_nic_priv
*priv
= netdev_priv(ndev
);
2396 if (ndev
->reg_state
!= NETREG_UNINITIALIZED
)
2397 unregister_netdev(ndev
);
2399 if (priv
->ring_data
)
2400 hns_nic_uninit_ring_data(priv
);
2401 priv
->ring_data
= NULL
;
2404 phy_disconnect(ndev
->phydev
);
2406 if (!IS_ERR_OR_NULL(priv
->ae_handle
))
2407 hnae_put_handle(priv
->ae_handle
);
2408 priv
->ae_handle
= NULL
;
2409 if (priv
->notifier_block
.notifier_call
)
2410 hnae_unregister_notifier(&priv
->notifier_block
);
2411 priv
->notifier_block
.notifier_call
= NULL
;
2413 set_bit(NIC_STATE_REMOVING
, &priv
->state
);
2414 (void)cancel_work_sync(&priv
->service_task
);
2416 /* safe for ACPI FW */
2417 of_node_put(to_of_node(priv
->fwnode
));
2423 static const struct of_device_id hns_enet_of_match
[] = {
2424 {.compatible
= "hisilicon,hns-nic-v1",},
2425 {.compatible
= "hisilicon,hns-nic-v2",},
2429 MODULE_DEVICE_TABLE(of
, hns_enet_of_match
);
2431 static struct platform_driver hns_nic_dev_driver
= {
2434 .of_match_table
= hns_enet_of_match
,
2435 .acpi_match_table
= ACPI_PTR(hns_enet_acpi_match
),
2437 .probe
= hns_nic_dev_probe
,
2438 .remove
= hns_nic_dev_remove
,
2441 module_platform_driver(hns_nic_dev_driver
);
2443 MODULE_DESCRIPTION("HISILICON HNS Ethernet driver");
2444 MODULE_AUTHOR("Hisilicon, Inc.");
2445 MODULE_LICENSE("GPL");
2446 MODULE_ALIAS("platform:hns-nic");