1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright (c) 2016-2017 Hisilicon Limited.
4 #include <linux/dma-mapping.h>
5 #include <linux/etherdevice.h>
6 #include <linux/interrupt.h>
7 #ifdef CONFIG_RFS_ACCEL
8 #include <linux/cpu_rmap.h>
10 #include <linux/if_vlan.h>
11 #include <linux/irq.h>
13 #include <linux/ipv6.h>
14 #include <linux/module.h>
15 #include <linux/pci.h>
16 #include <linux/aer.h>
17 #include <linux/skbuff.h>
18 #include <linux/sctp.h>
20 #include <net/ip6_checksum.h>
21 #include <net/pkt_cls.h>
23 #include <net/vxlan.h>
26 #include "hns3_enet.h"
27 /* All hns3 tracepoints are defined by the include below, which
28 * must be included exactly once across the whole kernel with
29 * CREATE_TRACE_POINTS defined
31 #define CREATE_TRACE_POINTS
32 #include "hns3_trace.h"
34 #define hns3_set_field(origin, shift, val) ((origin) |= ((val) << (shift)))
35 #define hns3_tx_bd_count(S) DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)
37 #define hns3_rl_err(fmt, ...) \
39 if (net_ratelimit()) \
40 netdev_err(fmt, ##__VA_ARGS__); \
43 static void hns3_clear_all_ring(struct hnae3_handle
*h
, bool force
);
45 static const char hns3_driver_name
[] = "hns3";
46 static const char hns3_driver_string
[] =
47 "Hisilicon Ethernet Network Driver for Hip08 Family";
48 static const char hns3_copyright
[] = "Copyright (c) 2017 Huawei Corporation.";
49 static struct hnae3_client client
;
51 static int debug
= -1;
52 module_param(debug
, int, 0);
53 MODULE_PARM_DESC(debug
, " Network interface message level setting");
55 #define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \
56 NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)
58 #define HNS3_INNER_VLAN_TAG 1
59 #define HNS3_OUTER_VLAN_TAG 2
61 #define HNS3_MIN_TX_LEN 33U
63 /* hns3_pci_tbl - PCI Device ID Table
65 * Last entry must be all 0s
67 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
68 * Class, Class Mask, private data (not used) }
70 static const struct pci_device_id hns3_pci_tbl
[] = {
71 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_GE
), 0},
72 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_25GE
), 0},
73 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_25GE_RDMA
),
74 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS
},
75 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_25GE_RDMA_MACSEC
),
76 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS
},
77 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_50GE_RDMA
),
78 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS
},
79 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_50GE_RDMA_MACSEC
),
80 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS
},
81 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_100G_RDMA_MACSEC
),
82 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS
},
83 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_100G_VF
), 0},
84 {PCI_VDEVICE(HUAWEI
, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF
),
85 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS
},
86 /* required last entry */
89 MODULE_DEVICE_TABLE(pci
, hns3_pci_tbl
);
91 static irqreturn_t
hns3_irq_handle(int irq
, void *vector
)
93 struct hns3_enet_tqp_vector
*tqp_vector
= vector
;
95 napi_schedule_irqoff(&tqp_vector
->napi
);
100 static void hns3_nic_uninit_irq(struct hns3_nic_priv
*priv
)
102 struct hns3_enet_tqp_vector
*tqp_vectors
;
105 for (i
= 0; i
< priv
->vector_num
; i
++) {
106 tqp_vectors
= &priv
->tqp_vector
[i
];
108 if (tqp_vectors
->irq_init_flag
!= HNS3_VECTOR_INITED
)
111 /* clear the affinity mask */
112 irq_set_affinity_hint(tqp_vectors
->vector_irq
, NULL
);
114 /* release the irq resource */
115 free_irq(tqp_vectors
->vector_irq
, tqp_vectors
);
116 tqp_vectors
->irq_init_flag
= HNS3_VECTOR_NOT_INITED
;
120 static int hns3_nic_init_irq(struct hns3_nic_priv
*priv
)
122 struct hns3_enet_tqp_vector
*tqp_vectors
;
123 int txrx_int_idx
= 0;
129 for (i
= 0; i
< priv
->vector_num
; i
++) {
130 tqp_vectors
= &priv
->tqp_vector
[i
];
132 if (tqp_vectors
->irq_init_flag
== HNS3_VECTOR_INITED
)
135 if (tqp_vectors
->tx_group
.ring
&& tqp_vectors
->rx_group
.ring
) {
136 snprintf(tqp_vectors
->name
, HNAE3_INT_NAME_LEN
,
137 "%s-%s-%s-%d", hns3_driver_name
,
138 pci_name(priv
->ae_handle
->pdev
),
139 "TxRx", txrx_int_idx
++);
141 } else if (tqp_vectors
->rx_group
.ring
) {
142 snprintf(tqp_vectors
->name
, HNAE3_INT_NAME_LEN
,
143 "%s-%s-%s-%d", hns3_driver_name
,
144 pci_name(priv
->ae_handle
->pdev
),
146 } else if (tqp_vectors
->tx_group
.ring
) {
147 snprintf(tqp_vectors
->name
, HNAE3_INT_NAME_LEN
,
148 "%s-%s-%s-%d", hns3_driver_name
,
149 pci_name(priv
->ae_handle
->pdev
),
152 /* Skip this unused q_vector */
156 tqp_vectors
->name
[HNAE3_INT_NAME_LEN
- 1] = '\0';
158 irq_set_status_flags(tqp_vectors
->vector_irq
, IRQ_NOAUTOEN
);
159 ret
= request_irq(tqp_vectors
->vector_irq
, hns3_irq_handle
, 0,
160 tqp_vectors
->name
, tqp_vectors
);
162 netdev_err(priv
->netdev
, "request irq(%d) fail\n",
163 tqp_vectors
->vector_irq
);
164 hns3_nic_uninit_irq(priv
);
168 irq_set_affinity_hint(tqp_vectors
->vector_irq
,
169 &tqp_vectors
->affinity_mask
);
171 tqp_vectors
->irq_init_flag
= HNS3_VECTOR_INITED
;
177 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector
*tqp_vector
,
180 writel(mask_en
, tqp_vector
->mask_addr
);
183 static void hns3_vector_enable(struct hns3_enet_tqp_vector
*tqp_vector
)
185 napi_enable(&tqp_vector
->napi
);
186 enable_irq(tqp_vector
->vector_irq
);
189 hns3_mask_vector_irq(tqp_vector
, 1);
192 static void hns3_vector_disable(struct hns3_enet_tqp_vector
*tqp_vector
)
195 hns3_mask_vector_irq(tqp_vector
, 0);
197 disable_irq(tqp_vector
->vector_irq
);
198 napi_disable(&tqp_vector
->napi
);
201 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector
*tqp_vector
,
204 u32 rl_reg
= hns3_rl_usec_to_reg(rl_value
);
206 /* this defines the configuration for RL (Interrupt Rate Limiter).
207 * Rl defines rate of interrupts i.e. number of interrupts-per-second
208 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
211 if (rl_reg
> 0 && !tqp_vector
->tx_group
.coal
.gl_adapt_enable
&&
212 !tqp_vector
->rx_group
.coal
.gl_adapt_enable
)
213 /* According to the hardware, the range of rl_reg is
214 * 0-59 and the unit is 4.
216 rl_reg
|= HNS3_INT_RL_ENABLE_MASK
;
218 writel(rl_reg
, tqp_vector
->mask_addr
+ HNS3_VECTOR_RL_OFFSET
);
221 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector
*tqp_vector
,
224 u32 rx_gl_reg
= hns3_gl_usec_to_reg(gl_value
);
226 writel(rx_gl_reg
, tqp_vector
->mask_addr
+ HNS3_VECTOR_GL0_OFFSET
);
229 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector
*tqp_vector
,
232 u32 tx_gl_reg
= hns3_gl_usec_to_reg(gl_value
);
234 writel(tx_gl_reg
, tqp_vector
->mask_addr
+ HNS3_VECTOR_GL1_OFFSET
);
237 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector
*tqp_vector
,
238 struct hns3_nic_priv
*priv
)
240 /* initialize the configuration for interrupt coalescing.
241 * 1. GL (Interrupt Gap Limiter)
242 * 2. RL (Interrupt Rate Limiter)
244 * Default: enable interrupt coalescing self-adaptive and GL
246 tqp_vector
->tx_group
.coal
.gl_adapt_enable
= 1;
247 tqp_vector
->rx_group
.coal
.gl_adapt_enable
= 1;
249 tqp_vector
->tx_group
.coal
.int_gl
= HNS3_INT_GL_50K
;
250 tqp_vector
->rx_group
.coal
.int_gl
= HNS3_INT_GL_50K
;
252 tqp_vector
->rx_group
.coal
.flow_level
= HNS3_FLOW_LOW
;
253 tqp_vector
->tx_group
.coal
.flow_level
= HNS3_FLOW_LOW
;
256 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector
*tqp_vector
,
257 struct hns3_nic_priv
*priv
)
259 struct hnae3_handle
*h
= priv
->ae_handle
;
261 hns3_set_vector_coalesce_tx_gl(tqp_vector
,
262 tqp_vector
->tx_group
.coal
.int_gl
);
263 hns3_set_vector_coalesce_rx_gl(tqp_vector
,
264 tqp_vector
->rx_group
.coal
.int_gl
);
265 hns3_set_vector_coalesce_rl(tqp_vector
, h
->kinfo
.int_rl_setting
);
268 static int hns3_nic_set_real_num_queue(struct net_device
*netdev
)
270 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
271 struct hnae3_knic_private_info
*kinfo
= &h
->kinfo
;
272 unsigned int queue_size
= kinfo
->rss_size
* kinfo
->num_tc
;
275 if (kinfo
->num_tc
<= 1) {
276 netdev_reset_tc(netdev
);
278 ret
= netdev_set_num_tc(netdev
, kinfo
->num_tc
);
281 "netdev_set_num_tc fail, ret=%d!\n", ret
);
285 for (i
= 0; i
< HNAE3_MAX_TC
; i
++) {
286 if (!kinfo
->tc_info
[i
].enable
)
289 netdev_set_tc_queue(netdev
,
290 kinfo
->tc_info
[i
].tc
,
291 kinfo
->tc_info
[i
].tqp_count
,
292 kinfo
->tc_info
[i
].tqp_offset
);
296 ret
= netif_set_real_num_tx_queues(netdev
, queue_size
);
299 "netif_set_real_num_tx_queues fail, ret=%d!\n", ret
);
303 ret
= netif_set_real_num_rx_queues(netdev
, queue_size
);
306 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret
);
313 static u16
hns3_get_max_available_channels(struct hnae3_handle
*h
)
315 u16 alloc_tqps
, max_rss_size
, rss_size
;
317 h
->ae_algo
->ops
->get_tqps_and_rss_info(h
, &alloc_tqps
, &max_rss_size
);
318 rss_size
= alloc_tqps
/ h
->kinfo
.num_tc
;
320 return min_t(u16
, rss_size
, max_rss_size
);
323 static void hns3_tqp_enable(struct hnae3_queue
*tqp
)
327 rcb_reg
= hns3_read_dev(tqp
, HNS3_RING_EN_REG
);
328 rcb_reg
|= BIT(HNS3_RING_EN_B
);
329 hns3_write_dev(tqp
, HNS3_RING_EN_REG
, rcb_reg
);
332 static void hns3_tqp_disable(struct hnae3_queue
*tqp
)
336 rcb_reg
= hns3_read_dev(tqp
, HNS3_RING_EN_REG
);
337 rcb_reg
&= ~BIT(HNS3_RING_EN_B
);
338 hns3_write_dev(tqp
, HNS3_RING_EN_REG
, rcb_reg
);
341 static void hns3_free_rx_cpu_rmap(struct net_device
*netdev
)
343 #ifdef CONFIG_RFS_ACCEL
344 free_irq_cpu_rmap(netdev
->rx_cpu_rmap
);
345 netdev
->rx_cpu_rmap
= NULL
;
349 static int hns3_set_rx_cpu_rmap(struct net_device
*netdev
)
351 #ifdef CONFIG_RFS_ACCEL
352 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
353 struct hns3_enet_tqp_vector
*tqp_vector
;
356 if (!netdev
->rx_cpu_rmap
) {
357 netdev
->rx_cpu_rmap
= alloc_irq_cpu_rmap(priv
->vector_num
);
358 if (!netdev
->rx_cpu_rmap
)
362 for (i
= 0; i
< priv
->vector_num
; i
++) {
363 tqp_vector
= &priv
->tqp_vector
[i
];
364 ret
= irq_cpu_rmap_add(netdev
->rx_cpu_rmap
,
365 tqp_vector
->vector_irq
);
367 hns3_free_rx_cpu_rmap(netdev
);
375 static int hns3_nic_net_up(struct net_device
*netdev
)
377 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
378 struct hnae3_handle
*h
= priv
->ae_handle
;
382 ret
= hns3_nic_reset_all_ring(h
);
386 clear_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
);
388 /* enable the vectors */
389 for (i
= 0; i
< priv
->vector_num
; i
++)
390 hns3_vector_enable(&priv
->tqp_vector
[i
]);
393 for (j
= 0; j
< h
->kinfo
.num_tqps
; j
++)
394 hns3_tqp_enable(h
->kinfo
.tqp
[j
]);
396 /* start the ae_dev */
397 ret
= h
->ae_algo
->ops
->start
? h
->ae_algo
->ops
->start(h
) : 0;
399 set_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
);
401 hns3_tqp_disable(h
->kinfo
.tqp
[j
]);
403 for (j
= i
- 1; j
>= 0; j
--)
404 hns3_vector_disable(&priv
->tqp_vector
[j
]);
410 static void hns3_config_xps(struct hns3_nic_priv
*priv
)
414 for (i
= 0; i
< priv
->vector_num
; i
++) {
415 struct hns3_enet_tqp_vector
*tqp_vector
= &priv
->tqp_vector
[i
];
416 struct hns3_enet_ring
*ring
= tqp_vector
->tx_group
.ring
;
421 ret
= netif_set_xps_queue(priv
->netdev
,
422 &tqp_vector
->affinity_mask
,
423 ring
->tqp
->tqp_index
);
425 netdev_warn(priv
->netdev
,
426 "set xps queue failed: %d", ret
);
433 static int hns3_nic_net_open(struct net_device
*netdev
)
435 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
436 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
437 struct hnae3_knic_private_info
*kinfo
;
440 if (hns3_nic_resetting(netdev
))
443 netif_carrier_off(netdev
);
445 ret
= hns3_nic_set_real_num_queue(netdev
);
449 ret
= hns3_nic_net_up(netdev
);
451 netdev_err(netdev
, "net up fail, ret=%d!\n", ret
);
456 for (i
= 0; i
< HNAE3_MAX_USER_PRIO
; i
++)
457 netdev_set_prio_tc_map(netdev
, i
, kinfo
->prio_tc
[i
]);
459 if (h
->ae_algo
->ops
->set_timer_task
)
460 h
->ae_algo
->ops
->set_timer_task(priv
->ae_handle
, true);
462 hns3_config_xps(priv
);
464 netif_dbg(h
, drv
, netdev
, "net open\n");
469 static void hns3_reset_tx_queue(struct hnae3_handle
*h
)
471 struct net_device
*ndev
= h
->kinfo
.netdev
;
472 struct hns3_nic_priv
*priv
= netdev_priv(ndev
);
473 struct netdev_queue
*dev_queue
;
476 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++) {
477 dev_queue
= netdev_get_tx_queue(ndev
,
478 priv
->ring
[i
].queue_index
);
479 netdev_tx_reset_queue(dev_queue
);
483 static void hns3_nic_net_down(struct net_device
*netdev
)
485 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
486 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
487 const struct hnae3_ae_ops
*ops
;
490 /* disable vectors */
491 for (i
= 0; i
< priv
->vector_num
; i
++)
492 hns3_vector_disable(&priv
->tqp_vector
[i
]);
495 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++)
496 hns3_tqp_disable(h
->kinfo
.tqp
[i
]);
499 ops
= priv
->ae_handle
->ae_algo
->ops
;
501 ops
->stop(priv
->ae_handle
);
503 /* delay ring buffer clearing to hns3_reset_notify_uninit_enet
504 * during reset process, because driver may not be able
505 * to disable the ring through firmware when downing the netdev.
507 if (!hns3_nic_resetting(netdev
))
508 hns3_clear_all_ring(priv
->ae_handle
, false);
510 hns3_reset_tx_queue(priv
->ae_handle
);
513 static int hns3_nic_net_stop(struct net_device
*netdev
)
515 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
516 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
518 if (test_and_set_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
))
521 netif_dbg(h
, drv
, netdev
, "net stop\n");
523 if (h
->ae_algo
->ops
->set_timer_task
)
524 h
->ae_algo
->ops
->set_timer_task(priv
->ae_handle
, false);
526 netif_tx_stop_all_queues(netdev
);
527 netif_carrier_off(netdev
);
529 hns3_nic_net_down(netdev
);
534 static int hns3_nic_uc_sync(struct net_device
*netdev
,
535 const unsigned char *addr
)
537 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
539 if (h
->ae_algo
->ops
->add_uc_addr
)
540 return h
->ae_algo
->ops
->add_uc_addr(h
, addr
);
545 static int hns3_nic_uc_unsync(struct net_device
*netdev
,
546 const unsigned char *addr
)
548 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
550 /* need ignore the request of removing device address, because
551 * we store the device address and other addresses of uc list
552 * in the function's mac filter list.
554 if (ether_addr_equal(addr
, netdev
->dev_addr
))
557 if (h
->ae_algo
->ops
->rm_uc_addr
)
558 return h
->ae_algo
->ops
->rm_uc_addr(h
, addr
);
563 static int hns3_nic_mc_sync(struct net_device
*netdev
,
564 const unsigned char *addr
)
566 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
568 if (h
->ae_algo
->ops
->add_mc_addr
)
569 return h
->ae_algo
->ops
->add_mc_addr(h
, addr
);
574 static int hns3_nic_mc_unsync(struct net_device
*netdev
,
575 const unsigned char *addr
)
577 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
579 if (h
->ae_algo
->ops
->rm_mc_addr
)
580 return h
->ae_algo
->ops
->rm_mc_addr(h
, addr
);
585 static u8
hns3_get_netdev_flags(struct net_device
*netdev
)
589 if (netdev
->flags
& IFF_PROMISC
) {
590 flags
= HNAE3_USER_UPE
| HNAE3_USER_MPE
| HNAE3_BPE
;
592 flags
|= HNAE3_VLAN_FLTR
;
593 if (netdev
->flags
& IFF_ALLMULTI
)
594 flags
|= HNAE3_USER_MPE
;
600 static void hns3_nic_set_rx_mode(struct net_device
*netdev
)
602 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
605 new_flags
= hns3_get_netdev_flags(netdev
);
607 __dev_uc_sync(netdev
, hns3_nic_uc_sync
, hns3_nic_uc_unsync
);
608 __dev_mc_sync(netdev
, hns3_nic_mc_sync
, hns3_nic_mc_unsync
);
610 /* User mode Promisc mode enable and vlan filtering is disabled to
611 * let all packets in.
613 h
->netdev_flags
= new_flags
;
614 hns3_request_update_promisc_mode(h
);
617 void hns3_request_update_promisc_mode(struct hnae3_handle
*handle
)
619 const struct hnae3_ae_ops
*ops
= handle
->ae_algo
->ops
;
621 if (ops
->request_update_promisc_mode
)
622 ops
->request_update_promisc_mode(handle
);
625 int hns3_update_promisc_mode(struct net_device
*netdev
, u8 promisc_flags
)
627 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
628 struct hnae3_handle
*h
= priv
->ae_handle
;
630 if (h
->ae_algo
->ops
->set_promisc_mode
) {
631 return h
->ae_algo
->ops
->set_promisc_mode(h
,
632 promisc_flags
& HNAE3_UPE
,
633 promisc_flags
& HNAE3_MPE
);
639 void hns3_enable_vlan_filter(struct net_device
*netdev
, bool enable
)
641 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
642 struct hnae3_handle
*h
= priv
->ae_handle
;
645 if (h
->pdev
->revision
>= 0x21 && h
->ae_algo
->ops
->enable_vlan_filter
) {
646 last_state
= h
->netdev_flags
& HNAE3_VLAN_FLTR
? true : false;
647 if (enable
!= last_state
) {
650 enable
? "enable" : "disable");
651 h
->ae_algo
->ops
->enable_vlan_filter(h
, enable
);
656 static int hns3_set_tso(struct sk_buff
*skb
, u32
*paylen
,
657 u16
*mss
, u32
*type_cs_vlan_tso
)
659 u32 l4_offset
, hdr_len
;
660 union l3_hdr_info l3
;
661 union l4_hdr_info l4
;
665 if (!skb_is_gso(skb
))
668 ret
= skb_cow_head(skb
, 0);
669 if (unlikely(ret
< 0))
672 l3
.hdr
= skb_network_header(skb
);
673 l4
.hdr
= skb_transport_header(skb
);
675 /* Software should clear the IPv4's checksum field when tso is
678 if (l3
.v4
->version
== 4)
682 if (skb_shinfo(skb
)->gso_type
& (SKB_GSO_GRE
|
685 SKB_GSO_UDP_TUNNEL_CSUM
)) {
686 if ((!(skb_shinfo(skb
)->gso_type
&
688 (skb_shinfo(skb
)->gso_type
&
689 SKB_GSO_UDP_TUNNEL_CSUM
)) {
690 /* Software should clear the udp's checksum
691 * field when tso is needed.
695 /* reset l3&l4 pointers from outer to inner headers */
696 l3
.hdr
= skb_inner_network_header(skb
);
697 l4
.hdr
= skb_inner_transport_header(skb
);
699 /* Software should clear the IPv4's checksum field when
702 if (l3
.v4
->version
== 4)
706 /* normal or tunnel packet */
707 l4_offset
= l4
.hdr
- skb
->data
;
708 hdr_len
= (l4
.tcp
->doff
<< 2) + l4_offset
;
710 /* remove payload length from inner pseudo checksum when tso */
711 l4_paylen
= skb
->len
- l4_offset
;
712 csum_replace_by_diff(&l4
.tcp
->check
,
713 (__force __wsum
)htonl(l4_paylen
));
715 /* find the txbd field values */
716 *paylen
= skb
->len
- hdr_len
;
717 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_TSO_B
, 1);
719 /* get MSS for TSO */
720 *mss
= skb_shinfo(skb
)->gso_size
;
727 static int hns3_get_l4_protocol(struct sk_buff
*skb
, u8
*ol4_proto
,
730 union l3_hdr_info l3
;
731 unsigned char *l4_hdr
;
732 unsigned char *exthdr
;
736 /* find outer header point */
737 l3
.hdr
= skb_network_header(skb
);
738 l4_hdr
= skb_transport_header(skb
);
740 if (skb
->protocol
== htons(ETH_P_IPV6
)) {
741 exthdr
= l3
.hdr
+ sizeof(*l3
.v6
);
742 l4_proto_tmp
= l3
.v6
->nexthdr
;
743 if (l4_hdr
!= exthdr
)
744 ipv6_skip_exthdr(skb
, exthdr
- skb
->data
,
745 &l4_proto_tmp
, &frag_off
);
746 } else if (skb
->protocol
== htons(ETH_P_IP
)) {
747 l4_proto_tmp
= l3
.v4
->protocol
;
752 *ol4_proto
= l4_proto_tmp
;
755 if (!skb
->encapsulation
) {
760 /* find inner header point */
761 l3
.hdr
= skb_inner_network_header(skb
);
762 l4_hdr
= skb_inner_transport_header(skb
);
764 if (l3
.v6
->version
== 6) {
765 exthdr
= l3
.hdr
+ sizeof(*l3
.v6
);
766 l4_proto_tmp
= l3
.v6
->nexthdr
;
767 if (l4_hdr
!= exthdr
)
768 ipv6_skip_exthdr(skb
, exthdr
- skb
->data
,
769 &l4_proto_tmp
, &frag_off
);
770 } else if (l3
.v4
->version
== 4) {
771 l4_proto_tmp
= l3
.v4
->protocol
;
774 *il4_proto
= l4_proto_tmp
;
779 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
780 * and it is udp packet, which has a dest port as the IANA assigned.
781 * the hardware is expected to do the checksum offload, but the
782 * hardware will not do the checksum offload when udp dest port is
785 static bool hns3_tunnel_csum_bug(struct sk_buff
*skb
)
787 union l4_hdr_info l4
;
789 l4
.hdr
= skb_transport_header(skb
);
791 if (!(!skb
->encapsulation
&&
792 l4
.udp
->dest
== htons(IANA_VXLAN_UDP_PORT
)))
795 skb_checksum_help(skb
);
800 static void hns3_set_outer_l2l3l4(struct sk_buff
*skb
, u8 ol4_proto
,
801 u32
*ol_type_vlan_len_msec
)
803 u32 l2_len
, l3_len
, l4_len
;
804 unsigned char *il2_hdr
;
805 union l3_hdr_info l3
;
806 union l4_hdr_info l4
;
808 l3
.hdr
= skb_network_header(skb
);
809 l4
.hdr
= skb_transport_header(skb
);
811 /* compute OL2 header size, defined in 2 Bytes */
812 l2_len
= l3
.hdr
- skb
->data
;
813 hns3_set_field(*ol_type_vlan_len_msec
, HNS3_TXD_L2LEN_S
, l2_len
>> 1);
815 /* compute OL3 header size, defined in 4 Bytes */
816 l3_len
= l4
.hdr
- l3
.hdr
;
817 hns3_set_field(*ol_type_vlan_len_msec
, HNS3_TXD_L3LEN_S
, l3_len
>> 2);
819 il2_hdr
= skb_inner_mac_header(skb
);
820 /* compute OL4 header size, defined in 4 Bytes */
821 l4_len
= il2_hdr
- l4
.hdr
;
822 hns3_set_field(*ol_type_vlan_len_msec
, HNS3_TXD_L4LEN_S
, l4_len
>> 2);
824 /* define outer network header type */
825 if (skb
->protocol
== htons(ETH_P_IP
)) {
827 hns3_set_field(*ol_type_vlan_len_msec
,
829 HNS3_OL3T_IPV4_CSUM
);
831 hns3_set_field(*ol_type_vlan_len_msec
,
833 HNS3_OL3T_IPV4_NO_CSUM
);
835 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
836 hns3_set_field(*ol_type_vlan_len_msec
, HNS3_TXD_OL3T_S
,
840 if (ol4_proto
== IPPROTO_UDP
)
841 hns3_set_field(*ol_type_vlan_len_msec
, HNS3_TXD_TUNTYPE_S
,
842 HNS3_TUN_MAC_IN_UDP
);
843 else if (ol4_proto
== IPPROTO_GRE
)
844 hns3_set_field(*ol_type_vlan_len_msec
, HNS3_TXD_TUNTYPE_S
,
848 static int hns3_set_l2l3l4(struct sk_buff
*skb
, u8 ol4_proto
,
849 u8 il4_proto
, u32
*type_cs_vlan_tso
,
850 u32
*ol_type_vlan_len_msec
)
852 unsigned char *l2_hdr
= skb
->data
;
853 u32 l4_proto
= ol4_proto
;
854 union l4_hdr_info l4
;
855 union l3_hdr_info l3
;
858 l4
.hdr
= skb_transport_header(skb
);
859 l3
.hdr
= skb_network_header(skb
);
861 /* handle encapsulation skb */
862 if (skb
->encapsulation
) {
863 /* If this is a not UDP/GRE encapsulation skb */
864 if (!(ol4_proto
== IPPROTO_UDP
|| ol4_proto
== IPPROTO_GRE
)) {
865 /* drop the skb tunnel packet if hardware don't support,
866 * because hardware can't calculate csum when TSO.
871 /* the stack computes the IP header already,
872 * driver calculate l4 checksum when not TSO.
874 skb_checksum_help(skb
);
878 hns3_set_outer_l2l3l4(skb
, ol4_proto
, ol_type_vlan_len_msec
);
880 /* switch to inner header */
881 l2_hdr
= skb_inner_mac_header(skb
);
882 l3
.hdr
= skb_inner_network_header(skb
);
883 l4
.hdr
= skb_inner_transport_header(skb
);
884 l4_proto
= il4_proto
;
887 if (l3
.v4
->version
== 4) {
888 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L3T_S
,
891 /* the stack computes the IP header already, the only time we
892 * need the hardware to recompute it is in the case of TSO.
895 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L3CS_B
, 1);
896 } else if (l3
.v6
->version
== 6) {
897 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L3T_S
,
901 /* compute inner(/normal) L2 header size, defined in 2 Bytes */
902 l2_len
= l3
.hdr
- l2_hdr
;
903 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L2LEN_S
, l2_len
>> 1);
905 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
906 l3_len
= l4
.hdr
- l3
.hdr
;
907 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L3LEN_S
, l3_len
>> 2);
909 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
912 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4CS_B
, 1);
913 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4T_S
,
915 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4LEN_S
,
919 if (hns3_tunnel_csum_bug(skb
))
922 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4CS_B
, 1);
923 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4T_S
,
925 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4LEN_S
,
926 (sizeof(struct udphdr
) >> 2));
929 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4CS_B
, 1);
930 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4T_S
,
932 hns3_set_field(*type_cs_vlan_tso
, HNS3_TXD_L4LEN_S
,
933 (sizeof(struct sctphdr
) >> 2));
936 /* drop the skb tunnel packet if hardware don't support,
937 * because hardware can't calculate csum when TSO.
942 /* the stack computes the IP header already,
943 * driver calculate l4 checksum when not TSO.
945 skb_checksum_help(skb
);
952 static int hns3_handle_vtags(struct hns3_enet_ring
*tx_ring
,
955 struct hnae3_handle
*handle
= tx_ring
->tqp
->handle
;
956 struct vlan_ethhdr
*vhdr
;
959 if (!(skb
->protocol
== htons(ETH_P_8021Q
) ||
960 skb_vlan_tag_present(skb
)))
963 /* Since HW limitation, if port based insert VLAN enabled, only one VLAN
964 * header is allowed in skb, otherwise it will cause RAS error.
966 if (unlikely(skb_vlan_tagged_multi(skb
) &&
967 handle
->port_base_vlan_state
==
968 HNAE3_PORT_BASE_VLAN_ENABLE
))
971 if (skb
->protocol
== htons(ETH_P_8021Q
) &&
972 !(handle
->kinfo
.netdev
->features
& NETIF_F_HW_VLAN_CTAG_TX
)) {
973 /* When HW VLAN acceleration is turned off, and the stack
974 * sets the protocol to 802.1q, the driver just need to
975 * set the protocol to the encapsulated ethertype.
977 skb
->protocol
= vlan_get_protocol(skb
);
981 if (skb_vlan_tag_present(skb
)) {
982 /* Based on hw strategy, use out_vtag in two layer tag case,
983 * and use inner_vtag in one tag case.
985 if (skb
->protocol
== htons(ETH_P_8021Q
) &&
986 handle
->port_base_vlan_state
==
987 HNAE3_PORT_BASE_VLAN_DISABLE
)
988 rc
= HNS3_OUTER_VLAN_TAG
;
990 rc
= HNS3_INNER_VLAN_TAG
;
992 skb
->protocol
= vlan_get_protocol(skb
);
996 rc
= skb_cow_head(skb
, 0);
997 if (unlikely(rc
< 0))
1000 vhdr
= (struct vlan_ethhdr
*)skb
->data
;
1001 vhdr
->h_vlan_TCI
|= cpu_to_be16((skb
->priority
<< VLAN_PRIO_SHIFT
)
1004 skb
->protocol
= vlan_get_protocol(skb
);
1008 static int hns3_fill_skb_desc(struct hns3_enet_ring
*ring
,
1009 struct sk_buff
*skb
, struct hns3_desc
*desc
)
1011 u32 ol_type_vlan_len_msec
= 0;
1012 u32 type_cs_vlan_tso
= 0;
1013 u32 paylen
= skb
->len
;
1019 ret
= hns3_handle_vtags(ring
, skb
);
1020 if (unlikely(ret
< 0)) {
1021 u64_stats_update_begin(&ring
->syncp
);
1022 ring
->stats
.tx_vlan_err
++;
1023 u64_stats_update_end(&ring
->syncp
);
1025 } else if (ret
== HNS3_INNER_VLAN_TAG
) {
1026 inner_vtag
= skb_vlan_tag_get(skb
);
1027 inner_vtag
|= (skb
->priority
<< VLAN_PRIO_SHIFT
) &
1029 hns3_set_field(type_cs_vlan_tso
, HNS3_TXD_VLAN_B
, 1);
1030 } else if (ret
== HNS3_OUTER_VLAN_TAG
) {
1031 out_vtag
= skb_vlan_tag_get(skb
);
1032 out_vtag
|= (skb
->priority
<< VLAN_PRIO_SHIFT
) &
1034 hns3_set_field(ol_type_vlan_len_msec
, HNS3_TXD_OVLAN_B
,
1038 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1039 u8 ol4_proto
, il4_proto
;
1041 skb_reset_mac_len(skb
);
1043 ret
= hns3_get_l4_protocol(skb
, &ol4_proto
, &il4_proto
);
1044 if (unlikely(ret
< 0)) {
1045 u64_stats_update_begin(&ring
->syncp
);
1046 ring
->stats
.tx_l4_proto_err
++;
1047 u64_stats_update_end(&ring
->syncp
);
1051 ret
= hns3_set_l2l3l4(skb
, ol4_proto
, il4_proto
,
1053 &ol_type_vlan_len_msec
);
1054 if (unlikely(ret
< 0)) {
1055 u64_stats_update_begin(&ring
->syncp
);
1056 ring
->stats
.tx_l2l3l4_err
++;
1057 u64_stats_update_end(&ring
->syncp
);
1061 ret
= hns3_set_tso(skb
, &paylen
, &mss
,
1063 if (unlikely(ret
< 0)) {
1064 u64_stats_update_begin(&ring
->syncp
);
1065 ring
->stats
.tx_tso_err
++;
1066 u64_stats_update_end(&ring
->syncp
);
1072 desc
->tx
.ol_type_vlan_len_msec
=
1073 cpu_to_le32(ol_type_vlan_len_msec
);
1074 desc
->tx
.type_cs_vlan_tso_len
= cpu_to_le32(type_cs_vlan_tso
);
1075 desc
->tx
.paylen
= cpu_to_le32(paylen
);
1076 desc
->tx
.mss
= cpu_to_le16(mss
);
1077 desc
->tx
.vlan_tag
= cpu_to_le16(inner_vtag
);
1078 desc
->tx
.outer_vlan_tag
= cpu_to_le16(out_vtag
);
1083 static int hns3_fill_desc(struct hns3_enet_ring
*ring
, void *priv
,
1084 unsigned int size
, enum hns_desc_type type
)
1086 #define HNS3_LIKELY_BD_NUM 1
1088 struct hns3_desc_cb
*desc_cb
= &ring
->desc_cb
[ring
->next_to_use
];
1089 struct hns3_desc
*desc
= &ring
->desc
[ring
->next_to_use
];
1090 struct device
*dev
= ring_to_dev(ring
);
1092 unsigned int frag_buf_num
;
1096 if (type
== DESC_TYPE_FRAGLIST_SKB
||
1097 type
== DESC_TYPE_SKB
) {
1098 struct sk_buff
*skb
= (struct sk_buff
*)priv
;
1100 dma
= dma_map_single(dev
, skb
->data
, size
, DMA_TO_DEVICE
);
1102 frag
= (skb_frag_t
*)priv
;
1103 dma
= skb_frag_dma_map(dev
, frag
, 0, size
, DMA_TO_DEVICE
);
1106 if (unlikely(dma_mapping_error(dev
, dma
))) {
1107 u64_stats_update_begin(&ring
->syncp
);
1108 ring
->stats
.sw_err_cnt
++;
1109 u64_stats_update_end(&ring
->syncp
);
1113 desc_cb
->priv
= priv
;
1114 desc_cb
->length
= size
;
1116 desc_cb
->type
= type
;
1118 if (likely(size
<= HNS3_MAX_BD_SIZE
)) {
1119 desc
->addr
= cpu_to_le64(dma
);
1120 desc
->tx
.send_size
= cpu_to_le16(size
);
1121 desc
->tx
.bdtp_fe_sc_vld_ra_ri
=
1122 cpu_to_le16(BIT(HNS3_TXD_VLD_B
));
1124 trace_hns3_tx_desc(ring
, ring
->next_to_use
);
1125 ring_ptr_move_fw(ring
, next_to_use
);
1126 return HNS3_LIKELY_BD_NUM
;
1129 frag_buf_num
= hns3_tx_bd_count(size
);
1130 sizeoflast
= size
% HNS3_MAX_BD_SIZE
;
1131 sizeoflast
= sizeoflast
? sizeoflast
: HNS3_MAX_BD_SIZE
;
1133 /* When frag size is bigger than hardware limit, split this frag */
1134 for (k
= 0; k
< frag_buf_num
; k
++) {
1135 /* now, fill the descriptor */
1136 desc
->addr
= cpu_to_le64(dma
+ HNS3_MAX_BD_SIZE
* k
);
1137 desc
->tx
.send_size
= cpu_to_le16((k
== frag_buf_num
- 1) ?
1138 (u16
)sizeoflast
: (u16
)HNS3_MAX_BD_SIZE
);
1139 desc
->tx
.bdtp_fe_sc_vld_ra_ri
=
1140 cpu_to_le16(BIT(HNS3_TXD_VLD_B
));
1142 trace_hns3_tx_desc(ring
, ring
->next_to_use
);
1143 /* move ring pointer to next */
1144 ring_ptr_move_fw(ring
, next_to_use
);
1146 desc
= &ring
->desc
[ring
->next_to_use
];
1149 return frag_buf_num
;
1152 static unsigned int hns3_skb_bd_num(struct sk_buff
*skb
, unsigned int *bd_size
,
1153 unsigned int bd_num
)
1158 size
= skb_headlen(skb
);
1159 while (size
> HNS3_MAX_BD_SIZE
) {
1160 bd_size
[bd_num
++] = HNS3_MAX_BD_SIZE
;
1161 size
-= HNS3_MAX_BD_SIZE
;
1163 if (bd_num
> HNS3_MAX_TSO_BD_NUM
)
1168 bd_size
[bd_num
++] = size
;
1169 if (bd_num
> HNS3_MAX_TSO_BD_NUM
)
1173 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1174 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
1175 size
= skb_frag_size(frag
);
1179 while (size
> HNS3_MAX_BD_SIZE
) {
1180 bd_size
[bd_num
++] = HNS3_MAX_BD_SIZE
;
1181 size
-= HNS3_MAX_BD_SIZE
;
1183 if (bd_num
> HNS3_MAX_TSO_BD_NUM
)
1187 bd_size
[bd_num
++] = size
;
1188 if (bd_num
> HNS3_MAX_TSO_BD_NUM
)
1195 static unsigned int hns3_tx_bd_num(struct sk_buff
*skb
, unsigned int *bd_size
)
1197 struct sk_buff
*frag_skb
;
1198 unsigned int bd_num
= 0;
1200 /* If the total len is within the max bd limit */
1201 if (likely(skb
->len
<= HNS3_MAX_BD_SIZE
&& !skb_has_frag_list(skb
) &&
1202 skb_shinfo(skb
)->nr_frags
< HNS3_MAX_NON_TSO_BD_NUM
))
1203 return skb_shinfo(skb
)->nr_frags
+ 1U;
1205 /* The below case will always be linearized, return
1206 * HNS3_MAX_BD_NUM_TSO + 1U to make sure it is linearized.
1208 if (unlikely(skb
->len
> HNS3_MAX_TSO_SIZE
||
1209 (!skb_is_gso(skb
) && skb
->len
> HNS3_MAX_NON_TSO_SIZE
)))
1210 return HNS3_MAX_TSO_BD_NUM
+ 1U;
1212 bd_num
= hns3_skb_bd_num(skb
, bd_size
, bd_num
);
1214 if (!skb_has_frag_list(skb
) || bd_num
> HNS3_MAX_TSO_BD_NUM
)
1217 skb_walk_frags(skb
, frag_skb
) {
1218 bd_num
= hns3_skb_bd_num(frag_skb
, bd_size
, bd_num
);
1219 if (bd_num
> HNS3_MAX_TSO_BD_NUM
)
1226 static unsigned int hns3_gso_hdr_len(struct sk_buff
*skb
)
1228 if (!skb
->encapsulation
)
1229 return skb_transport_offset(skb
) + tcp_hdrlen(skb
);
1231 return skb_inner_transport_offset(skb
) + inner_tcp_hdrlen(skb
);
1234 /* HW need every continuous 8 buffer data to be larger than MSS,
1235 * we simplify it by ensuring skb_headlen + the first continuous
1236 * 7 frags to to be larger than gso header len + mss, and the remaining
1237 * continuous 7 frags to be larger than MSS except the last 7 frags.
1239 static bool hns3_skb_need_linearized(struct sk_buff
*skb
, unsigned int *bd_size
,
1240 unsigned int bd_num
)
1242 unsigned int tot_len
= 0;
1245 for (i
= 0; i
< HNS3_MAX_NON_TSO_BD_NUM
- 1U; i
++)
1246 tot_len
+= bd_size
[i
];
1248 /* ensure the first 8 frags is greater than mss + header */
1249 if (tot_len
+ bd_size
[HNS3_MAX_NON_TSO_BD_NUM
- 1U] <
1250 skb_shinfo(skb
)->gso_size
+ hns3_gso_hdr_len(skb
))
1253 /* ensure every continuous 7 buffer is greater than mss
1254 * except the last one.
1256 for (i
= 0; i
< bd_num
- HNS3_MAX_NON_TSO_BD_NUM
; i
++) {
1257 tot_len
-= bd_size
[i
];
1258 tot_len
+= bd_size
[i
+ HNS3_MAX_NON_TSO_BD_NUM
- 1U];
1260 if (tot_len
< skb_shinfo(skb
)->gso_size
)
1267 void hns3_shinfo_pack(struct skb_shared_info
*shinfo
, __u32
*size
)
1271 for (i
= 0; i
< MAX_SKB_FRAGS
; i
++)
1272 size
[i
] = skb_frag_size(&shinfo
->frags
[i
]);
1275 static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring
*ring
,
1276 struct net_device
*netdev
,
1277 struct sk_buff
*skb
)
1279 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
1280 unsigned int bd_size
[HNS3_MAX_TSO_BD_NUM
+ 1U];
1281 unsigned int bd_num
;
1283 bd_num
= hns3_tx_bd_num(skb
, bd_size
);
1284 if (unlikely(bd_num
> HNS3_MAX_NON_TSO_BD_NUM
)) {
1285 if (bd_num
<= HNS3_MAX_TSO_BD_NUM
&& skb_is_gso(skb
) &&
1286 !hns3_skb_need_linearized(skb
, bd_size
, bd_num
)) {
1287 trace_hns3_over_8bd(skb
);
1291 if (__skb_linearize(skb
))
1294 bd_num
= hns3_tx_bd_count(skb
->len
);
1295 if ((skb_is_gso(skb
) && bd_num
> HNS3_MAX_TSO_BD_NUM
) ||
1296 (!skb_is_gso(skb
) &&
1297 bd_num
> HNS3_MAX_NON_TSO_BD_NUM
)) {
1298 trace_hns3_over_8bd(skb
);
1302 u64_stats_update_begin(&ring
->syncp
);
1303 ring
->stats
.tx_copy
++;
1304 u64_stats_update_end(&ring
->syncp
);
1308 if (likely(ring_space(ring
) >= bd_num
))
1311 netif_stop_subqueue(netdev
, ring
->queue_index
);
1312 smp_mb(); /* Memory barrier before checking ring_space */
1314 /* Start queue in case hns3_clean_tx_ring has just made room
1315 * available and has not seen the queue stopped state performed
1316 * by netif_stop_subqueue above.
1318 if (ring_space(ring
) >= bd_num
&& netif_carrier_ok(netdev
) &&
1319 !test_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
)) {
1320 netif_start_subqueue(netdev
, ring
->queue_index
);
1327 static void hns3_clear_desc(struct hns3_enet_ring
*ring
, int next_to_use_orig
)
1329 struct device
*dev
= ring_to_dev(ring
);
1332 for (i
= 0; i
< ring
->desc_num
; i
++) {
1333 struct hns3_desc
*desc
= &ring
->desc
[ring
->next_to_use
];
1335 memset(desc
, 0, sizeof(*desc
));
1337 /* check if this is where we started */
1338 if (ring
->next_to_use
== next_to_use_orig
)
1342 ring_ptr_move_bw(ring
, next_to_use
);
1344 if (!ring
->desc_cb
[ring
->next_to_use
].dma
)
1347 /* unmap the descriptor dma address */
1348 if (ring
->desc_cb
[ring
->next_to_use
].type
== DESC_TYPE_SKB
||
1349 ring
->desc_cb
[ring
->next_to_use
].type
==
1350 DESC_TYPE_FRAGLIST_SKB
)
1351 dma_unmap_single(dev
,
1352 ring
->desc_cb
[ring
->next_to_use
].dma
,
1353 ring
->desc_cb
[ring
->next_to_use
].length
,
1355 else if (ring
->desc_cb
[ring
->next_to_use
].length
)
1357 ring
->desc_cb
[ring
->next_to_use
].dma
,
1358 ring
->desc_cb
[ring
->next_to_use
].length
,
1361 ring
->desc_cb
[ring
->next_to_use
].length
= 0;
1362 ring
->desc_cb
[ring
->next_to_use
].dma
= 0;
1363 ring
->desc_cb
[ring
->next_to_use
].type
= DESC_TYPE_UNKNOWN
;
1367 static int hns3_fill_skb_to_desc(struct hns3_enet_ring
*ring
,
1368 struct sk_buff
*skb
, enum hns_desc_type type
)
1370 unsigned int size
= skb_headlen(skb
);
1371 int i
, ret
, bd_num
= 0;
1374 ret
= hns3_fill_desc(ring
, skb
, size
, type
);
1375 if (unlikely(ret
< 0))
1381 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1382 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
1384 size
= skb_frag_size(frag
);
1388 ret
= hns3_fill_desc(ring
, frag
, size
, DESC_TYPE_PAGE
);
1389 if (unlikely(ret
< 0))
1398 netdev_tx_t
hns3_nic_net_xmit(struct sk_buff
*skb
, struct net_device
*netdev
)
1400 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
1401 struct hns3_enet_ring
*ring
= &priv
->ring
[skb
->queue_mapping
];
1402 struct netdev_queue
*dev_queue
;
1403 int pre_ntu
, next_to_use_head
;
1404 struct sk_buff
*frag_skb
;
1408 /* Hardware can only handle short frames above 32 bytes */
1409 if (skb_put_padto(skb
, HNS3_MIN_TX_LEN
))
1410 return NETDEV_TX_OK
;
1412 /* Prefetch the data used later */
1413 prefetch(skb
->data
);
1415 ret
= hns3_nic_maybe_stop_tx(ring
, netdev
, skb
);
1416 if (unlikely(ret
<= 0)) {
1417 if (ret
== -EBUSY
) {
1418 u64_stats_update_begin(&ring
->syncp
);
1419 ring
->stats
.tx_busy
++;
1420 u64_stats_update_end(&ring
->syncp
);
1421 return NETDEV_TX_BUSY
;
1422 } else if (ret
== -ENOMEM
) {
1423 u64_stats_update_begin(&ring
->syncp
);
1424 ring
->stats
.sw_err_cnt
++;
1425 u64_stats_update_end(&ring
->syncp
);
1428 hns3_rl_err(netdev
, "xmit error: %d!\n", ret
);
1432 next_to_use_head
= ring
->next_to_use
;
1434 ret
= hns3_fill_skb_desc(ring
, skb
, &ring
->desc
[ring
->next_to_use
]);
1435 if (unlikely(ret
< 0))
1438 ret
= hns3_fill_skb_to_desc(ring
, skb
, DESC_TYPE_SKB
);
1439 if (unlikely(ret
< 0))
1444 skb_walk_frags(skb
, frag_skb
) {
1445 ret
= hns3_fill_skb_to_desc(ring
, frag_skb
,
1446 DESC_TYPE_FRAGLIST_SKB
);
1447 if (unlikely(ret
< 0))
1453 pre_ntu
= ring
->next_to_use
? (ring
->next_to_use
- 1) :
1454 (ring
->desc_num
- 1);
1455 ring
->desc
[pre_ntu
].tx
.bdtp_fe_sc_vld_ra_ri
|=
1456 cpu_to_le16(BIT(HNS3_TXD_FE_B
));
1457 trace_hns3_tx_desc(ring
, pre_ntu
);
1459 /* Complete translate all packets */
1460 dev_queue
= netdev_get_tx_queue(netdev
, ring
->queue_index
);
1461 netdev_tx_sent_queue(dev_queue
, skb
->len
);
1463 wmb(); /* Commit all data before submit */
1465 hnae3_queue_xmit(ring
->tqp
, bd_num
);
1467 return NETDEV_TX_OK
;
1470 hns3_clear_desc(ring
, next_to_use_head
);
1473 dev_kfree_skb_any(skb
);
1474 return NETDEV_TX_OK
;
1477 static int hns3_nic_net_set_mac_address(struct net_device
*netdev
, void *p
)
1479 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1480 struct sockaddr
*mac_addr
= p
;
1483 if (!mac_addr
|| !is_valid_ether_addr((const u8
*)mac_addr
->sa_data
))
1484 return -EADDRNOTAVAIL
;
1486 if (ether_addr_equal(netdev
->dev_addr
, mac_addr
->sa_data
)) {
1487 netdev_info(netdev
, "already using mac address %pM\n",
1492 /* For VF device, if there is a perm_addr, then the user will not
1493 * be allowed to change the address.
1495 if (!hns3_is_phys_func(h
->pdev
) &&
1496 !is_zero_ether_addr(netdev
->perm_addr
)) {
1497 netdev_err(netdev
, "has permanent MAC %pM, user MAC %pM not allow\n",
1498 netdev
->perm_addr
, mac_addr
->sa_data
);
1502 ret
= h
->ae_algo
->ops
->set_mac_addr(h
, mac_addr
->sa_data
, false);
1504 netdev_err(netdev
, "set_mac_address fail, ret=%d!\n", ret
);
1508 ether_addr_copy(netdev
->dev_addr
, mac_addr
->sa_data
);
1513 static int hns3_nic_do_ioctl(struct net_device
*netdev
,
1514 struct ifreq
*ifr
, int cmd
)
1516 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1518 if (!netif_running(netdev
))
1521 if (!h
->ae_algo
->ops
->do_ioctl
)
1524 return h
->ae_algo
->ops
->do_ioctl(h
, ifr
, cmd
);
1527 static int hns3_nic_set_features(struct net_device
*netdev
,
1528 netdev_features_t features
)
1530 netdev_features_t changed
= netdev
->features
^ features
;
1531 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
1532 struct hnae3_handle
*h
= priv
->ae_handle
;
1536 if (changed
& (NETIF_F_GRO_HW
) && h
->ae_algo
->ops
->set_gro_en
) {
1537 enable
= !!(features
& NETIF_F_GRO_HW
);
1538 ret
= h
->ae_algo
->ops
->set_gro_en(h
, enable
);
1543 if ((changed
& NETIF_F_HW_VLAN_CTAG_RX
) &&
1544 h
->ae_algo
->ops
->enable_hw_strip_rxvtag
) {
1545 enable
= !!(features
& NETIF_F_HW_VLAN_CTAG_RX
);
1546 ret
= h
->ae_algo
->ops
->enable_hw_strip_rxvtag(h
, enable
);
1551 if ((changed
& NETIF_F_NTUPLE
) && h
->ae_algo
->ops
->enable_fd
) {
1552 enable
= !!(features
& NETIF_F_NTUPLE
);
1553 h
->ae_algo
->ops
->enable_fd(h
, enable
);
1556 netdev
->features
= features
;
1560 static netdev_features_t
hns3_features_check(struct sk_buff
*skb
,
1561 struct net_device
*dev
,
1562 netdev_features_t features
)
1564 #define HNS3_MAX_HDR_LEN 480U
1565 #define HNS3_MAX_L4_HDR_LEN 60U
1569 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1572 if (skb
->encapsulation
)
1573 len
= skb_inner_transport_header(skb
) - skb
->data
;
1575 len
= skb_transport_header(skb
) - skb
->data
;
1577 /* Assume L4 is 60 byte as TCP is the only protocol with a
1578 * a flexible value, and it's max len is 60 bytes.
1580 len
+= HNS3_MAX_L4_HDR_LEN
;
1582 /* Hardware only supports checksum on the skb with a max header
1585 if (len
> HNS3_MAX_HDR_LEN
)
1586 features
&= ~(NETIF_F_CSUM_MASK
| NETIF_F_GSO_MASK
);
1591 static void hns3_nic_get_stats64(struct net_device
*netdev
,
1592 struct rtnl_link_stats64
*stats
)
1594 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
1595 int queue_num
= priv
->ae_handle
->kinfo
.num_tqps
;
1596 struct hnae3_handle
*handle
= priv
->ae_handle
;
1597 struct hns3_enet_ring
*ring
;
1598 u64 rx_length_errors
= 0;
1599 u64 rx_crc_errors
= 0;
1600 u64 rx_multicast
= 0;
1612 if (test_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
))
1615 handle
->ae_algo
->ops
->update_stats(handle
, &netdev
->stats
);
1617 for (idx
= 0; idx
< queue_num
; idx
++) {
1618 /* fetch the tx stats */
1619 ring
= &priv
->ring
[idx
];
1621 start
= u64_stats_fetch_begin_irq(&ring
->syncp
);
1622 tx_bytes
+= ring
->stats
.tx_bytes
;
1623 tx_pkts
+= ring
->stats
.tx_pkts
;
1624 tx_drop
+= ring
->stats
.sw_err_cnt
;
1625 tx_drop
+= ring
->stats
.tx_vlan_err
;
1626 tx_drop
+= ring
->stats
.tx_l4_proto_err
;
1627 tx_drop
+= ring
->stats
.tx_l2l3l4_err
;
1628 tx_drop
+= ring
->stats
.tx_tso_err
;
1629 tx_errors
+= ring
->stats
.sw_err_cnt
;
1630 tx_errors
+= ring
->stats
.tx_vlan_err
;
1631 tx_errors
+= ring
->stats
.tx_l4_proto_err
;
1632 tx_errors
+= ring
->stats
.tx_l2l3l4_err
;
1633 tx_errors
+= ring
->stats
.tx_tso_err
;
1634 } while (u64_stats_fetch_retry_irq(&ring
->syncp
, start
));
1636 /* fetch the rx stats */
1637 ring
= &priv
->ring
[idx
+ queue_num
];
1639 start
= u64_stats_fetch_begin_irq(&ring
->syncp
);
1640 rx_bytes
+= ring
->stats
.rx_bytes
;
1641 rx_pkts
+= ring
->stats
.rx_pkts
;
1642 rx_drop
+= ring
->stats
.l2_err
;
1643 rx_errors
+= ring
->stats
.l2_err
;
1644 rx_errors
+= ring
->stats
.l3l4_csum_err
;
1645 rx_crc_errors
+= ring
->stats
.l2_err
;
1646 rx_multicast
+= ring
->stats
.rx_multicast
;
1647 rx_length_errors
+= ring
->stats
.err_pkt_len
;
1648 } while (u64_stats_fetch_retry_irq(&ring
->syncp
, start
));
1651 stats
->tx_bytes
= tx_bytes
;
1652 stats
->tx_packets
= tx_pkts
;
1653 stats
->rx_bytes
= rx_bytes
;
1654 stats
->rx_packets
= rx_pkts
;
1656 stats
->rx_errors
= rx_errors
;
1657 stats
->multicast
= rx_multicast
;
1658 stats
->rx_length_errors
= rx_length_errors
;
1659 stats
->rx_crc_errors
= rx_crc_errors
;
1660 stats
->rx_missed_errors
= netdev
->stats
.rx_missed_errors
;
1662 stats
->tx_errors
= tx_errors
;
1663 stats
->rx_dropped
= rx_drop
;
1664 stats
->tx_dropped
= tx_drop
;
1665 stats
->collisions
= netdev
->stats
.collisions
;
1666 stats
->rx_over_errors
= netdev
->stats
.rx_over_errors
;
1667 stats
->rx_frame_errors
= netdev
->stats
.rx_frame_errors
;
1668 stats
->rx_fifo_errors
= netdev
->stats
.rx_fifo_errors
;
1669 stats
->tx_aborted_errors
= netdev
->stats
.tx_aborted_errors
;
1670 stats
->tx_carrier_errors
= netdev
->stats
.tx_carrier_errors
;
1671 stats
->tx_fifo_errors
= netdev
->stats
.tx_fifo_errors
;
1672 stats
->tx_heartbeat_errors
= netdev
->stats
.tx_heartbeat_errors
;
1673 stats
->tx_window_errors
= netdev
->stats
.tx_window_errors
;
1674 stats
->rx_compressed
= netdev
->stats
.rx_compressed
;
1675 stats
->tx_compressed
= netdev
->stats
.tx_compressed
;
1678 static int hns3_setup_tc(struct net_device
*netdev
, void *type_data
)
1680 struct tc_mqprio_qopt_offload
*mqprio_qopt
= type_data
;
1681 u8
*prio_tc
= mqprio_qopt
->qopt
.prio_tc_map
;
1682 struct hnae3_knic_private_info
*kinfo
;
1683 u8 tc
= mqprio_qopt
->qopt
.num_tc
;
1684 u16 mode
= mqprio_qopt
->mode
;
1685 u8 hw
= mqprio_qopt
->qopt
.hw
;
1686 struct hnae3_handle
*h
;
1688 if (!((hw
== TC_MQPRIO_HW_OFFLOAD_TCS
&&
1689 mode
== TC_MQPRIO_MODE_CHANNEL
) || (!hw
&& tc
== 0)))
1692 if (tc
> HNAE3_MAX_TC
)
1698 h
= hns3_get_handle(netdev
);
1701 netif_dbg(h
, drv
, netdev
, "setup tc: num_tc=%u\n", tc
);
1703 return (kinfo
->dcb_ops
&& kinfo
->dcb_ops
->setup_tc
) ?
1704 kinfo
->dcb_ops
->setup_tc(h
, tc
? tc
: 1, prio_tc
) : -EOPNOTSUPP
;
1707 static int hns3_nic_setup_tc(struct net_device
*dev
, enum tc_setup_type type
,
1710 if (type
!= TC_SETUP_QDISC_MQPRIO
)
1713 return hns3_setup_tc(dev
, type_data
);
1716 static int hns3_vlan_rx_add_vid(struct net_device
*netdev
,
1717 __be16 proto
, u16 vid
)
1719 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1722 if (h
->ae_algo
->ops
->set_vlan_filter
)
1723 ret
= h
->ae_algo
->ops
->set_vlan_filter(h
, proto
, vid
, false);
1728 static int hns3_vlan_rx_kill_vid(struct net_device
*netdev
,
1729 __be16 proto
, u16 vid
)
1731 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1734 if (h
->ae_algo
->ops
->set_vlan_filter
)
1735 ret
= h
->ae_algo
->ops
->set_vlan_filter(h
, proto
, vid
, true);
1740 static int hns3_ndo_set_vf_vlan(struct net_device
*netdev
, int vf
, u16 vlan
,
1741 u8 qos
, __be16 vlan_proto
)
1743 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1746 netif_dbg(h
, drv
, netdev
,
1747 "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",
1748 vf
, vlan
, qos
, ntohs(vlan_proto
));
1750 if (h
->ae_algo
->ops
->set_vf_vlan_filter
)
1751 ret
= h
->ae_algo
->ops
->set_vf_vlan_filter(h
, vf
, vlan
,
1757 static int hns3_set_vf_spoofchk(struct net_device
*netdev
, int vf
, bool enable
)
1759 struct hnae3_handle
*handle
= hns3_get_handle(netdev
);
1761 if (hns3_nic_resetting(netdev
))
1764 if (!handle
->ae_algo
->ops
->set_vf_spoofchk
)
1767 return handle
->ae_algo
->ops
->set_vf_spoofchk(handle
, vf
, enable
);
1770 static int hns3_set_vf_trust(struct net_device
*netdev
, int vf
, bool enable
)
1772 struct hnae3_handle
*handle
= hns3_get_handle(netdev
);
1774 if (!handle
->ae_algo
->ops
->set_vf_trust
)
1777 return handle
->ae_algo
->ops
->set_vf_trust(handle
, vf
, enable
);
1780 static int hns3_nic_change_mtu(struct net_device
*netdev
, int new_mtu
)
1782 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1785 if (hns3_nic_resetting(netdev
))
1788 if (!h
->ae_algo
->ops
->set_mtu
)
1791 netif_dbg(h
, drv
, netdev
,
1792 "change mtu from %u to %d\n", netdev
->mtu
, new_mtu
);
1794 ret
= h
->ae_algo
->ops
->set_mtu(h
, new_mtu
);
1796 netdev_err(netdev
, "failed to change MTU in hardware %d\n",
1799 netdev
->mtu
= new_mtu
;
1804 static bool hns3_get_tx_timeo_queue_info(struct net_device
*ndev
)
1806 struct hns3_nic_priv
*priv
= netdev_priv(ndev
);
1807 struct hnae3_handle
*h
= hns3_get_handle(ndev
);
1808 struct hns3_enet_ring
*tx_ring
;
1809 struct napi_struct
*napi
;
1810 int timeout_queue
= 0;
1811 int hw_head
, hw_tail
;
1812 int fbd_num
, fbd_oft
;
1813 int ebd_num
, ebd_oft
;
1818 /* Find the stopped queue the same way the stack does */
1819 for (i
= 0; i
< ndev
->num_tx_queues
; i
++) {
1820 struct netdev_queue
*q
;
1821 unsigned long trans_start
;
1823 q
= netdev_get_tx_queue(ndev
, i
);
1824 trans_start
= q
->trans_start
;
1825 if (netif_xmit_stopped(q
) &&
1827 (trans_start
+ ndev
->watchdog_timeo
))) {
1829 netdev_info(ndev
, "queue state: 0x%lx, delta msecs: %u\n",
1831 jiffies_to_msecs(jiffies
- trans_start
));
1836 if (i
== ndev
->num_tx_queues
) {
1838 "no netdev TX timeout queue found, timeout count: %llu\n",
1839 priv
->tx_timeout_count
);
1843 priv
->tx_timeout_count
++;
1845 tx_ring
= &priv
->ring
[timeout_queue
];
1846 napi
= &tx_ring
->tqp_vector
->napi
;
1849 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",
1850 priv
->tx_timeout_count
, timeout_queue
, tx_ring
->next_to_use
,
1851 tx_ring
->next_to_clean
, napi
->state
);
1854 "tx_pkts: %llu, tx_bytes: %llu, io_err_cnt: %llu, sw_err_cnt: %llu\n",
1855 tx_ring
->stats
.tx_pkts
, tx_ring
->stats
.tx_bytes
,
1856 tx_ring
->stats
.io_err_cnt
, tx_ring
->stats
.sw_err_cnt
);
1859 "seg_pkt_cnt: %llu, tx_err_cnt: %llu, restart_queue: %llu, tx_busy: %llu\n",
1860 tx_ring
->stats
.seg_pkt_cnt
, tx_ring
->stats
.tx_err_cnt
,
1861 tx_ring
->stats
.restart_queue
, tx_ring
->stats
.tx_busy
);
1863 /* When mac received many pause frames continuous, it's unable to send
1864 * packets, which may cause tx timeout
1866 if (h
->ae_algo
->ops
->get_mac_stats
) {
1867 struct hns3_mac_stats mac_stats
;
1869 h
->ae_algo
->ops
->get_mac_stats(h
, &mac_stats
);
1870 netdev_info(ndev
, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",
1871 mac_stats
.tx_pause_cnt
, mac_stats
.rx_pause_cnt
);
1874 hw_head
= readl_relaxed(tx_ring
->tqp
->io_base
+
1875 HNS3_RING_TX_RING_HEAD_REG
);
1876 hw_tail
= readl_relaxed(tx_ring
->tqp
->io_base
+
1877 HNS3_RING_TX_RING_TAIL_REG
);
1878 fbd_num
= readl_relaxed(tx_ring
->tqp
->io_base
+
1879 HNS3_RING_TX_RING_FBDNUM_REG
);
1880 fbd_oft
= readl_relaxed(tx_ring
->tqp
->io_base
+
1881 HNS3_RING_TX_RING_OFFSET_REG
);
1882 ebd_num
= readl_relaxed(tx_ring
->tqp
->io_base
+
1883 HNS3_RING_TX_RING_EBDNUM_REG
);
1884 ebd_oft
= readl_relaxed(tx_ring
->tqp
->io_base
+
1885 HNS3_RING_TX_RING_EBD_OFFSET_REG
);
1886 bd_num
= readl_relaxed(tx_ring
->tqp
->io_base
+
1887 HNS3_RING_TX_RING_BD_NUM_REG
);
1888 bd_err
= readl_relaxed(tx_ring
->tqp
->io_base
+
1889 HNS3_RING_TX_RING_BD_ERR_REG
);
1890 ring_en
= readl_relaxed(tx_ring
->tqp
->io_base
+ HNS3_RING_EN_REG
);
1891 tc
= readl_relaxed(tx_ring
->tqp
->io_base
+ HNS3_RING_TX_RING_TC_REG
);
1894 "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",
1895 bd_num
, hw_head
, hw_tail
, bd_err
,
1896 readl(tx_ring
->tqp_vector
->mask_addr
));
1898 "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",
1899 ring_en
, tc
, fbd_num
, fbd_oft
, ebd_num
, ebd_oft
);
1904 static void hns3_nic_net_timeout(struct net_device
*ndev
, unsigned int txqueue
)
1906 struct hns3_nic_priv
*priv
= netdev_priv(ndev
);
1907 struct hnae3_handle
*h
= priv
->ae_handle
;
1909 if (!hns3_get_tx_timeo_queue_info(ndev
))
1912 /* request the reset, and let the hclge to determine
1913 * which reset level should be done
1915 if (h
->ae_algo
->ops
->reset_event
)
1916 h
->ae_algo
->ops
->reset_event(h
->pdev
, h
);
1919 #ifdef CONFIG_RFS_ACCEL
1920 static int hns3_rx_flow_steer(struct net_device
*dev
, const struct sk_buff
*skb
,
1921 u16 rxq_index
, u32 flow_id
)
1923 struct hnae3_handle
*h
= hns3_get_handle(dev
);
1924 struct flow_keys fkeys
;
1926 if (!h
->ae_algo
->ops
->add_arfs_entry
)
1929 if (skb
->encapsulation
)
1930 return -EPROTONOSUPPORT
;
1932 if (!skb_flow_dissect_flow_keys(skb
, &fkeys
, 0))
1933 return -EPROTONOSUPPORT
;
1935 if ((fkeys
.basic
.n_proto
!= htons(ETH_P_IP
) &&
1936 fkeys
.basic
.n_proto
!= htons(ETH_P_IPV6
)) ||
1937 (fkeys
.basic
.ip_proto
!= IPPROTO_TCP
&&
1938 fkeys
.basic
.ip_proto
!= IPPROTO_UDP
))
1939 return -EPROTONOSUPPORT
;
1941 return h
->ae_algo
->ops
->add_arfs_entry(h
, rxq_index
, flow_id
, &fkeys
);
1945 static int hns3_nic_get_vf_config(struct net_device
*ndev
, int vf
,
1946 struct ifla_vf_info
*ivf
)
1948 struct hnae3_handle
*h
= hns3_get_handle(ndev
);
1950 if (!h
->ae_algo
->ops
->get_vf_config
)
1953 return h
->ae_algo
->ops
->get_vf_config(h
, vf
, ivf
);
1956 static int hns3_nic_set_vf_link_state(struct net_device
*ndev
, int vf
,
1959 struct hnae3_handle
*h
= hns3_get_handle(ndev
);
1961 if (!h
->ae_algo
->ops
->set_vf_link_state
)
1964 return h
->ae_algo
->ops
->set_vf_link_state(h
, vf
, link_state
);
1967 static int hns3_nic_set_vf_rate(struct net_device
*ndev
, int vf
,
1968 int min_tx_rate
, int max_tx_rate
)
1970 struct hnae3_handle
*h
= hns3_get_handle(ndev
);
1972 if (!h
->ae_algo
->ops
->set_vf_rate
)
1975 return h
->ae_algo
->ops
->set_vf_rate(h
, vf
, min_tx_rate
, max_tx_rate
,
1979 static int hns3_nic_set_vf_mac(struct net_device
*netdev
, int vf_id
, u8
*mac
)
1981 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
1983 if (!h
->ae_algo
->ops
->set_vf_mac
)
1986 if (is_multicast_ether_addr(mac
)) {
1988 "Invalid MAC:%pM specified. Could not set MAC\n",
1993 return h
->ae_algo
->ops
->set_vf_mac(h
, vf_id
, mac
);
1996 static const struct net_device_ops hns3_nic_netdev_ops
= {
1997 .ndo_open
= hns3_nic_net_open
,
1998 .ndo_stop
= hns3_nic_net_stop
,
1999 .ndo_start_xmit
= hns3_nic_net_xmit
,
2000 .ndo_tx_timeout
= hns3_nic_net_timeout
,
2001 .ndo_set_mac_address
= hns3_nic_net_set_mac_address
,
2002 .ndo_do_ioctl
= hns3_nic_do_ioctl
,
2003 .ndo_change_mtu
= hns3_nic_change_mtu
,
2004 .ndo_set_features
= hns3_nic_set_features
,
2005 .ndo_features_check
= hns3_features_check
,
2006 .ndo_get_stats64
= hns3_nic_get_stats64
,
2007 .ndo_setup_tc
= hns3_nic_setup_tc
,
2008 .ndo_set_rx_mode
= hns3_nic_set_rx_mode
,
2009 .ndo_vlan_rx_add_vid
= hns3_vlan_rx_add_vid
,
2010 .ndo_vlan_rx_kill_vid
= hns3_vlan_rx_kill_vid
,
2011 .ndo_set_vf_vlan
= hns3_ndo_set_vf_vlan
,
2012 .ndo_set_vf_spoofchk
= hns3_set_vf_spoofchk
,
2013 .ndo_set_vf_trust
= hns3_set_vf_trust
,
2014 #ifdef CONFIG_RFS_ACCEL
2015 .ndo_rx_flow_steer
= hns3_rx_flow_steer
,
2017 .ndo_get_vf_config
= hns3_nic_get_vf_config
,
2018 .ndo_set_vf_link_state
= hns3_nic_set_vf_link_state
,
2019 .ndo_set_vf_rate
= hns3_nic_set_vf_rate
,
2020 .ndo_set_vf_mac
= hns3_nic_set_vf_mac
,
2023 bool hns3_is_phys_func(struct pci_dev
*pdev
)
2025 u32 dev_id
= pdev
->device
;
2028 case HNAE3_DEV_ID_GE
:
2029 case HNAE3_DEV_ID_25GE
:
2030 case HNAE3_DEV_ID_25GE_RDMA
:
2031 case HNAE3_DEV_ID_25GE_RDMA_MACSEC
:
2032 case HNAE3_DEV_ID_50GE_RDMA
:
2033 case HNAE3_DEV_ID_50GE_RDMA_MACSEC
:
2034 case HNAE3_DEV_ID_100G_RDMA_MACSEC
:
2036 case HNAE3_DEV_ID_100G_VF
:
2037 case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF
:
2040 dev_warn(&pdev
->dev
, "un-recognized pci device-id %u",
2047 static void hns3_disable_sriov(struct pci_dev
*pdev
)
2049 /* If our VFs are assigned we cannot shut down SR-IOV
2050 * without causing issues, so just leave the hardware
2051 * available but disabled
2053 if (pci_vfs_assigned(pdev
)) {
2054 dev_warn(&pdev
->dev
,
2055 "disabling driver while VFs are assigned\n");
2059 pci_disable_sriov(pdev
);
2062 static void hns3_get_dev_capability(struct pci_dev
*pdev
,
2063 struct hnae3_ae_dev
*ae_dev
)
2065 if (pdev
->revision
>= 0x21) {
2066 hnae3_set_bit(ae_dev
->flag
, HNAE3_DEV_SUPPORT_FD_B
, 1);
2067 hnae3_set_bit(ae_dev
->flag
, HNAE3_DEV_SUPPORT_GRO_B
, 1);
2071 /* hns3_probe - Device initialization routine
2072 * @pdev: PCI device information struct
2073 * @ent: entry in hns3_pci_tbl
2075 * hns3_probe initializes a PF identified by a pci_dev structure.
2076 * The OS initialization, configuring of the PF private structure,
2077 * and a hardware reset occur.
2079 * Returns 0 on success, negative on failure
2081 static int hns3_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
2083 struct hnae3_ae_dev
*ae_dev
;
2086 ae_dev
= devm_kzalloc(&pdev
->dev
, sizeof(*ae_dev
), GFP_KERNEL
);
2090 ae_dev
->pdev
= pdev
;
2091 ae_dev
->flag
= ent
->driver_data
;
2092 hns3_get_dev_capability(pdev
, ae_dev
);
2093 pci_set_drvdata(pdev
, ae_dev
);
2095 ret
= hnae3_register_ae_dev(ae_dev
);
2097 pci_set_drvdata(pdev
, NULL
);
2102 /* hns3_remove - Device removal routine
2103 * @pdev: PCI device information struct
2105 static void hns3_remove(struct pci_dev
*pdev
)
2107 struct hnae3_ae_dev
*ae_dev
= pci_get_drvdata(pdev
);
2109 if (hns3_is_phys_func(pdev
) && IS_ENABLED(CONFIG_PCI_IOV
))
2110 hns3_disable_sriov(pdev
);
2112 hnae3_unregister_ae_dev(ae_dev
);
2113 pci_set_drvdata(pdev
, NULL
);
2117 * hns3_pci_sriov_configure
2118 * @pdev: pointer to a pci_dev structure
2119 * @num_vfs: number of VFs to allocate
2121 * Enable or change the number of VFs. Called when the user updates the number
2124 static int hns3_pci_sriov_configure(struct pci_dev
*pdev
, int num_vfs
)
2128 if (!(hns3_is_phys_func(pdev
) && IS_ENABLED(CONFIG_PCI_IOV
))) {
2129 dev_warn(&pdev
->dev
, "Can not config SRIOV\n");
2134 ret
= pci_enable_sriov(pdev
, num_vfs
);
2136 dev_err(&pdev
->dev
, "SRIOV enable failed %d\n", ret
);
2139 } else if (!pci_vfs_assigned(pdev
)) {
2140 pci_disable_sriov(pdev
);
2142 dev_warn(&pdev
->dev
,
2143 "Unable to free VFs because some are assigned to VMs.\n");
2149 static void hns3_shutdown(struct pci_dev
*pdev
)
2151 struct hnae3_ae_dev
*ae_dev
= pci_get_drvdata(pdev
);
2153 hnae3_unregister_ae_dev(ae_dev
);
2154 pci_set_drvdata(pdev
, NULL
);
2156 if (system_state
== SYSTEM_POWER_OFF
)
2157 pci_set_power_state(pdev
, PCI_D3hot
);
2160 static pci_ers_result_t
hns3_error_detected(struct pci_dev
*pdev
,
2161 pci_channel_state_t state
)
2163 struct hnae3_ae_dev
*ae_dev
= pci_get_drvdata(pdev
);
2164 pci_ers_result_t ret
;
2166 dev_info(&pdev
->dev
, "PCI error detected, state(=%d)!!\n", state
);
2168 if (state
== pci_channel_io_perm_failure
)
2169 return PCI_ERS_RESULT_DISCONNECT
;
2171 if (!ae_dev
|| !ae_dev
->ops
) {
2173 "Can't recover - error happened before device initialized\n");
2174 return PCI_ERS_RESULT_NONE
;
2177 if (ae_dev
->ops
->handle_hw_ras_error
)
2178 ret
= ae_dev
->ops
->handle_hw_ras_error(ae_dev
);
2180 return PCI_ERS_RESULT_NONE
;
2185 static pci_ers_result_t
hns3_slot_reset(struct pci_dev
*pdev
)
2187 struct hnae3_ae_dev
*ae_dev
= pci_get_drvdata(pdev
);
2188 const struct hnae3_ae_ops
*ops
;
2189 enum hnae3_reset_type reset_type
;
2190 struct device
*dev
= &pdev
->dev
;
2192 if (!ae_dev
|| !ae_dev
->ops
)
2193 return PCI_ERS_RESULT_NONE
;
2196 /* request the reset */
2197 if (ops
->reset_event
&& ops
->get_reset_level
&&
2198 ops
->set_default_reset_request
) {
2199 if (ae_dev
->hw_err_reset_req
) {
2200 reset_type
= ops
->get_reset_level(ae_dev
,
2201 &ae_dev
->hw_err_reset_req
);
2202 ops
->set_default_reset_request(ae_dev
, reset_type
);
2203 dev_info(dev
, "requesting reset due to PCI error\n");
2204 ops
->reset_event(pdev
, NULL
);
2207 return PCI_ERS_RESULT_RECOVERED
;
2210 return PCI_ERS_RESULT_DISCONNECT
;
2213 static void hns3_reset_prepare(struct pci_dev
*pdev
)
2215 struct hnae3_ae_dev
*ae_dev
= pci_get_drvdata(pdev
);
2217 dev_info(&pdev
->dev
, "FLR prepare\n");
2218 if (ae_dev
&& ae_dev
->ops
&& ae_dev
->ops
->flr_prepare
)
2219 ae_dev
->ops
->flr_prepare(ae_dev
);
2222 static void hns3_reset_done(struct pci_dev
*pdev
)
2224 struct hnae3_ae_dev
*ae_dev
= pci_get_drvdata(pdev
);
2226 dev_info(&pdev
->dev
, "FLR done\n");
2227 if (ae_dev
&& ae_dev
->ops
&& ae_dev
->ops
->flr_done
)
2228 ae_dev
->ops
->flr_done(ae_dev
);
2231 static const struct pci_error_handlers hns3_err_handler
= {
2232 .error_detected
= hns3_error_detected
,
2233 .slot_reset
= hns3_slot_reset
,
2234 .reset_prepare
= hns3_reset_prepare
,
2235 .reset_done
= hns3_reset_done
,
2238 static struct pci_driver hns3_driver
= {
2239 .name
= hns3_driver_name
,
2240 .id_table
= hns3_pci_tbl
,
2241 .probe
= hns3_probe
,
2242 .remove
= hns3_remove
,
2243 .shutdown
= hns3_shutdown
,
2244 .sriov_configure
= hns3_pci_sriov_configure
,
2245 .err_handler
= &hns3_err_handler
,
2248 /* set default feature to hns3 */
2249 static void hns3_set_default_feature(struct net_device
*netdev
)
2251 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
2252 struct pci_dev
*pdev
= h
->pdev
;
2254 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
2256 netdev
->hw_enc_features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2257 NETIF_F_RXCSUM
| NETIF_F_SG
| NETIF_F_GSO
|
2258 NETIF_F_GRO
| NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_GSO_GRE
|
2259 NETIF_F_GSO_GRE_CSUM
| NETIF_F_GSO_UDP_TUNNEL
|
2260 NETIF_F_GSO_UDP_TUNNEL_CSUM
| NETIF_F_SCTP_CRC
|
2261 NETIF_F_TSO_MANGLEID
| NETIF_F_FRAGLIST
;
2263 netdev
->gso_partial_features
|= NETIF_F_GSO_GRE_CSUM
;
2265 netdev
->features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2266 NETIF_F_HW_VLAN_CTAG_FILTER
|
2267 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
|
2268 NETIF_F_RXCSUM
| NETIF_F_SG
| NETIF_F_GSO
|
2269 NETIF_F_GRO
| NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_GSO_GRE
|
2270 NETIF_F_GSO_GRE_CSUM
| NETIF_F_GSO_UDP_TUNNEL
|
2271 NETIF_F_GSO_UDP_TUNNEL_CSUM
| NETIF_F_SCTP_CRC
|
2274 netdev
->vlan_features
|=
2275 NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
| NETIF_F_RXCSUM
|
2276 NETIF_F_SG
| NETIF_F_GSO
| NETIF_F_GRO
|
2277 NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_GSO_GRE
|
2278 NETIF_F_GSO_GRE_CSUM
| NETIF_F_GSO_UDP_TUNNEL
|
2279 NETIF_F_GSO_UDP_TUNNEL_CSUM
| NETIF_F_SCTP_CRC
|
2282 netdev
->hw_features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2283 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
|
2284 NETIF_F_RXCSUM
| NETIF_F_SG
| NETIF_F_GSO
|
2285 NETIF_F_GRO
| NETIF_F_TSO
| NETIF_F_TSO6
| NETIF_F_GSO_GRE
|
2286 NETIF_F_GSO_GRE_CSUM
| NETIF_F_GSO_UDP_TUNNEL
|
2287 NETIF_F_GSO_UDP_TUNNEL_CSUM
| NETIF_F_SCTP_CRC
|
2290 if (pdev
->revision
>= 0x21) {
2291 netdev
->hw_features
|= NETIF_F_GRO_HW
;
2292 netdev
->features
|= NETIF_F_GRO_HW
;
2294 if (!(h
->flags
& HNAE3_SUPPORT_VF
)) {
2295 netdev
->hw_features
|= NETIF_F_NTUPLE
;
2296 netdev
->features
|= NETIF_F_NTUPLE
;
2301 static int hns3_alloc_buffer(struct hns3_enet_ring
*ring
,
2302 struct hns3_desc_cb
*cb
)
2304 unsigned int order
= hns3_page_order(ring
);
2307 p
= dev_alloc_pages(order
);
2312 cb
->page_offset
= 0;
2314 cb
->buf
= page_address(p
);
2315 cb
->length
= hns3_page_size(ring
);
2316 cb
->type
= DESC_TYPE_PAGE
;
2321 static void hns3_free_buffer(struct hns3_enet_ring
*ring
,
2322 struct hns3_desc_cb
*cb
)
2324 if (cb
->type
== DESC_TYPE_SKB
)
2325 dev_kfree_skb_any((struct sk_buff
*)cb
->priv
);
2326 else if (!HNAE3_IS_TX_RING(ring
))
2327 put_page((struct page
*)cb
->priv
);
2328 memset(cb
, 0, sizeof(*cb
));
2331 static int hns3_map_buffer(struct hns3_enet_ring
*ring
, struct hns3_desc_cb
*cb
)
2333 cb
->dma
= dma_map_page(ring_to_dev(ring
), cb
->priv
, 0,
2334 cb
->length
, ring_to_dma_dir(ring
));
2336 if (unlikely(dma_mapping_error(ring_to_dev(ring
), cb
->dma
)))
2342 static void hns3_unmap_buffer(struct hns3_enet_ring
*ring
,
2343 struct hns3_desc_cb
*cb
)
2345 if (cb
->type
== DESC_TYPE_SKB
|| cb
->type
== DESC_TYPE_FRAGLIST_SKB
)
2346 dma_unmap_single(ring_to_dev(ring
), cb
->dma
, cb
->length
,
2347 ring_to_dma_dir(ring
));
2348 else if (cb
->length
)
2349 dma_unmap_page(ring_to_dev(ring
), cb
->dma
, cb
->length
,
2350 ring_to_dma_dir(ring
));
2353 static void hns3_buffer_detach(struct hns3_enet_ring
*ring
, int i
)
2355 hns3_unmap_buffer(ring
, &ring
->desc_cb
[i
]);
2356 ring
->desc
[i
].addr
= 0;
2359 static void hns3_free_buffer_detach(struct hns3_enet_ring
*ring
, int i
)
2361 struct hns3_desc_cb
*cb
= &ring
->desc_cb
[i
];
2363 if (!ring
->desc_cb
[i
].dma
)
2366 hns3_buffer_detach(ring
, i
);
2367 hns3_free_buffer(ring
, cb
);
2370 static void hns3_free_buffers(struct hns3_enet_ring
*ring
)
2374 for (i
= 0; i
< ring
->desc_num
; i
++)
2375 hns3_free_buffer_detach(ring
, i
);
2378 /* free desc along with its attached buffer */
2379 static void hns3_free_desc(struct hns3_enet_ring
*ring
)
2381 int size
= ring
->desc_num
* sizeof(ring
->desc
[0]);
2383 hns3_free_buffers(ring
);
2386 dma_free_coherent(ring_to_dev(ring
), size
,
2387 ring
->desc
, ring
->desc_dma_addr
);
2392 static int hns3_alloc_desc(struct hns3_enet_ring
*ring
)
2394 int size
= ring
->desc_num
* sizeof(ring
->desc
[0]);
2396 ring
->desc
= dma_alloc_coherent(ring_to_dev(ring
), size
,
2397 &ring
->desc_dma_addr
, GFP_KERNEL
);
2404 static int hns3_alloc_and_map_buffer(struct hns3_enet_ring
*ring
,
2405 struct hns3_desc_cb
*cb
)
2409 ret
= hns3_alloc_buffer(ring
, cb
);
2413 ret
= hns3_map_buffer(ring
, cb
);
2420 hns3_free_buffer(ring
, cb
);
2425 static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring
*ring
, int i
)
2427 int ret
= hns3_alloc_and_map_buffer(ring
, &ring
->desc_cb
[i
]);
2432 ring
->desc
[i
].addr
= cpu_to_le64(ring
->desc_cb
[i
].dma
);
2437 /* Allocate memory for raw pkg, and map with dma */
2438 static int hns3_alloc_ring_buffers(struct hns3_enet_ring
*ring
)
2442 for (i
= 0; i
< ring
->desc_num
; i
++) {
2443 ret
= hns3_alloc_and_attach_buffer(ring
, i
);
2445 goto out_buffer_fail
;
2451 for (j
= i
- 1; j
>= 0; j
--)
2452 hns3_free_buffer_detach(ring
, j
);
2456 /* detach a in-used buffer and replace with a reserved one */
2457 static void hns3_replace_buffer(struct hns3_enet_ring
*ring
, int i
,
2458 struct hns3_desc_cb
*res_cb
)
2460 hns3_unmap_buffer(ring
, &ring
->desc_cb
[i
]);
2461 ring
->desc_cb
[i
] = *res_cb
;
2462 ring
->desc
[i
].addr
= cpu_to_le64(ring
->desc_cb
[i
].dma
);
2463 ring
->desc
[i
].rx
.bd_base_info
= 0;
2466 static void hns3_reuse_buffer(struct hns3_enet_ring
*ring
, int i
)
2468 ring
->desc_cb
[i
].reuse_flag
= 0;
2469 ring
->desc
[i
].addr
= cpu_to_le64(ring
->desc_cb
[i
].dma
+
2470 ring
->desc_cb
[i
].page_offset
);
2471 ring
->desc
[i
].rx
.bd_base_info
= 0;
2473 dma_sync_single_for_device(ring_to_dev(ring
),
2474 ring
->desc_cb
[i
].dma
+ ring
->desc_cb
[i
].page_offset
,
2475 hns3_buf_size(ring
),
2479 static void hns3_nic_reclaim_desc(struct hns3_enet_ring
*ring
, int head
,
2480 int *bytes
, int *pkts
)
2482 int ntc
= ring
->next_to_clean
;
2483 struct hns3_desc_cb
*desc_cb
;
2485 while (head
!= ntc
) {
2486 desc_cb
= &ring
->desc_cb
[ntc
];
2487 (*pkts
) += (desc_cb
->type
== DESC_TYPE_SKB
);
2488 (*bytes
) += desc_cb
->length
;
2489 /* desc_cb will be cleaned, after hnae3_free_buffer_detach */
2490 hns3_free_buffer_detach(ring
, ntc
);
2492 if (++ntc
== ring
->desc_num
)
2495 /* Issue prefetch for next Tx descriptor */
2496 prefetch(&ring
->desc_cb
[ntc
]);
2499 /* This smp_store_release() pairs with smp_load_acquire() in
2500 * ring_space called by hns3_nic_net_xmit.
2502 smp_store_release(&ring
->next_to_clean
, ntc
);
2505 static int is_valid_clean_head(struct hns3_enet_ring
*ring
, int h
)
2507 int u
= ring
->next_to_use
;
2508 int c
= ring
->next_to_clean
;
2510 if (unlikely(h
> ring
->desc_num
))
2513 return u
> c
? (h
> c
&& h
<= u
) : (h
> c
|| h
<= u
);
2516 void hns3_clean_tx_ring(struct hns3_enet_ring
*ring
)
2518 struct net_device
*netdev
= ring_to_netdev(ring
);
2519 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
2520 struct netdev_queue
*dev_queue
;
2524 head
= readl_relaxed(ring
->tqp
->io_base
+ HNS3_RING_TX_RING_HEAD_REG
);
2526 if (is_ring_empty(ring
) || head
== ring
->next_to_clean
)
2527 return; /* no data to poll */
2529 rmb(); /* Make sure head is ready before touch any data */
2531 if (unlikely(!is_valid_clean_head(ring
, head
))) {
2532 hns3_rl_err(netdev
, "wrong head (%d, %d-%d)\n", head
,
2533 ring
->next_to_use
, ring
->next_to_clean
);
2535 u64_stats_update_begin(&ring
->syncp
);
2536 ring
->stats
.io_err_cnt
++;
2537 u64_stats_update_end(&ring
->syncp
);
2543 hns3_nic_reclaim_desc(ring
, head
, &bytes
, &pkts
);
2545 ring
->tqp_vector
->tx_group
.total_bytes
+= bytes
;
2546 ring
->tqp_vector
->tx_group
.total_packets
+= pkts
;
2548 u64_stats_update_begin(&ring
->syncp
);
2549 ring
->stats
.tx_bytes
+= bytes
;
2550 ring
->stats
.tx_pkts
+= pkts
;
2551 u64_stats_update_end(&ring
->syncp
);
2553 dev_queue
= netdev_get_tx_queue(netdev
, ring
->tqp
->tqp_index
);
2554 netdev_tx_completed_queue(dev_queue
, pkts
, bytes
);
2556 if (unlikely(netif_carrier_ok(netdev
) &&
2557 ring_space(ring
) > HNS3_MAX_TSO_BD_NUM
)) {
2558 /* Make sure that anybody stopping the queue after this
2559 * sees the new next_to_clean.
2562 if (netif_tx_queue_stopped(dev_queue
) &&
2563 !test_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
)) {
2564 netif_tx_wake_queue(dev_queue
);
2565 ring
->stats
.restart_queue
++;
2570 static int hns3_desc_unused(struct hns3_enet_ring
*ring
)
2572 int ntc
= ring
->next_to_clean
;
2573 int ntu
= ring
->next_to_use
;
2575 return ((ntc
>= ntu
) ? 0 : ring
->desc_num
) + ntc
- ntu
;
2578 static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring
*ring
,
2581 struct hns3_desc_cb
*desc_cb
;
2582 struct hns3_desc_cb res_cbs
;
2585 for (i
= 0; i
< cleand_count
; i
++) {
2586 desc_cb
= &ring
->desc_cb
[ring
->next_to_use
];
2587 if (desc_cb
->reuse_flag
) {
2588 u64_stats_update_begin(&ring
->syncp
);
2589 ring
->stats
.reuse_pg_cnt
++;
2590 u64_stats_update_end(&ring
->syncp
);
2592 hns3_reuse_buffer(ring
, ring
->next_to_use
);
2594 ret
= hns3_alloc_and_map_buffer(ring
, &res_cbs
);
2596 u64_stats_update_begin(&ring
->syncp
);
2597 ring
->stats
.sw_err_cnt
++;
2598 u64_stats_update_end(&ring
->syncp
);
2600 hns3_rl_err(ring_to_netdev(ring
),
2601 "alloc rx buffer failed: %d\n",
2605 hns3_replace_buffer(ring
, ring
->next_to_use
, &res_cbs
);
2607 u64_stats_update_begin(&ring
->syncp
);
2608 ring
->stats
.non_reuse_pg
++;
2609 u64_stats_update_end(&ring
->syncp
);
2612 ring_ptr_move_fw(ring
, next_to_use
);
2615 wmb(); /* Make all data has been write before submit */
2616 writel_relaxed(i
, ring
->tqp
->io_base
+ HNS3_RING_RX_RING_HEAD_REG
);
2619 static bool hns3_page_is_reusable(struct page
*page
)
2621 return page_to_nid(page
) == numa_mem_id() &&
2622 !page_is_pfmemalloc(page
);
2625 static void hns3_nic_reuse_page(struct sk_buff
*skb
, int i
,
2626 struct hns3_enet_ring
*ring
, int pull_len
,
2627 struct hns3_desc_cb
*desc_cb
)
2629 struct hns3_desc
*desc
= &ring
->desc
[ring
->next_to_clean
];
2630 int size
= le16_to_cpu(desc
->rx
.size
);
2631 u32 truesize
= hns3_buf_size(ring
);
2633 skb_add_rx_frag(skb
, i
, desc_cb
->priv
, desc_cb
->page_offset
+ pull_len
,
2634 size
- pull_len
, truesize
);
2636 /* Avoid re-using remote pages, or the stack is still using the page
2637 * when page_offset rollback to zero, flag default unreuse
2639 if (unlikely(!hns3_page_is_reusable(desc_cb
->priv
)) ||
2640 (!desc_cb
->page_offset
&& page_count(desc_cb
->priv
) > 1))
2643 /* Move offset up to the next cache line */
2644 desc_cb
->page_offset
+= truesize
;
2646 if (desc_cb
->page_offset
+ truesize
<= hns3_page_size(ring
)) {
2647 desc_cb
->reuse_flag
= 1;
2648 /* Bump ref count on page before it is given */
2649 get_page(desc_cb
->priv
);
2650 } else if (page_count(desc_cb
->priv
) == 1) {
2651 desc_cb
->reuse_flag
= 1;
2652 desc_cb
->page_offset
= 0;
2653 get_page(desc_cb
->priv
);
2657 static int hns3_gro_complete(struct sk_buff
*skb
, u32 l234info
)
2659 __be16 type
= skb
->protocol
;
2663 while (eth_type_vlan(type
)) {
2664 struct vlan_hdr
*vh
;
2666 if ((depth
+ VLAN_HLEN
) > skb_headlen(skb
))
2669 vh
= (struct vlan_hdr
*)(skb
->data
+ depth
);
2670 type
= vh
->h_vlan_encapsulated_proto
;
2674 skb_set_network_header(skb
, depth
);
2676 if (type
== htons(ETH_P_IP
)) {
2677 const struct iphdr
*iph
= ip_hdr(skb
);
2679 depth
+= sizeof(struct iphdr
);
2680 skb_set_transport_header(skb
, depth
);
2682 th
->check
= ~tcp_v4_check(skb
->len
- depth
, iph
->saddr
,
2684 } else if (type
== htons(ETH_P_IPV6
)) {
2685 const struct ipv6hdr
*iph
= ipv6_hdr(skb
);
2687 depth
+= sizeof(struct ipv6hdr
);
2688 skb_set_transport_header(skb
, depth
);
2690 th
->check
= ~tcp_v6_check(skb
->len
- depth
, &iph
->saddr
,
2693 hns3_rl_err(skb
->dev
,
2694 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",
2695 be16_to_cpu(type
), depth
);
2699 skb_shinfo(skb
)->gso_segs
= NAPI_GRO_CB(skb
)->count
;
2701 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_ECN
;
2703 if (l234info
& BIT(HNS3_RXD_GRO_FIXID_B
))
2704 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_FIXEDID
;
2706 skb
->csum_start
= (unsigned char *)th
- skb
->head
;
2707 skb
->csum_offset
= offsetof(struct tcphdr
, check
);
2708 skb
->ip_summed
= CHECKSUM_PARTIAL
;
2710 trace_hns3_gro(skb
);
2715 static void hns3_rx_checksum(struct hns3_enet_ring
*ring
, struct sk_buff
*skb
,
2716 u32 l234info
, u32 bd_base_info
, u32 ol_info
)
2718 struct net_device
*netdev
= ring_to_netdev(ring
);
2719 int l3_type
, l4_type
;
2722 skb
->ip_summed
= CHECKSUM_NONE
;
2724 skb_checksum_none_assert(skb
);
2726 if (!(netdev
->features
& NETIF_F_RXCSUM
))
2729 /* check if hardware has done checksum */
2730 if (!(bd_base_info
& BIT(HNS3_RXD_L3L4P_B
)))
2733 if (unlikely(l234info
& (BIT(HNS3_RXD_L3E_B
) | BIT(HNS3_RXD_L4E_B
) |
2734 BIT(HNS3_RXD_OL3E_B
) |
2735 BIT(HNS3_RXD_OL4E_B
)))) {
2736 u64_stats_update_begin(&ring
->syncp
);
2737 ring
->stats
.l3l4_csum_err
++;
2738 u64_stats_update_end(&ring
->syncp
);
2743 ol4_type
= hnae3_get_field(ol_info
, HNS3_RXD_OL4ID_M
,
2746 case HNS3_OL4_TYPE_MAC_IN_UDP
:
2747 case HNS3_OL4_TYPE_NVGRE
:
2748 skb
->csum_level
= 1;
2750 case HNS3_OL4_TYPE_NO_TUN
:
2751 l3_type
= hnae3_get_field(l234info
, HNS3_RXD_L3ID_M
,
2753 l4_type
= hnae3_get_field(l234info
, HNS3_RXD_L4ID_M
,
2756 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2757 if ((l3_type
== HNS3_L3_TYPE_IPV4
||
2758 l3_type
== HNS3_L3_TYPE_IPV6
) &&
2759 (l4_type
== HNS3_L4_TYPE_UDP
||
2760 l4_type
== HNS3_L4_TYPE_TCP
||
2761 l4_type
== HNS3_L4_TYPE_SCTP
))
2762 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
2769 static void hns3_rx_skb(struct hns3_enet_ring
*ring
, struct sk_buff
*skb
)
2771 if (skb_has_frag_list(skb
))
2772 napi_gro_flush(&ring
->tqp_vector
->napi
, false);
2774 napi_gro_receive(&ring
->tqp_vector
->napi
, skb
);
2777 static bool hns3_parse_vlan_tag(struct hns3_enet_ring
*ring
,
2778 struct hns3_desc
*desc
, u32 l234info
,
2781 struct hnae3_handle
*handle
= ring
->tqp
->handle
;
2782 struct pci_dev
*pdev
= ring
->tqp
->handle
->pdev
;
2784 if (pdev
->revision
== 0x20) {
2785 *vlan_tag
= le16_to_cpu(desc
->rx
.ot_vlan_tag
);
2786 if (!(*vlan_tag
& VLAN_VID_MASK
))
2787 *vlan_tag
= le16_to_cpu(desc
->rx
.vlan_tag
);
2789 return (*vlan_tag
!= 0);
2792 #define HNS3_STRP_OUTER_VLAN 0x1
2793 #define HNS3_STRP_INNER_VLAN 0x2
2794 #define HNS3_STRP_BOTH 0x3
2796 /* Hardware always insert VLAN tag into RX descriptor when
2797 * remove the tag from packet, driver needs to determine
2798 * reporting which tag to stack.
2800 switch (hnae3_get_field(l234info
, HNS3_RXD_STRP_TAGP_M
,
2801 HNS3_RXD_STRP_TAGP_S
)) {
2802 case HNS3_STRP_OUTER_VLAN
:
2803 if (handle
->port_base_vlan_state
!=
2804 HNAE3_PORT_BASE_VLAN_DISABLE
)
2807 *vlan_tag
= le16_to_cpu(desc
->rx
.ot_vlan_tag
);
2809 case HNS3_STRP_INNER_VLAN
:
2810 if (handle
->port_base_vlan_state
!=
2811 HNAE3_PORT_BASE_VLAN_DISABLE
)
2814 *vlan_tag
= le16_to_cpu(desc
->rx
.vlan_tag
);
2816 case HNS3_STRP_BOTH
:
2817 if (handle
->port_base_vlan_state
==
2818 HNAE3_PORT_BASE_VLAN_DISABLE
)
2819 *vlan_tag
= le16_to_cpu(desc
->rx
.ot_vlan_tag
);
2821 *vlan_tag
= le16_to_cpu(desc
->rx
.vlan_tag
);
2829 static int hns3_alloc_skb(struct hns3_enet_ring
*ring
, unsigned int length
,
2832 struct hns3_desc_cb
*desc_cb
= &ring
->desc_cb
[ring
->next_to_clean
];
2833 struct net_device
*netdev
= ring_to_netdev(ring
);
2834 struct sk_buff
*skb
;
2836 ring
->skb
= napi_alloc_skb(&ring
->tqp_vector
->napi
, HNS3_RX_HEAD_SIZE
);
2838 if (unlikely(!skb
)) {
2839 hns3_rl_err(netdev
, "alloc rx skb fail\n");
2841 u64_stats_update_begin(&ring
->syncp
);
2842 ring
->stats
.sw_err_cnt
++;
2843 u64_stats_update_end(&ring
->syncp
);
2848 trace_hns3_rx_desc(ring
);
2849 prefetchw(skb
->data
);
2851 ring
->pending_buf
= 1;
2853 ring
->tail_skb
= NULL
;
2854 if (length
<= HNS3_RX_HEAD_SIZE
) {
2855 memcpy(__skb_put(skb
, length
), va
, ALIGN(length
, sizeof(long)));
2857 /* We can reuse buffer as-is, just make sure it is local */
2858 if (likely(hns3_page_is_reusable(desc_cb
->priv
)))
2859 desc_cb
->reuse_flag
= 1;
2860 else /* This page cannot be reused so discard it */
2861 put_page(desc_cb
->priv
);
2863 ring_ptr_move_fw(ring
, next_to_clean
);
2866 u64_stats_update_begin(&ring
->syncp
);
2867 ring
->stats
.seg_pkt_cnt
++;
2868 u64_stats_update_end(&ring
->syncp
);
2870 ring
->pull_len
= eth_get_headlen(netdev
, va
, HNS3_RX_HEAD_SIZE
);
2871 __skb_put(skb
, ring
->pull_len
);
2872 hns3_nic_reuse_page(skb
, ring
->frag_num
++, ring
, ring
->pull_len
,
2874 ring_ptr_move_fw(ring
, next_to_clean
);
2879 static int hns3_add_frag(struct hns3_enet_ring
*ring
)
2881 struct sk_buff
*skb
= ring
->skb
;
2882 struct sk_buff
*head_skb
= skb
;
2883 struct sk_buff
*new_skb
;
2884 struct hns3_desc_cb
*desc_cb
;
2885 struct hns3_desc
*desc
;
2889 desc
= &ring
->desc
[ring
->next_to_clean
];
2890 desc_cb
= &ring
->desc_cb
[ring
->next_to_clean
];
2891 bd_base_info
= le32_to_cpu(desc
->rx
.bd_base_info
);
2892 /* make sure HW write desc complete */
2894 if (!(bd_base_info
& BIT(HNS3_RXD_VLD_B
)))
2897 if (unlikely(ring
->frag_num
>= MAX_SKB_FRAGS
)) {
2898 new_skb
= napi_alloc_skb(&ring
->tqp_vector
->napi
, 0);
2899 if (unlikely(!new_skb
)) {
2900 hns3_rl_err(ring_to_netdev(ring
),
2901 "alloc rx fraglist skb fail\n");
2906 if (ring
->tail_skb
) {
2907 ring
->tail_skb
->next
= new_skb
;
2908 ring
->tail_skb
= new_skb
;
2910 skb_shinfo(skb
)->frag_list
= new_skb
;
2911 ring
->tail_skb
= new_skb
;
2915 if (ring
->tail_skb
) {
2916 head_skb
->truesize
+= hns3_buf_size(ring
);
2917 head_skb
->data_len
+= le16_to_cpu(desc
->rx
.size
);
2918 head_skb
->len
+= le16_to_cpu(desc
->rx
.size
);
2919 skb
= ring
->tail_skb
;
2922 dma_sync_single_for_cpu(ring_to_dev(ring
),
2923 desc_cb
->dma
+ desc_cb
->page_offset
,
2924 hns3_buf_size(ring
),
2927 hns3_nic_reuse_page(skb
, ring
->frag_num
++, ring
, 0, desc_cb
);
2928 trace_hns3_rx_desc(ring
);
2929 ring_ptr_move_fw(ring
, next_to_clean
);
2930 ring
->pending_buf
++;
2931 } while (!(bd_base_info
& BIT(HNS3_RXD_FE_B
)));
2936 static int hns3_set_gro_and_checksum(struct hns3_enet_ring
*ring
,
2937 struct sk_buff
*skb
, u32 l234info
,
2938 u32 bd_base_info
, u32 ol_info
)
2942 skb_shinfo(skb
)->gso_size
= hnae3_get_field(bd_base_info
,
2943 HNS3_RXD_GRO_SIZE_M
,
2944 HNS3_RXD_GRO_SIZE_S
);
2945 /* if there is no HW GRO, do not set gro params */
2946 if (!skb_shinfo(skb
)->gso_size
) {
2947 hns3_rx_checksum(ring
, skb
, l234info
, bd_base_info
, ol_info
);
2951 NAPI_GRO_CB(skb
)->count
= hnae3_get_field(l234info
,
2952 HNS3_RXD_GRO_COUNT_M
,
2953 HNS3_RXD_GRO_COUNT_S
);
2955 l3_type
= hnae3_get_field(l234info
, HNS3_RXD_L3ID_M
, HNS3_RXD_L3ID_S
);
2956 if (l3_type
== HNS3_L3_TYPE_IPV4
)
2957 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
2958 else if (l3_type
== HNS3_L3_TYPE_IPV6
)
2959 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
2963 return hns3_gro_complete(skb
, l234info
);
2966 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring
*ring
,
2967 struct sk_buff
*skb
, u32 rss_hash
)
2969 struct hnae3_handle
*handle
= ring
->tqp
->handle
;
2970 enum pkt_hash_types rss_type
;
2973 rss_type
= handle
->kinfo
.rss_type
;
2975 rss_type
= PKT_HASH_TYPE_NONE
;
2977 skb_set_hash(skb
, rss_hash
, rss_type
);
2980 static int hns3_handle_bdinfo(struct hns3_enet_ring
*ring
, struct sk_buff
*skb
)
2982 struct net_device
*netdev
= ring_to_netdev(ring
);
2983 enum hns3_pkt_l2t_type l2_frame_type
;
2984 u32 bd_base_info
, l234info
, ol_info
;
2985 struct hns3_desc
*desc
;
2989 /* bdinfo handled below is only valid on the last BD of the
2990 * current packet, and ring->next_to_clean indicates the first
2991 * descriptor of next packet, so need - 1 below.
2993 pre_ntc
= ring
->next_to_clean
? (ring
->next_to_clean
- 1) :
2994 (ring
->desc_num
- 1);
2995 desc
= &ring
->desc
[pre_ntc
];
2996 bd_base_info
= le32_to_cpu(desc
->rx
.bd_base_info
);
2997 l234info
= le32_to_cpu(desc
->rx
.l234_info
);
2998 ol_info
= le32_to_cpu(desc
->rx
.ol_info
);
3000 /* Based on hw strategy, the tag offloaded will be stored at
3001 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
3002 * in one layer tag case.
3004 if (netdev
->features
& NETIF_F_HW_VLAN_CTAG_RX
) {
3007 if (hns3_parse_vlan_tag(ring
, desc
, l234info
, &vlan_tag
))
3008 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
3012 if (unlikely(!desc
->rx
.pkt_len
|| (l234info
& (BIT(HNS3_RXD_TRUNCAT_B
) |
3013 BIT(HNS3_RXD_L2E_B
))))) {
3014 u64_stats_update_begin(&ring
->syncp
);
3015 if (l234info
& BIT(HNS3_RXD_L2E_B
))
3016 ring
->stats
.l2_err
++;
3018 ring
->stats
.err_pkt_len
++;
3019 u64_stats_update_end(&ring
->syncp
);
3026 /* Do update ip stack process */
3027 skb
->protocol
= eth_type_trans(skb
, netdev
);
3029 /* This is needed in order to enable forwarding support */
3030 ret
= hns3_set_gro_and_checksum(ring
, skb
, l234info
,
3031 bd_base_info
, ol_info
);
3032 if (unlikely(ret
)) {
3033 u64_stats_update_begin(&ring
->syncp
);
3034 ring
->stats
.rx_err_cnt
++;
3035 u64_stats_update_end(&ring
->syncp
);
3039 l2_frame_type
= hnae3_get_field(l234info
, HNS3_RXD_DMAC_M
,
3042 u64_stats_update_begin(&ring
->syncp
);
3043 ring
->stats
.rx_pkts
++;
3044 ring
->stats
.rx_bytes
+= len
;
3046 if (l2_frame_type
== HNS3_L2_TYPE_MULTICAST
)
3047 ring
->stats
.rx_multicast
++;
3049 u64_stats_update_end(&ring
->syncp
);
3051 ring
->tqp_vector
->rx_group
.total_bytes
+= len
;
3053 hns3_set_rx_skb_rss_type(ring
, skb
, le32_to_cpu(desc
->rx
.rss_hash
));
3057 static int hns3_handle_rx_bd(struct hns3_enet_ring
*ring
)
3059 struct sk_buff
*skb
= ring
->skb
;
3060 struct hns3_desc_cb
*desc_cb
;
3061 struct hns3_desc
*desc
;
3062 unsigned int length
;
3066 desc
= &ring
->desc
[ring
->next_to_clean
];
3067 desc_cb
= &ring
->desc_cb
[ring
->next_to_clean
];
3071 length
= le16_to_cpu(desc
->rx
.size
);
3072 bd_base_info
= le32_to_cpu(desc
->rx
.bd_base_info
);
3074 /* Check valid BD */
3075 if (unlikely(!(bd_base_info
& BIT(HNS3_RXD_VLD_B
))))
3079 ring
->va
= desc_cb
->buf
+ desc_cb
->page_offset
;
3081 dma_sync_single_for_cpu(ring_to_dev(ring
),
3082 desc_cb
->dma
+ desc_cb
->page_offset
,
3083 hns3_buf_size(ring
),
3087 /* Prefetch first cache line of first page
3088 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
3089 * line size is 64B so need to prefetch twice to make it 128B. But in
3090 * actual we can have greater size of caches with 128B Level 1 cache
3091 * lines. In such a case, single fetch would suffice to cache in the
3092 * relevant part of the header.
3095 #if L1_CACHE_BYTES < 128
3096 prefetch(ring
->va
+ L1_CACHE_BYTES
);
3100 ret
= hns3_alloc_skb(ring
, length
, ring
->va
);
3103 if (ret
< 0) /* alloc buffer fail */
3105 if (!(bd_base_info
& BIT(HNS3_RXD_FE_B
))) { /* need add frag */
3106 ret
= hns3_add_frag(ring
);
3111 ret
= hns3_add_frag(ring
);
3116 /* As the head data may be changed when GRO enable, copy
3117 * the head data in after other data rx completed
3119 if (skb
->len
> HNS3_RX_HEAD_SIZE
)
3120 memcpy(skb
->data
, ring
->va
,
3121 ALIGN(ring
->pull_len
, sizeof(long)));
3123 ret
= hns3_handle_bdinfo(ring
, skb
);
3124 if (unlikely(ret
)) {
3125 dev_kfree_skb_any(skb
);
3129 skb_record_rx_queue(skb
, ring
->tqp
->tqp_index
);
3133 int hns3_clean_rx_ring(struct hns3_enet_ring
*ring
, int budget
,
3134 void (*rx_fn
)(struct hns3_enet_ring
*, struct sk_buff
*))
3136 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
3137 int unused_count
= hns3_desc_unused(ring
);
3142 num
= readl_relaxed(ring
->tqp
->io_base
+ HNS3_RING_RX_RING_FBDNUM_REG
);
3143 num
-= unused_count
;
3144 unused_count
-= ring
->pending_buf
;
3149 rmb(); /* Make sure num taken effect before the other data is touched */
3151 while (recv_pkts
< budget
&& recv_bds
< num
) {
3152 /* Reuse or realloc buffers */
3153 if (unused_count
>= RCB_NOF_ALLOC_RX_BUFF_ONCE
) {
3154 hns3_nic_alloc_rx_buffers(ring
, unused_count
);
3155 unused_count
= hns3_desc_unused(ring
) -
3160 err
= hns3_handle_rx_bd(ring
);
3161 /* Do not get FE for the packet or failed to alloc skb */
3162 if (unlikely(!ring
->skb
|| err
== -ENXIO
)) {
3164 } else if (likely(!err
)) {
3165 rx_fn(ring
, ring
->skb
);
3169 recv_bds
+= ring
->pending_buf
;
3170 unused_count
+= ring
->pending_buf
;
3172 ring
->pending_buf
= 0;
3176 /* Make all data has been write before submit */
3177 if (unused_count
> 0)
3178 hns3_nic_alloc_rx_buffers(ring
, unused_count
);
3183 static bool hns3_get_new_flow_lvl(struct hns3_enet_ring_group
*ring_group
)
3185 #define HNS3_RX_LOW_BYTE_RATE 10000
3186 #define HNS3_RX_MID_BYTE_RATE 20000
3187 #define HNS3_RX_ULTRA_PACKET_RATE 40
3189 enum hns3_flow_level_range new_flow_level
;
3190 struct hns3_enet_tqp_vector
*tqp_vector
;
3191 int packets_per_msecs
, bytes_per_msecs
;
3194 tqp_vector
= ring_group
->ring
->tqp_vector
;
3196 jiffies_to_msecs(jiffies
- tqp_vector
->last_jiffies
);
3197 if (!time_passed_ms
)
3200 do_div(ring_group
->total_packets
, time_passed_ms
);
3201 packets_per_msecs
= ring_group
->total_packets
;
3203 do_div(ring_group
->total_bytes
, time_passed_ms
);
3204 bytes_per_msecs
= ring_group
->total_bytes
;
3206 new_flow_level
= ring_group
->coal
.flow_level
;
3208 /* Simple throttlerate management
3209 * 0-10MB/s lower (50000 ints/s)
3210 * 10-20MB/s middle (20000 ints/s)
3211 * 20-1249MB/s high (18000 ints/s)
3212 * > 40000pps ultra (8000 ints/s)
3214 switch (new_flow_level
) {
3216 if (bytes_per_msecs
> HNS3_RX_LOW_BYTE_RATE
)
3217 new_flow_level
= HNS3_FLOW_MID
;
3220 if (bytes_per_msecs
> HNS3_RX_MID_BYTE_RATE
)
3221 new_flow_level
= HNS3_FLOW_HIGH
;
3222 else if (bytes_per_msecs
<= HNS3_RX_LOW_BYTE_RATE
)
3223 new_flow_level
= HNS3_FLOW_LOW
;
3225 case HNS3_FLOW_HIGH
:
3226 case HNS3_FLOW_ULTRA
:
3228 if (bytes_per_msecs
<= HNS3_RX_MID_BYTE_RATE
)
3229 new_flow_level
= HNS3_FLOW_MID
;
3233 if (packets_per_msecs
> HNS3_RX_ULTRA_PACKET_RATE
&&
3234 &tqp_vector
->rx_group
== ring_group
)
3235 new_flow_level
= HNS3_FLOW_ULTRA
;
3237 ring_group
->total_bytes
= 0;
3238 ring_group
->total_packets
= 0;
3239 ring_group
->coal
.flow_level
= new_flow_level
;
3244 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group
*ring_group
)
3246 struct hns3_enet_tqp_vector
*tqp_vector
;
3249 if (!ring_group
->ring
)
3252 tqp_vector
= ring_group
->ring
->tqp_vector
;
3253 if (!tqp_vector
->last_jiffies
)
3256 if (ring_group
->total_packets
== 0) {
3257 ring_group
->coal
.int_gl
= HNS3_INT_GL_50K
;
3258 ring_group
->coal
.flow_level
= HNS3_FLOW_LOW
;
3262 if (!hns3_get_new_flow_lvl(ring_group
))
3265 new_int_gl
= ring_group
->coal
.int_gl
;
3266 switch (ring_group
->coal
.flow_level
) {
3268 new_int_gl
= HNS3_INT_GL_50K
;
3271 new_int_gl
= HNS3_INT_GL_20K
;
3273 case HNS3_FLOW_HIGH
:
3274 new_int_gl
= HNS3_INT_GL_18K
;
3276 case HNS3_FLOW_ULTRA
:
3277 new_int_gl
= HNS3_INT_GL_8K
;
3283 if (new_int_gl
!= ring_group
->coal
.int_gl
) {
3284 ring_group
->coal
.int_gl
= new_int_gl
;
3290 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector
*tqp_vector
)
3292 struct hns3_enet_ring_group
*rx_group
= &tqp_vector
->rx_group
;
3293 struct hns3_enet_ring_group
*tx_group
= &tqp_vector
->tx_group
;
3294 bool rx_update
, tx_update
;
3296 /* update param every 1000ms */
3297 if (time_before(jiffies
,
3298 tqp_vector
->last_jiffies
+ msecs_to_jiffies(1000)))
3301 if (rx_group
->coal
.gl_adapt_enable
) {
3302 rx_update
= hns3_get_new_int_gl(rx_group
);
3304 hns3_set_vector_coalesce_rx_gl(tqp_vector
,
3305 rx_group
->coal
.int_gl
);
3308 if (tx_group
->coal
.gl_adapt_enable
) {
3309 tx_update
= hns3_get_new_int_gl(tx_group
);
3311 hns3_set_vector_coalesce_tx_gl(tqp_vector
,
3312 tx_group
->coal
.int_gl
);
3315 tqp_vector
->last_jiffies
= jiffies
;
3318 static int hns3_nic_common_poll(struct napi_struct
*napi
, int budget
)
3320 struct hns3_nic_priv
*priv
= netdev_priv(napi
->dev
);
3321 struct hns3_enet_ring
*ring
;
3322 int rx_pkt_total
= 0;
3324 struct hns3_enet_tqp_vector
*tqp_vector
=
3325 container_of(napi
, struct hns3_enet_tqp_vector
, napi
);
3326 bool clean_complete
= true;
3327 int rx_budget
= budget
;
3329 if (unlikely(test_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
))) {
3330 napi_complete(napi
);
3334 /* Since the actual Tx work is minimal, we can give the Tx a larger
3335 * budget and be more aggressive about cleaning up the Tx descriptors.
3337 hns3_for_each_ring(ring
, tqp_vector
->tx_group
)
3338 hns3_clean_tx_ring(ring
);
3340 /* make sure rx ring budget not smaller than 1 */
3341 if (tqp_vector
->num_tqps
> 1)
3342 rx_budget
= max(budget
/ tqp_vector
->num_tqps
, 1);
3344 hns3_for_each_ring(ring
, tqp_vector
->rx_group
) {
3345 int rx_cleaned
= hns3_clean_rx_ring(ring
, rx_budget
,
3348 if (rx_cleaned
>= rx_budget
)
3349 clean_complete
= false;
3351 rx_pkt_total
+= rx_cleaned
;
3354 tqp_vector
->rx_group
.total_packets
+= rx_pkt_total
;
3356 if (!clean_complete
)
3359 if (napi_complete(napi
) &&
3360 likely(!test_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
))) {
3361 hns3_update_new_int_gl(tqp_vector
);
3362 hns3_mask_vector_irq(tqp_vector
, 1);
3365 return rx_pkt_total
;
3368 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector
*tqp_vector
,
3369 struct hnae3_ring_chain_node
*head
)
3371 struct pci_dev
*pdev
= tqp_vector
->handle
->pdev
;
3372 struct hnae3_ring_chain_node
*cur_chain
= head
;
3373 struct hnae3_ring_chain_node
*chain
;
3374 struct hns3_enet_ring
*tx_ring
;
3375 struct hns3_enet_ring
*rx_ring
;
3377 tx_ring
= tqp_vector
->tx_group
.ring
;
3379 cur_chain
->tqp_index
= tx_ring
->tqp
->tqp_index
;
3380 hnae3_set_bit(cur_chain
->flag
, HNAE3_RING_TYPE_B
,
3381 HNAE3_RING_TYPE_TX
);
3382 hnae3_set_field(cur_chain
->int_gl_idx
, HNAE3_RING_GL_IDX_M
,
3383 HNAE3_RING_GL_IDX_S
, HNAE3_RING_GL_TX
);
3385 cur_chain
->next
= NULL
;
3387 while (tx_ring
->next
) {
3388 tx_ring
= tx_ring
->next
;
3390 chain
= devm_kzalloc(&pdev
->dev
, sizeof(*chain
),
3393 goto err_free_chain
;
3395 cur_chain
->next
= chain
;
3396 chain
->tqp_index
= tx_ring
->tqp
->tqp_index
;
3397 hnae3_set_bit(chain
->flag
, HNAE3_RING_TYPE_B
,
3398 HNAE3_RING_TYPE_TX
);
3399 hnae3_set_field(chain
->int_gl_idx
,
3400 HNAE3_RING_GL_IDX_M
,
3401 HNAE3_RING_GL_IDX_S
,
3408 rx_ring
= tqp_vector
->rx_group
.ring
;
3409 if (!tx_ring
&& rx_ring
) {
3410 cur_chain
->next
= NULL
;
3411 cur_chain
->tqp_index
= rx_ring
->tqp
->tqp_index
;
3412 hnae3_set_bit(cur_chain
->flag
, HNAE3_RING_TYPE_B
,
3413 HNAE3_RING_TYPE_RX
);
3414 hnae3_set_field(cur_chain
->int_gl_idx
, HNAE3_RING_GL_IDX_M
,
3415 HNAE3_RING_GL_IDX_S
, HNAE3_RING_GL_RX
);
3417 rx_ring
= rx_ring
->next
;
3421 chain
= devm_kzalloc(&pdev
->dev
, sizeof(*chain
), GFP_KERNEL
);
3423 goto err_free_chain
;
3425 cur_chain
->next
= chain
;
3426 chain
->tqp_index
= rx_ring
->tqp
->tqp_index
;
3427 hnae3_set_bit(chain
->flag
, HNAE3_RING_TYPE_B
,
3428 HNAE3_RING_TYPE_RX
);
3429 hnae3_set_field(chain
->int_gl_idx
, HNAE3_RING_GL_IDX_M
,
3430 HNAE3_RING_GL_IDX_S
, HNAE3_RING_GL_RX
);
3434 rx_ring
= rx_ring
->next
;
3440 cur_chain
= head
->next
;
3442 chain
= cur_chain
->next
;
3443 devm_kfree(&pdev
->dev
, cur_chain
);
3451 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector
*tqp_vector
,
3452 struct hnae3_ring_chain_node
*head
)
3454 struct pci_dev
*pdev
= tqp_vector
->handle
->pdev
;
3455 struct hnae3_ring_chain_node
*chain_tmp
, *chain
;
3460 chain_tmp
= chain
->next
;
3461 devm_kfree(&pdev
->dev
, chain
);
3466 static void hns3_add_ring_to_group(struct hns3_enet_ring_group
*group
,
3467 struct hns3_enet_ring
*ring
)
3469 ring
->next
= group
->ring
;
3475 static void hns3_nic_set_cpumask(struct hns3_nic_priv
*priv
)
3477 struct pci_dev
*pdev
= priv
->ae_handle
->pdev
;
3478 struct hns3_enet_tqp_vector
*tqp_vector
;
3479 int num_vectors
= priv
->vector_num
;
3483 numa_node
= dev_to_node(&pdev
->dev
);
3485 for (vector_i
= 0; vector_i
< num_vectors
; vector_i
++) {
3486 tqp_vector
= &priv
->tqp_vector
[vector_i
];
3487 cpumask_set_cpu(cpumask_local_spread(vector_i
, numa_node
),
3488 &tqp_vector
->affinity_mask
);
3492 static int hns3_nic_init_vector_data(struct hns3_nic_priv
*priv
)
3494 struct hnae3_ring_chain_node vector_ring_chain
;
3495 struct hnae3_handle
*h
= priv
->ae_handle
;
3496 struct hns3_enet_tqp_vector
*tqp_vector
;
3500 hns3_nic_set_cpumask(priv
);
3502 for (i
= 0; i
< priv
->vector_num
; i
++) {
3503 tqp_vector
= &priv
->tqp_vector
[i
];
3504 hns3_vector_gl_rl_init_hw(tqp_vector
, priv
);
3505 tqp_vector
->num_tqps
= 0;
3508 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++) {
3509 u16 vector_i
= i
% priv
->vector_num
;
3510 u16 tqp_num
= h
->kinfo
.num_tqps
;
3512 tqp_vector
= &priv
->tqp_vector
[vector_i
];
3514 hns3_add_ring_to_group(&tqp_vector
->tx_group
,
3517 hns3_add_ring_to_group(&tqp_vector
->rx_group
,
3518 &priv
->ring
[i
+ tqp_num
]);
3520 priv
->ring
[i
].tqp_vector
= tqp_vector
;
3521 priv
->ring
[i
+ tqp_num
].tqp_vector
= tqp_vector
;
3522 tqp_vector
->num_tqps
++;
3525 for (i
= 0; i
< priv
->vector_num
; i
++) {
3526 tqp_vector
= &priv
->tqp_vector
[i
];
3528 tqp_vector
->rx_group
.total_bytes
= 0;
3529 tqp_vector
->rx_group
.total_packets
= 0;
3530 tqp_vector
->tx_group
.total_bytes
= 0;
3531 tqp_vector
->tx_group
.total_packets
= 0;
3532 tqp_vector
->handle
= h
;
3534 ret
= hns3_get_vector_ring_chain(tqp_vector
,
3535 &vector_ring_chain
);
3539 ret
= h
->ae_algo
->ops
->map_ring_to_vector(h
,
3540 tqp_vector
->vector_irq
, &vector_ring_chain
);
3542 hns3_free_vector_ring_chain(tqp_vector
, &vector_ring_chain
);
3547 netif_napi_add(priv
->netdev
, &tqp_vector
->napi
,
3548 hns3_nic_common_poll
, NAPI_POLL_WEIGHT
);
3555 netif_napi_del(&priv
->tqp_vector
[i
].napi
);
3560 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv
*priv
)
3562 #define HNS3_VECTOR_PF_MAX_NUM 64
3564 struct hnae3_handle
*h
= priv
->ae_handle
;
3565 struct hns3_enet_tqp_vector
*tqp_vector
;
3566 struct hnae3_vector_info
*vector
;
3567 struct pci_dev
*pdev
= h
->pdev
;
3568 u16 tqp_num
= h
->kinfo
.num_tqps
;
3573 /* RSS size, cpu online and vector_num should be the same */
3574 /* Should consider 2p/4p later */
3575 vector_num
= min_t(u16
, num_online_cpus(), tqp_num
);
3576 vector_num
= min_t(u16
, vector_num
, HNS3_VECTOR_PF_MAX_NUM
);
3578 vector
= devm_kcalloc(&pdev
->dev
, vector_num
, sizeof(*vector
),
3583 /* save the actual available vector number */
3584 vector_num
= h
->ae_algo
->ops
->get_vector(h
, vector_num
, vector
);
3586 priv
->vector_num
= vector_num
;
3587 priv
->tqp_vector
= (struct hns3_enet_tqp_vector
*)
3588 devm_kcalloc(&pdev
->dev
, vector_num
, sizeof(*priv
->tqp_vector
),
3590 if (!priv
->tqp_vector
) {
3595 for (i
= 0; i
< priv
->vector_num
; i
++) {
3596 tqp_vector
= &priv
->tqp_vector
[i
];
3597 tqp_vector
->idx
= i
;
3598 tqp_vector
->mask_addr
= vector
[i
].io_addr
;
3599 tqp_vector
->vector_irq
= vector
[i
].vector
;
3600 hns3_vector_gl_rl_init(tqp_vector
, priv
);
3604 devm_kfree(&pdev
->dev
, vector
);
3608 static void hns3_clear_ring_group(struct hns3_enet_ring_group
*group
)
3614 static void hns3_nic_uninit_vector_data(struct hns3_nic_priv
*priv
)
3616 struct hnae3_ring_chain_node vector_ring_chain
;
3617 struct hnae3_handle
*h
= priv
->ae_handle
;
3618 struct hns3_enet_tqp_vector
*tqp_vector
;
3621 for (i
= 0; i
< priv
->vector_num
; i
++) {
3622 tqp_vector
= &priv
->tqp_vector
[i
];
3624 if (!tqp_vector
->rx_group
.ring
&& !tqp_vector
->tx_group
.ring
)
3627 /* Since the mapping can be overwritten, when fail to get the
3628 * chain between vector and ring, we should go on to deal with
3629 * the remaining options.
3631 if (hns3_get_vector_ring_chain(tqp_vector
, &vector_ring_chain
))
3632 dev_warn(priv
->dev
, "failed to get ring chain\n");
3634 h
->ae_algo
->ops
->unmap_ring_from_vector(h
,
3635 tqp_vector
->vector_irq
, &vector_ring_chain
);
3637 hns3_free_vector_ring_chain(tqp_vector
, &vector_ring_chain
);
3639 hns3_clear_ring_group(&tqp_vector
->rx_group
);
3640 hns3_clear_ring_group(&tqp_vector
->tx_group
);
3641 netif_napi_del(&priv
->tqp_vector
[i
].napi
);
3645 static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv
*priv
)
3647 struct hnae3_handle
*h
= priv
->ae_handle
;
3648 struct pci_dev
*pdev
= h
->pdev
;
3651 for (i
= 0; i
< priv
->vector_num
; i
++) {
3652 struct hns3_enet_tqp_vector
*tqp_vector
;
3654 tqp_vector
= &priv
->tqp_vector
[i
];
3655 ret
= h
->ae_algo
->ops
->put_vector(h
, tqp_vector
->vector_irq
);
3660 devm_kfree(&pdev
->dev
, priv
->tqp_vector
);
3663 static void hns3_ring_get_cfg(struct hnae3_queue
*q
, struct hns3_nic_priv
*priv
,
3664 unsigned int ring_type
)
3666 int queue_num
= priv
->ae_handle
->kinfo
.num_tqps
;
3667 struct hns3_enet_ring
*ring
;
3670 if (ring_type
== HNAE3_RING_TYPE_TX
) {
3671 ring
= &priv
->ring
[q
->tqp_index
];
3672 desc_num
= priv
->ae_handle
->kinfo
.num_tx_desc
;
3673 ring
->queue_index
= q
->tqp_index
;
3674 ring
->io_base
= (u8 __iomem
*)q
->io_base
+ HNS3_TX_REG_OFFSET
;
3676 ring
= &priv
->ring
[q
->tqp_index
+ queue_num
];
3677 desc_num
= priv
->ae_handle
->kinfo
.num_rx_desc
;
3678 ring
->queue_index
= q
->tqp_index
;
3679 ring
->io_base
= q
->io_base
;
3682 hnae3_set_bit(ring
->flag
, HNAE3_RING_TYPE_B
, ring_type
);
3686 ring
->desc_cb
= NULL
;
3687 ring
->dev
= priv
->dev
;
3688 ring
->desc_dma_addr
= 0;
3689 ring
->buf_size
= q
->buf_size
;
3690 ring
->desc_num
= desc_num
;
3691 ring
->next_to_use
= 0;
3692 ring
->next_to_clean
= 0;
3695 static void hns3_queue_to_ring(struct hnae3_queue
*tqp
,
3696 struct hns3_nic_priv
*priv
)
3698 hns3_ring_get_cfg(tqp
, priv
, HNAE3_RING_TYPE_TX
);
3699 hns3_ring_get_cfg(tqp
, priv
, HNAE3_RING_TYPE_RX
);
3702 static int hns3_get_ring_config(struct hns3_nic_priv
*priv
)
3704 struct hnae3_handle
*h
= priv
->ae_handle
;
3705 struct pci_dev
*pdev
= h
->pdev
;
3708 priv
->ring
= devm_kzalloc(&pdev
->dev
,
3709 array3_size(h
->kinfo
.num_tqps
,
3710 sizeof(*priv
->ring
), 2),
3715 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++)
3716 hns3_queue_to_ring(h
->kinfo
.tqp
[i
], priv
);
3721 static void hns3_put_ring_config(struct hns3_nic_priv
*priv
)
3726 devm_kfree(priv
->dev
, priv
->ring
);
3730 static int hns3_alloc_ring_memory(struct hns3_enet_ring
*ring
)
3734 if (ring
->desc_num
<= 0 || ring
->buf_size
<= 0)
3737 ring
->desc_cb
= devm_kcalloc(ring_to_dev(ring
), ring
->desc_num
,
3738 sizeof(ring
->desc_cb
[0]), GFP_KERNEL
);
3739 if (!ring
->desc_cb
) {
3744 ret
= hns3_alloc_desc(ring
);
3746 goto out_with_desc_cb
;
3748 if (!HNAE3_IS_TX_RING(ring
)) {
3749 ret
= hns3_alloc_ring_buffers(ring
);
3757 hns3_free_desc(ring
);
3759 devm_kfree(ring_to_dev(ring
), ring
->desc_cb
);
3760 ring
->desc_cb
= NULL
;
3765 void hns3_fini_ring(struct hns3_enet_ring
*ring
)
3767 hns3_free_desc(ring
);
3768 devm_kfree(ring_to_dev(ring
), ring
->desc_cb
);
3769 ring
->desc_cb
= NULL
;
3770 ring
->next_to_clean
= 0;
3771 ring
->next_to_use
= 0;
3772 ring
->pending_buf
= 0;
3774 dev_kfree_skb_any(ring
->skb
);
3779 static int hns3_buf_size2type(u32 buf_size
)
3785 bd_size_type
= HNS3_BD_SIZE_512_TYPE
;
3788 bd_size_type
= HNS3_BD_SIZE_1024_TYPE
;
3791 bd_size_type
= HNS3_BD_SIZE_2048_TYPE
;
3794 bd_size_type
= HNS3_BD_SIZE_4096_TYPE
;
3797 bd_size_type
= HNS3_BD_SIZE_2048_TYPE
;
3800 return bd_size_type
;
3803 static void hns3_init_ring_hw(struct hns3_enet_ring
*ring
)
3805 dma_addr_t dma
= ring
->desc_dma_addr
;
3806 struct hnae3_queue
*q
= ring
->tqp
;
3808 if (!HNAE3_IS_TX_RING(ring
)) {
3809 hns3_write_dev(q
, HNS3_RING_RX_RING_BASEADDR_L_REG
, (u32
)dma
);
3810 hns3_write_dev(q
, HNS3_RING_RX_RING_BASEADDR_H_REG
,
3811 (u32
)((dma
>> 31) >> 1));
3813 hns3_write_dev(q
, HNS3_RING_RX_RING_BD_LEN_REG
,
3814 hns3_buf_size2type(ring
->buf_size
));
3815 hns3_write_dev(q
, HNS3_RING_RX_RING_BD_NUM_REG
,
3816 ring
->desc_num
/ 8 - 1);
3819 hns3_write_dev(q
, HNS3_RING_TX_RING_BASEADDR_L_REG
,
3821 hns3_write_dev(q
, HNS3_RING_TX_RING_BASEADDR_H_REG
,
3822 (u32
)((dma
>> 31) >> 1));
3824 hns3_write_dev(q
, HNS3_RING_TX_RING_BD_NUM_REG
,
3825 ring
->desc_num
/ 8 - 1);
3829 static void hns3_init_tx_ring_tc(struct hns3_nic_priv
*priv
)
3831 struct hnae3_knic_private_info
*kinfo
= &priv
->ae_handle
->kinfo
;
3834 for (i
= 0; i
< HNAE3_MAX_TC
; i
++) {
3835 struct hnae3_tc_info
*tc_info
= &kinfo
->tc_info
[i
];
3838 if (!tc_info
->enable
)
3841 for (j
= 0; j
< tc_info
->tqp_count
; j
++) {
3842 struct hnae3_queue
*q
;
3844 q
= priv
->ring
[tc_info
->tqp_offset
+ j
].tqp
;
3845 hns3_write_dev(q
, HNS3_RING_TX_RING_TC_REG
,
3851 int hns3_init_all_ring(struct hns3_nic_priv
*priv
)
3853 struct hnae3_handle
*h
= priv
->ae_handle
;
3854 int ring_num
= h
->kinfo
.num_tqps
* 2;
3858 for (i
= 0; i
< ring_num
; i
++) {
3859 ret
= hns3_alloc_ring_memory(&priv
->ring
[i
]);
3862 "Alloc ring memory fail! ret=%d\n", ret
);
3863 goto out_when_alloc_ring_memory
;
3866 u64_stats_init(&priv
->ring
[i
].syncp
);
3871 out_when_alloc_ring_memory
:
3872 for (j
= i
- 1; j
>= 0; j
--)
3873 hns3_fini_ring(&priv
->ring
[j
]);
3878 int hns3_uninit_all_ring(struct hns3_nic_priv
*priv
)
3880 struct hnae3_handle
*h
= priv
->ae_handle
;
3883 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++) {
3884 hns3_fini_ring(&priv
->ring
[i
]);
3885 hns3_fini_ring(&priv
->ring
[i
+ h
->kinfo
.num_tqps
]);
3890 /* Set mac addr if it is configured. or leave it to the AE driver */
3891 static int hns3_init_mac_addr(struct net_device
*netdev
)
3893 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
3894 struct hnae3_handle
*h
= priv
->ae_handle
;
3895 u8 mac_addr_temp
[ETH_ALEN
];
3898 if (h
->ae_algo
->ops
->get_mac_addr
)
3899 h
->ae_algo
->ops
->get_mac_addr(h
, mac_addr_temp
);
3901 /* Check if the MAC address is valid, if not get a random one */
3902 if (!is_valid_ether_addr(mac_addr_temp
)) {
3903 eth_hw_addr_random(netdev
);
3904 dev_warn(priv
->dev
, "using random MAC address %pM\n",
3906 } else if (!ether_addr_equal(netdev
->dev_addr
, mac_addr_temp
)) {
3907 ether_addr_copy(netdev
->dev_addr
, mac_addr_temp
);
3908 ether_addr_copy(netdev
->perm_addr
, mac_addr_temp
);
3913 if (h
->ae_algo
->ops
->set_mac_addr
)
3914 ret
= h
->ae_algo
->ops
->set_mac_addr(h
, netdev
->dev_addr
, true);
3919 static int hns3_init_phy(struct net_device
*netdev
)
3921 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
3924 if (h
->ae_algo
->ops
->mac_connect_phy
)
3925 ret
= h
->ae_algo
->ops
->mac_connect_phy(h
);
3930 static void hns3_uninit_phy(struct net_device
*netdev
)
3932 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
3934 if (h
->ae_algo
->ops
->mac_disconnect_phy
)
3935 h
->ae_algo
->ops
->mac_disconnect_phy(h
);
3938 static void hns3_del_all_fd_rules(struct net_device
*netdev
, bool clear_list
)
3940 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
3942 if (h
->ae_algo
->ops
->del_all_fd_entries
)
3943 h
->ae_algo
->ops
->del_all_fd_entries(h
, clear_list
);
3946 static int hns3_client_start(struct hnae3_handle
*handle
)
3948 if (!handle
->ae_algo
->ops
->client_start
)
3951 return handle
->ae_algo
->ops
->client_start(handle
);
3954 static void hns3_client_stop(struct hnae3_handle
*handle
)
3956 if (!handle
->ae_algo
->ops
->client_stop
)
3959 handle
->ae_algo
->ops
->client_stop(handle
);
3962 static void hns3_info_show(struct hns3_nic_priv
*priv
)
3964 struct hnae3_knic_private_info
*kinfo
= &priv
->ae_handle
->kinfo
;
3966 dev_info(priv
->dev
, "MAC address: %pM\n", priv
->netdev
->dev_addr
);
3967 dev_info(priv
->dev
, "Task queue pairs numbers: %u\n", kinfo
->num_tqps
);
3968 dev_info(priv
->dev
, "RSS size: %u\n", kinfo
->rss_size
);
3969 dev_info(priv
->dev
, "Allocated RSS size: %u\n", kinfo
->req_rss_size
);
3970 dev_info(priv
->dev
, "RX buffer length: %u\n", kinfo
->rx_buf_len
);
3971 dev_info(priv
->dev
, "Desc num per TX queue: %u\n", kinfo
->num_tx_desc
);
3972 dev_info(priv
->dev
, "Desc num per RX queue: %u\n", kinfo
->num_rx_desc
);
3973 dev_info(priv
->dev
, "Total number of enabled TCs: %u\n", kinfo
->num_tc
);
3974 dev_info(priv
->dev
, "Max mtu size: %u\n", priv
->netdev
->max_mtu
);
3977 static int hns3_client_init(struct hnae3_handle
*handle
)
3979 struct pci_dev
*pdev
= handle
->pdev
;
3980 u16 alloc_tqps
, max_rss_size
;
3981 struct hns3_nic_priv
*priv
;
3982 struct net_device
*netdev
;
3985 handle
->ae_algo
->ops
->get_tqps_and_rss_info(handle
, &alloc_tqps
,
3987 netdev
= alloc_etherdev_mq(sizeof(struct hns3_nic_priv
), alloc_tqps
);
3991 priv
= netdev_priv(netdev
);
3992 priv
->dev
= &pdev
->dev
;
3993 priv
->netdev
= netdev
;
3994 priv
->ae_handle
= handle
;
3995 priv
->tx_timeout_count
= 0;
3996 set_bit(HNS3_NIC_STATE_DOWN
, &priv
->state
);
3998 handle
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_LEVEL
);
4000 handle
->kinfo
.netdev
= netdev
;
4001 handle
->priv
= (void *)priv
;
4003 hns3_init_mac_addr(netdev
);
4005 hns3_set_default_feature(netdev
);
4007 netdev
->watchdog_timeo
= HNS3_TX_TIMEOUT
;
4008 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
4009 netdev
->netdev_ops
= &hns3_nic_netdev_ops
;
4010 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
4011 hns3_ethtool_set_ops(netdev
);
4013 /* Carrier off reporting is important to ethtool even BEFORE open */
4014 netif_carrier_off(netdev
);
4016 ret
= hns3_get_ring_config(priv
);
4019 goto out_get_ring_cfg
;
4022 ret
= hns3_nic_alloc_vector_data(priv
);
4025 goto out_alloc_vector_data
;
4028 ret
= hns3_nic_init_vector_data(priv
);
4031 goto out_init_vector_data
;
4034 ret
= hns3_init_all_ring(priv
);
4040 ret
= hns3_init_phy(netdev
);
4044 ret
= register_netdev(netdev
);
4046 dev_err(priv
->dev
, "probe register netdev fail!\n");
4047 goto out_reg_netdev_fail
;
4050 /* the device can work without cpu rmap, only aRFS needs it */
4051 ret
= hns3_set_rx_cpu_rmap(netdev
);
4053 dev_warn(priv
->dev
, "set rx cpu rmap fail, ret=%d\n", ret
);
4055 ret
= hns3_nic_init_irq(priv
);
4057 dev_err(priv
->dev
, "init irq failed! ret=%d\n", ret
);
4058 hns3_free_rx_cpu_rmap(netdev
);
4059 goto out_init_irq_fail
;
4062 ret
= hns3_client_start(handle
);
4064 dev_err(priv
->dev
, "hns3_client_start fail! ret=%d\n", ret
);
4065 goto out_client_start
;
4068 hns3_dcbnl_setup(handle
);
4070 hns3_dbg_init(handle
);
4072 /* MTU range: (ETH_MIN_MTU(kernel default) - 9702) */
4073 netdev
->max_mtu
= HNS3_MAX_MTU
;
4075 set_bit(HNS3_NIC_STATE_INITED
, &priv
->state
);
4077 if (netif_msg_drv(handle
))
4078 hns3_info_show(priv
);
4083 hns3_free_rx_cpu_rmap(netdev
);
4084 hns3_nic_uninit_irq(priv
);
4086 unregister_netdev(netdev
);
4087 out_reg_netdev_fail
:
4088 hns3_uninit_phy(netdev
);
4090 hns3_uninit_all_ring(priv
);
4092 hns3_nic_uninit_vector_data(priv
);
4093 out_init_vector_data
:
4094 hns3_nic_dealloc_vector_data(priv
);
4095 out_alloc_vector_data
:
4098 priv
->ae_handle
= NULL
;
4099 free_netdev(netdev
);
4103 static void hns3_client_uninit(struct hnae3_handle
*handle
, bool reset
)
4105 struct net_device
*netdev
= handle
->kinfo
.netdev
;
4106 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
4109 if (netdev
->reg_state
!= NETREG_UNINITIALIZED
)
4110 unregister_netdev(netdev
);
4112 hns3_client_stop(handle
);
4114 hns3_uninit_phy(netdev
);
4116 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED
, &priv
->state
)) {
4117 netdev_warn(netdev
, "already uninitialized\n");
4118 goto out_netdev_free
;
4121 hns3_free_rx_cpu_rmap(netdev
);
4123 hns3_nic_uninit_irq(priv
);
4125 hns3_del_all_fd_rules(netdev
, true);
4127 hns3_clear_all_ring(handle
, true);
4129 hns3_nic_uninit_vector_data(priv
);
4131 hns3_nic_dealloc_vector_data(priv
);
4133 ret
= hns3_uninit_all_ring(priv
);
4135 netdev_err(netdev
, "uninit ring error\n");
4137 hns3_put_ring_config(priv
);
4140 hns3_dbg_uninit(handle
);
4141 free_netdev(netdev
);
4144 static void hns3_link_status_change(struct hnae3_handle
*handle
, bool linkup
)
4146 struct net_device
*netdev
= handle
->kinfo
.netdev
;
4152 netif_tx_wake_all_queues(netdev
);
4153 netif_carrier_on(netdev
);
4154 if (netif_msg_link(handle
))
4155 netdev_info(netdev
, "link up\n");
4157 netif_carrier_off(netdev
);
4158 netif_tx_stop_all_queues(netdev
);
4159 if (netif_msg_link(handle
))
4160 netdev_info(netdev
, "link down\n");
4164 static int hns3_client_setup_tc(struct hnae3_handle
*handle
, u8 tc
)
4166 struct hnae3_knic_private_info
*kinfo
= &handle
->kinfo
;
4167 struct net_device
*ndev
= kinfo
->netdev
;
4169 if (tc
> HNAE3_MAX_TC
)
4175 return hns3_nic_set_real_num_queue(ndev
);
4178 static void hns3_clear_tx_ring(struct hns3_enet_ring
*ring
)
4180 while (ring
->next_to_clean
!= ring
->next_to_use
) {
4181 ring
->desc
[ring
->next_to_clean
].tx
.bdtp_fe_sc_vld_ra_ri
= 0;
4182 hns3_free_buffer_detach(ring
, ring
->next_to_clean
);
4183 ring_ptr_move_fw(ring
, next_to_clean
);
4187 static int hns3_clear_rx_ring(struct hns3_enet_ring
*ring
)
4189 struct hns3_desc_cb res_cbs
;
4192 while (ring
->next_to_use
!= ring
->next_to_clean
) {
4193 /* When a buffer is not reused, it's memory has been
4194 * freed in hns3_handle_rx_bd or will be freed by
4195 * stack, so we need to replace the buffer here.
4197 if (!ring
->desc_cb
[ring
->next_to_use
].reuse_flag
) {
4198 ret
= hns3_alloc_and_map_buffer(ring
, &res_cbs
);
4200 u64_stats_update_begin(&ring
->syncp
);
4201 ring
->stats
.sw_err_cnt
++;
4202 u64_stats_update_end(&ring
->syncp
);
4203 /* if alloc new buffer fail, exit directly
4204 * and reclear in up flow.
4206 netdev_warn(ring_to_netdev(ring
),
4207 "reserve buffer map failed, ret = %d\n",
4211 hns3_replace_buffer(ring
, ring
->next_to_use
, &res_cbs
);
4213 ring_ptr_move_fw(ring
, next_to_use
);
4216 /* Free the pending skb in rx ring */
4218 dev_kfree_skb_any(ring
->skb
);
4220 ring
->pending_buf
= 0;
4226 static void hns3_force_clear_rx_ring(struct hns3_enet_ring
*ring
)
4228 while (ring
->next_to_use
!= ring
->next_to_clean
) {
4229 /* When a buffer is not reused, it's memory has been
4230 * freed in hns3_handle_rx_bd or will be freed by
4231 * stack, so only need to unmap the buffer here.
4233 if (!ring
->desc_cb
[ring
->next_to_use
].reuse_flag
) {
4234 hns3_unmap_buffer(ring
,
4235 &ring
->desc_cb
[ring
->next_to_use
]);
4236 ring
->desc_cb
[ring
->next_to_use
].dma
= 0;
4239 ring_ptr_move_fw(ring
, next_to_use
);
4243 static void hns3_clear_all_ring(struct hnae3_handle
*h
, bool force
)
4245 struct net_device
*ndev
= h
->kinfo
.netdev
;
4246 struct hns3_nic_priv
*priv
= netdev_priv(ndev
);
4249 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++) {
4250 struct hns3_enet_ring
*ring
;
4252 ring
= &priv
->ring
[i
];
4253 hns3_clear_tx_ring(ring
);
4255 ring
= &priv
->ring
[i
+ h
->kinfo
.num_tqps
];
4256 /* Continue to clear other rings even if clearing some
4260 hns3_force_clear_rx_ring(ring
);
4262 hns3_clear_rx_ring(ring
);
4266 int hns3_nic_reset_all_ring(struct hnae3_handle
*h
)
4268 struct net_device
*ndev
= h
->kinfo
.netdev
;
4269 struct hns3_nic_priv
*priv
= netdev_priv(ndev
);
4270 struct hns3_enet_ring
*rx_ring
;
4274 for (i
= 0; i
< h
->kinfo
.num_tqps
; i
++) {
4275 ret
= h
->ae_algo
->ops
->reset_queue(h
, i
);
4279 hns3_init_ring_hw(&priv
->ring
[i
]);
4281 /* We need to clear tx ring here because self test will
4282 * use the ring and will not run down before up
4284 hns3_clear_tx_ring(&priv
->ring
[i
]);
4285 priv
->ring
[i
].next_to_clean
= 0;
4286 priv
->ring
[i
].next_to_use
= 0;
4288 rx_ring
= &priv
->ring
[i
+ h
->kinfo
.num_tqps
];
4289 hns3_init_ring_hw(rx_ring
);
4290 ret
= hns3_clear_rx_ring(rx_ring
);
4294 /* We can not know the hardware head and tail when this
4295 * function is called in reset flow, so we reuse all desc.
4297 for (j
= 0; j
< rx_ring
->desc_num
; j
++)
4298 hns3_reuse_buffer(rx_ring
, j
);
4300 rx_ring
->next_to_clean
= 0;
4301 rx_ring
->next_to_use
= 0;
4304 hns3_init_tx_ring_tc(priv
);
4309 static void hns3_store_coal(struct hns3_nic_priv
*priv
)
4311 /* ethtool only support setting and querying one coal
4312 * configuration for now, so save the vector 0' coal
4313 * configuration here in order to restore it.
4315 memcpy(&priv
->tx_coal
, &priv
->tqp_vector
[0].tx_group
.coal
,
4316 sizeof(struct hns3_enet_coalesce
));
4317 memcpy(&priv
->rx_coal
, &priv
->tqp_vector
[0].rx_group
.coal
,
4318 sizeof(struct hns3_enet_coalesce
));
4321 static void hns3_restore_coal(struct hns3_nic_priv
*priv
)
4323 u16 vector_num
= priv
->vector_num
;
4326 for (i
= 0; i
< vector_num
; i
++) {
4327 memcpy(&priv
->tqp_vector
[i
].tx_group
.coal
, &priv
->tx_coal
,
4328 sizeof(struct hns3_enet_coalesce
));
4329 memcpy(&priv
->tqp_vector
[i
].rx_group
.coal
, &priv
->rx_coal
,
4330 sizeof(struct hns3_enet_coalesce
));
4334 static int hns3_reset_notify_down_enet(struct hnae3_handle
*handle
)
4336 struct hnae3_knic_private_info
*kinfo
= &handle
->kinfo
;
4337 struct net_device
*ndev
= kinfo
->netdev
;
4338 struct hns3_nic_priv
*priv
= netdev_priv(ndev
);
4340 if (test_and_set_bit(HNS3_NIC_STATE_RESETTING
, &priv
->state
))
4343 if (!netif_running(ndev
))
4346 return hns3_nic_net_stop(ndev
);
4349 static int hns3_reset_notify_up_enet(struct hnae3_handle
*handle
)
4351 struct hnae3_knic_private_info
*kinfo
= &handle
->kinfo
;
4352 struct hns3_nic_priv
*priv
= netdev_priv(kinfo
->netdev
);
4355 clear_bit(HNS3_NIC_STATE_RESETTING
, &priv
->state
);
4357 if (netif_running(kinfo
->netdev
)) {
4358 ret
= hns3_nic_net_open(kinfo
->netdev
);
4360 set_bit(HNS3_NIC_STATE_RESETTING
, &priv
->state
);
4361 netdev_err(kinfo
->netdev
,
4362 "net up fail, ret=%d!\n", ret
);
4370 static int hns3_reset_notify_init_enet(struct hnae3_handle
*handle
)
4372 struct net_device
*netdev
= handle
->kinfo
.netdev
;
4373 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
4376 /* Carrier off reporting is important to ethtool even BEFORE open */
4377 netif_carrier_off(netdev
);
4379 ret
= hns3_get_ring_config(priv
);
4383 ret
= hns3_nic_alloc_vector_data(priv
);
4387 hns3_restore_coal(priv
);
4389 ret
= hns3_nic_init_vector_data(priv
);
4391 goto err_dealloc_vector
;
4393 ret
= hns3_init_all_ring(priv
);
4395 goto err_uninit_vector
;
4397 /* the device can work without cpu rmap, only aRFS needs it */
4398 ret
= hns3_set_rx_cpu_rmap(netdev
);
4400 dev_warn(priv
->dev
, "set rx cpu rmap fail, ret=%d\n", ret
);
4402 ret
= hns3_nic_init_irq(priv
);
4404 dev_err(priv
->dev
, "init irq failed! ret=%d\n", ret
);
4405 hns3_free_rx_cpu_rmap(netdev
);
4406 goto err_init_irq_fail
;
4409 if (!hns3_is_phys_func(handle
->pdev
))
4410 hns3_init_mac_addr(netdev
);
4412 ret
= hns3_client_start(handle
);
4414 dev_err(priv
->dev
, "hns3_client_start fail! ret=%d\n", ret
);
4415 goto err_client_start_fail
;
4418 set_bit(HNS3_NIC_STATE_INITED
, &priv
->state
);
4422 err_client_start_fail
:
4423 hns3_free_rx_cpu_rmap(netdev
);
4424 hns3_nic_uninit_irq(priv
);
4426 hns3_uninit_all_ring(priv
);
4428 hns3_nic_uninit_vector_data(priv
);
4430 hns3_nic_dealloc_vector_data(priv
);
4432 hns3_put_ring_config(priv
);
4437 static int hns3_reset_notify_uninit_enet(struct hnae3_handle
*handle
)
4439 struct net_device
*netdev
= handle
->kinfo
.netdev
;
4440 struct hns3_nic_priv
*priv
= netdev_priv(netdev
);
4443 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED
, &priv
->state
)) {
4444 netdev_warn(netdev
, "already uninitialized\n");
4448 hns3_free_rx_cpu_rmap(netdev
);
4449 hns3_nic_uninit_irq(priv
);
4450 hns3_clear_all_ring(handle
, true);
4451 hns3_reset_tx_queue(priv
->ae_handle
);
4453 hns3_nic_uninit_vector_data(priv
);
4455 hns3_store_coal(priv
);
4457 hns3_nic_dealloc_vector_data(priv
);
4459 ret
= hns3_uninit_all_ring(priv
);
4461 netdev_err(netdev
, "uninit ring error\n");
4463 hns3_put_ring_config(priv
);
4468 static int hns3_reset_notify(struct hnae3_handle
*handle
,
4469 enum hnae3_reset_notify_type type
)
4474 case HNAE3_UP_CLIENT
:
4475 ret
= hns3_reset_notify_up_enet(handle
);
4477 case HNAE3_DOWN_CLIENT
:
4478 ret
= hns3_reset_notify_down_enet(handle
);
4480 case HNAE3_INIT_CLIENT
:
4481 ret
= hns3_reset_notify_init_enet(handle
);
4483 case HNAE3_UNINIT_CLIENT
:
4484 ret
= hns3_reset_notify_uninit_enet(handle
);
4493 static int hns3_change_channels(struct hnae3_handle
*handle
, u32 new_tqp_num
,
4494 bool rxfh_configured
)
4498 ret
= handle
->ae_algo
->ops
->set_channels(handle
, new_tqp_num
,
4501 dev_err(&handle
->pdev
->dev
,
4502 "Change tqp num(%u) fail.\n", new_tqp_num
);
4506 ret
= hns3_reset_notify(handle
, HNAE3_INIT_CLIENT
);
4510 ret
= hns3_reset_notify(handle
, HNAE3_UP_CLIENT
);
4512 hns3_reset_notify(handle
, HNAE3_UNINIT_CLIENT
);
4517 int hns3_set_channels(struct net_device
*netdev
,
4518 struct ethtool_channels
*ch
)
4520 struct hnae3_handle
*h
= hns3_get_handle(netdev
);
4521 struct hnae3_knic_private_info
*kinfo
= &h
->kinfo
;
4522 bool rxfh_configured
= netif_is_rxfh_configured(netdev
);
4523 u32 new_tqp_num
= ch
->combined_count
;
4527 if (hns3_nic_resetting(netdev
))
4530 if (ch
->rx_count
|| ch
->tx_count
)
4533 if (new_tqp_num
> hns3_get_max_available_channels(h
) ||
4535 dev_err(&netdev
->dev
,
4536 "Change tqps fail, the tqp range is from 1 to %u",
4537 hns3_get_max_available_channels(h
));
4541 if (kinfo
->rss_size
== new_tqp_num
)
4544 netif_dbg(h
, drv
, netdev
,
4545 "set channels: tqp_num=%u, rxfh=%d\n",
4546 new_tqp_num
, rxfh_configured
);
4548 ret
= hns3_reset_notify(h
, HNAE3_DOWN_CLIENT
);
4552 ret
= hns3_reset_notify(h
, HNAE3_UNINIT_CLIENT
);
4556 org_tqp_num
= h
->kinfo
.num_tqps
;
4557 ret
= hns3_change_channels(h
, new_tqp_num
, rxfh_configured
);
4562 "Change channels fail, revert to old value\n");
4563 ret1
= hns3_change_channels(h
, org_tqp_num
, rxfh_configured
);
4566 "revert to old channel fail\n");
4576 static const struct hns3_hw_error_info hns3_hw_err
[] = {
4577 { .type
= HNAE3_PPU_POISON_ERROR
,
4578 .msg
= "PPU poison" },
4579 { .type
= HNAE3_CMDQ_ECC_ERROR
,
4580 .msg
= "IMP CMDQ error" },
4581 { .type
= HNAE3_IMP_RD_POISON_ERROR
,
4582 .msg
= "IMP RD poison" },
4585 static void hns3_process_hw_error(struct hnae3_handle
*handle
,
4586 enum hnae3_hw_error_type type
)
4590 for (i
= 0; i
< ARRAY_SIZE(hns3_hw_err
); i
++) {
4591 if (hns3_hw_err
[i
].type
== type
) {
4592 dev_err(&handle
->pdev
->dev
, "Detected %s!\n",
4593 hns3_hw_err
[i
].msg
);
4599 static const struct hnae3_client_ops client_ops
= {
4600 .init_instance
= hns3_client_init
,
4601 .uninit_instance
= hns3_client_uninit
,
4602 .link_status_change
= hns3_link_status_change
,
4603 .setup_tc
= hns3_client_setup_tc
,
4604 .reset_notify
= hns3_reset_notify
,
4605 .process_hw_error
= hns3_process_hw_error
,
4608 /* hns3_init_module - Driver registration routine
4609 * hns3_init_module is the first routine called when the driver is
4610 * loaded. All it does is register with the PCI subsystem.
4612 static int __init
hns3_init_module(void)
4616 pr_info("%s: %s - version\n", hns3_driver_name
, hns3_driver_string
);
4617 pr_info("%s: %s\n", hns3_driver_name
, hns3_copyright
);
4619 client
.type
= HNAE3_CLIENT_KNIC
;
4620 snprintf(client
.name
, HNAE3_CLIENT_NAME_LENGTH
, "%s",
4623 client
.ops
= &client_ops
;
4625 INIT_LIST_HEAD(&client
.node
);
4627 hns3_dbg_register_debugfs(hns3_driver_name
);
4629 ret
= hnae3_register_client(&client
);
4631 goto err_reg_client
;
4633 ret
= pci_register_driver(&hns3_driver
);
4635 goto err_reg_driver
;
4640 hnae3_unregister_client(&client
);
4642 hns3_dbg_unregister_debugfs();
4645 module_init(hns3_init_module
);
4647 /* hns3_exit_module - Driver exit cleanup routine
4648 * hns3_exit_module is called just before the driver is removed
4651 static void __exit
hns3_exit_module(void)
4653 pci_unregister_driver(&hns3_driver
);
4654 hnae3_unregister_client(&client
);
4655 hns3_dbg_unregister_debugfs();
4657 module_exit(hns3_exit_module
);
4659 MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4660 MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4661 MODULE_LICENSE("GPL");
4662 MODULE_ALIAS("pci:hns-nic");